1""" 2Utility functions for salt.cloud 3""" 4 5 6import codecs 7import copy 8import errno 9import hashlib 10import logging 11import multiprocessing 12import os 13import pipes 14import re 15import shutil 16import socket 17import stat 18import subprocess 19import tempfile 20import time 21import traceback 22import uuid 23 24import salt.client 25import salt.cloud 26import salt.config 27import salt.crypt 28import salt.loader 29import salt.template 30import salt.utils.compat 31import salt.utils.crypt 32import salt.utils.data 33import salt.utils.event 34import salt.utils.files 35import salt.utils.msgpack 36import salt.utils.path 37import salt.utils.platform 38import salt.utils.stringutils 39import salt.utils.versions 40import salt.utils.vt 41import salt.utils.yaml 42from jinja2 import Template 43from salt.exceptions import ( 44 SaltCloudConfigError, 45 SaltCloudException, 46 SaltCloudExecutionFailure, 47 SaltCloudExecutionTimeout, 48 SaltCloudPasswordError, 49 SaltCloudSystemExit, 50) 51from salt.utils.nb_popen import NonBlockingPopen 52from salt.utils.validate.path import is_writeable 53 54try: 55 import salt.utils.smb 56 57 HAS_SMB = True 58except ImportError: 59 HAS_SMB = False 60 61try: 62 from pypsexec.client import Client as PsExecClient 63 from pypsexec.scmr import Service as ScmrService 64 from pypsexec.exceptions import SCMRException 65 from smbprotocol.tree import TreeConnect 66 from smbprotocol.exceptions import SMBResponseException 67 68 logging.getLogger("smbprotocol").setLevel(logging.WARNING) 69 logging.getLogger("pypsexec").setLevel(logging.WARNING) 70 HAS_PSEXEC = True 71except ImportError: 72 HAS_PSEXEC = False 73 74 75# Set the minimum version of PyWinrm. 76WINRM_MIN_VER = "0.3.0" 77 78 79try: 80 import winrm 81 from winrm.exceptions import WinRMTransportError 82 83 # Verify WinRM 0.3.0 or greater 84 import pkg_resources # pylint: disable=3rd-party-module-not-gated 85 86 winrm_pkg = pkg_resources.get_distribution("pywinrm") 87 if not salt.utils.versions.compare(winrm_pkg.version, ">=", WINRM_MIN_VER): 88 HAS_WINRM = False 89 else: 90 HAS_WINRM = True 91 92except ImportError: 93 HAS_WINRM = False 94 95 96# Let's import pwd and catch the ImportError. We'll raise it if this is not 97# Windows. This import has to be below where we import salt.utils.platform! 98try: 99 import pwd 100except ImportError: 101 if not salt.utils.platform.is_windows(): 102 raise 103 104try: 105 import getpass 106 107 HAS_GETPASS = True 108except ImportError: 109 HAS_GETPASS = False 110 111# This is required to support international characters in AWS EC2 tags or any 112# other kind of metadata provided by particular Cloud vendor. 113MSGPACK_ENCODING = "utf-8" 114 115NSTATES = { 116 0: "running", 117 1: "rebooting", 118 2: "terminated", 119 3: "pending", 120} 121 122SSH_PASSWORD_PROMP_RE = re.compile(r"(?:.*)[Pp]assword(?: for .*)?:\ *$", re.M) 123SSH_PASSWORD_PROMP_SUDO_RE = re.compile( 124 r"(?:.*sudo)(?:.*)[Pp]assword(?: for .*)?:", re.M 125) 126 127SERVER_ALIVE_INTERVAL = 60 128SERVER_ALIVE_COUNT_MAX = 3 129 130# Get logging started 131log = logging.getLogger(__name__) 132 133 134def __render_script(path, vm_=None, opts=None, minion=""): 135 """ 136 Return the rendered script 137 """ 138 log.info("Rendering deploy script: %s", path) 139 try: 140 with salt.utils.files.fopen(path, "r") as fp_: 141 template = Template(salt.utils.stringutils.to_unicode(fp_.read())) 142 return str(template.render(opts=opts, vm=vm_, minion=minion)) 143 except AttributeError: 144 # Specified renderer was not found 145 with salt.utils.files.fopen(path, "r") as fp_: 146 return str(fp_.read()) 147 148 149def __ssh_gateway_config_dict(gateway): 150 """ 151 Return a dictionary with gateway options. The result is used 152 to provide arguments to __ssh_gateway_arguments method. 153 """ 154 extended_kwargs = {} 155 if gateway: 156 extended_kwargs["ssh_gateway"] = gateway["ssh_gateway"] 157 extended_kwargs["ssh_gateway_key"] = gateway["ssh_gateway_key"] 158 extended_kwargs["ssh_gateway_user"] = gateway["ssh_gateway_user"] 159 extended_kwargs["ssh_gateway_command"] = gateway["ssh_gateway_command"] 160 return extended_kwargs 161 162 163def __ssh_gateway_arguments(kwargs): 164 """ 165 Return ProxyCommand configuration string for ssh/scp command. 166 167 All gateway options should not include quotes (' or "). To support 168 future user configuration, please make sure to update the dictionary 169 from __ssh_gateway_config_dict and get_ssh_gateway_config (ec2.py) 170 """ 171 extended_arguments = "" 172 173 ssh_gateway = kwargs.get("ssh_gateway", "") 174 ssh_gateway_port = 22 175 if ":" in ssh_gateway: 176 ssh_gateway, ssh_gateway_port = ssh_gateway.split(":") 177 ssh_gateway_command = kwargs.get("ssh_gateway_command", "nc -q0 %h %p") 178 179 if ssh_gateway: 180 ssh_gateway_port = kwargs.get("ssh_gateway_port", ssh_gateway_port) 181 ssh_gateway_key = ( 182 "-i {}".format(kwargs["ssh_gateway_key"]) 183 if "ssh_gateway_key" in kwargs 184 else "" 185 ) 186 ssh_gateway_user = kwargs.get("ssh_gateway_user", "root") 187 188 # Setup ProxyCommand 189 extended_arguments = " ".join( 190 ( 191 "ssh", 192 "-oStrictHostKeyChecking=no", 193 "-oServerAliveInterval={}".format( 194 kwargs.get("server_alive_interval", SERVER_ALIVE_INTERVAL) 195 ), 196 "-oServerAliveCountMax={}".format( 197 kwargs.get("server_alive_count_max", SERVER_ALIVE_COUNT_MAX) 198 ), 199 "-oUserKnownHostsFile=/dev/null", 200 "-oControlPath=none", 201 str(ssh_gateway_key), 202 "{}@{}".format(ssh_gateway_user, ssh_gateway), 203 "-p", 204 str(ssh_gateway_port), 205 str(ssh_gateway_command), 206 ) 207 ) 208 209 log.info( 210 "Using SSH gateway %s@%s:%s %s", 211 ssh_gateway_user, 212 ssh_gateway, 213 ssh_gateway_port, 214 ssh_gateway_command, 215 ) 216 217 return extended_arguments 218 219 220def os_script(os_, vm_=None, opts=None, minion=""): 221 """ 222 Return the script as a string for the specific os 223 """ 224 if minion: 225 minion = salt_config_to_yaml(minion) 226 227 if os.path.isabs(os_): 228 # The user provided an absolute path to the deploy script, let's use it 229 return __render_script(os_, vm_, opts, minion) 230 231 if os.path.isabs("{}.sh".format(os_)): 232 # The user provided an absolute path to the deploy script, although no 233 # extension was provided. Let's use it anyway. 234 return __render_script("{}.sh".format(os_), vm_, opts, minion) 235 236 for search_path in opts["deploy_scripts_search_path"]: 237 if os.path.isfile(os.path.join(search_path, os_)): 238 return __render_script(os.path.join(search_path, os_), vm_, opts, minion) 239 240 if os.path.isfile(os.path.join(search_path, "{}.sh".format(os_))): 241 return __render_script( 242 os.path.join(search_path, "{}.sh".format(os_)), vm_, opts, minion 243 ) 244 # No deploy script was found, return an empty string 245 return "" 246 247 248def gen_keys(keysize=2048): 249 """ 250 Generate Salt minion keys and return them as PEM file strings 251 """ 252 # Mandate that keys are at least 2048 in size 253 if keysize < 2048: 254 keysize = 2048 255 tdir = tempfile.mkdtemp() 256 257 salt.crypt.gen_keys(tdir, "minion", keysize) 258 priv_path = os.path.join(tdir, "minion.pem") 259 pub_path = os.path.join(tdir, "minion.pub") 260 with salt.utils.files.fopen(priv_path) as fp_: 261 priv = salt.utils.stringutils.to_unicode(fp_.read()) 262 with salt.utils.files.fopen(pub_path) as fp_: 263 pub = salt.utils.stringutils.to_unicode(fp_.read()) 264 shutil.rmtree(tdir) 265 return priv, pub 266 267 268def accept_key(pki_dir, pub, id_): 269 """ 270 If the master config was available then we will have a pki_dir key in 271 the opts directory, this method places the pub key in the accepted 272 keys dir and removes it from the unaccepted keys dir if that is the case. 273 """ 274 for key_dir in "minions", "minions_pre", "minions_rejected": 275 key_path = os.path.join(pki_dir, key_dir) 276 if not os.path.exists(key_path): 277 os.makedirs(key_path) 278 279 key = os.path.join(pki_dir, "minions", id_) 280 with salt.utils.files.fopen(key, "w+") as fp_: 281 fp_.write(salt.utils.stringutils.to_str(pub)) 282 283 oldkey = os.path.join(pki_dir, "minions_pre", id_) 284 if os.path.isfile(oldkey): 285 with salt.utils.files.fopen(oldkey) as fp_: 286 if fp_.read() == pub: 287 os.remove(oldkey) 288 289 290def remove_key(pki_dir, id_): 291 """ 292 This method removes a specified key from the accepted keys dir 293 """ 294 key = os.path.join(pki_dir, "minions", id_) 295 if os.path.isfile(key): 296 os.remove(key) 297 log.debug("Deleted '%s'", key) 298 299 300def rename_key(pki_dir, id_, new_id): 301 """ 302 Rename a key, when an instance has also been renamed 303 """ 304 oldkey = os.path.join(pki_dir, "minions", id_) 305 newkey = os.path.join(pki_dir, "minions", new_id) 306 if os.path.isfile(oldkey): 307 os.rename(oldkey, newkey) 308 309 310def minion_config(opts, vm_): 311 """ 312 Return a minion's configuration for the provided options and VM 313 """ 314 315 # Don't start with a copy of the default minion opts; they're not always 316 # what we need. Some default options are Null, let's set a reasonable default 317 minion = { 318 "master": "salt", 319 "log_level": "info", 320 "hash_type": "sha256", 321 } 322 323 # Now, let's update it to our needs 324 minion["id"] = vm_["name"] 325 master_finger = salt.config.get_cloud_config_value("master_finger", vm_, opts) 326 if master_finger is not None: 327 minion["master_finger"] = master_finger 328 minion.update( 329 # Get ANY defined minion settings, merging data, in the following order 330 # 1. VM config 331 # 2. Profile config 332 # 3. Global configuration 333 salt.config.get_cloud_config_value( 334 "minion", vm_, opts, default={}, search_global=True 335 ) 336 ) 337 338 make_master = salt.config.get_cloud_config_value("make_master", vm_, opts) 339 if "master" not in minion and make_master is not True: 340 raise SaltCloudConfigError( 341 "A master setting was not defined in the minion's configuration." 342 ) 343 344 # Get ANY defined grains settings, merging data, in the following order 345 # 1. VM config 346 # 2. Profile config 347 # 3. Global configuration 348 minion.setdefault("grains", {}).update( 349 salt.config.get_cloud_config_value( 350 "grains", vm_, opts, default={}, search_global=True 351 ) 352 ) 353 return minion 354 355 356def master_config(opts, vm_): 357 """ 358 Return a master's configuration for the provided options and VM 359 """ 360 # Let's get a copy of the salt master default options 361 master = copy.deepcopy(salt.config.DEFAULT_MASTER_OPTS) 362 # Some default options are Null, let's set a reasonable default 363 master.update(log_level="info", log_level_logfile="info", hash_type="sha256") 364 365 # Get ANY defined master setting, merging data, in the following order 366 # 1. VM config 367 # 2. Profile config 368 # 3. Global configuration 369 master.update( 370 salt.config.get_cloud_config_value( 371 "master", vm_, opts, default={}, search_global=True 372 ) 373 ) 374 return master 375 376 377def salt_config_to_yaml(configuration, line_break="\n"): 378 """ 379 Return a salt configuration dictionary, master or minion, as a yaml dump 380 """ 381 return salt.utils.yaml.safe_dump( 382 configuration, line_break=line_break, default_flow_style=False 383 ) 384 385 386def bootstrap(vm_, opts=None): 387 """ 388 This is the primary entry point for logging into any system (POSIX or 389 Windows) to install Salt. It will make the decision on its own as to which 390 deploy function to call. 391 """ 392 if opts is None: 393 opts = __opts__ 394 deploy_config = salt.config.get_cloud_config_value( 395 "deploy", vm_, opts, default=False 396 ) 397 inline_script_config = salt.config.get_cloud_config_value( 398 "inline_script", vm_, opts, default=None 399 ) 400 if deploy_config is False and inline_script_config is None: 401 return {"Error": {"No Deploy": "'deploy' is not enabled. Not deploying."}} 402 403 if vm_.get("driver") == "saltify": 404 saltify_driver = True 405 else: 406 saltify_driver = False 407 408 key_filename = salt.config.get_cloud_config_value( 409 "key_filename", 410 vm_, 411 opts, 412 search_global=False, 413 default=salt.config.get_cloud_config_value( 414 "ssh_keyfile", vm_, opts, search_global=False, default=None 415 ), 416 ) 417 if key_filename is not None and not os.path.isfile(key_filename): 418 raise SaltCloudConfigError( 419 "The defined ssh_keyfile '{}' does not exist".format(key_filename) 420 ) 421 has_ssh_agent = False 422 if ( 423 opts.get("ssh_agent", False) 424 and "SSH_AUTH_SOCK" in os.environ 425 and stat.S_ISSOCK(os.stat(os.environ["SSH_AUTH_SOCK"]).st_mode) 426 ): 427 has_ssh_agent = True 428 429 if ( 430 key_filename is None 431 and salt.config.get_cloud_config_value("password", vm_, opts, default=None) 432 is None 433 and salt.config.get_cloud_config_value("win_password", vm_, opts, default=None) 434 is None 435 and has_ssh_agent is False 436 ): 437 raise SaltCloudSystemExit( 438 "Cannot deploy Salt in a VM if the 'key_filename' setting " 439 "is not set and there is no password set for the VM. " 440 "Check the provider docs for 'change_password' option if it " 441 "is supported by your provider." 442 ) 443 444 ret = {} 445 446 minion_conf = minion_config(opts, vm_) 447 deploy_script_code = os_script( 448 salt.config.get_cloud_config_value("os", vm_, opts, default="bootstrap-salt"), 449 vm_, 450 opts, 451 minion_conf, 452 ) 453 454 ssh_username = salt.config.get_cloud_config_value( 455 "ssh_username", vm_, opts, default="root" 456 ) 457 458 if "file_transport" not in opts: 459 opts["file_transport"] = vm_.get("file_transport", "sftp") 460 461 # If we haven't generated any keys yet, do so now. 462 if "pub_key" not in vm_ and "priv_key" not in vm_: 463 log.debug("Generating keys for '%s'", vm_["name"]) 464 465 vm_["priv_key"], vm_["pub_key"] = gen_keys( 466 salt.config.get_cloud_config_value("keysize", vm_, opts) 467 ) 468 469 key_id = vm_.get("name") 470 if "append_domain" in vm_: 471 key_id = ".".join([key_id, vm_["append_domain"]]) 472 473 accept_key(opts["pki_dir"], vm_["pub_key"], key_id) 474 475 if "os" not in vm_: 476 vm_["os"] = salt.config.get_cloud_config_value("script", vm_, opts) 477 478 # NOTE: deploy_kwargs is also used to pass inline_script variable content 479 # to run_inline_script function 480 host = salt.config.get_cloud_config_value("ssh_host", vm_, opts) 481 deploy_kwargs = { 482 "opts": opts, 483 "host": host, 484 "port": salt.config.get_cloud_config_value("ssh_port", vm_, opts, default=22), 485 "salt_host": vm_.get("salt_host", host), 486 "username": ssh_username, 487 "script": deploy_script_code, 488 "inline_script": inline_script_config, 489 "name": vm_["name"], 490 "has_ssh_agent": has_ssh_agent, 491 "tmp_dir": salt.config.get_cloud_config_value( 492 "tmp_dir", vm_, opts, default="/tmp/.saltcloud" 493 ), 494 "vm_": vm_, 495 "start_action": opts["start_action"], 496 "parallel": opts["parallel"], 497 "sock_dir": opts["sock_dir"], 498 "conf_file": opts["conf_file"], 499 "minion_pem": vm_["priv_key"], 500 "minion_pub": vm_["pub_key"], 501 "master_sign_pub_file": salt.config.get_cloud_config_value( 502 "master_sign_pub_file", vm_, opts, default=None 503 ), 504 "keep_tmp": opts["keep_tmp"], 505 "sudo": salt.config.get_cloud_config_value( 506 "sudo", vm_, opts, default=(ssh_username != "root") 507 ), 508 "sudo_password": salt.config.get_cloud_config_value( 509 "sudo_password", vm_, opts, default=None 510 ), 511 "tty": salt.config.get_cloud_config_value("tty", vm_, opts, default=True), 512 "password": salt.config.get_cloud_config_value( 513 "password", vm_, opts, search_global=False 514 ), 515 "key_filename": key_filename, 516 "script_args": salt.config.get_cloud_config_value("script_args", vm_, opts), 517 "script_env": salt.config.get_cloud_config_value("script_env", vm_, opts), 518 "minion_conf": minion_conf, 519 "force_minion_config": salt.config.get_cloud_config_value( 520 "force_minion_config", vm_, opts, default=False 521 ), 522 "preseed_minion_keys": vm_.get("preseed_minion_keys", None), 523 "display_ssh_output": salt.config.get_cloud_config_value( 524 "display_ssh_output", vm_, opts, default=True 525 ), 526 "known_hosts_file": salt.config.get_cloud_config_value( 527 "known_hosts_file", vm_, opts, default="/dev/null" 528 ), 529 "file_map": salt.config.get_cloud_config_value( 530 "file_map", vm_, opts, default=None 531 ), 532 "maxtries": salt.config.get_cloud_config_value( 533 "wait_for_passwd_maxtries", vm_, opts, default=15 534 ), 535 "preflight_cmds": salt.config.get_cloud_config_value( 536 "preflight_cmds", vm_, opts, default=[] 537 ), 538 "cloud_grains": { 539 "driver": vm_["driver"], 540 "provider": vm_["provider"], 541 "profile": vm_["profile"], 542 }, 543 } 544 545 inline_script_kwargs = deploy_kwargs.copy() # make a copy at this point 546 547 # forward any info about possible ssh gateway to deploy script 548 # as some providers need also a 'gateway' configuration 549 if "gateway" in vm_: 550 deploy_kwargs.update({"gateway": vm_["gateway"]}) 551 552 # Deploy salt-master files, if necessary 553 if salt.config.get_cloud_config_value("make_master", vm_, opts) is True: 554 deploy_kwargs["make_master"] = True 555 deploy_kwargs["master_pub"] = vm_["master_pub"] 556 deploy_kwargs["master_pem"] = vm_["master_pem"] 557 master_conf = master_config(opts, vm_) 558 deploy_kwargs["master_conf"] = master_conf 559 560 if master_conf.get("syndic_master", None): 561 deploy_kwargs["make_syndic"] = True 562 563 deploy_kwargs["make_minion"] = salt.config.get_cloud_config_value( 564 "make_minion", vm_, opts, default=True 565 ) 566 567 if saltify_driver: 568 deploy_kwargs[ 569 "wait_for_passwd_maxtries" 570 ] = 0 # No need to wait/retry with Saltify 571 572 win_installer = salt.config.get_cloud_config_value("win_installer", vm_, opts) 573 if win_installer: 574 deploy_kwargs["port"] = salt.config.get_cloud_config_value( 575 "smb_port", vm_, opts, default=445 576 ) 577 deploy_kwargs["win_installer"] = win_installer 578 minion = minion_config(opts, vm_) 579 deploy_kwargs["master"] = minion["master"] 580 deploy_kwargs["username"] = salt.config.get_cloud_config_value( 581 "win_username", vm_, opts, default="Administrator" 582 ) 583 win_pass = salt.config.get_cloud_config_value( 584 "win_password", vm_, opts, default="" 585 ) 586 if win_pass: 587 deploy_kwargs["password"] = win_pass 588 deploy_kwargs["use_winrm"] = salt.config.get_cloud_config_value( 589 "use_winrm", vm_, opts, default=False 590 ) 591 deploy_kwargs["winrm_port"] = salt.config.get_cloud_config_value( 592 "winrm_port", vm_, opts, default=5986 593 ) 594 deploy_kwargs["winrm_use_ssl"] = salt.config.get_cloud_config_value( 595 "winrm_use_ssl", vm_, opts, default=True 596 ) 597 deploy_kwargs["winrm_verify_ssl"] = salt.config.get_cloud_config_value( 598 "winrm_verify_ssl", vm_, opts, default=True 599 ) 600 if saltify_driver: 601 deploy_kwargs["port_timeout"] = 1 # No need to wait/retry with Saltify 602 603 # Store what was used to the deploy the VM 604 event_kwargs = copy.deepcopy(deploy_kwargs) 605 del event_kwargs["opts"] 606 del event_kwargs["minion_pem"] 607 del event_kwargs["minion_pub"] 608 del event_kwargs["sudo_password"] 609 if "password" in event_kwargs: 610 del event_kwargs["password"] 611 ret["deploy_kwargs"] = event_kwargs 612 613 fire_event( 614 "event", 615 "executing deploy script", 616 "salt/cloud/{}/deploying".format(vm_["name"]), 617 args={"kwargs": salt.utils.data.simple_types_filter(event_kwargs)}, 618 sock_dir=opts.get("sock_dir", os.path.join(__opts__["sock_dir"], "master")), 619 transport=opts.get("transport", "zeromq"), 620 ) 621 622 if inline_script_config and deploy_config is False: 623 inline_script_deployed = run_inline_script(**inline_script_kwargs) 624 if inline_script_deployed is not False: 625 log.info("Inline script(s) ha(s|ve) run on %s", vm_["name"]) 626 ret["deployed"] = False 627 return ret 628 else: 629 if win_installer: 630 deployed = deploy_windows(**deploy_kwargs) 631 else: 632 deployed = deploy_script(**deploy_kwargs) 633 634 if inline_script_config: 635 inline_script_deployed = run_inline_script(**inline_script_kwargs) 636 if inline_script_deployed is not False: 637 log.info("Inline script(s) ha(s|ve) run on %s", vm_["name"]) 638 639 if deployed is not False: 640 ret["deployed"] = True 641 if deployed is not True: 642 ret.update(deployed) 643 log.info("Salt installed on %s", vm_["name"]) 644 return ret 645 646 log.error("Failed to start Salt on host %s", vm_["name"]) 647 return { 648 "Error": {"Not Deployed": "Failed to start Salt on host {}".format(vm_["name"])} 649 } 650 651 652def ssh_usernames(vm_, opts, default_users=None): 653 """ 654 Return the ssh_usernames. Defaults to a built-in list of users for trying. 655 """ 656 if default_users is None: 657 default_users = ["root"] 658 659 usernames = salt.config.get_cloud_config_value("ssh_username", vm_, opts) 660 661 if not isinstance(usernames, list): 662 usernames = [usernames] 663 664 # get rid of None's or empty names 665 usernames = [x for x in usernames if x] 666 # Keep a copy of the usernames the user might have provided 667 initial = usernames[:] 668 669 # Add common usernames to the list to be tested 670 for name in default_users: 671 if name not in usernames: 672 usernames.append(name) 673 # Add the user provided usernames to the end of the list since enough time 674 # might need to pass before the remote service is available for logins and 675 # the proper username might have passed its iteration. 676 # This has detected in a CentOS 5.7 EC2 image 677 usernames.extend(initial) 678 return usernames 679 680 681def wait_for_fun(fun, timeout=900, **kwargs): 682 """ 683 Wait until a function finishes, or times out 684 """ 685 start = time.time() 686 log.debug("Attempting function %s", fun) 687 trycount = 0 688 while True: 689 trycount += 1 690 try: 691 response = fun(**kwargs) 692 if not isinstance(response, bool): 693 return response 694 except Exception as exc: # pylint: disable=broad-except 695 log.debug("Caught exception in wait_for_fun: %s", exc) 696 time.sleep(1) 697 log.debug("Retrying function %s on (try %s)", fun, trycount) 698 if time.time() - start > timeout: 699 log.error("Function timed out: %s", timeout) 700 return False 701 702 703def wait_for_port( 704 host, 705 port=22, 706 timeout=900, 707 gateway=None, 708 server_alive_interval=SERVER_ALIVE_INTERVAL, 709 server_alive_count_max=SERVER_ALIVE_COUNT_MAX, 710): 711 """ 712 Wait until a connection to the specified port can be made on a specified 713 host. This is usually port 22 (for SSH), but in the case of Windows 714 installations, it might be port 445 (for psexec). It may also be an 715 alternate port for SSH, depending on the base image. 716 """ 717 start = time.time() 718 # Assign test ports because if a gateway is defined 719 # we first want to test the gateway before the host. 720 test_ssh_host = host 721 test_ssh_port = port 722 723 if gateway: 724 ssh_gateway = gateway["ssh_gateway"] 725 ssh_gateway_port = 22 726 if ":" in ssh_gateway: 727 ssh_gateway, ssh_gateway_port = ssh_gateway.split(":") 728 if "ssh_gateway_port" in gateway: 729 ssh_gateway_port = gateway["ssh_gateway_port"] 730 test_ssh_host = ssh_gateway 731 test_ssh_port = ssh_gateway_port 732 log.debug( 733 "Attempting connection to host %s on port %s via gateway %s on port %s", 734 host, 735 port, 736 ssh_gateway, 737 ssh_gateway_port, 738 ) 739 else: 740 log.debug("Attempting connection to host %s on port %s", host, port) 741 trycount = 0 742 while True: 743 trycount += 1 744 try: 745 if socket.inet_pton(socket.AF_INET6, host): 746 sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) 747 else: 748 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 749 except OSError: 750 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 751 try: 752 sock.settimeout(5) 753 sock.connect((test_ssh_host, int(test_ssh_port))) 754 # Stop any remaining reads/writes on the socket 755 sock.shutdown(socket.SHUT_RDWR) 756 # Close it! 757 sock.close() 758 break 759 except OSError as exc: 760 log.debug("Caught exception in wait_for_port: %s", exc) 761 time.sleep(1) 762 if time.time() - start > timeout: 763 log.error("Port connection timed out: %s", timeout) 764 return False 765 log.debug( 766 "Retrying connection to %s %s on port %s (try %s)", 767 "gateway" if gateway else "host", 768 test_ssh_host, 769 test_ssh_port, 770 trycount, 771 ) 772 if not gateway: 773 return True 774 # Let the user know that his gateway is good! 775 log.debug("Gateway %s on port %s is reachable.", test_ssh_host, test_ssh_port) 776 777 # Now we need to test the host via the gateway. 778 # We will use netcat on the gateway to test the port 779 ssh_args = [] 780 ssh_args.extend( 781 [ 782 # Don't add new hosts to the host key database 783 "-oStrictHostKeyChecking=no", 784 # make sure ssh can time out on connection lose 785 "-oServerAliveInterval={}".format(server_alive_interval), 786 "-oServerAliveCountMax={}".format(server_alive_count_max), 787 # Set hosts key database path to /dev/null, i.e., non-existing 788 "-oUserKnownHostsFile=/dev/null", 789 # Don't re-use the SSH connection. Less failures. 790 "-oControlPath=none", 791 ] 792 ) 793 # There should never be both a password and an ssh key passed in, so 794 if "ssh_gateway_key" in gateway: 795 ssh_args.extend( 796 [ 797 # tell SSH to skip password authentication 798 "-oPasswordAuthentication=no", 799 "-oChallengeResponseAuthentication=no", 800 # Make sure public key authentication is enabled 801 "-oPubkeyAuthentication=yes", 802 # do only use the provided identity file 803 "-oIdentitiesOnly=yes", 804 # No Keyboard interaction! 805 "-oKbdInteractiveAuthentication=no", 806 # Also, specify the location of the key file 807 "-i {}".format(gateway["ssh_gateway_key"]), 808 ] 809 ) 810 # Netcat command testing remote port 811 command = "nc -z -w5 -q0 {} {}".format(host, port) 812 # SSH command 813 pcmd = "ssh {} {}@{} -p {} {}".format( 814 " ".join(ssh_args), 815 gateway["ssh_gateway_user"], 816 ssh_gateway, 817 ssh_gateway_port, 818 pipes.quote("date"), 819 ) 820 cmd = "ssh {} {}@{} -p {} {}".format( 821 " ".join(ssh_args), 822 gateway["ssh_gateway_user"], 823 ssh_gateway, 824 ssh_gateway_port, 825 pipes.quote(command), 826 ) 827 log.debug("SSH command: '%s'", cmd) 828 829 kwargs = { 830 "display_ssh_output": False, 831 "password": gateway.get("ssh_gateway_password", None), 832 } 833 trycount = 0 834 usable_gateway = False 835 gateway_retries = 5 836 while True: 837 trycount += 1 838 # test gateway usage 839 if not usable_gateway: 840 pstatus = _exec_ssh_cmd(pcmd, allow_failure=True, **kwargs) 841 if pstatus == 0: 842 usable_gateway = True 843 else: 844 gateway_retries -= 1 845 log.error( 846 "Gateway usage seems to be broken, password error ? Tries left: %s", 847 gateway_retries, 848 ) 849 if not gateway_retries: 850 raise SaltCloudExecutionFailure( 851 "SSH gateway is reachable but we can not login" 852 ) 853 # then try to reach out the target 854 if usable_gateway: 855 status = _exec_ssh_cmd(cmd, allow_failure=True, **kwargs) 856 # Get the exit code of the SSH command. 857 # If 0 then the port is open. 858 if status == 0: 859 return True 860 time.sleep(1) 861 if time.time() - start > timeout: 862 log.error("Port connection timed out: %s", timeout) 863 return False 864 log.debug( 865 "Retrying connection to host %s on port %s " 866 "via gateway %s on port %s. (try %s)", 867 host, 868 port, 869 ssh_gateway, 870 ssh_gateway_port, 871 trycount, 872 ) 873 874 875class Client: 876 """ 877 Wrap pypsexec.client.Client to fix some stability issues: 878 879 - Set the service name from a keyword arg, this allows multiple service 880 instances to be created in a single process. 881 - Keep trying service and file deletes since they may not succeed on the 882 first try. Raises an exception if they do not succeed after a timeout 883 period. 884 """ 885 886 def __init__( 887 self, 888 server, 889 username=None, 890 password=None, 891 port=445, 892 encrypt=True, 893 service_name=None, 894 ): 895 self.service_name = service_name 896 self._exe_file = "{}.exe".format(self.service_name) 897 self._client = PsExecClient(server, username, password, port, encrypt) 898 self._client._service = ScmrService(self.service_name, self._client.session) 899 900 def connect(self): 901 return self._client.connect() 902 903 def disconnect(self): 904 return self._client.disconnect() 905 906 def create_service(self): 907 return self._client.create_service() 908 909 def run_executable(self, *args, **kwargs): 910 return self._client.run_executable(*args, **kwargs) 911 912 def remove_service(self, wait_timeout=10, sleep_wait=1): 913 """ 914 Removes the PAExec service and executable that was created as part of 915 the create_service function. This does not remove any older executables 916 or services from previous runs, use cleanup() instead for that purpose. 917 """ 918 919 # Stops/remove the PAExec service and removes the executable 920 log.debug("Deleting PAExec service at the end of the process") 921 wait_start = time.time() 922 while True: 923 try: 924 self._client._service.delete() 925 except SCMRException as exc: 926 log.debug("Exception encountered while deleting service %s", repr(exc)) 927 if time.time() - wait_start > wait_timeout: 928 raise exc 929 time.sleep(sleep_wait) 930 continue 931 break 932 933 # delete the PAExec executable 934 smb_tree = TreeConnect( 935 self._client.session, 936 r"\\{}\ADMIN$".format(self._client.connection.server_name), 937 ) 938 log.info("Connecting to SMB Tree %s", smb_tree.share_name) 939 smb_tree.connect() 940 941 wait_start = time.time() 942 while True: 943 try: 944 log.info("Creating open to PAExec file with delete on close flags") 945 self._client._delete_file(smb_tree, self._exe_file) 946 except SMBResponseException as exc: 947 log.debug("Exception deleting file %s %s", self._exe_file, repr(exc)) 948 if time.time() - wait_start > wait_timeout: 949 raise exc 950 time.sleep(sleep_wait) 951 continue 952 break 953 log.info("Disconnecting from SMB Tree %s", smb_tree.share_name) 954 smb_tree.disconnect() 955 956 957def run_winexe_command(cmd, args, host, username, password, port=445): 958 """ 959 Run a command remotly via the winexe executable 960 """ 961 creds = "-U '{}%{}' //{}".format(username, password, host) 962 logging_creds = "-U '{}%XXX-REDACTED-XXX' //{}".format(username, host) 963 cmd = "winexe {} {} {}".format(creds, cmd, args) 964 logging_cmd = "winexe {} {} {}".format(logging_creds, cmd, args) 965 return win_cmd(cmd, logging_command=logging_cmd) 966 967 968def run_psexec_command(cmd, args, host, username, password, port=445): 969 """ 970 Run a command remotly using the psexec protocol 971 """ 972 service_name = "PS-Exec-{}".format(uuid.uuid4()) 973 stdout, stderr, ret_code = "", "", None 974 client = Client( 975 host, username, password, port=port, encrypt=False, service_name=service_name 976 ) 977 client.connect() 978 try: 979 client.create_service() 980 stdout, stderr, ret_code = client.run_executable(cmd, args) 981 finally: 982 client.remove_service() 983 client.disconnect() 984 return stdout, stderr, ret_code 985 986 987def wait_for_winexe(host, port, username, password, timeout=900): 988 """ 989 Wait until winexe connection can be established. 990 """ 991 start = time.time() 992 log.debug("Attempting winexe connection to host %s on port %s", host, port) 993 try_count = 0 994 while True: 995 try_count += 1 996 try: 997 # Shell out to winexe to check %TEMP% 998 ret_code = run_winexe_command( 999 "sc", "query winexesvc", host, username, password, port 1000 ) 1001 if ret_code == 0: 1002 log.debug("winexe connected...") 1003 return True 1004 log.debug("Return code was %s", ret_code) 1005 except OSError as exc: 1006 log.debug("Caught exception in wait_for_winexesvc: %s", exc) 1007 1008 if time.time() - start > timeout: 1009 return False 1010 time.sleep(1) 1011 1012 1013def wait_for_psexecsvc(host, port, username, password, timeout=900): 1014 """ 1015 Wait until psexec connection can be established. 1016 """ 1017 start = time.time() 1018 try_count = 0 1019 while True: 1020 try_count += 1 1021 ret_code = 1 1022 try: 1023 stdout, stderr, ret_code = run_psexec_command( 1024 "cmd.exe", "/c hostname", host, username, password, port=port 1025 ) 1026 except Exception as exc: # pylint: disable=broad-except 1027 log.exception("Unable to execute command") 1028 if ret_code == 0: 1029 log.debug("psexec connected...") 1030 return True 1031 if time.time() - start > timeout: 1032 return False 1033 log.debug( 1034 "Retrying psexec connection to host %s on port %s (try %s)", 1035 host, 1036 port, 1037 try_count, 1038 ) 1039 time.sleep(1) 1040 1041 1042def wait_for_winrm( 1043 host, port, username, password, timeout=900, use_ssl=True, verify=True 1044): 1045 """ 1046 Wait until WinRM connection can be established. 1047 """ 1048 # Ensure the winrm service is listening before attempting to connect 1049 wait_for_port(host=host, port=port, timeout=timeout) 1050 1051 start = time.time() 1052 log.debug("Attempting WinRM connection to host %s on port %s", host, port) 1053 transport = "ssl" 1054 if not use_ssl: 1055 transport = "ntlm" 1056 trycount = 0 1057 while True: 1058 trycount += 1 1059 try: 1060 winrm_kwargs = { 1061 "target": host, 1062 "auth": (username, password), 1063 "transport": transport, 1064 } 1065 if not verify: 1066 log.debug("SSL validation for WinRM disabled.") 1067 winrm_kwargs["server_cert_validation"] = "ignore" 1068 s = winrm.Session(**winrm_kwargs) 1069 if hasattr(s.protocol, "set_timeout"): 1070 s.protocol.set_timeout(15) 1071 log.trace("WinRM endpoint url: %s", s.url) 1072 r = s.run_cmd("sc query winrm") 1073 if r.status_code == 0: 1074 log.debug("WinRM session connected...") 1075 return s 1076 log.debug("Return code was %s", r.status_code) 1077 except WinRMTransportError as exc: 1078 log.debug("Caught exception in wait_for_winrm: %s", exc) 1079 1080 if time.time() - start > timeout: 1081 log.error("WinRM connection timed out: %s", timeout) 1082 return None 1083 log.debug( 1084 "Retrying WinRM connection to host %s on port %s (try %s)", 1085 host, 1086 port, 1087 trycount, 1088 ) 1089 time.sleep(1) 1090 1091 1092def validate_windows_cred_winexe( 1093 host, username="Administrator", password=None, retries=10, retry_delay=1 1094): 1095 """ 1096 Check if the windows credentials are valid 1097 """ 1098 cmd = "winexe -U '{}%{}' //{} \"hostname\"".format(username, password, host) 1099 logging_cmd = "winexe -U '{}%XXX-REDACTED-XXX' //{} \"hostname\"".format( 1100 username, host 1101 ) 1102 for i in range(retries): 1103 ret_code = win_cmd(cmd, logging_command=logging_cmd) 1104 return ret_code == 0 1105 1106 1107def validate_windows_cred( 1108 host, username="Administrator", password=None, retries=10, retry_delay=1 1109): 1110 """ 1111 Check if the windows credentials are valid 1112 """ 1113 for i in range(retries): 1114 ret_code = 1 1115 try: 1116 stdout, stderr, ret_code = run_psexec_command( 1117 "cmd.exe", "/c hostname", host, username, password, port=445 1118 ) 1119 except Exception as exc: # pylint: disable=broad-except 1120 log.exception("Exceoption while executing psexec") 1121 if ret_code == 0: 1122 break 1123 time.sleep(retry_delay) 1124 return ret_code == 0 1125 1126 1127def wait_for_passwd( 1128 host, 1129 port=22, 1130 ssh_timeout=15, 1131 username="root", 1132 password=None, 1133 key_filename=None, 1134 maxtries=15, 1135 trysleep=1, 1136 display_ssh_output=True, 1137 gateway=None, 1138 known_hosts_file="/dev/null", 1139 hard_timeout=None, 1140): 1141 """ 1142 Wait until ssh connection can be accessed via password or ssh key 1143 """ 1144 trycount = 0 1145 while trycount < maxtries: 1146 connectfail = False 1147 try: 1148 kwargs = { 1149 "hostname": host, 1150 "port": port, 1151 "username": username, 1152 "password_retries": maxtries, 1153 "timeout": ssh_timeout, 1154 "display_ssh_output": display_ssh_output, 1155 "known_hosts_file": known_hosts_file, 1156 "ssh_timeout": ssh_timeout, 1157 "hard_timeout": hard_timeout, 1158 } 1159 kwargs.update(__ssh_gateway_config_dict(gateway)) 1160 1161 if key_filename: 1162 if not os.path.isfile(key_filename): 1163 raise SaltCloudConfigError( 1164 "The defined key_filename '{}' does not exist".format( 1165 key_filename 1166 ) 1167 ) 1168 kwargs["key_filename"] = key_filename 1169 log.debug("Using %s as the key_filename", key_filename) 1170 elif password: 1171 kwargs["password"] = password 1172 log.debug("Using password authentication") 1173 1174 trycount += 1 1175 log.debug( 1176 "Attempting to authenticate as %s (try %s of %s)", 1177 username, 1178 trycount, 1179 maxtries, 1180 ) 1181 1182 status = root_cmd("date", tty=False, sudo=False, **kwargs) 1183 if status != 0: 1184 connectfail = True 1185 if trycount < maxtries: 1186 time.sleep(trysleep) 1187 continue 1188 1189 log.error("Authentication failed: status code %s", status) 1190 return False 1191 if connectfail is False: 1192 return True 1193 return False 1194 except SaltCloudPasswordError: 1195 raise 1196 except Exception: # pylint: disable=broad-except 1197 if trycount >= maxtries: 1198 return False 1199 time.sleep(trysleep) 1200 1201 1202def deploy_windows( 1203 host, 1204 port=445, 1205 timeout=900, 1206 username="Administrator", 1207 password=None, 1208 name=None, 1209 sock_dir=None, 1210 conf_file=None, 1211 start_action=None, 1212 parallel=False, 1213 minion_pub=None, 1214 minion_pem=None, 1215 minion_conf=None, 1216 keep_tmp=False, 1217 script_args=None, 1218 script_env=None, 1219 port_timeout=15, 1220 preseed_minion_keys=None, 1221 win_installer=None, 1222 master=None, 1223 tmp_dir="C:\\salttmp", 1224 opts=None, 1225 master_sign_pub_file=None, 1226 use_winrm=False, 1227 winrm_port=5986, 1228 winrm_use_ssl=True, 1229 winrm_verify_ssl=True, 1230 **kwargs 1231): 1232 """ 1233 Copy the install files to a remote Windows box, and execute them 1234 """ 1235 if not isinstance(opts, dict): 1236 opts = {} 1237 1238 if use_winrm and not HAS_WINRM: 1239 log.error( 1240 "WinRM requested but module winrm could not be imported. " 1241 "Ensure you are using version %s or higher.", 1242 WINRM_MIN_VER, 1243 ) 1244 return False 1245 1246 starttime = time.mktime(time.localtime()) 1247 log.debug("Deploying %s at %s (Windows)", host, starttime) 1248 log.trace("HAS_WINRM: %s, use_winrm: %s", HAS_WINRM, use_winrm) 1249 1250 port_available = wait_for_port(host=host, port=port, timeout=port_timeout * 60) 1251 1252 if not port_available: 1253 return False 1254 1255 service_available = False 1256 winrm_session = None 1257 1258 if HAS_WINRM and use_winrm: 1259 winrm_session = wait_for_winrm( 1260 host=host, 1261 port=winrm_port, 1262 username=username, 1263 password=password, 1264 timeout=port_timeout * 60, 1265 use_ssl=winrm_use_ssl, 1266 verify=winrm_verify_ssl, 1267 ) 1268 if winrm_session is not None: 1269 service_available = True 1270 else: 1271 service_available = wait_for_psexecsvc( 1272 host=host, 1273 port=port, 1274 username=username, 1275 password=password, 1276 timeout=port_timeout * 60, 1277 ) 1278 1279 if port_available and service_available: 1280 log.debug("SMB port %s on %s is available", port, host) 1281 log.debug("Logging into %s:%s as %s", host, port, username) 1282 smb_conn = salt.utils.smb.get_conn(host, username, password, port) 1283 if smb_conn is False: 1284 log.error("Please install smbprotocol to enable SMB functionality") 1285 return False 1286 1287 salt.utils.smb.mkdirs("salttemp", conn=smb_conn) 1288 salt.utils.smb.mkdirs("salt/conf/pki/minion", conn=smb_conn) 1289 1290 if minion_pub: 1291 salt.utils.smb.put_str( 1292 minion_pub, "salt\\conf\\pki\\minion\\minion.pub", conn=smb_conn 1293 ) 1294 1295 if minion_pem: 1296 salt.utils.smb.put_str( 1297 minion_pem, "salt\\conf\\pki\\minion\\minion.pem", conn=smb_conn 1298 ) 1299 1300 if master_sign_pub_file: 1301 # Read master-sign.pub file 1302 log.debug( 1303 "Copying master_sign.pub file from %s to minion", master_sign_pub_file 1304 ) 1305 try: 1306 salt.utils.smb.put_file( 1307 master_sign_pub_file, 1308 "salt\\conf\\pki\\minion\\master_sign.pub", 1309 "C$", 1310 conn=smb_conn, 1311 ) 1312 except Exception as e: # pylint: disable=broad-except 1313 log.debug( 1314 "Exception copying master_sign.pub file %s to minion", 1315 master_sign_pub_file, 1316 ) 1317 1318 # Copy over win_installer 1319 # win_installer refers to a file such as: 1320 # /root/Salt-Minion-0.17.0-win32-Setup.exe 1321 # ..which exists on the same machine as salt-cloud 1322 comps = win_installer.split("/") 1323 local_path = "/".join(comps[:-1]) 1324 installer = comps[-1] 1325 salt.utils.smb.put_file( 1326 win_installer, 1327 "salttemp\\{}".format(installer), 1328 "C$", 1329 conn=smb_conn, 1330 ) 1331 1332 if use_winrm: 1333 winrm_cmd( 1334 winrm_session, 1335 "c:\\salttemp\\{}".format(installer), 1336 ["/S", "/master={}".format(master), "/minion-name={}".format(name)], 1337 ) 1338 else: 1339 cmd = "c:\\salttemp\\{}".format(installer) 1340 args = "/S /master={} /minion-name={}".format(master, name) 1341 stdout, stderr, ret_code = run_psexec_command( 1342 cmd, args, host, username, password 1343 ) 1344 1345 if ret_code != 0: 1346 raise Exception("Fail installer {}".format(ret_code)) 1347 1348 # Copy over minion_conf 1349 if minion_conf: 1350 if not isinstance(minion_conf, dict): 1351 # Let's not just fail regarding this change, specially 1352 # since we can handle it 1353 raise DeprecationWarning( 1354 "`salt.utils.cloud.deploy_windows` now only accepts " 1355 "dictionaries for its `minion_conf` parameter. " 1356 "Loading YAML..." 1357 ) 1358 minion_grains = minion_conf.pop("grains", {}) 1359 if minion_grains: 1360 salt.utils.smb.put_str( 1361 salt_config_to_yaml(minion_grains, line_break="\r\n"), 1362 "salt\\conf\\grains", 1363 conn=smb_conn, 1364 ) 1365 # Add special windows minion configuration 1366 # that must be in the minion config file 1367 windows_minion_conf = { 1368 "ipc_mode": "tcp", 1369 "root_dir": "c:\\salt", 1370 "pki_dir": "/conf/pki/minion", 1371 "multiprocessing": False, 1372 } 1373 minion_conf = dict(minion_conf, **windows_minion_conf) 1374 salt.utils.smb.put_str( 1375 salt_config_to_yaml(minion_conf, line_break="\r\n"), 1376 "salt\\conf\\minion", 1377 conn=smb_conn, 1378 ) 1379 # Delete C:\salttmp\ and installer file 1380 # Unless keep_tmp is True 1381 if not keep_tmp: 1382 if use_winrm: 1383 winrm_cmd(winrm_session, "rmdir", ["/Q", "/S", "C:\\salttemp\\"]) 1384 else: 1385 salt.utils.smb.delete_file( 1386 "salttemp\\{}".format(installer), "C$", conn=smb_conn 1387 ) 1388 salt.utils.smb.delete_directory("salttemp", "C$", conn=smb_conn) 1389 # Shell out to psexec to ensure salt-minion service started 1390 if use_winrm: 1391 winrm_cmd(winrm_session, "sc", ["stop", "salt-minion"]) 1392 time.sleep(5) 1393 winrm_cmd(winrm_session, "sc", ["start", "salt-minion"]) 1394 else: 1395 stdout, stderr, ret_code = run_psexec_command( 1396 "cmd.exe", "/c sc stop salt-minion", host, username, password 1397 ) 1398 if ret_code != 0: 1399 return False 1400 1401 time.sleep(5) 1402 1403 log.debug("Run psexec: sc start salt-minion") 1404 stdout, stderr, ret_code = run_psexec_command( 1405 "cmd.exe", "/c sc start salt-minion", host, username, password 1406 ) 1407 if ret_code != 0: 1408 return False 1409 1410 # Fire deploy action 1411 fire_event( 1412 "event", 1413 "{} has been deployed at {}".format(name, host), 1414 "salt/cloud/{}/deploy_windows".format(name), 1415 args={"name": name}, 1416 sock_dir=opts.get("sock_dir", os.path.join(__opts__["sock_dir"], "master")), 1417 transport=opts.get("transport", "zeromq"), 1418 ) 1419 1420 return True 1421 return False 1422 1423 1424def deploy_script( 1425 host, 1426 port=22, 1427 timeout=900, 1428 username="root", 1429 password=None, 1430 key_filename=None, 1431 script=None, 1432 name=None, 1433 sock_dir=None, 1434 provider=None, 1435 conf_file=None, 1436 start_action=None, 1437 make_master=False, 1438 master_pub=None, 1439 master_pem=None, 1440 master_conf=None, 1441 minion_pub=None, 1442 minion_pem=None, 1443 minion_conf=None, 1444 keep_tmp=False, 1445 script_args=None, 1446 script_env=None, 1447 ssh_timeout=15, 1448 maxtries=15, 1449 make_syndic=False, 1450 make_minion=True, 1451 display_ssh_output=True, 1452 preseed_minion_keys=None, 1453 parallel=False, 1454 sudo_password=None, 1455 sudo=False, 1456 tty=None, 1457 vm_=None, 1458 opts=None, 1459 tmp_dir="/tmp/.saltcloud", 1460 file_map=None, 1461 master_sign_pub_file=None, 1462 cloud_grains=None, 1463 force_minion_config=False, 1464 **kwargs 1465): 1466 """ 1467 Copy a deploy script to a remote server, execute it, and remove it 1468 """ 1469 if not isinstance(opts, dict): 1470 opts = {} 1471 vm_ = vm_ or {} # if None, default to empty dict 1472 cloud_grains = cloud_grains or {} 1473 1474 tmp_dir = "{}-{}".format(tmp_dir.rstrip("/"), uuid.uuid4()) 1475 deploy_command = salt.config.get_cloud_config_value( 1476 "deploy_command", vm_, opts, default=os.path.join(tmp_dir, "deploy.sh") 1477 ) 1478 if key_filename is not None and not os.path.isfile(key_filename): 1479 raise SaltCloudConfigError( 1480 "The defined key_filename '{}' does not exist".format(key_filename) 1481 ) 1482 1483 gateway = None 1484 if "gateway" in kwargs: 1485 gateway = kwargs["gateway"] 1486 1487 starttime = time.localtime() 1488 log.debug("Deploying %s at %s", host, time.strftime("%Y-%m-%d %H:%M:%S", starttime)) 1489 known_hosts_file = kwargs.get("known_hosts_file", "/dev/null") 1490 hard_timeout = opts.get("hard_timeout", None) 1491 1492 if wait_for_port(host=host, port=port, gateway=gateway): 1493 log.debug("SSH port %s on %s is available", port, host) 1494 if wait_for_passwd( 1495 host, 1496 port=port, 1497 username=username, 1498 password=password, 1499 key_filename=key_filename, 1500 ssh_timeout=ssh_timeout, 1501 display_ssh_output=display_ssh_output, 1502 gateway=gateway, 1503 known_hosts_file=known_hosts_file, 1504 maxtries=maxtries, 1505 hard_timeout=hard_timeout, 1506 ): 1507 1508 log.debug("Logging into %s:%s as %s", host, port, username) 1509 ssh_kwargs = { 1510 "hostname": host, 1511 "port": port, 1512 "username": username, 1513 "timeout": ssh_timeout, 1514 "ssh_timeout": ssh_timeout, 1515 "display_ssh_output": display_ssh_output, 1516 "sudo_password": sudo_password, 1517 "sftp": opts.get("use_sftp", False), 1518 } 1519 ssh_kwargs.update(__ssh_gateway_config_dict(gateway)) 1520 if key_filename: 1521 log.debug("Using %s as the key_filename", key_filename) 1522 ssh_kwargs["key_filename"] = key_filename 1523 elif password and kwargs.get("has_ssh_agent", False) is False: 1524 ssh_kwargs["password"] = password 1525 1526 if root_cmd( 1527 "test -e '{}'".format(tmp_dir), 1528 tty, 1529 sudo, 1530 allow_failure=True, 1531 **ssh_kwargs 1532 ): 1533 ret = root_cmd( 1534 "sh -c \"( mkdir -p -m 700 '{}' )\"".format(tmp_dir), 1535 tty, 1536 sudo, 1537 **ssh_kwargs 1538 ) 1539 if ret: 1540 raise SaltCloudSystemExit( 1541 "Can't create temporary directory in {} !".format(tmp_dir) 1542 ) 1543 if sudo: 1544 comps = tmp_dir.lstrip("/").rstrip("/").split("/") 1545 if comps: 1546 if len(comps) > 1 or comps[0] != "tmp": 1547 ret = root_cmd( 1548 'chown {} "{}"'.format(username, tmp_dir), 1549 tty, 1550 sudo, 1551 **ssh_kwargs 1552 ) 1553 if ret: 1554 raise SaltCloudSystemExit( 1555 "Cant set {} ownership on {}".format(username, tmp_dir) 1556 ) 1557 1558 if not isinstance(file_map, dict): 1559 file_map = {} 1560 1561 # Copy an arbitrary group of files to the target system 1562 remote_dirs = [] 1563 file_map_success = [] 1564 file_map_fail = [] 1565 for map_item in file_map: 1566 local_file = map_item 1567 remote_file = file_map[map_item] 1568 if not os.path.exists(map_item): 1569 log.error( 1570 'The local file "%s" does not exist, and will not be ' 1571 'copied to "%s" on the target system', 1572 local_file, 1573 remote_file, 1574 ) 1575 file_map_fail.append({local_file: remote_file}) 1576 continue 1577 1578 if os.path.isdir(local_file): 1579 dir_name = os.path.basename(local_file) 1580 remote_dir = os.path.join(os.path.dirname(remote_file), dir_name) 1581 else: 1582 remote_dir = os.path.dirname(remote_file) 1583 1584 if remote_dir not in remote_dirs: 1585 root_cmd( 1586 "mkdir -p '{}'".format(remote_dir), tty, sudo, **ssh_kwargs 1587 ) 1588 if ssh_kwargs["username"] != "root": 1589 root_cmd( 1590 "chown {} '{}'".format(ssh_kwargs["username"], remote_dir), 1591 tty, 1592 sudo, 1593 **ssh_kwargs 1594 ) 1595 remote_dirs.append(remote_dir) 1596 ssh_file(opts, remote_file, kwargs=ssh_kwargs, local_file=local_file) 1597 file_map_success.append({local_file: remote_file}) 1598 1599 # Minion configuration 1600 if minion_pem: 1601 ssh_file(opts, "{}/minion.pem".format(tmp_dir), minion_pem, ssh_kwargs) 1602 ret = root_cmd( 1603 "chmod 600 '{}/minion.pem'".format(tmp_dir), tty, sudo, **ssh_kwargs 1604 ) 1605 if ret: 1606 raise SaltCloudSystemExit( 1607 "Can't set perms on {}/minion.pem".format(tmp_dir) 1608 ) 1609 if minion_pub: 1610 ssh_file(opts, "{}/minion.pub".format(tmp_dir), minion_pub, ssh_kwargs) 1611 1612 if master_sign_pub_file: 1613 ssh_file( 1614 opts, 1615 "{}/master_sign.pub".format(tmp_dir), 1616 kwargs=ssh_kwargs, 1617 local_file=master_sign_pub_file, 1618 ) 1619 1620 if minion_conf: 1621 if not isinstance(minion_conf, dict): 1622 # Let's not just fail regarding this change, specially 1623 # since we can handle it 1624 raise DeprecationWarning( 1625 "`salt.utils.cloud.deploy_script now only accepts " 1626 "dictionaries for it's `minion_conf` parameter. " 1627 "Loading YAML..." 1628 ) 1629 minion_grains = minion_conf.pop("grains", {}) 1630 if minion_grains: 1631 ssh_file( 1632 opts, 1633 "{}/grains".format(tmp_dir), 1634 salt_config_to_yaml(minion_grains), 1635 ssh_kwargs, 1636 ) 1637 if cloud_grains and opts.get("enable_cloud_grains", True): 1638 minion_conf["grains"] = {"salt-cloud": cloud_grains} 1639 ssh_file( 1640 opts, 1641 "{}/minion".format(tmp_dir), 1642 salt_config_to_yaml(minion_conf), 1643 ssh_kwargs, 1644 ) 1645 1646 # Master configuration 1647 if master_pem: 1648 ssh_file(opts, "{}/master.pem".format(tmp_dir), master_pem, ssh_kwargs) 1649 ret = root_cmd( 1650 "chmod 600 '{}/master.pem'".format(tmp_dir), tty, sudo, **ssh_kwargs 1651 ) 1652 if ret: 1653 raise SaltCloudSystemExit( 1654 "Cant set perms on {}/master.pem".format(tmp_dir) 1655 ) 1656 1657 if master_pub: 1658 ssh_file(opts, "{}/master.pub".format(tmp_dir), master_pub, ssh_kwargs) 1659 1660 if master_conf: 1661 if not isinstance(master_conf, dict): 1662 # Let's not just fail regarding this change, specially 1663 # since we can handle it 1664 raise DeprecationWarning( 1665 "`salt.utils.cloud.deploy_script now only accepts " 1666 "dictionaries for it's `master_conf` parameter. " 1667 "Loading from YAML ..." 1668 ) 1669 1670 ssh_file( 1671 opts, 1672 "{}/master".format(tmp_dir), 1673 salt_config_to_yaml(master_conf), 1674 ssh_kwargs, 1675 ) 1676 1677 # XXX: We need to make these paths configurable 1678 preseed_minion_keys_tempdir = "{}/preseed-minion-keys".format(tmp_dir) 1679 if preseed_minion_keys is not None: 1680 # Create remote temp dir 1681 ret = root_cmd( 1682 "mkdir '{}'".format(preseed_minion_keys_tempdir), 1683 tty, 1684 sudo, 1685 **ssh_kwargs 1686 ) 1687 if ret: 1688 raise SaltCloudSystemExit( 1689 "Cant create {}".format(preseed_minion_keys_tempdir) 1690 ) 1691 ret = root_cmd( 1692 "chmod 700 '{}'".format(preseed_minion_keys_tempdir), 1693 tty, 1694 sudo, 1695 **ssh_kwargs 1696 ) 1697 if ret: 1698 raise SaltCloudSystemExit( 1699 "Can't set perms on {}".format(preseed_minion_keys_tempdir) 1700 ) 1701 if ssh_kwargs["username"] != "root": 1702 root_cmd( 1703 "chown {} '{}'".format( 1704 ssh_kwargs["username"], preseed_minion_keys_tempdir 1705 ), 1706 tty, 1707 sudo, 1708 **ssh_kwargs 1709 ) 1710 1711 # Copy pre-seed minion keys 1712 for minion_id, minion_key in preseed_minion_keys.items(): 1713 rpath = os.path.join(preseed_minion_keys_tempdir, minion_id) 1714 ssh_file(opts, rpath, minion_key, ssh_kwargs) 1715 1716 if ssh_kwargs["username"] != "root": 1717 root_cmd( 1718 "chown -R root '{}'".format(preseed_minion_keys_tempdir), 1719 tty, 1720 sudo, 1721 **ssh_kwargs 1722 ) 1723 if ret: 1724 raise SaltCloudSystemExit( 1725 "Can't set ownership for {}".format( 1726 preseed_minion_keys_tempdir 1727 ) 1728 ) 1729 1730 # Run any pre-flight commands before running deploy scripts 1731 preflight_cmds = kwargs.get("preflight_cmds", []) 1732 for command in preflight_cmds: 1733 cmd_ret = root_cmd(command, tty, sudo, **ssh_kwargs) 1734 if cmd_ret: 1735 raise SaltCloudSystemExit( 1736 "Pre-flight command failed: '{}'".format(command) 1737 ) 1738 1739 # The actual deploy script 1740 if script: 1741 # got strange escaping issues with sudoer, going onto a 1742 # subshell fixes that 1743 ssh_file(opts, "{}/deploy.sh".format(tmp_dir), script, ssh_kwargs) 1744 ret = root_cmd( 1745 "sh -c \"( chmod +x '{}/deploy.sh' )\";exit $?".format(tmp_dir), 1746 tty, 1747 sudo, 1748 **ssh_kwargs 1749 ) 1750 if ret: 1751 raise SaltCloudSystemExit( 1752 "Can't set perms on {}/deploy.sh".format(tmp_dir) 1753 ) 1754 1755 time_used = time.mktime(time.localtime()) - time.mktime(starttime) 1756 newtimeout = timeout - time_used 1757 queue = None 1758 process = None 1759 # Consider this code experimental. It causes Salt Cloud to wait 1760 # for the minion to check in, and then fire a startup event. 1761 # Disabled if parallel because it doesn't work! 1762 if start_action and not parallel: 1763 queue = multiprocessing.Queue() 1764 process = multiprocessing.Process( 1765 target=check_auth, 1766 kwargs=dict( 1767 name=name, sock_dir=sock_dir, timeout=newtimeout, queue=queue 1768 ), 1769 ) 1770 log.debug("Starting new process to wait for salt-minion") 1771 process.start() 1772 1773 # Run the deploy script 1774 if script: 1775 if "bootstrap-salt" in script: 1776 deploy_command += " -c '{}'".format(tmp_dir) 1777 if force_minion_config: 1778 deploy_command += " -F" 1779 if make_syndic is True: 1780 deploy_command += " -S" 1781 if make_master is True: 1782 deploy_command += " -M" 1783 if make_minion is False: 1784 deploy_command += " -N" 1785 if keep_tmp is True: 1786 deploy_command += " -K" 1787 if preseed_minion_keys is not None: 1788 deploy_command += " -k '{}'".format(preseed_minion_keys_tempdir) 1789 if script_args: 1790 deploy_command += " {}".format(script_args) 1791 1792 if script_env: 1793 if not isinstance(script_env, dict): 1794 raise SaltCloudSystemExit( 1795 "The 'script_env' configuration setting NEEDS " 1796 "to be a dictionary not a {}".format(type(script_env)) 1797 ) 1798 environ_script_contents = ["#!/bin/sh"] 1799 for key, value in script_env.items(): 1800 environ_script_contents.append( 1801 "setenv {0} '{1}' >/dev/null 2>&1 || " 1802 "export {0}='{1}'".format(key, value) 1803 ) 1804 environ_script_contents.append(deploy_command) 1805 1806 # Upload our environ setter wrapper 1807 ssh_file( 1808 opts, 1809 "{}/environ-deploy-wrapper.sh".format(tmp_dir), 1810 "\n".join(environ_script_contents), 1811 ssh_kwargs, 1812 ) 1813 root_cmd( 1814 "chmod +x '{}/environ-deploy-wrapper.sh'".format(tmp_dir), 1815 tty, 1816 sudo, 1817 **ssh_kwargs 1818 ) 1819 # The deploy command is now our wrapper 1820 deploy_command = "'{}/environ-deploy-wrapper.sh'".format( 1821 tmp_dir, 1822 ) 1823 if root_cmd(deploy_command, tty, sudo, **ssh_kwargs) != 0: 1824 raise SaltCloudSystemExit( 1825 "Executing the command '{}' failed".format(deploy_command) 1826 ) 1827 log.debug("Executed command '%s'", deploy_command) 1828 1829 # Remove the deploy script 1830 if not keep_tmp: 1831 root_cmd( 1832 "rm -f '{}/deploy.sh'".format(tmp_dir), tty, sudo, **ssh_kwargs 1833 ) 1834 log.debug("Removed %s/deploy.sh", tmp_dir) 1835 if script_env: 1836 root_cmd( 1837 "rm -f '{}/environ-deploy-wrapper.sh'".format(tmp_dir), 1838 tty, 1839 sudo, 1840 **ssh_kwargs 1841 ) 1842 log.debug("Removed %s/environ-deploy-wrapper.sh", tmp_dir) 1843 1844 if keep_tmp: 1845 log.debug("Not removing deployment files from %s/", tmp_dir) 1846 else: 1847 # Remove minion configuration 1848 if minion_pub: 1849 root_cmd( 1850 "rm -f '{}/minion.pub'".format(tmp_dir), tty, sudo, **ssh_kwargs 1851 ) 1852 log.debug("Removed %s/minion.pub", tmp_dir) 1853 if minion_pem: 1854 root_cmd( 1855 "rm -f '{}/minion.pem'".format(tmp_dir), tty, sudo, **ssh_kwargs 1856 ) 1857 log.debug("Removed %s/minion.pem", tmp_dir) 1858 if minion_conf: 1859 root_cmd( 1860 "rm -f '{}/grains'".format(tmp_dir), tty, sudo, **ssh_kwargs 1861 ) 1862 log.debug("Removed %s/grains", tmp_dir) 1863 root_cmd( 1864 "rm -f '{}/minion'".format(tmp_dir), tty, sudo, **ssh_kwargs 1865 ) 1866 log.debug("Removed %s/minion", tmp_dir) 1867 if master_sign_pub_file: 1868 root_cmd( 1869 "rm -f {}/master_sign.pub".format(tmp_dir), 1870 tty, 1871 sudo, 1872 **ssh_kwargs 1873 ) 1874 log.debug("Removed %s/master_sign.pub", tmp_dir) 1875 1876 # Remove master configuration 1877 if master_pub: 1878 root_cmd( 1879 "rm -f '{}/master.pub'".format(tmp_dir), tty, sudo, **ssh_kwargs 1880 ) 1881 log.debug("Removed %s/master.pub", tmp_dir) 1882 if master_pem: 1883 root_cmd( 1884 "rm -f '{}/master.pem'".format(tmp_dir), tty, sudo, **ssh_kwargs 1885 ) 1886 log.debug("Removed %s/master.pem", tmp_dir) 1887 if master_conf: 1888 root_cmd( 1889 "rm -f '{}/master'".format(tmp_dir), tty, sudo, **ssh_kwargs 1890 ) 1891 log.debug("Removed %s/master", tmp_dir) 1892 1893 # Remove pre-seed keys directory 1894 if preseed_minion_keys is not None: 1895 root_cmd( 1896 "rm -rf '{}'".format(preseed_minion_keys_tempdir), 1897 tty, 1898 sudo, 1899 **ssh_kwargs 1900 ) 1901 log.debug("Removed %s", preseed_minion_keys_tempdir) 1902 1903 if start_action and not parallel: 1904 queuereturn = queue.get() 1905 process.join() 1906 if queuereturn and start_action: 1907 # client = salt.client.LocalClient(conf_file) 1908 # output = client.cmd_iter( 1909 # host, 'state.highstate', timeout=timeout 1910 # ) 1911 # for line in output: 1912 # print(line) 1913 log.info("Executing %s on the salt-minion", start_action) 1914 root_cmd( 1915 "salt-call {}".format(start_action), tty, sudo, **ssh_kwargs 1916 ) 1917 log.info("Finished executing %s on the salt-minion", start_action) 1918 # Fire deploy action 1919 fire_event( 1920 "event", 1921 "{} has been deployed at {}".format(name, host), 1922 "salt/cloud/{}/deploy_script".format(name), 1923 args={"name": name, "host": host}, 1924 sock_dir=opts.get( 1925 "sock_dir", os.path.join(__opts__["sock_dir"], "master") 1926 ), 1927 transport=opts.get("transport", "zeromq"), 1928 ) 1929 if file_map_fail or file_map_success: 1930 return { 1931 "File Upload Success": file_map_success, 1932 "File Upload Failure": file_map_fail, 1933 } 1934 return True 1935 return False 1936 1937 1938def run_inline_script( 1939 host, 1940 name=None, 1941 port=22, 1942 timeout=900, 1943 username="root", 1944 key_filename=None, 1945 inline_script=None, 1946 ssh_timeout=15, 1947 display_ssh_output=True, 1948 parallel=False, 1949 sudo_password=None, 1950 sudo=False, 1951 password=None, 1952 tty=None, 1953 opts=None, 1954 tmp_dir="/tmp/.saltcloud-inline_script", 1955 **kwargs 1956): 1957 """ 1958 Run the inline script commands, one by one 1959 :**kwargs: catch all other things we may get but don't actually need/use 1960 """ 1961 1962 gateway = None 1963 if "gateway" in kwargs: 1964 gateway = kwargs["gateway"] 1965 1966 starttime = time.mktime(time.localtime()) 1967 log.debug("Deploying %s at %s", host, starttime) 1968 1969 known_hosts_file = kwargs.get("known_hosts_file", "/dev/null") 1970 1971 if wait_for_port(host=host, port=port, gateway=gateway): 1972 log.debug("SSH port %s on %s is available", port, host) 1973 newtimeout = timeout - (time.mktime(time.localtime()) - starttime) 1974 if wait_for_passwd( 1975 host, 1976 port=port, 1977 username=username, 1978 password=password, 1979 key_filename=key_filename, 1980 ssh_timeout=ssh_timeout, 1981 display_ssh_output=display_ssh_output, 1982 gateway=gateway, 1983 known_hosts_file=known_hosts_file, 1984 ): 1985 1986 log.debug("Logging into %s:%s as %s", host, port, username) 1987 newtimeout = timeout - (time.mktime(time.localtime()) - starttime) 1988 ssh_kwargs = { 1989 "hostname": host, 1990 "port": port, 1991 "username": username, 1992 "timeout": ssh_timeout, 1993 "display_ssh_output": display_ssh_output, 1994 "sudo_password": sudo_password, 1995 "sftp": opts.get("use_sftp", False), 1996 } 1997 ssh_kwargs.update(__ssh_gateway_config_dict(gateway)) 1998 if key_filename: 1999 log.debug("Using %s as the key_filename", key_filename) 2000 ssh_kwargs["key_filename"] = key_filename 2001 elif ( 2002 password 2003 and "has_ssh_agent" in kwargs 2004 and kwargs["has_ssh_agent"] is False 2005 ): 2006 ssh_kwargs["password"] = password 2007 2008 # TODO: write some tests ??? 2009 # TODO: check edge cases (e.g. ssh gateways, salt deploy disabled, etc.) 2010 if ( 2011 root_cmd( 2012 'test -e \\"{}\\"'.format(tmp_dir), 2013 tty, 2014 sudo, 2015 allow_failure=True, 2016 **ssh_kwargs 2017 ) 2018 and inline_script 2019 ): 2020 log.debug("Found inline script to execute.") 2021 for cmd_line in inline_script: 2022 log.info("Executing inline command: %s", cmd_line) 2023 ret = root_cmd( 2024 'sh -c "( {} )"'.format(cmd_line), 2025 tty, 2026 sudo, 2027 allow_failure=True, 2028 **ssh_kwargs 2029 ) 2030 if ret: 2031 log.info("[%s] Output: %s", cmd_line, ret) 2032 2033 # TODO: ensure we send the correct return value 2034 return True 2035 2036 2037def filter_event(tag, data, defaults): 2038 """ 2039 Accept a tag, a dict and a list of default keys to return from the dict, and 2040 check them against the cloud configuration for that tag 2041 """ 2042 ret = {} 2043 keys = [] 2044 use_defaults = True 2045 2046 for ktag in __opts__.get("filter_events", {}): 2047 if tag != ktag: 2048 continue 2049 keys = __opts__["filter_events"][ktag]["keys"] 2050 use_defaults = __opts__["filter_events"][ktag].get("use_defaults", True) 2051 2052 if use_defaults is False: 2053 defaults = [] 2054 2055 # For PY3, if something like ".keys()" or ".values()" is used on a dictionary, 2056 # it returns a dict_view and not a list like in PY2. "defaults" should be passed 2057 # in with the correct data type, but don't stack-trace in case it wasn't. 2058 if not isinstance(defaults, list): 2059 defaults = list(defaults) 2060 2061 defaults = list(set(defaults + keys)) 2062 2063 for key in defaults: 2064 if key in data: 2065 ret[key] = data[key] 2066 2067 return ret 2068 2069 2070def fire_event(key, msg, tag, sock_dir, args=None, transport="zeromq"): 2071 """ 2072 Fire deploy action 2073 """ 2074 with salt.utils.event.get_event( 2075 "master", sock_dir, transport, listen=False 2076 ) as event: 2077 try: 2078 event.fire_event(msg, tag) 2079 except ValueError: 2080 # We're using at least a 0.17.x version of salt 2081 if isinstance(args, dict): 2082 args[key] = msg 2083 else: 2084 args = {key: msg} 2085 event.fire_event(args, tag) 2086 2087 # https://github.com/zeromq/pyzmq/issues/173#issuecomment-4037083 2088 # Assertion failed: get_load () == 0 (poller_base.cpp:32) 2089 time.sleep(0.025) 2090 2091 2092def _exec_ssh_cmd(cmd, error_msg=None, allow_failure=False, **kwargs): 2093 if error_msg is None: 2094 error_msg = "A wrong password has been issued while establishing ssh session." 2095 password_retries = kwargs.get("password_retries", 3) 2096 try: 2097 stdout, stderr = None, None 2098 proc = salt.utils.vt.Terminal( 2099 cmd, 2100 shell=True, 2101 log_stdout=True, 2102 log_stderr=True, 2103 stream_stdout=kwargs.get("display_ssh_output", True), 2104 stream_stderr=kwargs.get("display_ssh_output", True), 2105 ) 2106 sent_password = 0 2107 while proc.has_unread_data: 2108 stdout, stderr = proc.recv() 2109 if stdout and SSH_PASSWORD_PROMP_RE.search(stdout): 2110 # if authenticating with an SSH key and 'sudo' is found 2111 # in the password prompt 2112 if ( 2113 "key_filename" in kwargs 2114 and kwargs["key_filename"] 2115 and SSH_PASSWORD_PROMP_SUDO_RE.search(stdout) 2116 ): 2117 proc.sendline(kwargs["sudo_password"]) 2118 # elif authenticating via password and haven't exhausted our 2119 # password_retires 2120 elif kwargs.get("password", None) and ( 2121 sent_password < password_retries 2122 ): 2123 sent_password += 1 2124 proc.sendline(kwargs["password"]) 2125 # else raise an error as we are not authenticating properly 2126 # * not authenticating with an SSH key 2127 # * not authenticating with a Password 2128 else: 2129 raise SaltCloudPasswordError(error_msg) 2130 # 0.0125 is really too fast on some systems 2131 time.sleep(0.5) 2132 if proc.exitstatus != 0 and allow_failure is False: 2133 raise SaltCloudSystemExit( 2134 "Command '{}' failed. Exit code: {}".format(cmd, proc.exitstatus) 2135 ) 2136 return proc.exitstatus 2137 except salt.utils.vt.TerminalException as err: 2138 trace = traceback.format_exc() 2139 log.error( 2140 error_msg.format(cmd, err, trace) 2141 ) # pylint: disable=str-format-in-logging 2142 finally: 2143 proc.close(terminate=True, kill=True) 2144 # Signal an error 2145 return 1 2146 2147 2148def scp_file(dest_path, contents=None, kwargs=None, local_file=None): 2149 """ 2150 Use scp or sftp to copy a file to a server 2151 """ 2152 file_to_upload = None 2153 try: 2154 if contents is not None: 2155 try: 2156 tmpfd, file_to_upload = tempfile.mkstemp() 2157 os.write(tmpfd, contents) 2158 finally: 2159 try: 2160 os.close(tmpfd) 2161 except OSError as exc: 2162 if exc.errno != errno.EBADF: 2163 raise 2164 2165 log.debug("Uploading %s to %s", dest_path, kwargs["hostname"]) 2166 2167 ssh_args = [ 2168 # Don't add new hosts to the host key database 2169 "-oStrictHostKeyChecking=no", 2170 # make sure ssh can time out on connection lose 2171 "-oServerAliveInterval={}".format( 2172 kwargs.get("server_alive_interval", SERVER_ALIVE_INTERVAL) 2173 ), 2174 "-oServerAliveCountMax={}".format( 2175 kwargs.get("server_alive_count_max", SERVER_ALIVE_COUNT_MAX) 2176 ), 2177 # Set hosts key database path to /dev/null, i.e., non-existing 2178 "-oUserKnownHostsFile=/dev/null", 2179 # Don't re-use the SSH connection. Less failures. 2180 "-oControlPath=none", 2181 ] 2182 2183 if local_file is not None: 2184 file_to_upload = local_file 2185 if os.path.isdir(local_file): 2186 ssh_args.append("-r") 2187 2188 if "key_filename" in kwargs: 2189 # There should never be both a password and an ssh key passed in, so 2190 ssh_args.extend( 2191 [ 2192 # tell SSH to skip password authentication 2193 "-oPasswordAuthentication=no", 2194 "-oChallengeResponseAuthentication=no", 2195 # Make sure public key authentication is enabled 2196 "-oPubkeyAuthentication=yes", 2197 # do only use the provided identity file 2198 "-oIdentitiesOnly=yes", 2199 # No Keyboard interaction! 2200 "-oKbdInteractiveAuthentication=no", 2201 # Also, specify the location of the key file 2202 "-i {}".format(kwargs["key_filename"]), 2203 ] 2204 ) 2205 2206 if "port" in kwargs: 2207 ssh_args.append("-oPort={}".format(kwargs["port"])) 2208 2209 ssh_args.append(__ssh_gateway_arguments(kwargs)) 2210 2211 try: 2212 if socket.inet_pton(socket.AF_INET6, kwargs["hostname"]): 2213 ipaddr = "[{}]".format(kwargs["hostname"]) 2214 else: 2215 ipaddr = kwargs["hostname"] 2216 except OSError: 2217 ipaddr = kwargs["hostname"] 2218 2219 if file_to_upload is None: 2220 log.warning( 2221 "No source file to upload. Please make sure that either file " 2222 "contents or the path to a local file are provided." 2223 ) 2224 cmd = ( 2225 "scp {0} {1} {2[username]}@{4}:{3} || " 2226 'echo "put {1} {3}" | sftp {0} {2[username]}@{4} || ' 2227 'rsync -avz -e "ssh {0}" {1} {2[username]}@{2[hostname]}:{3}'.format( 2228 " ".join(ssh_args), file_to_upload, kwargs, dest_path, ipaddr 2229 ) 2230 ) 2231 2232 log.debug("SCP command: '%s'", cmd) 2233 retcode = _exec_ssh_cmd( 2234 cmd, 2235 error_msg="Failed to upload file '{0}': {1}\n{2}", 2236 password_retries=3, 2237 **kwargs 2238 ) 2239 finally: 2240 if contents is not None: 2241 try: 2242 os.remove(file_to_upload) 2243 except OSError as exc: 2244 if exc.errno != errno.ENOENT: 2245 raise 2246 return retcode 2247 2248 2249def ssh_file(opts, dest_path, contents=None, kwargs=None, local_file=None): 2250 """ 2251 Copies a file to the remote SSH target using either sftp or scp, as 2252 configured. 2253 """ 2254 if opts.get("file_transport", "sftp") == "sftp": 2255 return sftp_file(dest_path, contents, kwargs, local_file) 2256 return scp_file(dest_path, contents, kwargs, local_file) 2257 2258 2259def sftp_file(dest_path, contents=None, kwargs=None, local_file=None): 2260 """ 2261 Use sftp to upload a file to a server 2262 """ 2263 put_args = [] 2264 2265 if kwargs is None: 2266 kwargs = {} 2267 2268 file_to_upload = None 2269 try: 2270 if contents is not None: 2271 try: 2272 tmpfd, file_to_upload = tempfile.mkstemp() 2273 if isinstance(contents, str): 2274 os.write(tmpfd, contents.encode(__salt_system_encoding__)) 2275 else: 2276 os.write(tmpfd, contents) 2277 finally: 2278 try: 2279 os.close(tmpfd) 2280 except OSError as exc: 2281 if exc.errno != errno.EBADF: 2282 raise 2283 2284 if local_file is not None: 2285 file_to_upload = local_file 2286 if os.path.isdir(local_file): 2287 put_args = ["-r"] 2288 2289 log.debug("Uploading %s to %s (sftp)", dest_path, kwargs.get("hostname")) 2290 2291 ssh_args = [ 2292 # Don't add new hosts to the host key database 2293 "-oStrictHostKeyChecking=no", 2294 # make sure ssh can time out on connection lose 2295 "-oServerAliveInterval={}".format( 2296 kwargs.get("server_alive_interval", SERVER_ALIVE_INTERVAL) 2297 ), 2298 "-oServerAliveCountMax={}".format( 2299 kwargs.get("server_alive_count_max", SERVER_ALIVE_COUNT_MAX) 2300 ), 2301 # Set hosts key database path to /dev/null, i.e., non-existing 2302 "-oUserKnownHostsFile=/dev/null", 2303 # Don't re-use the SSH connection. Less failures. 2304 "-oControlPath=none", 2305 ] 2306 if "key_filename" in kwargs: 2307 # There should never be both a password and an ssh key passed in, so 2308 ssh_args.extend( 2309 [ 2310 # tell SSH to skip password authentication 2311 "-oPasswordAuthentication=no", 2312 "-oChallengeResponseAuthentication=no", 2313 # Make sure public key authentication is enabled 2314 "-oPubkeyAuthentication=yes", 2315 # do only use the provided identity file 2316 "-oIdentitiesOnly=yes", 2317 # No Keyboard interaction! 2318 "-oKbdInteractiveAuthentication=no", 2319 # Also, specify the location of the key file 2320 "-oIdentityFile={}".format(kwargs["key_filename"]), 2321 ] 2322 ) 2323 2324 if "port" in kwargs: 2325 ssh_args.append("-oPort={}".format(kwargs["port"])) 2326 2327 ssh_args.append(__ssh_gateway_arguments(kwargs)) 2328 2329 try: 2330 if socket.inet_pton(socket.AF_INET6, kwargs["hostname"]): 2331 ipaddr = "[{}]".format(kwargs["hostname"]) 2332 else: 2333 ipaddr = kwargs["hostname"] 2334 except OSError: 2335 ipaddr = kwargs["hostname"] 2336 2337 if file_to_upload is None: 2338 log.warning( 2339 "No source file to upload. Please make sure that either file " 2340 "contents or the path to a local file are provided." 2341 ) 2342 cmd = 'echo "put {0} {1} {2}" | sftp {3} {4[username]}@{5}'.format( 2343 " ".join(put_args), 2344 file_to_upload, 2345 dest_path, 2346 " ".join(ssh_args), 2347 kwargs, 2348 ipaddr, 2349 ) 2350 log.debug("SFTP command: '%s'", cmd) 2351 retcode = _exec_ssh_cmd( 2352 cmd, 2353 error_msg="Failed to upload file '{0}': {1}\n{2}", 2354 password_retries=3, 2355 **kwargs 2356 ) 2357 finally: 2358 if contents is not None: 2359 try: 2360 os.remove(file_to_upload) 2361 except OSError as exc: 2362 if exc.errno != errno.ENOENT: 2363 raise 2364 return retcode 2365 2366 2367def win_cmd(command, **kwargs): 2368 """ 2369 Wrapper for commands to be run against Windows boxes 2370 """ 2371 logging_command = kwargs.get("logging_command", None) 2372 2373 try: 2374 proc = NonBlockingPopen( 2375 command, 2376 shell=True, 2377 stderr=subprocess.PIPE, 2378 stdout=subprocess.PIPE, 2379 stream_stds=kwargs.get("display_ssh_output", True), 2380 logging_command=logging_command, 2381 ) 2382 2383 if logging_command is None: 2384 log.debug("Executing command(PID %s): '%s'", proc.pid, command) 2385 else: 2386 log.debug("Executing command(PID %s): '%s'", proc.pid, logging_command) 2387 2388 proc.poll_and_read_until_finish() 2389 proc.communicate() 2390 return proc.returncode 2391 except Exception as err: # pylint: disable=broad-except 2392 log.exception("Failed to execute command '%s'", logging_command) 2393 # Signal an error 2394 return 1 2395 2396 2397def winrm_cmd(session, command, flags, **kwargs): 2398 """ 2399 Wrapper for commands to be run against Windows boxes using WinRM. 2400 """ 2401 log.debug("Executing WinRM command: %s %s", command, flags) 2402 r = session.run_cmd(command, flags) 2403 return r.status_code 2404 2405 2406def root_cmd(command, tty, sudo, allow_failure=False, **kwargs): 2407 """ 2408 Wrapper for commands to be run as root 2409 """ 2410 logging_command = command 2411 sudo_password = kwargs.get("sudo_password", None) 2412 2413 if sudo: 2414 if sudo_password is None: 2415 command = "sudo {}".format(command) 2416 logging_command = command 2417 else: 2418 logging_command = 'sudo -S "XXX-REDACTED-XXX" {}'.format(command) 2419 command = "sudo -S {}".format(command) 2420 2421 log.debug("Using sudo to run command %s", logging_command) 2422 2423 ssh_args = [] 2424 2425 if tty: 2426 # Use double `-t` on the `ssh` command, it's necessary when `sudo` has 2427 # `requiretty` enforced. 2428 ssh_args.extend(["-t", "-t"]) 2429 2430 known_hosts_file = kwargs.get("known_hosts_file", "/dev/null") 2431 host_key_checking = "no" 2432 if known_hosts_file != "/dev/null": 2433 host_key_checking = "yes" 2434 2435 ssh_args.extend( 2436 [ 2437 # Don't add new hosts to the host key database 2438 "-oStrictHostKeyChecking={}".format(host_key_checking), 2439 # Set hosts key database path to /dev/null, i.e., non-existing 2440 "-oUserKnownHostsFile={}".format(known_hosts_file), 2441 # Don't re-use the SSH connection. Less failures. 2442 "-oControlPath=none", 2443 ] 2444 ) 2445 2446 if "key_filename" in kwargs: 2447 # There should never be both a password and an ssh key passed in, so 2448 ssh_args.extend( 2449 [ 2450 # tell SSH to skip password authentication 2451 "-oPasswordAuthentication=no", 2452 "-oChallengeResponseAuthentication=no", 2453 # Make sure public key authentication is enabled 2454 "-oPubkeyAuthentication=yes", 2455 # do only use the provided identity file 2456 "-oIdentitiesOnly=yes", 2457 # No Keyboard interaction! 2458 "-oKbdInteractiveAuthentication=no", 2459 # Also, specify the location of the key file 2460 "-i {}".format(kwargs["key_filename"]), 2461 ] 2462 ) 2463 if "ssh_timeout" in kwargs: 2464 ssh_args.extend(["-oConnectTimeout={}".format(kwargs["ssh_timeout"])]) 2465 2466 ssh_args.extend([__ssh_gateway_arguments(kwargs)]) 2467 2468 if "port" in kwargs: 2469 ssh_args.extend(["-p {}".format(kwargs["port"])]) 2470 2471 cmd = "ssh {0} {1[username]}@{1[hostname]} ".format(" ".join(ssh_args), kwargs) 2472 logging_command = cmd + logging_command 2473 cmd = cmd + pipes.quote(command) 2474 2475 hard_timeout = kwargs.get("hard_timeout") 2476 if hard_timeout is not None: 2477 logging_command = "timeout {} {}".format(hard_timeout, logging_command) 2478 cmd = "timeout {} {}".format(hard_timeout, cmd) 2479 2480 log.debug("SSH command: '%s'", logging_command) 2481 2482 retcode = _exec_ssh_cmd(cmd, allow_failure=allow_failure, **kwargs) 2483 return retcode 2484 2485 2486def check_auth(name, sock_dir=None, queue=None, timeout=300): 2487 """ 2488 This function is called from a multiprocess instance, to wait for a minion 2489 to become available to receive salt commands 2490 """ 2491 with salt.utils.event.SaltEvent("master", sock_dir, listen=True) as event: 2492 starttime = time.mktime(time.localtime()) 2493 newtimeout = timeout 2494 log.debug("In check_auth, waiting for %s to become available", name) 2495 while newtimeout > 0: 2496 newtimeout = timeout - (time.mktime(time.localtime()) - starttime) 2497 ret = event.get_event(full=True) 2498 if ret is None: 2499 continue 2500 if ret["tag"] == "salt/minion/{}/start".format(name): 2501 queue.put(name) 2502 newtimeout = 0 2503 log.debug("Minion %s is ready to receive commands", name) 2504 2505 2506def ip_to_int(ip): 2507 """ 2508 Converts an IP address to an integer 2509 """ 2510 ret = 0 2511 for octet in ip.split("."): 2512 ret = ret * 256 + int(octet) 2513 return ret 2514 2515 2516def is_public_ip(ip): 2517 """ 2518 Determines whether an IP address falls within one of the private IP ranges 2519 """ 2520 if ":" in ip: 2521 # ipv6 2522 if ip.startswith("fe80:"): 2523 # ipv6 link local 2524 return False 2525 return True 2526 addr = ip_to_int(ip) 2527 if 167772160 < addr < 184549375: 2528 # 10.0.0.0/8 2529 return False 2530 elif 3232235520 < addr < 3232301055: 2531 # 192.168.0.0/16 2532 return False 2533 elif 2886729728 < addr < 2887778303: 2534 # 172.16.0.0/12 2535 return False 2536 elif 2130706432 < addr < 2147483647: 2537 # 127.0.0.0/8 2538 return False 2539 return True 2540 2541 2542def check_name(name, safe_chars): 2543 """ 2544 Check whether the specified name contains invalid characters 2545 """ 2546 regexp = re.compile("[^{}]".format(safe_chars)) 2547 if regexp.search(name): 2548 raise SaltCloudException( 2549 "{} contains characters not supported by this cloud provider. " 2550 "Valid characters are: {}".format(name, safe_chars) 2551 ) 2552 2553 2554def remove_sshkey(host, known_hosts=None): 2555 """ 2556 Remove a host from the known_hosts file 2557 """ 2558 if known_hosts is None: 2559 if "HOME" in os.environ: 2560 known_hosts = "{}/.ssh/known_hosts".format(os.environ["HOME"]) 2561 else: 2562 try: 2563 known_hosts = "{}/.ssh/known_hosts".format( 2564 pwd.getpwuid(os.getuid()).pwd_dir 2565 ) 2566 except Exception: # pylint: disable=broad-except 2567 pass 2568 2569 if known_hosts is not None: 2570 log.debug("Removing ssh key for %s from known hosts file %s", host, known_hosts) 2571 else: 2572 log.debug("Removing ssh key for %s from known hosts file", host) 2573 2574 subprocess.call(["ssh-keygen", "-R", host]) 2575 2576 2577def wait_for_ip( 2578 update_callback, 2579 update_args=None, 2580 update_kwargs=None, 2581 timeout=5 * 60, 2582 interval=5, 2583 interval_multiplier=1, 2584 max_failures=10, 2585): 2586 """ 2587 Helper function that waits for an IP address for a specific maximum amount 2588 of time. 2589 2590 :param update_callback: callback function which queries the cloud provider 2591 for the VM ip address. It must return None if the 2592 required data, IP included, is not available yet. 2593 :param update_args: Arguments to pass to update_callback 2594 :param update_kwargs: Keyword arguments to pass to update_callback 2595 :param timeout: The maximum amount of time(in seconds) to wait for the IP 2596 address. 2597 :param interval: The looping interval, i.e., the amount of time to sleep 2598 before the next iteration. 2599 :param interval_multiplier: Increase the interval by this multiplier after 2600 each request; helps with throttling 2601 :param max_failures: If update_callback returns ``False`` it's considered 2602 query failure. This value is the amount of failures 2603 accepted before giving up. 2604 :returns: The update_callback returned data 2605 :raises: SaltCloudExecutionTimeout 2606 2607 """ 2608 if update_args is None: 2609 update_args = () 2610 if update_kwargs is None: 2611 update_kwargs = {} 2612 2613 duration = timeout 2614 while True: 2615 log.debug( 2616 "Waiting for VM IP. Giving up in 00:%02d:%02d.", 2617 int(timeout // 60), 2618 int(timeout % 60), 2619 ) 2620 data = update_callback(*update_args, **update_kwargs) 2621 if data is False: 2622 log.debug( 2623 "'update_callback' has returned 'False', which is " 2624 "considered a failure. Remaining Failures: %s.", 2625 max_failures, 2626 ) 2627 max_failures -= 1 2628 if max_failures <= 0: 2629 raise SaltCloudExecutionFailure( 2630 "Too many failures occurred while waiting for the IP address." 2631 ) 2632 elif data is not None: 2633 return data 2634 2635 if timeout < 0: 2636 raise SaltCloudExecutionTimeout( 2637 "Unable to get IP for 00:{:02d}:{:02d}.".format( 2638 int(duration // 60), int(duration % 60) 2639 ) 2640 ) 2641 time.sleep(interval) 2642 timeout -= interval 2643 2644 if interval_multiplier > 1: 2645 interval *= interval_multiplier 2646 if interval > timeout: 2647 interval = timeout + 1 2648 log.info("Interval multiplier in effect; interval is now %ss.", interval) 2649 2650 2651def list_nodes_select(nodes, selection, call=None): 2652 """ 2653 Return a list of the VMs that are on the provider, with select fields 2654 """ 2655 if call == "action": 2656 raise SaltCloudSystemExit( 2657 "The list_nodes_select function must be called with -f or --function." 2658 ) 2659 2660 if "error" in nodes: 2661 raise SaltCloudSystemExit( 2662 "An error occurred while listing nodes: {}".format( 2663 nodes["error"]["Errors"]["Error"]["Message"] 2664 ) 2665 ) 2666 2667 ret = {} 2668 for node in nodes: 2669 pairs = {} 2670 data = nodes[node] 2671 for key in data: 2672 if str(key) in selection: 2673 value = data[key] 2674 pairs[key] = value 2675 ret[node] = pairs 2676 2677 return ret 2678 2679 2680def lock_file(filename, interval=0.5, timeout=15): 2681 """ 2682 Lock a file; if it is already locked, then wait for it to become available 2683 before locking it. 2684 2685 Note that these locks are only recognized by Salt Cloud, and not other 2686 programs or platforms. 2687 """ 2688 log.trace("Attempting to obtain lock for %s", filename) 2689 lock = filename + ".lock" 2690 start = time.time() 2691 while True: 2692 if os.path.exists(lock): 2693 if time.time() - start >= timeout: 2694 log.warning("Unable to obtain lock for %s", filename) 2695 return False 2696 time.sleep(interval) 2697 else: 2698 break 2699 2700 with salt.utils.files.fopen(lock, "a"): 2701 pass 2702 2703 2704def unlock_file(filename): 2705 """ 2706 Unlock a locked file 2707 2708 Note that these locks are only recognized by Salt Cloud, and not other 2709 programs or platforms. 2710 """ 2711 log.trace("Removing lock for %s", filename) 2712 lock = filename + ".lock" 2713 try: 2714 os.remove(lock) 2715 except OSError as exc: 2716 log.trace("Unable to remove lock for %s: %s", filename, exc) 2717 2718 2719def cachedir_index_add(minion_id, profile, driver, provider, base=None): 2720 """ 2721 Add an entry to the cachedir index. This generally only needs to happen when 2722 a new instance is created. This entry should contain: 2723 2724 .. code-block:: yaml 2725 2726 - minion_id 2727 - profile used to create the instance 2728 - provider and driver name 2729 2730 The intent of this function is to speed up lookups for the cloud roster for 2731 salt-ssh. However, other code that makes use of profile information can also 2732 make use of this function. 2733 """ 2734 base = init_cachedir(base) 2735 index_file = os.path.join(base, "index.p") 2736 lock_file(index_file) 2737 2738 if os.path.exists(index_file): 2739 with salt.utils.files.fopen(index_file, "rb") as fh_: 2740 index = salt.utils.data.decode( 2741 salt.utils.msgpack.load(fh_, encoding=MSGPACK_ENCODING) 2742 ) 2743 else: 2744 index = {} 2745 2746 prov_comps = provider.split(":") 2747 2748 index.update( 2749 { 2750 minion_id: { 2751 "id": minion_id, 2752 "profile": profile, 2753 "driver": driver, 2754 "provider": prov_comps[0], 2755 } 2756 } 2757 ) 2758 2759 with salt.utils.files.fopen(index_file, "wb") as fh_: 2760 salt.utils.msgpack.dump(index, fh_, encoding=MSGPACK_ENCODING) 2761 2762 unlock_file(index_file) 2763 2764 2765def cachedir_index_del(minion_id, base=None): 2766 """ 2767 Delete an entry from the cachedir index. This generally only needs to happen 2768 when an instance is deleted. 2769 """ 2770 base = init_cachedir(base) 2771 index_file = os.path.join(base, "index.p") 2772 lock_file(index_file) 2773 2774 if os.path.exists(index_file): 2775 with salt.utils.files.fopen(index_file, "rb") as fh_: 2776 index = salt.utils.data.decode( 2777 salt.utils.msgpack.load(fh_, encoding=MSGPACK_ENCODING) 2778 ) 2779 else: 2780 return 2781 2782 if minion_id in index: 2783 del index[minion_id] 2784 2785 with salt.utils.files.fopen(index_file, "wb") as fh_: 2786 salt.utils.msgpack.dump(index, fh_, encoding=MSGPACK_ENCODING) 2787 2788 unlock_file(index_file) 2789 2790 2791def init_cachedir(base=None): 2792 """ 2793 Initialize the cachedir needed for Salt Cloud to keep track of minions 2794 """ 2795 if base is None: 2796 base = __opts__["cachedir"] 2797 needed_dirs = (base, os.path.join(base, "requested"), os.path.join(base, "active")) 2798 for dir_ in needed_dirs: 2799 if not os.path.exists(dir_): 2800 os.makedirs(dir_) 2801 os.chmod(base, 0o755) 2802 2803 return base 2804 2805 2806# FIXME: This function seems used nowhere. Dead code? 2807def request_minion_cachedir( 2808 minion_id, 2809 opts=None, 2810 fingerprint="", 2811 pubkey=None, 2812 provider=None, 2813 base=None, 2814): 2815 """ 2816 Creates an entry in the requested/ cachedir. This means that Salt Cloud has 2817 made a request to a cloud provider to create an instance, but it has not 2818 yet verified that the instance properly exists. 2819 2820 If the fingerprint is unknown, a raw pubkey can be passed in, and a 2821 fingerprint will be calculated. If both are empty, then the fingerprint 2822 will be set to None. 2823 """ 2824 if base is None: 2825 base = __opts__["cachedir"] 2826 2827 if not fingerprint and pubkey is not None: 2828 fingerprint = salt.utils.crypt.pem_finger( 2829 key=pubkey, sum_type=(opts and opts.get("hash_type") or "sha256") 2830 ) 2831 2832 init_cachedir(base) 2833 2834 data = { 2835 "minion_id": minion_id, 2836 "fingerprint": fingerprint, 2837 "provider": provider, 2838 } 2839 2840 fname = "{}.p".format(minion_id) 2841 path = os.path.join(base, "requested", fname) 2842 with salt.utils.files.fopen(path, "wb") as fh_: 2843 salt.utils.msgpack.dump(data, fh_, encoding=MSGPACK_ENCODING) 2844 2845 2846def change_minion_cachedir( 2847 minion_id, 2848 cachedir, 2849 data=None, 2850 base=None, 2851): 2852 """ 2853 Changes the info inside a minion's cachedir entry. The type of cachedir 2854 must be specified (i.e., 'requested' or 'active'). A dict is also passed in 2855 which contains the data to be changed. 2856 2857 Example: 2858 2859 change_minion_cachedir( 2860 'myminion', 2861 'requested', 2862 {'fingerprint': '26:5c:8c:de:be:fe:89:c0:02:ed:27:65:0e:bb:be:60'}, 2863 ) 2864 """ 2865 if not isinstance(data, dict): 2866 return False 2867 2868 if base is None: 2869 base = __opts__["cachedir"] 2870 2871 fname = "{}.p".format(minion_id) 2872 path = os.path.join(base, cachedir, fname) 2873 2874 with salt.utils.files.fopen(path, "r") as fh_: 2875 cache_data = salt.utils.data.decode( 2876 salt.utils.msgpack.load(fh_, encoding=MSGPACK_ENCODING) 2877 ) 2878 2879 cache_data.update(data) 2880 2881 with salt.utils.files.fopen(path, "w") as fh_: 2882 salt.utils.msgpack.dump(cache_data, fh_, encoding=MSGPACK_ENCODING) 2883 2884 2885def activate_minion_cachedir(minion_id, base=None): 2886 """ 2887 Moves a minion from the requested/ cachedir into the active/ cachedir. This 2888 means that Salt Cloud has verified that a requested instance properly 2889 exists, and should be expected to exist from here on out. 2890 """ 2891 if base is None: 2892 base = __opts__["cachedir"] 2893 2894 fname = "{}.p".format(minion_id) 2895 src = os.path.join(base, "requested", fname) 2896 dst = os.path.join(base, "active") 2897 shutil.move(src, dst) 2898 2899 2900def delete_minion_cachedir(minion_id, provider, opts, base=None): 2901 """ 2902 Deletes a minion's entry from the cloud cachedir. It will search through 2903 all cachedirs to find the minion's cache file. 2904 Needs `update_cachedir` set to True. 2905 """ 2906 if isinstance(opts, dict): 2907 __opts__.update(opts) 2908 2909 if __opts__.get("update_cachedir", False) is False: 2910 return 2911 2912 if base is None: 2913 base = __opts__["cachedir"] 2914 2915 driver = next(iter(__opts__["providers"][provider].keys())) 2916 fname = "{}.p".format(minion_id) 2917 for cachedir in "requested", "active": 2918 path = os.path.join(base, cachedir, driver, provider, fname) 2919 log.debug("path: %s", path) 2920 if os.path.exists(path): 2921 os.remove(path) 2922 2923 2924def list_cache_nodes_full(opts=None, provider=None, base=None): 2925 """ 2926 Return a list of minion data from the cloud cache, rather from the cloud 2927 providers themselves. This is the cloud cache version of list_nodes_full(). 2928 """ 2929 if opts is None: 2930 opts = __opts__ 2931 if opts.get("update_cachedir", False) is False: 2932 return 2933 2934 if base is None: 2935 base = os.path.join(opts["cachedir"], "active") 2936 2937 minions = {} 2938 # First, get a list of all drivers in use 2939 for driver in os.listdir(base): 2940 minions[driver] = {} 2941 prov_dir = os.path.join(base, driver) 2942 # Then, get a list of all providers per driver 2943 for prov in os.listdir(prov_dir): 2944 # If a specific provider is requested, filter out everyone else 2945 if provider and provider != prov: 2946 continue 2947 minions[driver][prov] = {} 2948 min_dir = os.path.join(prov_dir, prov) 2949 # Get a list of all nodes per provider 2950 for fname in os.listdir(min_dir): 2951 # Finally, get a list of full minion data 2952 fpath = os.path.join(min_dir, fname) 2953 minion_id = fname[:-2] # strip '.p' from end of msgpack filename 2954 with salt.utils.files.fopen(fpath, "rb") as fh_: 2955 minions[driver][prov][minion_id] = salt.utils.data.decode( 2956 salt.utils.msgpack.load(fh_, encoding=MSGPACK_ENCODING) 2957 ) 2958 2959 return minions 2960 2961 2962def update_bootstrap(config, url=None): 2963 """ 2964 Update the salt-bootstrap script 2965 2966 url can be one of: 2967 2968 - The URL to fetch the bootstrap script from 2969 - The absolute path to the bootstrap 2970 - The content of the bootstrap script 2971 """ 2972 default_url = config.get("bootstrap_script_url", "https://bootstrap.saltstack.com") 2973 if not url: 2974 url = default_url 2975 if not url: 2976 raise ValueError("Cant get any source to update") 2977 if url.startswith("http") or "://" in url: 2978 log.debug("Updating the bootstrap-salt.sh script to latest stable") 2979 try: 2980 import requests 2981 except ImportError: 2982 return { 2983 "error": ( 2984 "Updating the bootstrap-salt.sh script requires the " 2985 "Python requests library to be installed" 2986 ) 2987 } 2988 req = requests.get(url) 2989 if req.status_code != 200: 2990 return { 2991 "error": ( 2992 "Failed to download the latest stable version of the " 2993 "bootstrap-salt.sh script from {}. HTTP error: " 2994 "{}".format(url, req.status_code) 2995 ) 2996 } 2997 script_content = req.text 2998 if url == default_url: 2999 script_name = "bootstrap-salt.sh" 3000 else: 3001 script_name = os.path.basename(url) 3002 elif os.path.exists(url): 3003 with salt.utils.files.fopen(url) as fic: 3004 script_content = salt.utils.stringutils.to_unicode(fic.read()) 3005 script_name = os.path.basename(url) 3006 # in last case, assuming we got a script content 3007 else: 3008 script_content = url 3009 script_name = "{}.sh".format(hashlib.sha1(script_content).hexdigest()) 3010 3011 if not script_content: 3012 raise ValueError("No content in bootstrap script !") 3013 3014 # Get the path to the built-in deploy scripts directory 3015 builtin_deploy_dir = os.path.join(os.path.dirname(__file__), "deploy") 3016 3017 # Compute the search path from the current loaded opts conf_file 3018 # value 3019 deploy_d_from_conf_file = os.path.join( 3020 os.path.dirname(config["conf_file"]), "cloud.deploy.d" 3021 ) 3022 3023 # Compute the search path using the install time defined 3024 # syspaths.CONF_DIR 3025 deploy_d_from_syspaths = os.path.join(config["config_dir"], "cloud.deploy.d") 3026 3027 # Get a copy of any defined search paths, flagging them not to 3028 # create parent 3029 deploy_scripts_search_paths = [] 3030 for entry in config.get("deploy_scripts_search_path", []): 3031 if entry.startswith(builtin_deploy_dir): 3032 # We won't write the updated script to the built-in deploy 3033 # directory 3034 continue 3035 3036 if entry in (deploy_d_from_conf_file, deploy_d_from_syspaths): 3037 # Allow parent directories to be made 3038 deploy_scripts_search_paths.append((entry, True)) 3039 else: 3040 deploy_scripts_search_paths.append((entry, False)) 3041 3042 # In case the user is not using defaults and the computed 3043 # 'cloud.deploy.d' from conf_file and syspaths is not included, add 3044 # them 3045 if deploy_d_from_conf_file not in deploy_scripts_search_paths: 3046 deploy_scripts_search_paths.append((deploy_d_from_conf_file, True)) 3047 if deploy_d_from_syspaths not in deploy_scripts_search_paths: 3048 deploy_scripts_search_paths.append((deploy_d_from_syspaths, True)) 3049 3050 finished = [] 3051 finished_full = [] 3052 for entry, makedirs in deploy_scripts_search_paths: 3053 # This handles duplicate entries, which are likely to appear 3054 if entry in finished: 3055 continue 3056 else: 3057 finished.append(entry) 3058 3059 if makedirs and not os.path.isdir(entry): 3060 try: 3061 os.makedirs(entry) 3062 except OSError as err: 3063 log.info("Failed to create directory '%s'", entry) 3064 continue 3065 3066 if not is_writeable(entry): 3067 log.debug("The '%s' is not writeable. Continuing...", entry) 3068 continue 3069 3070 deploy_path = os.path.join(entry, script_name) 3071 try: 3072 finished_full.append(deploy_path) 3073 with salt.utils.files.fopen(deploy_path, "w") as fp_: 3074 fp_.write(salt.utils.stringutils.to_str(script_content)) 3075 except OSError as err: 3076 log.debug("Failed to write the updated script: %s", err) 3077 continue 3078 3079 return {"Success": {"Files updated": finished_full}} 3080 3081 3082def cache_node_list(nodes, provider, opts): 3083 """ 3084 If configured to do so, update the cloud cachedir with the current list of 3085 nodes. Also fires configured events pertaining to the node list. 3086 3087 .. versionadded:: 2014.7.0 3088 """ 3089 if "update_cachedir" not in opts or not opts["update_cachedir"]: 3090 return 3091 3092 base = os.path.join(init_cachedir(), "active") 3093 driver = next(iter(opts["providers"][provider].keys())) 3094 prov_dir = os.path.join(base, driver, provider) 3095 if not os.path.exists(prov_dir): 3096 os.makedirs(prov_dir) 3097 3098 # Check to see if any nodes in the cache are not in the new list 3099 missing_node_cache(prov_dir, nodes, provider, opts) 3100 3101 for node in nodes: 3102 diff_node_cache(prov_dir, node, nodes[node], opts) 3103 path = os.path.join(prov_dir, "{}.p".format(node)) 3104 with salt.utils.files.fopen(path, "wb") as fh_: 3105 salt.utils.msgpack.dump(nodes[node], fh_, encoding=MSGPACK_ENCODING) 3106 3107 3108def cache_node(node, provider, opts): 3109 """ 3110 Cache node individually 3111 3112 .. versionadded:: 2014.7.0 3113 """ 3114 if isinstance(opts, dict): 3115 __opts__.update(opts) 3116 3117 if "update_cachedir" not in __opts__ or not __opts__["update_cachedir"]: 3118 return 3119 3120 if not os.path.exists(os.path.join(__opts__["cachedir"], "active")): 3121 init_cachedir() 3122 3123 base = os.path.join(__opts__["cachedir"], "active") 3124 provider, driver = provider.split(":") 3125 prov_dir = os.path.join(base, driver, provider) 3126 if not os.path.exists(prov_dir): 3127 os.makedirs(prov_dir) 3128 path = os.path.join(prov_dir, "{}.p".format(node["name"])) 3129 with salt.utils.files.fopen(path, "wb") as fh_: 3130 salt.utils.msgpack.dump(node, fh_, encoding=MSGPACK_ENCODING) 3131 3132 3133def missing_node_cache(prov_dir, node_list, provider, opts): 3134 """ 3135 Check list of nodes to see if any nodes which were previously known about 3136 in the cache have been removed from the node list. 3137 3138 This function will only run if configured to do so in the main Salt Cloud 3139 configuration file (normally /etc/salt/cloud). 3140 3141 .. code-block:: yaml 3142 3143 diff_cache_events: True 3144 3145 .. versionadded:: 2014.7.0 3146 """ 3147 cached_nodes = [] 3148 for node in os.listdir(prov_dir): 3149 cached_nodes.append(os.path.splitext(node)[0]) 3150 3151 for node in cached_nodes: 3152 if node not in node_list: 3153 delete_minion_cachedir(node, provider, opts) 3154 if "diff_cache_events" in opts and opts["diff_cache_events"]: 3155 fire_event( 3156 "event", 3157 "cached node missing from provider", 3158 "salt/cloud/{}/cache_node_missing".format(node), 3159 args={"missing node": node}, 3160 sock_dir=opts.get( 3161 "sock_dir", os.path.join(__opts__["sock_dir"], "master") 3162 ), 3163 transport=opts.get("transport", "zeromq"), 3164 ) 3165 3166 3167def diff_node_cache(prov_dir, node, new_data, opts): 3168 """ 3169 Check new node data against current cache. If data differ, fire an event 3170 which consists of the new node data. 3171 3172 This function will only run if configured to do so in the main Salt Cloud 3173 configuration file (normally /etc/salt/cloud). 3174 3175 .. code-block:: yaml 3176 3177 diff_cache_events: True 3178 3179 .. versionadded:: 2014.7.0 3180 """ 3181 if "diff_cache_events" not in opts or not opts["diff_cache_events"]: 3182 return 3183 3184 if node is None: 3185 return 3186 path = "{}.p".format(os.path.join(prov_dir, node)) 3187 3188 if not os.path.exists(path): 3189 event_data = _strip_cache_events(new_data, opts) 3190 3191 fire_event( 3192 "event", 3193 "new node found", 3194 "salt/cloud/{}/cache_node_new".format(node), 3195 args={"new_data": event_data}, 3196 sock_dir=opts.get("sock_dir", os.path.join(__opts__["sock_dir"], "master")), 3197 transport=opts.get("transport", "zeromq"), 3198 ) 3199 return 3200 3201 with salt.utils.files.fopen(path, "r") as fh_: 3202 try: 3203 cache_data = salt.utils.data.decode( 3204 salt.utils.msgpack.load(fh_, encoding=MSGPACK_ENCODING) 3205 ) 3206 except ValueError: 3207 log.warning("Cache for %s was corrupt: Deleting", node) 3208 cache_data = {} 3209 3210 # Perform a simple diff between the old and the new data, and if it differs, 3211 # return both dicts. 3212 # TODO: Return an actual diff 3213 diff = salt.utils.compat.cmp(new_data, cache_data) 3214 if diff != 0: 3215 fire_event( 3216 "event", 3217 "node data differs", 3218 "salt/cloud/{}/cache_node_diff".format(node), 3219 args={ 3220 "new_data": _strip_cache_events(new_data, opts), 3221 "cache_data": _strip_cache_events(cache_data, opts), 3222 }, 3223 sock_dir=opts.get("sock_dir", os.path.join(__opts__["sock_dir"], "master")), 3224 transport=opts.get("transport", "zeromq"), 3225 ) 3226 3227 3228def _strip_cache_events(data, opts): 3229 """ 3230 Strip out user-configured sensitive event data. The fields to be stripped 3231 are configured in the main Salt Cloud configuration file, usually 3232 ``/etc/salt/cloud``. 3233 3234 .. code-block:: yaml 3235 3236 cache_event_strip_fields: 3237 - password 3238 - priv_key 3239 3240 .. versionadded:: 2014.7.0 3241 """ 3242 event_data = copy.deepcopy(data) 3243 strip_fields = opts.get("cache_event_strip_fields", []) 3244 for field in strip_fields: 3245 if field in event_data: 3246 del event_data[field] 3247 3248 return event_data 3249 3250 3251def _salt_cloud_force_ascii(exc): 3252 """ 3253 Helper method to try its best to convert any Unicode text into ASCII 3254 without stack tracing since salt internally does not handle Unicode strings 3255 3256 This method is not supposed to be used directly. Once 3257 `py:module: salt.utils.cloud` is imported this method register's with 3258 python's codecs module for proper automatic conversion in case of encoding 3259 errors. 3260 """ 3261 if not isinstance(exc, (UnicodeEncodeError, UnicodeTranslateError)): 3262 raise TypeError("Can't handle {}".format(exc)) 3263 3264 unicode_trans = { 3265 # Convert non-breaking space to space 3266 "\xa0": " ", 3267 # Convert en dash to dash 3268 "\u2013": "-", 3269 } 3270 3271 if exc.object[exc.start : exc.end] in unicode_trans: 3272 return unicode_trans[exc.object[exc.start : exc.end]], exc.end 3273 3274 # There's nothing else we can do, raise the exception 3275 raise exc 3276 3277 3278codecs.register_error("salt-cloud-force-ascii", _salt_cloud_force_ascii) 3279 3280 3281def retrieve_password_from_keyring(credential_id, username): 3282 """ 3283 Retrieve particular user's password for a specified credential set from system keyring. 3284 """ 3285 try: 3286 import keyring # pylint: disable=import-error 3287 3288 return keyring.get_password(credential_id, username) 3289 except ImportError: 3290 log.error( 3291 "USE_KEYRING configured as a password, but no keyring module is installed" 3292 ) 3293 return False 3294 3295 3296def _save_password_in_keyring(credential_id, username, password): 3297 """ 3298 Saves provider password in system keyring 3299 """ 3300 try: 3301 import keyring # pylint: disable=import-error 3302 3303 return keyring.set_password(credential_id, username, password) 3304 except ImportError: 3305 log.error( 3306 "Tried to store password in keyring, but no keyring module is installed" 3307 ) 3308 return False 3309 3310 3311def store_password_in_keyring(credential_id, username, password=None): 3312 """ 3313 Interactively prompts user for a password and stores it in system keyring 3314 """ 3315 try: 3316 # pylint: disable=import-error 3317 import keyring 3318 import keyring.errors 3319 3320 # pylint: enable=import-error 3321 if password is None: 3322 prompt = "Please enter password for {}: ".format(credential_id) 3323 try: 3324 password = getpass.getpass(prompt) 3325 except EOFError: 3326 password = None 3327 3328 if not password: 3329 # WE should raise something else here to be able to use this 3330 # as/from an API 3331 raise RuntimeError("Invalid password provided.") 3332 3333 try: 3334 _save_password_in_keyring(credential_id, username, password) 3335 except keyring.errors.PasswordSetError as exc: 3336 log.debug("Problem saving password in the keyring: %s", exc) 3337 except ImportError: 3338 log.error( 3339 "Tried to store password in keyring, but no keyring module is installed" 3340 ) 3341 return False 3342 3343 3344def _unwrap_dict(dictionary, index_string): 3345 """ 3346 Accepts index in form of a string 3347 Returns final value 3348 Example: dictionary = {'a': {'b': {'c': 'foobar'}}} 3349 index_string = 'a,b,c' 3350 returns 'foobar' 3351 """ 3352 index = index_string.split(",") 3353 for k in index: 3354 dictionary = dictionary[k] 3355 return dictionary 3356 3357 3358def run_func_until_ret_arg( 3359 fun, 3360 kwargs, 3361 fun_call=None, 3362 argument_being_watched=None, 3363 required_argument_response=None, 3364): 3365 """ 3366 Waits until the function retrieves some required argument. 3367 NOTE: Tested with ec2 describe_volumes and describe_snapshots only. 3368 """ 3369 status = None 3370 while status != required_argument_response: 3371 f_result = fun(kwargs, call=fun_call) 3372 r_set = {} 3373 for d in f_result: 3374 if isinstance(d, list): 3375 d0 = d[0] 3376 if isinstance(d0, dict): 3377 for k, v in d0.items(): 3378 r_set[k] = v 3379 status = _unwrap_dict(r_set, argument_being_watched) 3380 log.debug( 3381 "Function: %s, Watched arg: %s, Response: %s", 3382 str(fun).split(" ")[1], 3383 argument_being_watched, 3384 status, 3385 ) 3386 time.sleep(5) 3387 3388 return True 3389 3390 3391def get_salt_interface(vm_, opts): 3392 """ 3393 Return the salt_interface type to connect to. Either 'public_ips' (default) 3394 or 'private_ips'. 3395 """ 3396 salt_host = salt.config.get_cloud_config_value( 3397 "salt_interface", vm_, opts, default=False, search_global=False 3398 ) 3399 3400 if salt_host is False: 3401 salt_host = salt.config.get_cloud_config_value( 3402 "ssh_interface", vm_, opts, default="public_ips", search_global=False 3403 ) 3404 3405 return salt_host 3406 3407 3408def check_key_path_and_mode(provider, key_path): 3409 """ 3410 Checks that the key_path exists and the key_mode is either 0400 or 0600. 3411 3412 Returns True or False. 3413 3414 .. versionadded:: 2016.3.0 3415 3416 provider 3417 The provider name that the key_path to check belongs to. 3418 3419 key_path 3420 The key_path to ensure that it exists and to check adequate permissions 3421 against. 3422 """ 3423 if not os.path.exists(key_path): 3424 log.error( 3425 "The key file '%s' used in the '%s' provider configuration " 3426 "does not exist.\n", 3427 key_path, 3428 provider, 3429 ) 3430 return False 3431 3432 key_mode = stat.S_IMODE(os.stat(key_path).st_mode) 3433 if key_mode not in (0o400, 0o600): 3434 log.error( 3435 "The key file '%s' used in the '%s' provider configuration " 3436 "needs to be set to mode 0400 or 0600.\n", 3437 key_path, 3438 provider, 3439 ) 3440 return False 3441 3442 return True 3443 3444 3445def userdata_template(opts, vm_, userdata): 3446 """ 3447 Use the configured templating engine to template the userdata file 3448 """ 3449 # No userdata, no need to template anything 3450 if userdata is None: 3451 return userdata 3452 3453 userdata_template = salt.config.get_cloud_config_value( 3454 "userdata_template", vm_, opts, search_global=False, default=None 3455 ) 3456 if userdata_template is False: 3457 return userdata 3458 # Use the cloud profile's userdata_template, otherwise get it from the 3459 # master configuration file. 3460 renderer = ( 3461 opts.get("userdata_template") 3462 if userdata_template is None 3463 else userdata_template 3464 ) 3465 if renderer is None: 3466 return userdata 3467 else: 3468 render_opts = opts.copy() 3469 render_opts.update(vm_) 3470 rend = salt.loader.render(render_opts, {}) 3471 blacklist = opts["renderer_blacklist"] 3472 whitelist = opts["renderer_whitelist"] 3473 templated = salt.template.compile_template( 3474 ":string:", 3475 rend, 3476 renderer, 3477 blacklist, 3478 whitelist, 3479 input_data=userdata, 3480 ) 3481 if not isinstance(templated, str): 3482 # template renderers like "jinja" should return a StringIO 3483 try: 3484 templated = "".join(templated.readlines()) 3485 except AttributeError: 3486 log.warning( 3487 "Templated userdata resulted in non-string result (%s), " 3488 "converting to string", 3489 templated, 3490 ) 3491 templated = str(templated) 3492 3493 return templated 3494