1""" 2This module provides the point of entry to SPM, the Salt Package Manager 3 4.. versionadded:: 2015.8.0 5""" 6 7import hashlib 8import logging 9import os 10import shutil 11import sys 12import tarfile 13 14import salt.cache 15import salt.client 16import salt.config 17import salt.loader 18import salt.syspaths as syspaths 19import salt.utils.files 20import salt.utils.http as http 21import salt.utils.path 22import salt.utils.platform 23import salt.utils.win_functions 24import salt.utils.yaml 25from salt.template import compile_template 26 27try: 28 import pwd 29 import grp 30except ImportError: 31 pass 32 33 34log = logging.getLogger(__name__) 35 36FILE_TYPES = ("c", "d", "g", "l", "r", "s", "m") 37# c: config file 38# d: documentation file 39# g: ghost file (i.e. the file contents are not included in the package payload) 40# l: license file 41# r: readme file 42# s: SLS file 43# m: Salt module 44 45 46class SPMException(Exception): 47 """ 48 Base class for SPMClient exceptions 49 """ 50 51 52class SPMInvocationError(SPMException): 53 """ 54 Wrong number of arguments or other usage error 55 """ 56 57 58class SPMPackageError(SPMException): 59 """ 60 Problem with package file or package installation 61 """ 62 63 64class SPMDatabaseError(SPMException): 65 """ 66 SPM database not found, etc 67 """ 68 69 70class SPMOperationCanceled(SPMException): 71 """ 72 SPM install or uninstall was canceled 73 """ 74 75 76class SPMClient: 77 """ 78 Provide an SPM Client 79 """ 80 81 def __init__(self, ui, opts=None): # pylint: disable=W0231 82 self.ui = ui 83 if not opts: 84 opts = salt.config.spm_config(os.path.join(syspaths.CONFIG_DIR, "spm")) 85 self.opts = opts 86 self.db_prov = self.opts.get("spm_db_provider", "sqlite3") 87 self.files_prov = self.opts.get("spm_files_provider", "local") 88 self._prep_pkgdb() 89 self._prep_pkgfiles() 90 self.db_conn = None 91 self.files_conn = None 92 self._init() 93 94 def _prep_pkgdb(self): 95 self.pkgdb = salt.loader.pkgdb(self.opts) 96 97 def _prep_pkgfiles(self): 98 self.pkgfiles = salt.loader.pkgfiles(self.opts) 99 100 def _init(self): 101 if not self.db_conn: 102 self.db_conn = self._pkgdb_fun("init") 103 if not self.files_conn: 104 self.files_conn = self._pkgfiles_fun("init") 105 106 def _close(self): 107 if self.db_conn: 108 self.db_conn.close() 109 110 def run(self, args): 111 """ 112 Run the SPM command 113 """ 114 command = args[0] 115 try: 116 if command == "install": 117 self._install(args) 118 elif command == "local": 119 self._local(args) 120 elif command == "repo": 121 self._repo(args) 122 elif command == "remove": 123 self._remove(args) 124 elif command == "build": 125 self._build(args) 126 elif command == "update_repo": 127 self._download_repo_metadata(args) 128 elif command == "create_repo": 129 self._create_repo(args) 130 elif command == "files": 131 self._list_files(args) 132 elif command == "info": 133 self._info(args) 134 elif command == "list": 135 self._list(args) 136 elif command == "close": 137 self._close() 138 else: 139 raise SPMInvocationError("Invalid command '{}'".format(command)) 140 except SPMException as exc: 141 self.ui.error(str(exc)) 142 143 def _pkgdb_fun(self, func, *args, **kwargs): 144 try: 145 return getattr(getattr(self.pkgdb, self.db_prov), func)(*args, **kwargs) 146 except AttributeError: 147 return self.pkgdb["{}.{}".format(self.db_prov, func)](*args, **kwargs) 148 149 def _pkgfiles_fun(self, func, *args, **kwargs): 150 try: 151 return getattr(getattr(self.pkgfiles, self.files_prov), func)( 152 *args, **kwargs 153 ) 154 except AttributeError: 155 return self.pkgfiles["{}.{}".format(self.files_prov, func)](*args, **kwargs) 156 157 def _list(self, args): 158 """ 159 Process local commands 160 """ 161 args.pop(0) 162 command = args[0] 163 if command == "packages": 164 self._list_packages(args) 165 elif command == "files": 166 self._list_files(args) 167 elif command == "repos": 168 self._repo_list(args) 169 else: 170 raise SPMInvocationError("Invalid list command '{}'".format(command)) 171 172 def _local(self, args): 173 """ 174 Process local commands 175 """ 176 args.pop(0) 177 command = args[0] 178 if command == "install": 179 self._local_install(args) 180 elif command == "files": 181 self._local_list_files(args) 182 elif command == "info": 183 self._local_info(args) 184 else: 185 raise SPMInvocationError("Invalid local command '{}'".format(command)) 186 187 def _repo(self, args): 188 """ 189 Process repo commands 190 """ 191 args.pop(0) 192 command = args[0] 193 if command == "list": 194 self._repo_list(args) 195 elif command == "packages": 196 self._repo_packages(args) 197 elif command == "search": 198 self._repo_packages(args, search=True) 199 elif command == "update": 200 self._download_repo_metadata(args) 201 elif command == "create": 202 self._create_repo(args) 203 else: 204 raise SPMInvocationError("Invalid repo command '{}'".format(command)) 205 206 def _repo_packages(self, args, search=False): 207 """ 208 List packages for one or more configured repos 209 """ 210 packages = [] 211 repo_metadata = self._get_repo_metadata() 212 for repo in repo_metadata: 213 for pkg in repo_metadata[repo]["packages"]: 214 if args[1] in pkg: 215 version = repo_metadata[repo]["packages"][pkg]["info"]["version"] 216 release = repo_metadata[repo]["packages"][pkg]["info"]["release"] 217 packages.append((pkg, version, release, repo)) 218 for pkg in sorted(packages): 219 self.ui.status("{}\t{}-{}\t{}".format(pkg[0], pkg[1], pkg[2], pkg[3])) 220 return packages 221 222 def _repo_list(self, args): 223 """ 224 List configured repos 225 226 This can be called either as a ``repo`` command or a ``list`` command 227 """ 228 repo_metadata = self._get_repo_metadata() 229 for repo in repo_metadata: 230 self.ui.status(repo) 231 232 def _install(self, args): 233 """ 234 Install a package from a repo 235 """ 236 if len(args) < 2: 237 raise SPMInvocationError("A package must be specified") 238 239 caller_opts = self.opts.copy() 240 caller_opts["file_client"] = "local" 241 self.caller = salt.client.Caller(mopts=caller_opts) 242 self.client = salt.client.get_local_client(self.opts["conf_file"]) 243 cache = salt.cache.Cache(self.opts) 244 245 packages = args[1:] 246 file_map = {} 247 optional = [] 248 recommended = [] 249 to_install = [] 250 for pkg in packages: 251 if pkg.endswith(".spm"): 252 if self._pkgfiles_fun("path_exists", pkg): 253 comps = pkg.split("-") 254 comps = os.path.split("-".join(comps[:-2])) 255 pkg_name = comps[-1] 256 257 formula_tar = tarfile.open(pkg, "r:bz2") 258 formula_ref = formula_tar.extractfile("{}/FORMULA".format(pkg_name)) 259 formula_def = salt.utils.yaml.safe_load(formula_ref) 260 261 file_map[pkg_name] = pkg 262 to_, op_, re_ = self._check_all_deps( 263 pkg_name=pkg_name, pkg_file=pkg, formula_def=formula_def 264 ) 265 to_install.extend(to_) 266 optional.extend(op_) 267 recommended.extend(re_) 268 formula_tar.close() 269 else: 270 raise SPMInvocationError("Package file {} not found".format(pkg)) 271 else: 272 to_, op_, re_ = self._check_all_deps(pkg_name=pkg) 273 to_install.extend(to_) 274 optional.extend(op_) 275 recommended.extend(re_) 276 277 optional = set(filter(len, optional)) 278 if optional: 279 self.ui.status( 280 "The following dependencies are optional:\n\t{}\n".format( 281 "\n\t".join(optional) 282 ) 283 ) 284 recommended = set(filter(len, recommended)) 285 if recommended: 286 self.ui.status( 287 "The following dependencies are recommended:\n\t{}\n".format( 288 "\n\t".join(recommended) 289 ) 290 ) 291 292 to_install = set(filter(len, to_install)) 293 msg = "Installing packages:\n\t{}\n".format("\n\t".join(to_install)) 294 if not self.opts["assume_yes"]: 295 self.ui.confirm(msg) 296 297 repo_metadata = self._get_repo_metadata() 298 299 dl_list = {} 300 for package in to_install: 301 if package in file_map: 302 self._install_indv_pkg(package, file_map[package]) 303 else: 304 for repo in repo_metadata: 305 repo_info = repo_metadata[repo] 306 if package in repo_info["packages"]: 307 dl_package = False 308 repo_ver = repo_info["packages"][package]["info"]["version"] 309 repo_rel = repo_info["packages"][package]["info"]["release"] 310 repo_url = repo_info["info"]["url"] 311 if package in dl_list: 312 # Check package version, replace if newer version 313 if repo_ver == dl_list[package]["version"]: 314 # Version is the same, check release 315 if repo_rel > dl_list[package]["release"]: 316 dl_package = True 317 elif repo_rel == dl_list[package]["release"]: 318 # Version and release are the same, give 319 # preference to local (file://) repos 320 if dl_list[package]["source"].startswith("file://"): 321 if not repo_url.startswith("file://"): 322 dl_package = True 323 elif repo_ver > dl_list[package]["version"]: 324 dl_package = True 325 else: 326 dl_package = True 327 328 if dl_package is True: 329 # Put together download directory 330 cache_path = os.path.join(self.opts["spm_cache_dir"], repo) 331 332 # Put together download paths 333 dl_url = "{}/{}".format( 334 repo_info["info"]["url"], 335 repo_info["packages"][package]["filename"], 336 ) 337 out_file = os.path.join( 338 cache_path, repo_info["packages"][package]["filename"] 339 ) 340 dl_list[package] = { 341 "version": repo_ver, 342 "release": repo_rel, 343 "source": dl_url, 344 "dest_dir": cache_path, 345 "dest_file": out_file, 346 } 347 348 for package in dl_list: 349 dl_url = dl_list[package]["source"] 350 cache_path = dl_list[package]["dest_dir"] 351 out_file = dl_list[package]["dest_file"] 352 353 # Make sure download directory exists 354 if not os.path.exists(cache_path): 355 os.makedirs(cache_path) 356 357 # Download the package 358 if dl_url.startswith("file://"): 359 dl_url = dl_url.replace("file://", "") 360 shutil.copyfile(dl_url, out_file) 361 else: 362 with salt.utils.files.fopen(out_file, "wb") as outf: 363 outf.write( 364 self._query_http(dl_url, repo_info["info"], decode_body=False) 365 ) 366 367 # First we download everything, then we install 368 for package in dl_list: 369 out_file = dl_list[package]["dest_file"] 370 # Kick off the install 371 self._install_indv_pkg(package, out_file) 372 return 373 374 def _local_install(self, args, pkg_name=None): 375 """ 376 Install a package from a file 377 """ 378 if len(args) < 2: 379 raise SPMInvocationError("A package file must be specified") 380 381 self._install(args) 382 383 def _check_all_deps(self, pkg_name=None, pkg_file=None, formula_def=None): 384 """ 385 Starting with one package, check all packages for dependencies 386 """ 387 if pkg_file and not os.path.exists(pkg_file): 388 raise SPMInvocationError("Package file {} not found".format(pkg_file)) 389 390 self.repo_metadata = self._get_repo_metadata() 391 if not formula_def: 392 for repo in self.repo_metadata: 393 if not isinstance(self.repo_metadata[repo]["packages"], dict): 394 continue 395 if pkg_name in self.repo_metadata[repo]["packages"]: 396 formula_def = self.repo_metadata[repo]["packages"][pkg_name]["info"] 397 398 if not formula_def: 399 raise SPMInvocationError("Unable to read formula for {}".format(pkg_name)) 400 401 # Check to see if the package is already installed 402 pkg_info = self._pkgdb_fun("info", pkg_name, self.db_conn) 403 pkgs_to_install = [] 404 if pkg_info is None or self.opts["force"]: 405 pkgs_to_install.append(pkg_name) 406 elif pkg_info is not None and not self.opts["force"]: 407 raise SPMPackageError( 408 "Package {} already installed, not installing again".format( 409 formula_def["name"] 410 ) 411 ) 412 413 optional_install = [] 414 recommended_install = [] 415 if ( 416 "dependencies" in formula_def 417 or "optional" in formula_def 418 or "recommended" in formula_def 419 ): 420 self.avail_pkgs = {} 421 for repo in self.repo_metadata: 422 if not isinstance(self.repo_metadata[repo]["packages"], dict): 423 continue 424 for pkg in self.repo_metadata[repo]["packages"]: 425 self.avail_pkgs[pkg] = repo 426 427 needs, unavail, optional, recommended = self._resolve_deps(formula_def) 428 429 if len(unavail) > 0: 430 raise SPMPackageError( 431 "Cannot install {}, the following dependencies are needed:\n\n{}".format( 432 formula_def["name"], "\n".join(unavail) 433 ) 434 ) 435 436 if optional: 437 optional_install.extend(optional) 438 for dep_pkg in optional: 439 pkg_info = self._pkgdb_fun("info", formula_def["name"]) 440 msg = dep_pkg 441 if isinstance(pkg_info, dict): 442 msg = "{} [Installed]".format(dep_pkg) 443 optional_install.append(msg) 444 445 if recommended: 446 recommended_install.extend(recommended) 447 for dep_pkg in recommended: 448 pkg_info = self._pkgdb_fun("info", formula_def["name"]) 449 msg = dep_pkg 450 if isinstance(pkg_info, dict): 451 msg = "{} [Installed]".format(dep_pkg) 452 recommended_install.append(msg) 453 454 if needs: 455 pkgs_to_install.extend(needs) 456 for dep_pkg in needs: 457 pkg_info = self._pkgdb_fun("info", formula_def["name"]) 458 msg = dep_pkg 459 if isinstance(pkg_info, dict): 460 msg = "{} [Installed]".format(dep_pkg) 461 462 return pkgs_to_install, optional_install, recommended_install 463 464 def _install_indv_pkg(self, pkg_name, pkg_file): 465 """ 466 Install one individual package 467 """ 468 self.ui.status("... installing {}".format(pkg_name)) 469 formula_tar = tarfile.open(pkg_file, "r:bz2") 470 formula_ref = formula_tar.extractfile("{}/FORMULA".format(pkg_name)) 471 formula_def = salt.utils.yaml.safe_load(formula_ref) 472 473 for field in ("version", "release", "summary", "description"): 474 if field not in formula_def: 475 raise SPMPackageError( 476 "Invalid package: the {} was not found".format(field) 477 ) 478 479 pkg_files = formula_tar.getmembers() 480 481 # First pass: check for files that already exist 482 existing_files = self._pkgfiles_fun( 483 "check_existing", pkg_name, pkg_files, formula_def 484 ) 485 486 if existing_files and not self.opts["force"]: 487 raise SPMPackageError( 488 "Not installing {} due to existing files:\n\n{}".format( 489 pkg_name, "\n".join(existing_files) 490 ) 491 ) 492 493 # We've decided to install 494 self._pkgdb_fun("register_pkg", pkg_name, formula_def, self.db_conn) 495 496 # Run the pre_local_state script, if present 497 if "pre_local_state" in formula_def: 498 high_data = self._render(formula_def["pre_local_state"], formula_def) 499 ret = self.caller.cmd("state.high", data=high_data) 500 if "pre_tgt_state" in formula_def: 501 log.debug("Executing pre_tgt_state script") 502 high_data = self._render(formula_def["pre_tgt_state"]["data"], formula_def) 503 tgt = formula_def["pre_tgt_state"]["tgt"] 504 ret = self.client.run_job( 505 tgt=formula_def["pre_tgt_state"]["tgt"], 506 fun="state.high", 507 tgt_type=formula_def["pre_tgt_state"].get("tgt_type", "glob"), 508 timout=self.opts["timeout"], 509 data=high_data, 510 ) 511 512 # No defaults for this in config.py; default to the current running 513 # user and group 514 if salt.utils.platform.is_windows(): 515 uname = gname = salt.utils.win_functions.get_current_user() 516 uname_sid = salt.utils.win_functions.get_sid_from_name(uname) 517 uid = self.opts.get("spm_uid", uname_sid) 518 gid = self.opts.get("spm_gid", uname_sid) 519 else: 520 uid = self.opts.get("spm_uid", os.getuid()) 521 gid = self.opts.get("spm_gid", os.getgid()) 522 uname = pwd.getpwuid(uid)[0] 523 gname = grp.getgrgid(gid)[0] 524 525 # Second pass: install the files 526 for member in pkg_files: 527 member.uid = uid 528 member.gid = gid 529 member.uname = uname 530 member.gname = gname 531 532 out_path = self._pkgfiles_fun( 533 "install_file", 534 pkg_name, 535 formula_tar, 536 member, 537 formula_def, 538 self.files_conn, 539 ) 540 if out_path is not False: 541 if member.isdir(): 542 digest = "" 543 else: 544 self._verbose( 545 "Installing file {} to {}".format(member.name, out_path), 546 log.trace, 547 ) 548 file_hash = hashlib.sha1() 549 digest = self._pkgfiles_fun( 550 "hash_file", 551 os.path.join(out_path, member.name), 552 file_hash, 553 self.files_conn, 554 ) 555 self._pkgdb_fun( 556 "register_file", pkg_name, member, out_path, digest, self.db_conn 557 ) 558 559 # Run the post_local_state script, if present 560 if "post_local_state" in formula_def: 561 log.debug("Executing post_local_state script") 562 high_data = self._render(formula_def["post_local_state"], formula_def) 563 self.caller.cmd("state.high", data=high_data) 564 if "post_tgt_state" in formula_def: 565 log.debug("Executing post_tgt_state script") 566 high_data = self._render(formula_def["post_tgt_state"]["data"], formula_def) 567 tgt = formula_def["post_tgt_state"]["tgt"] 568 ret = self.client.run_job( 569 tgt=formula_def["post_tgt_state"]["tgt"], 570 fun="state.high", 571 tgt_type=formula_def["post_tgt_state"].get("tgt_type", "glob"), 572 timout=self.opts["timeout"], 573 data=high_data, 574 ) 575 576 formula_tar.close() 577 578 def _resolve_deps(self, formula_def): 579 """ 580 Return a list of packages which need to be installed, to resolve all 581 dependencies 582 """ 583 pkg_info = self.pkgdb["{}.info".format(self.db_prov)](formula_def["name"]) 584 if not isinstance(pkg_info, dict): 585 pkg_info = {} 586 587 can_has = {} 588 cant_has = [] 589 if "dependencies" in formula_def and formula_def["dependencies"] is None: 590 formula_def["dependencies"] = "" 591 for dep in formula_def.get("dependencies", "").split(","): 592 dep = dep.strip() 593 if not dep: 594 continue 595 if self.pkgdb["{}.info".format(self.db_prov)](dep): 596 continue 597 598 if dep in self.avail_pkgs: 599 can_has[dep] = self.avail_pkgs[dep] 600 else: 601 cant_has.append(dep) 602 603 optional = formula_def.get("optional", "").split(",") 604 recommended = formula_def.get("recommended", "").split(",") 605 606 inspected = [] 607 to_inspect = can_has.copy() 608 while len(to_inspect) > 0: 609 dep = next(iter(to_inspect.keys())) 610 del to_inspect[dep] 611 612 # Don't try to resolve the same package more than once 613 if dep in inspected: 614 continue 615 inspected.append(dep) 616 617 repo_contents = self.repo_metadata.get(can_has[dep], {}) 618 repo_packages = repo_contents.get("packages", {}) 619 dep_formula = repo_packages.get(dep, {}).get("info", {}) 620 621 also_can, also_cant, opt_dep, rec_dep = self._resolve_deps(dep_formula) 622 can_has.update(also_can) 623 cant_has = sorted(set(cant_has + also_cant)) 624 optional = sorted(set(optional + opt_dep)) 625 recommended = sorted(set(recommended + rec_dep)) 626 627 return can_has, cant_has, optional, recommended 628 629 def _traverse_repos(self, callback, repo_name=None): 630 """ 631 Traverse through all repo files and apply the functionality provided in 632 the callback to them 633 """ 634 repo_files = [] 635 if os.path.exists(self.opts["spm_repos_config"]): 636 repo_files.append(self.opts["spm_repos_config"]) 637 638 for (dirpath, dirnames, filenames) in salt.utils.path.os_walk( 639 "{}.d".format(self.opts["spm_repos_config"]) 640 ): 641 for repo_file in filenames: 642 if not repo_file.endswith(".repo"): 643 continue 644 repo_files.append(repo_file) 645 646 for repo_file in repo_files: 647 repo_path = "{}.d/{}".format(self.opts["spm_repos_config"], repo_file) 648 with salt.utils.files.fopen(repo_path) as rph: 649 repo_data = salt.utils.yaml.safe_load(rph) 650 for repo in repo_data: 651 if repo_data[repo].get("enabled", True) is False: 652 continue 653 if repo_name is not None and repo != repo_name: 654 continue 655 callback(repo, repo_data[repo]) 656 657 def _query_http(self, dl_path, repo_info, decode_body=True): 658 """ 659 Download files via http 660 """ 661 query = None 662 response = None 663 664 try: 665 if "username" in repo_info: 666 try: 667 if "password" in repo_info: 668 query = http.query( 669 dl_path, 670 text=True, 671 username=repo_info["username"], 672 password=repo_info["password"], 673 decode_body=decode_body, 674 ) 675 else: 676 raise SPMException( 677 "Auth defined, but password is not set for username: '{}'".format( 678 repo_info["username"] 679 ) 680 ) 681 except SPMException as exc: 682 self.ui.error(str(exc)) 683 else: 684 query = http.query(dl_path, text=True, decode_body=decode_body) 685 except SPMException as exc: 686 self.ui.error(str(exc)) 687 688 try: 689 if query: 690 if "SPM-METADATA" in dl_path: 691 response = salt.utils.yaml.safe_load(query.get("text", "{}")) 692 else: 693 response = query.get("text") 694 else: 695 raise SPMException("Response is empty, please check for Errors above.") 696 except SPMException as exc: 697 self.ui.error(str(exc)) 698 699 return response 700 701 def _download_repo_metadata(self, args): 702 """ 703 Connect to all repos and download metadata 704 """ 705 cache = salt.cache.Cache(self.opts, self.opts["spm_cache_dir"]) 706 707 def _update_metadata(repo, repo_info): 708 dl_path = "{}/SPM-METADATA".format(repo_info["url"]) 709 if dl_path.startswith("file://"): 710 dl_path = dl_path.replace("file://", "") 711 with salt.utils.files.fopen(dl_path, "r") as rpm: 712 metadata = salt.utils.yaml.safe_load(rpm) 713 else: 714 metadata = self._query_http(dl_path, repo_info) 715 716 cache.store(".", repo, metadata) 717 718 repo_name = args[1] if len(args) > 1 else None 719 self._traverse_repos(_update_metadata, repo_name) 720 721 def _get_repo_metadata(self): 722 """ 723 Return cached repo metadata 724 """ 725 cache = salt.cache.Cache(self.opts, self.opts["spm_cache_dir"]) 726 metadata = {} 727 728 def _read_metadata(repo, repo_info): 729 if cache.updated(".", repo) is None: 730 log.warning("Updating repo metadata") 731 self._download_repo_metadata({}) 732 733 metadata[repo] = { 734 "info": repo_info, 735 "packages": cache.fetch(".", repo), 736 } 737 738 self._traverse_repos(_read_metadata) 739 return metadata 740 741 def _create_repo(self, args): 742 """ 743 Scan a directory and create an SPM-METADATA file which describes 744 all of the SPM files in that directory. 745 """ 746 if len(args) < 2: 747 raise SPMInvocationError("A path to a directory must be specified") 748 749 if args[1] == ".": 750 repo_path = os.getcwdu() 751 else: 752 repo_path = args[1] 753 754 old_files = [] 755 repo_metadata = {} 756 for (dirpath, dirnames, filenames) in salt.utils.path.os_walk(repo_path): 757 for spm_file in filenames: 758 if not spm_file.endswith(".spm"): 759 continue 760 spm_path = "{}/{}".format(repo_path, spm_file) 761 if not tarfile.is_tarfile(spm_path): 762 continue 763 comps = spm_file.split("-") 764 spm_name = "-".join(comps[:-2]) 765 spm_fh = tarfile.open(spm_path, "r:bz2") 766 formula_handle = spm_fh.extractfile("{}/FORMULA".format(spm_name)) 767 formula_conf = salt.utils.yaml.safe_load(formula_handle.read()) 768 769 use_formula = True 770 if spm_name in repo_metadata: 771 # This package is already in the repo; use the latest 772 cur_info = repo_metadata[spm_name]["info"] 773 new_info = formula_conf 774 if int(new_info["version"]) == int(cur_info["version"]): 775 # Version is the same, check release 776 if int(new_info["release"]) < int(cur_info["release"]): 777 # This is an old release; don't use it 778 use_formula = False 779 elif int(new_info["version"]) < int(cur_info["version"]): 780 # This is an old version; don't use it 781 use_formula = False 782 783 if use_formula is True: 784 # Ignore/archive/delete the old version 785 log.debug( 786 "%s %s-%s had been added, but %s-%s will replace it", 787 spm_name, 788 cur_info["version"], 789 cur_info["release"], 790 new_info["version"], 791 new_info["release"], 792 ) 793 old_files.append(repo_metadata[spm_name]["filename"]) 794 else: 795 # Ignore/archive/delete the new version 796 log.debug( 797 "%s %s-%s has been found, but is older than %s-%s", 798 spm_name, 799 new_info["version"], 800 new_info["release"], 801 cur_info["version"], 802 cur_info["release"], 803 ) 804 old_files.append(spm_file) 805 806 if use_formula is True: 807 log.debug( 808 "adding %s-%s-%s to the repo", 809 formula_conf["name"], 810 formula_conf["version"], 811 formula_conf["release"], 812 ) 813 repo_metadata[spm_name] = { 814 "info": formula_conf.copy(), 815 } 816 repo_metadata[spm_name]["filename"] = spm_file 817 818 metadata_filename = "{}/SPM-METADATA".format(repo_path) 819 with salt.utils.files.fopen(metadata_filename, "w") as mfh: 820 salt.utils.yaml.safe_dump( 821 repo_metadata, 822 mfh, 823 indent=4, 824 canonical=False, 825 default_flow_style=False, 826 ) 827 828 log.debug("Wrote %s", metadata_filename) 829 830 for file_ in old_files: 831 if self.opts["spm_repo_dups"] == "ignore": 832 # ignore old packages, but still only add the latest 833 log.debug("%s will be left in the directory", file_) 834 elif self.opts["spm_repo_dups"] == "archive": 835 # spm_repo_archive_path is where old packages are moved 836 if not os.path.exists("./archive"): 837 try: 838 os.makedirs("./archive") 839 log.debug("%s has been archived", file_) 840 except OSError: 841 log.error("Unable to create archive directory") 842 try: 843 shutil.move(file_, "./archive") 844 except OSError: 845 log.error("Unable to archive %s", file_) 846 elif self.opts["spm_repo_dups"] == "delete": 847 # delete old packages from the repo 848 try: 849 os.remove(file_) 850 log.debug("%s has been deleted", file_) 851 except OSError: 852 log.error("Unable to delete %s", file_) 853 except OSError: # pylint: disable=duplicate-except 854 # The file has already been deleted 855 pass 856 857 def _remove(self, args): 858 """ 859 Remove a package 860 """ 861 if len(args) < 2: 862 raise SPMInvocationError("A package must be specified") 863 864 packages = args[1:] 865 msg = "Removing packages:\n\t{}".format("\n\t".join(packages)) 866 867 if not self.opts["assume_yes"]: 868 self.ui.confirm(msg) 869 870 for package in packages: 871 self.ui.status("... removing {}".format(package)) 872 873 if not self._pkgdb_fun("db_exists", self.opts["spm_db"]): 874 raise SPMDatabaseError( 875 "No database at {}, cannot remove {}".format( 876 self.opts["spm_db"], package 877 ) 878 ) 879 880 # Look at local repo index 881 pkg_info = self._pkgdb_fun("info", package, self.db_conn) 882 if pkg_info is None: 883 raise SPMInvocationError("Package {} not installed".format(package)) 884 885 # Find files that have not changed and remove them 886 files = self._pkgdb_fun("list_files", package, self.db_conn) 887 dirs = [] 888 for filerow in files: 889 if self._pkgfiles_fun("path_isdir", filerow[0]): 890 dirs.append(filerow[0]) 891 continue 892 file_hash = hashlib.sha1() 893 digest = self._pkgfiles_fun( 894 "hash_file", filerow[0], file_hash, self.files_conn 895 ) 896 if filerow[1] == digest: 897 self._verbose("Removing file {}".format(filerow[0]), log.trace) 898 self._pkgfiles_fun("remove_file", filerow[0], self.files_conn) 899 else: 900 self._verbose("Not removing file {}".format(filerow[0]), log.trace) 901 self._pkgdb_fun("unregister_file", filerow[0], package, self.db_conn) 902 903 # Clean up directories 904 for dir_ in sorted(dirs, reverse=True): 905 self._pkgdb_fun("unregister_file", dir_, package, self.db_conn) 906 try: 907 self._verbose("Removing directory {}".format(dir_), log.trace) 908 os.rmdir(dir_) 909 except OSError: 910 # Leave directories in place that still have files in them 911 self._verbose( 912 "Cannot remove directory {}, probably not empty".format(dir_), 913 log.trace, 914 ) 915 916 self._pkgdb_fun("unregister_pkg", package, self.db_conn) 917 918 def _verbose(self, msg, level=log.debug): 919 """ 920 Display verbose information 921 """ 922 if self.opts.get("verbose", False) is True: 923 self.ui.status(msg) 924 level(msg) 925 926 def _local_info(self, args): 927 """ 928 List info for a package file 929 """ 930 if len(args) < 2: 931 raise SPMInvocationError("A package filename must be specified") 932 933 pkg_file = args[1] 934 935 if not os.path.exists(pkg_file): 936 raise SPMInvocationError("Package file {} not found".format(pkg_file)) 937 938 comps = pkg_file.split("-") 939 comps = "-".join(comps[:-2]).split("/") 940 name = comps[-1] 941 942 formula_tar = tarfile.open(pkg_file, "r:bz2") 943 formula_ref = formula_tar.extractfile("{}/FORMULA".format(name)) 944 formula_def = salt.utils.yaml.safe_load(formula_ref) 945 946 self.ui.status(self._get_info(formula_def)) 947 formula_tar.close() 948 949 def _info(self, args): 950 """ 951 List info for a package 952 """ 953 if len(args) < 2: 954 raise SPMInvocationError("A package must be specified") 955 956 package = args[1] 957 958 pkg_info = self._pkgdb_fun("info", package, self.db_conn) 959 if pkg_info is None: 960 raise SPMPackageError("package {} not installed".format(package)) 961 self.ui.status(self._get_info(pkg_info)) 962 963 def _get_info(self, formula_def): 964 """ 965 Get package info 966 """ 967 fields = ( 968 "name", 969 "os", 970 "os_family", 971 "release", 972 "version", 973 "dependencies", 974 "os_dependencies", 975 "os_family_dependencies", 976 "summary", 977 "description", 978 ) 979 for item in fields: 980 if item not in formula_def: 981 formula_def[item] = "None" 982 983 if "installed" not in formula_def: 984 formula_def["installed"] = "Not installed" 985 986 return ( 987 "Name: {name}\n" 988 "Version: {version}\n" 989 "Release: {release}\n" 990 "Install Date: {installed}\n" 991 "Supported OSes: {os}\n" 992 "Supported OS families: {os_family}\n" 993 "Dependencies: {dependencies}\n" 994 "OS Dependencies: {os_dependencies}\n" 995 "OS Family Dependencies: {os_family_dependencies}\n" 996 "Summary: {summary}\n" 997 "Description:\n" 998 "{description}".format(**formula_def) 999 ) 1000 1001 def _local_list_files(self, args): 1002 """ 1003 List files for a package file 1004 """ 1005 if len(args) < 2: 1006 raise SPMInvocationError("A package filename must be specified") 1007 1008 pkg_file = args[1] 1009 if not os.path.exists(pkg_file): 1010 raise SPMPackageError("Package file {} not found".format(pkg_file)) 1011 formula_tar = tarfile.open(pkg_file, "r:bz2") 1012 pkg_files = formula_tar.getmembers() 1013 1014 for member in pkg_files: 1015 self.ui.status(member.name) 1016 1017 def _list_packages(self, args): 1018 """ 1019 List files for an installed package 1020 """ 1021 packages = self._pkgdb_fun("list_packages", self.db_conn) 1022 for package in packages: 1023 if self.opts["verbose"]: 1024 status_msg = ",".join(package) 1025 else: 1026 status_msg = package[0] 1027 self.ui.status(status_msg) 1028 1029 def _list_files(self, args): 1030 """ 1031 List files for an installed package 1032 """ 1033 if len(args) < 2: 1034 raise SPMInvocationError("A package name must be specified") 1035 1036 package = args[-1] 1037 1038 files = self._pkgdb_fun("list_files", package, self.db_conn) 1039 if files is None: 1040 raise SPMPackageError("package {} not installed".format(package)) 1041 else: 1042 for file_ in files: 1043 if self.opts["verbose"]: 1044 status_msg = ",".join(file_) 1045 else: 1046 status_msg = file_[0] 1047 self.ui.status(status_msg) 1048 1049 def _build(self, args): 1050 """ 1051 Build a package 1052 """ 1053 if len(args) < 2: 1054 raise SPMInvocationError("A path to a formula must be specified") 1055 1056 self.abspath = args[1].rstrip("/") 1057 comps = self.abspath.split("/") 1058 self.relpath = comps[-1] 1059 1060 formula_path = "{}/FORMULA".format(self.abspath) 1061 if not os.path.exists(formula_path): 1062 raise SPMPackageError("Formula file {} not found".format(formula_path)) 1063 with salt.utils.files.fopen(formula_path) as fp_: 1064 formula_conf = salt.utils.yaml.safe_load(fp_) 1065 1066 for field in ("name", "version", "release", "summary", "description"): 1067 if field not in formula_conf: 1068 raise SPMPackageError( 1069 "Invalid package: a {} must be defined".format(field) 1070 ) 1071 1072 out_path = "{}/{}-{}-{}.spm".format( 1073 self.opts["spm_build_dir"], 1074 formula_conf["name"], 1075 formula_conf["version"], 1076 formula_conf["release"], 1077 ) 1078 1079 if not os.path.exists(self.opts["spm_build_dir"]): 1080 os.mkdir(self.opts["spm_build_dir"]) 1081 1082 self.formula_conf = formula_conf 1083 1084 formula_tar = tarfile.open(out_path, "w:bz2") 1085 1086 if "files" in formula_conf: 1087 # This allows files to be added to the SPM file in a specific order. 1088 # It also allows for files to be tagged as a certain type, as with 1089 # RPM files. This tag is ignored here, but is used when installing 1090 # the SPM file. 1091 if isinstance(formula_conf["files"], list): 1092 formula_dir = tarfile.TarInfo(formula_conf["name"]) 1093 formula_dir.type = tarfile.DIRTYPE 1094 formula_tar.addfile(formula_dir) 1095 for file_ in formula_conf["files"]: 1096 for ftype in FILE_TYPES: 1097 if file_.startswith("{}|".format(ftype)): 1098 file_ = file_.lstrip("{}|".format(ftype)) 1099 formula_tar.add( 1100 os.path.join(os.getcwd(), file_), 1101 os.path.join(formula_conf["name"], file_), 1102 ) 1103 else: 1104 # If no files are specified, then the whole directory will be added. 1105 try: 1106 formula_tar.add( 1107 formula_path, formula_conf["name"], filter=self._exclude 1108 ) 1109 formula_tar.add( 1110 self.abspath, formula_conf["name"], filter=self._exclude 1111 ) 1112 except TypeError: 1113 formula_tar.add( 1114 formula_path, formula_conf["name"], exclude=self._exclude 1115 ) 1116 formula_tar.add( 1117 self.abspath, formula_conf["name"], exclude=self._exclude 1118 ) 1119 formula_tar.close() 1120 1121 self.ui.status("Built package {}".format(out_path)) 1122 1123 def _exclude(self, member): 1124 """ 1125 Exclude based on opts 1126 """ 1127 if isinstance(member, str): 1128 return None 1129 1130 for item in self.opts["spm_build_exclude"]: 1131 if member.name.startswith("{}/{}".format(self.formula_conf["name"], item)): 1132 return None 1133 elif member.name.startswith("{}/{}".format(self.abspath, item)): 1134 return None 1135 return member 1136 1137 def _render(self, data, formula_def): 1138 """ 1139 Render a [pre|post]_local_state or [pre|post]_tgt_state script 1140 """ 1141 # FORMULA can contain a renderer option 1142 renderer = formula_def.get("renderer", self.opts.get("renderer", "jinja|yaml")) 1143 rend = salt.loader.render(self.opts, {}) 1144 blacklist = self.opts.get("renderer_blacklist") 1145 whitelist = self.opts.get("renderer_whitelist") 1146 template_vars = formula_def.copy() 1147 template_vars["opts"] = self.opts.copy() 1148 return compile_template( 1149 ":string:", 1150 rend, 1151 renderer, 1152 blacklist, 1153 whitelist, 1154 input_data=data, 1155 **template_vars 1156 ) 1157 1158 1159class SPMUserInterface: 1160 """ 1161 Handle user interaction with an SPMClient object 1162 """ 1163 1164 def status(self, msg): 1165 """ 1166 Report an SPMClient status message 1167 """ 1168 raise NotImplementedError() 1169 1170 def error(self, msg): 1171 """ 1172 Report an SPM error message 1173 """ 1174 raise NotImplementedError() 1175 1176 def confirm(self, action): 1177 """ 1178 Get confirmation from the user before performing an SPMClient action. 1179 Return if the action is confirmed, or raise SPMOperationCanceled(<msg>) 1180 if canceled. 1181 """ 1182 raise NotImplementedError() 1183 1184 1185class SPMCmdlineInterface(SPMUserInterface): 1186 """ 1187 Command-line interface to SPMClient 1188 """ 1189 1190 def status(self, msg): 1191 print(msg) 1192 1193 def error(self, msg): 1194 print(msg, file=sys.stderr) 1195 1196 def confirm(self, action): 1197 print(action) 1198 res = input("Proceed? [N/y] ") 1199 if not res.lower().startswith("y"): 1200 raise SPMOperationCanceled("canceled") 1201