1""" 2Runner to manage Windows software repo 3""" 4 5# WARNING: Any modules imported here must also be added to 6# salt/modules/win_repo.py 7 8 9import logging 10import os 11 12import salt.loader 13import salt.minion 14import salt.template 15import salt.utils.files 16import salt.utils.gitfs 17import salt.utils.msgpack 18import salt.utils.path 19from salt.exceptions import CommandExecutionError, SaltRenderError 20 21log = logging.getLogger(__name__) 22 23# Global parameters which can be overridden on a per-remote basis 24PER_REMOTE_OVERRIDES = ("ssl_verify", "refspecs", "fallback") 25 26# Fall back to default per-remote-only. This isn't technically needed since 27# salt.utils.gitfs.GitBase.__init__ will default to 28# salt.utils.gitfs.PER_REMOTE_ONLY for this value, so this is mainly for 29# runners and other modules that import salt.runners.winrepo. 30PER_REMOTE_ONLY = salt.utils.gitfs.PER_REMOTE_ONLY 31GLOBAL_ONLY = ("branch",) 32 33 34def genrepo(opts=None, fire_event=True): 35 """ 36 Generate winrepo_cachefile based on sls files in the winrepo_dir 37 38 opts 39 Specify an alternate opts dict. Should not be used unless this function 40 is imported into an execution module. 41 42 fire_event : True 43 Fire an event on failure. Only supported on the master. 44 45 CLI Example: 46 47 .. code-block:: bash 48 49 salt-run winrepo.genrepo 50 """ 51 if opts is None: 52 opts = __opts__ 53 54 winrepo_dir = opts["winrepo_dir"] 55 winrepo_cachefile = opts["winrepo_cachefile"] 56 57 ret = {} 58 if not os.path.exists(winrepo_dir): 59 os.makedirs(winrepo_dir) 60 renderers = salt.loader.render(opts, __salt__) 61 for root, _, files in salt.utils.path.os_walk(winrepo_dir): 62 for name in files: 63 if name.endswith(".sls"): 64 try: 65 config = salt.template.compile_template( 66 os.path.join(root, name), 67 renderers, 68 opts["renderer"], 69 opts["renderer_blacklist"], 70 opts["renderer_whitelist"], 71 ) 72 except SaltRenderError as exc: 73 log.debug("Failed to render %s.", os.path.join(root, name)) 74 log.debug("Error: %s.", exc) 75 continue 76 if config: 77 revmap = {} 78 for pkgname, versions in config.items(): 79 log.debug("Compiling winrepo data for package '%s'", pkgname) 80 for version, repodata in versions.items(): 81 log.debug( 82 "Compiling winrepo data for %s version %s", 83 pkgname, 84 version, 85 ) 86 if not isinstance(version, str): 87 config[pkgname][str(version)] = config[pkgname].pop( 88 version 89 ) 90 if not isinstance(repodata, dict): 91 msg = "Failed to compile {}.".format( 92 os.path.join(root, name) 93 ) 94 log.debug(msg) 95 if fire_event: 96 try: 97 __jid_event__.fire_event( 98 {"error": msg}, "progress" 99 ) 100 except NameError: 101 log.error( 102 "Attempted to fire the an event " 103 "with the following error, but " 104 "event firing is not supported: %s", 105 msg, 106 ) 107 continue 108 revmap[repodata["full_name"]] = pkgname 109 ret.setdefault("repo", {}).update(config) 110 ret.setdefault("name_map", {}).update(revmap) 111 with salt.utils.files.fopen( 112 os.path.join(winrepo_dir, winrepo_cachefile), "w+b" 113 ) as repo: 114 repo.write(salt.utils.msgpack.dumps(ret)) 115 return ret 116 117 118def update_git_repos(opts=None, clean=False, masterless=False): 119 """ 120 Checkout git repos containing Windows Software Package Definitions 121 122 opts 123 Specify an alternate opts dict. Should not be used unless this function 124 is imported into an execution module. 125 126 clean : False 127 Clean repo cachedirs which are not configured under 128 :conf_master:`winrepo_remotes`. 129 130 .. warning:: 131 This argument should not be set to ``True`` if a mix of git and 132 non-git repo definitions are being used, as it will result in the 133 non-git repo definitions being removed. 134 135 .. versionadded:: 2015.8.0 136 137 CLI Examples: 138 139 .. code-block:: bash 140 141 salt-run winrepo.update_git_repos 142 salt-run winrepo.update_git_repos clean=True 143 """ 144 if opts is None: 145 opts = __opts__ 146 147 winrepo_dir = opts["winrepo_dir"] 148 winrepo_remotes = opts["winrepo_remotes"] 149 150 winrepo_cfg = [ 151 (winrepo_remotes, winrepo_dir), 152 (opts["winrepo_remotes_ng"], opts["winrepo_dir_ng"]), 153 ] 154 155 ret = {} 156 for remotes, base_dir in winrepo_cfg: 157 if not any( 158 (salt.utils.gitfs.GITPYTHON_VERSION, salt.utils.gitfs.PYGIT2_VERSION) 159 ): 160 # Use legacy code 161 winrepo_result = {} 162 for remote_info in remotes: 163 if "/" in remote_info: 164 targetname = remote_info.split("/")[-1] 165 else: 166 targetname = remote_info 167 rev = "HEAD" 168 # If a revision is specified, use it. 169 try: 170 rev, remote_url = remote_info.strip().split() 171 except ValueError: 172 remote_url = remote_info 173 gittarget = os.path.join(base_dir, targetname).replace(".", "_") 174 if masterless: 175 result = __salt__["state.single"]( 176 "git.latest", 177 name=remote_url, 178 rev=rev, 179 branch="winrepo", 180 target=gittarget, 181 force_checkout=True, 182 force_reset=True, 183 ) 184 if isinstance(result, list): 185 # Errors were detected 186 raise CommandExecutionError( 187 "Failed up update winrepo remotes: {}".format( 188 "\n".join(result) 189 ) 190 ) 191 if "name" not in result: 192 # Highstate output dict, the results are actually nested 193 # one level down. 194 key = next(iter(result)) 195 result = result[key] 196 else: 197 mminion = salt.minion.MasterMinion(opts) 198 result = mminion.states["git.latest"]( 199 remote_url, 200 rev=rev, 201 branch="winrepo", 202 target=gittarget, 203 force_checkout=True, 204 force_reset=True, 205 ) 206 winrepo_result[result["name"]] = result["result"] 207 ret.update(winrepo_result) 208 else: 209 # New winrepo code utilizing salt.utils.gitfs 210 try: 211 winrepo = salt.utils.gitfs.WinRepo( 212 opts, 213 remotes, 214 per_remote_overrides=PER_REMOTE_OVERRIDES, 215 per_remote_only=PER_REMOTE_ONLY, 216 global_only=GLOBAL_ONLY, 217 cache_root=base_dir, 218 ) 219 winrepo.fetch_remotes() 220 # Since we're not running update(), we need to manually call 221 # clear_old_remotes() to remove directories from remotes that 222 # have been removed from configuration. 223 if clean: 224 winrepo.clear_old_remotes() 225 winrepo.checkout() 226 except Exception as exc: # pylint: disable=broad-except 227 msg = "Failed to update winrepo_remotes: {}".format(exc) 228 log.error(msg, exc_info_on_loglevel=logging.DEBUG) 229 return msg 230 ret.update(winrepo.winrepo_dirs) 231 return ret 232