1""" 2Functions used to sync external modules 3""" 4 5import logging 6import os 7import shutil 8 9import salt.fileclient 10import salt.utils.files 11import salt.utils.hashutils 12import salt.utils.path 13import salt.utils.url 14 15log = logging.getLogger(__name__) 16 17 18def _list_emptydirs(rootdir): 19 emptydirs = [] 20 for root, dirs, files in salt.utils.path.os_walk(rootdir): 21 if not files and not dirs: 22 emptydirs.append(root) 23 return emptydirs 24 25 26def _listdir_recursively(rootdir): 27 file_list = [] 28 for root, dirs, files in salt.utils.path.os_walk(rootdir): 29 for filename in files: 30 relpath = os.path.relpath(root, rootdir).strip(".") 31 file_list.append(os.path.join(relpath, filename)) 32 return file_list 33 34 35def sync(opts, form, saltenv=None, extmod_whitelist=None, extmod_blacklist=None): 36 """ 37 Sync custom modules into the extension_modules directory 38 """ 39 if saltenv is None: 40 saltenv = ["base"] 41 42 if extmod_whitelist is None: 43 extmod_whitelist = opts["extmod_whitelist"] 44 elif isinstance(extmod_whitelist, str): 45 extmod_whitelist = {form: extmod_whitelist.split(",")} 46 elif not isinstance(extmod_whitelist, dict): 47 log.error( 48 "extmod_whitelist must be a string or dictionary: %s", extmod_whitelist 49 ) 50 51 if extmod_blacklist is None: 52 extmod_blacklist = opts["extmod_blacklist"] 53 elif isinstance(extmod_blacklist, str): 54 extmod_blacklist = {form: extmod_blacklist.split(",")} 55 elif not isinstance(extmod_blacklist, dict): 56 log.error( 57 "extmod_blacklist must be a string or dictionary: %s", extmod_blacklist 58 ) 59 60 if isinstance(saltenv, str): 61 saltenv = saltenv.split(",") 62 ret = [] 63 remote = set() 64 source = salt.utils.url.create("_" + form) 65 mod_dir = os.path.join(opts["extension_modules"], "{}".format(form)) 66 touched = False 67 with salt.utils.files.set_umask(0o077): 68 try: 69 if not os.path.isdir(mod_dir): 70 log.info("Creating module dir '%s'", mod_dir) 71 try: 72 os.makedirs(mod_dir) 73 except OSError: 74 log.error( 75 "Cannot create cache module directory %s. Check permissions.", 76 mod_dir, 77 ) 78 fileclient = salt.fileclient.get_file_client(opts) 79 for sub_env in saltenv: 80 log.info("Syncing %s for environment '%s'", form, sub_env) 81 cache = [] 82 log.info("Loading cache from %s, for %s", source, sub_env) 83 # Grab only the desired files (.py, .pyx, .so) 84 cache.extend( 85 fileclient.cache_dir( 86 source, 87 sub_env, 88 include_empty=False, 89 include_pat=r"E@\.(pyx?|so|zip)$", 90 exclude_pat=None, 91 ) 92 ) 93 local_cache_dir = os.path.join( 94 opts["cachedir"], "files", sub_env, "_{}".format(form) 95 ) 96 log.debug("Local cache dir: '%s'", local_cache_dir) 97 for fn_ in cache: 98 relpath = os.path.relpath(fn_, local_cache_dir) 99 relname = os.path.splitext(relpath)[0].replace(os.sep, ".") 100 if ( 101 extmod_whitelist 102 and form in extmod_whitelist 103 and relname not in extmod_whitelist[form] 104 ): 105 continue 106 if ( 107 extmod_blacklist 108 and form in extmod_blacklist 109 and relname in extmod_blacklist[form] 110 ): 111 continue 112 remote.add(relpath) 113 dest = os.path.join(mod_dir, relpath) 114 log.info("Copying '%s' to '%s'", fn_, dest) 115 if os.path.isfile(dest): 116 # The file is present, if the sum differs replace it 117 hash_type = opts.get("hash_type", "md5") 118 src_digest = salt.utils.hashutils.get_hash(fn_, hash_type) 119 dst_digest = salt.utils.hashutils.get_hash(dest, hash_type) 120 if src_digest != dst_digest: 121 # The downloaded file differs, replace! 122 shutil.copyfile(fn_, dest) 123 ret.append("{}.{}".format(form, relname)) 124 else: 125 dest_dir = os.path.dirname(dest) 126 if not os.path.isdir(dest_dir): 127 os.makedirs(dest_dir) 128 shutil.copyfile(fn_, dest) 129 ret.append("{}.{}".format(form, relname)) 130 131 touched = bool(ret) 132 if opts["clean_dynamic_modules"] is True: 133 current = set(_listdir_recursively(mod_dir)) 134 for fn_ in current - remote: 135 full = os.path.join(mod_dir, fn_) 136 if os.path.isfile(full): 137 touched = True 138 os.remove(full) 139 # Cleanup empty dirs 140 while True: 141 emptydirs = _list_emptydirs(mod_dir) 142 if not emptydirs: 143 break 144 for emptydir in emptydirs: 145 touched = True 146 shutil.rmtree(emptydir, ignore_errors=True) 147 except Exception as exc: # pylint: disable=broad-except 148 log.error("Failed to sync %s module: %s", form, exc) 149 return ret, touched 150