1"""
2Functions used to sync external modules
3"""
4
5import logging
6import os
7import shutil
8
9import salt.fileclient
10import salt.utils.files
11import salt.utils.hashutils
12import salt.utils.path
13import salt.utils.url
14
15log = logging.getLogger(__name__)
16
17
18def _list_emptydirs(rootdir):
19    emptydirs = []
20    for root, dirs, files in salt.utils.path.os_walk(rootdir):
21        if not files and not dirs:
22            emptydirs.append(root)
23    return emptydirs
24
25
26def _listdir_recursively(rootdir):
27    file_list = []
28    for root, dirs, files in salt.utils.path.os_walk(rootdir):
29        for filename in files:
30            relpath = os.path.relpath(root, rootdir).strip(".")
31            file_list.append(os.path.join(relpath, filename))
32    return file_list
33
34
35def sync(opts, form, saltenv=None, extmod_whitelist=None, extmod_blacklist=None):
36    """
37    Sync custom modules into the extension_modules directory
38    """
39    if saltenv is None:
40        saltenv = ["base"]
41
42    if extmod_whitelist is None:
43        extmod_whitelist = opts["extmod_whitelist"]
44    elif isinstance(extmod_whitelist, str):
45        extmod_whitelist = {form: extmod_whitelist.split(",")}
46    elif not isinstance(extmod_whitelist, dict):
47        log.error(
48            "extmod_whitelist must be a string or dictionary: %s", extmod_whitelist
49        )
50
51    if extmod_blacklist is None:
52        extmod_blacklist = opts["extmod_blacklist"]
53    elif isinstance(extmod_blacklist, str):
54        extmod_blacklist = {form: extmod_blacklist.split(",")}
55    elif not isinstance(extmod_blacklist, dict):
56        log.error(
57            "extmod_blacklist must be a string or dictionary: %s", extmod_blacklist
58        )
59
60    if isinstance(saltenv, str):
61        saltenv = saltenv.split(",")
62    ret = []
63    remote = set()
64    source = salt.utils.url.create("_" + form)
65    mod_dir = os.path.join(opts["extension_modules"], "{}".format(form))
66    touched = False
67    with salt.utils.files.set_umask(0o077):
68        try:
69            if not os.path.isdir(mod_dir):
70                log.info("Creating module dir '%s'", mod_dir)
71                try:
72                    os.makedirs(mod_dir)
73                except OSError:
74                    log.error(
75                        "Cannot create cache module directory %s. Check permissions.",
76                        mod_dir,
77                    )
78            fileclient = salt.fileclient.get_file_client(opts)
79            for sub_env in saltenv:
80                log.info("Syncing %s for environment '%s'", form, sub_env)
81                cache = []
82                log.info("Loading cache from %s, for %s", source, sub_env)
83                # Grab only the desired files (.py, .pyx, .so)
84                cache.extend(
85                    fileclient.cache_dir(
86                        source,
87                        sub_env,
88                        include_empty=False,
89                        include_pat=r"E@\.(pyx?|so|zip)$",
90                        exclude_pat=None,
91                    )
92                )
93                local_cache_dir = os.path.join(
94                    opts["cachedir"], "files", sub_env, "_{}".format(form)
95                )
96                log.debug("Local cache dir: '%s'", local_cache_dir)
97                for fn_ in cache:
98                    relpath = os.path.relpath(fn_, local_cache_dir)
99                    relname = os.path.splitext(relpath)[0].replace(os.sep, ".")
100                    if (
101                        extmod_whitelist
102                        and form in extmod_whitelist
103                        and relname not in extmod_whitelist[form]
104                    ):
105                        continue
106                    if (
107                        extmod_blacklist
108                        and form in extmod_blacklist
109                        and relname in extmod_blacklist[form]
110                    ):
111                        continue
112                    remote.add(relpath)
113                    dest = os.path.join(mod_dir, relpath)
114                    log.info("Copying '%s' to '%s'", fn_, dest)
115                    if os.path.isfile(dest):
116                        # The file is present, if the sum differs replace it
117                        hash_type = opts.get("hash_type", "md5")
118                        src_digest = salt.utils.hashutils.get_hash(fn_, hash_type)
119                        dst_digest = salt.utils.hashutils.get_hash(dest, hash_type)
120                        if src_digest != dst_digest:
121                            # The downloaded file differs, replace!
122                            shutil.copyfile(fn_, dest)
123                            ret.append("{}.{}".format(form, relname))
124                    else:
125                        dest_dir = os.path.dirname(dest)
126                        if not os.path.isdir(dest_dir):
127                            os.makedirs(dest_dir)
128                        shutil.copyfile(fn_, dest)
129                        ret.append("{}.{}".format(form, relname))
130
131            touched = bool(ret)
132            if opts["clean_dynamic_modules"] is True:
133                current = set(_listdir_recursively(mod_dir))
134                for fn_ in current - remote:
135                    full = os.path.join(mod_dir, fn_)
136                    if os.path.isfile(full):
137                        touched = True
138                        os.remove(full)
139                # Cleanup empty dirs
140                while True:
141                    emptydirs = _list_emptydirs(mod_dir)
142                    if not emptydirs:
143                        break
144                    for emptydir in emptydirs:
145                        touched = True
146                        shutil.rmtree(emptydir, ignore_errors=True)
147        except Exception as exc:  # pylint: disable=broad-except
148            log.error("Failed to sync %s module: %s", form, exc)
149    return ret, touched
150