1"""DVC config objects."""
2
3from __future__ import unicode_literals
4
5from dvc.utils.compat import str, open
6
7import os
8import errno
9import configobj
10from schema import Schema, Optional, And, Use, Regex
11
12import dvc.logger as logger
13from dvc.exceptions import DvcException
14
15
16class ConfigError(DvcException):
17    """DVC config exception.
18
19    Args:
20        msg (str): error message.
21        ex (Exception): optional exception that has caused this error.
22    """
23
24    def __init__(self, msg, ex=None):
25        super(ConfigError, self).__init__(
26            "config file error: {}".format(msg), ex
27        )
28
29
30def supported_cache_type(types):
31    """Checks if link type config option has a valid value.
32
33    Args:
34        types (list/string): type(s) of links that dvc should try out.
35    """
36    if isinstance(types, str):
37        types = [typ.strip() for typ in types.split(",")]
38    for typ in types:
39        if typ not in ["reflink", "hardlink", "symlink", "copy"]:
40            return False
41    return True
42
43
44def supported_loglevel(level):
45    """Checks if log level config option has a valid value.
46
47    Args:
48        level (str): log level name.
49    """
50    return level in ["info", "debug", "warning", "error"]
51
52
53def supported_cloud(cloud):
54    """Checks if obsoleted cloud option has a valid value.
55
56    Args:
57        cloud (str): cloud type name.
58    """
59    return cloud in ["aws", "gcp", "local", ""]
60
61
62def is_bool(val):
63    """Checks that value is a boolean.
64
65    Args:
66        val (str): string value verify.
67
68    Returns:
69        bool: True if value stands for boolean, False otherwise.
70    """
71    return val.lower() in ["true", "false"]
72
73
74def to_bool(val):
75    """Converts value to boolean.
76
77    Args:
78        val (str): string to convert to boolean.
79
80    Returns:
81        bool: True if value.lower() == 'true', False otherwise.
82    """
83    return val.lower() == "true"
84
85
86def is_whole(val):
87    """Checks that value is a whole integer.
88
89    Args:
90        val (str): number string to verify.
91
92    Returns:
93        bool: True if val is a whole number, False otherwise.
94    """
95    return int(val) >= 0
96
97
98def is_percent(val):
99    """Checks that value is a percent.
100
101    Args:
102        val (str): number string to verify.
103
104    Returns:
105        bool: True if 0<=value<=100, False otherwise.
106    """
107    return int(val) >= 0 and int(val) <= 100
108
109
110class Config(object):  # pylint: disable=too-many-instance-attributes
111    """Class that manages configuration files for a dvc repo.
112
113    Args:
114        dvc_dir (str): optional path to `.dvc` directory, that is used to
115            access repo-specific configs like .dvc/config and
116            .dvc/config.local.
117        validate (bool): optional flag to tell dvc if it should validate the
118            config or just load it as is. 'True' by default.
119
120
121    Raises:
122        ConfigError: thrown when config has an invalid format.
123    """
124
125    APPNAME = "dvc"
126    APPAUTHOR = "iterative"
127
128    # NOTE: used internally in RemoteLOCAL to know config
129    # location, that url should resolved relative to.
130    PRIVATE_CWD = "_cwd"
131
132    CONFIG = "config"
133    CONFIG_LOCAL = "config.local"
134
135    SECTION_CORE = "core"
136    SECTION_CORE_LOGLEVEL = "loglevel"
137    SECTION_CORE_LOGLEVEL_SCHEMA = And(Use(str.lower), supported_loglevel)
138    SECTION_CORE_REMOTE = "remote"
139    SECTION_CORE_INTERACTIVE_SCHEMA = And(str, is_bool, Use(to_bool))
140    SECTION_CORE_INTERACTIVE = "interactive"
141    SECTION_CORE_ANALYTICS = "analytics"
142    SECTION_CORE_ANALYTICS_SCHEMA = And(str, is_bool, Use(to_bool))
143
144    SECTION_CACHE = "cache"
145    SECTION_CACHE_DIR = "dir"
146    SECTION_CACHE_TYPE = "type"
147    SECTION_CACHE_TYPE_SCHEMA = supported_cache_type
148    SECTION_CACHE_PROTECTED = "protected"
149    SECTION_CACHE_LOCAL = "local"
150    SECTION_CACHE_S3 = "s3"
151    SECTION_CACHE_GS = "gs"
152    SECTION_CACHE_SSH = "ssh"
153    SECTION_CACHE_HDFS = "hdfs"
154    SECTION_CACHE_AZURE = "azure"
155    SECTION_CACHE_SCHEMA = {
156        Optional(SECTION_CACHE_LOCAL): str,
157        Optional(SECTION_CACHE_S3): str,
158        Optional(SECTION_CACHE_GS): str,
159        Optional(SECTION_CACHE_HDFS): str,
160        Optional(SECTION_CACHE_SSH): str,
161        Optional(SECTION_CACHE_AZURE): str,
162        Optional(SECTION_CACHE_DIR): str,
163        Optional(SECTION_CACHE_TYPE, default=None): SECTION_CACHE_TYPE_SCHEMA,
164        Optional(SECTION_CACHE_PROTECTED, default=False): And(
165            str, is_bool, Use(to_bool)
166        ),
167        Optional(PRIVATE_CWD): str,
168    }
169
170    # backward compatibility
171    SECTION_CORE_CLOUD = "cloud"
172    SECTION_CORE_CLOUD_SCHEMA = And(Use(str.lower), supported_cloud)
173    SECTION_CORE_STORAGEPATH = "storagepath"
174
175    SECTION_CORE_SCHEMA = {
176        Optional(SECTION_CORE_LOGLEVEL, default="info"): And(
177            str, Use(str.lower), SECTION_CORE_LOGLEVEL_SCHEMA
178        ),
179        Optional(SECTION_CORE_REMOTE, default=""): And(str, Use(str.lower)),
180        Optional(
181            SECTION_CORE_INTERACTIVE, default=False
182        ): SECTION_CORE_INTERACTIVE_SCHEMA,
183        Optional(
184            SECTION_CORE_ANALYTICS, default=True
185        ): SECTION_CORE_ANALYTICS_SCHEMA,
186        # backward compatibility
187        Optional(SECTION_CORE_CLOUD, default=""): SECTION_CORE_CLOUD_SCHEMA,
188        Optional(SECTION_CORE_STORAGEPATH, default=""): str,
189    }
190
191    # backward compatibility
192    SECTION_AWS = "aws"
193    SECTION_AWS_STORAGEPATH = "storagepath"
194    SECTION_AWS_CREDENTIALPATH = "credentialpath"
195    SECTION_AWS_ENDPOINT_URL = "endpointurl"
196    SECTION_AWS_REGION = "region"
197    SECTION_AWS_PROFILE = "profile"
198    SECTION_AWS_USE_SSL = "use_ssl"
199    SECTION_AWS_SCHEMA = {
200        SECTION_AWS_STORAGEPATH: str,
201        Optional(SECTION_AWS_REGION): str,
202        Optional(SECTION_AWS_PROFILE): str,
203        Optional(SECTION_AWS_CREDENTIALPATH): str,
204        Optional(SECTION_AWS_ENDPOINT_URL): str,
205        Optional(SECTION_AWS_USE_SSL, default=True): And(
206            str, is_bool, Use(to_bool)
207        ),
208    }
209
210    # backward compatibility
211    SECTION_GCP = "gcp"
212    SECTION_GCP_STORAGEPATH = SECTION_AWS_STORAGEPATH
213    SECTION_GCP_CREDENTIALPATH = SECTION_AWS_CREDENTIALPATH
214    SECTION_GCP_PROJECTNAME = "projectname"
215    SECTION_GCP_SCHEMA = {
216        SECTION_GCP_STORAGEPATH: str,
217        Optional(SECTION_GCP_PROJECTNAME): str,
218    }
219
220    # backward compatibility
221    SECTION_LOCAL = "local"
222    SECTION_LOCAL_STORAGEPATH = SECTION_AWS_STORAGEPATH
223    SECTION_LOCAL_SCHEMA = {SECTION_LOCAL_STORAGEPATH: str}
224
225    SECTION_AZURE_CONNECTION_STRING = "connection_string"
226
227    SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P<name>.*)"\s*$'
228    SECTION_REMOTE_FMT = 'remote "{}"'
229    SECTION_REMOTE_URL = "url"
230    SECTION_REMOTE_USER = "user"
231    SECTION_REMOTE_PORT = "port"
232    SECTION_REMOTE_KEY_FILE = "keyfile"
233    SECTION_REMOTE_TIMEOUT = "timeout"
234    SECTION_REMOTE_PASSWORD = "password"
235    SECTION_REMOTE_ASK_PASSWORD = "ask_password"
236    SECTION_REMOTE_SCHEMA = {
237        SECTION_REMOTE_URL: str,
238        Optional(SECTION_AWS_REGION): str,
239        Optional(SECTION_AWS_PROFILE): str,
240        Optional(SECTION_AWS_CREDENTIALPATH): str,
241        Optional(SECTION_AWS_ENDPOINT_URL): str,
242        Optional(SECTION_AWS_USE_SSL, default=True): And(
243            str, is_bool, Use(to_bool)
244        ),
245        Optional(SECTION_GCP_PROJECTNAME): str,
246        Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA,
247        Optional(SECTION_CACHE_PROTECTED, default=False): And(
248            str, is_bool, Use(to_bool)
249        ),
250        Optional(SECTION_REMOTE_USER): str,
251        Optional(SECTION_REMOTE_PORT): Use(int),
252        Optional(SECTION_REMOTE_KEY_FILE): str,
253        Optional(SECTION_REMOTE_TIMEOUT): Use(int),
254        Optional(SECTION_REMOTE_PASSWORD): str,
255        Optional(SECTION_REMOTE_ASK_PASSWORD): And(str, is_bool, Use(to_bool)),
256        Optional(SECTION_AZURE_CONNECTION_STRING): str,
257        Optional(PRIVATE_CWD): str,
258    }
259
260    SECTION_STATE = "state"
261    SECTION_STATE_ROW_LIMIT = "row_limit"
262    SECTION_STATE_ROW_CLEANUP_QUOTA = "row_cleanup_quota"
263    SECTION_STATE_SCHEMA = {
264        Optional(SECTION_STATE_ROW_LIMIT): And(Use(int), is_whole),
265        Optional(SECTION_STATE_ROW_CLEANUP_QUOTA): And(Use(int), is_percent),
266    }
267
268    SCHEMA = {
269        Optional(SECTION_CORE, default={}): SECTION_CORE_SCHEMA,
270        Optional(Regex(SECTION_REMOTE_REGEX)): SECTION_REMOTE_SCHEMA,
271        Optional(SECTION_CACHE, default={}): SECTION_CACHE_SCHEMA,
272        Optional(SECTION_STATE, default={}): SECTION_STATE_SCHEMA,
273        # backward compatibility
274        Optional(SECTION_AWS, default={}): SECTION_AWS_SCHEMA,
275        Optional(SECTION_GCP, default={}): SECTION_GCP_SCHEMA,
276        Optional(SECTION_LOCAL, default={}): SECTION_LOCAL_SCHEMA,
277    }
278
279    def __init__(self, dvc_dir=None, validate=True):
280        self.system_config_file = os.path.join(
281            self.get_system_config_dir(), self.CONFIG
282        )
283        self.global_config_file = os.path.join(
284            self.get_global_config_dir(), self.CONFIG
285        )
286
287        if dvc_dir is not None:
288            self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
289            self.config_file = os.path.join(dvc_dir, self.CONFIG)
290            self.config_local_file = os.path.join(dvc_dir, self.CONFIG_LOCAL)
291        else:
292            self.dvc_dir = None
293            self.config_file = None
294            self.config_local_file = None
295
296        self._system_config = None
297        self._global_config = None
298        self._repo_config = None
299        self._local_config = None
300
301        self.config = None
302
303        self.load(validate=validate)
304
305    @staticmethod
306    def get_global_config_dir():
307        """Returns global config location. E.g. ~/.config/dvc/config.
308
309        Returns:
310            str: path to the global config directory.
311        """
312        from appdirs import user_config_dir
313
314        return user_config_dir(
315            appname=Config.APPNAME, appauthor=Config.APPAUTHOR
316        )
317
318    @staticmethod
319    def get_system_config_dir():
320        """Returns system config location. E.g. /etc/dvc.conf.
321
322        Returns:
323            str: path to the system config directory.
324        """
325        from appdirs import site_config_dir
326
327        return site_config_dir(
328            appname=Config.APPNAME, appauthor=Config.APPAUTHOR
329        )
330
331    @staticmethod
332    def init(dvc_dir):
333        """Initializes dvc config.
334
335        Args:
336            dvc_dir (str): path to .dvc directory.
337
338        Returns:
339            dvc.config.Config: config object.
340        """
341        config_file = os.path.join(dvc_dir, Config.CONFIG)
342        open(config_file, "w+").close()
343        return Config(dvc_dir)
344
345    def _load(self):
346        self._system_config = configobj.ConfigObj(self.system_config_file)
347        self._global_config = configobj.ConfigObj(self.global_config_file)
348
349        if self.config_file is not None:
350            self._repo_config = configobj.ConfigObj(self.config_file)
351        else:
352            self._repo_config = configobj.ConfigObj()
353
354        if self.config_local_file is not None:
355            self._local_config = configobj.ConfigObj(self.config_local_file)
356        else:
357            self._local_config = configobj.ConfigObj()
358
359        self.config = None
360
361    def _load_config(self, path):
362        config = configobj.ConfigObj(path)
363        config = self._lower(config)
364        self._resolve_paths(config, path)
365        return config
366
367    @staticmethod
368    def _resolve_path(path, config_file):
369        assert os.path.isabs(config_file)
370        config_dir = os.path.dirname(config_file)
371        return os.path.abspath(os.path.join(config_dir, path))
372
373    def _resolve_cache_path(self, config, fname):
374        cache = config.get(self.SECTION_CACHE)
375        if cache is None:
376            return
377
378        cache_dir = cache.get(self.SECTION_CACHE_DIR)
379        if cache_dir is None:
380            return
381
382        cache[self.PRIVATE_CWD] = os.path.dirname(fname)
383
384    def _resolve_paths(self, config, fname):
385        if fname is None:
386            return
387
388        self._resolve_cache_path(config, fname)
389        for section in config.values():
390            if self.SECTION_REMOTE_URL not in section.keys():
391                continue
392
393            section[self.PRIVATE_CWD] = os.path.dirname(fname)
394
395    def load(self, validate=True):
396        """Loads config from all the config files.
397
398        Args:
399            validate (bool): optional flag to tell dvc if it should validate
400                the config or just load it as is. 'True' by default.
401
402
403        Raises:
404            dvc.config.ConfigError: thrown if config has invalid format.
405        """
406        self._load()
407        try:
408            self.config = self._load_config(self.system_config_file)
409            user = self._load_config(self.global_config_file)
410            config = self._load_config(self.config_file)
411            local = self._load_config(self.config_local_file)
412
413            # NOTE: schema doesn't support ConfigObj.Section validation, so we
414            # need to convert our config to dict before passing it to
415            for conf in [user, config, local]:
416                self.config = self._merge(self.config, conf)
417
418            if validate:
419                self.config = Schema(self.SCHEMA).validate(self.config)
420
421            # NOTE: now converting back to ConfigObj
422            self.config = configobj.ConfigObj(
423                self.config, write_empty_values=True
424            )
425            self.config.filename = self.config_file
426            self._resolve_paths(self.config, self.config_file)
427        except Exception as ex:
428            raise ConfigError(ex)
429
430    @staticmethod
431    def _get_key(conf, name, add=False):
432        for k in conf.keys():
433            if k.lower() == name.lower():
434                return k
435
436        if add:
437            conf[name] = {}
438            return name
439
440        return None
441
442    def save(self, config=None):
443        """Saves config to config files.
444
445        Args:
446            config (configobj.ConfigObj): optional config object to save.
447
448        Raises:
449            dvc.config.ConfigError: thrown if failed to write config file.
450        """
451        if config is not None:
452            clist = [config]
453        else:
454            clist = [
455                self._system_config,
456                self._global_config,
457                self._repo_config,
458                self._local_config,
459            ]
460
461        for conf in clist:
462            if conf.filename is None:
463                continue
464
465            try:
466                logger.debug("Writing '{}'.".format(conf.filename))
467                dname = os.path.dirname(os.path.abspath(conf.filename))
468                try:
469                    os.makedirs(dname)
470                except OSError as exc:
471                    if exc.errno != errno.EEXIST:
472                        raise
473                conf.write()
474            except Exception as exc:
475                msg = "failed to write config '{}'".format(conf.filename)
476                raise ConfigError(msg, exc)
477
478    @staticmethod
479    def unset(config, section, opt=None):
480        """Unsets specified option and/or section in the config.
481
482        Args:
483            config (configobj.ConfigObj): config to work on.
484            section (str): section name.
485            opt (str): optional option name.
486        """
487        if section not in config.keys():
488            raise ConfigError("section '{}' doesn't exist".format(section))
489
490        if opt is None:
491            del config[section]
492            return
493
494        if opt not in config[section].keys():
495            raise ConfigError(
496                "option '{}.{}' doesn't exist".format(section, opt)
497            )
498        del config[section][opt]
499
500        if not config[section]:
501            del config[section]
502
503    @staticmethod
504    def set(config, section, opt, value):
505        """Sets specified option in the config.
506
507        Args:
508            config (configobj.ConfigObj): config to work on.
509            section (str): section name.
510            opt (str): option name.
511            value: value to set option to.
512        """
513        if section not in config.keys():
514            config[section] = {}
515
516        config[section][opt] = value
517
518    @staticmethod
519    def show(config, section, opt):
520        """Prints option value from the config.
521
522        Args:
523            config (configobj.ConfigObj): config to work on.
524            section (str): section name.
525            opt (str): option name.
526        """
527        if section not in config.keys():
528            raise ConfigError("section '{}' doesn't exist".format(section))
529
530        if opt not in config[section].keys():
531            raise ConfigError(
532                "option '{}.{}' doesn't exist".format(section, opt)
533            )
534
535        logger.info(config[section][opt])
536
537    @staticmethod
538    def _merge(first, second):
539        res = {}
540        sections = list(first.keys()) + list(second.keys())
541        for section in sections:
542            first_copy = first.get(section, {}).copy()
543            second_copy = second.get(section, {}).copy()
544            first_copy.update(second_copy)
545            res[section] = first_copy
546        return res
547
548    @staticmethod
549    def _lower(config):
550        new_config = {}
551        for s_key, s_value in config.items():
552            new_s = {}
553            for key, value in s_value.items():
554                new_s[key.lower()] = str(value)
555            new_config[s_key.lower()] = new_s
556        return new_config
557