1"""DVC config objects.""" 2 3from __future__ import unicode_literals 4 5from dvc.utils.compat import str, open 6 7import os 8import errno 9import configobj 10from schema import Schema, Optional, And, Use, Regex 11 12import dvc.logger as logger 13from dvc.exceptions import DvcException 14 15 16class ConfigError(DvcException): 17 """DVC config exception. 18 19 Args: 20 msg (str): error message. 21 ex (Exception): optional exception that has caused this error. 22 """ 23 24 def __init__(self, msg, ex=None): 25 super(ConfigError, self).__init__( 26 "config file error: {}".format(msg), ex 27 ) 28 29 30def supported_cache_type(types): 31 """Checks if link type config option has a valid value. 32 33 Args: 34 types (list/string): type(s) of links that dvc should try out. 35 """ 36 if isinstance(types, str): 37 types = [typ.strip() for typ in types.split(",")] 38 for typ in types: 39 if typ not in ["reflink", "hardlink", "symlink", "copy"]: 40 return False 41 return True 42 43 44def supported_loglevel(level): 45 """Checks if log level config option has a valid value. 46 47 Args: 48 level (str): log level name. 49 """ 50 return level in ["info", "debug", "warning", "error"] 51 52 53def supported_cloud(cloud): 54 """Checks if obsoleted cloud option has a valid value. 55 56 Args: 57 cloud (str): cloud type name. 58 """ 59 return cloud in ["aws", "gcp", "local", ""] 60 61 62def is_bool(val): 63 """Checks that value is a boolean. 64 65 Args: 66 val (str): string value verify. 67 68 Returns: 69 bool: True if value stands for boolean, False otherwise. 70 """ 71 return val.lower() in ["true", "false"] 72 73 74def to_bool(val): 75 """Converts value to boolean. 76 77 Args: 78 val (str): string to convert to boolean. 79 80 Returns: 81 bool: True if value.lower() == 'true', False otherwise. 82 """ 83 return val.lower() == "true" 84 85 86def is_whole(val): 87 """Checks that value is a whole integer. 88 89 Args: 90 val (str): number string to verify. 91 92 Returns: 93 bool: True if val is a whole number, False otherwise. 94 """ 95 return int(val) >= 0 96 97 98def is_percent(val): 99 """Checks that value is a percent. 100 101 Args: 102 val (str): number string to verify. 103 104 Returns: 105 bool: True if 0<=value<=100, False otherwise. 106 """ 107 return int(val) >= 0 and int(val) <= 100 108 109 110class Config(object): # pylint: disable=too-many-instance-attributes 111 """Class that manages configuration files for a dvc repo. 112 113 Args: 114 dvc_dir (str): optional path to `.dvc` directory, that is used to 115 access repo-specific configs like .dvc/config and 116 .dvc/config.local. 117 validate (bool): optional flag to tell dvc if it should validate the 118 config or just load it as is. 'True' by default. 119 120 121 Raises: 122 ConfigError: thrown when config has an invalid format. 123 """ 124 125 APPNAME = "dvc" 126 APPAUTHOR = "iterative" 127 128 # NOTE: used internally in RemoteLOCAL to know config 129 # location, that url should resolved relative to. 130 PRIVATE_CWD = "_cwd" 131 132 CONFIG = "config" 133 CONFIG_LOCAL = "config.local" 134 135 SECTION_CORE = "core" 136 SECTION_CORE_LOGLEVEL = "loglevel" 137 SECTION_CORE_LOGLEVEL_SCHEMA = And(Use(str.lower), supported_loglevel) 138 SECTION_CORE_REMOTE = "remote" 139 SECTION_CORE_INTERACTIVE_SCHEMA = And(str, is_bool, Use(to_bool)) 140 SECTION_CORE_INTERACTIVE = "interactive" 141 SECTION_CORE_ANALYTICS = "analytics" 142 SECTION_CORE_ANALYTICS_SCHEMA = And(str, is_bool, Use(to_bool)) 143 144 SECTION_CACHE = "cache" 145 SECTION_CACHE_DIR = "dir" 146 SECTION_CACHE_TYPE = "type" 147 SECTION_CACHE_TYPE_SCHEMA = supported_cache_type 148 SECTION_CACHE_PROTECTED = "protected" 149 SECTION_CACHE_LOCAL = "local" 150 SECTION_CACHE_S3 = "s3" 151 SECTION_CACHE_GS = "gs" 152 SECTION_CACHE_SSH = "ssh" 153 SECTION_CACHE_HDFS = "hdfs" 154 SECTION_CACHE_AZURE = "azure" 155 SECTION_CACHE_SCHEMA = { 156 Optional(SECTION_CACHE_LOCAL): str, 157 Optional(SECTION_CACHE_S3): str, 158 Optional(SECTION_CACHE_GS): str, 159 Optional(SECTION_CACHE_HDFS): str, 160 Optional(SECTION_CACHE_SSH): str, 161 Optional(SECTION_CACHE_AZURE): str, 162 Optional(SECTION_CACHE_DIR): str, 163 Optional(SECTION_CACHE_TYPE, default=None): SECTION_CACHE_TYPE_SCHEMA, 164 Optional(SECTION_CACHE_PROTECTED, default=False): And( 165 str, is_bool, Use(to_bool) 166 ), 167 Optional(PRIVATE_CWD): str, 168 } 169 170 # backward compatibility 171 SECTION_CORE_CLOUD = "cloud" 172 SECTION_CORE_CLOUD_SCHEMA = And(Use(str.lower), supported_cloud) 173 SECTION_CORE_STORAGEPATH = "storagepath" 174 175 SECTION_CORE_SCHEMA = { 176 Optional(SECTION_CORE_LOGLEVEL, default="info"): And( 177 str, Use(str.lower), SECTION_CORE_LOGLEVEL_SCHEMA 178 ), 179 Optional(SECTION_CORE_REMOTE, default=""): And(str, Use(str.lower)), 180 Optional( 181 SECTION_CORE_INTERACTIVE, default=False 182 ): SECTION_CORE_INTERACTIVE_SCHEMA, 183 Optional( 184 SECTION_CORE_ANALYTICS, default=True 185 ): SECTION_CORE_ANALYTICS_SCHEMA, 186 # backward compatibility 187 Optional(SECTION_CORE_CLOUD, default=""): SECTION_CORE_CLOUD_SCHEMA, 188 Optional(SECTION_CORE_STORAGEPATH, default=""): str, 189 } 190 191 # backward compatibility 192 SECTION_AWS = "aws" 193 SECTION_AWS_STORAGEPATH = "storagepath" 194 SECTION_AWS_CREDENTIALPATH = "credentialpath" 195 SECTION_AWS_ENDPOINT_URL = "endpointurl" 196 SECTION_AWS_REGION = "region" 197 SECTION_AWS_PROFILE = "profile" 198 SECTION_AWS_USE_SSL = "use_ssl" 199 SECTION_AWS_SCHEMA = { 200 SECTION_AWS_STORAGEPATH: str, 201 Optional(SECTION_AWS_REGION): str, 202 Optional(SECTION_AWS_PROFILE): str, 203 Optional(SECTION_AWS_CREDENTIALPATH): str, 204 Optional(SECTION_AWS_ENDPOINT_URL): str, 205 Optional(SECTION_AWS_USE_SSL, default=True): And( 206 str, is_bool, Use(to_bool) 207 ), 208 } 209 210 # backward compatibility 211 SECTION_GCP = "gcp" 212 SECTION_GCP_STORAGEPATH = SECTION_AWS_STORAGEPATH 213 SECTION_GCP_CREDENTIALPATH = SECTION_AWS_CREDENTIALPATH 214 SECTION_GCP_PROJECTNAME = "projectname" 215 SECTION_GCP_SCHEMA = { 216 SECTION_GCP_STORAGEPATH: str, 217 Optional(SECTION_GCP_PROJECTNAME): str, 218 } 219 220 # backward compatibility 221 SECTION_LOCAL = "local" 222 SECTION_LOCAL_STORAGEPATH = SECTION_AWS_STORAGEPATH 223 SECTION_LOCAL_SCHEMA = {SECTION_LOCAL_STORAGEPATH: str} 224 225 SECTION_AZURE_CONNECTION_STRING = "connection_string" 226 227 SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P<name>.*)"\s*$' 228 SECTION_REMOTE_FMT = 'remote "{}"' 229 SECTION_REMOTE_URL = "url" 230 SECTION_REMOTE_USER = "user" 231 SECTION_REMOTE_PORT = "port" 232 SECTION_REMOTE_KEY_FILE = "keyfile" 233 SECTION_REMOTE_TIMEOUT = "timeout" 234 SECTION_REMOTE_PASSWORD = "password" 235 SECTION_REMOTE_ASK_PASSWORD = "ask_password" 236 SECTION_REMOTE_SCHEMA = { 237 SECTION_REMOTE_URL: str, 238 Optional(SECTION_AWS_REGION): str, 239 Optional(SECTION_AWS_PROFILE): str, 240 Optional(SECTION_AWS_CREDENTIALPATH): str, 241 Optional(SECTION_AWS_ENDPOINT_URL): str, 242 Optional(SECTION_AWS_USE_SSL, default=True): And( 243 str, is_bool, Use(to_bool) 244 ), 245 Optional(SECTION_GCP_PROJECTNAME): str, 246 Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA, 247 Optional(SECTION_CACHE_PROTECTED, default=False): And( 248 str, is_bool, Use(to_bool) 249 ), 250 Optional(SECTION_REMOTE_USER): str, 251 Optional(SECTION_REMOTE_PORT): Use(int), 252 Optional(SECTION_REMOTE_KEY_FILE): str, 253 Optional(SECTION_REMOTE_TIMEOUT): Use(int), 254 Optional(SECTION_REMOTE_PASSWORD): str, 255 Optional(SECTION_REMOTE_ASK_PASSWORD): And(str, is_bool, Use(to_bool)), 256 Optional(SECTION_AZURE_CONNECTION_STRING): str, 257 Optional(PRIVATE_CWD): str, 258 } 259 260 SECTION_STATE = "state" 261 SECTION_STATE_ROW_LIMIT = "row_limit" 262 SECTION_STATE_ROW_CLEANUP_QUOTA = "row_cleanup_quota" 263 SECTION_STATE_SCHEMA = { 264 Optional(SECTION_STATE_ROW_LIMIT): And(Use(int), is_whole), 265 Optional(SECTION_STATE_ROW_CLEANUP_QUOTA): And(Use(int), is_percent), 266 } 267 268 SCHEMA = { 269 Optional(SECTION_CORE, default={}): SECTION_CORE_SCHEMA, 270 Optional(Regex(SECTION_REMOTE_REGEX)): SECTION_REMOTE_SCHEMA, 271 Optional(SECTION_CACHE, default={}): SECTION_CACHE_SCHEMA, 272 Optional(SECTION_STATE, default={}): SECTION_STATE_SCHEMA, 273 # backward compatibility 274 Optional(SECTION_AWS, default={}): SECTION_AWS_SCHEMA, 275 Optional(SECTION_GCP, default={}): SECTION_GCP_SCHEMA, 276 Optional(SECTION_LOCAL, default={}): SECTION_LOCAL_SCHEMA, 277 } 278 279 def __init__(self, dvc_dir=None, validate=True): 280 self.system_config_file = os.path.join( 281 self.get_system_config_dir(), self.CONFIG 282 ) 283 self.global_config_file = os.path.join( 284 self.get_global_config_dir(), self.CONFIG 285 ) 286 287 if dvc_dir is not None: 288 self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir)) 289 self.config_file = os.path.join(dvc_dir, self.CONFIG) 290 self.config_local_file = os.path.join(dvc_dir, self.CONFIG_LOCAL) 291 else: 292 self.dvc_dir = None 293 self.config_file = None 294 self.config_local_file = None 295 296 self._system_config = None 297 self._global_config = None 298 self._repo_config = None 299 self._local_config = None 300 301 self.config = None 302 303 self.load(validate=validate) 304 305 @staticmethod 306 def get_global_config_dir(): 307 """Returns global config location. E.g. ~/.config/dvc/config. 308 309 Returns: 310 str: path to the global config directory. 311 """ 312 from appdirs import user_config_dir 313 314 return user_config_dir( 315 appname=Config.APPNAME, appauthor=Config.APPAUTHOR 316 ) 317 318 @staticmethod 319 def get_system_config_dir(): 320 """Returns system config location. E.g. /etc/dvc.conf. 321 322 Returns: 323 str: path to the system config directory. 324 """ 325 from appdirs import site_config_dir 326 327 return site_config_dir( 328 appname=Config.APPNAME, appauthor=Config.APPAUTHOR 329 ) 330 331 @staticmethod 332 def init(dvc_dir): 333 """Initializes dvc config. 334 335 Args: 336 dvc_dir (str): path to .dvc directory. 337 338 Returns: 339 dvc.config.Config: config object. 340 """ 341 config_file = os.path.join(dvc_dir, Config.CONFIG) 342 open(config_file, "w+").close() 343 return Config(dvc_dir) 344 345 def _load(self): 346 self._system_config = configobj.ConfigObj(self.system_config_file) 347 self._global_config = configobj.ConfigObj(self.global_config_file) 348 349 if self.config_file is not None: 350 self._repo_config = configobj.ConfigObj(self.config_file) 351 else: 352 self._repo_config = configobj.ConfigObj() 353 354 if self.config_local_file is not None: 355 self._local_config = configobj.ConfigObj(self.config_local_file) 356 else: 357 self._local_config = configobj.ConfigObj() 358 359 self.config = None 360 361 def _load_config(self, path): 362 config = configobj.ConfigObj(path) 363 config = self._lower(config) 364 self._resolve_paths(config, path) 365 return config 366 367 @staticmethod 368 def _resolve_path(path, config_file): 369 assert os.path.isabs(config_file) 370 config_dir = os.path.dirname(config_file) 371 return os.path.abspath(os.path.join(config_dir, path)) 372 373 def _resolve_cache_path(self, config, fname): 374 cache = config.get(self.SECTION_CACHE) 375 if cache is None: 376 return 377 378 cache_dir = cache.get(self.SECTION_CACHE_DIR) 379 if cache_dir is None: 380 return 381 382 cache[self.PRIVATE_CWD] = os.path.dirname(fname) 383 384 def _resolve_paths(self, config, fname): 385 if fname is None: 386 return 387 388 self._resolve_cache_path(config, fname) 389 for section in config.values(): 390 if self.SECTION_REMOTE_URL not in section.keys(): 391 continue 392 393 section[self.PRIVATE_CWD] = os.path.dirname(fname) 394 395 def load(self, validate=True): 396 """Loads config from all the config files. 397 398 Args: 399 validate (bool): optional flag to tell dvc if it should validate 400 the config or just load it as is. 'True' by default. 401 402 403 Raises: 404 dvc.config.ConfigError: thrown if config has invalid format. 405 """ 406 self._load() 407 try: 408 self.config = self._load_config(self.system_config_file) 409 user = self._load_config(self.global_config_file) 410 config = self._load_config(self.config_file) 411 local = self._load_config(self.config_local_file) 412 413 # NOTE: schema doesn't support ConfigObj.Section validation, so we 414 # need to convert our config to dict before passing it to 415 for conf in [user, config, local]: 416 self.config = self._merge(self.config, conf) 417 418 if validate: 419 self.config = Schema(self.SCHEMA).validate(self.config) 420 421 # NOTE: now converting back to ConfigObj 422 self.config = configobj.ConfigObj( 423 self.config, write_empty_values=True 424 ) 425 self.config.filename = self.config_file 426 self._resolve_paths(self.config, self.config_file) 427 except Exception as ex: 428 raise ConfigError(ex) 429 430 @staticmethod 431 def _get_key(conf, name, add=False): 432 for k in conf.keys(): 433 if k.lower() == name.lower(): 434 return k 435 436 if add: 437 conf[name] = {} 438 return name 439 440 return None 441 442 def save(self, config=None): 443 """Saves config to config files. 444 445 Args: 446 config (configobj.ConfigObj): optional config object to save. 447 448 Raises: 449 dvc.config.ConfigError: thrown if failed to write config file. 450 """ 451 if config is not None: 452 clist = [config] 453 else: 454 clist = [ 455 self._system_config, 456 self._global_config, 457 self._repo_config, 458 self._local_config, 459 ] 460 461 for conf in clist: 462 if conf.filename is None: 463 continue 464 465 try: 466 logger.debug("Writing '{}'.".format(conf.filename)) 467 dname = os.path.dirname(os.path.abspath(conf.filename)) 468 try: 469 os.makedirs(dname) 470 except OSError as exc: 471 if exc.errno != errno.EEXIST: 472 raise 473 conf.write() 474 except Exception as exc: 475 msg = "failed to write config '{}'".format(conf.filename) 476 raise ConfigError(msg, exc) 477 478 @staticmethod 479 def unset(config, section, opt=None): 480 """Unsets specified option and/or section in the config. 481 482 Args: 483 config (configobj.ConfigObj): config to work on. 484 section (str): section name. 485 opt (str): optional option name. 486 """ 487 if section not in config.keys(): 488 raise ConfigError("section '{}' doesn't exist".format(section)) 489 490 if opt is None: 491 del config[section] 492 return 493 494 if opt not in config[section].keys(): 495 raise ConfigError( 496 "option '{}.{}' doesn't exist".format(section, opt) 497 ) 498 del config[section][opt] 499 500 if not config[section]: 501 del config[section] 502 503 @staticmethod 504 def set(config, section, opt, value): 505 """Sets specified option in the config. 506 507 Args: 508 config (configobj.ConfigObj): config to work on. 509 section (str): section name. 510 opt (str): option name. 511 value: value to set option to. 512 """ 513 if section not in config.keys(): 514 config[section] = {} 515 516 config[section][opt] = value 517 518 @staticmethod 519 def show(config, section, opt): 520 """Prints option value from the config. 521 522 Args: 523 config (configobj.ConfigObj): config to work on. 524 section (str): section name. 525 opt (str): option name. 526 """ 527 if section not in config.keys(): 528 raise ConfigError("section '{}' doesn't exist".format(section)) 529 530 if opt not in config[section].keys(): 531 raise ConfigError( 532 "option '{}.{}' doesn't exist".format(section, opt) 533 ) 534 535 logger.info(config[section][opt]) 536 537 @staticmethod 538 def _merge(first, second): 539 res = {} 540 sections = list(first.keys()) + list(second.keys()) 541 for section in sections: 542 first_copy = first.get(section, {}).copy() 543 second_copy = second.get(section, {}).copy() 544 first_copy.update(second_copy) 545 res[section] = first_copy 546 return res 547 548 @staticmethod 549 def _lower(config): 550 new_config = {} 551 for s_key, s_value in config.items(): 552 new_s = {} 553 for key, value in s_value.items(): 554 new_s[key.lower()] = str(value) 555 new_config[s_key.lower()] = new_s 556 return new_config 557