1"""Configuration management setup 2 3Some terminology: 4- name 5 As written in config files. 6- value 7 Value associated with a name 8- key 9 Name combined with it's section (section.name) 10- variant 11 A single word describing where the configuration key-value pair came from 12""" 13 14import locale 15import logging 16import os 17import sys 18 19from pip._vendor.six.moves import configparser 20 21from pip._internal.exceptions import ( 22 ConfigurationError, ConfigurationFileCouldNotBeLoaded, 23) 24from pip._internal.utils import appdirs 25from pip._internal.utils.compat import WINDOWS, expanduser 26from pip._internal.utils.misc import ensure_dir, enum 27from pip._internal.utils.typing import MYPY_CHECK_RUNNING 28 29if MYPY_CHECK_RUNNING: 30 from typing import ( 31 Any, Dict, Iterable, List, NewType, Optional, Tuple 32 ) 33 34 RawConfigParser = configparser.RawConfigParser # Shorthand 35 Kind = NewType("Kind", str) 36 37logger = logging.getLogger(__name__) 38 39 40# NOTE: Maybe use the optionx attribute to normalize keynames. 41def _normalize_name(name): 42 # type: (str) -> str 43 """Make a name consistent regardless of source (environment or file) 44 """ 45 name = name.lower().replace('_', '-') 46 if name.startswith('--'): 47 name = name[2:] # only prefer long opts 48 return name 49 50 51def _disassemble_key(name): 52 # type: (str) -> List[str] 53 if "." not in name: 54 error_message = ( 55 "Key does not contain dot separated section and key. " 56 "Perhaps you wanted to use 'global.{}' instead?" 57 ).format(name) 58 raise ConfigurationError(error_message) 59 return name.split(".", 1) 60 61 62# The kinds of configurations there are. 63kinds = enum( 64 USER="user", # User Specific 65 GLOBAL="global", # System Wide 66 SITE="site", # [Virtual] Environment Specific 67 ENV="env", # from PIP_CONFIG_FILE 68 ENV_VAR="env-var", # from Environment Variables 69) 70 71 72CONFIG_BASENAME = 'pip.ini' if WINDOWS else 'pip.conf' 73 74 75def get_configuration_files(): 76 global_config_files = [ 77 os.path.join(path, CONFIG_BASENAME) 78 for path in appdirs.site_config_dirs('pip') 79 ] 80 81 site_config_file = os.path.join(sys.prefix, CONFIG_BASENAME) 82 legacy_config_file = os.path.join( 83 expanduser('~'), 84 'pip' if WINDOWS else '.pip', 85 CONFIG_BASENAME, 86 ) 87 new_config_file = os.path.join( 88 appdirs.user_config_dir("pip"), CONFIG_BASENAME 89 ) 90 return { 91 kinds.GLOBAL: global_config_files, 92 kinds.SITE: [site_config_file], 93 kinds.USER: [legacy_config_file, new_config_file], 94 } 95 96 97class Configuration(object): 98 """Handles management of configuration. 99 100 Provides an interface to accessing and managing configuration files. 101 102 This class converts provides an API that takes "section.key-name" style 103 keys and stores the value associated with it as "key-name" under the 104 section "section". 105 106 This allows for a clean interface wherein the both the section and the 107 key-name are preserved in an easy to manage form in the configuration files 108 and the data stored is also nice. 109 """ 110 111 def __init__(self, isolated, load_only=None): 112 # type: (bool, Kind) -> None 113 super(Configuration, self).__init__() 114 115 _valid_load_only = [kinds.USER, kinds.GLOBAL, kinds.SITE, None] 116 if load_only not in _valid_load_only: 117 raise ConfigurationError( 118 "Got invalid value for load_only - should be one of {}".format( 119 ", ".join(map(repr, _valid_load_only[:-1])) 120 ) 121 ) 122 self.isolated = isolated # type: bool 123 self.load_only = load_only # type: Optional[Kind] 124 125 # The order here determines the override order. 126 self._override_order = [ 127 kinds.GLOBAL, kinds.USER, kinds.SITE, kinds.ENV, kinds.ENV_VAR 128 ] 129 130 self._ignore_env_names = ["version", "help"] 131 132 # Because we keep track of where we got the data from 133 self._parsers = { 134 variant: [] for variant in self._override_order 135 } # type: Dict[Kind, List[Tuple[str, RawConfigParser]]] 136 self._config = { 137 variant: {} for variant in self._override_order 138 } # type: Dict[Kind, Dict[str, Any]] 139 self._modified_parsers = [] # type: List[Tuple[str, RawConfigParser]] 140 141 def load(self): 142 # type: () -> None 143 """Loads configuration from configuration files and environment 144 """ 145 self._load_config_files() 146 if not self.isolated: 147 self._load_environment_vars() 148 149 def get_file_to_edit(self): 150 # type: () -> Optional[str] 151 """Returns the file with highest priority in configuration 152 """ 153 assert self.load_only is not None, \ 154 "Need to be specified a file to be editing" 155 156 try: 157 return self._get_parser_to_modify()[0] 158 except IndexError: 159 return None 160 161 def items(self): 162 # type: () -> Iterable[Tuple[str, Any]] 163 """Returns key-value pairs like dict.items() representing the loaded 164 configuration 165 """ 166 return self._dictionary.items() 167 168 def get_value(self, key): 169 # type: (str) -> Any 170 """Get a value from the configuration. 171 """ 172 try: 173 return self._dictionary[key] 174 except KeyError: 175 raise ConfigurationError("No such key - {}".format(key)) 176 177 def set_value(self, key, value): 178 # type: (str, Any) -> None 179 """Modify a value in the configuration. 180 """ 181 self._ensure_have_load_only() 182 183 fname, parser = self._get_parser_to_modify() 184 185 if parser is not None: 186 section, name = _disassemble_key(key) 187 188 # Modify the parser and the configuration 189 if not parser.has_section(section): 190 parser.add_section(section) 191 parser.set(section, name, value) 192 193 self._config[self.load_only][key] = value 194 self._mark_as_modified(fname, parser) 195 196 def unset_value(self, key): 197 # type: (str) -> None 198 """Unset a value in the configuration. 199 """ 200 self._ensure_have_load_only() 201 202 if key not in self._config[self.load_only]: 203 raise ConfigurationError("No such key - {}".format(key)) 204 205 fname, parser = self._get_parser_to_modify() 206 207 if parser is not None: 208 section, name = _disassemble_key(key) 209 210 # Remove the key in the parser 211 modified_something = False 212 if parser.has_section(section): 213 # Returns whether the option was removed or not 214 modified_something = parser.remove_option(section, name) 215 216 if modified_something: 217 # name removed from parser, section may now be empty 218 section_iter = iter(parser.items(section)) 219 try: 220 val = next(section_iter) 221 except StopIteration: 222 val = None 223 224 if val is None: 225 parser.remove_section(section) 226 227 self._mark_as_modified(fname, parser) 228 else: 229 raise ConfigurationError( 230 "Fatal Internal error [id=1]. Please report as a bug." 231 ) 232 233 del self._config[self.load_only][key] 234 235 def save(self): 236 # type: () -> None 237 """Save the current in-memory state. 238 """ 239 self._ensure_have_load_only() 240 241 for fname, parser in self._modified_parsers: 242 logger.info("Writing to %s", fname) 243 244 # Ensure directory exists. 245 ensure_dir(os.path.dirname(fname)) 246 247 with open(fname, "w") as f: 248 parser.write(f) 249 250 # 251 # Private routines 252 # 253 254 def _ensure_have_load_only(self): 255 # type: () -> None 256 if self.load_only is None: 257 raise ConfigurationError("Needed a specific file to be modifying.") 258 logger.debug("Will be working with %s variant only", self.load_only) 259 260 @property 261 def _dictionary(self): 262 # type: () -> Dict[str, Any] 263 """A dictionary representing the loaded configuration. 264 """ 265 # NOTE: Dictionaries are not populated if not loaded. So, conditionals 266 # are not needed here. 267 retval = {} 268 269 for variant in self._override_order: 270 retval.update(self._config[variant]) 271 272 return retval 273 274 def _load_config_files(self): 275 # type: () -> None 276 """Loads configuration from configuration files 277 """ 278 config_files = dict(self._iter_config_files()) 279 if config_files[kinds.ENV][0:1] == [os.devnull]: 280 logger.debug( 281 "Skipping loading configuration files due to " 282 "environment's PIP_CONFIG_FILE being os.devnull" 283 ) 284 return 285 286 for variant, files in config_files.items(): 287 for fname in files: 288 # If there's specific variant set in `load_only`, load only 289 # that variant, not the others. 290 if self.load_only is not None and variant != self.load_only: 291 logger.debug( 292 "Skipping file '%s' (variant: %s)", fname, variant 293 ) 294 continue 295 296 parser = self._load_file(variant, fname) 297 298 # Keeping track of the parsers used 299 self._parsers[variant].append((fname, parser)) 300 301 def _load_file(self, variant, fname): 302 # type: (Kind, str) -> RawConfigParser 303 logger.debug("For variant '%s', will try loading '%s'", variant, fname) 304 parser = self._construct_parser(fname) 305 306 for section in parser.sections(): 307 items = parser.items(section) 308 self._config[variant].update(self._normalized_keys(section, items)) 309 310 return parser 311 312 def _construct_parser(self, fname): 313 # type: (str) -> RawConfigParser 314 parser = configparser.RawConfigParser() 315 # If there is no such file, don't bother reading it but create the 316 # parser anyway, to hold the data. 317 # Doing this is useful when modifying and saving files, where we don't 318 # need to construct a parser. 319 if os.path.exists(fname): 320 try: 321 parser.read(fname) 322 except UnicodeDecodeError: 323 # See https://github.com/pypa/pip/issues/4963 324 raise ConfigurationFileCouldNotBeLoaded( 325 reason="contains invalid {} characters".format( 326 locale.getpreferredencoding(False) 327 ), 328 fname=fname, 329 ) 330 except configparser.Error as error: 331 # See https://github.com/pypa/pip/issues/4893 332 raise ConfigurationFileCouldNotBeLoaded(error=error) 333 return parser 334 335 def _load_environment_vars(self): 336 # type: () -> None 337 """Loads configuration from environment variables 338 """ 339 self._config[kinds.ENV_VAR].update( 340 self._normalized_keys(":env:", self._get_environ_vars()) 341 ) 342 343 def _normalized_keys(self, section, items): 344 # type: (str, Iterable[Tuple[str, Any]]) -> Dict[str, Any] 345 """Normalizes items to construct a dictionary with normalized keys. 346 347 This routine is where the names become keys and are made the same 348 regardless of source - configuration files or environment. 349 """ 350 normalized = {} 351 for name, val in items: 352 key = section + "." + _normalize_name(name) 353 normalized[key] = val 354 return normalized 355 356 def _get_environ_vars(self): 357 # type: () -> Iterable[Tuple[str, str]] 358 """Returns a generator with all environmental vars with prefix PIP_""" 359 for key, val in os.environ.items(): 360 should_be_yielded = ( 361 key.startswith("PIP_") and 362 key[4:].lower() not in self._ignore_env_names 363 ) 364 if should_be_yielded: 365 yield key[4:].lower(), val 366 367 # XXX: This is patched in the tests. 368 def _iter_config_files(self): 369 # type: () -> Iterable[Tuple[Kind, List[str]]] 370 """Yields variant and configuration files associated with it. 371 372 This should be treated like items of a dictionary. 373 """ 374 # SMELL: Move the conditions out of this function 375 376 # environment variables have the lowest priority 377 config_file = os.environ.get('PIP_CONFIG_FILE', None) 378 if config_file is not None: 379 yield kinds.ENV, [config_file] 380 else: 381 yield kinds.ENV, [] 382 383 config_files = get_configuration_files() 384 385 # at the base we have any global configuration 386 yield kinds.GLOBAL, config_files[kinds.GLOBAL] 387 388 # per-user configuration next 389 should_load_user_config = not self.isolated and not ( 390 config_file and os.path.exists(config_file) 391 ) 392 if should_load_user_config: 393 # The legacy config file is overridden by the new config file 394 yield kinds.USER, config_files[kinds.USER] 395 396 # finally virtualenv configuration first trumping others 397 yield kinds.SITE, config_files[kinds.SITE] 398 399 def _get_parser_to_modify(self): 400 # type: () -> Tuple[str, RawConfigParser] 401 # Determine which parser to modify 402 parsers = self._parsers[self.load_only] 403 if not parsers: 404 # This should not happen if everything works correctly. 405 raise ConfigurationError( 406 "Fatal Internal error [id=2]. Please report as a bug." 407 ) 408 409 # Use the highest priority parser. 410 return parsers[-1] 411 412 # XXX: This is patched in the tests. 413 def _mark_as_modified(self, fname, parser): 414 # type: (str, RawConfigParser) -> None 415 file_parser_tuple = (fname, parser) 416 if file_parser_tuple not in self._modified_parsers: 417 self._modified_parsers.append(file_parser_tuple) 418