1"""Configuration management setup 2 3Some terminology: 4- name 5 As written in config files. 6- value 7 Value associated with a name 8- key 9 Name combined with it's section (section.name) 10- variant 11 A single word describing where the configuration key-value pair came from 12""" 13 14import configparser 15import locale 16import logging 17import os 18import sys 19 20from pip._internal.exceptions import ( 21 ConfigurationError, 22 ConfigurationFileCouldNotBeLoaded, 23) 24from pip._internal.utils import appdirs 25from pip._internal.utils.compat import WINDOWS 26from pip._internal.utils.misc import ensure_dir, enum 27from pip._internal.utils.typing import MYPY_CHECK_RUNNING 28 29if MYPY_CHECK_RUNNING: 30 from typing import Any, Dict, Iterable, List, NewType, Optional, Tuple 31 32 RawConfigParser = configparser.RawConfigParser # Shorthand 33 Kind = NewType("Kind", str) 34 35CONFIG_BASENAME = 'pip.ini' if WINDOWS else 'pip.conf' 36ENV_NAMES_IGNORED = "version", "help" 37 38# The kinds of configurations there are. 39kinds = enum( 40 USER="user", # User Specific 41 GLOBAL="global", # System Wide 42 SITE="site", # [Virtual] Environment Specific 43 ENV="env", # from PIP_CONFIG_FILE 44 ENV_VAR="env-var", # from Environment Variables 45) 46OVERRIDE_ORDER = kinds.GLOBAL, kinds.USER, kinds.SITE, kinds.ENV, kinds.ENV_VAR 47VALID_LOAD_ONLY = kinds.USER, kinds.GLOBAL, kinds.SITE 48 49logger = logging.getLogger(__name__) 50 51 52# NOTE: Maybe use the optionx attribute to normalize keynames. 53def _normalize_name(name): 54 # type: (str) -> str 55 """Make a name consistent regardless of source (environment or file) 56 """ 57 name = name.lower().replace('_', '-') 58 if name.startswith('--'): 59 name = name[2:] # only prefer long opts 60 return name 61 62 63def _disassemble_key(name): 64 # type: (str) -> List[str] 65 if "." not in name: 66 error_message = ( 67 "Key does not contain dot separated section and key. " 68 "Perhaps you wanted to use 'global.{}' instead?" 69 ).format(name) 70 raise ConfigurationError(error_message) 71 return name.split(".", 1) 72 73 74def get_configuration_files(): 75 # type: () -> Dict[Kind, List[str]] 76 global_config_files = [ 77 os.path.join(path, CONFIG_BASENAME) 78 for path in appdirs.site_config_dirs('pip') 79 ] 80 81 site_config_file = os.path.join(sys.prefix, CONFIG_BASENAME) 82 legacy_config_file = os.path.join( 83 os.path.expanduser('~'), 84 'pip' if WINDOWS else '.pip', 85 CONFIG_BASENAME, 86 ) 87 new_config_file = os.path.join( 88 appdirs.user_config_dir("pip"), CONFIG_BASENAME 89 ) 90 return { 91 kinds.GLOBAL: global_config_files, 92 kinds.SITE: [site_config_file], 93 kinds.USER: [legacy_config_file, new_config_file], 94 } 95 96 97class Configuration: 98 """Handles management of configuration. 99 100 Provides an interface to accessing and managing configuration files. 101 102 This class converts provides an API that takes "section.key-name" style 103 keys and stores the value associated with it as "key-name" under the 104 section "section". 105 106 This allows for a clean interface wherein the both the section and the 107 key-name are preserved in an easy to manage form in the configuration files 108 and the data stored is also nice. 109 """ 110 111 def __init__(self, isolated, load_only=None): 112 # type: (bool, Optional[Kind]) -> None 113 super().__init__() 114 115 if load_only is not None and load_only not in VALID_LOAD_ONLY: 116 raise ConfigurationError( 117 "Got invalid value for load_only - should be one of {}".format( 118 ", ".join(map(repr, VALID_LOAD_ONLY)) 119 ) 120 ) 121 self.isolated = isolated 122 self.load_only = load_only 123 124 # Because we keep track of where we got the data from 125 self._parsers = { 126 variant: [] for variant in OVERRIDE_ORDER 127 } # type: Dict[Kind, List[Tuple[str, RawConfigParser]]] 128 self._config = { 129 variant: {} for variant in OVERRIDE_ORDER 130 } # type: Dict[Kind, Dict[str, Any]] 131 self._modified_parsers = [] # type: List[Tuple[str, RawConfigParser]] 132 133 def load(self): 134 # type: () -> None 135 """Loads configuration from configuration files and environment 136 """ 137 self._load_config_files() 138 if not self.isolated: 139 self._load_environment_vars() 140 141 def get_file_to_edit(self): 142 # type: () -> Optional[str] 143 """Returns the file with highest priority in configuration 144 """ 145 assert self.load_only is not None, \ 146 "Need to be specified a file to be editing" 147 148 try: 149 return self._get_parser_to_modify()[0] 150 except IndexError: 151 return None 152 153 def items(self): 154 # type: () -> Iterable[Tuple[str, Any]] 155 """Returns key-value pairs like dict.items() representing the loaded 156 configuration 157 """ 158 return self._dictionary.items() 159 160 def get_value(self, key): 161 # type: (str) -> Any 162 """Get a value from the configuration. 163 """ 164 try: 165 return self._dictionary[key] 166 except KeyError: 167 raise ConfigurationError(f"No such key - {key}") 168 169 def set_value(self, key, value): 170 # type: (str, Any) -> None 171 """Modify a value in the configuration. 172 """ 173 self._ensure_have_load_only() 174 175 assert self.load_only 176 fname, parser = self._get_parser_to_modify() 177 178 if parser is not None: 179 section, name = _disassemble_key(key) 180 181 # Modify the parser and the configuration 182 if not parser.has_section(section): 183 parser.add_section(section) 184 parser.set(section, name, value) 185 186 self._config[self.load_only][key] = value 187 self._mark_as_modified(fname, parser) 188 189 def unset_value(self, key): 190 # type: (str) -> None 191 """Unset a value in the configuration.""" 192 self._ensure_have_load_only() 193 194 assert self.load_only 195 if key not in self._config[self.load_only]: 196 raise ConfigurationError(f"No such key - {key}") 197 198 fname, parser = self._get_parser_to_modify() 199 200 if parser is not None: 201 section, name = _disassemble_key(key) 202 if not (parser.has_section(section) 203 and parser.remove_option(section, name)): 204 # The option was not removed. 205 raise ConfigurationError( 206 "Fatal Internal error [id=1]. Please report as a bug." 207 ) 208 209 # The section may be empty after the option was removed. 210 if not parser.items(section): 211 parser.remove_section(section) 212 self._mark_as_modified(fname, parser) 213 214 del self._config[self.load_only][key] 215 216 def save(self): 217 # type: () -> None 218 """Save the current in-memory state. 219 """ 220 self._ensure_have_load_only() 221 222 for fname, parser in self._modified_parsers: 223 logger.info("Writing to %s", fname) 224 225 # Ensure directory exists. 226 ensure_dir(os.path.dirname(fname)) 227 228 with open(fname, "w") as f: 229 parser.write(f) 230 231 # 232 # Private routines 233 # 234 235 def _ensure_have_load_only(self): 236 # type: () -> None 237 if self.load_only is None: 238 raise ConfigurationError("Needed a specific file to be modifying.") 239 logger.debug("Will be working with %s variant only", self.load_only) 240 241 @property 242 def _dictionary(self): 243 # type: () -> Dict[str, Any] 244 """A dictionary representing the loaded configuration. 245 """ 246 # NOTE: Dictionaries are not populated if not loaded. So, conditionals 247 # are not needed here. 248 retval = {} 249 250 for variant in OVERRIDE_ORDER: 251 retval.update(self._config[variant]) 252 253 return retval 254 255 def _load_config_files(self): 256 # type: () -> None 257 """Loads configuration from configuration files 258 """ 259 config_files = dict(self.iter_config_files()) 260 if config_files[kinds.ENV][0:1] == [os.devnull]: 261 logger.debug( 262 "Skipping loading configuration files due to " 263 "environment's PIP_CONFIG_FILE being os.devnull" 264 ) 265 return 266 267 for variant, files in config_files.items(): 268 for fname in files: 269 # If there's specific variant set in `load_only`, load only 270 # that variant, not the others. 271 if self.load_only is not None and variant != self.load_only: 272 logger.debug( 273 "Skipping file '%s' (variant: %s)", fname, variant 274 ) 275 continue 276 277 parser = self._load_file(variant, fname) 278 279 # Keeping track of the parsers used 280 self._parsers[variant].append((fname, parser)) 281 282 def _load_file(self, variant, fname): 283 # type: (Kind, str) -> RawConfigParser 284 logger.debug("For variant '%s', will try loading '%s'", variant, fname) 285 parser = self._construct_parser(fname) 286 287 for section in parser.sections(): 288 items = parser.items(section) 289 self._config[variant].update(self._normalized_keys(section, items)) 290 291 return parser 292 293 def _construct_parser(self, fname): 294 # type: (str) -> RawConfigParser 295 parser = configparser.RawConfigParser() 296 # If there is no such file, don't bother reading it but create the 297 # parser anyway, to hold the data. 298 # Doing this is useful when modifying and saving files, where we don't 299 # need to construct a parser. 300 if os.path.exists(fname): 301 try: 302 parser.read(fname) 303 except UnicodeDecodeError: 304 # See https://github.com/pypa/pip/issues/4963 305 raise ConfigurationFileCouldNotBeLoaded( 306 reason="contains invalid {} characters".format( 307 locale.getpreferredencoding(False) 308 ), 309 fname=fname, 310 ) 311 except configparser.Error as error: 312 # See https://github.com/pypa/pip/issues/4893 313 raise ConfigurationFileCouldNotBeLoaded(error=error) 314 return parser 315 316 def _load_environment_vars(self): 317 # type: () -> None 318 """Loads configuration from environment variables 319 """ 320 self._config[kinds.ENV_VAR].update( 321 self._normalized_keys(":env:", self.get_environ_vars()) 322 ) 323 324 def _normalized_keys(self, section, items): 325 # type: (str, Iterable[Tuple[str, Any]]) -> Dict[str, Any] 326 """Normalizes items to construct a dictionary with normalized keys. 327 328 This routine is where the names become keys and are made the same 329 regardless of source - configuration files or environment. 330 """ 331 normalized = {} 332 for name, val in items: 333 key = section + "." + _normalize_name(name) 334 normalized[key] = val 335 return normalized 336 337 def get_environ_vars(self): 338 # type: () -> Iterable[Tuple[str, str]] 339 """Returns a generator with all environmental vars with prefix PIP_""" 340 for key, val in os.environ.items(): 341 if key.startswith("PIP_"): 342 name = key[4:].lower() 343 if name not in ENV_NAMES_IGNORED: 344 yield name, val 345 346 # XXX: This is patched in the tests. 347 def iter_config_files(self): 348 # type: () -> Iterable[Tuple[Kind, List[str]]] 349 """Yields variant and configuration files associated with it. 350 351 This should be treated like items of a dictionary. 352 """ 353 # SMELL: Move the conditions out of this function 354 355 # environment variables have the lowest priority 356 config_file = os.environ.get('PIP_CONFIG_FILE', None) 357 if config_file is not None: 358 yield kinds.ENV, [config_file] 359 else: 360 yield kinds.ENV, [] 361 362 config_files = get_configuration_files() 363 364 # at the base we have any global configuration 365 yield kinds.GLOBAL, config_files[kinds.GLOBAL] 366 367 # per-user configuration next 368 should_load_user_config = not self.isolated and not ( 369 config_file and os.path.exists(config_file) 370 ) 371 if should_load_user_config: 372 # The legacy config file is overridden by the new config file 373 yield kinds.USER, config_files[kinds.USER] 374 375 # finally virtualenv configuration first trumping others 376 yield kinds.SITE, config_files[kinds.SITE] 377 378 def get_values_in_config(self, variant): 379 # type: (Kind) -> Dict[str, Any] 380 """Get values present in a config file""" 381 return self._config[variant] 382 383 def _get_parser_to_modify(self): 384 # type: () -> Tuple[str, RawConfigParser] 385 # Determine which parser to modify 386 assert self.load_only 387 parsers = self._parsers[self.load_only] 388 if not parsers: 389 # This should not happen if everything works correctly. 390 raise ConfigurationError( 391 "Fatal Internal error [id=2]. Please report as a bug." 392 ) 393 394 # Use the highest priority parser. 395 return parsers[-1] 396 397 # XXX: This is patched in the tests. 398 def _mark_as_modified(self, fname, parser): 399 # type: (str, RawConfigParser) -> None 400 file_parser_tuple = (fname, parser) 401 if file_parser_tuple not in self._modified_parsers: 402 self._modified_parsers.append(file_parser_tuple) 403 404 def __repr__(self): 405 # type: () -> str 406 return f"{self.__class__.__name__}({self._dictionary!r})" 407