1"""Configuration management setup
2
3Some terminology:
4- name
5  As written in config files.
6- value
7  Value associated with a name
8- key
9  Name combined with it's section (section.name)
10- variant
11  A single word describing where the configuration key-value pair came from
12"""
13
14import configparser
15import locale
16import logging
17import os
18import sys
19
20from pip._internal.exceptions import (
21    ConfigurationError,
22    ConfigurationFileCouldNotBeLoaded,
23)
24from pip._internal.utils import appdirs
25from pip._internal.utils.compat import WINDOWS
26from pip._internal.utils.misc import ensure_dir, enum
27from pip._internal.utils.typing import MYPY_CHECK_RUNNING
28
29if MYPY_CHECK_RUNNING:
30    from typing import Any, Dict, Iterable, List, NewType, Optional, Tuple
31
32    RawConfigParser = configparser.RawConfigParser  # Shorthand
33    Kind = NewType("Kind", str)
34
35CONFIG_BASENAME = 'pip.ini' if WINDOWS else 'pip.conf'
36ENV_NAMES_IGNORED = "version", "help"
37
38# The kinds of configurations there are.
39kinds = enum(
40    USER="user",        # User Specific
41    GLOBAL="global",    # System Wide
42    SITE="site",        # [Virtual] Environment Specific
43    ENV="env",          # from PIP_CONFIG_FILE
44    ENV_VAR="env-var",  # from Environment Variables
45)
46OVERRIDE_ORDER = kinds.GLOBAL, kinds.USER, kinds.SITE, kinds.ENV, kinds.ENV_VAR
47VALID_LOAD_ONLY = kinds.USER, kinds.GLOBAL, kinds.SITE
48
49logger = logging.getLogger(__name__)
50
51
52# NOTE: Maybe use the optionx attribute to normalize keynames.
53def _normalize_name(name):
54    # type: (str) -> str
55    """Make a name consistent regardless of source (environment or file)
56    """
57    name = name.lower().replace('_', '-')
58    if name.startswith('--'):
59        name = name[2:]  # only prefer long opts
60    return name
61
62
63def _disassemble_key(name):
64    # type: (str) -> List[str]
65    if "." not in name:
66        error_message = (
67            "Key does not contain dot separated section and key. "
68            "Perhaps you wanted to use 'global.{}' instead?"
69        ).format(name)
70        raise ConfigurationError(error_message)
71    return name.split(".", 1)
72
73
74def get_configuration_files():
75    # type: () -> Dict[Kind, List[str]]
76    global_config_files = [
77        os.path.join(path, CONFIG_BASENAME)
78        for path in appdirs.site_config_dirs('pip')
79    ]
80
81    site_config_file = os.path.join(sys.prefix, CONFIG_BASENAME)
82    legacy_config_file = os.path.join(
83        os.path.expanduser('~'),
84        'pip' if WINDOWS else '.pip',
85        CONFIG_BASENAME,
86    )
87    new_config_file = os.path.join(
88        appdirs.user_config_dir("pip"), CONFIG_BASENAME
89    )
90    return {
91        kinds.GLOBAL: global_config_files,
92        kinds.SITE: [site_config_file],
93        kinds.USER: [legacy_config_file, new_config_file],
94    }
95
96
97class Configuration:
98    """Handles management of configuration.
99
100    Provides an interface to accessing and managing configuration files.
101
102    This class converts provides an API that takes "section.key-name" style
103    keys and stores the value associated with it as "key-name" under the
104    section "section".
105
106    This allows for a clean interface wherein the both the section and the
107    key-name are preserved in an easy to manage form in the configuration files
108    and the data stored is also nice.
109    """
110
111    def __init__(self, isolated, load_only=None):
112        # type: (bool, Optional[Kind]) -> None
113        super().__init__()
114
115        if load_only is not None and load_only not in VALID_LOAD_ONLY:
116            raise ConfigurationError(
117                "Got invalid value for load_only - should be one of {}".format(
118                    ", ".join(map(repr, VALID_LOAD_ONLY))
119                )
120            )
121        self.isolated = isolated
122        self.load_only = load_only
123
124        # Because we keep track of where we got the data from
125        self._parsers = {
126            variant: [] for variant in OVERRIDE_ORDER
127        }  # type: Dict[Kind, List[Tuple[str, RawConfigParser]]]
128        self._config = {
129            variant: {} for variant in OVERRIDE_ORDER
130        }  # type: Dict[Kind, Dict[str, Any]]
131        self._modified_parsers = []  # type: List[Tuple[str, RawConfigParser]]
132
133    def load(self):
134        # type: () -> None
135        """Loads configuration from configuration files and environment
136        """
137        self._load_config_files()
138        if not self.isolated:
139            self._load_environment_vars()
140
141    def get_file_to_edit(self):
142        # type: () -> Optional[str]
143        """Returns the file with highest priority in configuration
144        """
145        assert self.load_only is not None, \
146            "Need to be specified a file to be editing"
147
148        try:
149            return self._get_parser_to_modify()[0]
150        except IndexError:
151            return None
152
153    def items(self):
154        # type: () -> Iterable[Tuple[str, Any]]
155        """Returns key-value pairs like dict.items() representing the loaded
156        configuration
157        """
158        return self._dictionary.items()
159
160    def get_value(self, key):
161        # type: (str) -> Any
162        """Get a value from the configuration.
163        """
164        try:
165            return self._dictionary[key]
166        except KeyError:
167            raise ConfigurationError(f"No such key - {key}")
168
169    def set_value(self, key, value):
170        # type: (str, Any) -> None
171        """Modify a value in the configuration.
172        """
173        self._ensure_have_load_only()
174
175        assert self.load_only
176        fname, parser = self._get_parser_to_modify()
177
178        if parser is not None:
179            section, name = _disassemble_key(key)
180
181            # Modify the parser and the configuration
182            if not parser.has_section(section):
183                parser.add_section(section)
184            parser.set(section, name, value)
185
186        self._config[self.load_only][key] = value
187        self._mark_as_modified(fname, parser)
188
189    def unset_value(self, key):
190        # type: (str) -> None
191        """Unset a value in the configuration."""
192        self._ensure_have_load_only()
193
194        assert self.load_only
195        if key not in self._config[self.load_only]:
196            raise ConfigurationError(f"No such key - {key}")
197
198        fname, parser = self._get_parser_to_modify()
199
200        if parser is not None:
201            section, name = _disassemble_key(key)
202            if not (parser.has_section(section)
203                    and parser.remove_option(section, name)):
204                # The option was not removed.
205                raise ConfigurationError(
206                    "Fatal Internal error [id=1]. Please report as a bug."
207                )
208
209            # The section may be empty after the option was removed.
210            if not parser.items(section):
211                parser.remove_section(section)
212            self._mark_as_modified(fname, parser)
213
214        del self._config[self.load_only][key]
215
216    def save(self):
217        # type: () -> None
218        """Save the current in-memory state.
219        """
220        self._ensure_have_load_only()
221
222        for fname, parser in self._modified_parsers:
223            logger.info("Writing to %s", fname)
224
225            # Ensure directory exists.
226            ensure_dir(os.path.dirname(fname))
227
228            with open(fname, "w") as f:
229                parser.write(f)
230
231    #
232    # Private routines
233    #
234
235    def _ensure_have_load_only(self):
236        # type: () -> None
237        if self.load_only is None:
238            raise ConfigurationError("Needed a specific file to be modifying.")
239        logger.debug("Will be working with %s variant only", self.load_only)
240
241    @property
242    def _dictionary(self):
243        # type: () -> Dict[str, Any]
244        """A dictionary representing the loaded configuration.
245        """
246        # NOTE: Dictionaries are not populated if not loaded. So, conditionals
247        #       are not needed here.
248        retval = {}
249
250        for variant in OVERRIDE_ORDER:
251            retval.update(self._config[variant])
252
253        return retval
254
255    def _load_config_files(self):
256        # type: () -> None
257        """Loads configuration from configuration files
258        """
259        config_files = dict(self.iter_config_files())
260        if config_files[kinds.ENV][0:1] == [os.devnull]:
261            logger.debug(
262                "Skipping loading configuration files due to "
263                "environment's PIP_CONFIG_FILE being os.devnull"
264            )
265            return
266
267        for variant, files in config_files.items():
268            for fname in files:
269                # If there's specific variant set in `load_only`, load only
270                # that variant, not the others.
271                if self.load_only is not None and variant != self.load_only:
272                    logger.debug(
273                        "Skipping file '%s' (variant: %s)", fname, variant
274                    )
275                    continue
276
277                parser = self._load_file(variant, fname)
278
279                # Keeping track of the parsers used
280                self._parsers[variant].append((fname, parser))
281
282    def _load_file(self, variant, fname):
283        # type: (Kind, str) -> RawConfigParser
284        logger.debug("For variant '%s', will try loading '%s'", variant, fname)
285        parser = self._construct_parser(fname)
286
287        for section in parser.sections():
288            items = parser.items(section)
289            self._config[variant].update(self._normalized_keys(section, items))
290
291        return parser
292
293    def _construct_parser(self, fname):
294        # type: (str) -> RawConfigParser
295        parser = configparser.RawConfigParser()
296        # If there is no such file, don't bother reading it but create the
297        # parser anyway, to hold the data.
298        # Doing this is useful when modifying and saving files, where we don't
299        # need to construct a parser.
300        if os.path.exists(fname):
301            try:
302                parser.read(fname)
303            except UnicodeDecodeError:
304                # See https://github.com/pypa/pip/issues/4963
305                raise ConfigurationFileCouldNotBeLoaded(
306                    reason="contains invalid {} characters".format(
307                        locale.getpreferredencoding(False)
308                    ),
309                    fname=fname,
310                )
311            except configparser.Error as error:
312                # See https://github.com/pypa/pip/issues/4893
313                raise ConfigurationFileCouldNotBeLoaded(error=error)
314        return parser
315
316    def _load_environment_vars(self):
317        # type: () -> None
318        """Loads configuration from environment variables
319        """
320        self._config[kinds.ENV_VAR].update(
321            self._normalized_keys(":env:", self.get_environ_vars())
322        )
323
324    def _normalized_keys(self, section, items):
325        # type: (str, Iterable[Tuple[str, Any]]) -> Dict[str, Any]
326        """Normalizes items to construct a dictionary with normalized keys.
327
328        This routine is where the names become keys and are made the same
329        regardless of source - configuration files or environment.
330        """
331        normalized = {}
332        for name, val in items:
333            key = section + "." + _normalize_name(name)
334            normalized[key] = val
335        return normalized
336
337    def get_environ_vars(self):
338        # type: () -> Iterable[Tuple[str, str]]
339        """Returns a generator with all environmental vars with prefix PIP_"""
340        for key, val in os.environ.items():
341            if key.startswith("PIP_"):
342                name = key[4:].lower()
343                if name not in ENV_NAMES_IGNORED:
344                    yield name, val
345
346    # XXX: This is patched in the tests.
347    def iter_config_files(self):
348        # type: () -> Iterable[Tuple[Kind, List[str]]]
349        """Yields variant and configuration files associated with it.
350
351        This should be treated like items of a dictionary.
352        """
353        # SMELL: Move the conditions out of this function
354
355        # environment variables have the lowest priority
356        config_file = os.environ.get('PIP_CONFIG_FILE', None)
357        if config_file is not None:
358            yield kinds.ENV, [config_file]
359        else:
360            yield kinds.ENV, []
361
362        config_files = get_configuration_files()
363
364        # at the base we have any global configuration
365        yield kinds.GLOBAL, config_files[kinds.GLOBAL]
366
367        # per-user configuration next
368        should_load_user_config = not self.isolated and not (
369            config_file and os.path.exists(config_file)
370        )
371        if should_load_user_config:
372            # The legacy config file is overridden by the new config file
373            yield kinds.USER, config_files[kinds.USER]
374
375        # finally virtualenv configuration first trumping others
376        yield kinds.SITE, config_files[kinds.SITE]
377
378    def get_values_in_config(self, variant):
379        # type: (Kind) -> Dict[str, Any]
380        """Get values present in a config file"""
381        return self._config[variant]
382
383    def _get_parser_to_modify(self):
384        # type: () -> Tuple[str, RawConfigParser]
385        # Determine which parser to modify
386        assert self.load_only
387        parsers = self._parsers[self.load_only]
388        if not parsers:
389            # This should not happen if everything works correctly.
390            raise ConfigurationError(
391                "Fatal Internal error [id=2]. Please report as a bug."
392            )
393
394        # Use the highest priority parser.
395        return parsers[-1]
396
397    # XXX: This is patched in the tests.
398    def _mark_as_modified(self, fname, parser):
399        # type: (str, RawConfigParser) -> None
400        file_parser_tuple = (fname, parser)
401        if file_parser_tuple not in self._modified_parsers:
402            self._modified_parsers.append(file_parser_tuple)
403
404    def __repr__(self):
405        # type: () -> str
406        return f"{self.__class__.__name__}({self._dictionary!r})"
407