1"""Configuration management setup
2
3Some terminology:
4- name
5  As written in config files.
6- value
7  Value associated with a name
8- key
9  Name combined with it's section (section.name)
10- variant
11  A single word describing where the configuration key-value pair came from
12"""
13
14import locale
15import logging
16import os
17import sys
18
19from pip._vendor.six.moves import configparser
20
21from pip._internal.exceptions import (
22    ConfigurationError, ConfigurationFileCouldNotBeLoaded,
23)
24from pip._internal.utils import appdirs
25from pip._internal.utils.compat import WINDOWS, expanduser
26from pip._internal.utils.misc import ensure_dir, enum
27from pip._internal.utils.typing import MYPY_CHECK_RUNNING
28
29if MYPY_CHECK_RUNNING:
30    from typing import (
31        Any, Dict, Iterable, List, NewType, Optional, Tuple
32    )
33
34    RawConfigParser = configparser.RawConfigParser  # Shorthand
35    Kind = NewType("Kind", str)
36
37logger = logging.getLogger(__name__)
38
39
40# NOTE: Maybe use the optionx attribute to normalize keynames.
41def _normalize_name(name):
42    # type: (str) -> str
43    """Make a name consistent regardless of source (environment or file)
44    """
45    name = name.lower().replace('_', '-')
46    if name.startswith('--'):
47        name = name[2:]  # only prefer long opts
48    return name
49
50
51def _disassemble_key(name):
52    # type: (str) -> List[str]
53    if "." not in name:
54        error_message = (
55            "Key does not contain dot separated section and key. "
56            "Perhaps you wanted to use 'global.{}' instead?"
57        ).format(name)
58        raise ConfigurationError(error_message)
59    return name.split(".", 1)
60
61
62# The kinds of configurations there are.
63kinds = enum(
64    USER="user",        # User Specific
65    GLOBAL="global",    # System Wide
66    SITE="site",        # [Virtual] Environment Specific
67    ENV="env",          # from PIP_CONFIG_FILE
68    ENV_VAR="env-var",  # from Environment Variables
69)
70
71
72CONFIG_BASENAME = 'pip.ini' if WINDOWS else 'pip.conf'
73
74
75def get_configuration_files():
76    global_config_files = [
77        os.path.join(path, CONFIG_BASENAME)
78        for path in appdirs.site_config_dirs('pip')
79    ]
80
81    site_config_file = os.path.join(sys.prefix, CONFIG_BASENAME)
82    legacy_config_file = os.path.join(
83        expanduser('~'),
84        'pip' if WINDOWS else '.pip',
85        CONFIG_BASENAME,
86    )
87    new_config_file = os.path.join(
88        appdirs.user_config_dir("pip"), CONFIG_BASENAME
89    )
90    return {
91        kinds.GLOBAL: global_config_files,
92        kinds.SITE: [site_config_file],
93        kinds.USER: [legacy_config_file, new_config_file],
94    }
95
96
97class Configuration(object):
98    """Handles management of configuration.
99
100    Provides an interface to accessing and managing configuration files.
101
102    This class converts provides an API that takes "section.key-name" style
103    keys and stores the value associated with it as "key-name" under the
104    section "section".
105
106    This allows for a clean interface wherein the both the section and the
107    key-name are preserved in an easy to manage form in the configuration files
108    and the data stored is also nice.
109    """
110
111    def __init__(self, isolated, load_only=None):
112        # type: (bool, Kind) -> None
113        super(Configuration, self).__init__()
114
115        _valid_load_only = [kinds.USER, kinds.GLOBAL, kinds.SITE, None]
116        if load_only not in _valid_load_only:
117            raise ConfigurationError(
118                "Got invalid value for load_only - should be one of {}".format(
119                    ", ".join(map(repr, _valid_load_only[:-1]))
120                )
121            )
122        self.isolated = isolated  # type: bool
123        self.load_only = load_only  # type: Optional[Kind]
124
125        # The order here determines the override order.
126        self._override_order = [
127            kinds.GLOBAL, kinds.USER, kinds.SITE, kinds.ENV, kinds.ENV_VAR
128        ]
129
130        self._ignore_env_names = ["version", "help"]
131
132        # Because we keep track of where we got the data from
133        self._parsers = {
134            variant: [] for variant in self._override_order
135        }  # type: Dict[Kind, List[Tuple[str, RawConfigParser]]]
136        self._config = {
137            variant: {} for variant in self._override_order
138        }  # type: Dict[Kind, Dict[str, Any]]
139        self._modified_parsers = []  # type: List[Tuple[str, RawConfigParser]]
140
141    def load(self):
142        # type: () -> None
143        """Loads configuration from configuration files and environment
144        """
145        self._load_config_files()
146        if not self.isolated:
147            self._load_environment_vars()
148
149    def get_file_to_edit(self):
150        # type: () -> Optional[str]
151        """Returns the file with highest priority in configuration
152        """
153        assert self.load_only is not None, \
154            "Need to be specified a file to be editing"
155
156        try:
157            return self._get_parser_to_modify()[0]
158        except IndexError:
159            return None
160
161    def items(self):
162        # type: () -> Iterable[Tuple[str, Any]]
163        """Returns key-value pairs like dict.items() representing the loaded
164        configuration
165        """
166        return self._dictionary.items()
167
168    def get_value(self, key):
169        # type: (str) -> Any
170        """Get a value from the configuration.
171        """
172        try:
173            return self._dictionary[key]
174        except KeyError:
175            raise ConfigurationError("No such key - {}".format(key))
176
177    def set_value(self, key, value):
178        # type: (str, Any) -> None
179        """Modify a value in the configuration.
180        """
181        self._ensure_have_load_only()
182
183        fname, parser = self._get_parser_to_modify()
184
185        if parser is not None:
186            section, name = _disassemble_key(key)
187
188            # Modify the parser and the configuration
189            if not parser.has_section(section):
190                parser.add_section(section)
191            parser.set(section, name, value)
192
193        self._config[self.load_only][key] = value
194        self._mark_as_modified(fname, parser)
195
196    def unset_value(self, key):
197        # type: (str) -> None
198        """Unset a value in the configuration.
199        """
200        self._ensure_have_load_only()
201
202        if key not in self._config[self.load_only]:
203            raise ConfigurationError("No such key - {}".format(key))
204
205        fname, parser = self._get_parser_to_modify()
206
207        if parser is not None:
208            section, name = _disassemble_key(key)
209
210            # Remove the key in the parser
211            modified_something = False
212            if parser.has_section(section):
213                # Returns whether the option was removed or not
214                modified_something = parser.remove_option(section, name)
215
216            if modified_something:
217                # name removed from parser, section may now be empty
218                section_iter = iter(parser.items(section))
219                try:
220                    val = next(section_iter)
221                except StopIteration:
222                    val = None
223
224                if val is None:
225                    parser.remove_section(section)
226
227                self._mark_as_modified(fname, parser)
228            else:
229                raise ConfigurationError(
230                    "Fatal Internal error [id=1]. Please report as a bug."
231                )
232
233        del self._config[self.load_only][key]
234
235    def save(self):
236        # type: () -> None
237        """Save the current in-memory state.
238        """
239        self._ensure_have_load_only()
240
241        for fname, parser in self._modified_parsers:
242            logger.info("Writing to %s", fname)
243
244            # Ensure directory exists.
245            ensure_dir(os.path.dirname(fname))
246
247            with open(fname, "w") as f:
248                parser.write(f)
249
250    #
251    # Private routines
252    #
253
254    def _ensure_have_load_only(self):
255        # type: () -> None
256        if self.load_only is None:
257            raise ConfigurationError("Needed a specific file to be modifying.")
258        logger.debug("Will be working with %s variant only", self.load_only)
259
260    @property
261    def _dictionary(self):
262        # type: () -> Dict[str, Any]
263        """A dictionary representing the loaded configuration.
264        """
265        # NOTE: Dictionaries are not populated if not loaded. So, conditionals
266        #       are not needed here.
267        retval = {}
268
269        for variant in self._override_order:
270            retval.update(self._config[variant])
271
272        return retval
273
274    def _load_config_files(self):
275        # type: () -> None
276        """Loads configuration from configuration files
277        """
278        config_files = dict(self._iter_config_files())
279        if config_files[kinds.ENV][0:1] == [os.devnull]:
280            logger.debug(
281                "Skipping loading configuration files due to "
282                "environment's PIP_CONFIG_FILE being os.devnull"
283            )
284            return
285
286        for variant, files in config_files.items():
287            for fname in files:
288                # If there's specific variant set in `load_only`, load only
289                # that variant, not the others.
290                if self.load_only is not None and variant != self.load_only:
291                    logger.debug(
292                        "Skipping file '%s' (variant: %s)", fname, variant
293                    )
294                    continue
295
296                parser = self._load_file(variant, fname)
297
298                # Keeping track of the parsers used
299                self._parsers[variant].append((fname, parser))
300
301    def _load_file(self, variant, fname):
302        # type: (Kind, str) -> RawConfigParser
303        logger.debug("For variant '%s', will try loading '%s'", variant, fname)
304        parser = self._construct_parser(fname)
305
306        for section in parser.sections():
307            items = parser.items(section)
308            self._config[variant].update(self._normalized_keys(section, items))
309
310        return parser
311
312    def _construct_parser(self, fname):
313        # type: (str) -> RawConfigParser
314        parser = configparser.RawConfigParser()
315        # If there is no such file, don't bother reading it but create the
316        # parser anyway, to hold the data.
317        # Doing this is useful when modifying and saving files, where we don't
318        # need to construct a parser.
319        if os.path.exists(fname):
320            try:
321                parser.read(fname)
322            except UnicodeDecodeError:
323                # See https://github.com/pypa/pip/issues/4963
324                raise ConfigurationFileCouldNotBeLoaded(
325                    reason="contains invalid {} characters".format(
326                        locale.getpreferredencoding(False)
327                    ),
328                    fname=fname,
329                )
330            except configparser.Error as error:
331                # See https://github.com/pypa/pip/issues/4893
332                raise ConfigurationFileCouldNotBeLoaded(error=error)
333        return parser
334
335    def _load_environment_vars(self):
336        # type: () -> None
337        """Loads configuration from environment variables
338        """
339        self._config[kinds.ENV_VAR].update(
340            self._normalized_keys(":env:", self._get_environ_vars())
341        )
342
343    def _normalized_keys(self, section, items):
344        # type: (str, Iterable[Tuple[str, Any]]) -> Dict[str, Any]
345        """Normalizes items to construct a dictionary with normalized keys.
346
347        This routine is where the names become keys and are made the same
348        regardless of source - configuration files or environment.
349        """
350        normalized = {}
351        for name, val in items:
352            key = section + "." + _normalize_name(name)
353            normalized[key] = val
354        return normalized
355
356    def _get_environ_vars(self):
357        # type: () -> Iterable[Tuple[str, str]]
358        """Returns a generator with all environmental vars with prefix PIP_"""
359        for key, val in os.environ.items():
360            should_be_yielded = (
361                key.startswith("PIP_") and
362                key[4:].lower() not in self._ignore_env_names
363            )
364            if should_be_yielded:
365                yield key[4:].lower(), val
366
367    # XXX: This is patched in the tests.
368    def _iter_config_files(self):
369        # type: () -> Iterable[Tuple[Kind, List[str]]]
370        """Yields variant and configuration files associated with it.
371
372        This should be treated like items of a dictionary.
373        """
374        # SMELL: Move the conditions out of this function
375
376        # environment variables have the lowest priority
377        config_file = os.environ.get('PIP_CONFIG_FILE', None)
378        if config_file is not None:
379            yield kinds.ENV, [config_file]
380        else:
381            yield kinds.ENV, []
382
383        config_files = get_configuration_files()
384
385        # at the base we have any global configuration
386        yield kinds.GLOBAL, config_files[kinds.GLOBAL]
387
388        # per-user configuration next
389        should_load_user_config = not self.isolated and not (
390            config_file and os.path.exists(config_file)
391        )
392        if should_load_user_config:
393            # The legacy config file is overridden by the new config file
394            yield kinds.USER, config_files[kinds.USER]
395
396        # finally virtualenv configuration first trumping others
397        yield kinds.SITE, config_files[kinds.SITE]
398
399    def _get_parser_to_modify(self):
400        # type: () -> Tuple[str, RawConfigParser]
401        # Determine which parser to modify
402        parsers = self._parsers[self.load_only]
403        if not parsers:
404            # This should not happen if everything works correctly.
405            raise ConfigurationError(
406                "Fatal Internal error [id=2]. Please report as a bug."
407            )
408
409        # Use the highest priority parser.
410        return parsers[-1]
411
412    # XXX: This is patched in the tests.
413    def _mark_as_modified(self, fname, parser):
414        # type: (str, RawConfigParser) -> None
415        file_parser_tuple = (fname, parser)
416        if file_parser_tuple not in self._modified_parsers:
417            self._modified_parsers.append(file_parser_tuple)
418