1"""
2Requirements file parsing
3"""
4
5from __future__ import absolute_import
6
7import os
8import re
9import shlex
10import sys
11import optparse
12import warnings
13
14from pip9._vendor.six.moves.urllib import parse as urllib_parse
15from pip9._vendor.six.moves import filterfalse
16
17import pip9
18from pip9.download import get_file_content
19from pip9.req.req_install import InstallRequirement
20from pip9.exceptions import (RequirementsFileParseError)
21from pip9.utils.deprecation import RemovedInPip10Warning
22from pip9 import cmdoptions
23
24__all__ = ['parse_requirements']
25
26SCHEME_RE = re.compile(r'^(http|https|file):', re.I)
27COMMENT_RE = re.compile(r'(^|\s)+#.*$')
28
29SUPPORTED_OPTIONS = [
30    cmdoptions.constraints,
31    cmdoptions.editable,
32    cmdoptions.requirements,
33    cmdoptions.no_index,
34    cmdoptions.index_url,
35    cmdoptions.find_links,
36    cmdoptions.extra_index_url,
37    cmdoptions.allow_external,
38    cmdoptions.allow_all_external,
39    cmdoptions.no_allow_external,
40    cmdoptions.allow_unsafe,
41    cmdoptions.no_allow_unsafe,
42    cmdoptions.use_wheel,
43    cmdoptions.no_use_wheel,
44    cmdoptions.always_unzip,
45    cmdoptions.no_binary,
46    cmdoptions.only_binary,
47    cmdoptions.pre,
48    cmdoptions.process_dependency_links,
49    cmdoptions.trusted_host,
50    cmdoptions.require_hashes,
51]
52
53# options to be passed to requirements
54SUPPORTED_OPTIONS_REQ = [
55    cmdoptions.install_options,
56    cmdoptions.global_options,
57    cmdoptions.hash,
58]
59
60# the 'dest' string values
61SUPPORTED_OPTIONS_REQ_DEST = [o().dest for o in SUPPORTED_OPTIONS_REQ]
62
63
64def parse_requirements(filename, finder=None, comes_from=None, options=None,
65                       session=None, constraint=False, wheel_cache=None):
66    """Parse a requirements file and yield InstallRequirement instances.
67
68    :param filename:    Path or url of requirements file.
69    :param finder:      Instance of pip9.index.PackageFinder.
70    :param comes_from:  Origin description of requirements.
71    :param options:     cli options.
72    :param session:     Instance of pip9.download.PipSession.
73    :param constraint:  If true, parsing a constraint file rather than
74        requirements file.
75    :param wheel_cache: Instance of pip9.wheel.WheelCache
76    """
77    if session is None:
78        raise TypeError(
79            "parse_requirements() missing 1 required keyword argument: "
80            "'session'"
81        )
82
83    _, content = get_file_content(
84        filename, comes_from=comes_from, session=session
85    )
86
87    lines_enum = preprocess(content, options)
88
89    for line_number, line in lines_enum:
90        req_iter = process_line(line, filename, line_number, finder,
91                                comes_from, options, session, wheel_cache,
92                                constraint=constraint)
93        for req in req_iter:
94            yield req
95
96
97def preprocess(content, options):
98    """Split, filter, and join lines, and return a line iterator
99
100    :param content: the content of the requirements file
101    :param options: cli options
102    """
103    lines_enum = enumerate(content.splitlines(), start=1)
104    lines_enum = join_lines(lines_enum)
105    lines_enum = ignore_comments(lines_enum)
106    lines_enum = skip_regex(lines_enum, options)
107    return lines_enum
108
109
110def process_line(line, filename, line_number, finder=None, comes_from=None,
111                 options=None, session=None, wheel_cache=None,
112                 constraint=False):
113    """Process a single requirements line; This can result in creating/yielding
114    requirements, or updating the finder.
115
116    For lines that contain requirements, the only options that have an effect
117    are from SUPPORTED_OPTIONS_REQ, and they are scoped to the
118    requirement. Other options from SUPPORTED_OPTIONS may be present, but are
119    ignored.
120
121    For lines that do not contain requirements, the only options that have an
122    effect are from SUPPORTED_OPTIONS. Options from SUPPORTED_OPTIONS_REQ may
123    be present, but are ignored. These lines may contain multiple options
124    (although our docs imply only one is supported), and all our parsed and
125    affect the finder.
126
127    :param constraint: If True, parsing a constraints file.
128    :param options: OptionParser options that we may update
129    """
130    parser = build_parser()
131    defaults = parser.get_default_values()
132    defaults.index_url = None
133    if finder:
134        # `finder.format_control` will be updated during parsing
135        defaults.format_control = finder.format_control
136    args_str, options_str = break_args_options(line)
137    if sys.version_info < (2, 7, 3):
138        # Prior to 2.7.3, shlex cannot deal with unicode entries
139        options_str = options_str.encode('utf8')
140    opts, _ = parser.parse_args(shlex.split(options_str), defaults)
141
142    # preserve for the nested code path
143    line_comes_from = '%s %s (line %s)' % (
144        '-c' if constraint else '-r', filename, line_number)
145
146    # yield a line requirement
147    if args_str:
148        isolated = options.isolated_mode if options else False
149        if options:
150            cmdoptions.check_install_build_global(options, opts)
151        # get the options that apply to requirements
152        req_options = {}
153        for dest in SUPPORTED_OPTIONS_REQ_DEST:
154            if dest in opts.__dict__ and opts.__dict__[dest]:
155                req_options[dest] = opts.__dict__[dest]
156        yield InstallRequirement.from_line(
157            args_str, line_comes_from, constraint=constraint,
158            isolated=isolated, options=req_options, wheel_cache=wheel_cache
159        )
160
161    # yield an editable requirement
162    elif opts.editables:
163        isolated = options.isolated_mode if options else False
164        default_vcs = options.default_vcs if options else None
165        yield InstallRequirement.from_editable(
166            opts.editables[0], comes_from=line_comes_from,
167            constraint=constraint, default_vcs=default_vcs, isolated=isolated,
168            wheel_cache=wheel_cache
169        )
170
171    # parse a nested requirements file
172    elif opts.requirements or opts.constraints:
173        if opts.requirements:
174            req_path = opts.requirements[0]
175            nested_constraint = False
176        else:
177            req_path = opts.constraints[0]
178            nested_constraint = True
179        # original file is over http
180        if SCHEME_RE.search(filename):
181            # do a url join so relative paths work
182            req_path = urllib_parse.urljoin(filename, req_path)
183        # original file and nested file are paths
184        elif not SCHEME_RE.search(req_path):
185            # do a join so relative paths work
186            req_path = os.path.join(os.path.dirname(filename), req_path)
187        # TODO: Why not use `comes_from='-r {} (line {})'` here as well?
188        parser = parse_requirements(
189            req_path, finder, comes_from, options, session,
190            constraint=nested_constraint, wheel_cache=wheel_cache
191        )
192        for req in parser:
193            yield req
194
195    # percolate hash-checking option upward
196    elif opts.require_hashes:
197        options.require_hashes = opts.require_hashes
198
199    # set finder options
200    elif finder:
201        if opts.allow_external:
202            warnings.warn(
203                "--allow-external has been deprecated and will be removed in "
204                "the future. Due to changes in the repository protocol, it no "
205                "longer has any effect.",
206                RemovedInPip10Warning,
207            )
208
209        if opts.allow_all_external:
210            warnings.warn(
211                "--allow-all-external has been deprecated and will be removed "
212                "in the future. Due to changes in the repository protocol, it "
213                "no longer has any effect.",
214                RemovedInPip10Warning,
215            )
216
217        if opts.allow_unverified:
218            warnings.warn(
219                "--allow-unverified has been deprecated and will be removed "
220                "in the future. Due to changes in the repository protocol, it "
221                "no longer has any effect.",
222                RemovedInPip10Warning,
223            )
224
225        if opts.index_url:
226            finder.index_urls = [opts.index_url]
227        if opts.use_wheel is False:
228            finder.use_wheel = False
229            pip9.index.fmt_ctl_no_use_wheel(finder.format_control)
230        if opts.no_index is True:
231            finder.index_urls = []
232        if opts.extra_index_urls:
233            finder.index_urls.extend(opts.extra_index_urls)
234        if opts.find_links:
235            # FIXME: it would be nice to keep track of the source
236            # of the find_links: support a find-links local path
237            # relative to a requirements file.
238            value = opts.find_links[0]
239            req_dir = os.path.dirname(os.path.abspath(filename))
240            relative_to_reqs_file = os.path.join(req_dir, value)
241            if os.path.exists(relative_to_reqs_file):
242                value = relative_to_reqs_file
243            finder.find_links.append(value)
244        if opts.pre:
245            finder.allow_all_prereleases = True
246        if opts.process_dependency_links:
247            finder.process_dependency_links = True
248        if opts.trusted_hosts:
249            finder.secure_origins.extend(
250                ("*", host, "*") for host in opts.trusted_hosts)
251
252
253def break_args_options(line):
254    """Break up the line into an args and options string.  We only want to shlex
255    (and then optparse) the options, not the args.  args can contain markers
256    which are corrupted by shlex.
257    """
258    tokens = line.split(' ')
259    args = []
260    options = tokens[:]
261    for token in tokens:
262        if token.startswith('-') or token.startswith('--'):
263            break
264        else:
265            args.append(token)
266            options.pop(0)
267    return ' '.join(args), ' '.join(options)
268
269
270def build_parser():
271    """
272    Return a parser for parsing requirement lines
273    """
274    parser = optparse.OptionParser(add_help_option=False)
275
276    option_factories = SUPPORTED_OPTIONS + SUPPORTED_OPTIONS_REQ
277    for option_factory in option_factories:
278        option = option_factory()
279        parser.add_option(option)
280
281    # By default optparse sys.exits on parsing errors. We want to wrap
282    # that in our own exception.
283    def parser_exit(self, msg):
284        raise RequirementsFileParseError(msg)
285    parser.exit = parser_exit
286
287    return parser
288
289
290def join_lines(lines_enum):
291    """Joins a line ending in '\' with the previous line (except when following
292    comments).  The joined line takes on the index of the first line.
293    """
294    primary_line_number = None
295    new_line = []
296    for line_number, line in lines_enum:
297        if not line.endswith('\\') or COMMENT_RE.match(line):
298            if COMMENT_RE.match(line):
299                # this ensures comments are always matched later
300                line = ' ' + line
301            if new_line:
302                new_line.append(line)
303                yield primary_line_number, ''.join(new_line)
304                new_line = []
305            else:
306                yield line_number, line
307        else:
308            if not new_line:
309                primary_line_number = line_number
310            new_line.append(line.strip('\\'))
311
312    # last line contains \
313    if new_line:
314        yield primary_line_number, ''.join(new_line)
315
316    # TODO: handle space after '\'.
317
318
319def ignore_comments(lines_enum):
320    """
321    Strips comments and filter empty lines.
322    """
323    for line_number, line in lines_enum:
324        line = COMMENT_RE.sub('', line)
325        line = line.strip()
326        if line:
327            yield line_number, line
328
329
330def skip_regex(lines_enum, options):
331    """
332    Skip lines that match '--skip-requirements-regex' pattern
333
334    Note: the regex pattern is only built once
335    """
336    skip_regex = options.skip_requirements_regex if options else None
337    if skip_regex:
338        pattern = re.compile(skip_regex)
339        lines_enum = filterfalse(
340            lambda e: pattern.search(e[1]),
341            lines_enum)
342    return lines_enum
343