1# -*- coding: utf-8 -*-
2from __future__ import unicode_literals, absolute_import
3from collections import OrderedDict
4import re
5import yaml
6
7from io import StringIO
8
9from configparser import SafeConfigParser, NoOptionError
10
11
12from .regex import URL_REGEX, HASH_REGEX
13
14from .dependencies import DependencyFile, Dependency
15from packaging.requirements import Requirement as PackagingRequirement, InvalidRequirement
16from . import filetypes
17import toml
18from packaging.specifiers import SpecifierSet
19import json
20
21
22# this is a backport from setuptools 26.1
23def setuptools_parse_requirements_backport(strs):  # pragma: no cover
24    # Copyright (C) 2016 Jason R Coombs <jaraco@jaraco.com>
25    #
26    # Permission is hereby granted, free of charge, to any person obtaining a copy of
27    # this software and associated documentation files (the "Software"), to deal in
28    # the Software without restriction, including without limitation the rights to
29    # use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
30    # of the Software, and to permit persons to whom the Software is furnished to do
31    # so, subject to the following conditions:
32    #
33    # The above copyright notice and this permission notice shall be included in all
34    # copies or substantial portions of the Software.
35    #
36    # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
37    # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
38    # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
39    # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
40    # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
41    # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
42    # SOFTWARE.
43    """Yield ``Requirement`` objects for each specification in `strs`
44
45    `strs` must be a string, or a (possibly-nested) iterable thereof.
46    """
47    # create a steppable iterator, so we can handle \-continuations
48    def yield_lines(strs):
49        """Yield non-empty/non-comment lines of a string or sequence"""
50        if isinstance(strs, str):
51            for s in strs.splitlines():
52                s = s.strip()
53                # skip blank lines/comments
54                if s and not s.startswith('#'):
55                    yield s
56        else:
57            for ss in strs:
58                for s in yield_lines(ss):
59                    yield s
60    lines = iter(yield_lines(strs))
61
62    for line in lines:
63        # Drop comments -- a hash without a space may be in a URL.
64        if ' #' in line:
65            line = line[:line.find(' #')]
66        # If there is a line continuation, drop it, and append the next line.
67        if line.endswith('\\'):
68            line = line[:-2].strip()
69            line += next(lines)
70        yield PackagingRequirement(line)
71
72
73class RequirementsTXTLineParser(object):
74    """
75
76    """
77
78    @classmethod
79    def parse(cls, line):
80        """
81
82        :param line:
83        :return:
84        """
85        try:
86            # setuptools requires a space before the comment. If this isn't the case, add it.
87            if "\t#" in line:
88                parsed, = setuptools_parse_requirements_backport(line.replace("\t#", "\t #"))
89            else:
90                parsed, = setuptools_parse_requirements_backport(line)
91        except InvalidRequirement:
92            return None
93        dep = Dependency(
94            name=parsed.name,
95            specs=parsed.specifier,
96            line=line,
97            extras=parsed.extras,
98            dependency_type=filetypes.requirements_txt
99        )
100        return dep
101
102
103class Parser(object):
104    """
105
106    """
107
108    def __init__(self, obj):
109        """
110
111        :param obj:
112        """
113        self.obj = obj
114        self._lines = None
115
116    def iter_lines(self, lineno=0):
117        """
118
119        :param lineno:
120        :return:
121        """
122        for line in self.lines[lineno:]:
123            yield line
124
125    @property
126    def lines(self):
127        """
128
129        :return:
130        """
131        if self._lines is None:
132            self._lines = self.obj.content.splitlines()
133        return self._lines
134
135    @property
136    def is_marked_file(self):
137        """
138
139        :return:
140        """
141        for n, line in enumerate(self.iter_lines()):
142            for marker in self.obj.file_marker:
143                if marker in line:
144                    return True
145            if n >= 2:
146                break
147        return False
148
149    def is_marked_line(self, line):
150        """
151
152        :param line:
153        :return:
154        """
155        for marker in self.obj.line_marker:
156            if marker in line:
157                return True
158        return False
159
160    @classmethod
161    def parse_hashes(cls, line):
162        """
163
164        :param line:
165        :return:
166        """
167        hashes = []
168        for match in re.finditer(HASH_REGEX, line):
169            hashes.append(line[match.start():match.end()])
170        return re.sub(HASH_REGEX, "", line).strip(), hashes
171
172    @classmethod
173    def parse_index_server(cls, line):
174        """
175
176        :param line:
177        :return:
178        """
179        matches = URL_REGEX.findall(line)
180        if matches:
181            url = matches[0]
182            return url if url.endswith("/") else url + "/"
183        return None
184
185    @classmethod
186    def resolve_file(cls, file_path, line):
187        """
188
189        :param file_path:
190        :param line:
191        :return:
192        """
193        line = line.replace("-r ", "").replace("--requirement ", "")
194        parts = file_path.split("/")
195        if " #" in line:
196            line = line.split("#")[0].strip()
197        if len(parts) == 1:
198            return line
199        return "/".join(parts[:-1]) + "/" + line
200
201
202class RequirementsTXTParser(Parser):
203    """
204
205    """
206
207    def parse(self):
208        """
209        Parses a requirements.txt-like file
210        """
211        index_server = None
212        for num, line in enumerate(self.iter_lines()):
213            line = line.rstrip()
214            if not line:
215                continue
216            if line.startswith('#'):
217                # comments are lines that start with # only
218                continue
219            if line.startswith('-i') or \
220                line.startswith('--index-url') or \
221                line.startswith('--extra-index-url'):
222                # this file is using a private index server, try to parse it
223                index_server = self.parse_index_server(line)
224                continue
225            elif self.obj.path and (line.startswith('-r') or line.startswith('--requirement')):
226                self.obj.resolved_files.append(self.resolve_file(self.obj.path, line))
227            elif line.startswith('-f') or line.startswith('--find-links') or \
228                line.startswith('--no-index') or line.startswith('--allow-external') or \
229                line.startswith('--allow-unverified') or line.startswith('-Z') or \
230                line.startswith('--always-unzip'):
231                continue
232            elif self.is_marked_line(line):
233                continue
234            else:
235                try:
236
237                    parseable_line = line
238
239                    # multiline requirements are not parseable
240                    if "\\" in line:
241                        parseable_line = line.replace("\\", "")
242                        for next_line in self.iter_lines(num + 1):
243                            parseable_line += next_line.strip().replace("\\", "")
244                            line += "\n" + next_line
245                            if "\\" in next_line:
246                                continue
247                            break
248                        # ignore multiline requirements if they are marked
249                        if self.is_marked_line(parseable_line):
250                            continue
251
252                    hashes = []
253                    if "--hash" in parseable_line:
254                        parseable_line, hashes = Parser.parse_hashes(parseable_line)
255
256                    req = RequirementsTXTLineParser.parse(parseable_line)
257                    if req:
258                        req.hashes = hashes
259                        req.index_server = index_server
260                        # replace the requirements line with the 'real' line
261                        req.line = line
262                        self.obj.dependencies.append(req)
263                except ValueError:
264                    continue
265
266
267class ToxINIParser(Parser):
268    """
269
270    """
271
272    def parse(self):
273        """
274
275        :return:
276        """
277        parser = SafeConfigParser()
278        parser.readfp(StringIO(self.obj.content))
279        for section in parser.sections():
280            try:
281                content = parser.get(section=section, option="deps")
282                for n, line in enumerate(content.splitlines()):
283                    if self.is_marked_line(line):
284                        continue
285                    if line:
286                        req = RequirementsTXTLineParser.parse(line)
287                        if req:
288                            req.dependency_type = self.obj.file_type
289                            self.obj.dependencies.append(req)
290            except NoOptionError:
291                pass
292
293
294class CondaYMLParser(Parser):
295    """
296
297    """
298
299    def parse(self):
300        """
301
302        :return:
303        """
304        try:
305            data = yaml.safe_load(self.obj.content)
306            if data and 'dependencies' in data and isinstance(data['dependencies'], list):
307                for dep in data['dependencies']:
308                    if isinstance(dep, dict) and 'pip' in dep:
309                        for n, line in enumerate(dep['pip']):
310                            if self.is_marked_line(line):
311                                continue
312                            req = RequirementsTXTLineParser.parse(line)
313                            if req:
314                                req.dependency_type = self.obj.file_type
315                                self.obj.dependencies.append(req)
316        except yaml.YAMLError:
317            pass
318
319
320class PipfileParser(Parser):
321
322    def parse(self):
323        """
324        Parse a Pipfile (as seen in pipenv)
325        :return:
326        """
327        try:
328            data = toml.loads(self.obj.content, _dict=OrderedDict)
329            if data:
330                for package_type in ['packages', 'dev-packages']:
331                    if package_type in data:
332                        for name, specs in data[package_type].items():
333                            # skip on VCS dependencies
334                            if not isinstance(specs, str):
335                                continue
336                            if specs == '*':
337                                specs = ''
338                            self.obj.dependencies.append(
339                                Dependency(
340                                    name=name, specs=SpecifierSet(specs),
341                                    dependency_type=filetypes.pipfile,
342                                    line=''.join([name, specs]),
343                                    section=package_type
344                                )
345                            )
346        except (toml.TomlDecodeError, IndexError) as e:
347            pass
348
349class PipfileLockParser(Parser):
350
351    def parse(self):
352        """
353        Parse a Pipfile.lock (as seen in pipenv)
354        :return:
355        """
356        try:
357            data = json.loads(self.obj.content, object_pairs_hook=OrderedDict)
358            if data:
359                for package_type in ['default', 'develop']:
360                    if package_type in data:
361                        for name, meta in data[package_type].items():
362                            # skip VCS dependencies
363                            if 'version' not in meta:
364                                continue
365                            specs = meta['version']
366                            hashes = meta['hashes']
367                            self.obj.dependencies.append(
368                                Dependency(
369                                    name=name, specs=SpecifierSet(specs),
370                                    dependency_type=filetypes.pipfile_lock,
371                                    hashes=hashes,
372                                    line=''.join([name, specs]),
373                                    section=package_type
374                                )
375                            )
376        except ValueError:
377            pass
378
379
380class SetupCfgParser(Parser):
381    def parse(self):
382        parser = SafeConfigParser()
383        parser.readfp(StringIO(self.obj.content))
384        for section in parser.values():
385            if section.name == 'options':
386                options = 'install_requires', 'setup_requires', 'test_require'
387                for name in options:
388                    content = section.get(name)
389                    if not content:
390                        continue
391                    self._parse_content(content)
392            elif section.name == 'options.extras_require':
393                for content in section.values():
394                    self._parse_content(content)
395
396    def _parse_content(self, content):
397        for n, line in enumerate(content.splitlines()):
398            if self.is_marked_line(line):
399                continue
400            if line:
401                req = RequirementsTXTLineParser.parse(line)
402                if req:
403                    req.dependency_type = self.obj.file_type
404                    self.obj.dependencies.append(req)
405
406
407def parse(content, file_type=None, path=None, sha=None, marker=((), ()), parser=None):
408    """
409
410    :param content:
411    :param file_type:
412    :param path:
413    :param sha:
414    :param marker:
415    :param parser:
416    :return:
417    """
418    dep_file = DependencyFile(
419        content=content,
420        path=path,
421        sha=sha,
422        marker=marker,
423        file_type=file_type,
424        parser=parser
425    )
426
427    return dep_file.parse()
428