1# -*- coding: utf-8 -*- 2from __future__ import unicode_literals, absolute_import 3from collections import OrderedDict 4import re 5import yaml 6 7from io import StringIO 8 9from configparser import SafeConfigParser, NoOptionError 10 11 12from .regex import URL_REGEX, HASH_REGEX 13 14from .dependencies import DependencyFile, Dependency 15from packaging.requirements import Requirement as PackagingRequirement, InvalidRequirement 16from . import filetypes 17import toml 18from packaging.specifiers import SpecifierSet 19import json 20 21 22# this is a backport from setuptools 26.1 23def setuptools_parse_requirements_backport(strs): # pragma: no cover 24 # Copyright (C) 2016 Jason R Coombs <jaraco@jaraco.com> 25 # 26 # Permission is hereby granted, free of charge, to any person obtaining a copy of 27 # this software and associated documentation files (the "Software"), to deal in 28 # the Software without restriction, including without limitation the rights to 29 # use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 30 # of the Software, and to permit persons to whom the Software is furnished to do 31 # so, subject to the following conditions: 32 # 33 # The above copyright notice and this permission notice shall be included in all 34 # copies or substantial portions of the Software. 35 # 36 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 37 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 38 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 39 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 40 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 41 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 42 # SOFTWARE. 43 """Yield ``Requirement`` objects for each specification in `strs` 44 45 `strs` must be a string, or a (possibly-nested) iterable thereof. 46 """ 47 # create a steppable iterator, so we can handle \-continuations 48 def yield_lines(strs): 49 """Yield non-empty/non-comment lines of a string or sequence""" 50 if isinstance(strs, str): 51 for s in strs.splitlines(): 52 s = s.strip() 53 # skip blank lines/comments 54 if s and not s.startswith('#'): 55 yield s 56 else: 57 for ss in strs: 58 for s in yield_lines(ss): 59 yield s 60 lines = iter(yield_lines(strs)) 61 62 for line in lines: 63 # Drop comments -- a hash without a space may be in a URL. 64 if ' #' in line: 65 line = line[:line.find(' #')] 66 # If there is a line continuation, drop it, and append the next line. 67 if line.endswith('\\'): 68 line = line[:-2].strip() 69 line += next(lines) 70 yield PackagingRequirement(line) 71 72 73class RequirementsTXTLineParser(object): 74 """ 75 76 """ 77 78 @classmethod 79 def parse(cls, line): 80 """ 81 82 :param line: 83 :return: 84 """ 85 try: 86 # setuptools requires a space before the comment. If this isn't the case, add it. 87 if "\t#" in line: 88 parsed, = setuptools_parse_requirements_backport(line.replace("\t#", "\t #")) 89 else: 90 parsed, = setuptools_parse_requirements_backport(line) 91 except InvalidRequirement: 92 return None 93 dep = Dependency( 94 name=parsed.name, 95 specs=parsed.specifier, 96 line=line, 97 extras=parsed.extras, 98 dependency_type=filetypes.requirements_txt 99 ) 100 return dep 101 102 103class Parser(object): 104 """ 105 106 """ 107 108 def __init__(self, obj): 109 """ 110 111 :param obj: 112 """ 113 self.obj = obj 114 self._lines = None 115 116 def iter_lines(self, lineno=0): 117 """ 118 119 :param lineno: 120 :return: 121 """ 122 for line in self.lines[lineno:]: 123 yield line 124 125 @property 126 def lines(self): 127 """ 128 129 :return: 130 """ 131 if self._lines is None: 132 self._lines = self.obj.content.splitlines() 133 return self._lines 134 135 @property 136 def is_marked_file(self): 137 """ 138 139 :return: 140 """ 141 for n, line in enumerate(self.iter_lines()): 142 for marker in self.obj.file_marker: 143 if marker in line: 144 return True 145 if n >= 2: 146 break 147 return False 148 149 def is_marked_line(self, line): 150 """ 151 152 :param line: 153 :return: 154 """ 155 for marker in self.obj.line_marker: 156 if marker in line: 157 return True 158 return False 159 160 @classmethod 161 def parse_hashes(cls, line): 162 """ 163 164 :param line: 165 :return: 166 """ 167 hashes = [] 168 for match in re.finditer(HASH_REGEX, line): 169 hashes.append(line[match.start():match.end()]) 170 return re.sub(HASH_REGEX, "", line).strip(), hashes 171 172 @classmethod 173 def parse_index_server(cls, line): 174 """ 175 176 :param line: 177 :return: 178 """ 179 matches = URL_REGEX.findall(line) 180 if matches: 181 url = matches[0] 182 return url if url.endswith("/") else url + "/" 183 return None 184 185 @classmethod 186 def resolve_file(cls, file_path, line): 187 """ 188 189 :param file_path: 190 :param line: 191 :return: 192 """ 193 line = line.replace("-r ", "").replace("--requirement ", "") 194 parts = file_path.split("/") 195 if " #" in line: 196 line = line.split("#")[0].strip() 197 if len(parts) == 1: 198 return line 199 return "/".join(parts[:-1]) + "/" + line 200 201 202class RequirementsTXTParser(Parser): 203 """ 204 205 """ 206 207 def parse(self): 208 """ 209 Parses a requirements.txt-like file 210 """ 211 index_server = None 212 for num, line in enumerate(self.iter_lines()): 213 line = line.rstrip() 214 if not line: 215 continue 216 if line.startswith('#'): 217 # comments are lines that start with # only 218 continue 219 if line.startswith('-i') or \ 220 line.startswith('--index-url') or \ 221 line.startswith('--extra-index-url'): 222 # this file is using a private index server, try to parse it 223 index_server = self.parse_index_server(line) 224 continue 225 elif self.obj.path and (line.startswith('-r') or line.startswith('--requirement')): 226 self.obj.resolved_files.append(self.resolve_file(self.obj.path, line)) 227 elif line.startswith('-f') or line.startswith('--find-links') or \ 228 line.startswith('--no-index') or line.startswith('--allow-external') or \ 229 line.startswith('--allow-unverified') or line.startswith('-Z') or \ 230 line.startswith('--always-unzip'): 231 continue 232 elif self.is_marked_line(line): 233 continue 234 else: 235 try: 236 237 parseable_line = line 238 239 # multiline requirements are not parseable 240 if "\\" in line: 241 parseable_line = line.replace("\\", "") 242 for next_line in self.iter_lines(num + 1): 243 parseable_line += next_line.strip().replace("\\", "") 244 line += "\n" + next_line 245 if "\\" in next_line: 246 continue 247 break 248 # ignore multiline requirements if they are marked 249 if self.is_marked_line(parseable_line): 250 continue 251 252 hashes = [] 253 if "--hash" in parseable_line: 254 parseable_line, hashes = Parser.parse_hashes(parseable_line) 255 256 req = RequirementsTXTLineParser.parse(parseable_line) 257 if req: 258 req.hashes = hashes 259 req.index_server = index_server 260 # replace the requirements line with the 'real' line 261 req.line = line 262 self.obj.dependencies.append(req) 263 except ValueError: 264 continue 265 266 267class ToxINIParser(Parser): 268 """ 269 270 """ 271 272 def parse(self): 273 """ 274 275 :return: 276 """ 277 parser = SafeConfigParser() 278 parser.readfp(StringIO(self.obj.content)) 279 for section in parser.sections(): 280 try: 281 content = parser.get(section=section, option="deps") 282 for n, line in enumerate(content.splitlines()): 283 if self.is_marked_line(line): 284 continue 285 if line: 286 req = RequirementsTXTLineParser.parse(line) 287 if req: 288 req.dependency_type = self.obj.file_type 289 self.obj.dependencies.append(req) 290 except NoOptionError: 291 pass 292 293 294class CondaYMLParser(Parser): 295 """ 296 297 """ 298 299 def parse(self): 300 """ 301 302 :return: 303 """ 304 try: 305 data = yaml.safe_load(self.obj.content) 306 if data and 'dependencies' in data and isinstance(data['dependencies'], list): 307 for dep in data['dependencies']: 308 if isinstance(dep, dict) and 'pip' in dep: 309 for n, line in enumerate(dep['pip']): 310 if self.is_marked_line(line): 311 continue 312 req = RequirementsTXTLineParser.parse(line) 313 if req: 314 req.dependency_type = self.obj.file_type 315 self.obj.dependencies.append(req) 316 except yaml.YAMLError: 317 pass 318 319 320class PipfileParser(Parser): 321 322 def parse(self): 323 """ 324 Parse a Pipfile (as seen in pipenv) 325 :return: 326 """ 327 try: 328 data = toml.loads(self.obj.content, _dict=OrderedDict) 329 if data: 330 for package_type in ['packages', 'dev-packages']: 331 if package_type in data: 332 for name, specs in data[package_type].items(): 333 # skip on VCS dependencies 334 if not isinstance(specs, str): 335 continue 336 if specs == '*': 337 specs = '' 338 self.obj.dependencies.append( 339 Dependency( 340 name=name, specs=SpecifierSet(specs), 341 dependency_type=filetypes.pipfile, 342 line=''.join([name, specs]), 343 section=package_type 344 ) 345 ) 346 except (toml.TomlDecodeError, IndexError) as e: 347 pass 348 349class PipfileLockParser(Parser): 350 351 def parse(self): 352 """ 353 Parse a Pipfile.lock (as seen in pipenv) 354 :return: 355 """ 356 try: 357 data = json.loads(self.obj.content, object_pairs_hook=OrderedDict) 358 if data: 359 for package_type in ['default', 'develop']: 360 if package_type in data: 361 for name, meta in data[package_type].items(): 362 # skip VCS dependencies 363 if 'version' not in meta: 364 continue 365 specs = meta['version'] 366 hashes = meta['hashes'] 367 self.obj.dependencies.append( 368 Dependency( 369 name=name, specs=SpecifierSet(specs), 370 dependency_type=filetypes.pipfile_lock, 371 hashes=hashes, 372 line=''.join([name, specs]), 373 section=package_type 374 ) 375 ) 376 except ValueError: 377 pass 378 379 380class SetupCfgParser(Parser): 381 def parse(self): 382 parser = SafeConfigParser() 383 parser.readfp(StringIO(self.obj.content)) 384 for section in parser.values(): 385 if section.name == 'options': 386 options = 'install_requires', 'setup_requires', 'test_require' 387 for name in options: 388 content = section.get(name) 389 if not content: 390 continue 391 self._parse_content(content) 392 elif section.name == 'options.extras_require': 393 for content in section.values(): 394 self._parse_content(content) 395 396 def _parse_content(self, content): 397 for n, line in enumerate(content.splitlines()): 398 if self.is_marked_line(line): 399 continue 400 if line: 401 req = RequirementsTXTLineParser.parse(line) 402 if req: 403 req.dependency_type = self.obj.file_type 404 self.obj.dependencies.append(req) 405 406 407def parse(content, file_type=None, path=None, sha=None, marker=((), ()), parser=None): 408 """ 409 410 :param content: 411 :param file_type: 412 :param path: 413 :param sha: 414 :param marker: 415 :param parser: 416 :return: 417 """ 418 dep_file = DependencyFile( 419 content=content, 420 path=path, 421 sha=sha, 422 marker=marker, 423 file_type=file_type, 424 parser=parser 425 ) 426 427 return dep_file.parse() 428