1""" 2Requirements file parsing 3""" 4 5from __future__ import absolute_import 6 7import os 8import re 9import shlex 10import sys 11import optparse 12import warnings 13 14from pip9._vendor.six.moves.urllib import parse as urllib_parse 15from pip9._vendor.six.moves import filterfalse 16 17import pip9 18from pip9.download import get_file_content 19from pip9.req.req_install import InstallRequirement 20from pip9.exceptions import (RequirementsFileParseError) 21from pip9.utils.deprecation import RemovedInPip10Warning 22from pip9 import cmdoptions 23 24__all__ = ['parse_requirements'] 25 26SCHEME_RE = re.compile(r'^(http|https|file):', re.I) 27COMMENT_RE = re.compile(r'(^|\s)+#.*$') 28 29SUPPORTED_OPTIONS = [ 30 cmdoptions.constraints, 31 cmdoptions.editable, 32 cmdoptions.requirements, 33 cmdoptions.no_index, 34 cmdoptions.index_url, 35 cmdoptions.find_links, 36 cmdoptions.extra_index_url, 37 cmdoptions.allow_external, 38 cmdoptions.allow_all_external, 39 cmdoptions.no_allow_external, 40 cmdoptions.allow_unsafe, 41 cmdoptions.no_allow_unsafe, 42 cmdoptions.use_wheel, 43 cmdoptions.no_use_wheel, 44 cmdoptions.always_unzip, 45 cmdoptions.no_binary, 46 cmdoptions.only_binary, 47 cmdoptions.pre, 48 cmdoptions.process_dependency_links, 49 cmdoptions.trusted_host, 50 cmdoptions.require_hashes, 51] 52 53# options to be passed to requirements 54SUPPORTED_OPTIONS_REQ = [ 55 cmdoptions.install_options, 56 cmdoptions.global_options, 57 cmdoptions.hash, 58] 59 60# the 'dest' string values 61SUPPORTED_OPTIONS_REQ_DEST = [o().dest for o in SUPPORTED_OPTIONS_REQ] 62 63 64def parse_requirements(filename, finder=None, comes_from=None, options=None, 65 session=None, constraint=False, wheel_cache=None): 66 """Parse a requirements file and yield InstallRequirement instances. 67 68 :param filename: Path or url of requirements file. 69 :param finder: Instance of pip9.index.PackageFinder. 70 :param comes_from: Origin description of requirements. 71 :param options: cli options. 72 :param session: Instance of pip9.download.PipSession. 73 :param constraint: If true, parsing a constraint file rather than 74 requirements file. 75 :param wheel_cache: Instance of pip9.wheel.WheelCache 76 """ 77 if session is None: 78 raise TypeError( 79 "parse_requirements() missing 1 required keyword argument: " 80 "'session'" 81 ) 82 83 _, content = get_file_content( 84 filename, comes_from=comes_from, session=session 85 ) 86 87 lines_enum = preprocess(content, options) 88 89 for line_number, line in lines_enum: 90 req_iter = process_line(line, filename, line_number, finder, 91 comes_from, options, session, wheel_cache, 92 constraint=constraint) 93 for req in req_iter: 94 yield req 95 96 97def preprocess(content, options): 98 """Split, filter, and join lines, and return a line iterator 99 100 :param content: the content of the requirements file 101 :param options: cli options 102 """ 103 lines_enum = enumerate(content.splitlines(), start=1) 104 lines_enum = join_lines(lines_enum) 105 lines_enum = ignore_comments(lines_enum) 106 lines_enum = skip_regex(lines_enum, options) 107 return lines_enum 108 109 110def process_line(line, filename, line_number, finder=None, comes_from=None, 111 options=None, session=None, wheel_cache=None, 112 constraint=False): 113 """Process a single requirements line; This can result in creating/yielding 114 requirements, or updating the finder. 115 116 For lines that contain requirements, the only options that have an effect 117 are from SUPPORTED_OPTIONS_REQ, and they are scoped to the 118 requirement. Other options from SUPPORTED_OPTIONS may be present, but are 119 ignored. 120 121 For lines that do not contain requirements, the only options that have an 122 effect are from SUPPORTED_OPTIONS. Options from SUPPORTED_OPTIONS_REQ may 123 be present, but are ignored. These lines may contain multiple options 124 (although our docs imply only one is supported), and all our parsed and 125 affect the finder. 126 127 :param constraint: If True, parsing a constraints file. 128 :param options: OptionParser options that we may update 129 """ 130 parser = build_parser() 131 defaults = parser.get_default_values() 132 defaults.index_url = None 133 if finder: 134 # `finder.format_control` will be updated during parsing 135 defaults.format_control = finder.format_control 136 args_str, options_str = break_args_options(line) 137 if sys.version_info < (2, 7, 3): 138 # Prior to 2.7.3, shlex cannot deal with unicode entries 139 options_str = options_str.encode('utf8') 140 opts, _ = parser.parse_args(shlex.split(options_str), defaults) 141 142 # preserve for the nested code path 143 line_comes_from = '%s %s (line %s)' % ( 144 '-c' if constraint else '-r', filename, line_number) 145 146 # yield a line requirement 147 if args_str: 148 isolated = options.isolated_mode if options else False 149 if options: 150 cmdoptions.check_install_build_global(options, opts) 151 # get the options that apply to requirements 152 req_options = {} 153 for dest in SUPPORTED_OPTIONS_REQ_DEST: 154 if dest in opts.__dict__ and opts.__dict__[dest]: 155 req_options[dest] = opts.__dict__[dest] 156 yield InstallRequirement.from_line( 157 args_str, line_comes_from, constraint=constraint, 158 isolated=isolated, options=req_options, wheel_cache=wheel_cache 159 ) 160 161 # yield an editable requirement 162 elif opts.editables: 163 isolated = options.isolated_mode if options else False 164 default_vcs = options.default_vcs if options else None 165 yield InstallRequirement.from_editable( 166 opts.editables[0], comes_from=line_comes_from, 167 constraint=constraint, default_vcs=default_vcs, isolated=isolated, 168 wheel_cache=wheel_cache 169 ) 170 171 # parse a nested requirements file 172 elif opts.requirements or opts.constraints: 173 if opts.requirements: 174 req_path = opts.requirements[0] 175 nested_constraint = False 176 else: 177 req_path = opts.constraints[0] 178 nested_constraint = True 179 # original file is over http 180 if SCHEME_RE.search(filename): 181 # do a url join so relative paths work 182 req_path = urllib_parse.urljoin(filename, req_path) 183 # original file and nested file are paths 184 elif not SCHEME_RE.search(req_path): 185 # do a join so relative paths work 186 req_path = os.path.join(os.path.dirname(filename), req_path) 187 # TODO: Why not use `comes_from='-r {} (line {})'` here as well? 188 parser = parse_requirements( 189 req_path, finder, comes_from, options, session, 190 constraint=nested_constraint, wheel_cache=wheel_cache 191 ) 192 for req in parser: 193 yield req 194 195 # percolate hash-checking option upward 196 elif opts.require_hashes: 197 options.require_hashes = opts.require_hashes 198 199 # set finder options 200 elif finder: 201 if opts.allow_external: 202 warnings.warn( 203 "--allow-external has been deprecated and will be removed in " 204 "the future. Due to changes in the repository protocol, it no " 205 "longer has any effect.", 206 RemovedInPip10Warning, 207 ) 208 209 if opts.allow_all_external: 210 warnings.warn( 211 "--allow-all-external has been deprecated and will be removed " 212 "in the future. Due to changes in the repository protocol, it " 213 "no longer has any effect.", 214 RemovedInPip10Warning, 215 ) 216 217 if opts.allow_unverified: 218 warnings.warn( 219 "--allow-unverified has been deprecated and will be removed " 220 "in the future. Due to changes in the repository protocol, it " 221 "no longer has any effect.", 222 RemovedInPip10Warning, 223 ) 224 225 if opts.index_url: 226 finder.index_urls = [opts.index_url] 227 if opts.use_wheel is False: 228 finder.use_wheel = False 229 pip9.index.fmt_ctl_no_use_wheel(finder.format_control) 230 if opts.no_index is True: 231 finder.index_urls = [] 232 if opts.extra_index_urls: 233 finder.index_urls.extend(opts.extra_index_urls) 234 if opts.find_links: 235 # FIXME: it would be nice to keep track of the source 236 # of the find_links: support a find-links local path 237 # relative to a requirements file. 238 value = opts.find_links[0] 239 req_dir = os.path.dirname(os.path.abspath(filename)) 240 relative_to_reqs_file = os.path.join(req_dir, value) 241 if os.path.exists(relative_to_reqs_file): 242 value = relative_to_reqs_file 243 finder.find_links.append(value) 244 if opts.pre: 245 finder.allow_all_prereleases = True 246 if opts.process_dependency_links: 247 finder.process_dependency_links = True 248 if opts.trusted_hosts: 249 finder.secure_origins.extend( 250 ("*", host, "*") for host in opts.trusted_hosts) 251 252 253def break_args_options(line): 254 """Break up the line into an args and options string. We only want to shlex 255 (and then optparse) the options, not the args. args can contain markers 256 which are corrupted by shlex. 257 """ 258 tokens = line.split(' ') 259 args = [] 260 options = tokens[:] 261 for token in tokens: 262 if token.startswith('-') or token.startswith('--'): 263 break 264 else: 265 args.append(token) 266 options.pop(0) 267 return ' '.join(args), ' '.join(options) 268 269 270def build_parser(): 271 """ 272 Return a parser for parsing requirement lines 273 """ 274 parser = optparse.OptionParser(add_help_option=False) 275 276 option_factories = SUPPORTED_OPTIONS + SUPPORTED_OPTIONS_REQ 277 for option_factory in option_factories: 278 option = option_factory() 279 parser.add_option(option) 280 281 # By default optparse sys.exits on parsing errors. We want to wrap 282 # that in our own exception. 283 def parser_exit(self, msg): 284 raise RequirementsFileParseError(msg) 285 parser.exit = parser_exit 286 287 return parser 288 289 290def join_lines(lines_enum): 291 """Joins a line ending in '\' with the previous line (except when following 292 comments). The joined line takes on the index of the first line. 293 """ 294 primary_line_number = None 295 new_line = [] 296 for line_number, line in lines_enum: 297 if not line.endswith('\\') or COMMENT_RE.match(line): 298 if COMMENT_RE.match(line): 299 # this ensures comments are always matched later 300 line = ' ' + line 301 if new_line: 302 new_line.append(line) 303 yield primary_line_number, ''.join(new_line) 304 new_line = [] 305 else: 306 yield line_number, line 307 else: 308 if not new_line: 309 primary_line_number = line_number 310 new_line.append(line.strip('\\')) 311 312 # last line contains \ 313 if new_line: 314 yield primary_line_number, ''.join(new_line) 315 316 # TODO: handle space after '\'. 317 318 319def ignore_comments(lines_enum): 320 """ 321 Strips comments and filter empty lines. 322 """ 323 for line_number, line in lines_enum: 324 line = COMMENT_RE.sub('', line) 325 line = line.strip() 326 if line: 327 yield line_number, line 328 329 330def skip_regex(lines_enum, options): 331 """ 332 Skip lines that match '--skip-requirements-regex' pattern 333 334 Note: the regex pattern is only built once 335 """ 336 skip_regex = options.skip_requirements_regex if options else None 337 if skip_regex: 338 pattern = re.compile(skip_regex) 339 lines_enum = filterfalse( 340 lambda e: pattern.search(e[1]), 341 lines_enum) 342 return lines_enum 343