1# coding: utf-8 2from __future__ import absolute_import, division, print_function, unicode_literals 3 4import sys 5from collections import OrderedDict 6from itertools import chain 7 8from click.utils import LazyFile 9from pip._internal.req.constructors import install_req_from_line 10from pip._internal.utils.misc import redact_auth_from_url 11from pip._internal.vcs import is_url 12from pip._vendor import six 13from pip._vendor.six.moves import shlex_quote 14 15from .click import style 16 17UNSAFE_PACKAGES = {"setuptools", "distribute", "pip"} 18COMPILE_EXCLUDE_OPTIONS = { 19 "--dry-run", 20 "--quiet", 21 "--rebuild", 22 "--upgrade", 23 "--upgrade-package", 24 "--verbose", 25 "--cache-dir", 26 "--no-reuse-hashes", 27} 28 29 30def key_from_ireq(ireq): 31 """Get a standardized key for an InstallRequirement.""" 32 if ireq.req is None and ireq.link is not None: 33 return str(ireq.link) 34 else: 35 return key_from_req(ireq.req) 36 37 38def key_from_req(req): 39 """Get an all-lowercase version of the requirement's name.""" 40 if hasattr(req, "key"): 41 # from pkg_resources, such as installed dists for pip-sync 42 key = req.key 43 else: 44 # from packaging, such as install requirements from requirements.txt 45 key = req.name 46 47 key = key.replace("_", "-").lower() 48 return key 49 50 51def comment(text): 52 return style(text, fg="green") 53 54 55def make_install_requirement(name, version, extras, constraint=False): 56 # If no extras are specified, the extras string is blank 57 extras_string = "" 58 if extras: 59 # Sort extras for stability 60 extras_string = "[{}]".format(",".join(sorted(extras))) 61 62 return install_req_from_line( 63 str("{}{}=={}".format(name, extras_string, version)), constraint=constraint 64 ) 65 66 67def is_url_requirement(ireq): 68 """ 69 Return True if requirement was specified as a path or URL. 70 ireq.original_link will have been set by InstallRequirement.__init__ 71 """ 72 return bool(ireq.original_link) 73 74 75def format_requirement(ireq, marker=None, hashes=None): 76 """ 77 Generic formatter for pretty printing InstallRequirements to the terminal 78 in a less verbose way than using its `__str__` method. 79 """ 80 if ireq.editable: 81 line = "-e {}".format(ireq.link.url) 82 elif is_url_requirement(ireq): 83 line = ireq.link.url 84 else: 85 line = str(ireq.req).lower() 86 87 if marker: 88 line = "{} ; {}".format(line, marker) 89 90 if hashes: 91 for hash_ in sorted(hashes): 92 line += " \\\n --hash={}".format(hash_) 93 94 return line 95 96 97def format_specifier(ireq): 98 """ 99 Generic formatter for pretty printing the specifier part of 100 InstallRequirements to the terminal. 101 """ 102 # TODO: Ideally, this is carried over to the pip library itself 103 specs = ireq.specifier if ireq.req is not None else [] 104 specs = sorted(specs, key=lambda x: x.version) 105 return ",".join(str(s) for s in specs) or "<any>" 106 107 108def is_pinned_requirement(ireq): 109 """ 110 Returns whether an InstallRequirement is a "pinned" requirement. 111 112 An InstallRequirement is considered pinned if: 113 114 - Is not editable 115 - It has exactly one specifier 116 - That specifier is "==" 117 - The version does not contain a wildcard 118 119 Examples: 120 django==1.8 # pinned 121 django>1.8 # NOT pinned 122 django~=1.8 # NOT pinned 123 django==1.* # NOT pinned 124 """ 125 if ireq.editable: 126 return False 127 128 if ireq.req is None or len(ireq.specifier) != 1: 129 return False 130 131 spec = next(iter(ireq.specifier)) 132 return spec.operator in {"==", "==="} and not spec.version.endswith(".*") 133 134 135def as_tuple(ireq): 136 """ 137 Pulls out the (name: str, version:str, extras:(str)) tuple from 138 the pinned InstallRequirement. 139 """ 140 if not is_pinned_requirement(ireq): 141 raise TypeError("Expected a pinned InstallRequirement, got {}".format(ireq)) 142 143 name = key_from_ireq(ireq) 144 version = next(iter(ireq.specifier)).version 145 extras = tuple(sorted(ireq.extras)) 146 return name, version, extras 147 148 149def flat_map(fn, collection): 150 """Map a function over a collection and flatten the result by one-level""" 151 return chain.from_iterable(map(fn, collection)) 152 153 154def lookup_table(values, key=None, keyval=None, unique=False, use_lists=False): 155 """ 156 Builds a dict-based lookup table (index) elegantly. 157 158 Supports building normal and unique lookup tables. For example: 159 160 >>> assert lookup_table( 161 ... ['foo', 'bar', 'baz', 'qux', 'quux'], lambda s: s[0]) == { 162 ... 'b': {'bar', 'baz'}, 163 ... 'f': {'foo'}, 164 ... 'q': {'quux', 'qux'} 165 ... } 166 167 For key functions that uniquely identify values, set unique=True: 168 169 >>> assert lookup_table( 170 ... ['foo', 'bar', 'baz', 'qux', 'quux'], lambda s: s[0], 171 ... unique=True) == { 172 ... 'b': 'baz', 173 ... 'f': 'foo', 174 ... 'q': 'quux' 175 ... } 176 177 For the values represented as lists, set use_lists=True: 178 179 >>> assert lookup_table( 180 ... ['foo', 'bar', 'baz', 'qux', 'quux'], lambda s: s[0], 181 ... use_lists=True) == { 182 ... 'b': ['bar', 'baz'], 183 ... 'f': ['foo'], 184 ... 'q': ['qux', 'quux'] 185 ... } 186 187 The values of the resulting lookup table will be lists, not sets. 188 189 For extra power, you can even change the values while building up the LUT. 190 To do so, use the `keyval` function instead of the `key` arg: 191 192 >>> assert lookup_table( 193 ... ['foo', 'bar', 'baz', 'qux', 'quux'], 194 ... keyval=lambda s: (s[0], s[1:])) == { 195 ... 'b': {'ar', 'az'}, 196 ... 'f': {'oo'}, 197 ... 'q': {'uux', 'ux'} 198 ... } 199 200 """ 201 if keyval is None: 202 if key is None: 203 204 def keyval(v): 205 return v 206 207 else: 208 209 def keyval(v): 210 return (key(v), v) 211 212 if unique: 213 return dict(keyval(v) for v in values) 214 215 lut = {} 216 for value in values: 217 k, v = keyval(value) 218 try: 219 s = lut[k] 220 except KeyError: 221 if use_lists: 222 s = lut[k] = list() 223 else: 224 s = lut[k] = set() 225 if use_lists: 226 s.append(v) 227 else: 228 s.add(v) 229 return dict(lut) 230 231 232def dedup(iterable): 233 """Deduplicate an iterable object like iter(set(iterable)) but 234 order-preserved. 235 """ 236 return iter(OrderedDict.fromkeys(iterable)) 237 238 239def name_from_req(req): 240 """Get the name of the requirement""" 241 if hasattr(req, "project_name"): 242 # from pkg_resources, such as installed dists for pip-sync 243 return req.project_name 244 else: 245 # from packaging, such as install requirements from requirements.txt 246 return req.name 247 248 249def fs_str(string): 250 """ 251 Convert given string to a correctly encoded filesystem string. 252 253 On Python 2, if the input string is unicode, converts it to bytes 254 encoded with the filesystem encoding. 255 256 On Python 3 returns the string as is, since Python 3 uses unicode 257 paths and the input string shouldn't be bytes. 258 259 :type string: str|unicode 260 :rtype: str 261 """ 262 if isinstance(string, str): 263 return string 264 if isinstance(string, bytes): 265 raise TypeError("fs_str() argument must not be bytes") 266 return string.encode(_fs_encoding) 267 268 269_fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() 270 271 272def get_hashes_from_ireq(ireq): 273 """ 274 Given an InstallRequirement, return a list of string hashes in 275 the format "{algorithm}:{hash}". Return an empty list if there are no hashes 276 in the requirement options. 277 """ 278 result = [] 279 for algorithm, hexdigests in ireq.hash_options.items(): 280 for hash_ in hexdigests: 281 result.append("{}:{}".format(algorithm, hash_)) 282 return result 283 284 285def force_text(s): 286 """ 287 Return a string representing `s`. 288 """ 289 if s is None: 290 return "" 291 if not isinstance(s, six.string_types): 292 return six.text_type(s) 293 return s 294 295 296def get_compile_command(click_ctx): 297 """ 298 Returns a normalized compile command depending on cli context. 299 300 The command will be normalized by: 301 - expanding options short to long 302 - removing values that are already default 303 - sorting the arguments 304 - removing one-off arguments like '--upgrade' 305 - removing arguments that don't change build behaviour like '--verbose' 306 """ 307 from piptools.scripts.compile import cli 308 309 # Map of the compile cli options (option name -> click.Option) 310 compile_options = {option.name: option for option in cli.params} 311 312 left_args = [] 313 right_args = [] 314 315 for option_name, value in click_ctx.params.items(): 316 option = compile_options[option_name] 317 318 # Collect variadic args separately, they will be added 319 # at the end of the command later 320 if option.nargs < 0: 321 # These will necessarily be src_files 322 # Re-add click-stripped '--' if any start with '-' 323 if any(val.startswith("-") and val != "-" for val in value): 324 right_args.append("--") 325 right_args.extend([shlex_quote(force_text(val)) for val in value]) 326 continue 327 328 # Get the latest option name (usually it'll be a long name) 329 option_long_name = option.opts[-1] 330 331 # Exclude one-off options (--upgrade/--upgrade-package/--rebuild/...) 332 # or options that don't change compile behaviour (--verbose/--dry-run/...) 333 if option_long_name in COMPILE_EXCLUDE_OPTIONS: 334 continue 335 336 # Skip options without a value 337 if option.default is None and not value: 338 continue 339 340 # Skip options with a default value 341 if option.default == value: 342 continue 343 344 # Use a file name for file-like objects 345 if isinstance(value, LazyFile): 346 value = value.name 347 348 # Convert value to the list 349 if not isinstance(value, (tuple, list)): 350 value = [value] 351 352 for val in value: 353 # Flags don't have a value, thus add to args true or false option long name 354 if option.is_flag: 355 # If there are false-options, choose an option name depending on a value 356 if option.secondary_opts: 357 # Get the latest false-option 358 secondary_option_long_name = option.secondary_opts[-1] 359 arg = option_long_name if val else secondary_option_long_name 360 # There are no false-options, use true-option 361 else: 362 arg = option_long_name 363 left_args.append(shlex_quote(arg)) 364 # Append to args the option with a value 365 else: 366 if isinstance(val, six.string_types) and is_url(val): 367 val = redact_auth_from_url(val) 368 if option.name == "pip_args": 369 # shlex_quote would produce functional but noisily quoted results, 370 # e.g. --pip-args='--cache-dir='"'"'/tmp/with spaces'"'"'' 371 # Instead, we try to get more legible quoting via repr: 372 left_args.append( 373 "{option}={value}".format( 374 option=option_long_name, value=repr(fs_str(force_text(val))) 375 ) 376 ) 377 else: 378 left_args.append( 379 "{option}={value}".format( 380 option=option_long_name, value=shlex_quote(force_text(val)) 381 ) 382 ) 383 384 return " ".join(["pip-compile"] + sorted(left_args) + sorted(right_args)) 385