1# coding: utf-8 2from __future__ import absolute_import, division, print_function, unicode_literals 3 4import sys 5from collections import OrderedDict 6from itertools import chain, groupby 7 8import six 9from click.utils import LazyFile 10from six.moves import shlex_quote 11 12from ._compat import PIP_VERSION, InstallCommand, install_req_from_line 13from .click import style 14 15UNSAFE_PACKAGES = {"setuptools", "distribute", "pip"} 16COMPILE_EXCLUDE_OPTIONS = { 17 "--dry-run", 18 "--quiet", 19 "--rebuild", 20 "--upgrade", 21 "--upgrade-package", 22 "--verbose", 23 "--cache-dir", 24} 25 26 27def key_from_ireq(ireq): 28 """Get a standardized key for an InstallRequirement.""" 29 if ireq.req is None and ireq.link is not None: 30 return str(ireq.link) 31 else: 32 return key_from_req(ireq.req) 33 34 35def key_from_req(req): 36 """Get an all-lowercase version of the requirement's name.""" 37 if hasattr(req, "key"): 38 # from pkg_resources, such as installed dists for pip-sync 39 key = req.key 40 else: 41 # from packaging, such as install requirements from requirements.txt 42 key = req.name 43 44 key = key.replace("_", "-").lower() 45 return key 46 47 48def comment(text): 49 return style(text, fg="green") 50 51 52def make_install_requirement(name, version, extras, constraint=False): 53 # If no extras are specified, the extras string is blank 54 extras_string = "" 55 if extras: 56 # Sort extras for stability 57 extras_string = "[{}]".format(",".join(sorted(extras))) 58 59 return install_req_from_line( 60 str("{}{}=={}".format(name, extras_string, version)), constraint=constraint 61 ) 62 63 64def is_url_requirement(ireq): 65 """ 66 Return True if requirement was specified as a path or URL. 67 ireq.original_link will have been set by InstallRequirement.__init__ 68 """ 69 return bool(ireq.original_link) 70 71 72def format_requirement(ireq, marker=None, hashes=None): 73 """ 74 Generic formatter for pretty printing InstallRequirements to the terminal 75 in a less verbose way than using its `__str__` method. 76 """ 77 if ireq.editable: 78 line = "-e {}".format(ireq.link.url) 79 elif is_url_requirement(ireq): 80 line = ireq.link.url 81 else: 82 line = str(ireq.req).lower() 83 84 if marker: 85 line = "{} ; {}".format(line, marker) 86 87 if hashes: 88 for hash_ in sorted(hashes): 89 line += " \\\n --hash={}".format(hash_) 90 91 return line 92 93 94def format_specifier(ireq): 95 """ 96 Generic formatter for pretty printing the specifier part of 97 InstallRequirements to the terminal. 98 """ 99 # TODO: Ideally, this is carried over to the pip library itself 100 specs = ireq.specifier._specs if ireq.req is not None else [] 101 specs = sorted(specs, key=lambda x: x._spec[1]) 102 return ",".join(str(s) for s in specs) or "<any>" 103 104 105def is_pinned_requirement(ireq): 106 """ 107 Returns whether an InstallRequirement is a "pinned" requirement. 108 109 An InstallRequirement is considered pinned if: 110 111 - Is not editable 112 - It has exactly one specifier 113 - That specifier is "==" 114 - The version does not contain a wildcard 115 116 Examples: 117 django==1.8 # pinned 118 django>1.8 # NOT pinned 119 django~=1.8 # NOT pinned 120 django==1.* # NOT pinned 121 """ 122 if ireq.editable: 123 return False 124 125 if ireq.req is None or len(ireq.specifier._specs) != 1: 126 return False 127 128 op, version = next(iter(ireq.specifier._specs))._spec 129 return (op == "==" or op == "===") and not version.endswith(".*") 130 131 132def as_tuple(ireq): 133 """ 134 Pulls out the (name: str, version:str, extras:(str)) tuple from 135 the pinned InstallRequirement. 136 """ 137 if not is_pinned_requirement(ireq): 138 raise TypeError("Expected a pinned InstallRequirement, got {}".format(ireq)) 139 140 name = key_from_ireq(ireq) 141 version = next(iter(ireq.specifier._specs))._spec[1] 142 extras = tuple(sorted(ireq.extras)) 143 return name, version, extras 144 145 146def full_groupby(iterable, key=None): 147 """Like groupby(), but sorts the input on the group key first.""" 148 return groupby(sorted(iterable, key=key), key=key) 149 150 151def flat_map(fn, collection): 152 """Map a function over a collection and flatten the result by one-level""" 153 return chain.from_iterable(map(fn, collection)) 154 155 156def lookup_table(values, key=None, keyval=None, unique=False, use_lists=False): 157 """ 158 Builds a dict-based lookup table (index) elegantly. 159 160 Supports building normal and unique lookup tables. For example: 161 162 >>> assert lookup_table( 163 ... ['foo', 'bar', 'baz', 'qux', 'quux'], lambda s: s[0]) == { 164 ... 'b': {'bar', 'baz'}, 165 ... 'f': {'foo'}, 166 ... 'q': {'quux', 'qux'} 167 ... } 168 169 For key functions that uniquely identify values, set unique=True: 170 171 >>> assert lookup_table( 172 ... ['foo', 'bar', 'baz', 'qux', 'quux'], lambda s: s[0], 173 ... unique=True) == { 174 ... 'b': 'baz', 175 ... 'f': 'foo', 176 ... 'q': 'quux' 177 ... } 178 179 For the values represented as lists, set use_lists=True: 180 181 >>> assert lookup_table( 182 ... ['foo', 'bar', 'baz', 'qux', 'quux'], lambda s: s[0], 183 ... use_lists=True) == { 184 ... 'b': ['bar', 'baz'], 185 ... 'f': ['foo'], 186 ... 'q': ['qux', 'quux'] 187 ... } 188 189 The values of the resulting lookup table will be lists, not sets. 190 191 For extra power, you can even change the values while building up the LUT. 192 To do so, use the `keyval` function instead of the `key` arg: 193 194 >>> assert lookup_table( 195 ... ['foo', 'bar', 'baz', 'qux', 'quux'], 196 ... keyval=lambda s: (s[0], s[1:])) == { 197 ... 'b': {'ar', 'az'}, 198 ... 'f': {'oo'}, 199 ... 'q': {'uux', 'ux'} 200 ... } 201 202 """ 203 if keyval is None: 204 if key is None: 205 206 def keyval(v): 207 return v 208 209 else: 210 211 def keyval(v): 212 return (key(v), v) 213 214 if unique: 215 return dict(keyval(v) for v in values) 216 217 lut = {} 218 for value in values: 219 k, v = keyval(value) 220 try: 221 s = lut[k] 222 except KeyError: 223 if use_lists: 224 s = lut[k] = list() 225 else: 226 s = lut[k] = set() 227 if use_lists: 228 s.append(v) 229 else: 230 s.add(v) 231 return dict(lut) 232 233 234def dedup(iterable): 235 """Deduplicate an iterable object like iter(set(iterable)) but 236 order-reserved. 237 """ 238 return iter(OrderedDict.fromkeys(iterable)) 239 240 241def name_from_req(req): 242 """Get the name of the requirement""" 243 if hasattr(req, "project_name"): 244 # from pkg_resources, such as installed dists for pip-sync 245 return req.project_name 246 else: 247 # from packaging, such as install requirements from requirements.txt 248 return req.name 249 250 251def fs_str(string): 252 """ 253 Convert given string to a correctly encoded filesystem string. 254 255 On Python 2, if the input string is unicode, converts it to bytes 256 encoded with the filesystem encoding. 257 258 On Python 3 returns the string as is, since Python 3 uses unicode 259 paths and the input string shouldn't be bytes. 260 261 :type string: str|unicode 262 :rtype: str 263 """ 264 if isinstance(string, str): 265 return string 266 if isinstance(string, bytes): 267 raise AssertionError 268 return string.encode(_fs_encoding) 269 270 271_fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() 272 273 274def get_hashes_from_ireq(ireq): 275 """ 276 Given an InstallRequirement, return a list of string hashes in 277 the format "{algorithm}:{hash}". Return an empty list if there are no hashes 278 in the requirement options. 279 """ 280 result = [] 281 ireq_hashes = ireq.options.get("hashes", {}) 282 for algorithm, hexdigests in ireq_hashes.items(): 283 for hash_ in hexdigests: 284 result.append("{}:{}".format(algorithm, hash_)) 285 return result 286 287 288def force_text(s): 289 """ 290 Return a string representing `s`. 291 """ 292 if s is None: 293 return "" 294 if not isinstance(s, six.string_types): 295 return six.text_type(s) 296 return s 297 298 299def get_compile_command(click_ctx): 300 """ 301 Returns a normalized compile command depending on cli context. 302 303 The command will be normalized by: 304 - expanding options short to long 305 - removing values that are already default 306 - sorting the arguments 307 - removing one-off arguments like '--upgrade' 308 - removing arguments that don't change build behaviour like '--verbose' 309 """ 310 from piptools.scripts.compile import cli 311 312 # Map of the compile cli options (option name -> click.Option) 313 compile_options = {option.name: option for option in cli.params} 314 315 left_args = [] 316 right_args = [] 317 318 for option_name, value in click_ctx.params.items(): 319 option = compile_options[option_name] 320 321 # Get the latest option name (usually it'll be a long name) 322 option_long_name = option.opts[-1] 323 324 # Collect variadic args separately, they will be added 325 # at the end of the command later 326 if option.nargs < 0: 327 right_args.extend([shlex_quote(force_text(val)) for val in value]) 328 continue 329 330 # Exclude one-off options (--upgrade/--upgrade-package/--rebuild/...) 331 # or options that don't change compile behaviour (--verbose/--dry-run/...) 332 if option_long_name in COMPILE_EXCLUDE_OPTIONS: 333 continue 334 335 # Skip options without a value 336 if option.default is None and not value: 337 continue 338 339 # Skip options with a default value 340 if option.default == value: 341 continue 342 343 # Use a file name for file-like objects 344 if isinstance(value, LazyFile): 345 value = value.name 346 347 # Convert value to the list 348 if not isinstance(value, (tuple, list)): 349 value = [value] 350 351 for val in value: 352 # Flags don't have a value, thus add to args true or false option long name 353 if option.is_flag: 354 # If there are false-options, choose an option name depending on a value 355 if option.secondary_opts: 356 # Get the latest false-option 357 secondary_option_long_name = option.secondary_opts[-1] 358 arg = option_long_name if val else secondary_option_long_name 359 # There are no false-options, use true-option 360 else: 361 arg = option_long_name 362 left_args.append(shlex_quote(arg)) 363 # Append to args the option with a value 364 else: 365 left_args.append( 366 "{option}={value}".format( 367 option=option_long_name, value=shlex_quote(force_text(val)) 368 ) 369 ) 370 371 return " ".join(["pip-compile"] + sorted(left_args) + sorted(right_args)) 372 373 374def create_install_command(): 375 """ 376 Return an instance of InstallCommand. 377 """ 378 if PIP_VERSION < (19, 3): 379 return InstallCommand() 380 381 from pip._internal.commands import create_command 382 383 return create_command("install") 384 385 386def get_trusted_hosts(finder): 387 """ 388 Returns an iterable of trusted hosts from a given finder. 389 """ 390 if PIP_VERSION < (19, 2): 391 return (host for _, host, _ in finder.secure_origins) 392 393 return finder.trusted_hosts 394