1# -*- coding: utf-8 -*- 2# 3# Copyright (C) 2013-2020 Vinay Sajip. 4# Licensed to the Python Software Foundation under a contributor agreement. 5# See LICENSE.txt and CONTRIBUTORS.txt. 6# 7from __future__ import unicode_literals 8 9import base64 10import codecs 11import datetime 12from email import message_from_file 13import hashlib 14import imp 15import json 16import logging 17import os 18import posixpath 19import re 20import shutil 21import sys 22import tempfile 23import zipfile 24 25from . import __version__, DistlibException 26from .compat import sysconfig, ZipFile, fsdecode, text_type, filter 27from .database import InstalledDistribution 28from .metadata import (Metadata, METADATA_FILENAME, WHEEL_METADATA_FILENAME, 29 LEGACY_METADATA_FILENAME) 30from .util import (FileOperator, convert_path, CSVReader, CSVWriter, Cache, 31 cached_property, get_cache_base, read_exports, tempdir, 32 get_platform) 33from .version import NormalizedVersion, UnsupportedVersionError 34 35logger = logging.getLogger(__name__) 36 37cache = None # created when needed 38 39if hasattr(sys, 'pypy_version_info'): # pragma: no cover 40 IMP_PREFIX = 'pp' 41elif sys.platform.startswith('java'): # pragma: no cover 42 IMP_PREFIX = 'jy' 43elif sys.platform == 'cli': # pragma: no cover 44 IMP_PREFIX = 'ip' 45else: 46 IMP_PREFIX = 'cp' 47 48VER_SUFFIX = sysconfig.get_config_var('py_version_nodot') 49if not VER_SUFFIX: # pragma: no cover 50 if sys.version_info[1] >= 10: 51 VER_SUFFIX = '%s_%s' % sys.version_info[:2] # PEP 641 (draft) 52 else: 53 VER_SUFFIX = '%s%s' % sys.version_info[:2] 54PYVER = 'py' + VER_SUFFIX 55IMPVER = IMP_PREFIX + VER_SUFFIX 56 57ARCH = get_platform().replace('-', '_').replace('.', '_') 58 59ABI = sysconfig.get_config_var('SOABI') 60if ABI and ABI.startswith('cpython-'): 61 ABI = ABI.replace('cpython-', 'cp').split('-')[0] 62else: 63 def _derive_abi(): 64 parts = ['cp', VER_SUFFIX] 65 if sysconfig.get_config_var('Py_DEBUG'): 66 parts.append('d') 67 if sysconfig.get_config_var('WITH_PYMALLOC'): 68 parts.append('m') 69 if sysconfig.get_config_var('Py_UNICODE_SIZE') == 4: 70 parts.append('u') 71 return ''.join(parts) 72 ABI = _derive_abi() 73 del _derive_abi 74 75FILENAME_RE = re.compile(r''' 76(?P<nm>[^-]+) 77-(?P<vn>\d+[^-]*) 78(-(?P<bn>\d+[^-]*))? 79-(?P<py>\w+\d+(\.\w+\d+)*) 80-(?P<bi>\w+) 81-(?P<ar>\w+(\.\w+)*) 82\.whl$ 83''', re.IGNORECASE | re.VERBOSE) 84 85NAME_VERSION_RE = re.compile(r''' 86(?P<nm>[^-]+) 87-(?P<vn>\d+[^-]*) 88(-(?P<bn>\d+[^-]*))?$ 89''', re.IGNORECASE | re.VERBOSE) 90 91SHEBANG_RE = re.compile(br'\s*#![^\r\n]*') 92SHEBANG_DETAIL_RE = re.compile(br'^(\s*#!("[^"]+"|\S+))\s+(.*)$') 93SHEBANG_PYTHON = b'#!python' 94SHEBANG_PYTHONW = b'#!pythonw' 95 96if os.sep == '/': 97 to_posix = lambda o: o 98else: 99 to_posix = lambda o: o.replace(os.sep, '/') 100 101 102class Mounter(object): 103 def __init__(self): 104 self.impure_wheels = {} 105 self.libs = {} 106 107 def add(self, pathname, extensions): 108 self.impure_wheels[pathname] = extensions 109 self.libs.update(extensions) 110 111 def remove(self, pathname): 112 extensions = self.impure_wheels.pop(pathname) 113 for k, v in extensions: 114 if k in self.libs: 115 del self.libs[k] 116 117 def find_module(self, fullname, path=None): 118 if fullname in self.libs: 119 result = self 120 else: 121 result = None 122 return result 123 124 def load_module(self, fullname): 125 if fullname in sys.modules: 126 result = sys.modules[fullname] 127 else: 128 if fullname not in self.libs: 129 raise ImportError('unable to find extension for %s' % fullname) 130 result = imp.load_dynamic(fullname, self.libs[fullname]) 131 result.__loader__ = self 132 parts = fullname.rsplit('.', 1) 133 if len(parts) > 1: 134 result.__package__ = parts[0] 135 return result 136 137_hook = Mounter() 138 139 140class Wheel(object): 141 """ 142 Class to build and install from Wheel files (PEP 427). 143 """ 144 145 wheel_version = (1, 1) 146 hash_kind = 'sha256' 147 148 def __init__(self, filename=None, sign=False, verify=False): 149 """ 150 Initialise an instance using a (valid) filename. 151 """ 152 self.sign = sign 153 self.should_verify = verify 154 self.buildver = '' 155 self.pyver = [PYVER] 156 self.abi = ['none'] 157 self.arch = ['any'] 158 self.dirname = os.getcwd() 159 if filename is None: 160 self.name = 'dummy' 161 self.version = '0.1' 162 self._filename = self.filename 163 else: 164 m = NAME_VERSION_RE.match(filename) 165 if m: 166 info = m.groupdict('') 167 self.name = info['nm'] 168 # Reinstate the local version separator 169 self.version = info['vn'].replace('_', '-') 170 self.buildver = info['bn'] 171 self._filename = self.filename 172 else: 173 dirname, filename = os.path.split(filename) 174 m = FILENAME_RE.match(filename) 175 if not m: 176 raise DistlibException('Invalid name or ' 177 'filename: %r' % filename) 178 if dirname: 179 self.dirname = os.path.abspath(dirname) 180 self._filename = filename 181 info = m.groupdict('') 182 self.name = info['nm'] 183 self.version = info['vn'] 184 self.buildver = info['bn'] 185 self.pyver = info['py'].split('.') 186 self.abi = info['bi'].split('.') 187 self.arch = info['ar'].split('.') 188 189 @property 190 def filename(self): 191 """ 192 Build and return a filename from the various components. 193 """ 194 if self.buildver: 195 buildver = '-' + self.buildver 196 else: 197 buildver = '' 198 pyver = '.'.join(self.pyver) 199 abi = '.'.join(self.abi) 200 arch = '.'.join(self.arch) 201 # replace - with _ as a local version separator 202 version = self.version.replace('-', '_') 203 return '%s-%s%s-%s-%s-%s.whl' % (self.name, version, buildver, 204 pyver, abi, arch) 205 206 @property 207 def exists(self): 208 path = os.path.join(self.dirname, self.filename) 209 return os.path.isfile(path) 210 211 @property 212 def tags(self): 213 for pyver in self.pyver: 214 for abi in self.abi: 215 for arch in self.arch: 216 yield pyver, abi, arch 217 218 @cached_property 219 def metadata(self): 220 pathname = os.path.join(self.dirname, self.filename) 221 name_ver = '%s-%s' % (self.name, self.version) 222 info_dir = '%s.dist-info' % name_ver 223 wrapper = codecs.getreader('utf-8') 224 with ZipFile(pathname, 'r') as zf: 225 wheel_metadata = self.get_wheel_metadata(zf) 226 wv = wheel_metadata['Wheel-Version'].split('.', 1) 227 file_version = tuple([int(i) for i in wv]) 228 # if file_version < (1, 1): 229 # fns = [WHEEL_METADATA_FILENAME, METADATA_FILENAME, 230 # LEGACY_METADATA_FILENAME] 231 # else: 232 # fns = [WHEEL_METADATA_FILENAME, METADATA_FILENAME] 233 fns = [WHEEL_METADATA_FILENAME, LEGACY_METADATA_FILENAME] 234 result = None 235 for fn in fns: 236 try: 237 metadata_filename = posixpath.join(info_dir, fn) 238 with zf.open(metadata_filename) as bf: 239 wf = wrapper(bf) 240 result = Metadata(fileobj=wf) 241 if result: 242 break 243 except KeyError: 244 pass 245 if not result: 246 raise ValueError('Invalid wheel, because metadata is ' 247 'missing: looked in %s' % ', '.join(fns)) 248 return result 249 250 def get_wheel_metadata(self, zf): 251 name_ver = '%s-%s' % (self.name, self.version) 252 info_dir = '%s.dist-info' % name_ver 253 metadata_filename = posixpath.join(info_dir, 'WHEEL') 254 with zf.open(metadata_filename) as bf: 255 wf = codecs.getreader('utf-8')(bf) 256 message = message_from_file(wf) 257 return dict(message) 258 259 @cached_property 260 def info(self): 261 pathname = os.path.join(self.dirname, self.filename) 262 with ZipFile(pathname, 'r') as zf: 263 result = self.get_wheel_metadata(zf) 264 return result 265 266 def process_shebang(self, data): 267 m = SHEBANG_RE.match(data) 268 if m: 269 end = m.end() 270 shebang, data_after_shebang = data[:end], data[end:] 271 # Preserve any arguments after the interpreter 272 if b'pythonw' in shebang.lower(): 273 shebang_python = SHEBANG_PYTHONW 274 else: 275 shebang_python = SHEBANG_PYTHON 276 m = SHEBANG_DETAIL_RE.match(shebang) 277 if m: 278 args = b' ' + m.groups()[-1] 279 else: 280 args = b'' 281 shebang = shebang_python + args 282 data = shebang + data_after_shebang 283 else: 284 cr = data.find(b'\r') 285 lf = data.find(b'\n') 286 if cr < 0 or cr > lf: 287 term = b'\n' 288 else: 289 if data[cr:cr + 2] == b'\r\n': 290 term = b'\r\n' 291 else: 292 term = b'\r' 293 data = SHEBANG_PYTHON + term + data 294 return data 295 296 def get_hash(self, data, hash_kind=None): 297 if hash_kind is None: 298 hash_kind = self.hash_kind 299 try: 300 hasher = getattr(hashlib, hash_kind) 301 except AttributeError: 302 raise DistlibException('Unsupported hash algorithm: %r' % hash_kind) 303 result = hasher(data).digest() 304 result = base64.urlsafe_b64encode(result).rstrip(b'=').decode('ascii') 305 return hash_kind, result 306 307 def write_record(self, records, record_path, base): 308 records = list(records) # make a copy, as mutated 309 p = to_posix(os.path.relpath(record_path, base)) 310 records.append((p, '', '')) 311 with CSVWriter(record_path) as writer: 312 for row in records: 313 writer.writerow(row) 314 315 def write_records(self, info, libdir, archive_paths): 316 records = [] 317 distinfo, info_dir = info 318 hasher = getattr(hashlib, self.hash_kind) 319 for ap, p in archive_paths: 320 with open(p, 'rb') as f: 321 data = f.read() 322 digest = '%s=%s' % self.get_hash(data) 323 size = os.path.getsize(p) 324 records.append((ap, digest, size)) 325 326 p = os.path.join(distinfo, 'RECORD') 327 self.write_record(records, p, libdir) 328 ap = to_posix(os.path.join(info_dir, 'RECORD')) 329 archive_paths.append((ap, p)) 330 331 def build_zip(self, pathname, archive_paths): 332 with ZipFile(pathname, 'w', zipfile.ZIP_DEFLATED) as zf: 333 for ap, p in archive_paths: 334 logger.debug('Wrote %s to %s in wheel', p, ap) 335 zf.write(p, ap) 336 337 def build(self, paths, tags=None, wheel_version=None): 338 """ 339 Build a wheel from files in specified paths, and use any specified tags 340 when determining the name of the wheel. 341 """ 342 if tags is None: 343 tags = {} 344 345 libkey = list(filter(lambda o: o in paths, ('purelib', 'platlib')))[0] 346 if libkey == 'platlib': 347 is_pure = 'false' 348 default_pyver = [IMPVER] 349 default_abi = [ABI] 350 default_arch = [ARCH] 351 else: 352 is_pure = 'true' 353 default_pyver = [PYVER] 354 default_abi = ['none'] 355 default_arch = ['any'] 356 357 self.pyver = tags.get('pyver', default_pyver) 358 self.abi = tags.get('abi', default_abi) 359 self.arch = tags.get('arch', default_arch) 360 361 libdir = paths[libkey] 362 363 name_ver = '%s-%s' % (self.name, self.version) 364 data_dir = '%s.data' % name_ver 365 info_dir = '%s.dist-info' % name_ver 366 367 archive_paths = [] 368 369 # First, stuff which is not in site-packages 370 for key in ('data', 'headers', 'scripts'): 371 if key not in paths: 372 continue 373 path = paths[key] 374 if os.path.isdir(path): 375 for root, dirs, files in os.walk(path): 376 for fn in files: 377 p = fsdecode(os.path.join(root, fn)) 378 rp = os.path.relpath(p, path) 379 ap = to_posix(os.path.join(data_dir, key, rp)) 380 archive_paths.append((ap, p)) 381 if key == 'scripts' and not p.endswith('.exe'): 382 with open(p, 'rb') as f: 383 data = f.read() 384 data = self.process_shebang(data) 385 with open(p, 'wb') as f: 386 f.write(data) 387 388 # Now, stuff which is in site-packages, other than the 389 # distinfo stuff. 390 path = libdir 391 distinfo = None 392 for root, dirs, files in os.walk(path): 393 if root == path: 394 # At the top level only, save distinfo for later 395 # and skip it for now 396 for i, dn in enumerate(dirs): 397 dn = fsdecode(dn) 398 if dn.endswith('.dist-info'): 399 distinfo = os.path.join(root, dn) 400 del dirs[i] 401 break 402 assert distinfo, '.dist-info directory expected, not found' 403 404 for fn in files: 405 # comment out next suite to leave .pyc files in 406 if fsdecode(fn).endswith(('.pyc', '.pyo')): 407 continue 408 p = os.path.join(root, fn) 409 rp = to_posix(os.path.relpath(p, path)) 410 archive_paths.append((rp, p)) 411 412 # Now distinfo. Assumed to be flat, i.e. os.listdir is enough. 413 files = os.listdir(distinfo) 414 for fn in files: 415 if fn not in ('RECORD', 'INSTALLER', 'SHARED', 'WHEEL'): 416 p = fsdecode(os.path.join(distinfo, fn)) 417 ap = to_posix(os.path.join(info_dir, fn)) 418 archive_paths.append((ap, p)) 419 420 wheel_metadata = [ 421 'Wheel-Version: %d.%d' % (wheel_version or self.wheel_version), 422 'Generator: distlib %s' % __version__, 423 'Root-Is-Purelib: %s' % is_pure, 424 ] 425 for pyver, abi, arch in self.tags: 426 wheel_metadata.append('Tag: %s-%s-%s' % (pyver, abi, arch)) 427 p = os.path.join(distinfo, 'WHEEL') 428 with open(p, 'w') as f: 429 f.write('\n'.join(wheel_metadata)) 430 ap = to_posix(os.path.join(info_dir, 'WHEEL')) 431 archive_paths.append((ap, p)) 432 433 # sort the entries by archive path. Not needed by any spec, but it 434 # keeps the archive listing and RECORD tidier than they would otherwise 435 # be. Use the number of path segments to keep directory entries together, 436 # and keep the dist-info stuff at the end. 437 def sorter(t): 438 ap = t[0] 439 n = ap.count('/') 440 if '.dist-info' in ap: 441 n += 10000 442 return (n, ap) 443 archive_paths = sorted(archive_paths, key=sorter) 444 445 # Now, at last, RECORD. 446 # Paths in here are archive paths - nothing else makes sense. 447 self.write_records((distinfo, info_dir), libdir, archive_paths) 448 # Now, ready to build the zip file 449 pathname = os.path.join(self.dirname, self.filename) 450 self.build_zip(pathname, archive_paths) 451 return pathname 452 453 def skip_entry(self, arcname): 454 """ 455 Determine whether an archive entry should be skipped when verifying 456 or installing. 457 """ 458 # The signature file won't be in RECORD, 459 # and we don't currently don't do anything with it 460 # We also skip directories, as they won't be in RECORD 461 # either. See: 462 # 463 # https://github.com/pypa/wheel/issues/294 464 # https://github.com/pypa/wheel/issues/287 465 # https://github.com/pypa/wheel/pull/289 466 # 467 return arcname.endswith(('/', '/RECORD.jws')) 468 469 def install(self, paths, maker, **kwargs): 470 """ 471 Install a wheel to the specified paths. If kwarg ``warner`` is 472 specified, it should be a callable, which will be called with two 473 tuples indicating the wheel version of this software and the wheel 474 version in the file, if there is a discrepancy in the versions. 475 This can be used to issue any warnings to raise any exceptions. 476 If kwarg ``lib_only`` is True, only the purelib/platlib files are 477 installed, and the headers, scripts, data and dist-info metadata are 478 not written. If kwarg ``bytecode_hashed_invalidation`` is True, written 479 bytecode will try to use file-hash based invalidation (PEP-552) on 480 supported interpreter versions (CPython 2.7+). 481 482 The return value is a :class:`InstalledDistribution` instance unless 483 ``options.lib_only`` is True, in which case the return value is ``None``. 484 """ 485 486 dry_run = maker.dry_run 487 warner = kwargs.get('warner') 488 lib_only = kwargs.get('lib_only', False) 489 bc_hashed_invalidation = kwargs.get('bytecode_hashed_invalidation', False) 490 491 pathname = os.path.join(self.dirname, self.filename) 492 name_ver = '%s-%s' % (self.name, self.version) 493 data_dir = '%s.data' % name_ver 494 info_dir = '%s.dist-info' % name_ver 495 496 metadata_name = posixpath.join(info_dir, LEGACY_METADATA_FILENAME) 497 wheel_metadata_name = posixpath.join(info_dir, 'WHEEL') 498 record_name = posixpath.join(info_dir, 'RECORD') 499 500 wrapper = codecs.getreader('utf-8') 501 502 with ZipFile(pathname, 'r') as zf: 503 with zf.open(wheel_metadata_name) as bwf: 504 wf = wrapper(bwf) 505 message = message_from_file(wf) 506 wv = message['Wheel-Version'].split('.', 1) 507 file_version = tuple([int(i) for i in wv]) 508 if (file_version != self.wheel_version) and warner: 509 warner(self.wheel_version, file_version) 510 511 if message['Root-Is-Purelib'] == 'true': 512 libdir = paths['purelib'] 513 else: 514 libdir = paths['platlib'] 515 516 records = {} 517 with zf.open(record_name) as bf: 518 with CSVReader(stream=bf) as reader: 519 for row in reader: 520 p = row[0] 521 records[p] = row 522 523 data_pfx = posixpath.join(data_dir, '') 524 info_pfx = posixpath.join(info_dir, '') 525 script_pfx = posixpath.join(data_dir, 'scripts', '') 526 527 # make a new instance rather than a copy of maker's, 528 # as we mutate it 529 fileop = FileOperator(dry_run=dry_run) 530 fileop.record = True # so we can rollback if needed 531 532 bc = not sys.dont_write_bytecode # Double negatives. Lovely! 533 534 outfiles = [] # for RECORD writing 535 536 # for script copying/shebang processing 537 workdir = tempfile.mkdtemp() 538 # set target dir later 539 # we default add_launchers to False, as the 540 # Python Launcher should be used instead 541 maker.source_dir = workdir 542 maker.target_dir = None 543 try: 544 for zinfo in zf.infolist(): 545 arcname = zinfo.filename 546 if isinstance(arcname, text_type): 547 u_arcname = arcname 548 else: 549 u_arcname = arcname.decode('utf-8') 550 if self.skip_entry(u_arcname): 551 continue 552 row = records[u_arcname] 553 if row[2] and str(zinfo.file_size) != row[2]: 554 raise DistlibException('size mismatch for ' 555 '%s' % u_arcname) 556 if row[1]: 557 kind, value = row[1].split('=', 1) 558 with zf.open(arcname) as bf: 559 data = bf.read() 560 _, digest = self.get_hash(data, kind) 561 if digest != value: 562 raise DistlibException('digest mismatch for ' 563 '%s' % arcname) 564 565 if lib_only and u_arcname.startswith((info_pfx, data_pfx)): 566 logger.debug('lib_only: skipping %s', u_arcname) 567 continue 568 is_script = (u_arcname.startswith(script_pfx) 569 and not u_arcname.endswith('.exe')) 570 571 if u_arcname.startswith(data_pfx): 572 _, where, rp = u_arcname.split('/', 2) 573 outfile = os.path.join(paths[where], convert_path(rp)) 574 else: 575 # meant for site-packages. 576 if u_arcname in (wheel_metadata_name, record_name): 577 continue 578 outfile = os.path.join(libdir, convert_path(u_arcname)) 579 if not is_script: 580 with zf.open(arcname) as bf: 581 fileop.copy_stream(bf, outfile) 582 # Issue #147: permission bits aren't preserved. Using 583 # zf.extract(zinfo, libdir) should have worked, but didn't, 584 # see https://www.thetopsites.net/article/53834422.shtml 585 # So ... manually preserve permission bits as given in zinfo 586 if os.name == 'posix': 587 # just set the normal permission bits 588 os.chmod(outfile, (zinfo.external_attr >> 16) & 0x1FF) 589 outfiles.append(outfile) 590 # Double check the digest of the written file 591 if not dry_run and row[1]: 592 with open(outfile, 'rb') as bf: 593 data = bf.read() 594 _, newdigest = self.get_hash(data, kind) 595 if newdigest != digest: 596 raise DistlibException('digest mismatch ' 597 'on write for ' 598 '%s' % outfile) 599 if bc and outfile.endswith('.py'): 600 try: 601 pyc = fileop.byte_compile(outfile, 602 hashed_invalidation=bc_hashed_invalidation) 603 outfiles.append(pyc) 604 except Exception: 605 # Don't give up if byte-compilation fails, 606 # but log it and perhaps warn the user 607 logger.warning('Byte-compilation failed', 608 exc_info=True) 609 else: 610 fn = os.path.basename(convert_path(arcname)) 611 workname = os.path.join(workdir, fn) 612 with zf.open(arcname) as bf: 613 fileop.copy_stream(bf, workname) 614 615 dn, fn = os.path.split(outfile) 616 maker.target_dir = dn 617 filenames = maker.make(fn) 618 fileop.set_executable_mode(filenames) 619 outfiles.extend(filenames) 620 621 if lib_only: 622 logger.debug('lib_only: returning None') 623 dist = None 624 else: 625 # Generate scripts 626 627 # Try to get pydist.json so we can see if there are 628 # any commands to generate. If this fails (e.g. because 629 # of a legacy wheel), log a warning but don't give up. 630 commands = None 631 file_version = self.info['Wheel-Version'] 632 if file_version == '1.0': 633 # Use legacy info 634 ep = posixpath.join(info_dir, 'entry_points.txt') 635 try: 636 with zf.open(ep) as bwf: 637 epdata = read_exports(bwf) 638 commands = {} 639 for key in ('console', 'gui'): 640 k = '%s_scripts' % key 641 if k in epdata: 642 commands['wrap_%s' % key] = d = {} 643 for v in epdata[k].values(): 644 s = '%s:%s' % (v.prefix, v.suffix) 645 if v.flags: 646 s += ' [%s]' % ','.join(v.flags) 647 d[v.name] = s 648 except Exception: 649 logger.warning('Unable to read legacy script ' 650 'metadata, so cannot generate ' 651 'scripts') 652 else: 653 try: 654 with zf.open(metadata_name) as bwf: 655 wf = wrapper(bwf) 656 commands = json.load(wf).get('extensions') 657 if commands: 658 commands = commands.get('python.commands') 659 except Exception: 660 logger.warning('Unable to read JSON metadata, so ' 661 'cannot generate scripts') 662 if commands: 663 console_scripts = commands.get('wrap_console', {}) 664 gui_scripts = commands.get('wrap_gui', {}) 665 if console_scripts or gui_scripts: 666 script_dir = paths.get('scripts', '') 667 if not os.path.isdir(script_dir): 668 raise ValueError('Valid script path not ' 669 'specified') 670 maker.target_dir = script_dir 671 for k, v in console_scripts.items(): 672 script = '%s = %s' % (k, v) 673 filenames = maker.make(script) 674 fileop.set_executable_mode(filenames) 675 676 if gui_scripts: 677 options = {'gui': True } 678 for k, v in gui_scripts.items(): 679 script = '%s = %s' % (k, v) 680 filenames = maker.make(script, options) 681 fileop.set_executable_mode(filenames) 682 683 p = os.path.join(libdir, info_dir) 684 dist = InstalledDistribution(p) 685 686 # Write SHARED 687 paths = dict(paths) # don't change passed in dict 688 del paths['purelib'] 689 del paths['platlib'] 690 paths['lib'] = libdir 691 p = dist.write_shared_locations(paths, dry_run) 692 if p: 693 outfiles.append(p) 694 695 # Write RECORD 696 dist.write_installed_files(outfiles, paths['prefix'], 697 dry_run) 698 return dist 699 except Exception: # pragma: no cover 700 logger.exception('installation failed.') 701 fileop.rollback() 702 raise 703 finally: 704 shutil.rmtree(workdir) 705 706 def _get_dylib_cache(self): 707 global cache 708 if cache is None: 709 # Use native string to avoid issues on 2.x: see Python #20140. 710 base = os.path.join(get_cache_base(), str('dylib-cache'), 711 '%s.%s' % sys.version_info[:2]) 712 cache = Cache(base) 713 return cache 714 715 def _get_extensions(self): 716 pathname = os.path.join(self.dirname, self.filename) 717 name_ver = '%s-%s' % (self.name, self.version) 718 info_dir = '%s.dist-info' % name_ver 719 arcname = posixpath.join(info_dir, 'EXTENSIONS') 720 wrapper = codecs.getreader('utf-8') 721 result = [] 722 with ZipFile(pathname, 'r') as zf: 723 try: 724 with zf.open(arcname) as bf: 725 wf = wrapper(bf) 726 extensions = json.load(wf) 727 cache = self._get_dylib_cache() 728 prefix = cache.prefix_to_dir(pathname) 729 cache_base = os.path.join(cache.base, prefix) 730 if not os.path.isdir(cache_base): 731 os.makedirs(cache_base) 732 for name, relpath in extensions.items(): 733 dest = os.path.join(cache_base, convert_path(relpath)) 734 if not os.path.exists(dest): 735 extract = True 736 else: 737 file_time = os.stat(dest).st_mtime 738 file_time = datetime.datetime.fromtimestamp(file_time) 739 info = zf.getinfo(relpath) 740 wheel_time = datetime.datetime(*info.date_time) 741 extract = wheel_time > file_time 742 if extract: 743 zf.extract(relpath, cache_base) 744 result.append((name, dest)) 745 except KeyError: 746 pass 747 return result 748 749 def is_compatible(self): 750 """ 751 Determine if a wheel is compatible with the running system. 752 """ 753 return is_compatible(self) 754 755 def is_mountable(self): 756 """ 757 Determine if a wheel is asserted as mountable by its metadata. 758 """ 759 return True # for now - metadata details TBD 760 761 def mount(self, append=False): 762 pathname = os.path.abspath(os.path.join(self.dirname, self.filename)) 763 if not self.is_compatible(): 764 msg = 'Wheel %s not compatible with this Python.' % pathname 765 raise DistlibException(msg) 766 if not self.is_mountable(): 767 msg = 'Wheel %s is marked as not mountable.' % pathname 768 raise DistlibException(msg) 769 if pathname in sys.path: 770 logger.debug('%s already in path', pathname) 771 else: 772 if append: 773 sys.path.append(pathname) 774 else: 775 sys.path.insert(0, pathname) 776 extensions = self._get_extensions() 777 if extensions: 778 if _hook not in sys.meta_path: 779 sys.meta_path.append(_hook) 780 _hook.add(pathname, extensions) 781 782 def unmount(self): 783 pathname = os.path.abspath(os.path.join(self.dirname, self.filename)) 784 if pathname not in sys.path: 785 logger.debug('%s not in path', pathname) 786 else: 787 sys.path.remove(pathname) 788 if pathname in _hook.impure_wheels: 789 _hook.remove(pathname) 790 if not _hook.impure_wheels: 791 if _hook in sys.meta_path: 792 sys.meta_path.remove(_hook) 793 794 def verify(self): 795 pathname = os.path.join(self.dirname, self.filename) 796 name_ver = '%s-%s' % (self.name, self.version) 797 data_dir = '%s.data' % name_ver 798 info_dir = '%s.dist-info' % name_ver 799 800 metadata_name = posixpath.join(info_dir, LEGACY_METADATA_FILENAME) 801 wheel_metadata_name = posixpath.join(info_dir, 'WHEEL') 802 record_name = posixpath.join(info_dir, 'RECORD') 803 804 wrapper = codecs.getreader('utf-8') 805 806 with ZipFile(pathname, 'r') as zf: 807 with zf.open(wheel_metadata_name) as bwf: 808 wf = wrapper(bwf) 809 message = message_from_file(wf) 810 wv = message['Wheel-Version'].split('.', 1) 811 file_version = tuple([int(i) for i in wv]) 812 # TODO version verification 813 814 records = {} 815 with zf.open(record_name) as bf: 816 with CSVReader(stream=bf) as reader: 817 for row in reader: 818 p = row[0] 819 records[p] = row 820 821 for zinfo in zf.infolist(): 822 arcname = zinfo.filename 823 if isinstance(arcname, text_type): 824 u_arcname = arcname 825 else: 826 u_arcname = arcname.decode('utf-8') 827 # See issue #115: some wheels have .. in their entries, but 828 # in the filename ... e.g. __main__..py ! So the check is 829 # updated to look for .. in the directory portions 830 p = u_arcname.split('/') 831 if '..' in p: 832 raise DistlibException('invalid entry in ' 833 'wheel: %r' % u_arcname) 834 835 if self.skip_entry(u_arcname): 836 continue 837 row = records[u_arcname] 838 if row[2] and str(zinfo.file_size) != row[2]: 839 raise DistlibException('size mismatch for ' 840 '%s' % u_arcname) 841 if row[1]: 842 kind, value = row[1].split('=', 1) 843 with zf.open(arcname) as bf: 844 data = bf.read() 845 _, digest = self.get_hash(data, kind) 846 if digest != value: 847 raise DistlibException('digest mismatch for ' 848 '%s' % arcname) 849 850 def update(self, modifier, dest_dir=None, **kwargs): 851 """ 852 Update the contents of a wheel in a generic way. The modifier should 853 be a callable which expects a dictionary argument: its keys are 854 archive-entry paths, and its values are absolute filesystem paths 855 where the contents the corresponding archive entries can be found. The 856 modifier is free to change the contents of the files pointed to, add 857 new entries and remove entries, before returning. This method will 858 extract the entire contents of the wheel to a temporary location, call 859 the modifier, and then use the passed (and possibly updated) 860 dictionary to write a new wheel. If ``dest_dir`` is specified, the new 861 wheel is written there -- otherwise, the original wheel is overwritten. 862 863 The modifier should return True if it updated the wheel, else False. 864 This method returns the same value the modifier returns. 865 """ 866 867 def get_version(path_map, info_dir): 868 version = path = None 869 key = '%s/%s' % (info_dir, LEGACY_METADATA_FILENAME) 870 if key not in path_map: 871 key = '%s/PKG-INFO' % info_dir 872 if key in path_map: 873 path = path_map[key] 874 version = Metadata(path=path).version 875 return version, path 876 877 def update_version(version, path): 878 updated = None 879 try: 880 v = NormalizedVersion(version) 881 i = version.find('-') 882 if i < 0: 883 updated = '%s+1' % version 884 else: 885 parts = [int(s) for s in version[i + 1:].split('.')] 886 parts[-1] += 1 887 updated = '%s+%s' % (version[:i], 888 '.'.join(str(i) for i in parts)) 889 except UnsupportedVersionError: 890 logger.debug('Cannot update non-compliant (PEP-440) ' 891 'version %r', version) 892 if updated: 893 md = Metadata(path=path) 894 md.version = updated 895 legacy = path.endswith(LEGACY_METADATA_FILENAME) 896 md.write(path=path, legacy=legacy) 897 logger.debug('Version updated from %r to %r', version, 898 updated) 899 900 pathname = os.path.join(self.dirname, self.filename) 901 name_ver = '%s-%s' % (self.name, self.version) 902 info_dir = '%s.dist-info' % name_ver 903 record_name = posixpath.join(info_dir, 'RECORD') 904 with tempdir() as workdir: 905 with ZipFile(pathname, 'r') as zf: 906 path_map = {} 907 for zinfo in zf.infolist(): 908 arcname = zinfo.filename 909 if isinstance(arcname, text_type): 910 u_arcname = arcname 911 else: 912 u_arcname = arcname.decode('utf-8') 913 if u_arcname == record_name: 914 continue 915 if '..' in u_arcname: 916 raise DistlibException('invalid entry in ' 917 'wheel: %r' % u_arcname) 918 zf.extract(zinfo, workdir) 919 path = os.path.join(workdir, convert_path(u_arcname)) 920 path_map[u_arcname] = path 921 922 # Remember the version. 923 original_version, _ = get_version(path_map, info_dir) 924 # Files extracted. Call the modifier. 925 modified = modifier(path_map, **kwargs) 926 if modified: 927 # Something changed - need to build a new wheel. 928 current_version, path = get_version(path_map, info_dir) 929 if current_version and (current_version == original_version): 930 # Add or update local version to signify changes. 931 update_version(current_version, path) 932 # Decide where the new wheel goes. 933 if dest_dir is None: 934 fd, newpath = tempfile.mkstemp(suffix='.whl', 935 prefix='wheel-update-', 936 dir=workdir) 937 os.close(fd) 938 else: 939 if not os.path.isdir(dest_dir): 940 raise DistlibException('Not a directory: %r' % dest_dir) 941 newpath = os.path.join(dest_dir, self.filename) 942 archive_paths = list(path_map.items()) 943 distinfo = os.path.join(workdir, info_dir) 944 info = distinfo, info_dir 945 self.write_records(info, workdir, archive_paths) 946 self.build_zip(newpath, archive_paths) 947 if dest_dir is None: 948 shutil.copyfile(newpath, pathname) 949 return modified 950 951def _get_glibc_version(): 952 import platform 953 ver = platform.libc_ver() 954 result = [] 955 if ver[0] == 'glibc': 956 for s in ver[1].split('.'): 957 result.append(int(s) if s.isdigit() else 0) 958 result = tuple(result) 959 return result 960 961def compatible_tags(): 962 """ 963 Return (pyver, abi, arch) tuples compatible with this Python. 964 """ 965 versions = [VER_SUFFIX] 966 major = VER_SUFFIX[0] 967 for minor in range(sys.version_info[1] - 1, - 1, -1): 968 versions.append(''.join([major, str(minor)])) 969 970 abis = [] 971 for suffix, _, _ in imp.get_suffixes(): 972 if suffix.startswith('.abi'): 973 abis.append(suffix.split('.', 2)[1]) 974 abis.sort() 975 if ABI != 'none': 976 abis.insert(0, ABI) 977 abis.append('none') 978 result = [] 979 980 arches = [ARCH] 981 if sys.platform == 'darwin': 982 m = re.match(r'(\w+)_(\d+)_(\d+)_(\w+)$', ARCH) 983 if m: 984 name, major, minor, arch = m.groups() 985 minor = int(minor) 986 matches = [arch] 987 if arch in ('i386', 'ppc'): 988 matches.append('fat') 989 if arch in ('i386', 'ppc', 'x86_64'): 990 matches.append('fat3') 991 if arch in ('ppc64', 'x86_64'): 992 matches.append('fat64') 993 if arch in ('i386', 'x86_64'): 994 matches.append('intel') 995 if arch in ('i386', 'x86_64', 'intel', 'ppc', 'ppc64'): 996 matches.append('universal') 997 while minor >= 0: 998 for match in matches: 999 s = '%s_%s_%s_%s' % (name, major, minor, match) 1000 if s != ARCH: # already there 1001 arches.append(s) 1002 minor -= 1 1003 1004 # Most specific - our Python version, ABI and arch 1005 for abi in abis: 1006 for arch in arches: 1007 result.append((''.join((IMP_PREFIX, versions[0])), abi, arch)) 1008 # manylinux 1009 if abi != 'none' and sys.platform.startswith('linux'): 1010 arch = arch.replace('linux_', '') 1011 parts = _get_glibc_version() 1012 if len(parts) == 2: 1013 if parts >= (2, 5): 1014 result.append((''.join((IMP_PREFIX, versions[0])), abi, 1015 'manylinux1_%s' % arch)) 1016 if parts >= (2, 12): 1017 result.append((''.join((IMP_PREFIX, versions[0])), abi, 1018 'manylinux2010_%s' % arch)) 1019 if parts >= (2, 17): 1020 result.append((''.join((IMP_PREFIX, versions[0])), abi, 1021 'manylinux2014_%s' % arch)) 1022 result.append((''.join((IMP_PREFIX, versions[0])), abi, 1023 'manylinux_%s_%s_%s' % (parts[0], parts[1], 1024 arch))) 1025 1026 # where no ABI / arch dependency, but IMP_PREFIX dependency 1027 for i, version in enumerate(versions): 1028 result.append((''.join((IMP_PREFIX, version)), 'none', 'any')) 1029 if i == 0: 1030 result.append((''.join((IMP_PREFIX, version[0])), 'none', 'any')) 1031 1032 # no IMP_PREFIX, ABI or arch dependency 1033 for i, version in enumerate(versions): 1034 result.append((''.join(('py', version)), 'none', 'any')) 1035 if i == 0: 1036 result.append((''.join(('py', version[0])), 'none', 'any')) 1037 1038 return set(result) 1039 1040 1041COMPATIBLE_TAGS = compatible_tags() 1042 1043del compatible_tags 1044 1045 1046def is_compatible(wheel, tags=None): 1047 if not isinstance(wheel, Wheel): 1048 wheel = Wheel(wheel) # assume it's a filename 1049 result = False 1050 if tags is None: 1051 tags = COMPATIBLE_TAGS 1052 for ver, abi, arch in tags: 1053 if ver in wheel.pyver and abi in wheel.abi and arch in wheel.arch: 1054 result = True 1055 break 1056 return result 1057