1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5# This file contains code for populating the virtualenv environment for 6# Mozilla's build system. It is typically called as part of configure. 7 8from __future__ import absolute_import, print_function, unicode_literals 9 10import distutils.sysconfig 11import os 12import shutil 13import subprocess 14import sys 15import warnings 16 17from distutils.version import LooseVersion 18 19IS_NATIVE_WIN = (sys.platform == 'win32' and os.sep == '\\') 20IS_MSYS2 = (sys.platform == 'win32' and os.sep == '/') 21IS_CYGWIN = (sys.platform == 'cygwin') 22 23# Minimum version of Python required to build. 24MINIMUM_PYTHON_VERSION = LooseVersion('2.7.3') 25MINIMUM_PYTHON_MAJOR = 2 26 27 28UPGRADE_WINDOWS = ''' 29Please upgrade to the latest MozillaBuild development environment. See 30https://developer.mozilla.org/en-US/docs/Developer_Guide/Build_Instructions/Windows_Prerequisites 31'''.lstrip() 32 33UPGRADE_OTHER = ''' 34Run |mach bootstrap| to ensure your system is up to date. 35 36If you still receive this error, your shell environment is likely detecting 37another Python version. Ensure a modern Python can be found in the paths 38defined by the $PATH environment variable and try again. 39'''.lstrip() 40 41 42class VirtualenvManager(object): 43 """Contains logic for managing virtualenvs for building the tree.""" 44 45 def __init__(self, topsrcdir, topobjdir, virtualenv_path, log_handle, 46 manifest_path): 47 """Create a new manager. 48 49 Each manager is associated with a source directory, a path where you 50 want the virtualenv to be created, and a handle to write output to. 51 """ 52 assert os.path.isabs(manifest_path), "manifest_path must be an absolute path: %s" % (manifest_path) 53 self.topsrcdir = topsrcdir 54 self.topobjdir = topobjdir 55 self.virtualenv_root = virtualenv_path 56 57 # Record the Python executable that was used to create the Virtualenv 58 # so we can check this against sys.executable when verifying the 59 # integrity of the virtualenv. 60 self.exe_info_path = os.path.join(self.virtualenv_root, 61 'python_exe.txt') 62 63 self.log_handle = log_handle 64 self.manifest_path = manifest_path 65 66 @property 67 def virtualenv_script_path(self): 68 """Path to virtualenv's own populator script.""" 69 return os.path.join(self.topsrcdir, 'third_party', 'python', 70 'virtualenv', 'virtualenv.py') 71 72 @property 73 def bin_path(self): 74 # virtualenv.py provides a similar API via path_locations(). However, 75 # we have a bit of a chicken-and-egg problem and can't reliably 76 # import virtualenv. The functionality is trivial, so just implement 77 # it here. 78 if IS_CYGWIN or IS_NATIVE_WIN: 79 return os.path.join(self.virtualenv_root, 'Scripts') 80 81 return os.path.join(self.virtualenv_root, 'bin') 82 83 @property 84 def python_path(self): 85 binary = 'python' 86 if sys.platform in ('win32', 'cygwin'): 87 binary += '.exe' 88 89 return os.path.join(self.bin_path, binary) 90 91 @property 92 def activate_path(self): 93 return os.path.join(self.bin_path, 'activate_this.py') 94 95 def get_exe_info(self): 96 """Returns the version and file size of the python executable that was in 97 use when this virutalenv was created. 98 """ 99 with open(self.exe_info_path, 'r') as fh: 100 version, size = fh.read().splitlines() 101 return int(version), int(size) 102 103 def write_exe_info(self, python): 104 """Records the the version of the python executable that was in use when 105 this virutalenv was created. We record this explicitly because 106 on OS X our python path may end up being a different or modified 107 executable. 108 """ 109 ver = subprocess.check_output([python, '-c', 'import sys; print(sys.hexversion)']).rstrip() 110 with open(self.exe_info_path, 'w') as fh: 111 fh.write("%s\n" % ver) 112 fh.write("%s\n" % os.path.getsize(python)) 113 114 def up_to_date(self, python=sys.executable): 115 """Returns whether the virtualenv is present and up to date.""" 116 117 deps = [self.manifest_path, __file__] 118 119 # check if virtualenv exists 120 if not os.path.exists(self.virtualenv_root) or \ 121 not os.path.exists(self.activate_path): 122 123 return False 124 125 # check modification times 126 activate_mtime = os.path.getmtime(self.activate_path) 127 dep_mtime = max(os.path.getmtime(p) for p in deps) 128 if dep_mtime > activate_mtime: 129 return False 130 131 # Verify that the Python we're checking here is either the virutalenv 132 # python, or we have the Python version that was used to create the 133 # virtualenv. If this fails, it is likely system Python has been 134 # upgraded, and our virtualenv would not be usable. 135 python_size = os.path.getsize(python) 136 if ((python, python_size) != (self.python_path, os.path.getsize(self.python_path)) and 137 (sys.hexversion, python_size) != self.get_exe_info()): 138 return False 139 140 # recursively check sub packages.txt files 141 submanifests = [i[1] for i in self.packages() 142 if i[0] == 'packages.txt'] 143 for submanifest in submanifests: 144 submanifest = os.path.join(self.topsrcdir, submanifest) 145 submanager = VirtualenvManager(self.topsrcdir, 146 self.topobjdir, 147 self.virtualenv_root, 148 self.log_handle, 149 submanifest) 150 if not submanager.up_to_date(python): 151 return False 152 153 return True 154 155 def ensure(self, python=sys.executable): 156 """Ensure the virtualenv is present and up to date. 157 158 If the virtualenv is up to date, this does nothing. Otherwise, it 159 creates and populates the virtualenv as necessary. 160 161 This should be the main API used from this class as it is the 162 highest-level. 163 """ 164 if self.up_to_date(python): 165 return self.virtualenv_root 166 return self.build(python) 167 168 def _log_process_output(self, *args, **kwargs): 169 if hasattr(self.log_handle, 'fileno'): 170 return subprocess.call(*args, stdout=self.log_handle, 171 stderr=subprocess.STDOUT, **kwargs) 172 173 proc = subprocess.Popen(*args, stdout=subprocess.PIPE, 174 stderr=subprocess.STDOUT, **kwargs) 175 176 for line in proc.stdout: 177 self.log_handle.write(line) 178 179 return proc.wait() 180 181 def create(self, python=sys.executable): 182 """Create a new, empty virtualenv. 183 184 Receives the path to virtualenv's virtualenv.py script (which will be 185 called out to), the path to create the virtualenv in, and a handle to 186 write output to. 187 """ 188 env = dict(os.environ) 189 env.pop('PYTHONDONTWRITEBYTECODE', None) 190 191 args = [python, self.virtualenv_script_path, 192 # Without this, virtualenv.py may attempt to contact the outside 193 # world and search for or download a newer version of pip, 194 # setuptools, or wheel. This is bad for security, reproducibility, 195 # and speed. 196 '--no-download', 197 self.virtualenv_root] 198 199 result = self._log_process_output(args, env=env) 200 201 if result: 202 raise Exception( 203 'Failed to create virtualenv: %s' % self.virtualenv_root) 204 205 self.write_exe_info(python) 206 207 return self.virtualenv_root 208 209 def packages(self): 210 with file(self.manifest_path, 'rU') as fh: 211 packages = [line.rstrip().split(':') 212 for line in fh] 213 return packages 214 215 def populate(self): 216 """Populate the virtualenv. 217 218 The manifest file consists of colon-delimited fields. The first field 219 specifies the action. The remaining fields are arguments to that 220 action. The following actions are supported: 221 222 setup.py -- Invoke setup.py for a package. Expects the arguments: 223 1. relative path directory containing setup.py. 224 2. argument(s) to setup.py. e.g. "develop". Each program argument 225 is delimited by a colon. Arguments with colons are not yet 226 supported. 227 228 filename.pth -- Adds the path given as argument to filename.pth under 229 the virtualenv site packages directory. 230 231 optional -- This denotes the action as optional. The requested action 232 is attempted. If it fails, we issue a warning and go on. The 233 initial "optional" field is stripped then the remaining line is 234 processed like normal. e.g. 235 "optional:setup.py:python/foo:built_ext:-i" 236 237 copy -- Copies the given file in the virtualenv site packages 238 directory. 239 240 packages.txt -- Denotes that the specified path is a child manifest. It 241 will be read and processed as if its contents were concatenated 242 into the manifest being read. 243 244 objdir -- Denotes a relative path in the object directory to add to the 245 search path. e.g. "objdir:build" will add $topobjdir/build to the 246 search path. 247 248 Note that the Python interpreter running this function should be the 249 one from the virtualenv. If it is the system Python or if the 250 environment is not configured properly, packages could be installed 251 into the wrong place. This is how virtualenv's work. 252 """ 253 254 packages = self.packages() 255 python_lib = distutils.sysconfig.get_python_lib() 256 257 def handle_package(package): 258 if package[0] == 'setup.py': 259 assert len(package) >= 2 260 261 self.call_setup(os.path.join(self.topsrcdir, package[1]), 262 package[2:]) 263 264 return True 265 266 if package[0] == 'copy': 267 assert len(package) == 2 268 269 src = os.path.join(self.topsrcdir, package[1]) 270 dst = os.path.join(python_lib, os.path.basename(package[1])) 271 272 shutil.copy(src, dst) 273 274 return True 275 276 if package[0] == 'packages.txt': 277 assert len(package) == 2 278 279 src = os.path.join(self.topsrcdir, package[1]) 280 assert os.path.isfile(src), "'%s' does not exist" % src 281 submanager = VirtualenvManager(self.topsrcdir, 282 self.topobjdir, 283 self.virtualenv_root, 284 self.log_handle, 285 src) 286 submanager.populate() 287 288 return True 289 290 if package[0].endswith('.pth'): 291 assert len(package) == 2 292 293 path = os.path.join(self.topsrcdir, package[1]) 294 295 with open(os.path.join(python_lib, package[0]), 'a') as f: 296 # This path is relative to the .pth file. Using a 297 # relative path allows the srcdir/objdir combination 298 # to be moved around (as long as the paths relative to 299 # each other remain the same). 300 try: 301 f.write("%s\n" % os.path.relpath(path, python_lib)) 302 except ValueError: 303 # When objdir is on a separate drive, relpath throws 304 f.write("%s\n" % os.path.join(python_lib, path)) 305 306 return True 307 308 if package[0] == 'optional': 309 try: 310 handle_package(package[1:]) 311 return True 312 except: 313 print('Error processing command. Ignoring', \ 314 'because optional. (%s)' % ':'.join(package), 315 file=self.log_handle) 316 return False 317 318 if package[0] == 'objdir': 319 assert len(package) == 2 320 path = os.path.join(self.topobjdir, package[1]) 321 322 with open(os.path.join(python_lib, 'objdir.pth'), 'a') as f: 323 f.write('%s\n' % path) 324 325 return True 326 327 raise Exception('Unknown action: %s' % package[0]) 328 329 # We always target the OS X deployment target that Python itself was 330 # built with, regardless of what's in the current environment. If we 331 # don't do # this, we may run into a Python bug. See 332 # http://bugs.python.org/issue9516 and bug 659881. 333 # 334 # Note that this assumes that nothing compiled in the virtualenv is 335 # shipped as part of a distribution. If we do ship anything, the 336 # deployment target here may be different from what's targeted by the 337 # shipping binaries and # virtualenv-produced binaries may fail to 338 # work. 339 # 340 # We also ignore environment variables that may have been altered by 341 # configure or a mozconfig activated in the current shell. We trust 342 # Python is smart enough to find a proper compiler and to use the 343 # proper compiler flags. If it isn't your Python is likely broken. 344 IGNORE_ENV_VARIABLES = ('CC', 'CXX', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS', 345 'PYTHONDONTWRITEBYTECODE') 346 347 try: 348 old_target = os.environ.get('MACOSX_DEPLOYMENT_TARGET', None) 349 sysconfig_target = \ 350 distutils.sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') 351 352 if sysconfig_target is not None: 353 os.environ['MACOSX_DEPLOYMENT_TARGET'] = sysconfig_target 354 355 old_env_variables = {} 356 for k in IGNORE_ENV_VARIABLES: 357 if k not in os.environ: 358 continue 359 360 old_env_variables[k] = os.environ[k] 361 del os.environ[k] 362 363 # HACK ALERT. 364 # 365 # The following adjustment to the VSNNCOMNTOOLS environment 366 # variables are wrong. This is done as a hack to facilitate the 367 # building of binary Python packages - notably psutil - on Windows 368 # machines that don't have the Visual Studio 2008 binaries 369 # installed. This hack assumes the Python on that system was built 370 # with Visual Studio 2008. The hack is wrong for the reasons 371 # explained at 372 # http://stackoverflow.com/questions/3047542/building-lxml-for-python-2-7-on-windows/5122521#5122521. 373 if sys.platform in ('win32', 'cygwin') and \ 374 'VS90COMNTOOLS' not in os.environ: 375 376 warnings.warn('Hacking environment to allow binary Python ' 377 'extensions to build. You can make this warning go away ' 378 'by installing Visual Studio 2008. You can download the ' 379 'Express Edition installer from ' 380 'http://go.microsoft.com/?linkid=7729279') 381 382 # We list in order from oldest to newest to prefer the closest 383 # to 2008 so differences are minimized. 384 for ver in ('100', '110', '120'): 385 var = 'VS%sCOMNTOOLS' % ver 386 if var in os.environ: 387 os.environ['VS90COMNTOOLS'] = os.environ[var] 388 break 389 390 for package in packages: 391 handle_package(package) 392 393 sitecustomize = os.path.join( 394 os.path.dirname(os.__file__), 'sitecustomize.py') 395 with open(sitecustomize, 'w') as f: 396 f.write( 397 '# Importing mach_bootstrap has the side effect of\n' 398 '# installing an import hook\n' 399 'import mach_bootstrap\n' 400 ) 401 402 finally: 403 os.environ.pop('MACOSX_DEPLOYMENT_TARGET', None) 404 405 if old_target is not None: 406 os.environ['MACOSX_DEPLOYMENT_TARGET'] = old_target 407 408 os.environ.update(old_env_variables) 409 410 def call_setup(self, directory, arguments): 411 """Calls setup.py in a directory.""" 412 setup = os.path.join(directory, 'setup.py') 413 414 program = [self.python_path, setup] 415 program.extend(arguments) 416 417 # We probably could call the contents of this file inside the context 418 # of this interpreter using execfile() or similar. However, if global 419 # variables like sys.path are adjusted, this could cause all kinds of 420 # havoc. While this may work, invoking a new process is safer. 421 422 try: 423 output = subprocess.check_output(program, cwd=directory, stderr=subprocess.STDOUT) 424 print(output) 425 except subprocess.CalledProcessError as e: 426 if 'Python.h: No such file or directory' in e.output: 427 print('WARNING: Python.h not found. Install Python development headers.') 428 else: 429 print(e.output) 430 431 raise Exception('Error installing package: %s' % directory) 432 433 def build(self, python=sys.executable): 434 """Build a virtualenv per tree conventions. 435 436 This returns the path of the created virtualenv. 437 """ 438 439 self.create(python) 440 441 # We need to populate the virtualenv using the Python executable in 442 # the virtualenv for paths to be proper. 443 444 args = [self.python_path, __file__, 'populate', self.topsrcdir, 445 self.topobjdir, self.virtualenv_root, self.manifest_path] 446 447 result = self._log_process_output(args, cwd=self.topsrcdir) 448 449 if result != 0: 450 raise Exception('Error populating virtualenv.') 451 452 os.utime(self.activate_path, None) 453 454 return self.virtualenv_root 455 456 def activate(self): 457 """Activate the virtualenv in this Python context. 458 459 If you run a random Python script and wish to "activate" the 460 virtualenv, you can simply instantiate an instance of this class 461 and call .ensure() and .activate() to make the virtualenv active. 462 """ 463 464 execfile(self.activate_path, dict(__file__=self.activate_path)) 465 if isinstance(os.environ['PATH'], unicode): 466 os.environ['PATH'] = os.environ['PATH'].encode('utf-8') 467 468 def install_pip_package(self, package): 469 """Install a package via pip. 470 471 The supplied package is specified using a pip requirement specifier. 472 e.g. 'foo' or 'foo==1.0'. 473 474 If the package is already installed, this is a no-op. 475 """ 476 from pip.req import InstallRequirement 477 478 req = InstallRequirement.from_line(package) 479 req.check_if_exists() 480 if req.satisfied_by is not None: 481 return 482 483 args = [ 484 'install', 485 '--use-wheel', 486 package, 487 ] 488 489 return self._run_pip(args) 490 491 def install_pip_requirements(self, path, require_hashes=True): 492 """Install a pip requirements.txt file. 493 494 The supplied path is a text file containing pip requirement 495 specifiers. 496 497 If require_hashes is True, each specifier must contain the 498 expected hash of the downloaded package. See: 499 https://pip.pypa.io/en/stable/reference/pip_install/#hash-checking-mode 500 """ 501 502 if not os.path.isabs(path): 503 path = os.path.join(self.topsrcdir, path) 504 505 args = [ 506 'install', 507 '--requirement', 508 path, 509 ] 510 511 if require_hashes: 512 args.append('--require-hashes') 513 514 return self._run_pip(args) 515 516 def _run_pip(self, args): 517 # It's tempting to call pip natively via pip.main(). However, 518 # the current Python interpreter may not be the virtualenv python. 519 # This will confuse pip and cause the package to attempt to install 520 # against the executing interpreter. By creating a new process, we 521 # force the virtualenv's interpreter to be used and all is well. 522 # It /might/ be possible to cheat and set sys.executable to 523 # self.python_path. However, this seems more risk than it's worth. 524 subprocess.check_call([os.path.join(self.bin_path, 'pip')] + args, 525 stderr=subprocess.STDOUT) 526 527 528def verify_python_version(log_handle): 529 """Ensure the current version of Python is sufficient.""" 530 major, minor, micro = sys.version_info[:3] 531 532 our = LooseVersion('%d.%d.%d' % (major, minor, micro)) 533 534 if major != MINIMUM_PYTHON_MAJOR or our < MINIMUM_PYTHON_VERSION: 535 log_handle.write('Python %s or greater (but not Python 3) is ' 536 'required to build. ' % MINIMUM_PYTHON_VERSION) 537 log_handle.write('You are running Python %s.\n' % our) 538 539 if os.name in ('nt', 'ce'): 540 log_handle.write(UPGRADE_WINDOWS) 541 else: 542 log_handle.write(UPGRADE_OTHER) 543 544 sys.exit(1) 545 546 547if __name__ == '__main__': 548 if len(sys.argv) < 5: 549 print('Usage: populate_virtualenv.py /path/to/topsrcdir /path/to/topobjdir /path/to/virtualenv /path/to/virtualenv_manifest') 550 sys.exit(1) 551 552 verify_python_version(sys.stdout) 553 554 topsrcdir, topobjdir, virtualenv_path, manifest_path = sys.argv[1:5] 555 populate = False 556 557 # This should only be called internally. 558 if sys.argv[1] == 'populate': 559 populate = True 560 topsrcdir, topobjdir, virtualenv_path, manifest_path = sys.argv[2:] 561 562 manager = VirtualenvManager(topsrcdir, topobjdir, virtualenv_path, 563 sys.stdout, manifest_path) 564 565 if populate: 566 manager.populate() 567 else: 568 manager.ensure() 569 570