1# This file builds official Windows binaries of PycURL and all of its dependencies. 2# 3# It is written to be run on a system dedicated to building pycurl, but can be configured 4# for any system that has the required tools installed. 5# 6# Generally, the workflow of building pycurl binaries is as follows: 7# 1. Install git for windows. Use it to check out pycurl repository on the build system. 8# 2. There must be a python installation already present on the build system 9# in order to execute this file at all. It doesn't matter what the python 10# version of the bootstrap python is. The first step is to install some 11# version of python. It saves effort to install one of the versions that will be used 12# to build pycurl later, however if this is done the target path should be 13# in line with where all other pythons are going to be installed (i.e. c:/dev/{32,64}/pythonXY by default). 14# Try these binaries: 15# https://www.python.org/ftp/python/3.8.0/python-3.8.0.exe 16# https://www.python.org/ftp/python/3.8.0/python-3.8.0-amd64.exe 17# Then execute: 18# c:\dev\python-3.8.0.exe /norestart /passive InstallAllUsers=1 Include_test=0 Include_doc=0 Include_launcher=0 Include_tcltk=0 TargetDir=c:\dev\32\python38 19# 3. Define python versions to build for in the configuration below, then 20# run `python winbuild.py download` and `python winbuild.py installpy` to install them. 21# 4. Download and install visual studio. Any edition of 2015 or newer should work; 22# 2019 in particular (including community edition) provides batch files to set up a 2015 build environment, 23# such that there is no reason to get an older version. 24# 5. You may need to install platform sdk/windows sdk, especially if you installed community edition of 25# visual studio as opposed to a fuller edition. Try https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk. 26# 6. You may also need to install windows 8.1 sdk for building nghttp2 with cmake. 27# See https://developer.microsoft.com/en-us/windows/downloads/sdk-archive. 28# 7. Download and install perl. This script is tested with activestate perl, although 29# other distributions may also work. activestate perl can be downloaded at http://www.activestate.com/activeperl/downloads, 30# although it now requires registration to download thus using a third party download site may be preferable. 31# 8. Download and install nasm: https://www.nasm.us/pub/nasm/releasebuilds/?C=M;O=D 32# (homepage: http://www.nasm.us/) 33# 9a. Not needed since nghttp2 is currently built using gmake: download and install cmake: https://cmake.org/download/ 34# 9b. Download and install gmake: http://gnuwin32.sourceforge.net/packages/make.htm 35# 10. Run `python winbuild.py builddeps` to compile all dependencies for all environments (32/64 bit and python versions). 36# 11. Optional: run `python winbuild.py assembledeps` to assemble all dependencies into archives suitable for use in appveyor. 37# 12. Run `python winbuild.py installvirtualenv` to install virtualenv in all python interpreters. 38# 13. Run `python winbuild.py createvirtualenvs` to create virtualenvs used for pycurl compilation. 39# 14. Run `python winbuild.py` to compile pycurl in all defined configurations. 40# 15. Optional: run `python winbuild.py assemble` to assemble all built versions of pycurl in the current directory. 41 42class Config: 43 '''User-adjustable configuration. 44 45 This class contains version numbers for dependencies, 46 which dependencies to use, 47 and where various binaries, headers and libraries are located in the filesystem. 48 ''' 49 50 # work directory for downloading dependencies and building everything 51 root = 'c:/dev/build-pycurl' 52 # where msysgit is installed 53 git_root = 'c:/program files/git' 54 msysgit_bin_paths = [ 55 "c:\\Program Files\\Git\\bin", 56 "c:\\Program Files\\Git\\usr\\bin", 57 #"c:\\Program Files\\Git\\mingw64\\bin", 58 ] 59 # where NASM is installed, for building OpenSSL 60 nasm_path = ('c:/dev/nasm', 'c:/program files/nasm', 'c:/program files (x86)/nasm') 61 cmake_path = r"c:\Program Files\CMake\bin\cmake.exe" 62 gmake_path = r"c:\Program Files (x86)\GnuWin32\bin\make.exe" 63 # where ActiveState Perl is installed, for building 64-bit OpenSSL 64 activestate_perl_path = ('c:/perl64', r'c:\dev\perl64') 65 # which versions of python to build against 66 #python_versions = ['2.7.10', '3.2.5', '3.3.5', '3.4.3', '3.5.4', '3.6.2'] 67 # these require only vc9 and vc14 68 python_versions = ['3.5.4', '3.6.8', '3.7.6', '3.8.1'] 69 # where pythons are installed 70 python_path_template = 'c:/dev/%(bitness)s/python%(python_release)s/python' 71 # overrides only, defaults are given in default_vc_paths below 72 vc_paths = { 73 # where msvc 9/vs 2008 is installed, for python 2.6 through 3.2 74 'vc9': None, 75 # where msvc 10/vs 2010 is installed, for python 3.3 through 3.4 76 'vc10': None, 77 # where msvc 14/vs 2015 is installed, for python 3.5 through 3.8 78 'vc14': None, 79 } 80 # whether to link libcurl against zlib 81 use_zlib = True 82 # which version of zlib to use, will be downloaded from internet 83 zlib_version = '1.2.11' 84 # whether to use openssl instead of winssl 85 use_openssl = True 86 # which version of openssl to use, will be downloaded from internet 87 openssl_version = '1.1.1d' 88 # whether to use c-ares 89 use_cares = True 90 cares_version = '1.15.0' 91 # whether to use libssh2 92 use_libssh2 = True 93 libssh2_version = '1.9.0' 94 use_nghttp2 = True 95 nghttp2_version = '1.40.0' 96 use_libidn = False 97 libiconv_version = '1.16' 98 libidn_version = '1.35' 99 # which version of libcurl to use, will be downloaded from internet 100 libcurl_version = '7.68.0' 101 # virtualenv version 102 virtualenv_version = '15.1.0' 103 # whether to build binary wheels 104 build_wheels = True 105 # pycurl version to build, we should know this ourselves 106 pycurl_version = '7.44.1' 107 108 # Sometimes vc14 does not include windows sdk path in vcvars which breaks stuff. 109 # another application for this is to supply normaliz.lib for vc9 110 # which has an older version that doesn't have the symbols we need 111 windows_sdk_path = 'c:\\program files (x86)\\microsoft sdks\\windows\\v7.1a' 112 113 # See the note below about VCTargetsPath and 114 # https://stackoverflow.com/questions/16092169/why-does-msbuild-look-in-c-for-microsoft-cpp-default-props-instead-of-c-progr. 115 # Since we are targeting vc14, use the v140 path. 116 vc_targets_path = "c:\\Program Files (x86)\\MSBuild\\Microsoft.Cpp\\v4.0\\v140" 117 #vc_targets_path = "c:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Community\\MSBuild\\Current" 118 119 # Where the msbuild that is part of visual studio lives 120 msbuild_bin_path = "c:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Community\\MSBuild\\Current\\Bin" 121 122# *** 123# No user-serviceable parts beyond this point. 124# *** 125 126# OpenSSL build resources including 64-bit builds: 127# http://stackoverflow.com/questions/158232/how-do-you-compile-openssl-for-x64 128# https://wiki.openssl.org/index.php/Compilation_and_Installation 129# http://developer.covenanteyes.com/building-openssl-for-visual-studio/ 130 131import os, os.path, sys, subprocess, shutil, contextlib, zipfile, re 132from winbuild.utils import * 133from winbuild.config import * 134from winbuild.builder import * 135from winbuild.nghttp_gmake import * 136from winbuild.tools import * 137from winbuild.zlib import * 138from winbuild.openssl import * 139from winbuild.cares import * 140from winbuild.ssh import * 141from winbuild.curl import * 142from winbuild.pycurl import * 143 144user_config = {} 145for attr in dir(Config): 146 if attr.startswith('_'): 147 continue 148 user_config[attr] = getattr(Config, attr) 149 150# This must be at top level as __file__ can be a relative path 151# and changing current directory will break it 152DIR_HERE = os.path.abspath(os.path.dirname(__file__)) 153 154def fetch_to_archives(url): 155 mkdir_p(config.archives_path) 156 path = os.path.join(config.archives_path, os.path.basename(url)) 157 fetch(url, path) 158 159@contextlib.contextmanager 160def step(step_fn, args, target_dir): 161 #step = step_fn.__name__ 162 state_tag = target_dir 163 mkdir_p(config.state_path) 164 state_file_path = os.path.join(config.state_path, state_tag) 165 if not os.path.exists(state_file_path) or not os.path.exists(target_dir): 166 step_fn(*args) 167 with open(state_file_path, 'w'): 168 pass 169 170def dep_builders(bconf): 171 builders = [] 172 if config.use_zlib: 173 builders.append(ZlibBuilder) 174 if config.use_openssl: 175 builders.append(OpensslBuilder) 176 if config.use_cares: 177 builders.append(CaresBuilder) 178 if config.use_libssh2: 179 builders.append(Libssh2Builder) 180 if config.use_nghttp2: 181 builders.append(Nghttp2Builder) 182 if config.use_libidn: 183 builders.append(LibiconvBuilder) 184 builders.append(LibidnBuilder) 185 builders.append(LibcurlBuilder) 186 builders = [ 187 cls(bconf=bconf) 188 for cls in builders 189 ] 190 return builders 191 192def build_dependencies(config): 193 if config.use_libssh2: 194 if not config.use_zlib: 195 # technically we can build libssh2 without zlib but I don't want to bother 196 raise ValueError('use_zlib must be true if use_libssh2 is true') 197 if not config.use_openssl: 198 raise ValueError('use_openssl must be true if use_libssh2 is true') 199 200 if config.git_bin_path: 201 os.environ['PATH'] += ";%s" % config.git_bin_path 202 mkdir_p(config.archives_path) 203 with in_dir(config.archives_path): 204 for bconf in buildconfigs(): 205 if opts.verbose: 206 print('Builddep for %s, %s-bit' % (bconf.vc_version, bconf.bitness)) 207 for builder in dep_builders(bconf): 208 step(builder.build, (), builder.state_tag) 209 210def build(config): 211 # note: adds git_bin_path to PATH if necessary, and creates archives_path 212 build_dependencies(config) 213 with in_dir(config.archives_path): 214 for bitness in config.bitnesses: 215 for python_release in config.python_releases: 216 targets = ['bdist', 'bdist_wininst', 'bdist_msi'] 217 vc_version = PYTHON_VC_VERSIONS[python_release] 218 bconf = BuildConfig(config, bitness=bitness, vc_version=vc_version) 219 builder = PycurlBuilder(bconf=bconf, python_release=python_release) 220 builder.prepare_tree() 221 builder.build(targets) 222 223def assemble(config): 224 rm_rf(config, 'dist') 225 mkdir_p('dist') 226 for bitness in config.bitnesses: 227 for python_release in config.python_releases: 228 vc_version = PYTHON_VC_VERSIONS[python_release] 229 bconf = BuildConfig(config, bitness=bitness, vc_version=vc_version) 230 builder = PycurlBuilder(bconf=bconf, python_release=python_release) 231 print(builder.build_dir_name) 232 sys.stdout.flush() 233 src = os.path.join(config.archives_path, builder.build_dir_name, 'dist') 234 cp_r(config, src, '.') 235 236def python_metas(): 237 metas = [] 238 for version in config.python_versions: 239 parts = [int(part) for part in version.split('.')] 240 if parts[0] >= 3 and parts[1] >= 5: 241 ext = 'exe' 242 amd64_suffix = '-amd64' 243 else: 244 ext = 'msi' 245 amd64_suffix = '.amd64' 246 url_32 = 'https://www.python.org/ftp/python/%s/python-%s.%s' % (version, version, ext) 247 url_64 = 'https://www.python.org/ftp/python/%s/python-%s%s.%s' % (version, version, amd64_suffix, ext) 248 meta = dict( 249 version=version, ext=ext, amd64_suffix=amd64_suffix, 250 url_32=url_32, url_64=url_64, 251 installed_path_32 = 'c:\\dev\\32\\python%d%d' % (parts[0], parts[1]), 252 installed_path_64 = 'c:\\dev\\64\\python%d%d' % (parts[0], parts[1]), 253 ) 254 metas.append(meta) 255 return metas 256 257def download_pythons(config): 258 for meta in python_metas(): 259 for bitness in config.bitnesses: 260 fetch_to_archives(meta['url_%d' % bitness]) 261 262def install_pythons(config): 263 for meta in python_metas(): 264 for bitness in config.bitnesses: 265 if not os.path.exists(meta['installed_path_%d' % bitness]): 266 install_python(config, meta, bitness) 267 268# http://eddiejackson.net/wp/?p=10276 269def install_python(config, meta, bitness): 270 archive_path = fix_slashes(os.path.join(config.archives_path, os.path.basename(meta['url_%d' % bitness]))) 271 if meta['ext'] == 'exe': 272 cmd = [archive_path] 273 else: 274 cmd = ['msiexec', '/i', archive_path, '/norestart'] 275 cmd += ['/passive', 'InstallAllUsers=1', 276 'Include_test=0', 'Include_doc=0', 'Include_launcher=0', 277 'Include_tcltk=0', 278 'TargetDir=%s' % meta['installed_path_%d' % bitness], 279 ] 280 sys.stdout.write('Installing python %s (%d bit)\n' % (meta['version'], bitness)) 281 print(' '.join(cmd)) 282 sys.stdout.flush() 283 check_call(cmd) 284 285def download_bootstrap_python(config): 286 version = config.python_versions[-2] 287 url = 'https://www.python.org/ftp/python/%s/python-%s.msi' % (version, version) 288 fetch(url) 289 290def install_virtualenv(config): 291 with in_dir(config.archives_path): 292 #fetch('https://pypi.python.org/packages/source/v/virtualenv/virtualenv-%s.tar.gz' % virtualenv_version) 293 fetch('https://pypi.python.org/packages/d4/0c/9840c08189e030873387a73b90ada981885010dd9aea134d6de30cd24cb8/virtualenv-15.1.0.tar.gz') 294 for bitness in config.bitnesses: 295 for python_release in config.python_releases: 296 print('Installing virtualenv %s for Python %s (%s bit)' % (config.virtualenv_version, python_release, bitness)) 297 sys.stdout.flush() 298 untar(config, 'virtualenv-%s' % config.virtualenv_version) 299 with in_dir('virtualenv-%s' % config.virtualenv_version): 300 python_binary = PythonBinary(python_release, bitness) 301 cmd = [python_binary.executable_path(config), 'setup.py', 'install'] 302 check_call(cmd) 303 304def create_virtualenvs(config): 305 for bitness in config.bitnesses: 306 for python_release in config.python_releases: 307 print('Creating a virtualenv for Python %s (%s bit)' % (python_release, bitness)) 308 sys.stdout.flush() 309 with in_dir(config.archives_path): 310 python_binary = PythonBinary(python_release, bitness) 311 venv_basename = 'venv-%s-%s' % (python_release, bitness) 312 cmd = [python_binary.executable_path(config), '-m', 'virtualenv', venv_basename] 313 check_call(cmd) 314 315def assemble_deps(config): 316 rm_rf(config, 'deps') 317 os.mkdir('deps') 318 for bconf in buildconfigs(): 319 print(bconf.vc_tag) 320 sys.stdout.flush() 321 dest = os.path.join('deps', bconf.vc_tag) 322 os.mkdir(dest) 323 for builder in dep_builders(bconf): 324 cp_r(config, builder.include_path, dest) 325 cp_r(config, builder.lib_path, dest) 326 with zipfile.ZipFile(os.path.join('deps', bconf.vc_tag + '.zip'), 'w', zipfile.ZIP_DEFLATED) as zip: 327 for root, dirs, files in os.walk(dest): 328 for file in files: 329 path = os.path.join(root, file) 330 zip_name = path[len(dest)+1:] 331 zip.write(path, zip_name) 332 333def get_deps(): 334 import struct 335 336 python_release = sys.version_info[:2] 337 vc_version = PYTHON_VC_VERSIONS['.'.join(map(str, python_release))] 338 bitness = struct.calcsize('P') * 8 339 vc_tag = '%s-%d' % (vc_version, bitness) 340 fetch('https://dl.bintray.com/pycurl/deps/%s.zip' % vc_tag) 341 check_call(['unzip', '-d', 'deps', vc_tag + '.zip']) 342 343import optparse 344 345parser = optparse.OptionParser() 346parser.add_option('-b', '--bitness', help='Bitnesses build for, comma separated') 347parser.add_option('-p', '--python', help='Python versions to build for, comma separated') 348parser.add_option('-v', '--verbose', help='Print what is being done', action='store_true') 349opts, args = parser.parse_args() 350 351if opts.bitness: 352 chosen_bitnesses = [int(bitness) for bitness in opts.bitness.split(',')] 353 for bitness in chosen_bitnesses: 354 if bitness not in BITNESSES: 355 print('Invalid bitness %d' % bitness) 356 exit(2) 357else: 358 chosen_bitnesses = BITNESSES 359 360if opts.python: 361 chosen_pythons = opts.python.split(',') 362 chosen_python_versions = [] 363 for python in chosen_pythons: 364 python = python.replace('.', '') 365 python = python[0] + '.' + python[1] + '.' 366 ok = False 367 for python_version in Config.python_versions: 368 if python_version.startswith(python): 369 chosen_python_versions.append(python_version) 370 ok = True 371 if not ok: 372 print('Invalid python %s' % python) 373 exit(2) 374else: 375 chosen_python_versions = Config.python_versions 376 377config = ExtendedConfig(user_config, 378 bitnesses=chosen_bitnesses, 379 python_versions=chosen_python_versions, 380 winbuild_root=DIR_HERE, 381) 382 383def buildconfigs(): 384 return [BuildConfig(config, bitness=bitness, vc_version=vc_version) 385 for bitness in config.bitnesses 386 for vc_version in needed_vc_versions(config, config.python_versions) 387 ] 388 389if len(args) > 0: 390 if args[0] == 'download': 391 download_pythons(config) 392 elif args[0] == 'bootstrap': 393 download_bootstrap_python(config) 394 elif args[0] == 'installpy': 395 install_pythons(config) 396 elif args[0] == 'builddeps': 397 build_dependencies(config) 398 elif args[0] == 'installvirtualenv': 399 install_virtualenv(config) 400 elif args[0] == 'createvirtualenvs': 401 create_virtualenvs(config) 402 elif args[0] == 'assembledeps': 403 assemble_deps(config) 404 elif args[0] == 'assemble': 405 assemble(config) 406 elif args[0] == 'getdeps': 407 get_deps() 408 else: 409 print('Unknown command: %s' % args[0]) 410 exit(2) 411else: 412 build(config) 413