1# This file builds official Windows binaries of PycURL and all of its dependencies.
2#
3# It is written to be run on a system dedicated to building pycurl, but can be configured
4# for any system that has the required tools installed.
5#
6# Generally, the workflow of building pycurl binaries is as follows:
7#  1. Install git for windows. Use it to check out pycurl repository on the build system.
8#  2. There must be a python installation already present on the build system
9#     in order to execute this file at all. It doesn't matter what the python
10#     version of the bootstrap python is. The first step is to install some
11#     version of python. It saves effort to install one of the versions that will be used
12#     to build pycurl later, however if this is done the target path should be
13#     in line with where all other pythons are going to be installed (i.e. c:/dev/{32,64}/pythonXY by default).
14#     Try these binaries:
15#     https://www.python.org/ftp/python/3.8.0/python-3.8.0.exe
16#     https://www.python.org/ftp/python/3.8.0/python-3.8.0-amd64.exe
17#     Then execute:
18#     c:\dev\python-3.8.0.exe /norestart /passive InstallAllUsers=1 Include_test=0 Include_doc=0 Include_launcher=0 Include_tcltk=0 TargetDir=c:\dev\32\python38
19#  3. Define python versions to build for in the configuration below, then
20#     run `python winbuild.py download` and `python winbuild.py installpy` to install them.
21#  4. Download and install visual studio. Any edition of 2015 or newer should work;
22#     2019 in particular (including community edition) provides batch files to set up a 2015 build environment,
23#     such that there is no reason to get an older version.
24#  5. You may need to install platform sdk/windows sdk, especially if you installed community edition of
25#     visual studio as opposed to a fuller edition. Try https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk.
26#  6. You may also need to install windows 8.1 sdk for building nghttp2 with cmake.
27#     See https://developer.microsoft.com/en-us/windows/downloads/sdk-archive.
28#  7. Download and install perl. This script is tested with activestate perl, although
29#     other distributions may also work. activestate perl can be downloaded at http://www.activestate.com/activeperl/downloads,
30#     although it now requires registration to download thus using a third party download site may be preferable.
31#  8. Download and install nasm: https://www.nasm.us/pub/nasm/releasebuilds/?C=M;O=D
32#     (homepage: http://www.nasm.us/)
33# 9a. Not needed since nghttp2 is currently built using gmake: download and install cmake: https://cmake.org/download/
34# 9b. Download and install gmake: http://gnuwin32.sourceforge.net/packages/make.htm
35# 10. Run `python winbuild.py builddeps` to compile all dependencies for all environments (32/64 bit and python versions).
36# 11. Optional: run `python winbuild.py assembledeps` to assemble all dependencies into archives suitable for use in appveyor.
37# 12. Run `python winbuild.py installvirtualenv` to install virtualenv in all python interpreters.
38# 13. Run `python winbuild.py createvirtualenvs` to create virtualenvs used for pycurl compilation.
39# 14. Run `python winbuild.py` to compile pycurl in all defined configurations.
40# 15. Optional: run `python winbuild.py assemble` to assemble all built versions of pycurl in the current directory.
41
42class Config:
43    '''User-adjustable configuration.
44
45    This class contains version numbers for dependencies,
46    which dependencies to use,
47    and where various binaries, headers and libraries are located in the filesystem.
48    '''
49
50    # work directory for downloading dependencies and building everything
51    root = 'c:/dev/build-pycurl'
52    # where msysgit is installed
53    git_root = 'c:/program files/git'
54    msysgit_bin_paths = [
55        "c:\\Program Files\\Git\\bin",
56        "c:\\Program Files\\Git\\usr\\bin",
57        #"c:\\Program Files\\Git\\mingw64\\bin",
58    ]
59    # where NASM is installed, for building OpenSSL
60    nasm_path = ('c:/dev/nasm', 'c:/program files/nasm', 'c:/program files (x86)/nasm')
61    cmake_path = r"c:\Program Files\CMake\bin\cmake.exe"
62    gmake_path = r"c:\Program Files (x86)\GnuWin32\bin\make.exe"
63    # where ActiveState Perl is installed, for building 64-bit OpenSSL
64    activestate_perl_path = ('c:/perl64', r'c:\dev\perl64')
65    # which versions of python to build against
66    #python_versions = ['2.7.10', '3.2.5', '3.3.5', '3.4.3', '3.5.4', '3.6.2']
67    # these require only vc9 and vc14
68    python_versions = ['3.5.4', '3.6.8', '3.7.6', '3.8.1']
69    # where pythons are installed
70    python_path_template = 'c:/dev/%(bitness)s/python%(python_release)s/python'
71    # overrides only, defaults are given in default_vc_paths below
72    vc_paths = {
73        # where msvc 9/vs 2008 is installed, for python 2.6 through 3.2
74        'vc9': None,
75        # where msvc 10/vs 2010 is installed, for python 3.3 through 3.4
76        'vc10': None,
77        # where msvc 14/vs 2015 is installed, for python 3.5 through 3.8
78        'vc14': None,
79    }
80    # whether to link libcurl against zlib
81    use_zlib = True
82    # which version of zlib to use, will be downloaded from internet
83    zlib_version = '1.2.11'
84    # whether to use openssl instead of winssl
85    use_openssl = True
86    # which version of openssl to use, will be downloaded from internet
87    openssl_version = '1.1.1d'
88    # whether to use c-ares
89    use_cares = True
90    cares_version = '1.15.0'
91    # whether to use libssh2
92    use_libssh2 = True
93    libssh2_version = '1.9.0'
94    use_nghttp2 = True
95    nghttp2_version = '1.40.0'
96    use_libidn = False
97    libiconv_version = '1.16'
98    libidn_version = '1.35'
99    # which version of libcurl to use, will be downloaded from internet
100    libcurl_version = '7.68.0'
101    # virtualenv version
102    virtualenv_version = '15.1.0'
103    # whether to build binary wheels
104    build_wheels = True
105    # pycurl version to build, we should know this ourselves
106    pycurl_version = '7.44.1'
107
108    # Sometimes vc14 does not include windows sdk path in vcvars which breaks stuff.
109    # another application for this is to supply normaliz.lib for vc9
110    # which has an older version that doesn't have the symbols we need
111    windows_sdk_path = 'c:\\program files (x86)\\microsoft sdks\\windows\\v7.1a'
112
113    # See the note below about VCTargetsPath and
114    # https://stackoverflow.com/questions/16092169/why-does-msbuild-look-in-c-for-microsoft-cpp-default-props-instead-of-c-progr.
115    # Since we are targeting vc14, use the v140 path.
116    vc_targets_path = "c:\\Program Files (x86)\\MSBuild\\Microsoft.Cpp\\v4.0\\v140"
117    #vc_targets_path = "c:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Community\\MSBuild\\Current"
118
119    # Where the msbuild that is part of visual studio lives
120    msbuild_bin_path = "c:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Community\\MSBuild\\Current\\Bin"
121
122# ***
123# No user-serviceable parts beyond this point.
124# ***
125
126# OpenSSL build resources including 64-bit builds:
127# http://stackoverflow.com/questions/158232/how-do-you-compile-openssl-for-x64
128# https://wiki.openssl.org/index.php/Compilation_and_Installation
129# http://developer.covenanteyes.com/building-openssl-for-visual-studio/
130
131import os, os.path, sys, subprocess, shutil, contextlib, zipfile, re
132from winbuild.utils import *
133from winbuild.config import *
134from winbuild.builder import *
135from winbuild.nghttp_gmake import *
136from winbuild.tools import *
137from winbuild.zlib import *
138from winbuild.openssl import *
139from winbuild.cares import *
140from winbuild.ssh import *
141from winbuild.curl import *
142from winbuild.pycurl import *
143
144user_config = {}
145for attr in dir(Config):
146    if attr.startswith('_'):
147        continue
148    user_config[attr] = getattr(Config, attr)
149
150# This must be at top level as __file__ can be a relative path
151# and changing current directory will break it
152DIR_HERE = os.path.abspath(os.path.dirname(__file__))
153
154def fetch_to_archives(url):
155    mkdir_p(config.archives_path)
156    path = os.path.join(config.archives_path, os.path.basename(url))
157    fetch(url, path)
158
159@contextlib.contextmanager
160def step(step_fn, args, target_dir):
161    #step = step_fn.__name__
162    state_tag = target_dir
163    mkdir_p(config.state_path)
164    state_file_path = os.path.join(config.state_path, state_tag)
165    if not os.path.exists(state_file_path) or not os.path.exists(target_dir):
166        step_fn(*args)
167    with open(state_file_path, 'w'):
168        pass
169
170def dep_builders(bconf):
171    builders = []
172    if config.use_zlib:
173        builders.append(ZlibBuilder)
174    if config.use_openssl:
175        builders.append(OpensslBuilder)
176    if config.use_cares:
177        builders.append(CaresBuilder)
178    if config.use_libssh2:
179        builders.append(Libssh2Builder)
180    if config.use_nghttp2:
181        builders.append(Nghttp2Builder)
182    if config.use_libidn:
183        builders.append(LibiconvBuilder)
184        builders.append(LibidnBuilder)
185    builders.append(LibcurlBuilder)
186    builders = [
187        cls(bconf=bconf)
188        for cls in builders
189    ]
190    return builders
191
192def build_dependencies(config):
193    if config.use_libssh2:
194        if not config.use_zlib:
195            # technically we can build libssh2 without zlib but I don't want to bother
196            raise ValueError('use_zlib must be true if use_libssh2 is true')
197        if not config.use_openssl:
198            raise ValueError('use_openssl must be true if use_libssh2 is true')
199
200    if config.git_bin_path:
201        os.environ['PATH'] += ";%s" % config.git_bin_path
202    mkdir_p(config.archives_path)
203    with in_dir(config.archives_path):
204        for bconf in buildconfigs():
205                if opts.verbose:
206                    print('Builddep for %s, %s-bit' % (bconf.vc_version, bconf.bitness))
207                for builder in dep_builders(bconf):
208                    step(builder.build, (), builder.state_tag)
209
210def build(config):
211    # note: adds git_bin_path to PATH if necessary, and creates archives_path
212    build_dependencies(config)
213    with in_dir(config.archives_path):
214        for bitness in config.bitnesses:
215            for python_release in config.python_releases:
216                targets = ['bdist', 'bdist_wininst', 'bdist_msi']
217                vc_version = PYTHON_VC_VERSIONS[python_release]
218                bconf = BuildConfig(config, bitness=bitness, vc_version=vc_version)
219                builder = PycurlBuilder(bconf=bconf, python_release=python_release)
220                builder.prepare_tree()
221                builder.build(targets)
222
223def assemble(config):
224    rm_rf(config, 'dist')
225    mkdir_p('dist')
226    for bitness in config.bitnesses:
227        for python_release in config.python_releases:
228            vc_version = PYTHON_VC_VERSIONS[python_release]
229            bconf = BuildConfig(config, bitness=bitness, vc_version=vc_version)
230            builder = PycurlBuilder(bconf=bconf, python_release=python_release)
231            print(builder.build_dir_name)
232            sys.stdout.flush()
233            src = os.path.join(config.archives_path, builder.build_dir_name, 'dist')
234            cp_r(config, src, '.')
235
236def python_metas():
237    metas = []
238    for version in config.python_versions:
239        parts = [int(part) for part in version.split('.')]
240        if parts[0] >= 3 and parts[1] >= 5:
241            ext = 'exe'
242            amd64_suffix = '-amd64'
243        else:
244            ext = 'msi'
245            amd64_suffix = '.amd64'
246        url_32 = 'https://www.python.org/ftp/python/%s/python-%s.%s' % (version, version, ext)
247        url_64 = 'https://www.python.org/ftp/python/%s/python-%s%s.%s' % (version, version, amd64_suffix, ext)
248        meta = dict(
249            version=version, ext=ext, amd64_suffix=amd64_suffix,
250            url_32=url_32, url_64=url_64,
251            installed_path_32 = 'c:\\dev\\32\\python%d%d' % (parts[0], parts[1]),
252            installed_path_64 = 'c:\\dev\\64\\python%d%d' % (parts[0], parts[1]),
253        )
254        metas.append(meta)
255    return metas
256
257def download_pythons(config):
258    for meta in python_metas():
259        for bitness in config.bitnesses:
260            fetch_to_archives(meta['url_%d' % bitness])
261
262def install_pythons(config):
263    for meta in python_metas():
264        for bitness in config.bitnesses:
265            if not os.path.exists(meta['installed_path_%d' % bitness]):
266                install_python(config, meta, bitness)
267
268# http://eddiejackson.net/wp/?p=10276
269def install_python(config, meta, bitness):
270    archive_path = fix_slashes(os.path.join(config.archives_path, os.path.basename(meta['url_%d' % bitness])))
271    if meta['ext'] == 'exe':
272        cmd = [archive_path]
273    else:
274        cmd = ['msiexec', '/i', archive_path, '/norestart']
275    cmd += ['/passive', 'InstallAllUsers=1',
276            'Include_test=0', 'Include_doc=0', 'Include_launcher=0',
277            'Include_tcltk=0',
278            'TargetDir=%s' % meta['installed_path_%d' % bitness],
279        ]
280    sys.stdout.write('Installing python %s (%d bit)\n' % (meta['version'], bitness))
281    print(' '.join(cmd))
282    sys.stdout.flush()
283    check_call(cmd)
284
285def download_bootstrap_python(config):
286    version = config.python_versions[-2]
287    url = 'https://www.python.org/ftp/python/%s/python-%s.msi' % (version, version)
288    fetch(url)
289
290def install_virtualenv(config):
291    with in_dir(config.archives_path):
292        #fetch('https://pypi.python.org/packages/source/v/virtualenv/virtualenv-%s.tar.gz' % virtualenv_version)
293        fetch('https://pypi.python.org/packages/d4/0c/9840c08189e030873387a73b90ada981885010dd9aea134d6de30cd24cb8/virtualenv-15.1.0.tar.gz')
294        for bitness in config.bitnesses:
295            for python_release in config.python_releases:
296                print('Installing virtualenv %s for Python %s (%s bit)' % (config.virtualenv_version, python_release, bitness))
297                sys.stdout.flush()
298                untar(config, 'virtualenv-%s' % config.virtualenv_version)
299                with in_dir('virtualenv-%s' % config.virtualenv_version):
300                    python_binary = PythonBinary(python_release, bitness)
301                    cmd = [python_binary.executable_path(config), 'setup.py', 'install']
302                    check_call(cmd)
303
304def create_virtualenvs(config):
305    for bitness in config.bitnesses:
306        for python_release in config.python_releases:
307            print('Creating a virtualenv for Python %s (%s bit)' % (python_release, bitness))
308            sys.stdout.flush()
309            with in_dir(config.archives_path):
310                python_binary = PythonBinary(python_release, bitness)
311                venv_basename = 'venv-%s-%s' % (python_release, bitness)
312                cmd = [python_binary.executable_path(config), '-m', 'virtualenv', venv_basename]
313                check_call(cmd)
314
315def assemble_deps(config):
316    rm_rf(config, 'deps')
317    os.mkdir('deps')
318    for bconf in buildconfigs():
319        print(bconf.vc_tag)
320        sys.stdout.flush()
321        dest = os.path.join('deps', bconf.vc_tag)
322        os.mkdir(dest)
323        for builder in dep_builders(bconf):
324            cp_r(config, builder.include_path, dest)
325            cp_r(config, builder.lib_path, dest)
326            with zipfile.ZipFile(os.path.join('deps', bconf.vc_tag + '.zip'), 'w', zipfile.ZIP_DEFLATED) as zip:
327                for root, dirs, files in os.walk(dest):
328                    for file in files:
329                        path = os.path.join(root, file)
330                        zip_name = path[len(dest)+1:]
331                        zip.write(path, zip_name)
332
333def get_deps():
334    import struct
335
336    python_release = sys.version_info[:2]
337    vc_version = PYTHON_VC_VERSIONS['.'.join(map(str, python_release))]
338    bitness = struct.calcsize('P') * 8
339    vc_tag = '%s-%d' % (vc_version, bitness)
340    fetch('https://dl.bintray.com/pycurl/deps/%s.zip' % vc_tag)
341    check_call(['unzip', '-d', 'deps', vc_tag + '.zip'])
342
343import optparse
344
345parser = optparse.OptionParser()
346parser.add_option('-b', '--bitness', help='Bitnesses build for, comma separated')
347parser.add_option('-p', '--python', help='Python versions to build for, comma separated')
348parser.add_option('-v', '--verbose', help='Print what is being done', action='store_true')
349opts, args = parser.parse_args()
350
351if opts.bitness:
352    chosen_bitnesses = [int(bitness) for bitness in opts.bitness.split(',')]
353    for bitness in chosen_bitnesses:
354        if bitness not in BITNESSES:
355            print('Invalid bitness %d' % bitness)
356            exit(2)
357else:
358    chosen_bitnesses = BITNESSES
359
360if opts.python:
361    chosen_pythons = opts.python.split(',')
362    chosen_python_versions = []
363    for python in chosen_pythons:
364        python = python.replace('.', '')
365        python = python[0] + '.' + python[1] + '.'
366        ok = False
367        for python_version in Config.python_versions:
368            if python_version.startswith(python):
369                chosen_python_versions.append(python_version)
370                ok = True
371        if not ok:
372            print('Invalid python %s' % python)
373            exit(2)
374else:
375    chosen_python_versions = Config.python_versions
376
377config = ExtendedConfig(user_config,
378    bitnesses=chosen_bitnesses,
379    python_versions=chosen_python_versions,
380    winbuild_root=DIR_HERE,
381)
382
383def buildconfigs():
384    return [BuildConfig(config, bitness=bitness, vc_version=vc_version)
385        for bitness in config.bitnesses
386        for vc_version in needed_vc_versions(config, config.python_versions)
387    ]
388
389if len(args) > 0:
390    if args[0] == 'download':
391        download_pythons(config)
392    elif args[0] == 'bootstrap':
393        download_bootstrap_python(config)
394    elif args[0] == 'installpy':
395        install_pythons(config)
396    elif args[0] == 'builddeps':
397        build_dependencies(config)
398    elif args[0] == 'installvirtualenv':
399        install_virtualenv(config)
400    elif args[0] == 'createvirtualenvs':
401        create_virtualenvs(config)
402    elif args[0] == 'assembledeps':
403        assemble_deps(config)
404    elif args[0] == 'assemble':
405        assemble(config)
406    elif args[0] == 'getdeps':
407        get_deps()
408    else:
409        print('Unknown command: %s' % args[0])
410        exit(2)
411else:
412    build(config)
413