1#!/usr/bin/env python
2# Copyright 2016 gRPC authors.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#     http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15"""Definition of targets to build artifacts."""
16
17import os.path
18import random
19import string
20import sys
21
22sys.path.insert(0, os.path.abspath('..'))
23import python_utils.jobset as jobset
24
25
26def create_docker_jobspec(name,
27                          dockerfile_dir,
28                          shell_command,
29                          environ={},
30                          flake_retries=0,
31                          timeout_retries=0,
32                          timeout_seconds=30 * 60,
33                          docker_base_image=None,
34                          extra_docker_args=None,
35                          verbose_success=False):
36    """Creates jobspec for a task running under docker."""
37    environ = environ.copy()
38    environ['RUN_COMMAND'] = shell_command
39    environ['ARTIFACTS_OUT'] = 'artifacts/%s' % name
40
41    docker_args = []
42    for k, v in environ.items():
43        docker_args += ['-e', '%s=%s' % (k, v)]
44    docker_env = {
45        'DOCKERFILE_DIR': dockerfile_dir,
46        'DOCKER_RUN_SCRIPT': 'tools/run_tests/dockerize/docker_run.sh',
47        'OUTPUT_DIR': 'artifacts'
48    }
49
50    if docker_base_image is not None:
51        docker_env['DOCKER_BASE_IMAGE'] = docker_base_image
52    if extra_docker_args is not None:
53        docker_env['EXTRA_DOCKER_ARGS'] = extra_docker_args
54    jobspec = jobset.JobSpec(
55        cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] +
56        docker_args,
57        environ=docker_env,
58        shortname='build_artifact.%s' % (name),
59        timeout_seconds=timeout_seconds,
60        flake_retries=flake_retries,
61        timeout_retries=timeout_retries,
62        verbose_success=verbose_success)
63    return jobspec
64
65
66def create_jobspec(name,
67                   cmdline,
68                   environ={},
69                   shell=False,
70                   flake_retries=0,
71                   timeout_retries=0,
72                   timeout_seconds=30 * 60,
73                   use_workspace=False,
74                   cpu_cost=1.0,
75                   verbose_success=False):
76    """Creates jobspec."""
77    environ = environ.copy()
78    if use_workspace:
79        environ['WORKSPACE_NAME'] = 'workspace_%s' % name
80        environ['ARTIFACTS_OUT'] = os.path.join('..', 'artifacts', name)
81        cmdline = ['bash', 'tools/run_tests/artifacts/run_in_workspace.sh'
82                  ] + cmdline
83    else:
84        environ['ARTIFACTS_OUT'] = os.path.join('artifacts', name)
85
86    jobspec = jobset.JobSpec(
87        cmdline=cmdline,
88        environ=environ,
89        shortname='build_artifact.%s' % (name),
90        timeout_seconds=timeout_seconds,
91        flake_retries=flake_retries,
92        timeout_retries=timeout_retries,
93        shell=shell,
94        cpu_cost=cpu_cost,
95        verbose_success=verbose_success)
96    return jobspec
97
98
99_MACOS_COMPAT_FLAG = '-mmacosx-version-min=10.7'
100
101_ARCH_FLAG_MAP = {'x86': '-m32', 'x64': '-m64'}
102
103
104class PythonArtifact:
105    """Builds Python artifacts."""
106
107    def __init__(self, platform, arch, py_version):
108        self.name = 'python_%s_%s_%s' % (platform, arch, py_version)
109        self.platform = platform
110        self.arch = arch
111        self.labels = ['artifact', 'python', platform, arch, py_version]
112        self.py_version = py_version
113
114    def pre_build_jobspecs(self):
115        return []
116
117    def build_jobspec(self):
118        environ = {}
119        if self.platform == 'linux_extra':
120            # Raspberry Pi build
121            environ['PYTHON'] = '/usr/local/bin/python{}'.format(
122                self.py_version)
123            environ['PIP'] = '/usr/local/bin/pip{}'.format(self.py_version)
124            # https://github.com/resin-io-projects/armv7hf-debian-qemu/issues/9
125            # A QEMU bug causes submodule update to hang, so we copy directly
126            environ['RELATIVE_COPY_PATH'] = '.'
127            # Parallel builds are counterproductive in emulated environment
128            environ['GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS'] = '1'
129            extra_args = ' --entrypoint=/usr/bin/qemu-arm-static '
130            return create_docker_jobspec(
131                self.name,
132                'tools/dockerfile/grpc_artifact_linux_{}'.format(self.arch),
133                'tools/run_tests/artifacts/build_artifact_python.sh',
134                environ=environ,
135                timeout_seconds=60 * 60 * 5,
136                docker_base_image='quay.io/grpc/raspbian_{}'.format(self.arch),
137                extra_docker_args=extra_args)
138        elif self.platform == 'linux':
139            if self.arch == 'x86':
140                environ['SETARCH_CMD'] = 'linux32'
141            # Inside the manylinux container, the python installations are located in
142            # special places...
143            environ['PYTHON'] = '/opt/python/{}/bin/python'.format(
144                self.py_version)
145            environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.py_version)
146            # Platform autodetection for the manylinux1 image breaks so we set the
147            # defines ourselves.
148            # TODO(atash) get better platform-detection support in core so we don't
149            # need to do this manually...
150            environ['CFLAGS'] = '-DGPR_MANYLINUX1=1'
151            environ['GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS'] = 'TRUE'
152            environ['GRPC_BUILD_MANYLINUX_WHEEL'] = 'TRUE'
153            return create_docker_jobspec(
154                self.name,
155                'tools/dockerfile/grpc_artifact_python_manylinux_%s' %
156                self.arch,
157                'tools/run_tests/artifacts/build_artifact_python.sh',
158                environ=environ,
159                timeout_seconds=60 * 60,
160                docker_base_image='quay.io/pypa/manylinux1_i686'
161                if self.arch == 'x86' else 'quay.io/pypa/manylinux1_x86_64')
162        elif self.platform == 'windows':
163            if 'Python27' in self.py_version or 'Python34' in self.py_version:
164                environ['EXT_COMPILER'] = 'mingw32'
165            else:
166                environ['EXT_COMPILER'] = 'msvc'
167            # For some reason, the batch script %random% always runs with the same
168            # seed.  We create a random temp-dir here
169            dir = ''.join(
170                random.choice(string.ascii_uppercase) for _ in range(10))
171            return create_jobspec(
172                self.name, [
173                    'tools\\run_tests\\artifacts\\build_artifact_python.bat',
174                    self.py_version, '32' if self.arch == 'x86' else '64'
175                ],
176                environ=environ,
177                timeout_seconds=45 * 60,
178                use_workspace=True)
179        else:
180            environ['PYTHON'] = self.py_version
181            environ['SKIP_PIP_INSTALL'] = 'TRUE'
182            return create_jobspec(
183                self.name,
184                ['tools/run_tests/artifacts/build_artifact_python.sh'],
185                environ=environ,
186                timeout_seconds=60 * 60 * 2,
187                use_workspace=True)
188
189    def __str__(self):
190        return self.name
191
192
193class RubyArtifact:
194    """Builds ruby native gem."""
195
196    def __init__(self, platform, arch):
197        self.name = 'ruby_native_gem_%s_%s' % (platform, arch)
198        self.platform = platform
199        self.arch = arch
200        self.labels = ['artifact', 'ruby', platform, arch]
201
202    def pre_build_jobspecs(self):
203        return []
204
205    def build_jobspec(self):
206        # Ruby build uses docker internally and docker cannot be nested.
207        # We are using a custom workspace instead.
208        return create_jobspec(
209            self.name, ['tools/run_tests/artifacts/build_artifact_ruby.sh'],
210            use_workspace=True,
211            timeout_seconds=45 * 60)
212
213
214class CSharpExtArtifact:
215    """Builds C# native extension library"""
216
217    def __init__(self, platform, arch, arch_abi=None):
218        self.name = 'csharp_ext_%s_%s' % (platform, arch)
219        self.platform = platform
220        self.arch = arch
221        self.arch_abi = arch_abi
222        self.labels = ['artifact', 'csharp', platform, arch]
223        if arch_abi:
224            self.name += '_%s' % arch_abi
225            self.labels.append(arch_abi)
226
227    def pre_build_jobspecs(self):
228        return []
229
230    def build_jobspec(self):
231        if self.arch == 'android':
232            return create_docker_jobspec(
233                self.name,
234                'tools/dockerfile/grpc_artifact_android_ndk',
235                'tools/run_tests/artifacts/build_artifact_csharp_android.sh',
236                environ={
237                    'ANDROID_ABI': self.arch_abi
238                })
239        elif self.arch == 'ios':
240            return create_jobspec(
241                self.name,
242                ['tools/run_tests/artifacts/build_artifact_csharp_ios.sh'],
243                use_workspace=True)
244        elif self.platform == 'windows':
245            return create_jobspec(
246                self.name, [
247                    'tools\\run_tests\\artifacts\\build_artifact_csharp.bat',
248                    self.arch
249                ],
250                use_workspace=True)
251        else:
252            if self.platform == 'linux':
253                cmake_arch_option = ''  # x64 is the default architecture
254                if self.arch == 'x86':
255                    # TODO(jtattermusch): more work needed to enable
256                    # boringssl assembly optimizations for 32-bit linux.
257                    # Problem: currently we are building the artifact under
258                    # 32-bit docker image, but CMAKE_SYSTEM_PROCESSOR is still
259                    # set to x86_64, so the resulting boringssl binary
260                    # would have undefined symbols.
261                    cmake_arch_option = '-DOPENSSL_NO_ASM=ON'
262                return create_docker_jobspec(
263                    self.name,
264                    'tools/dockerfile/grpc_artifact_linux_%s' % self.arch,
265                    'tools/run_tests/artifacts/build_artifact_csharp.sh',
266                    environ={
267                        'CMAKE_ARCH_OPTION': cmake_arch_option
268                    })
269            else:
270                cmake_arch_option = ''  # x64 is the default architecture
271                if self.arch == 'x86':
272                    cmake_arch_option = '-DCMAKE_OSX_ARCHITECTURES=i386'
273                return create_jobspec(
274                    self.name,
275                    ['tools/run_tests/artifacts/build_artifact_csharp.sh'],
276                    environ={'CMAKE_ARCH_OPTION': cmake_arch_option},
277                    use_workspace=True)
278
279    def __str__(self):
280        return self.name
281
282
283class PHPArtifact:
284    """Builds PHP PECL package"""
285
286    def __init__(self, platform, arch):
287        self.name = 'php_pecl_package_{0}_{1}'.format(platform, arch)
288        self.platform = platform
289        self.arch = arch
290        self.labels = ['artifact', 'php', platform, arch]
291
292    def pre_build_jobspecs(self):
293        return []
294
295    def build_jobspec(self):
296        return create_docker_jobspec(
297            self.name, 'tools/dockerfile/grpc_artifact_linux_{}'.format(
298                self.arch), 'tools/run_tests/artifacts/build_artifact_php.sh')
299
300
301class ProtocArtifact:
302    """Builds protoc and protoc-plugin artifacts"""
303
304    def __init__(self, platform, arch):
305        self.name = 'protoc_%s_%s' % (platform, arch)
306        self.platform = platform
307        self.arch = arch
308        self.labels = ['artifact', 'protoc', platform, arch]
309
310    def pre_build_jobspecs(self):
311        return []
312
313    def build_jobspec(self):
314        if self.platform != 'windows':
315            cxxflags = '-DNDEBUG %s' % _ARCH_FLAG_MAP[self.arch]
316            ldflags = '%s' % _ARCH_FLAG_MAP[self.arch]
317            if self.platform != 'macos':
318                ldflags += '  -static-libgcc -static-libstdc++ -s'
319            environ = {
320                'CONFIG': 'opt',
321                'CXXFLAGS': cxxflags,
322                'LDFLAGS': ldflags,
323                'PROTOBUF_LDFLAGS_EXTRA': ldflags
324            }
325            if self.platform == 'linux':
326                return create_docker_jobspec(
327                    self.name,
328                    'tools/dockerfile/grpc_artifact_protoc',
329                    'tools/run_tests/artifacts/build_artifact_protoc.sh',
330                    environ=environ)
331            else:
332                environ[
333                    'CXXFLAGS'] += ' -std=c++11 -stdlib=libc++ %s' % _MACOS_COMPAT_FLAG
334                return create_jobspec(
335                    self.name,
336                    ['tools/run_tests/artifacts/build_artifact_protoc.sh'],
337                    environ=environ,
338                    timeout_seconds=60 * 60,
339                    use_workspace=True)
340        else:
341            generator = 'Visual Studio 14 2015 Win64' if self.arch == 'x64' else 'Visual Studio 14 2015'
342            return create_jobspec(
343                self.name,
344                ['tools\\run_tests\\artifacts\\build_artifact_protoc.bat'],
345                environ={'generator': generator},
346                use_workspace=True)
347
348    def __str__(self):
349        return self.name
350
351
352def targets():
353    """Gets list of supported targets"""
354    return ([
355        Cls(platform, arch)
356        for Cls in (CSharpExtArtifact, ProtocArtifact)
357        for platform in ('linux', 'macos', 'windows') for arch in ('x86', 'x64')
358    ] + [
359        CSharpExtArtifact('linux', 'android', arch_abi='arm64-v8a'),
360        CSharpExtArtifact('linux', 'android', arch_abi='armeabi-v7a'),
361        CSharpExtArtifact('linux', 'android', arch_abi='x86'),
362        CSharpExtArtifact('macos', 'ios'),
363        PythonArtifact('linux', 'x86', 'cp27-cp27m'),
364        PythonArtifact('linux', 'x86', 'cp27-cp27mu'),
365        PythonArtifact('linux', 'x86', 'cp34-cp34m'),
366        PythonArtifact('linux', 'x86', 'cp35-cp35m'),
367        PythonArtifact('linux', 'x86', 'cp36-cp36m'),
368        PythonArtifact('linux', 'x86', 'cp37-cp37m'),
369        PythonArtifact('linux_extra', 'armv7', '2.7'),
370        PythonArtifact('linux_extra', 'armv7', '3.4'),
371        PythonArtifact('linux_extra', 'armv7', '3.5'),
372        PythonArtifact('linux_extra', 'armv7', '3.6'),
373        PythonArtifact('linux_extra', 'armv6', '2.7'),
374        PythonArtifact('linux_extra', 'armv6', '3.4'),
375        PythonArtifact('linux_extra', 'armv6', '3.5'),
376        PythonArtifact('linux_extra', 'armv6', '3.6'),
377        PythonArtifact('linux', 'x64', 'cp27-cp27m'),
378        PythonArtifact('linux', 'x64', 'cp27-cp27mu'),
379        PythonArtifact('linux', 'x64', 'cp34-cp34m'),
380        PythonArtifact('linux', 'x64', 'cp35-cp35m'),
381        PythonArtifact('linux', 'x64', 'cp36-cp36m'),
382        PythonArtifact('linux', 'x64', 'cp37-cp37m'),
383        PythonArtifact('macos', 'x64', 'python2.7'),
384        PythonArtifact('macos', 'x64', 'python3.4'),
385        PythonArtifact('macos', 'x64', 'python3.5'),
386        PythonArtifact('macos', 'x64', 'python3.6'),
387        PythonArtifact('macos', 'x64', 'python3.7'),
388        PythonArtifact('windows', 'x86', 'Python27_32bits'),
389        PythonArtifact('windows', 'x86', 'Python34_32bits'),
390        PythonArtifact('windows', 'x86', 'Python35_32bits'),
391        PythonArtifact('windows', 'x86', 'Python36_32bits'),
392        PythonArtifact('windows', 'x86', 'Python37_32bits'),
393        PythonArtifact('windows', 'x64', 'Python27'),
394        PythonArtifact('windows', 'x64', 'Python34'),
395        PythonArtifact('windows', 'x64', 'Python35'),
396        PythonArtifact('windows', 'x64', 'Python36'),
397        PythonArtifact('windows', 'x64', 'Python37'),
398        RubyArtifact('linux', 'x64'),
399        RubyArtifact('macos', 'x64'),
400        PHPArtifact('linux', 'x64')
401    ])
402