1#!/usr/bin/env python
2# Copyright 2016 gRPC authors.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#     http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15"""Definition of targets to build artifacts."""
16
17import os.path
18import random
19import string
20import sys
21
22sys.path.insert(0, os.path.abspath('..'))
23import python_utils.jobset as jobset
24
25
26def create_docker_jobspec(name,
27                          dockerfile_dir,
28                          shell_command,
29                          environ={},
30                          flake_retries=0,
31                          timeout_retries=0,
32                          timeout_seconds=30 * 60,
33                          docker_base_image=None,
34                          extra_docker_args=None,
35                          verbose_success=False):
36    """Creates jobspec for a task running under docker."""
37    environ = environ.copy()
38    environ['RUN_COMMAND'] = shell_command
39    environ['ARTIFACTS_OUT'] = 'artifacts/%s' % name
40
41    docker_args = []
42    for k, v in environ.items():
43        docker_args += ['-e', '%s=%s' % (k, v)]
44    docker_env = {
45        'DOCKERFILE_DIR': dockerfile_dir,
46        'DOCKER_RUN_SCRIPT': 'tools/run_tests/dockerize/docker_run.sh',
47        'OUTPUT_DIR': 'artifacts'
48    }
49
50    if docker_base_image is not None:
51        docker_env['DOCKER_BASE_IMAGE'] = docker_base_image
52    if extra_docker_args is not None:
53        docker_env['EXTRA_DOCKER_ARGS'] = extra_docker_args
54    jobspec = jobset.JobSpec(
55        cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] +
56        docker_args,
57        environ=docker_env,
58        shortname='build_artifact.%s' % (name),
59        timeout_seconds=timeout_seconds,
60        flake_retries=flake_retries,
61        timeout_retries=timeout_retries,
62        verbose_success=verbose_success)
63    return jobspec
64
65
66def create_jobspec(name,
67                   cmdline,
68                   environ={},
69                   shell=False,
70                   flake_retries=0,
71                   timeout_retries=0,
72                   timeout_seconds=30 * 60,
73                   use_workspace=False,
74                   cpu_cost=1.0,
75                   verbose_success=False):
76    """Creates jobspec."""
77    environ = environ.copy()
78    if use_workspace:
79        environ['WORKSPACE_NAME'] = 'workspace_%s' % name
80        environ['ARTIFACTS_OUT'] = os.path.join('..', 'artifacts', name)
81        cmdline = ['bash', 'tools/run_tests/artifacts/run_in_workspace.sh'
82                  ] + cmdline
83    else:
84        environ['ARTIFACTS_OUT'] = os.path.join('artifacts', name)
85
86    jobspec = jobset.JobSpec(cmdline=cmdline,
87                             environ=environ,
88                             shortname='build_artifact.%s' % (name),
89                             timeout_seconds=timeout_seconds,
90                             flake_retries=flake_retries,
91                             timeout_retries=timeout_retries,
92                             shell=shell,
93                             cpu_cost=cpu_cost,
94                             verbose_success=verbose_success)
95    return jobspec
96
97
98_MACOS_COMPAT_FLAG = '-mmacosx-version-min=10.10'
99
100_ARCH_FLAG_MAP = {'x86': '-m32', 'x64': '-m64'}
101
102
103class PythonArtifact:
104    """Builds Python artifacts."""
105
106    def __init__(self, platform, arch, py_version):
107        self.name = 'python_%s_%s_%s' % (platform, arch, py_version)
108        self.platform = platform
109        self.arch = arch
110        self.labels = ['artifact', 'python', platform, arch, py_version]
111        self.py_version = py_version
112        if 'manylinux' in platform:
113            self.labels.append('linux')
114
115    def pre_build_jobspecs(self):
116        return []
117
118    def build_jobspec(self):
119        environ = {}
120        if self.platform == 'linux_extra':
121            # Raspberry Pi build
122            environ['PYTHON'] = '/usr/local/bin/python{}'.format(
123                self.py_version)
124            environ['PIP'] = '/usr/local/bin/pip{}'.format(self.py_version)
125            # https://github.com/resin-io-projects/armv7hf-debian-qemu/issues/9
126            # A QEMU bug causes submodule update to hang, so we copy directly
127            environ['RELATIVE_COPY_PATH'] = '.'
128            # Parallel builds are counterproductive in emulated environment
129            environ['GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS'] = '1'
130            extra_args = ' --entrypoint=/usr/bin/qemu-arm-static '
131            return create_docker_jobspec(
132                self.name,
133                'tools/dockerfile/grpc_artifact_linux_{}'.format(self.arch),
134                'tools/run_tests/artifacts/build_artifact_python.sh',
135                environ=environ,
136                timeout_seconds=60 * 60 * 5,
137                docker_base_image='quay.io/grpc/raspbian_{}'.format(self.arch),
138                extra_docker_args=extra_args)
139        elif 'manylinux' in self.platform:
140            if self.arch == 'x86':
141                environ['SETARCH_CMD'] = 'linux32'
142            # Inside the manylinux container, the python installations are located in
143            # special places...
144            environ['PYTHON'] = '/opt/python/{}/bin/python'.format(
145                self.py_version)
146            environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.py_version)
147            environ['GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS'] = 'TRUE'
148            environ['GRPC_BUILD_MANYLINUX_WHEEL'] = 'TRUE'
149            return create_docker_jobspec(
150                self.name,
151                # NOTE(rbellevi): Do *not* update this without also ensuring the
152                # base_docker_image attribute is accurate.
153                'tools/dockerfile/grpc_artifact_python_%s_%s' %
154                (self.platform, self.arch),
155                'tools/run_tests/artifacts/build_artifact_python.sh',
156                environ=environ,
157                timeout_seconds=60 * 60)
158        elif self.platform == 'windows':
159            if 'Python27' in self.py_version:
160                environ['EXT_COMPILER'] = 'mingw32'
161            else:
162                environ['EXT_COMPILER'] = 'msvc'
163            # For some reason, the batch script %random% always runs with the same
164            # seed.  We create a random temp-dir here
165            dir = ''.join(
166                random.choice(string.ascii_uppercase) for _ in range(10))
167            return create_jobspec(self.name, [
168                'tools\\run_tests\\artifacts\\build_artifact_python.bat',
169                self.py_version, '32' if self.arch == 'x86' else '64'
170            ],
171                                  environ=environ,
172                                  timeout_seconds=45 * 60,
173                                  use_workspace=True)
174        else:
175            environ['PYTHON'] = self.py_version
176            environ['SKIP_PIP_INSTALL'] = 'TRUE'
177            return create_jobspec(
178                self.name,
179                ['tools/run_tests/artifacts/build_artifact_python.sh'],
180                environ=environ,
181                timeout_seconds=60 * 60 * 2,
182                use_workspace=True)
183
184    def __str__(self):
185        return self.name
186
187
188class RubyArtifact:
189    """Builds ruby native gem."""
190
191    def __init__(self, platform, arch):
192        self.name = 'ruby_native_gem_%s_%s' % (platform, arch)
193        self.platform = platform
194        self.arch = arch
195        self.labels = ['artifact', 'ruby', platform, arch]
196
197    def pre_build_jobspecs(self):
198        return []
199
200    def build_jobspec(self):
201        # Ruby build uses docker internally and docker cannot be nested.
202        # We are using a custom workspace instead.
203        return create_jobspec(
204            self.name, ['tools/run_tests/artifacts/build_artifact_ruby.sh'],
205            use_workspace=True,
206            timeout_seconds=45 * 60)
207
208
209class CSharpExtArtifact:
210    """Builds C# native extension library"""
211
212    def __init__(self, platform, arch, arch_abi=None):
213        self.name = 'csharp_ext_%s_%s' % (platform, arch)
214        self.platform = platform
215        self.arch = arch
216        self.arch_abi = arch_abi
217        self.labels = ['artifact', 'csharp', platform, arch]
218        if arch_abi:
219            self.name += '_%s' % arch_abi
220            self.labels.append(arch_abi)
221
222    def pre_build_jobspecs(self):
223        return []
224
225    def build_jobspec(self):
226        if self.arch == 'android':
227            return create_docker_jobspec(
228                self.name,
229                'tools/dockerfile/grpc_artifact_android_ndk',
230                'tools/run_tests/artifacts/build_artifact_csharp_android.sh',
231                environ={'ANDROID_ABI': self.arch_abi})
232        elif self.arch == 'ios':
233            return create_jobspec(
234                self.name,
235                ['tools/run_tests/artifacts/build_artifact_csharp_ios.sh'],
236                use_workspace=True)
237        elif self.platform == 'windows':
238            return create_jobspec(self.name, [
239                'tools\\run_tests\\artifacts\\build_artifact_csharp.bat',
240                self.arch
241            ],
242                                  use_workspace=True)
243        else:
244            if self.platform == 'linux':
245                cmake_arch_option = ''  # x64 is the default architecture
246                if self.arch == 'x86':
247                    # TODO(jtattermusch): more work needed to enable
248                    # boringssl assembly optimizations for 32-bit linux.
249                    # Problem: currently we are building the artifact under
250                    # 32-bit docker image, but CMAKE_SYSTEM_PROCESSOR is still
251                    # set to x86_64, so the resulting boringssl binary
252                    # would have undefined symbols.
253                    cmake_arch_option = '-DOPENSSL_NO_ASM=ON'
254                return create_docker_jobspec(
255                    self.name,
256                    'tools/dockerfile/grpc_artifact_centos6_{}'.format(
257                        self.arch),
258                    'tools/run_tests/artifacts/build_artifact_csharp.sh',
259                    environ={'CMAKE_ARCH_OPTION': cmake_arch_option})
260            else:
261                cmake_arch_option = ''  # x64 is the default architecture
262                if self.arch == 'x86':
263                    cmake_arch_option = '-DCMAKE_OSX_ARCHITECTURES=i386'
264                return create_jobspec(
265                    self.name,
266                    ['tools/run_tests/artifacts/build_artifact_csharp.sh'],
267                    environ={'CMAKE_ARCH_OPTION': cmake_arch_option},
268                    use_workspace=True)
269
270    def __str__(self):
271        return self.name
272
273
274class PHPArtifact:
275    """Builds PHP PECL package"""
276
277    def __init__(self, platform, arch):
278        self.name = 'php_pecl_package_{0}_{1}'.format(platform, arch)
279        self.platform = platform
280        self.arch = arch
281        self.labels = ['artifact', 'php', platform, arch]
282
283    def pre_build_jobspecs(self):
284        return []
285
286    def build_jobspec(self):
287        return create_docker_jobspec(
288            self.name,
289            'tools/dockerfile/test/php73_zts_stretch_{}'.format(self.arch),
290            'tools/run_tests/artifacts/build_artifact_php.sh')
291
292
293class ProtocArtifact:
294    """Builds protoc and protoc-plugin artifacts"""
295
296    def __init__(self, platform, arch):
297        self.name = 'protoc_%s_%s' % (platform, arch)
298        self.platform = platform
299        self.arch = arch
300        self.labels = ['artifact', 'protoc', platform, arch]
301
302    def pre_build_jobspecs(self):
303        return []
304
305    def build_jobspec(self):
306        if self.platform != 'windows':
307            cxxflags = '-DNDEBUG %s' % _ARCH_FLAG_MAP[self.arch]
308            ldflags = '%s' % _ARCH_FLAG_MAP[self.arch]
309            if self.platform != 'macos':
310                ldflags += '  -static-libgcc -static-libstdc++ -s'
311            environ = {
312                'CONFIG': 'opt',
313                'CXXFLAGS': cxxflags,
314                'LDFLAGS': ldflags,
315                'PROTOBUF_LDFLAGS_EXTRA': ldflags
316            }
317            if self.platform == 'linux':
318                return create_docker_jobspec(
319                    self.name,
320                    'tools/dockerfile/grpc_artifact_centos6_{}'.format(
321                        self.arch),
322                    'tools/run_tests/artifacts/build_artifact_protoc.sh',
323                    environ=environ)
324            else:
325                environ[
326                    'CXXFLAGS'] += ' -std=c++11 -stdlib=libc++ %s' % _MACOS_COMPAT_FLAG
327                return create_jobspec(
328                    self.name,
329                    ['tools/run_tests/artifacts/build_artifact_protoc.sh'],
330                    environ=environ,
331                    timeout_seconds=60 * 60,
332                    use_workspace=True)
333        else:
334            generator = 'Visual Studio 14 2015 Win64' if self.arch == 'x64' else 'Visual Studio 14 2015'
335            return create_jobspec(
336                self.name,
337                ['tools\\run_tests\\artifacts\\build_artifact_protoc.bat'],
338                environ={'generator': generator},
339                use_workspace=True)
340
341    def __str__(self):
342        return self.name
343
344
345def targets():
346    """Gets list of supported targets"""
347    return [
348        ProtocArtifact('linux', 'x64'),
349        ProtocArtifact('linux', 'x86'),
350        ProtocArtifact('macos', 'x64'),
351        ProtocArtifact('windows', 'x64'),
352        ProtocArtifact('windows', 'x86'),
353        CSharpExtArtifact('linux', 'x64'),
354        CSharpExtArtifact('macos', 'x64'),
355        CSharpExtArtifact('windows', 'x64'),
356        CSharpExtArtifact('windows', 'x86'),
357        CSharpExtArtifact('linux', 'android', arch_abi='arm64-v8a'),
358        CSharpExtArtifact('linux', 'android', arch_abi='armeabi-v7a'),
359        CSharpExtArtifact('linux', 'android', arch_abi='x86'),
360        CSharpExtArtifact('macos', 'ios'),
361        PythonArtifact('manylinux2014', 'x64', 'cp35-cp35m'),
362        PythonArtifact('manylinux2014', 'x64', 'cp36-cp36m'),
363        PythonArtifact('manylinux2014', 'x64', 'cp37-cp37m'),
364        PythonArtifact('manylinux2014', 'x64', 'cp38-cp38'),
365        PythonArtifact('manylinux2014', 'x64', 'cp39-cp39'),
366        PythonArtifact('manylinux2014', 'x86', 'cp35-cp35m'),
367        PythonArtifact('manylinux2014', 'x86', 'cp36-cp36m'),
368        PythonArtifact('manylinux2014', 'x86', 'cp37-cp37m'),
369        PythonArtifact('manylinux2014', 'x86', 'cp38-cp38'),
370        PythonArtifact('manylinux2014', 'x86', 'cp39-cp39'),
371        PythonArtifact('manylinux2010', 'x64', 'cp27-cp27m'),
372        PythonArtifact('manylinux2010', 'x64', 'cp27-cp27mu'),
373        PythonArtifact('manylinux2010', 'x64', 'cp35-cp35m'),
374        PythonArtifact('manylinux2010', 'x64', 'cp36-cp36m'),
375        PythonArtifact('manylinux2010', 'x64', 'cp37-cp37m'),
376        PythonArtifact('manylinux2010', 'x64', 'cp38-cp38'),
377        PythonArtifact('manylinux2010', 'x64', 'cp39-cp39'),
378        PythonArtifact('manylinux2010', 'x86', 'cp27-cp27m'),
379        PythonArtifact('manylinux2010', 'x86', 'cp27-cp27mu'),
380        PythonArtifact('manylinux2010', 'x86', 'cp35-cp35m'),
381        PythonArtifact('manylinux2010', 'x86', 'cp36-cp36m'),
382        PythonArtifact('manylinux2010', 'x86', 'cp37-cp37m'),
383        PythonArtifact('manylinux2010', 'x86', 'cp38-cp38'),
384        PythonArtifact('manylinux2010', 'x86', 'cp39-cp39'),
385        PythonArtifact('linux_extra', 'armv7', '2.7'),
386        PythonArtifact('linux_extra', 'armv7', '3.5'),
387        PythonArtifact('linux_extra', 'armv7', '3.6'),
388        PythonArtifact('linux_extra', 'armv6', '2.7'),
389        PythonArtifact('linux_extra', 'armv6', '3.5'),
390        PythonArtifact('linux_extra', 'armv6', '3.6'),
391        PythonArtifact('macos', 'x64', 'python2.7'),
392        PythonArtifact('macos', 'x64', 'python3.5'),
393        PythonArtifact('macos', 'x64', 'python3.6'),
394        PythonArtifact('macos', 'x64', 'python3.7'),
395        PythonArtifact('macos', 'x64', 'python3.8'),
396        PythonArtifact('macos', 'x64', 'python3.9'),
397        PythonArtifact('windows', 'x86', 'Python27_32bit'),
398        PythonArtifact('windows', 'x86', 'Python35_32bit'),
399        PythonArtifact('windows', 'x86', 'Python36_32bit'),
400        PythonArtifact('windows', 'x86', 'Python37_32bit'),
401        PythonArtifact('windows', 'x86', 'Python38_32bit'),
402        PythonArtifact('windows', 'x86', 'Python39_32bit'),
403        PythonArtifact('windows', 'x64', 'Python27'),
404        PythonArtifact('windows', 'x64', 'Python35'),
405        PythonArtifact('windows', 'x64', 'Python36'),
406        PythonArtifact('windows', 'x64', 'Python37'),
407        PythonArtifact('windows', 'x64', 'Python38'),
408        PythonArtifact('windows', 'x64', 'Python39'),
409        RubyArtifact('linux', 'x64'),
410        RubyArtifact('macos', 'x64'),
411        PHPArtifact('linux', 'x64')
412    ]
413