1#!/usr/bin/env python 2# Copyright 2016 gRPC authors. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Definition of targets to build artifacts.""" 16 17import os.path 18import random 19import string 20import sys 21 22sys.path.insert(0, os.path.abspath('..')) 23import python_utils.jobset as jobset 24 25 26def create_docker_jobspec(name, 27 dockerfile_dir, 28 shell_command, 29 environ={}, 30 flake_retries=0, 31 timeout_retries=0, 32 timeout_seconds=30 * 60, 33 docker_base_image=None, 34 extra_docker_args=None, 35 verbose_success=False): 36 """Creates jobspec for a task running under docker.""" 37 environ = environ.copy() 38 environ['RUN_COMMAND'] = shell_command 39 environ['ARTIFACTS_OUT'] = 'artifacts/%s' % name 40 41 docker_args = [] 42 for k, v in environ.items(): 43 docker_args += ['-e', '%s=%s' % (k, v)] 44 docker_env = { 45 'DOCKERFILE_DIR': dockerfile_dir, 46 'DOCKER_RUN_SCRIPT': 'tools/run_tests/dockerize/docker_run.sh', 47 'OUTPUT_DIR': 'artifacts' 48 } 49 50 if docker_base_image is not None: 51 docker_env['DOCKER_BASE_IMAGE'] = docker_base_image 52 if extra_docker_args is not None: 53 docker_env['EXTRA_DOCKER_ARGS'] = extra_docker_args 54 jobspec = jobset.JobSpec( 55 cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] + 56 docker_args, 57 environ=docker_env, 58 shortname='build_artifact.%s' % (name), 59 timeout_seconds=timeout_seconds, 60 flake_retries=flake_retries, 61 timeout_retries=timeout_retries, 62 verbose_success=verbose_success) 63 return jobspec 64 65 66def create_jobspec(name, 67 cmdline, 68 environ={}, 69 shell=False, 70 flake_retries=0, 71 timeout_retries=0, 72 timeout_seconds=30 * 60, 73 use_workspace=False, 74 cpu_cost=1.0, 75 verbose_success=False): 76 """Creates jobspec.""" 77 environ = environ.copy() 78 if use_workspace: 79 environ['WORKSPACE_NAME'] = 'workspace_%s' % name 80 environ['ARTIFACTS_OUT'] = os.path.join('..', 'artifacts', name) 81 cmdline = ['bash', 'tools/run_tests/artifacts/run_in_workspace.sh' 82 ] + cmdline 83 else: 84 environ['ARTIFACTS_OUT'] = os.path.join('artifacts', name) 85 86 jobspec = jobset.JobSpec(cmdline=cmdline, 87 environ=environ, 88 shortname='build_artifact.%s' % (name), 89 timeout_seconds=timeout_seconds, 90 flake_retries=flake_retries, 91 timeout_retries=timeout_retries, 92 shell=shell, 93 cpu_cost=cpu_cost, 94 verbose_success=verbose_success) 95 return jobspec 96 97 98_MACOS_COMPAT_FLAG = '-mmacosx-version-min=10.10' 99 100_ARCH_FLAG_MAP = {'x86': '-m32', 'x64': '-m64'} 101 102 103class PythonArtifact: 104 """Builds Python artifacts.""" 105 106 def __init__(self, platform, arch, py_version): 107 self.name = 'python_%s_%s_%s' % (platform, arch, py_version) 108 self.platform = platform 109 self.arch = arch 110 self.labels = ['artifact', 'python', platform, arch, py_version] 111 self.py_version = py_version 112 if 'manylinux' in platform: 113 self.labels.append('linux') 114 115 def pre_build_jobspecs(self): 116 return [] 117 118 def build_jobspec(self): 119 environ = {} 120 if self.platform == 'linux_extra': 121 # Raspberry Pi build 122 environ['PYTHON'] = '/usr/local/bin/python{}'.format( 123 self.py_version) 124 environ['PIP'] = '/usr/local/bin/pip{}'.format(self.py_version) 125 # https://github.com/resin-io-projects/armv7hf-debian-qemu/issues/9 126 # A QEMU bug causes submodule update to hang, so we copy directly 127 environ['RELATIVE_COPY_PATH'] = '.' 128 # Parallel builds are counterproductive in emulated environment 129 environ['GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS'] = '1' 130 extra_args = ' --entrypoint=/usr/bin/qemu-arm-static ' 131 return create_docker_jobspec( 132 self.name, 133 'tools/dockerfile/grpc_artifact_linux_{}'.format(self.arch), 134 'tools/run_tests/artifacts/build_artifact_python.sh', 135 environ=environ, 136 timeout_seconds=60 * 60 * 5, 137 docker_base_image='quay.io/grpc/raspbian_{}'.format(self.arch), 138 extra_docker_args=extra_args) 139 elif 'manylinux' in self.platform: 140 if self.arch == 'x86': 141 environ['SETARCH_CMD'] = 'linux32' 142 # Inside the manylinux container, the python installations are located in 143 # special places... 144 environ['PYTHON'] = '/opt/python/{}/bin/python'.format( 145 self.py_version) 146 environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.py_version) 147 environ['GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS'] = 'TRUE' 148 environ['GRPC_BUILD_MANYLINUX_WHEEL'] = 'TRUE' 149 return create_docker_jobspec( 150 self.name, 151 # NOTE(rbellevi): Do *not* update this without also ensuring the 152 # base_docker_image attribute is accurate. 153 'tools/dockerfile/grpc_artifact_python_%s_%s' % 154 (self.platform, self.arch), 155 'tools/run_tests/artifacts/build_artifact_python.sh', 156 environ=environ, 157 timeout_seconds=60 * 60) 158 elif self.platform == 'windows': 159 if 'Python27' in self.py_version: 160 environ['EXT_COMPILER'] = 'mingw32' 161 else: 162 environ['EXT_COMPILER'] = 'msvc' 163 # For some reason, the batch script %random% always runs with the same 164 # seed. We create a random temp-dir here 165 dir = ''.join( 166 random.choice(string.ascii_uppercase) for _ in range(10)) 167 return create_jobspec(self.name, [ 168 'tools\\run_tests\\artifacts\\build_artifact_python.bat', 169 self.py_version, '32' if self.arch == 'x86' else '64' 170 ], 171 environ=environ, 172 timeout_seconds=45 * 60, 173 use_workspace=True) 174 else: 175 environ['PYTHON'] = self.py_version 176 environ['SKIP_PIP_INSTALL'] = 'TRUE' 177 return create_jobspec( 178 self.name, 179 ['tools/run_tests/artifacts/build_artifact_python.sh'], 180 environ=environ, 181 timeout_seconds=60 * 60 * 2, 182 use_workspace=True) 183 184 def __str__(self): 185 return self.name 186 187 188class RubyArtifact: 189 """Builds ruby native gem.""" 190 191 def __init__(self, platform, arch): 192 self.name = 'ruby_native_gem_%s_%s' % (platform, arch) 193 self.platform = platform 194 self.arch = arch 195 self.labels = ['artifact', 'ruby', platform, arch] 196 197 def pre_build_jobspecs(self): 198 return [] 199 200 def build_jobspec(self): 201 # Ruby build uses docker internally and docker cannot be nested. 202 # We are using a custom workspace instead. 203 return create_jobspec( 204 self.name, ['tools/run_tests/artifacts/build_artifact_ruby.sh'], 205 use_workspace=True, 206 timeout_seconds=45 * 60) 207 208 209class CSharpExtArtifact: 210 """Builds C# native extension library""" 211 212 def __init__(self, platform, arch, arch_abi=None): 213 self.name = 'csharp_ext_%s_%s' % (platform, arch) 214 self.platform = platform 215 self.arch = arch 216 self.arch_abi = arch_abi 217 self.labels = ['artifact', 'csharp', platform, arch] 218 if arch_abi: 219 self.name += '_%s' % arch_abi 220 self.labels.append(arch_abi) 221 222 def pre_build_jobspecs(self): 223 return [] 224 225 def build_jobspec(self): 226 if self.arch == 'android': 227 return create_docker_jobspec( 228 self.name, 229 'tools/dockerfile/grpc_artifact_android_ndk', 230 'tools/run_tests/artifacts/build_artifact_csharp_android.sh', 231 environ={'ANDROID_ABI': self.arch_abi}) 232 elif self.arch == 'ios': 233 return create_jobspec( 234 self.name, 235 ['tools/run_tests/artifacts/build_artifact_csharp_ios.sh'], 236 use_workspace=True) 237 elif self.platform == 'windows': 238 return create_jobspec(self.name, [ 239 'tools\\run_tests\\artifacts\\build_artifact_csharp.bat', 240 self.arch 241 ], 242 use_workspace=True) 243 else: 244 if self.platform == 'linux': 245 cmake_arch_option = '' # x64 is the default architecture 246 if self.arch == 'x86': 247 # TODO(jtattermusch): more work needed to enable 248 # boringssl assembly optimizations for 32-bit linux. 249 # Problem: currently we are building the artifact under 250 # 32-bit docker image, but CMAKE_SYSTEM_PROCESSOR is still 251 # set to x86_64, so the resulting boringssl binary 252 # would have undefined symbols. 253 cmake_arch_option = '-DOPENSSL_NO_ASM=ON' 254 return create_docker_jobspec( 255 self.name, 256 'tools/dockerfile/grpc_artifact_centos6_{}'.format( 257 self.arch), 258 'tools/run_tests/artifacts/build_artifact_csharp.sh', 259 environ={'CMAKE_ARCH_OPTION': cmake_arch_option}) 260 else: 261 cmake_arch_option = '' # x64 is the default architecture 262 if self.arch == 'x86': 263 cmake_arch_option = '-DCMAKE_OSX_ARCHITECTURES=i386' 264 return create_jobspec( 265 self.name, 266 ['tools/run_tests/artifacts/build_artifact_csharp.sh'], 267 environ={'CMAKE_ARCH_OPTION': cmake_arch_option}, 268 use_workspace=True) 269 270 def __str__(self): 271 return self.name 272 273 274class PHPArtifact: 275 """Builds PHP PECL package""" 276 277 def __init__(self, platform, arch): 278 self.name = 'php_pecl_package_{0}_{1}'.format(platform, arch) 279 self.platform = platform 280 self.arch = arch 281 self.labels = ['artifact', 'php', platform, arch] 282 283 def pre_build_jobspecs(self): 284 return [] 285 286 def build_jobspec(self): 287 return create_docker_jobspec( 288 self.name, 289 'tools/dockerfile/test/php73_zts_stretch_{}'.format(self.arch), 290 'tools/run_tests/artifacts/build_artifact_php.sh') 291 292 293class ProtocArtifact: 294 """Builds protoc and protoc-plugin artifacts""" 295 296 def __init__(self, platform, arch): 297 self.name = 'protoc_%s_%s' % (platform, arch) 298 self.platform = platform 299 self.arch = arch 300 self.labels = ['artifact', 'protoc', platform, arch] 301 302 def pre_build_jobspecs(self): 303 return [] 304 305 def build_jobspec(self): 306 if self.platform != 'windows': 307 cxxflags = '-DNDEBUG %s' % _ARCH_FLAG_MAP[self.arch] 308 ldflags = '%s' % _ARCH_FLAG_MAP[self.arch] 309 if self.platform != 'macos': 310 ldflags += ' -static-libgcc -static-libstdc++ -s' 311 environ = { 312 'CONFIG': 'opt', 313 'CXXFLAGS': cxxflags, 314 'LDFLAGS': ldflags, 315 'PROTOBUF_LDFLAGS_EXTRA': ldflags 316 } 317 if self.platform == 'linux': 318 return create_docker_jobspec( 319 self.name, 320 'tools/dockerfile/grpc_artifact_centos6_{}'.format( 321 self.arch), 322 'tools/run_tests/artifacts/build_artifact_protoc.sh', 323 environ=environ) 324 else: 325 environ[ 326 'CXXFLAGS'] += ' -std=c++11 -stdlib=libc++ %s' % _MACOS_COMPAT_FLAG 327 return create_jobspec( 328 self.name, 329 ['tools/run_tests/artifacts/build_artifact_protoc.sh'], 330 environ=environ, 331 timeout_seconds=60 * 60, 332 use_workspace=True) 333 else: 334 generator = 'Visual Studio 14 2015 Win64' if self.arch == 'x64' else 'Visual Studio 14 2015' 335 return create_jobspec( 336 self.name, 337 ['tools\\run_tests\\artifacts\\build_artifact_protoc.bat'], 338 environ={'generator': generator}, 339 use_workspace=True) 340 341 def __str__(self): 342 return self.name 343 344 345def targets(): 346 """Gets list of supported targets""" 347 return [ 348 ProtocArtifact('linux', 'x64'), 349 ProtocArtifact('linux', 'x86'), 350 ProtocArtifact('macos', 'x64'), 351 ProtocArtifact('windows', 'x64'), 352 ProtocArtifact('windows', 'x86'), 353 CSharpExtArtifact('linux', 'x64'), 354 CSharpExtArtifact('macos', 'x64'), 355 CSharpExtArtifact('windows', 'x64'), 356 CSharpExtArtifact('windows', 'x86'), 357 CSharpExtArtifact('linux', 'android', arch_abi='arm64-v8a'), 358 CSharpExtArtifact('linux', 'android', arch_abi='armeabi-v7a'), 359 CSharpExtArtifact('linux', 'android', arch_abi='x86'), 360 CSharpExtArtifact('macos', 'ios'), 361 PythonArtifact('manylinux2014', 'x64', 'cp35-cp35m'), 362 PythonArtifact('manylinux2014', 'x64', 'cp36-cp36m'), 363 PythonArtifact('manylinux2014', 'x64', 'cp37-cp37m'), 364 PythonArtifact('manylinux2014', 'x64', 'cp38-cp38'), 365 PythonArtifact('manylinux2014', 'x64', 'cp39-cp39'), 366 PythonArtifact('manylinux2014', 'x86', 'cp35-cp35m'), 367 PythonArtifact('manylinux2014', 'x86', 'cp36-cp36m'), 368 PythonArtifact('manylinux2014', 'x86', 'cp37-cp37m'), 369 PythonArtifact('manylinux2014', 'x86', 'cp38-cp38'), 370 PythonArtifact('manylinux2014', 'x86', 'cp39-cp39'), 371 PythonArtifact('manylinux2010', 'x64', 'cp27-cp27m'), 372 PythonArtifact('manylinux2010', 'x64', 'cp27-cp27mu'), 373 PythonArtifact('manylinux2010', 'x64', 'cp35-cp35m'), 374 PythonArtifact('manylinux2010', 'x64', 'cp36-cp36m'), 375 PythonArtifact('manylinux2010', 'x64', 'cp37-cp37m'), 376 PythonArtifact('manylinux2010', 'x64', 'cp38-cp38'), 377 PythonArtifact('manylinux2010', 'x64', 'cp39-cp39'), 378 PythonArtifact('manylinux2010', 'x86', 'cp27-cp27m'), 379 PythonArtifact('manylinux2010', 'x86', 'cp27-cp27mu'), 380 PythonArtifact('manylinux2010', 'x86', 'cp35-cp35m'), 381 PythonArtifact('manylinux2010', 'x86', 'cp36-cp36m'), 382 PythonArtifact('manylinux2010', 'x86', 'cp37-cp37m'), 383 PythonArtifact('manylinux2010', 'x86', 'cp38-cp38'), 384 PythonArtifact('manylinux2010', 'x86', 'cp39-cp39'), 385 PythonArtifact('linux_extra', 'armv7', '2.7'), 386 PythonArtifact('linux_extra', 'armv7', '3.5'), 387 PythonArtifact('linux_extra', 'armv7', '3.6'), 388 PythonArtifact('linux_extra', 'armv6', '2.7'), 389 PythonArtifact('linux_extra', 'armv6', '3.5'), 390 PythonArtifact('linux_extra', 'armv6', '3.6'), 391 PythonArtifact('macos', 'x64', 'python2.7'), 392 PythonArtifact('macos', 'x64', 'python3.5'), 393 PythonArtifact('macos', 'x64', 'python3.6'), 394 PythonArtifact('macos', 'x64', 'python3.7'), 395 PythonArtifact('macos', 'x64', 'python3.8'), 396 PythonArtifact('macos', 'x64', 'python3.9'), 397 PythonArtifact('windows', 'x86', 'Python27_32bit'), 398 PythonArtifact('windows', 'x86', 'Python35_32bit'), 399 PythonArtifact('windows', 'x86', 'Python36_32bit'), 400 PythonArtifact('windows', 'x86', 'Python37_32bit'), 401 PythonArtifact('windows', 'x86', 'Python38_32bit'), 402 PythonArtifact('windows', 'x86', 'Python39_32bit'), 403 PythonArtifact('windows', 'x64', 'Python27'), 404 PythonArtifact('windows', 'x64', 'Python35'), 405 PythonArtifact('windows', 'x64', 'Python36'), 406 PythonArtifact('windows', 'x64', 'Python37'), 407 PythonArtifact('windows', 'x64', 'Python38'), 408 PythonArtifact('windows', 'x64', 'Python39'), 409 RubyArtifact('linux', 'x64'), 410 RubyArtifact('macos', 'x64'), 411 PHPArtifact('linux', 'x64') 412 ] 413