1#!/usr/bin/env python 2# Copyright 2016 gRPC authors. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Definition of targets to build artifacts.""" 16 17import os.path 18import random 19import string 20import sys 21 22sys.path.insert(0, os.path.abspath('..')) 23import python_utils.jobset as jobset 24 25 26def create_docker_jobspec(name, 27 dockerfile_dir, 28 shell_command, 29 environ={}, 30 flake_retries=0, 31 timeout_retries=0, 32 timeout_seconds=30 * 60, 33 docker_base_image=None, 34 extra_docker_args=None, 35 verbose_success=False): 36 """Creates jobspec for a task running under docker.""" 37 environ = environ.copy() 38 environ['RUN_COMMAND'] = shell_command 39 environ['ARTIFACTS_OUT'] = 'artifacts/%s' % name 40 41 docker_args = [] 42 for k, v in environ.items(): 43 docker_args += ['-e', '%s=%s' % (k, v)] 44 docker_env = { 45 'DOCKERFILE_DIR': dockerfile_dir, 46 'DOCKER_RUN_SCRIPT': 'tools/run_tests/dockerize/docker_run.sh', 47 'OUTPUT_DIR': 'artifacts' 48 } 49 50 if docker_base_image is not None: 51 docker_env['DOCKER_BASE_IMAGE'] = docker_base_image 52 if extra_docker_args is not None: 53 docker_env['EXTRA_DOCKER_ARGS'] = extra_docker_args 54 jobspec = jobset.JobSpec( 55 cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] + 56 docker_args, 57 environ=docker_env, 58 shortname='build_artifact.%s' % (name), 59 timeout_seconds=timeout_seconds, 60 flake_retries=flake_retries, 61 timeout_retries=timeout_retries, 62 verbose_success=verbose_success) 63 return jobspec 64 65 66def create_jobspec(name, 67 cmdline, 68 environ={}, 69 shell=False, 70 flake_retries=0, 71 timeout_retries=0, 72 timeout_seconds=30 * 60, 73 use_workspace=False, 74 cpu_cost=1.0, 75 verbose_success=False): 76 """Creates jobspec.""" 77 environ = environ.copy() 78 if use_workspace: 79 environ['WORKSPACE_NAME'] = 'workspace_%s' % name 80 environ['ARTIFACTS_OUT'] = os.path.join('..', 'artifacts', name) 81 cmdline = ['bash', 'tools/run_tests/artifacts/run_in_workspace.sh' 82 ] + cmdline 83 else: 84 environ['ARTIFACTS_OUT'] = os.path.join('artifacts', name) 85 86 jobspec = jobset.JobSpec( 87 cmdline=cmdline, 88 environ=environ, 89 shortname='build_artifact.%s' % (name), 90 timeout_seconds=timeout_seconds, 91 flake_retries=flake_retries, 92 timeout_retries=timeout_retries, 93 shell=shell, 94 cpu_cost=cpu_cost, 95 verbose_success=verbose_success) 96 return jobspec 97 98 99_MACOS_COMPAT_FLAG = '-mmacosx-version-min=10.7' 100 101_ARCH_FLAG_MAP = {'x86': '-m32', 'x64': '-m64'} 102 103 104class PythonArtifact: 105 """Builds Python artifacts.""" 106 107 def __init__(self, platform, arch, py_version): 108 self.name = 'python_%s_%s_%s' % (platform, arch, py_version) 109 self.platform = platform 110 self.arch = arch 111 self.labels = ['artifact', 'python', platform, arch, py_version] 112 self.py_version = py_version 113 114 def pre_build_jobspecs(self): 115 return [] 116 117 def build_jobspec(self): 118 environ = {} 119 if self.platform == 'linux_extra': 120 # Raspberry Pi build 121 environ['PYTHON'] = '/usr/local/bin/python{}'.format( 122 self.py_version) 123 environ['PIP'] = '/usr/local/bin/pip{}'.format(self.py_version) 124 # https://github.com/resin-io-projects/armv7hf-debian-qemu/issues/9 125 # A QEMU bug causes submodule update to hang, so we copy directly 126 environ['RELATIVE_COPY_PATH'] = '.' 127 # Parallel builds are counterproductive in emulated environment 128 environ['GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS'] = '1' 129 extra_args = ' --entrypoint=/usr/bin/qemu-arm-static ' 130 return create_docker_jobspec( 131 self.name, 132 'tools/dockerfile/grpc_artifact_linux_{}'.format(self.arch), 133 'tools/run_tests/artifacts/build_artifact_python.sh', 134 environ=environ, 135 timeout_seconds=60 * 60 * 5, 136 docker_base_image='quay.io/grpc/raspbian_{}'.format(self.arch), 137 extra_docker_args=extra_args) 138 elif self.platform == 'linux': 139 if self.arch == 'x86': 140 environ['SETARCH_CMD'] = 'linux32' 141 # Inside the manylinux container, the python installations are located in 142 # special places... 143 environ['PYTHON'] = '/opt/python/{}/bin/python'.format( 144 self.py_version) 145 environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.py_version) 146 # Platform autodetection for the manylinux1 image breaks so we set the 147 # defines ourselves. 148 # TODO(atash) get better platform-detection support in core so we don't 149 # need to do this manually... 150 environ['CFLAGS'] = '-DGPR_MANYLINUX1=1' 151 environ['GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS'] = 'TRUE' 152 environ['GRPC_BUILD_MANYLINUX_WHEEL'] = 'TRUE' 153 return create_docker_jobspec( 154 self.name, 155 'tools/dockerfile/grpc_artifact_python_manylinux_%s' % 156 self.arch, 157 'tools/run_tests/artifacts/build_artifact_python.sh', 158 environ=environ, 159 timeout_seconds=60 * 60, 160 docker_base_image='quay.io/pypa/manylinux1_i686' 161 if self.arch == 'x86' else 'quay.io/pypa/manylinux1_x86_64') 162 elif self.platform == 'windows': 163 if 'Python27' in self.py_version or 'Python34' in self.py_version: 164 environ['EXT_COMPILER'] = 'mingw32' 165 else: 166 environ['EXT_COMPILER'] = 'msvc' 167 # For some reason, the batch script %random% always runs with the same 168 # seed. We create a random temp-dir here 169 dir = ''.join( 170 random.choice(string.ascii_uppercase) for _ in range(10)) 171 return create_jobspec( 172 self.name, [ 173 'tools\\run_tests\\artifacts\\build_artifact_python.bat', 174 self.py_version, '32' if self.arch == 'x86' else '64' 175 ], 176 environ=environ, 177 timeout_seconds=45 * 60, 178 use_workspace=True) 179 else: 180 environ['PYTHON'] = self.py_version 181 environ['SKIP_PIP_INSTALL'] = 'TRUE' 182 return create_jobspec( 183 self.name, 184 ['tools/run_tests/artifacts/build_artifact_python.sh'], 185 environ=environ, 186 timeout_seconds=60 * 60 * 2, 187 use_workspace=True) 188 189 def __str__(self): 190 return self.name 191 192 193class RubyArtifact: 194 """Builds ruby native gem.""" 195 196 def __init__(self, platform, arch): 197 self.name = 'ruby_native_gem_%s_%s' % (platform, arch) 198 self.platform = platform 199 self.arch = arch 200 self.labels = ['artifact', 'ruby', platform, arch] 201 202 def pre_build_jobspecs(self): 203 return [] 204 205 def build_jobspec(self): 206 # Ruby build uses docker internally and docker cannot be nested. 207 # We are using a custom workspace instead. 208 return create_jobspec( 209 self.name, ['tools/run_tests/artifacts/build_artifact_ruby.sh'], 210 use_workspace=True, 211 timeout_seconds=45 * 60) 212 213 214class CSharpExtArtifact: 215 """Builds C# native extension library""" 216 217 def __init__(self, platform, arch, arch_abi=None): 218 self.name = 'csharp_ext_%s_%s' % (platform, arch) 219 self.platform = platform 220 self.arch = arch 221 self.arch_abi = arch_abi 222 self.labels = ['artifact', 'csharp', platform, arch] 223 if arch_abi: 224 self.name += '_%s' % arch_abi 225 self.labels.append(arch_abi) 226 227 def pre_build_jobspecs(self): 228 return [] 229 230 def build_jobspec(self): 231 if self.arch == 'android': 232 return create_docker_jobspec( 233 self.name, 234 'tools/dockerfile/grpc_artifact_android_ndk', 235 'tools/run_tests/artifacts/build_artifact_csharp_android.sh', 236 environ={ 237 'ANDROID_ABI': self.arch_abi 238 }) 239 elif self.arch == 'ios': 240 return create_jobspec( 241 self.name, 242 ['tools/run_tests/artifacts/build_artifact_csharp_ios.sh'], 243 use_workspace=True) 244 elif self.platform == 'windows': 245 return create_jobspec( 246 self.name, [ 247 'tools\\run_tests\\artifacts\\build_artifact_csharp.bat', 248 self.arch 249 ], 250 use_workspace=True) 251 else: 252 if self.platform == 'linux': 253 cmake_arch_option = '' # x64 is the default architecture 254 if self.arch == 'x86': 255 # TODO(jtattermusch): more work needed to enable 256 # boringssl assembly optimizations for 32-bit linux. 257 # Problem: currently we are building the artifact under 258 # 32-bit docker image, but CMAKE_SYSTEM_PROCESSOR is still 259 # set to x86_64, so the resulting boringssl binary 260 # would have undefined symbols. 261 cmake_arch_option = '-DOPENSSL_NO_ASM=ON' 262 return create_docker_jobspec( 263 self.name, 264 'tools/dockerfile/grpc_artifact_linux_%s' % self.arch, 265 'tools/run_tests/artifacts/build_artifact_csharp.sh', 266 environ={ 267 'CMAKE_ARCH_OPTION': cmake_arch_option 268 }) 269 else: 270 cmake_arch_option = '' # x64 is the default architecture 271 if self.arch == 'x86': 272 cmake_arch_option = '-DCMAKE_OSX_ARCHITECTURES=i386' 273 return create_jobspec( 274 self.name, 275 ['tools/run_tests/artifacts/build_artifact_csharp.sh'], 276 environ={'CMAKE_ARCH_OPTION': cmake_arch_option}, 277 use_workspace=True) 278 279 def __str__(self): 280 return self.name 281 282 283class PHPArtifact: 284 """Builds PHP PECL package""" 285 286 def __init__(self, platform, arch): 287 self.name = 'php_pecl_package_{0}_{1}'.format(platform, arch) 288 self.platform = platform 289 self.arch = arch 290 self.labels = ['artifact', 'php', platform, arch] 291 292 def pre_build_jobspecs(self): 293 return [] 294 295 def build_jobspec(self): 296 return create_docker_jobspec( 297 self.name, 'tools/dockerfile/grpc_artifact_linux_{}'.format( 298 self.arch), 'tools/run_tests/artifacts/build_artifact_php.sh') 299 300 301class ProtocArtifact: 302 """Builds protoc and protoc-plugin artifacts""" 303 304 def __init__(self, platform, arch): 305 self.name = 'protoc_%s_%s' % (platform, arch) 306 self.platform = platform 307 self.arch = arch 308 self.labels = ['artifact', 'protoc', platform, arch] 309 310 def pre_build_jobspecs(self): 311 return [] 312 313 def build_jobspec(self): 314 if self.platform != 'windows': 315 cxxflags = '-DNDEBUG %s' % _ARCH_FLAG_MAP[self.arch] 316 ldflags = '%s' % _ARCH_FLAG_MAP[self.arch] 317 if self.platform != 'macos': 318 ldflags += ' -static-libgcc -static-libstdc++ -s' 319 environ = { 320 'CONFIG': 'opt', 321 'CXXFLAGS': cxxflags, 322 'LDFLAGS': ldflags, 323 'PROTOBUF_LDFLAGS_EXTRA': ldflags 324 } 325 if self.platform == 'linux': 326 return create_docker_jobspec( 327 self.name, 328 'tools/dockerfile/grpc_artifact_protoc', 329 'tools/run_tests/artifacts/build_artifact_protoc.sh', 330 environ=environ) 331 else: 332 environ[ 333 'CXXFLAGS'] += ' -std=c++11 -stdlib=libc++ %s' % _MACOS_COMPAT_FLAG 334 return create_jobspec( 335 self.name, 336 ['tools/run_tests/artifacts/build_artifact_protoc.sh'], 337 environ=environ, 338 timeout_seconds=60 * 60, 339 use_workspace=True) 340 else: 341 generator = 'Visual Studio 14 2015 Win64' if self.arch == 'x64' else 'Visual Studio 14 2015' 342 return create_jobspec( 343 self.name, 344 ['tools\\run_tests\\artifacts\\build_artifact_protoc.bat'], 345 environ={'generator': generator}, 346 use_workspace=True) 347 348 def __str__(self): 349 return self.name 350 351 352def targets(): 353 """Gets list of supported targets""" 354 return ([ 355 Cls(platform, arch) 356 for Cls in (CSharpExtArtifact, ProtocArtifact) 357 for platform in ('linux', 'macos', 'windows') for arch in ('x86', 'x64') 358 ] + [ 359 CSharpExtArtifact('linux', 'android', arch_abi='arm64-v8a'), 360 CSharpExtArtifact('linux', 'android', arch_abi='armeabi-v7a'), 361 CSharpExtArtifact('linux', 'android', arch_abi='x86'), 362 CSharpExtArtifact('macos', 'ios'), 363 PythonArtifact('linux', 'x86', 'cp27-cp27m'), 364 PythonArtifact('linux', 'x86', 'cp27-cp27mu'), 365 PythonArtifact('linux', 'x86', 'cp34-cp34m'), 366 PythonArtifact('linux', 'x86', 'cp35-cp35m'), 367 PythonArtifact('linux', 'x86', 'cp36-cp36m'), 368 PythonArtifact('linux', 'x86', 'cp37-cp37m'), 369 PythonArtifact('linux_extra', 'armv7', '2.7'), 370 PythonArtifact('linux_extra', 'armv7', '3.4'), 371 PythonArtifact('linux_extra', 'armv7', '3.5'), 372 PythonArtifact('linux_extra', 'armv7', '3.6'), 373 PythonArtifact('linux_extra', 'armv6', '2.7'), 374 PythonArtifact('linux_extra', 'armv6', '3.4'), 375 PythonArtifact('linux_extra', 'armv6', '3.5'), 376 PythonArtifact('linux_extra', 'armv6', '3.6'), 377 PythonArtifact('linux', 'x64', 'cp27-cp27m'), 378 PythonArtifact('linux', 'x64', 'cp27-cp27mu'), 379 PythonArtifact('linux', 'x64', 'cp34-cp34m'), 380 PythonArtifact('linux', 'x64', 'cp35-cp35m'), 381 PythonArtifact('linux', 'x64', 'cp36-cp36m'), 382 PythonArtifact('linux', 'x64', 'cp37-cp37m'), 383 PythonArtifact('macos', 'x64', 'python2.7'), 384 PythonArtifact('macos', 'x64', 'python3.4'), 385 PythonArtifact('macos', 'x64', 'python3.5'), 386 PythonArtifact('macos', 'x64', 'python3.6'), 387 PythonArtifact('macos', 'x64', 'python3.7'), 388 PythonArtifact('windows', 'x86', 'Python27_32bits'), 389 PythonArtifact('windows', 'x86', 'Python34_32bits'), 390 PythonArtifact('windows', 'x86', 'Python35_32bits'), 391 PythonArtifact('windows', 'x86', 'Python36_32bits'), 392 PythonArtifact('windows', 'x86', 'Python37_32bits'), 393 PythonArtifact('windows', 'x64', 'Python27'), 394 PythonArtifact('windows', 'x64', 'Python34'), 395 PythonArtifact('windows', 'x64', 'Python35'), 396 PythonArtifact('windows', 'x64', 'Python36'), 397 PythonArtifact('windows', 'x64', 'Python37'), 398 RubyArtifact('linux', 'x64'), 399 RubyArtifact('macos', 'x64'), 400 PHPArtifact('linux', 'x64') 401 ]) 402