1# Copyright 2016 gRPC authors. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15from distutils import cygwinccompiler 16from distutils import extension 17from distutils import util 18import errno 19import os 20import os.path 21import platform 22import re 23import shlex 24import shutil 25import subprocess 26from subprocess import PIPE 27import sys 28import sysconfig 29 30import pkg_resources 31import setuptools 32from setuptools.command import build_ext 33 34# TODO(atash) add flag to disable Cython use 35 36_PACKAGE_PATH = os.path.realpath(os.path.dirname(__file__)) 37_README_PATH = os.path.join(_PACKAGE_PATH, 'README.rst') 38 39os.chdir(os.path.dirname(os.path.abspath(__file__))) 40sys.path.insert(0, os.path.abspath('.')) 41 42import _parallel_compile_patch 43import protoc_lib_deps 44 45import grpc_version 46 47_EXT_INIT_SYMBOL = None 48if sys.version_info[0] == 2: 49 _EXT_INIT_SYMBOL = "init_protoc_compiler" 50else: 51 _EXT_INIT_SYMBOL = "PyInit__protoc_compiler" 52 53_parallel_compile_patch.monkeypatch_compile_maybe() 54 55CLASSIFIERS = [ 56 'Development Status :: 5 - Production/Stable', 57 'Programming Language :: Python', 58 'Programming Language :: Python :: 3', 59 'License :: OSI Approved :: Apache Software License', 60] 61 62PY3 = sys.version_info.major == 3 63 64 65def _env_bool_value(env_name, default): 66 """Parses a bool option from an environment variable""" 67 return os.environ.get(env_name, default).upper() not in ['FALSE', '0', ''] 68 69 70# Environment variable to determine whether or not the Cython extension should 71# *use* Cython or use the generated C files. Note that this requires the C files 72# to have been generated by building first *with* Cython support. 73BUILD_WITH_CYTHON = _env_bool_value('GRPC_PYTHON_BUILD_WITH_CYTHON', 'False') 74 75# Export this variable to force building the python extension with a statically linked libstdc++. 76# At least on linux, this is normally not needed as we can build manylinux-compatible wheels on linux just fine 77# without statically linking libstdc++ (which leads to a slight increase in the wheel size). 78# This option is useful when crosscompiling wheels for aarch64 where 79# it's difficult to ensure that the crosscompilation toolchain has a high-enough version 80# of GCC (we require >4.9) but still uses old-enough libstdc++ symbols. 81# TODO(jtattermusch): remove this workaround once issues with crosscompiler version are resolved. 82BUILD_WITH_STATIC_LIBSTDCXX = _env_bool_value( 83 'GRPC_PYTHON_BUILD_WITH_STATIC_LIBSTDCXX', 'False') 84 85 86def check_linker_need_libatomic(): 87 """Test if linker on system needs libatomic.""" 88 code_test = (b'#include <atomic>\n' + 89 b'int main() { return std::atomic<int64_t>{}; }') 90 cxx = os.environ.get('CXX', 'c++') 91 cpp_test = subprocess.Popen([cxx, '-x', 'c++', '-std=c++11', '-'], 92 stdin=PIPE, 93 stdout=PIPE, 94 stderr=PIPE) 95 cpp_test.communicate(input=code_test) 96 if cpp_test.returncode == 0: 97 return False 98 # Double-check to see if -latomic actually can solve the problem. 99 # https://github.com/grpc/grpc/issues/22491 100 cpp_test = subprocess.Popen( 101 [cxx, '-x', 'c++', '-std=c++11', '-latomic', '-'], 102 stdin=PIPE, 103 stdout=PIPE, 104 stderr=PIPE) 105 cpp_test.communicate(input=code_test) 106 return cpp_test.returncode == 0 107 108 109class BuildExt(build_ext.build_ext): 110 """Custom build_ext command.""" 111 112 def get_ext_filename(self, ext_name): 113 # since python3.5, python extensions' shared libraries use a suffix that corresponds to the value 114 # of sysconfig.get_config_var('EXT_SUFFIX') and contains info about the architecture the library targets. 115 # E.g. on x64 linux the suffix is ".cpython-XYZ-x86_64-linux-gnu.so" 116 # When crosscompiling python wheels, we need to be able to override this suffix 117 # so that the resulting file name matches the target architecture and we end up with a well-formed 118 # wheel. 119 filename = build_ext.build_ext.get_ext_filename(self, ext_name) 120 orig_ext_suffix = sysconfig.get_config_var('EXT_SUFFIX') 121 new_ext_suffix = os.getenv('GRPC_PYTHON_OVERRIDE_EXT_SUFFIX') 122 if new_ext_suffix and filename.endswith(orig_ext_suffix): 123 filename = filename[:-len(orig_ext_suffix)] + new_ext_suffix 124 return filename 125 126 127# There are some situations (like on Windows) where CC, CFLAGS, and LDFLAGS are 128# entirely ignored/dropped/forgotten by distutils and its Cygwin/MinGW support. 129# We use these environment variables to thus get around that without locking 130# ourselves in w.r.t. the multitude of operating systems this ought to build on. 131# We can also use these variables as a way to inject environment-specific 132# compiler/linker flags. We assume GCC-like compilers and/or MinGW as a 133# reasonable default. 134EXTRA_ENV_COMPILE_ARGS = os.environ.get('GRPC_PYTHON_CFLAGS', None) 135EXTRA_ENV_LINK_ARGS = os.environ.get('GRPC_PYTHON_LDFLAGS', None) 136if EXTRA_ENV_COMPILE_ARGS is None: 137 EXTRA_ENV_COMPILE_ARGS = '-std=c++11' 138 if 'win32' in sys.platform: 139 if sys.version_info < (3, 5): 140 # We use define flags here and don't directly add to DEFINE_MACROS below to 141 # ensure that the expert user/builder has a way of turning it off (via the 142 # envvars) without adding yet more GRPC-specific envvars. 143 # See https://sourceforge.net/p/mingw-w64/bugs/363/ 144 if '32' in platform.architecture()[0]: 145 EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime32 -D_timeb=__timeb32 -D_ftime_s=_ftime32_s -D_hypot=hypot' 146 else: 147 EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime64 -D_timeb=__timeb64 -D_hypot=hypot' 148 else: 149 # We need to statically link the C++ Runtime, only the C runtime is 150 # available dynamically 151 EXTRA_ENV_COMPILE_ARGS += ' /MT' 152 elif "linux" in sys.platform or "darwin" in sys.platform: 153 EXTRA_ENV_COMPILE_ARGS += ' -fno-wrapv -frtti' 154if EXTRA_ENV_LINK_ARGS is None: 155 EXTRA_ENV_LINK_ARGS = '' 156 # NOTE(rbellevi): Clang on Mac OS will make all static symbols (both 157 # variables and objects) global weak symbols. When a process loads the 158 # protobuf wheel's shared object library before loading *this* C extension, 159 # the runtime linker will prefer the protobuf module's version of symbols. 160 # This results in the process using a mixture of symbols from the protobuf 161 # wheel and this wheel, which may be using different versions of 162 # libprotobuf. In the case that they *are* using different versions of 163 # libprotobuf *and* there has been a change in data layout (or in other 164 # invariants) segfaults, data corruption, or "bad things" may happen. 165 # 166 # This flag ensures that on Mac, the only global symbol is the one loaded by 167 # the Python interpreter. The problematic global weak symbols become local 168 # weak symbols. This is not required on Linux since the compiler does not 169 # produce global weak symbols. This is not required on Windows as our ".pyd" 170 # file does not contain any symbols. 171 # 172 # Finally, the leading underscore here is part of the Mach-O ABI. Unlike 173 # more modern ABIs (ELF et al.), Mach-O prepends an underscore to the names 174 # of C functions. 175 if "darwin" in sys.platform: 176 EXTRA_ENV_LINK_ARGS += ' -Wl,-exported_symbol,_{}'.format( 177 _EXT_INIT_SYMBOL) 178 if "linux" in sys.platform or "darwin" in sys.platform: 179 EXTRA_ENV_LINK_ARGS += ' -lpthread' 180 if check_linker_need_libatomic(): 181 EXTRA_ENV_LINK_ARGS += ' -latomic' 182 elif "win32" in sys.platform and sys.version_info < (3, 5): 183 msvcr = cygwinccompiler.get_msvcr()[0] 184 EXTRA_ENV_LINK_ARGS += ( 185 ' -static-libgcc -static-libstdc++ -mcrtdll={msvcr}' 186 ' -static -lshlwapi'.format(msvcr=msvcr)) 187 188EXTRA_COMPILE_ARGS = shlex.split(EXTRA_ENV_COMPILE_ARGS) 189EXTRA_LINK_ARGS = shlex.split(EXTRA_ENV_LINK_ARGS) 190 191if BUILD_WITH_STATIC_LIBSTDCXX: 192 EXTRA_LINK_ARGS.append('-static-libstdc++') 193 194CC_FILES = [os.path.normpath(cc_file) for cc_file in protoc_lib_deps.CC_FILES] 195PROTO_FILES = [ 196 os.path.normpath(proto_file) for proto_file in protoc_lib_deps.PROTO_FILES 197] 198CC_INCLUDE = os.path.normpath(protoc_lib_deps.CC_INCLUDE) 199PROTO_INCLUDE = os.path.normpath(protoc_lib_deps.PROTO_INCLUDE) 200 201GRPC_PYTHON_TOOLS_PACKAGE = 'grpc_tools' 202GRPC_PYTHON_PROTO_RESOURCES_NAME = '_proto' 203 204DEFINE_MACROS = () 205if "win32" in sys.platform: 206 DEFINE_MACROS += (('WIN32_LEAN_AND_MEAN', 1),) 207 if '64bit' in platform.architecture()[0]: 208 DEFINE_MACROS += (('MS_WIN64', 1),) 209elif "linux" in sys.platform or "darwin" in sys.platform: 210 DEFINE_MACROS += (('HAVE_PTHREAD', 1),) 211 212# By default, Python3 distutils enforces compatibility of 213# c plugins (.so files) with the OSX version Python was built with. 214# We need OSX 10.10, the oldest which supports C++ thread_local. 215if 'darwin' in sys.platform: 216 mac_target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') 217 if mac_target and (pkg_resources.parse_version(mac_target) < 218 pkg_resources.parse_version('10.10.0')): 219 os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.10' 220 os.environ['_PYTHON_HOST_PLATFORM'] = re.sub( 221 r'macosx-[0-9]+\.[0-9]+-(.+)', r'macosx-10.10-\1', 222 util.get_platform()) 223 224 225def package_data(): 226 tools_path = GRPC_PYTHON_TOOLS_PACKAGE.replace('.', os.path.sep) 227 proto_resources_path = os.path.join(tools_path, 228 GRPC_PYTHON_PROTO_RESOURCES_NAME) 229 proto_files = [] 230 for proto_file in PROTO_FILES: 231 source = os.path.join(PROTO_INCLUDE, proto_file) 232 target = os.path.join(proto_resources_path, proto_file) 233 relative_target = os.path.join(GRPC_PYTHON_PROTO_RESOURCES_NAME, 234 proto_file) 235 try: 236 os.makedirs(os.path.dirname(target)) 237 except OSError as error: 238 if error.errno == errno.EEXIST: 239 pass 240 else: 241 raise 242 shutil.copy(source, target) 243 proto_files.append(relative_target) 244 return {GRPC_PYTHON_TOOLS_PACKAGE: proto_files} 245 246 247def extension_modules(): 248 if BUILD_WITH_CYTHON: 249 plugin_sources = [os.path.join('grpc_tools', '_protoc_compiler.pyx')] 250 else: 251 plugin_sources = [os.path.join('grpc_tools', '_protoc_compiler.cpp')] 252 253 plugin_sources += [ 254 os.path.join('grpc_tools', 'main.cc'), 255 os.path.join('grpc_root', 'src', 'compiler', 'python_generator.cc') 256 ] + [os.path.join(CC_INCLUDE, cc_file) for cc_file in CC_FILES] 257 258 plugin_ext = extension.Extension( 259 name='grpc_tools._protoc_compiler', 260 sources=plugin_sources, 261 include_dirs=[ 262 '.', 263 'grpc_root', 264 os.path.join('grpc_root', 'include'), 265 CC_INCLUDE, 266 ], 267 language='c++', 268 define_macros=list(DEFINE_MACROS), 269 extra_compile_args=list(EXTRA_COMPILE_ARGS), 270 extra_link_args=list(EXTRA_LINK_ARGS), 271 ) 272 extensions = [plugin_ext] 273 if BUILD_WITH_CYTHON: 274 from Cython import Build 275 return Build.cythonize(extensions) 276 else: 277 return extensions 278 279 280setuptools.setup(name='grpcio-tools', 281 version=grpc_version.VERSION, 282 description='Protobuf code generator for gRPC', 283 long_description=open(_README_PATH, 'r').read(), 284 author='The gRPC Authors', 285 author_email='grpc-io@googlegroups.com', 286 url='https://grpc.io', 287 license='Apache License 2.0', 288 classifiers=CLASSIFIERS, 289 ext_modules=extension_modules(), 290 packages=setuptools.find_packages('.'), 291 python_requires='>=3.6', 292 install_requires=[ 293 'protobuf>=3.5.0.post1, < 4.0dev', 294 'grpcio>={version}'.format(version=grpc_version.VERSION), 295 'setuptools', 296 ], 297 package_data=package_data(), 298 cmdclass={ 299 'build_ext': BuildExt, 300 }) 301