1# Copyright 2016 gRPC authors.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15from distutils import cygwinccompiler
16from distutils import extension
17from distutils import util
18import errno
19import os
20import os.path
21import platform
22import re
23import shlex
24import shutil
25import subprocess
26from subprocess import PIPE
27import sys
28import sysconfig
29
30import pkg_resources
31import setuptools
32from setuptools.command import build_ext
33
34# TODO(atash) add flag to disable Cython use
35
36_PACKAGE_PATH = os.path.realpath(os.path.dirname(__file__))
37_README_PATH = os.path.join(_PACKAGE_PATH, 'README.rst')
38
39os.chdir(os.path.dirname(os.path.abspath(__file__)))
40sys.path.insert(0, os.path.abspath('.'))
41
42import _parallel_compile_patch
43import protoc_lib_deps
44
45import grpc_version
46
47_EXT_INIT_SYMBOL = None
48if sys.version_info[0] == 2:
49    _EXT_INIT_SYMBOL = "init_protoc_compiler"
50else:
51    _EXT_INIT_SYMBOL = "PyInit__protoc_compiler"
52
53_parallel_compile_patch.monkeypatch_compile_maybe()
54
55CLASSIFIERS = [
56    'Development Status :: 5 - Production/Stable',
57    'Programming Language :: Python',
58    'Programming Language :: Python :: 3',
59    'License :: OSI Approved :: Apache Software License',
60]
61
62PY3 = sys.version_info.major == 3
63
64
65def _env_bool_value(env_name, default):
66    """Parses a bool option from an environment variable"""
67    return os.environ.get(env_name, default).upper() not in ['FALSE', '0', '']
68
69
70# Environment variable to determine whether or not the Cython extension should
71# *use* Cython or use the generated C files. Note that this requires the C files
72# to have been generated by building first *with* Cython support.
73BUILD_WITH_CYTHON = _env_bool_value('GRPC_PYTHON_BUILD_WITH_CYTHON', 'False')
74
75# Export this variable to force building the python extension with a statically linked libstdc++.
76# At least on linux, this is normally not needed as we can build manylinux-compatible wheels on linux just fine
77# without statically linking libstdc++ (which leads to a slight increase in the wheel size).
78# This option is useful when crosscompiling wheels for aarch64 where
79# it's difficult to ensure that the crosscompilation toolchain has a high-enough version
80# of GCC (we require >4.9) but still uses old-enough libstdc++ symbols.
81# TODO(jtattermusch): remove this workaround once issues with crosscompiler version are resolved.
82BUILD_WITH_STATIC_LIBSTDCXX = _env_bool_value(
83    'GRPC_PYTHON_BUILD_WITH_STATIC_LIBSTDCXX', 'False')
84
85
86def check_linker_need_libatomic():
87    """Test if linker on system needs libatomic."""
88    code_test = (b'#include <atomic>\n' +
89                 b'int main() { return std::atomic<int64_t>{}; }')
90    cxx = os.environ.get('CXX', 'c++')
91    cpp_test = subprocess.Popen([cxx, '-x', 'c++', '-std=c++11', '-'],
92                                stdin=PIPE,
93                                stdout=PIPE,
94                                stderr=PIPE)
95    cpp_test.communicate(input=code_test)
96    if cpp_test.returncode == 0:
97        return False
98    # Double-check to see if -latomic actually can solve the problem.
99    # https://github.com/grpc/grpc/issues/22491
100    cpp_test = subprocess.Popen(
101        [cxx, '-x', 'c++', '-std=c++11', '-latomic', '-'],
102        stdin=PIPE,
103        stdout=PIPE,
104        stderr=PIPE)
105    cpp_test.communicate(input=code_test)
106    return cpp_test.returncode == 0
107
108
109class BuildExt(build_ext.build_ext):
110    """Custom build_ext command."""
111
112    def get_ext_filename(self, ext_name):
113        # since python3.5, python extensions' shared libraries use a suffix that corresponds to the value
114        # of sysconfig.get_config_var('EXT_SUFFIX') and contains info about the architecture the library targets.
115        # E.g. on x64 linux the suffix is ".cpython-XYZ-x86_64-linux-gnu.so"
116        # When crosscompiling python wheels, we need to be able to override this suffix
117        # so that the resulting file name matches the target architecture and we end up with a well-formed
118        # wheel.
119        filename = build_ext.build_ext.get_ext_filename(self, ext_name)
120        orig_ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')
121        new_ext_suffix = os.getenv('GRPC_PYTHON_OVERRIDE_EXT_SUFFIX')
122        if new_ext_suffix and filename.endswith(orig_ext_suffix):
123            filename = filename[:-len(orig_ext_suffix)] + new_ext_suffix
124        return filename
125
126
127# There are some situations (like on Windows) where CC, CFLAGS, and LDFLAGS are
128# entirely ignored/dropped/forgotten by distutils and its Cygwin/MinGW support.
129# We use these environment variables to thus get around that without locking
130# ourselves in w.r.t. the multitude of operating systems this ought to build on.
131# We can also use these variables as a way to inject environment-specific
132# compiler/linker flags. We assume GCC-like compilers and/or MinGW as a
133# reasonable default.
134EXTRA_ENV_COMPILE_ARGS = os.environ.get('GRPC_PYTHON_CFLAGS', None)
135EXTRA_ENV_LINK_ARGS = os.environ.get('GRPC_PYTHON_LDFLAGS', None)
136if EXTRA_ENV_COMPILE_ARGS is None:
137    EXTRA_ENV_COMPILE_ARGS = '-std=c++11'
138    if 'win32' in sys.platform:
139        if sys.version_info < (3, 5):
140            # We use define flags here and don't directly add to DEFINE_MACROS below to
141            # ensure that the expert user/builder has a way of turning it off (via the
142            # envvars) without adding yet more GRPC-specific envvars.
143            # See https://sourceforge.net/p/mingw-w64/bugs/363/
144            if '32' in platform.architecture()[0]:
145                EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime32 -D_timeb=__timeb32 -D_ftime_s=_ftime32_s -D_hypot=hypot'
146            else:
147                EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime64 -D_timeb=__timeb64 -D_hypot=hypot'
148        else:
149            # We need to statically link the C++ Runtime, only the C runtime is
150            # available dynamically
151            EXTRA_ENV_COMPILE_ARGS += ' /MT'
152    elif "linux" in sys.platform or "darwin" in sys.platform:
153        EXTRA_ENV_COMPILE_ARGS += ' -fno-wrapv -frtti'
154if EXTRA_ENV_LINK_ARGS is None:
155    EXTRA_ENV_LINK_ARGS = ''
156    # NOTE(rbellevi): Clang on Mac OS will make all static symbols (both
157    # variables and objects) global weak symbols. When a process loads the
158    # protobuf wheel's shared object library before loading *this* C extension,
159    # the runtime linker will prefer the protobuf module's version of symbols.
160    # This results in the process using a mixture of symbols from the protobuf
161    # wheel and this wheel, which may be using different versions of
162    # libprotobuf. In the case that they *are* using different versions of
163    # libprotobuf *and* there has been a change in data layout (or in other
164    # invariants) segfaults, data corruption, or "bad things" may happen.
165    #
166    # This flag ensures that on Mac, the only global symbol is the one loaded by
167    # the Python interpreter. The problematic global weak symbols become local
168    # weak symbols.  This is not required on Linux since the compiler does not
169    # produce global weak symbols. This is not required on Windows as our ".pyd"
170    # file does not contain any symbols.
171    #
172    # Finally, the leading underscore here is part of the Mach-O ABI. Unlike
173    # more modern ABIs (ELF et al.), Mach-O prepends an underscore to the names
174    # of C functions.
175    if "darwin" in sys.platform:
176        EXTRA_ENV_LINK_ARGS += ' -Wl,-exported_symbol,_{}'.format(
177            _EXT_INIT_SYMBOL)
178    if "linux" in sys.platform or "darwin" in sys.platform:
179        EXTRA_ENV_LINK_ARGS += ' -lpthread'
180        if check_linker_need_libatomic():
181            EXTRA_ENV_LINK_ARGS += ' -latomic'
182    elif "win32" in sys.platform and sys.version_info < (3, 5):
183        msvcr = cygwinccompiler.get_msvcr()[0]
184        EXTRA_ENV_LINK_ARGS += (
185            ' -static-libgcc -static-libstdc++ -mcrtdll={msvcr}'
186            ' -static -lshlwapi'.format(msvcr=msvcr))
187
188EXTRA_COMPILE_ARGS = shlex.split(EXTRA_ENV_COMPILE_ARGS)
189EXTRA_LINK_ARGS = shlex.split(EXTRA_ENV_LINK_ARGS)
190
191if BUILD_WITH_STATIC_LIBSTDCXX:
192    EXTRA_LINK_ARGS.append('-static-libstdc++')
193
194CC_FILES = [os.path.normpath(cc_file) for cc_file in protoc_lib_deps.CC_FILES]
195PROTO_FILES = [
196    os.path.normpath(proto_file) for proto_file in protoc_lib_deps.PROTO_FILES
197]
198CC_INCLUDE = os.path.normpath(protoc_lib_deps.CC_INCLUDE)
199PROTO_INCLUDE = os.path.normpath(protoc_lib_deps.PROTO_INCLUDE)
200
201GRPC_PYTHON_TOOLS_PACKAGE = 'grpc_tools'
202GRPC_PYTHON_PROTO_RESOURCES_NAME = '_proto'
203
204DEFINE_MACROS = ()
205if "win32" in sys.platform:
206    DEFINE_MACROS += (('WIN32_LEAN_AND_MEAN', 1),)
207    if '64bit' in platform.architecture()[0]:
208        DEFINE_MACROS += (('MS_WIN64', 1),)
209elif "linux" in sys.platform or "darwin" in sys.platform:
210    DEFINE_MACROS += (('HAVE_PTHREAD', 1),)
211
212# By default, Python3 distutils enforces compatibility of
213# c plugins (.so files) with the OSX version Python was built with.
214# We need OSX 10.10, the oldest which supports C++ thread_local.
215if 'darwin' in sys.platform:
216    mac_target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET')
217    if mac_target and (pkg_resources.parse_version(mac_target) <
218                       pkg_resources.parse_version('10.10.0')):
219        os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.10'
220        os.environ['_PYTHON_HOST_PLATFORM'] = re.sub(
221            r'macosx-[0-9]+\.[0-9]+-(.+)', r'macosx-10.10-\1',
222            util.get_platform())
223
224
225def package_data():
226    tools_path = GRPC_PYTHON_TOOLS_PACKAGE.replace('.', os.path.sep)
227    proto_resources_path = os.path.join(tools_path,
228                                        GRPC_PYTHON_PROTO_RESOURCES_NAME)
229    proto_files = []
230    for proto_file in PROTO_FILES:
231        source = os.path.join(PROTO_INCLUDE, proto_file)
232        target = os.path.join(proto_resources_path, proto_file)
233        relative_target = os.path.join(GRPC_PYTHON_PROTO_RESOURCES_NAME,
234                                       proto_file)
235        try:
236            os.makedirs(os.path.dirname(target))
237        except OSError as error:
238            if error.errno == errno.EEXIST:
239                pass
240            else:
241                raise
242        shutil.copy(source, target)
243        proto_files.append(relative_target)
244    return {GRPC_PYTHON_TOOLS_PACKAGE: proto_files}
245
246
247def extension_modules():
248    if BUILD_WITH_CYTHON:
249        plugin_sources = [os.path.join('grpc_tools', '_protoc_compiler.pyx')]
250    else:
251        plugin_sources = [os.path.join('grpc_tools', '_protoc_compiler.cpp')]
252
253    plugin_sources += [
254        os.path.join('grpc_tools', 'main.cc'),
255        os.path.join('grpc_root', 'src', 'compiler', 'python_generator.cc')
256    ] + [os.path.join(CC_INCLUDE, cc_file) for cc_file in CC_FILES]
257
258    plugin_ext = extension.Extension(
259        name='grpc_tools._protoc_compiler',
260        sources=plugin_sources,
261        include_dirs=[
262            '.',
263            'grpc_root',
264            os.path.join('grpc_root', 'include'),
265            CC_INCLUDE,
266        ],
267        language='c++',
268        define_macros=list(DEFINE_MACROS),
269        extra_compile_args=list(EXTRA_COMPILE_ARGS),
270        extra_link_args=list(EXTRA_LINK_ARGS),
271    )
272    extensions = [plugin_ext]
273    if BUILD_WITH_CYTHON:
274        from Cython import Build
275        return Build.cythonize(extensions)
276    else:
277        return extensions
278
279
280setuptools.setup(name='grpcio-tools',
281                 version=grpc_version.VERSION,
282                 description='Protobuf code generator for gRPC',
283                 long_description=open(_README_PATH, 'r').read(),
284                 author='The gRPC Authors',
285                 author_email='grpc-io@googlegroups.com',
286                 url='https://grpc.io',
287                 license='Apache License 2.0',
288                 classifiers=CLASSIFIERS,
289                 ext_modules=extension_modules(),
290                 packages=setuptools.find_packages('.'),
291                 python_requires='>=3.6',
292                 install_requires=[
293                     'protobuf>=3.5.0.post1, < 4.0dev',
294                     'grpcio>={version}'.format(version=grpc_version.VERSION),
295                     'setuptools',
296                 ],
297                 package_data=package_data(),
298                 cmdclass={
299                     'build_ext': BuildExt,
300                 })
301