1# Copyright 2015 gRPC authors. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14"""Provides distutils command classes for the GRPC Python setup process.""" 15 16from __future__ import print_function 17 18import distutils 19import glob 20import os 21import os.path 22import platform 23import re 24import shutil 25import subprocess 26import sys 27import sysconfig 28import traceback 29 30import setuptools 31from setuptools.command import build_ext 32from setuptools.command import build_py 33from setuptools.command import easy_install 34from setuptools.command import install 35from setuptools.command import test 36import support 37 38PYTHON_STEM = os.path.dirname(os.path.abspath(__file__)) 39GRPC_STEM = os.path.abspath(PYTHON_STEM + '../../../../') 40PROTO_STEM = os.path.join(GRPC_STEM, 'src', 'proto') 41PROTO_GEN_STEM = os.path.join(GRPC_STEM, 'src', 'python', 'gens') 42CYTHON_STEM = os.path.join(PYTHON_STEM, 'grpc', '_cython') 43 44 45class CommandError(Exception): 46 """Simple exception class for GRPC custom commands.""" 47 48 49# TODO(atash): Remove this once PyPI has better Linux bdist support. See 50# https://bitbucket.org/pypa/pypi/issues/120/binary-wheels-for-linux-are-not-supported 51def _get_grpc_custom_bdist(decorated_basename, target_bdist_basename): 52 """Returns a string path to a bdist file for Linux to install. 53 54 If we can retrieve a pre-compiled bdist from online, uses it. Else, emits a 55 warning and builds from source. 56 """ 57 # TODO(atash): somehow the name that's returned from `wheel` is different 58 # between different versions of 'wheel' (but from a compatibility standpoint, 59 # the names are compatible); we should have some way of determining name 60 # compatibility in the same way `wheel` does to avoid having to rename all of 61 # the custom wheels that we build/upload to GCS. 62 63 # Break import style to ensure that setup.py has had a chance to install the 64 # relevant package. 65 from six.moves.urllib import request 66 decorated_path = decorated_basename + GRPC_CUSTOM_BDIST_EXT 67 try: 68 url = BINARIES_REPOSITORY + '/{target}'.format(target=decorated_path) 69 bdist_data = request.urlopen(url).read() 70 except IOError as error: 71 raise CommandError('{}\n\nCould not find the bdist {}: {}'.format( 72 traceback.format_exc(), decorated_path, error.message)) 73 # Our chosen local bdist path. 74 bdist_path = target_bdist_basename + GRPC_CUSTOM_BDIST_EXT 75 try: 76 with open(bdist_path, 'w') as bdist_file: 77 bdist_file.write(bdist_data) 78 except IOError as error: 79 raise CommandError('{}\n\nCould not write grpcio bdist: {}'.format( 80 traceback.format_exc(), error.message)) 81 return bdist_path 82 83 84class SphinxDocumentation(setuptools.Command): 85 """Command to generate documentation via sphinx.""" 86 87 description = 'generate sphinx documentation' 88 user_options = [] 89 90 def initialize_options(self): 91 pass 92 93 def finalize_options(self): 94 pass 95 96 def run(self): 97 # We import here to ensure that setup.py has had a chance to install the 98 # relevant package eggs first. 99 import sphinx.cmd.build 100 source_dir = os.path.join(GRPC_STEM, 'doc', 'python', 'sphinx') 101 target_dir = os.path.join(GRPC_STEM, 'doc', 'build') 102 exit_code = sphinx.cmd.build.build_main( 103 ['-b', 'html', '-W', '--keep-going', source_dir, target_dir]) 104 if exit_code != 0: 105 raise CommandError( 106 "Documentation generation has warnings or errors") 107 108 109class BuildProjectMetadata(setuptools.Command): 110 """Command to generate project metadata in a module.""" 111 112 description = 'build grpcio project metadata files' 113 user_options = [] 114 115 def initialize_options(self): 116 pass 117 118 def finalize_options(self): 119 pass 120 121 def run(self): 122 with open(os.path.join(PYTHON_STEM, 'grpc/_grpcio_metadata.py'), 123 'w') as module_file: 124 module_file.write('__version__ = """{}"""'.format( 125 self.distribution.get_version())) 126 127 128class BuildPy(build_py.build_py): 129 """Custom project build command.""" 130 131 def run(self): 132 self.run_command('build_project_metadata') 133 build_py.build_py.run(self) 134 135 136def _poison_extensions(extensions, message): 137 """Includes a file that will always fail to compile in all extensions.""" 138 poison_filename = os.path.join(PYTHON_STEM, 'poison.c') 139 with open(poison_filename, 'w') as poison: 140 poison.write('#error {}'.format(message)) 141 for extension in extensions: 142 extension.sources = [poison_filename] 143 144 145def check_and_update_cythonization(extensions): 146 """Replace .pyx files with their generated counterparts and return whether or 147 not cythonization still needs to occur.""" 148 for extension in extensions: 149 generated_pyx_sources = [] 150 other_sources = [] 151 for source in extension.sources: 152 base, file_ext = os.path.splitext(source) 153 if file_ext == '.pyx': 154 generated_pyx_source = next((base + gen_ext for gen_ext in ( 155 '.c', 156 '.cpp', 157 ) if os.path.isfile(base + gen_ext)), None) 158 if generated_pyx_source: 159 generated_pyx_sources.append(generated_pyx_source) 160 else: 161 sys.stderr.write('Cython-generated files are missing...\n') 162 return False 163 else: 164 other_sources.append(source) 165 extension.sources = generated_pyx_sources + other_sources 166 sys.stderr.write('Found cython-generated files...\n') 167 return True 168 169 170def try_cythonize(extensions, linetracing=False, mandatory=True): 171 """Attempt to cythonize the extensions. 172 173 Args: 174 extensions: A list of `distutils.extension.Extension`. 175 linetracing: A bool indicating whether or not to enable linetracing. 176 mandatory: Whether or not having Cython-generated files is mandatory. If it 177 is, extensions will be poisoned when they can't be fully generated. 178 """ 179 try: 180 # Break import style to ensure we have access to Cython post-setup_requires 181 import Cython.Build 182 except ImportError: 183 if mandatory: 184 sys.stderr.write( 185 "This package needs to generate C files with Cython but it cannot. " 186 "Poisoning extension sources to disallow extension commands...") 187 _poison_extensions( 188 extensions, 189 "Extensions have been poisoned due to missing Cython-generated code." 190 ) 191 return extensions 192 cython_compiler_directives = {} 193 if linetracing: 194 additional_define_macros = [('CYTHON_TRACE_NOGIL', '1')] 195 cython_compiler_directives['linetrace'] = True 196 return Cython.Build.cythonize( 197 extensions, 198 include_path=[ 199 include_dir for extension in extensions 200 for include_dir in extension.include_dirs 201 ] + [CYTHON_STEM], 202 compiler_directives=cython_compiler_directives) 203 204 205class BuildExt(build_ext.build_ext): 206 """Custom build_ext command to enable compiler-specific flags.""" 207 208 C_OPTIONS = { 209 'unix': ('-pthread',), 210 'msvc': (), 211 } 212 LINK_OPTIONS = {} 213 214 def get_ext_filename(self, ext_name): 215 # since python3.5, python extensions' shared libraries use a suffix that corresponds to the value 216 # of sysconfig.get_config_var('EXT_SUFFIX') and contains info about the architecture the library targets. 217 # E.g. on x64 linux the suffix is ".cpython-XYZ-x86_64-linux-gnu.so" 218 # When crosscompiling python wheels, we need to be able to override this suffix 219 # so that the resulting file name matches the target architecture and we end up with a well-formed 220 # wheel. 221 filename = build_ext.build_ext.get_ext_filename(self, ext_name) 222 orig_ext_suffix = sysconfig.get_config_var('EXT_SUFFIX') 223 new_ext_suffix = os.getenv('GRPC_PYTHON_OVERRIDE_EXT_SUFFIX') 224 if new_ext_suffix and filename.endswith(orig_ext_suffix): 225 filename = filename[:-len(orig_ext_suffix)] + new_ext_suffix 226 return filename 227 228 def build_extensions(self): 229 230 def compiler_ok_with_extra_std(): 231 """Test if default compiler is okay with specifying c++ version 232 when invoked in C mode. GCC is okay with this, while clang is not. 233 """ 234 try: 235 # TODO(lidiz) Remove the generated a.out for success tests. 236 cc_test = subprocess.Popen(['cc', '-x', 'c', '-std=c++11', '-'], 237 stdin=subprocess.PIPE, 238 stdout=subprocess.PIPE, 239 stderr=subprocess.PIPE) 240 _, cc_err = cc_test.communicate(input=b'int main(){return 0;}') 241 return not 'invalid argument' in str(cc_err) 242 except: 243 sys.stderr.write('Non-fatal exception:' + 244 traceback.format_exc() + '\n') 245 return False 246 247 # This special conditioning is here due to difference of compiler 248 # behavior in gcc and clang. The clang doesn't take --stdc++11 249 # flags but gcc does. Since the setuptools of Python only support 250 # all C or all C++ compilation, the mix of C and C++ will crash. 251 # *By default*, macOS and FreBSD use clang and Linux use gcc 252 # 253 # If we are not using a permissive compiler that's OK with being 254 # passed wrong std flags, swap out compile function by adding a filter 255 # for it. 256 if not compiler_ok_with_extra_std(): 257 old_compile = self.compiler._compile 258 259 def new_compile(obj, src, ext, cc_args, extra_postargs, pp_opts): 260 if src.endswith('.c'): 261 extra_postargs = [ 262 arg for arg in extra_postargs if not '-std=c++' in arg 263 ] 264 elif src.endswith('.cc') or src.endswith('.cpp'): 265 extra_postargs = [ 266 arg for arg in extra_postargs if not '-std=gnu99' in arg 267 ] 268 return old_compile(obj, src, ext, cc_args, extra_postargs, 269 pp_opts) 270 271 self.compiler._compile = new_compile 272 273 compiler = self.compiler.compiler_type 274 if compiler in BuildExt.C_OPTIONS: 275 for extension in self.extensions: 276 extension.extra_compile_args += list( 277 BuildExt.C_OPTIONS[compiler]) 278 if compiler in BuildExt.LINK_OPTIONS: 279 for extension in self.extensions: 280 extension.extra_link_args += list( 281 BuildExt.LINK_OPTIONS[compiler]) 282 if not check_and_update_cythonization(self.extensions): 283 self.extensions = try_cythonize(self.extensions) 284 try: 285 build_ext.build_ext.build_extensions(self) 286 except Exception as error: 287 formatted_exception = traceback.format_exc() 288 support.diagnose_build_ext_error(self, error, formatted_exception) 289 raise CommandError( 290 "Failed `build_ext` step:\n{}".format(formatted_exception)) 291 292 293class Gather(setuptools.Command): 294 """Command to gather project dependencies.""" 295 296 description = 'gather dependencies for grpcio' 297 user_options = [ 298 ('test', 't', 'flag indicating to gather test dependencies'), 299 ('install', 'i', 'flag indicating to gather install dependencies') 300 ] 301 302 def initialize_options(self): 303 self.test = False 304 self.install = False 305 306 def finalize_options(self): 307 # distutils requires this override. 308 pass 309 310 def run(self): 311 if self.install and self.distribution.install_requires: 312 self.distribution.fetch_build_eggs( 313 self.distribution.install_requires) 314 if self.test and self.distribution.tests_require: 315 self.distribution.fetch_build_eggs(self.distribution.tests_require) 316 317 318class Clean(setuptools.Command): 319 """Command to clean build artifacts.""" 320 321 description = 'Clean build artifacts.' 322 user_options = [ 323 ('all', 'a', 'a phony flag to allow our script to continue'), 324 ] 325 326 _FILE_PATTERNS = ( 327 'python_build', 328 'src/python/grpcio/__pycache__/', 329 'src/python/grpcio/grpc/_cython/cygrpc.cpp', 330 'src/python/grpcio/grpc/_cython/*.so', 331 'src/python/grpcio/grpcio.egg-info/', 332 ) 333 _CURRENT_DIRECTORY = os.path.normpath( 334 os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../..")) 335 336 def initialize_options(self): 337 self.all = False 338 339 def finalize_options(self): 340 pass 341 342 def run(self): 343 for path_spec in self._FILE_PATTERNS: 344 this_glob = os.path.normpath( 345 os.path.join(Clean._CURRENT_DIRECTORY, path_spec)) 346 abs_paths = glob.glob(this_glob) 347 for path in abs_paths: 348 if not str(path).startswith(Clean._CURRENT_DIRECTORY): 349 raise ValueError( 350 "Cowardly refusing to delete {}.".format(path)) 351 print("Removing {}".format(os.path.relpath(path))) 352 if os.path.isfile(path): 353 os.remove(str(path)) 354 else: 355 shutil.rmtree(str(path)) 356