1# Licensed under a 3-clause BSD style license - see LICENSE.rst 2""" 3This module contains a number of utilities for use during 4setup/build/packaging that are useful to astropy as a whole. 5""" 6 7import os 8import sys 9import shutil 10import subprocess 11from distutils import log 12from collections import defaultdict 13from distutils.core import Extension 14 15from setuptools import find_packages 16from setuptools.config import read_configuration 17 18from ._distutils_helpers import get_compiler 19from ._utils import import_file, walk_skip_hidden 20 21__all__ = ['get_extensions', 'pkg_config'] 22 23 24def get_extensions(srcdir='.'): 25 """ 26 Collect all extensions from Cython files and ``setup_package.py`` files. 27 28 If numpy is importable, the numpy include path will be added to all Cython 29 extensions which are automatically generated. 30 31 This function obtains that information by iterating through all 32 packages in ``srcdir`` and locating a ``setup_package.py`` module. 33 This module can contain the ``get_extensions()`` function which returns 34 a list of :class:`distutils.core.Extension` objects. 35 36 """ 37 ext_modules = [] 38 packages = [] 39 package_dir = {} 40 41 # Use the find_packages tool to locate all packages and modules 42 packages = find_packages(srcdir) 43 44 # Update package_dir if the package lies in a subdirectory 45 if srcdir != '.': 46 package_dir[''] = srcdir 47 48 for setuppkg in iter_setup_packages(srcdir, packages): 49 # get_extensions must include any Cython extensions by their .pyx 50 # filename. 51 if hasattr(setuppkg, 'get_extensions'): 52 ext_modules.extend(setuppkg.get_extensions()) 53 54 # Locate any .pyx files not already specified, and add their extensions in. 55 # The default include dirs include numpy to facilitate numerical work. 56 includes = [] 57 try: 58 import numpy 59 includes = [numpy.get_include()] 60 except ImportError: 61 pass 62 63 ext_modules.extend(get_cython_extensions(srcdir, packages, ext_modules, includes)) 64 65 # Now remove extensions that have the special name 'skip_cython', as they 66 # exist Only to indicate that the cython extensions shouldn't be built 67 for i, ext in reversed(list(enumerate(ext_modules))): 68 if ext.name == 'skip_cython': 69 del ext_modules[i] 70 71 # On Microsoft compilers, we need to pass the '/MANIFEST' 72 # commandline argument. This was the default on MSVC 9.0, but is 73 # now required on MSVC 10.0, but it doesn't seem to hurt to add 74 # it unconditionally. 75 if get_compiler() == 'msvc': 76 for ext in ext_modules: 77 ext.extra_link_args.append('/MANIFEST') 78 79 if len(ext_modules) > 0: 80 main_package_dir = min(packages, key=len) 81 src_path = os.path.join(os.path.dirname(__file__), 'src') 82 shutil.copy(os.path.join(src_path, 'compiler.c'), 83 os.path.join(srcdir, main_package_dir, '_compiler.c')) 84 ext = Extension(main_package_dir + '.compiler_version', 85 [os.path.join(main_package_dir, '_compiler.c')]) 86 ext_modules.append(ext) 87 88 return ext_modules 89 90 91def iter_setup_packages(srcdir, packages): 92 """ A generator that finds and imports all of the ``setup_package.py`` 93 modules in the source packages. 94 95 Returns 96 ------- 97 modgen : generator 98 A generator that yields (modname, mod), where `mod` is the module and 99 `modname` is the module name for the ``setup_package.py`` modules. 100 101 """ 102 103 for packagename in packages: 104 package_parts = packagename.split('.') 105 package_path = os.path.join(srcdir, *package_parts) 106 setup_package = os.path.join(package_path, 'setup_package.py') 107 108 if os.path.isfile(setup_package): 109 module = import_file(setup_package, 110 name=packagename + '.setup_package') 111 yield module 112 113 114def iter_pyx_files(package_dir, package_name): 115 """ 116 A generator that yields Cython source files (ending in '.pyx') in the 117 source packages. 118 119 Returns 120 ------- 121 pyxgen : generator 122 A generator that yields (extmod, fullfn) where `extmod` is the 123 full name of the module that the .pyx file would live in based 124 on the source directory structure, and `fullfn` is the path to 125 the .pyx file. 126 """ 127 for dirpath, dirnames, filenames in walk_skip_hidden(package_dir): 128 for fn in filenames: 129 if fn.endswith('.pyx'): 130 fullfn = os.path.join(dirpath, fn) 131 # Package must match file name 132 extmod = '.'.join([package_name, fn[:-4]]) 133 yield (extmod, fullfn) 134 135 break # Don't recurse into subdirectories 136 137 138def get_cython_extensions(srcdir, packages, prevextensions=tuple(), 139 extincludedirs=None): 140 """ 141 Looks for Cython files and generates Extensions if needed. 142 143 Parameters 144 ---------- 145 srcdir : str 146 Path to the root of the source directory to search. 147 prevextensions : list 148 The extensions that are already defined, as a list of of 149 `~distutils.core.Extension` objects. Any .pyx files already here will 150 be ignored. 151 extincludedirs : list or None 152 Directories to include as the `include_dirs` argument to the generated 153 `~distutils.core.Extension` objects, as a list of strings. 154 155 Returns 156 ------- 157 exts : list 158 The new extensions that are needed to compile all .pyx files (does not 159 include any already in `prevextensions`). 160 """ 161 162 # Vanilla setuptools and old versions of distribute include Cython files 163 # as .c files in the sources, not .pyx, so we cannot simply look for 164 # existing .pyx sources in the previous sources, but we should also check 165 # for .c files with the same remaining filename. So we look for .pyx and 166 # .c files, and we strip the extension. 167 prevsourcepaths = [] 168 ext_modules = [] 169 170 for ext in prevextensions: 171 for s in ext.sources: 172 if s.endswith(('.pyx', '.c', '.cpp')): 173 sourcepath = os.path.realpath(os.path.splitext(s)[0]) 174 prevsourcepaths.append(sourcepath) 175 176 for package_name in packages: 177 package_parts = package_name.split('.') 178 package_path = os.path.join(srcdir, *package_parts) 179 180 for extmod, pyxfn in iter_pyx_files(package_path, package_name): 181 sourcepath = os.path.realpath(os.path.splitext(pyxfn)[0]) 182 if sourcepath not in prevsourcepaths: 183 ext_modules.append(Extension(extmod, [pyxfn], 184 include_dirs=extincludedirs)) 185 186 return ext_modules 187 188 189def pkg_config(packages, default_libraries, executable='pkg-config'): 190 """ 191 Uses pkg-config to update a set of distutils Extension arguments 192 to include the flags necessary to link against the given packages. 193 194 If the pkg-config lookup fails, default_libraries is applied to 195 libraries. 196 197 Parameters 198 ---------- 199 packages : list 200 The pkg-config packages to look up, as a list of strings. 201 202 default_libraries : list 203 The ibrary names to use if the pkg-config lookup fails, a a list of 204 strings. 205 206 Returns 207 ------- 208 config : dict 209 A dictionary containing keyword arguments to 210 :class:`~distutils.core.Extension`. These entries include: 211 212 - ``include_dirs``: A list of include directories 213 - ``library_dirs``: A list of library directories 214 - ``libraries``: A list of libraries 215 - ``define_macros``: A list of macro defines 216 - ``undef_macros``: A list of macros to undefine 217 - ``extra_compile_args``: A list of extra arguments to pass to 218 the compiler 219 """ 220 221 flag_map = {'-I': 'include_dirs', '-L': 'library_dirs', '-l': 'libraries', 222 '-D': 'define_macros', '-U': 'undef_macros'} 223 command = "{0} --libs --cflags {1}".format(executable, ' '.join(packages)), 224 225 result = defaultdict(list) 226 227 try: 228 pipe = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE) 229 output = pipe.communicate()[0].strip() 230 except subprocess.CalledProcessError as e: 231 lines = [ 232 ("{0} failed. This may cause the build to fail below." 233 .format(executable)), 234 " command: {0}".format(e.cmd), 235 " returncode: {0}".format(e.returncode), 236 " output: {0}".format(e.output) 237 ] 238 log.warn('\n'.join(lines)) 239 result['libraries'].extend(default_libraries) 240 else: 241 if pipe.returncode != 0: 242 lines = [ 243 "pkg-config could not lookup up package(s) {0}.".format( 244 ", ".join(packages)), 245 "This may cause the build to fail below." 246 ] 247 log.warn('\n'.join(lines)) 248 result['libraries'].extend(default_libraries) 249 else: 250 for token in output.split(): 251 # It's not clear what encoding the output of 252 # pkg-config will come to us in. It will probably be 253 # some combination of pure ASCII (for the compiler 254 # flags) and the filesystem encoding (for any argument 255 # that includes directories or filenames), but this is 256 # just conjecture, as the pkg-config documentation 257 # doesn't seem to address it. 258 arg = token[:2].decode('ascii') 259 value = token[2:].decode(sys.getfilesystemencoding()) 260 if arg in flag_map: 261 if arg == '-D': 262 value = tuple(value.split('=', 1)) 263 result[flag_map[arg]].append(value) 264 else: 265 result['extra_compile_args'].append(value) 266 267 return result 268