1# Licensed under a 3-clause BSD style license - see LICENSE.rst
2"""
3This module contains a number of utilities for use during
4setup/build/packaging that are useful to astropy as a whole.
5"""
6
7import os
8import sys
9import shutil
10import subprocess
11from distutils import log
12from collections import defaultdict
13from distutils.core import Extension
14
15from setuptools import find_packages
16from setuptools.config import read_configuration
17
18from ._distutils_helpers import get_compiler
19from ._utils import import_file, walk_skip_hidden
20
21__all__ = ['get_extensions', 'pkg_config']
22
23
24def get_extensions(srcdir='.'):
25    """
26    Collect all extensions from Cython files and ``setup_package.py`` files.
27
28    If numpy is importable, the numpy include path will be added to all Cython
29    extensions which are automatically generated.
30
31    This function obtains that information by iterating through all
32    packages in ``srcdir`` and locating a ``setup_package.py`` module.
33    This module can contain the ``get_extensions()`` function which returns
34    a list of :class:`distutils.core.Extension` objects.
35
36    """
37    ext_modules = []
38    packages = []
39    package_dir = {}
40
41    # Use the find_packages tool to locate all packages and modules
42    packages = find_packages(srcdir)
43
44    # Update package_dir if the package lies in a subdirectory
45    if srcdir != '.':
46        package_dir[''] = srcdir
47
48    for setuppkg in iter_setup_packages(srcdir, packages):
49        # get_extensions must include any Cython extensions by their .pyx
50        # filename.
51        if hasattr(setuppkg, 'get_extensions'):
52            ext_modules.extend(setuppkg.get_extensions())
53
54    # Locate any .pyx files not already specified, and add their extensions in.
55    # The default include dirs include numpy to facilitate numerical work.
56    includes = []
57    try:
58        import numpy
59        includes = [numpy.get_include()]
60    except ImportError:
61        pass
62
63    ext_modules.extend(get_cython_extensions(srcdir, packages, ext_modules, includes))
64
65    # Now remove extensions that have the special name 'skip_cython', as they
66    # exist Only to indicate that the cython extensions shouldn't be built
67    for i, ext in reversed(list(enumerate(ext_modules))):
68        if ext.name == 'skip_cython':
69            del ext_modules[i]
70
71    # On Microsoft compilers, we need to pass the '/MANIFEST'
72    # commandline argument.  This was the default on MSVC 9.0, but is
73    # now required on MSVC 10.0, but it doesn't seem to hurt to add
74    # it unconditionally.
75    if get_compiler() == 'msvc':
76        for ext in ext_modules:
77            ext.extra_link_args.append('/MANIFEST')
78
79    if len(ext_modules) > 0:
80        main_package_dir = min(packages, key=len)
81        src_path = os.path.join(os.path.dirname(__file__), 'src')
82        shutil.copy(os.path.join(src_path, 'compiler.c'),
83                    os.path.join(srcdir, main_package_dir, '_compiler.c'))
84        ext = Extension(main_package_dir + '.compiler_version',
85                        [os.path.join(main_package_dir, '_compiler.c')])
86        ext_modules.append(ext)
87
88    return ext_modules
89
90
91def iter_setup_packages(srcdir, packages):
92    """ A generator that finds and imports all of the ``setup_package.py``
93    modules in the source packages.
94
95    Returns
96    -------
97    modgen : generator
98        A generator that yields (modname, mod), where `mod` is the module and
99        `modname` is the module name for the ``setup_package.py`` modules.
100
101    """
102
103    for packagename in packages:
104        package_parts = packagename.split('.')
105        package_path = os.path.join(srcdir, *package_parts)
106        setup_package = os.path.join(package_path, 'setup_package.py')
107
108        if os.path.isfile(setup_package):
109            module = import_file(setup_package,
110                                 name=packagename + '.setup_package')
111            yield module
112
113
114def iter_pyx_files(package_dir, package_name):
115    """
116    A generator that yields Cython source files (ending in '.pyx') in the
117    source packages.
118
119    Returns
120    -------
121    pyxgen : generator
122        A generator that yields (extmod, fullfn) where `extmod` is the
123        full name of the module that the .pyx file would live in based
124        on the source directory structure, and `fullfn` is the path to
125        the .pyx file.
126    """
127    for dirpath, dirnames, filenames in walk_skip_hidden(package_dir):
128        for fn in filenames:
129            if fn.endswith('.pyx'):
130                fullfn = os.path.join(dirpath, fn)
131                # Package must match file name
132                extmod = '.'.join([package_name, fn[:-4]])
133                yield (extmod, fullfn)
134
135        break  # Don't recurse into subdirectories
136
137
138def get_cython_extensions(srcdir, packages, prevextensions=tuple(),
139                          extincludedirs=None):
140    """
141    Looks for Cython files and generates Extensions if needed.
142
143    Parameters
144    ----------
145    srcdir : str
146        Path to the root of the source directory to search.
147    prevextensions : list
148        The extensions that are already defined, as a list of of
149        `~distutils.core.Extension` objects.  Any .pyx files already here will
150        be ignored.
151    extincludedirs : list or None
152        Directories to include as the `include_dirs` argument to the generated
153        `~distutils.core.Extension` objects, as a list of strings.
154
155    Returns
156    -------
157    exts : list
158        The new extensions that are needed to compile all .pyx files (does not
159        include any already in `prevextensions`).
160    """
161
162    # Vanilla setuptools and old versions of distribute include Cython files
163    # as .c files in the sources, not .pyx, so we cannot simply look for
164    # existing .pyx sources in the previous sources, but we should also check
165    # for .c files with the same remaining filename. So we look for .pyx and
166    # .c files, and we strip the extension.
167    prevsourcepaths = []
168    ext_modules = []
169
170    for ext in prevextensions:
171        for s in ext.sources:
172            if s.endswith(('.pyx', '.c', '.cpp')):
173                sourcepath = os.path.realpath(os.path.splitext(s)[0])
174                prevsourcepaths.append(sourcepath)
175
176    for package_name in packages:
177        package_parts = package_name.split('.')
178        package_path = os.path.join(srcdir, *package_parts)
179
180        for extmod, pyxfn in iter_pyx_files(package_path, package_name):
181            sourcepath = os.path.realpath(os.path.splitext(pyxfn)[0])
182            if sourcepath not in prevsourcepaths:
183                ext_modules.append(Extension(extmod, [pyxfn],
184                                             include_dirs=extincludedirs))
185
186    return ext_modules
187
188
189def pkg_config(packages, default_libraries, executable='pkg-config'):
190    """
191    Uses pkg-config to update a set of distutils Extension arguments
192    to include the flags necessary to link against the given packages.
193
194    If the pkg-config lookup fails, default_libraries is applied to
195    libraries.
196
197    Parameters
198    ----------
199    packages : list
200        The pkg-config packages to look up, as a list of strings.
201
202    default_libraries : list
203        The ibrary names to use if the pkg-config lookup fails, a a list of
204        strings.
205
206    Returns
207    -------
208    config : dict
209        A dictionary containing keyword arguments to
210        :class:`~distutils.core.Extension`.  These entries include:
211
212        - ``include_dirs``: A list of include directories
213        - ``library_dirs``: A list of library directories
214        - ``libraries``: A list of libraries
215        - ``define_macros``: A list of macro defines
216        - ``undef_macros``: A list of macros to undefine
217        - ``extra_compile_args``: A list of extra arguments to pass to
218          the compiler
219    """
220
221    flag_map = {'-I': 'include_dirs', '-L': 'library_dirs', '-l': 'libraries',
222                '-D': 'define_macros', '-U': 'undef_macros'}
223    command = "{0} --libs --cflags {1}".format(executable, ' '.join(packages)),
224
225    result = defaultdict(list)
226
227    try:
228        pipe = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
229        output = pipe.communicate()[0].strip()
230    except subprocess.CalledProcessError as e:
231        lines = [
232            ("{0} failed. This may cause the build to fail below."
233             .format(executable)),
234            "  command: {0}".format(e.cmd),
235            "  returncode: {0}".format(e.returncode),
236            "  output: {0}".format(e.output)
237            ]
238        log.warn('\n'.join(lines))
239        result['libraries'].extend(default_libraries)
240    else:
241        if pipe.returncode != 0:
242            lines = [
243                "pkg-config could not lookup up package(s) {0}.".format(
244                    ", ".join(packages)),
245                "This may cause the build to fail below."
246                ]
247            log.warn('\n'.join(lines))
248            result['libraries'].extend(default_libraries)
249        else:
250            for token in output.split():
251                # It's not clear what encoding the output of
252                # pkg-config will come to us in.  It will probably be
253                # some combination of pure ASCII (for the compiler
254                # flags) and the filesystem encoding (for any argument
255                # that includes directories or filenames), but this is
256                # just conjecture, as the pkg-config documentation
257                # doesn't seem to address it.
258                arg = token[:2].decode('ascii')
259                value = token[2:].decode(sys.getfilesystemencoding())
260                if arg in flag_map:
261                    if arg == '-D':
262                        value = tuple(value.split('=', 1))
263                    result[flag_map[arg]].append(value)
264                else:
265                    result['extra_compile_args'].append(value)
266
267    return result
268