1"""
2To build with coverage of Cython files
3export SM_CYTHON_COVERAGE=1
4python setup.py develop
5pytest --cov=statsmodels statsmodels
6coverage html
7"""
8from setuptools import Extension, find_packages, setup
9from setuptools.dist import Distribution
10
11from collections import defaultdict
12from distutils.command.clean import clean
13import fnmatch
14import os
15from os.path import abspath, join as pjoin, relpath, split
16import shutil
17import sys
18
19import pkg_resources
20
21import versioneer
22
23try:
24    # SM_FORCE_C is a testing shim to force setup to use C source files
25    FORCE_C = int(os.environ.get("SM_FORCE_C", 0))
26    if FORCE_C:
27        raise ImportError("Force import error for testing")
28    from Cython import Tempita
29    from Cython.Build import cythonize
30    from Cython.Distutils import build_ext
31
32    HAS_CYTHON = True
33except ImportError:
34    from setuptools.command.build_ext import build_ext
35
36    HAS_CYTHON = False
37
38try:
39    import numpy  # noqa: F401
40
41    HAS_NUMPY = True
42except ImportError:
43    HAS_NUMPY = False
44
45###############################################################################
46# Key Values that Change Each Release
47###############################################################################
48SETUP_REQUIREMENTS = {
49    "numpy": "1.17",  # released July 2019
50    "scipy": "1.3",  # released May 2019
51}
52
53REQ_NOT_MET_MSG = """
54{0} is installed but older ({1}) than required ({2}). You must manually
55upgrade {0} before installing or install into a fresh virtualenv.
56"""
57for key in SETUP_REQUIREMENTS:
58    from distutils.version import LooseVersion
59    import importlib
60
61    req_ver = LooseVersion(SETUP_REQUIREMENTS[key])
62    try:
63        mod = importlib.import_module(key)
64        ver = LooseVersion(mod.__version__)
65        if ver < req_ver:
66            raise RuntimeError(REQ_NOT_MET_MSG.format(key, ver, req_ver))
67    except ImportError:
68        pass
69    except AttributeError:
70        raise RuntimeError(REQ_NOT_MET_MSG.format(key, ver, req_ver))
71
72INSTALL_REQUIREMENTS = SETUP_REQUIREMENTS.copy()
73INSTALL_REQUIREMENTS.update(
74    {
75        "pandas": "0.25",  # released July 2019
76        "patsy": "0.5.2",  # released January 2018
77    }
78)
79
80CYTHON_MIN_VER = "0.29.22"  # released 2020
81
82SETUP_REQUIRES = [k + ">=" + v for k, v in SETUP_REQUIREMENTS.items()]
83INSTALL_REQUIRES = [k + ">=" + v for k, v in INSTALL_REQUIREMENTS.items()]
84
85EXTRAS_REQUIRE = {
86    "build": ["cython>=" + CYTHON_MIN_VER],
87    "develop": ["cython>=" + CYTHON_MIN_VER],
88    "docs": [
89        "sphinx",
90        "nbconvert",
91        "jupyter_client",
92        "ipykernel",
93        "matplotlib",
94        "nbformat",
95        "numpydoc",
96        "pandas-datareader",
97    ],
98}
99
100###############################################################################
101# Values that rarely change
102###############################################################################
103DISTNAME = "statsmodels"
104DESCRIPTION = "Statistical computations and models for Python"
105SETUP_DIR = split(abspath(__file__))[0]
106with open(pjoin(SETUP_DIR, "README.rst")) as readme:
107    README = readme.read()
108LONG_DESCRIPTION = README
109MAINTAINER = "statsmodels Developers"
110MAINTAINER_EMAIL = "pystatsmodels@googlegroups.com"
111URL = "https://www.statsmodels.org/"
112LICENSE = "BSD License"
113DOWNLOAD_URL = ""
114PROJECT_URLS = {
115    "Bug Tracker": "https://github.com/statsmodels/statsmodels/issues",
116    "Documentation": "https://www.statsmodels.org/stable/index.html",
117    "Source Code": "https://github.com/statsmodels/statsmodels",
118}
119
120CLASSIFIERS = [
121    "Development Status :: 4 - Beta",
122    "Environment :: Console",
123    "Programming Language :: Cython",
124    "Programming Language :: Python :: 3.7",
125    "Programming Language :: Python :: 3.8",
126    "Programming Language :: Python :: 3.9",
127    "Operating System :: OS Independent",
128    "Intended Audience :: End Users/Desktop",
129    "Intended Audience :: Developers",
130    "Intended Audience :: Science/Research",
131    "Natural Language :: English",
132    "License :: OSI Approved :: BSD License",
133    "Topic :: Office/Business :: Financial",
134    "Topic :: Scientific/Engineering",
135]
136
137FILES_TO_INCLUDE_IN_PACKAGE = ["LICENSE.txt", "setup.cfg"]
138
139FILES_COPIED_TO_PACKAGE = []
140for filename in FILES_TO_INCLUDE_IN_PACKAGE:
141    if os.path.exists(filename):
142        dest = os.path.join("statsmodels", filename)
143        shutil.copy2(filename, dest)
144        FILES_COPIED_TO_PACKAGE.append(dest)
145
146STATESPACE_RESULTS = "statsmodels.tsa.statespace.tests.results"
147
148ADDITIONAL_PACKAGE_DATA = {
149    "statsmodels": FILES_TO_INCLUDE_IN_PACKAGE,
150    "statsmodels.datasets.tests": ["*.zip"],
151    "statsmodels.iolib.tests.results": ["*.dta"],
152    "statsmodels.stats.tests.results": ["*.json"],
153    "statsmodels.tsa.vector_ar.tests.results": ["*.npz", "*.dat"],
154    "statsmodels.stats.tests": ["*.txt"],
155    "statsmodels.stats.libqsturng": ["*.r", "*.txt", "*.dat"],
156    "statsmodels.stats.libqsturng.tests": ["*.csv", "*.dat"],
157    "statsmodels.sandbox.regression.tests": ["*.dta", "*.csv"],
158    STATESPACE_RESULTS: ["*.pkl", "*.csv"],
159    STATESPACE_RESULTS + ".frbny_nowcast": ["test*.mat"],
160    STATESPACE_RESULTS + ".frbny_nowcast.Nowcasting.data.US": ["*.csv"],
161}
162
163##############################################################################
164# Extension Building
165##############################################################################
166CYTHON_COVERAGE = os.environ.get("SM_CYTHON_COVERAGE", False)
167CYTHON_COVERAGE = CYTHON_COVERAGE in ("1", "true", '"true"')
168CYTHON_TRACE_NOGIL = str(int(CYTHON_COVERAGE))
169if CYTHON_COVERAGE:
170    print("Building with coverage for Cython code")
171COMPILER_DIRECTIVES = {"linetrace": CYTHON_COVERAGE}
172DEFINE_MACROS = [
173    ("CYTHON_TRACE_NOGIL", CYTHON_TRACE_NOGIL),
174    ("NPY_NO_DEPRECATED_API", "NPY_1_7_API_VERSION"),
175]
176
177
178exts = dict(
179    _stl={"source": "statsmodels/tsa/_stl.pyx"},
180    _exponential_smoothers={
181        "source": "statsmodels/tsa/holtwinters/_exponential_smoothers.pyx"
182    },  # noqa: E501
183    _ets_smooth={
184        "source": "statsmodels/tsa/exponential_smoothing/_ets_smooth.pyx"
185    },  # noqa: E501
186    _innovations={"source": "statsmodels/tsa/_innovations.pyx"},
187    _hamilton_filter={
188        "source": "statsmodels/tsa/regime_switching/_hamilton_filter.pyx.in"
189    },  # noqa: E501
190    _kim_smoother={
191        "source": "statsmodels/tsa/regime_switching/_kim_smoother.pyx.in"
192    },  # noqa: E501
193    _arma_innovations={
194        "source": "statsmodels/tsa/innovations/_arma_innovations.pyx.in"
195    },  # noqa: E501
196    linbin={"source": "statsmodels/nonparametric/linbin.pyx"},
197    _qn={"source": "statsmodels/robust/_qn.pyx"},
198    _smoothers_lowess={
199        "source": "statsmodels/nonparametric/_smoothers_lowess.pyx"
200    },  # noqa: E501
201)
202
203statespace_exts = [
204    "statsmodels/tsa/statespace/_initialization.pyx.in",
205    "statsmodels/tsa/statespace/_representation.pyx.in",
206    "statsmodels/tsa/statespace/_kalman_filter.pyx.in",
207    "statsmodels/tsa/statespace/_filters/_conventional.pyx.in",
208    "statsmodels/tsa/statespace/_filters/_inversions.pyx.in",
209    "statsmodels/tsa/statespace/_filters/_univariate.pyx.in",
210    "statsmodels/tsa/statespace/_filters/_univariate_diffuse.pyx.in",
211    "statsmodels/tsa/statespace/_kalman_smoother.pyx.in",
212    "statsmodels/tsa/statespace/_smoothers/_alternative.pyx.in",
213    "statsmodels/tsa/statespace/_smoothers/_classical.pyx.in",
214    "statsmodels/tsa/statespace/_smoothers/_conventional.pyx.in",
215    "statsmodels/tsa/statespace/_smoothers/_univariate.pyx.in",
216    "statsmodels/tsa/statespace/_smoothers/_univariate_diffuse.pyx.in",
217    "statsmodels/tsa/statespace/_simulation_smoother.pyx.in",
218    "statsmodels/tsa/statespace/_cfa_simulation_smoother.pyx.in",
219    "statsmodels/tsa/statespace/_tools.pyx.in",
220]
221
222
223class CleanCommand(clean):
224    def run(self):
225        msg = """
226
227python setup.py clean is not supported.
228
229Use one of:
230
231* `git clean -xdf` to clean all untracked files
232* `git clean -Xdf` to clean untracked files ignored by .gitignore
233"""
234        print(msg)
235        sys.exit(1)
236
237
238def update_extension(extension, requires_math=True):
239    import numpy  # noqa: F811
240    from numpy.distutils.log import set_verbosity
241    from numpy.distutils.misc_util import get_info
242
243    set_verbosity(1)
244
245    numpy_includes = [numpy.get_include()]
246    extra_incl = pkg_resources.resource_filename("numpy", "core/include")
247    numpy_includes += [extra_incl]
248    numpy_includes = list(set(numpy_includes))
249    numpy_math_libs = get_info("npymath")
250
251    if not hasattr(extension, "include_dirs"):
252        return
253    extension.include_dirs = list(set(extension.include_dirs + numpy_includes))
254    if requires_math:
255        extension.include_dirs += numpy_math_libs["include_dirs"]
256        extension.libraries += numpy_math_libs["libraries"]
257        extension.library_dirs += numpy_math_libs["library_dirs"]
258
259
260class DeferredBuildExt(build_ext):
261    """build_ext command for use when numpy headers are needed."""
262
263    def build_extensions(self):
264        self._update_extensions()
265        build_ext.build_extensions(self)
266
267    def _update_extensions(self):
268        for extension in self.extensions:
269            requires_math = extension.name in EXT_REQUIRES_NUMPY_MATH_LIBS
270            update_extension(extension, requires_math=requires_math)
271
272
273cmdclass = versioneer.get_cmdclass()
274if not HAS_NUMPY:
275    cmdclass["build_ext"] = DeferredBuildExt
276cmdclass["clean"] = CleanCommand
277
278
279def check_source(source_name):
280    """Chooses C or pyx source files, and raises if C is needed but missing"""
281    source_ext = ".pyx"
282    if not HAS_CYTHON:
283        source_name = source_name.replace(".pyx.in", ".c")
284        source_name = source_name.replace(".pyx", ".c")
285        source_ext = ".c"
286        if not os.path.exists(source_name):
287            msg = (
288                "C source not found.  You must have Cython installed to "
289                "build if the C source files have not been generated."
290            )
291            raise IOError(msg)
292    return source_name, source_ext
293
294
295def process_tempita(source_name):
296    """Runs pyx.in files through tempita is needed"""
297    if source_name.endswith("pyx.in"):
298        with open(source_name, "r") as templated:
299            pyx_template = templated.read()
300        pyx = Tempita.sub(pyx_template)
301        pyx_filename = source_name[:-3]
302        with open(pyx_filename, "w") as pyx_file:
303            pyx_file.write(pyx)
304        file_stats = os.stat(source_name)
305        try:
306            os.utime(
307                pyx_filename,
308                ns=(file_stats.st_atime_ns, file_stats.st_mtime_ns),
309            )
310        except AttributeError:
311            os.utime(pyx_filename, (file_stats.st_atime, file_stats.st_mtime))
312        source_name = pyx_filename
313    return source_name
314
315
316EXT_REQUIRES_NUMPY_MATH_LIBS = []
317extensions = []
318for config in exts.values():
319    uses_blas = True
320    source, ext = check_source(config["source"])
321    source = process_tempita(source)
322    name = source.replace("/", ".").replace(ext, "")
323    include_dirs = config.get("include_dirs", [])
324    depends = config.get("depends", [])
325    libraries = config.get("libraries", [])
326    library_dirs = config.get("library_dirs", [])
327
328    uses_numpy_libraries = config.get("numpy_libraries", False)
329    if uses_blas or uses_numpy_libraries:
330        EXT_REQUIRES_NUMPY_MATH_LIBS.append(name)
331
332    ext = Extension(
333        name,
334        [source],
335        include_dirs=include_dirs,
336        depends=depends,
337        libraries=libraries,
338        library_dirs=library_dirs,
339        define_macros=DEFINE_MACROS,
340    )
341    extensions.append(ext)
342
343for source in statespace_exts:
344    source, ext = check_source(source)
345    source = process_tempita(source)
346    name = source.replace("/", ".").replace(ext, "")
347
348    EXT_REQUIRES_NUMPY_MATH_LIBS.append(name)
349    ext = Extension(
350        name,
351        [source],
352        include_dirs=["statsmodels/src"],
353        depends=[],
354        libraries=[],
355        library_dirs=[],
356        define_macros=DEFINE_MACROS,
357    )
358    extensions.append(ext)
359
360if HAS_NUMPY:
361    for extension in extensions:
362        requires_math = extension.name in EXT_REQUIRES_NUMPY_MATH_LIBS
363        update_extension(extension, requires_math=requires_math)
364
365if HAS_CYTHON:
366    extensions = cythonize(
367        extensions,
368        compiler_directives=COMPILER_DIRECTIVES,
369        language_level=3,
370        force=CYTHON_COVERAGE,
371    )
372
373##############################################################################
374# Construct package data
375##############################################################################
376package_data = defaultdict(list)
377filetypes = ["*.csv", "*.txt", "*.dta"]
378for root, _, filenames in os.walk(
379    pjoin(os.getcwd(), "statsmodels", "datasets")
380):  # noqa: E501
381    matches = []
382    for filetype in filetypes:
383        for filename in fnmatch.filter(filenames, filetype):
384            matches.append(filename)
385    if matches:
386        package_data[".".join(relpath(root).split(os.path.sep))] = filetypes
387for root, _, _ in os.walk(pjoin(os.getcwd(), "statsmodels")):
388    if root.endswith("results"):
389        package_data[".".join(relpath(root).split(os.path.sep))] = filetypes
390
391for path, filetypes in ADDITIONAL_PACKAGE_DATA.items():
392    package_data[path].extend(filetypes)
393
394if os.path.exists("MANIFEST"):
395    os.unlink("MANIFEST")
396
397
398class BinaryDistribution(Distribution):
399    def is_pure(self):
400        return False
401
402
403setup(
404    name=DISTNAME,
405    version=versioneer.get_version(),
406    maintainer=MAINTAINER,
407    ext_modules=extensions,
408    maintainer_email=MAINTAINER_EMAIL,
409    description=DESCRIPTION,
410    license=LICENSE,
411    url=URL,
412    download_url=DOWNLOAD_URL,
413    project_urls=PROJECT_URLS,
414    long_description=LONG_DESCRIPTION,
415    classifiers=CLASSIFIERS,
416    platforms="any",
417    cmdclass=cmdclass,
418    packages=find_packages(),
419    package_data=package_data,
420    distclass=BinaryDistribution,
421    include_package_data=False,  # True will install all files in repo
422    setup_requires=SETUP_REQUIRES,
423    install_requires=INSTALL_REQUIRES,
424    extras_require=EXTRAS_REQUIRE,
425    zip_safe=False,
426    python_requires=">=3.7",
427)
428
429# Clean-up copied files
430for copy in FILES_COPIED_TO_PACKAGE:
431    os.unlink(copy)
432