1""" 2Utility functions for 3 4- building and importing modules on test time, using a temporary location 5- detecting if compilers are present 6 7""" 8import os 9import sys 10import subprocess 11import tempfile 12import shutil 13import atexit 14import textwrap 15import re 16import pytest 17 18from numpy.compat import asbytes, asstr 19from numpy.testing import temppath 20from importlib import import_module 21 22# 23# Maintaining a temporary module directory 24# 25 26_module_dir = None 27_module_num = 5403 28 29 30def _cleanup(): 31 global _module_dir 32 if _module_dir is not None: 33 try: 34 sys.path.remove(_module_dir) 35 except ValueError: 36 pass 37 try: 38 shutil.rmtree(_module_dir) 39 except (IOError, OSError): 40 pass 41 _module_dir = None 42 43 44def get_module_dir(): 45 global _module_dir 46 if _module_dir is None: 47 _module_dir = tempfile.mkdtemp() 48 atexit.register(_cleanup) 49 if _module_dir not in sys.path: 50 sys.path.insert(0, _module_dir) 51 return _module_dir 52 53 54def get_temp_module_name(): 55 # Assume single-threaded, and the module dir usable only by this thread 56 global _module_num 57 d = get_module_dir() 58 name = "_test_ext_module_%d" % _module_num 59 _module_num += 1 60 if name in sys.modules: 61 # this should not be possible, but check anyway 62 raise RuntimeError("Temporary module name already in use.") 63 return name 64 65 66def _memoize(func): 67 memo = {} 68 69 def wrapper(*a, **kw): 70 key = repr((a, kw)) 71 if key not in memo: 72 try: 73 memo[key] = func(*a, **kw) 74 except Exception as e: 75 memo[key] = e 76 raise 77 ret = memo[key] 78 if isinstance(ret, Exception): 79 raise ret 80 return ret 81 wrapper.__name__ = func.__name__ 82 return wrapper 83 84# 85# Building modules 86# 87 88 89@_memoize 90def build_module(source_files, options=[], skip=[], only=[], module_name=None): 91 """ 92 Compile and import a f2py module, built from the given files. 93 94 """ 95 96 code = ("import sys; sys.path = %s; import numpy.f2py as f2py2e; " 97 "f2py2e.main()" % repr(sys.path)) 98 99 d = get_module_dir() 100 101 # Copy files 102 dst_sources = [] 103 f2py_sources = [] 104 for fn in source_files: 105 if not os.path.isfile(fn): 106 raise RuntimeError("%s is not a file" % fn) 107 dst = os.path.join(d, os.path.basename(fn)) 108 shutil.copyfile(fn, dst) 109 dst_sources.append(dst) 110 111 base, ext = os.path.splitext(dst) 112 if ext in ('.f90', '.f', '.c', '.pyf'): 113 f2py_sources.append(dst) 114 115 # Prepare options 116 if module_name is None: 117 module_name = get_temp_module_name() 118 f2py_opts = ['-c', '-m', module_name] + options + f2py_sources 119 if skip: 120 f2py_opts += ['skip:'] + skip 121 if only: 122 f2py_opts += ['only:'] + only 123 124 # Build 125 cwd = os.getcwd() 126 try: 127 os.chdir(d) 128 cmd = [sys.executable, '-c', code] + f2py_opts 129 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, 130 stderr=subprocess.STDOUT) 131 out, err = p.communicate() 132 if p.returncode != 0: 133 raise RuntimeError("Running f2py failed: %s\n%s" 134 % (cmd[4:], asstr(out))) 135 finally: 136 os.chdir(cwd) 137 138 # Partial cleanup 139 for fn in dst_sources: 140 os.unlink(fn) 141 142 # Import 143 return import_module(module_name) 144 145 146@_memoize 147def build_code(source_code, options=[], skip=[], only=[], suffix=None, 148 module_name=None): 149 """ 150 Compile and import Fortran code using f2py. 151 152 """ 153 if suffix is None: 154 suffix = '.f' 155 with temppath(suffix=suffix) as path: 156 with open(path, 'w') as f: 157 f.write(source_code) 158 return build_module([path], options=options, skip=skip, only=only, 159 module_name=module_name) 160 161# 162# Check if compilers are available at all... 163# 164 165_compiler_status = None 166 167 168def _get_compiler_status(): 169 global _compiler_status 170 if _compiler_status is not None: 171 return _compiler_status 172 173 _compiler_status = (False, False, False) 174 175 # XXX: this is really ugly. But I don't know how to invoke Distutils 176 # in a safer way... 177 code = textwrap.dedent("""\ 178 import os 179 import sys 180 sys.path = %(syspath)s 181 182 def configuration(parent_name='',top_path=None): 183 global config 184 from numpy.distutils.misc_util import Configuration 185 config = Configuration('', parent_name, top_path) 186 return config 187 188 from numpy.distutils.core import setup 189 setup(configuration=configuration) 190 191 config_cmd = config.get_config_cmd() 192 have_c = config_cmd.try_compile('void foo() {}') 193 print('COMPILERS:%%d,%%d,%%d' %% (have_c, 194 config.have_f77c(), 195 config.have_f90c())) 196 sys.exit(99) 197 """) 198 code = code % dict(syspath=repr(sys.path)) 199 200 tmpdir = tempfile.mkdtemp() 201 try: 202 script = os.path.join(tmpdir, 'setup.py') 203 204 with open(script, 'w') as f: 205 f.write(code) 206 207 cmd = [sys.executable, 'setup.py', 'config'] 208 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, 209 stderr=subprocess.STDOUT, 210 cwd=tmpdir) 211 out, err = p.communicate() 212 finally: 213 shutil.rmtree(tmpdir) 214 215 m = re.search(br'COMPILERS:(\d+),(\d+),(\d+)', out) 216 if m: 217 _compiler_status = (bool(int(m.group(1))), bool(int(m.group(2))), 218 bool(int(m.group(3)))) 219 # Finished 220 return _compiler_status 221 222 223def has_c_compiler(): 224 return _get_compiler_status()[0] 225 226 227def has_f77_compiler(): 228 return _get_compiler_status()[1] 229 230 231def has_f90_compiler(): 232 return _get_compiler_status()[2] 233 234# 235# Building with distutils 236# 237 238 239@_memoize 240def build_module_distutils(source_files, config_code, module_name, **kw): 241 """ 242 Build a module via distutils and import it. 243 244 """ 245 from numpy.distutils.misc_util import Configuration 246 from numpy.distutils.core import setup 247 248 d = get_module_dir() 249 250 # Copy files 251 dst_sources = [] 252 for fn in source_files: 253 if not os.path.isfile(fn): 254 raise RuntimeError("%s is not a file" % fn) 255 dst = os.path.join(d, os.path.basename(fn)) 256 shutil.copyfile(fn, dst) 257 dst_sources.append(dst) 258 259 # Build script 260 config_code = textwrap.dedent(config_code).replace("\n", "\n ") 261 262 code = textwrap.dedent("""\ 263 import os 264 import sys 265 sys.path = %(syspath)s 266 267 def configuration(parent_name='',top_path=None): 268 from numpy.distutils.misc_util import Configuration 269 config = Configuration('', parent_name, top_path) 270 %(config_code)s 271 return config 272 273 if __name__ == "__main__": 274 from numpy.distutils.core import setup 275 setup(configuration=configuration) 276 """) % dict(config_code=config_code, syspath=repr(sys.path)) 277 278 script = os.path.join(d, get_temp_module_name() + '.py') 279 dst_sources.append(script) 280 with open(script, 'wb') as f: 281 f.write(asbytes(code)) 282 283 # Build 284 cwd = os.getcwd() 285 try: 286 os.chdir(d) 287 cmd = [sys.executable, script, 'build_ext', '-i'] 288 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, 289 stderr=subprocess.STDOUT) 290 out, err = p.communicate() 291 if p.returncode != 0: 292 raise RuntimeError("Running distutils build failed: %s\n%s" 293 % (cmd[4:], asstr(out))) 294 finally: 295 os.chdir(cwd) 296 297 # Partial cleanup 298 for fn in dst_sources: 299 os.unlink(fn) 300 301 # Import 302 __import__(module_name) 303 return sys.modules[module_name] 304 305# 306# Unittest convenience 307# 308 309 310class F2PyTest: 311 code = None 312 sources = None 313 options = [] 314 skip = [] 315 only = [] 316 suffix = '.f' 317 module = None 318 module_name = None 319 320 def setup(self): 321 if sys.platform == 'win32': 322 pytest.skip('Fails with MinGW64 Gfortran (Issue #9673)') 323 324 if self.module is not None: 325 return 326 327 # Check compiler availability first 328 if not has_c_compiler(): 329 pytest.skip("No C compiler available") 330 331 codes = [] 332 if self.sources: 333 codes.extend(self.sources) 334 if self.code is not None: 335 codes.append(self.suffix) 336 337 needs_f77 = False 338 needs_f90 = False 339 for fn in codes: 340 if fn.endswith('.f'): 341 needs_f77 = True 342 elif fn.endswith('.f90'): 343 needs_f90 = True 344 if needs_f77 and not has_f77_compiler(): 345 pytest.skip("No Fortran 77 compiler available") 346 if needs_f90 and not has_f90_compiler(): 347 pytest.skip("No Fortran 90 compiler available") 348 349 # Build the module 350 if self.code is not None: 351 self.module = build_code(self.code, options=self.options, 352 skip=self.skip, only=self.only, 353 suffix=self.suffix, 354 module_name=self.module_name) 355 356 if self.sources is not None: 357 self.module = build_module(self.sources, options=self.options, 358 skip=self.skip, only=self.only, 359 module_name=self.module_name) 360