1from __future__ import absolute_import, print_function 2 3import cython 4from .. import __version__ 5 6import collections 7import contextlib 8import hashlib 9import os 10import shutil 11import subprocess 12import re, sys, time 13import warnings 14from glob import iglob 15from io import open as io_open 16from os.path import relpath as _relpath 17from distutils.extension import Extension 18from distutils.util import strtobool 19import zipfile 20 21try: 22 from collections.abc import Iterable 23except ImportError: 24 from collections import Iterable 25 26try: 27 import gzip 28 gzip_open = gzip.open 29 gzip_ext = '.gz' 30except ImportError: 31 gzip_open = open 32 gzip_ext = '' 33 34try: 35 import zlib 36 zipfile_compression_mode = zipfile.ZIP_DEFLATED 37except ImportError: 38 zipfile_compression_mode = zipfile.ZIP_STORED 39 40try: 41 import pythran 42except: 43 pythran = None 44 45from .. import Utils 46from ..Utils import (cached_function, cached_method, path_exists, 47 safe_makedirs, copy_file_to_dir_if_newer, is_package_dir, replace_suffix) 48from ..Compiler.Main import Context, CompilationOptions, default_options 49 50join_path = cached_function(os.path.join) 51copy_once_if_newer = cached_function(copy_file_to_dir_if_newer) 52safe_makedirs_once = cached_function(safe_makedirs) 53 54if sys.version_info[0] < 3: 55 # stupid Py2 distutils enforces str type in list of sources 56 _fs_encoding = sys.getfilesystemencoding() 57 if _fs_encoding is None: 58 _fs_encoding = sys.getdefaultencoding() 59 def encode_filename_in_py2(filename): 60 if not isinstance(filename, bytes): 61 return filename.encode(_fs_encoding) 62 return filename 63else: 64 def encode_filename_in_py2(filename): 65 return filename 66 basestring = str 67 68 69def _make_relative(file_paths, base=None): 70 if not base: 71 base = os.getcwd() 72 if base[-1] != os.path.sep: 73 base += os.path.sep 74 return [_relpath(path, base) if path.startswith(base) else path 75 for path in file_paths] 76 77 78def extended_iglob(pattern): 79 if '{' in pattern: 80 m = re.match('(.*){([^}]+)}(.*)', pattern) 81 if m: 82 before, switch, after = m.groups() 83 for case in switch.split(','): 84 for path in extended_iglob(before + case + after): 85 yield path 86 return 87 if '**/' in pattern: 88 seen = set() 89 first, rest = pattern.split('**/', 1) 90 if first: 91 first = iglob(first+'/') 92 else: 93 first = [''] 94 for root in first: 95 for path in extended_iglob(join_path(root, rest)): 96 if path not in seen: 97 seen.add(path) 98 yield path 99 for path in extended_iglob(join_path(root, '*', '**/' + rest)): 100 if path not in seen: 101 seen.add(path) 102 yield path 103 else: 104 for path in iglob(pattern): 105 yield path 106 107 108def nonempty(it, error_msg="expected non-empty iterator"): 109 empty = True 110 for value in it: 111 empty = False 112 yield value 113 if empty: 114 raise ValueError(error_msg) 115 116 117@cached_function 118def file_hash(filename): 119 path = os.path.normpath(filename) 120 prefix = ('%d:%s' % (len(path), path)).encode("UTF-8") 121 m = hashlib.md5(prefix) 122 with open(path, 'rb') as f: 123 data = f.read(65000) 124 while data: 125 m.update(data) 126 data = f.read(65000) 127 return m.hexdigest() 128 129 130def update_pythran_extension(ext): 131 if pythran is None: 132 raise RuntimeError("You first need to install Pythran to use the np_pythran directive.") 133 try: 134 pythran_ext = pythran.config.make_extension(python=True) 135 except TypeError: # older pythran version only 136 pythran_ext = pythran.config.make_extension() 137 138 ext.include_dirs.extend(pythran_ext['include_dirs']) 139 ext.extra_compile_args.extend(pythran_ext['extra_compile_args']) 140 ext.extra_link_args.extend(pythran_ext['extra_link_args']) 141 ext.define_macros.extend(pythran_ext['define_macros']) 142 ext.undef_macros.extend(pythran_ext['undef_macros']) 143 ext.library_dirs.extend(pythran_ext['library_dirs']) 144 ext.libraries.extend(pythran_ext['libraries']) 145 ext.language = 'c++' 146 147 # These options are not compatible with the way normal Cython extensions work 148 for bad_option in ["-fwhole-program", "-fvisibility=hidden"]: 149 try: 150 ext.extra_compile_args.remove(bad_option) 151 except ValueError: 152 pass 153 154 155def parse_list(s): 156 """ 157 >>> parse_list("") 158 [] 159 >>> parse_list("a") 160 ['a'] 161 >>> parse_list("a b c") 162 ['a', 'b', 'c'] 163 >>> parse_list("[a, b, c]") 164 ['a', 'b', 'c'] 165 >>> parse_list('a " " b') 166 ['a', ' ', 'b'] 167 >>> parse_list('[a, ",a", "a,", ",", ]') 168 ['a', ',a', 'a,', ','] 169 """ 170 if len(s) >= 2 and s[0] == '[' and s[-1] == ']': 171 s = s[1:-1] 172 delimiter = ',' 173 else: 174 delimiter = ' ' 175 s, literals = strip_string_literals(s) 176 def unquote(literal): 177 literal = literal.strip() 178 if literal[0] in "'\"": 179 return literals[literal[1:-1]] 180 else: 181 return literal 182 return [unquote(item) for item in s.split(delimiter) if item.strip()] 183 184 185transitive_str = object() 186transitive_list = object() 187bool_or = object() 188 189distutils_settings = { 190 'name': str, 191 'sources': list, 192 'define_macros': list, 193 'undef_macros': list, 194 'libraries': transitive_list, 195 'library_dirs': transitive_list, 196 'runtime_library_dirs': transitive_list, 197 'include_dirs': transitive_list, 198 'extra_objects': list, 199 'extra_compile_args': transitive_list, 200 'extra_link_args': transitive_list, 201 'export_symbols': list, 202 'depends': transitive_list, 203 'language': transitive_str, 204 'np_pythran': bool_or 205} 206 207 208@cython.locals(start=cython.Py_ssize_t, end=cython.Py_ssize_t) 209def line_iter(source): 210 if isinstance(source, basestring): 211 start = 0 212 while True: 213 end = source.find('\n', start) 214 if end == -1: 215 yield source[start:] 216 return 217 yield source[start:end] 218 start = end+1 219 else: 220 for line in source: 221 yield line 222 223 224class DistutilsInfo(object): 225 226 def __init__(self, source=None, exn=None): 227 self.values = {} 228 if source is not None: 229 for line in line_iter(source): 230 line = line.lstrip() 231 if not line: 232 continue 233 if line[0] != '#': 234 break 235 line = line[1:].lstrip() 236 kind = next((k for k in ("distutils:","cython:") if line.startswith(k)), None) 237 if kind is not None: 238 key, _, value = [s.strip() for s in line[len(kind):].partition('=')] 239 type = distutils_settings.get(key, None) 240 if line.startswith("cython:") and type is None: continue 241 if type in (list, transitive_list): 242 value = parse_list(value) 243 if key == 'define_macros': 244 value = [tuple(macro.split('=', 1)) 245 if '=' in macro else (macro, None) 246 for macro in value] 247 if type is bool_or: 248 value = strtobool(value) 249 self.values[key] = value 250 elif exn is not None: 251 for key in distutils_settings: 252 if key in ('name', 'sources','np_pythran'): 253 continue 254 value = getattr(exn, key, None) 255 if value: 256 self.values[key] = value 257 258 def merge(self, other): 259 if other is None: 260 return self 261 for key, value in other.values.items(): 262 type = distutils_settings[key] 263 if type is transitive_str and key not in self.values: 264 self.values[key] = value 265 elif type is transitive_list: 266 if key in self.values: 267 # Change a *copy* of the list (Trac #845) 268 all = self.values[key][:] 269 for v in value: 270 if v not in all: 271 all.append(v) 272 value = all 273 self.values[key] = value 274 elif type is bool_or: 275 self.values[key] = self.values.get(key, False) | value 276 return self 277 278 def subs(self, aliases): 279 if aliases is None: 280 return self 281 resolved = DistutilsInfo() 282 for key, value in self.values.items(): 283 type = distutils_settings[key] 284 if type in [list, transitive_list]: 285 new_value_list = [] 286 for v in value: 287 if v in aliases: 288 v = aliases[v] 289 if isinstance(v, list): 290 new_value_list += v 291 else: 292 new_value_list.append(v) 293 value = new_value_list 294 else: 295 if value in aliases: 296 value = aliases[value] 297 resolved.values[key] = value 298 return resolved 299 300 def apply(self, extension): 301 for key, value in self.values.items(): 302 type = distutils_settings[key] 303 if type in [list, transitive_list]: 304 value = getattr(extension, key) + list(value) 305 setattr(extension, key, value) 306 307 308@cython.locals(start=cython.Py_ssize_t, q=cython.Py_ssize_t, 309 single_q=cython.Py_ssize_t, double_q=cython.Py_ssize_t, 310 hash_mark=cython.Py_ssize_t, end=cython.Py_ssize_t, 311 k=cython.Py_ssize_t, counter=cython.Py_ssize_t, quote_len=cython.Py_ssize_t) 312def strip_string_literals(code, prefix='__Pyx_L'): 313 """ 314 Normalizes every string literal to be of the form '__Pyx_Lxxx', 315 returning the normalized code and a mapping of labels to 316 string literals. 317 """ 318 new_code = [] 319 literals = {} 320 counter = 0 321 start = q = 0 322 in_quote = False 323 hash_mark = single_q = double_q = -1 324 code_len = len(code) 325 quote_type = quote_len = None 326 327 while True: 328 if hash_mark < q: 329 hash_mark = code.find('#', q) 330 if single_q < q: 331 single_q = code.find("'", q) 332 if double_q < q: 333 double_q = code.find('"', q) 334 q = min(single_q, double_q) 335 if q == -1: 336 q = max(single_q, double_q) 337 338 # We're done. 339 if q == -1 and hash_mark == -1: 340 new_code.append(code[start:]) 341 break 342 343 # Try to close the quote. 344 elif in_quote: 345 if code[q-1] == u'\\': 346 k = 2 347 while q >= k and code[q-k] == u'\\': 348 k += 1 349 if k % 2 == 0: 350 q += 1 351 continue 352 if code[q] == quote_type and ( 353 quote_len == 1 or (code_len > q + 2 and quote_type == code[q+1] == code[q+2])): 354 counter += 1 355 label = "%s%s_" % (prefix, counter) 356 literals[label] = code[start+quote_len:q] 357 full_quote = code[q:q+quote_len] 358 new_code.append(full_quote) 359 new_code.append(label) 360 new_code.append(full_quote) 361 q += quote_len 362 in_quote = False 363 start = q 364 else: 365 q += 1 366 367 # Process comment. 368 elif -1 != hash_mark and (hash_mark < q or q == -1): 369 new_code.append(code[start:hash_mark+1]) 370 end = code.find('\n', hash_mark) 371 counter += 1 372 label = "%s%s_" % (prefix, counter) 373 if end == -1: 374 end_or_none = None 375 else: 376 end_or_none = end 377 literals[label] = code[hash_mark+1:end_or_none] 378 new_code.append(label) 379 if end == -1: 380 break 381 start = q = end 382 383 # Open the quote. 384 else: 385 if code_len >= q+3 and (code[q] == code[q+1] == code[q+2]): 386 quote_len = 3 387 else: 388 quote_len = 1 389 in_quote = True 390 quote_type = code[q] 391 new_code.append(code[start:q]) 392 start = q 393 q += quote_len 394 395 return "".join(new_code), literals 396 397 398# We need to allow spaces to allow for conditional compilation like 399# IF ...: 400# cimport ... 401dependency_regex = re.compile(r"(?:^\s*from +([0-9a-zA-Z_.]+) +cimport)|" 402 r"(?:^\s*cimport +([0-9a-zA-Z_.]+(?: *, *[0-9a-zA-Z_.]+)*))|" 403 r"(?:^\s*cdef +extern +from +['\"]([^'\"]+)['\"])|" 404 r"(?:^\s*include +['\"]([^'\"]+)['\"])", re.M) 405dependency_after_from_regex = re.compile( 406 r"(?:^\s+\(([0-9a-zA-Z_., ]*)\)[#\n])|" 407 r"(?:^\s+([0-9a-zA-Z_., ]*)[#\n])", 408 re.M) 409 410 411def normalize_existing(base_path, rel_paths): 412 return normalize_existing0(os.path.dirname(base_path), tuple(set(rel_paths))) 413 414 415@cached_function 416def normalize_existing0(base_dir, rel_paths): 417 """ 418 Given some base directory ``base_dir`` and a list of path names 419 ``rel_paths``, normalize each relative path name ``rel`` by 420 replacing it by ``os.path.join(base, rel)`` if that file exists. 421 422 Return a couple ``(normalized, needed_base)`` where ``normalized`` 423 if the list of normalized file names and ``needed_base`` is 424 ``base_dir`` if we actually needed ``base_dir``. If no paths were 425 changed (for example, if all paths were already absolute), then 426 ``needed_base`` is ``None``. 427 """ 428 normalized = [] 429 needed_base = None 430 for rel in rel_paths: 431 if os.path.isabs(rel): 432 normalized.append(rel) 433 continue 434 path = join_path(base_dir, rel) 435 if path_exists(path): 436 normalized.append(os.path.normpath(path)) 437 needed_base = base_dir 438 else: 439 normalized.append(rel) 440 return (normalized, needed_base) 441 442 443def resolve_depends(depends, include_dirs): 444 include_dirs = tuple(include_dirs) 445 resolved = [] 446 for depend in depends: 447 path = resolve_depend(depend, include_dirs) 448 if path is not None: 449 resolved.append(path) 450 return resolved 451 452 453@cached_function 454def resolve_depend(depend, include_dirs): 455 if depend[0] == '<' and depend[-1] == '>': 456 return None 457 for dir in include_dirs: 458 path = join_path(dir, depend) 459 if path_exists(path): 460 return os.path.normpath(path) 461 return None 462 463 464@cached_function 465def package(filename): 466 dir = os.path.dirname(os.path.abspath(str(filename))) 467 if dir != filename and is_package_dir(dir): 468 return package(dir) + (os.path.basename(dir),) 469 else: 470 return () 471 472 473@cached_function 474def fully_qualified_name(filename): 475 module = os.path.splitext(os.path.basename(filename))[0] 476 return '.'.join(package(filename) + (module,)) 477 478 479@cached_function 480def parse_dependencies(source_filename): 481 # Actual parsing is way too slow, so we use regular expressions. 482 # The only catch is that we must strip comments and string 483 # literals ahead of time. 484 with Utils.open_source_file(source_filename, error_handling='ignore') as fh: 485 source = fh.read() 486 distutils_info = DistutilsInfo(source) 487 source, literals = strip_string_literals(source) 488 source = source.replace('\\\n', ' ').replace('\t', ' ') 489 490 # TODO: pure mode 491 cimports = [] 492 includes = [] 493 externs = [] 494 for m in dependency_regex.finditer(source): 495 cimport_from, cimport_list, extern, include = m.groups() 496 if cimport_from: 497 cimports.append(cimport_from) 498 m_after_from = dependency_after_from_regex.search(source, pos=m.end()) 499 if m_after_from: 500 multiline, one_line = m_after_from.groups() 501 subimports = multiline or one_line 502 cimports.extend("{0}.{1}".format(cimport_from, s.strip()) 503 for s in subimports.split(',')) 504 505 elif cimport_list: 506 cimports.extend(x.strip() for x in cimport_list.split(",")) 507 elif extern: 508 externs.append(literals[extern]) 509 else: 510 includes.append(literals[include]) 511 return cimports, includes, externs, distutils_info 512 513 514class DependencyTree(object): 515 516 def __init__(self, context, quiet=False): 517 self.context = context 518 self.quiet = quiet 519 self._transitive_cache = {} 520 521 def parse_dependencies(self, source_filename): 522 if path_exists(source_filename): 523 source_filename = os.path.normpath(source_filename) 524 return parse_dependencies(source_filename) 525 526 @cached_method 527 def included_files(self, filename): 528 # This is messy because included files are textually included, resolving 529 # cimports (but not includes) relative to the including file. 530 all = set() 531 for include in self.parse_dependencies(filename)[1]: 532 include_path = join_path(os.path.dirname(filename), include) 533 if not path_exists(include_path): 534 include_path = self.context.find_include_file(include, None) 535 if include_path: 536 if '.' + os.path.sep in include_path: 537 include_path = os.path.normpath(include_path) 538 all.add(include_path) 539 all.update(self.included_files(include_path)) 540 elif not self.quiet: 541 print("Unable to locate '%s' referenced from '%s'" % (filename, include)) 542 return all 543 544 @cached_method 545 def cimports_externs_incdirs(self, filename): 546 # This is really ugly. Nested cimports are resolved with respect to the 547 # includer, but includes are resolved with respect to the includee. 548 cimports, includes, externs = self.parse_dependencies(filename)[:3] 549 cimports = set(cimports) 550 externs = set(externs) 551 incdirs = set() 552 for include in self.included_files(filename): 553 included_cimports, included_externs, included_incdirs = self.cimports_externs_incdirs(include) 554 cimports.update(included_cimports) 555 externs.update(included_externs) 556 incdirs.update(included_incdirs) 557 externs, incdir = normalize_existing(filename, externs) 558 if incdir: 559 incdirs.add(incdir) 560 return tuple(cimports), externs, incdirs 561 562 def cimports(self, filename): 563 return self.cimports_externs_incdirs(filename)[0] 564 565 def package(self, filename): 566 return package(filename) 567 568 def fully_qualified_name(self, filename): 569 return fully_qualified_name(filename) 570 571 @cached_method 572 def find_pxd(self, module, filename=None): 573 is_relative = module[0] == '.' 574 if is_relative and not filename: 575 raise NotImplementedError("New relative imports.") 576 if filename is not None: 577 module_path = module.split('.') 578 if is_relative: 579 module_path.pop(0) # just explicitly relative 580 package_path = list(self.package(filename)) 581 while module_path and not module_path[0]: 582 try: 583 package_path.pop() 584 except IndexError: 585 return None # FIXME: error? 586 module_path.pop(0) 587 relative = '.'.join(package_path + module_path) 588 pxd = self.context.find_pxd_file(relative, None) 589 if pxd: 590 return pxd 591 if is_relative: 592 return None # FIXME: error? 593 return self.context.find_pxd_file(module, None) 594 595 @cached_method 596 def cimported_files(self, filename): 597 if filename[-4:] == '.pyx' and path_exists(filename[:-4] + '.pxd'): 598 pxd_list = [filename[:-4] + '.pxd'] 599 else: 600 pxd_list = [] 601 # Cimports generates all possible combinations package.module 602 # when imported as from package cimport module. 603 for module in self.cimports(filename): 604 if module[:7] == 'cython.' or module == 'cython': 605 continue 606 pxd_file = self.find_pxd(module, filename) 607 if pxd_file is not None: 608 pxd_list.append(pxd_file) 609 return tuple(pxd_list) 610 611 @cached_method 612 def immediate_dependencies(self, filename): 613 all = set([filename]) 614 all.update(self.cimported_files(filename)) 615 all.update(self.included_files(filename)) 616 return all 617 618 def all_dependencies(self, filename): 619 return self.transitive_merge(filename, self.immediate_dependencies, set.union) 620 621 @cached_method 622 def timestamp(self, filename): 623 return os.path.getmtime(filename) 624 625 def extract_timestamp(self, filename): 626 return self.timestamp(filename), filename 627 628 def newest_dependency(self, filename): 629 return max([self.extract_timestamp(f) for f in self.all_dependencies(filename)]) 630 631 def transitive_fingerprint(self, filename, module, compilation_options): 632 r""" 633 Return a fingerprint of a cython file that is about to be cythonized. 634 635 Fingerprints are looked up in future compilations. If the fingerprint 636 is found, the cythonization can be skipped. The fingerprint must 637 incorporate everything that has an influence on the generated code. 638 """ 639 try: 640 m = hashlib.md5(__version__.encode('UTF-8')) 641 m.update(file_hash(filename).encode('UTF-8')) 642 for x in sorted(self.all_dependencies(filename)): 643 if os.path.splitext(x)[1] not in ('.c', '.cpp', '.h'): 644 m.update(file_hash(x).encode('UTF-8')) 645 # Include the module attributes that change the compilation result 646 # in the fingerprint. We do not iterate over module.__dict__ and 647 # include almost everything here as users might extend Extension 648 # with arbitrary (random) attributes that would lead to cache 649 # misses. 650 m.update(str(( 651 module.language, 652 getattr(module, 'py_limited_api', False), 653 getattr(module, 'np_pythran', False) 654 )).encode('UTF-8')) 655 656 m.update(compilation_options.get_fingerprint().encode('UTF-8')) 657 return m.hexdigest() 658 except IOError: 659 return None 660 661 def distutils_info0(self, filename): 662 info = self.parse_dependencies(filename)[3] 663 kwds = info.values 664 cimports, externs, incdirs = self.cimports_externs_incdirs(filename) 665 basedir = os.getcwd() 666 # Add dependencies on "cdef extern from ..." files 667 if externs: 668 externs = _make_relative(externs, basedir) 669 if 'depends' in kwds: 670 kwds['depends'] = list(set(kwds['depends']).union(externs)) 671 else: 672 kwds['depends'] = list(externs) 673 # Add include_dirs to ensure that the C compiler will find the 674 # "cdef extern from ..." files 675 if incdirs: 676 include_dirs = list(kwds.get('include_dirs', [])) 677 for inc in _make_relative(incdirs, basedir): 678 if inc not in include_dirs: 679 include_dirs.append(inc) 680 kwds['include_dirs'] = include_dirs 681 return info 682 683 def distutils_info(self, filename, aliases=None, base=None): 684 return (self.transitive_merge(filename, self.distutils_info0, DistutilsInfo.merge) 685 .subs(aliases) 686 .merge(base)) 687 688 def transitive_merge(self, node, extract, merge): 689 try: 690 seen = self._transitive_cache[extract, merge] 691 except KeyError: 692 seen = self._transitive_cache[extract, merge] = {} 693 return self.transitive_merge_helper( 694 node, extract, merge, seen, {}, self.cimported_files)[0] 695 696 def transitive_merge_helper(self, node, extract, merge, seen, stack, outgoing): 697 if node in seen: 698 return seen[node], None 699 deps = extract(node) 700 if node in stack: 701 return deps, node 702 try: 703 stack[node] = len(stack) 704 loop = None 705 for next in outgoing(node): 706 sub_deps, sub_loop = self.transitive_merge_helper(next, extract, merge, seen, stack, outgoing) 707 if sub_loop is not None: 708 if loop is not None and stack[loop] < stack[sub_loop]: 709 pass 710 else: 711 loop = sub_loop 712 deps = merge(deps, sub_deps) 713 if loop == node: 714 loop = None 715 if loop is None: 716 seen[node] = deps 717 return deps, loop 718 finally: 719 del stack[node] 720 721 722_dep_tree = None 723 724def create_dependency_tree(ctx=None, quiet=False): 725 global _dep_tree 726 if _dep_tree is None: 727 if ctx is None: 728 ctx = Context(["."], CompilationOptions(default_options)) 729 _dep_tree = DependencyTree(ctx, quiet=quiet) 730 return _dep_tree 731 732 733# If this changes, change also docs/src/reference/compilation.rst 734# which mentions this function 735def default_create_extension(template, kwds): 736 if 'depends' in kwds: 737 include_dirs = kwds.get('include_dirs', []) + ["."] 738 depends = resolve_depends(kwds['depends'], include_dirs) 739 kwds['depends'] = sorted(set(depends + template.depends)) 740 741 t = template.__class__ 742 ext = t(**kwds) 743 metadata = dict(distutils=kwds, module_name=kwds['name']) 744 return (ext, metadata) 745 746 747# This may be useful for advanced users? 748def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet=False, language=None, 749 exclude_failures=False): 750 if language is not None: 751 print('Warning: passing language={0!r} to cythonize() is deprecated. ' 752 'Instead, put "# distutils: language={0}" in your .pyx or .pxd file(s)'.format(language)) 753 if exclude is None: 754 exclude = [] 755 if patterns is None: 756 return [], {} 757 elif isinstance(patterns, basestring) or not isinstance(patterns, Iterable): 758 patterns = [patterns] 759 explicit_modules = set([m.name for m in patterns if isinstance(m, Extension)]) 760 seen = set() 761 deps = create_dependency_tree(ctx, quiet=quiet) 762 to_exclude = set() 763 if not isinstance(exclude, list): 764 exclude = [exclude] 765 for pattern in exclude: 766 to_exclude.update(map(os.path.abspath, extended_iglob(pattern))) 767 768 module_list = [] 769 module_metadata = {} 770 771 # workaround for setuptools 772 if 'setuptools' in sys.modules: 773 Extension_distutils = sys.modules['setuptools.extension']._Extension 774 Extension_setuptools = sys.modules['setuptools'].Extension 775 else: 776 # dummy class, in case we do not have setuptools 777 Extension_distutils = Extension 778 class Extension_setuptools(Extension): pass 779 780 # if no create_extension() function is defined, use a simple 781 # default function. 782 create_extension = ctx.options.create_extension or default_create_extension 783 784 for pattern in patterns: 785 if isinstance(pattern, str): 786 filepattern = pattern 787 template = Extension(pattern, []) # Fake Extension without sources 788 name = '*' 789 base = None 790 ext_language = language 791 elif isinstance(pattern, (Extension_distutils, Extension_setuptools)): 792 cython_sources = [s for s in pattern.sources 793 if os.path.splitext(s)[1] in ('.py', '.pyx')] 794 if cython_sources: 795 filepattern = cython_sources[0] 796 if len(cython_sources) > 1: 797 print("Warning: Multiple cython sources found for extension '%s': %s\n" 798 "See http://cython.readthedocs.io/en/latest/src/userguide/sharing_declarations.html " 799 "for sharing declarations among Cython files." % (pattern.name, cython_sources)) 800 else: 801 # ignore non-cython modules 802 module_list.append(pattern) 803 continue 804 template = pattern 805 name = template.name 806 base = DistutilsInfo(exn=template) 807 ext_language = None # do not override whatever the Extension says 808 else: 809 msg = str("pattern is not of type str nor subclass of Extension (%s)" 810 " but of type %s and class %s" % (repr(Extension), 811 type(pattern), 812 pattern.__class__)) 813 raise TypeError(msg) 814 815 for file in nonempty(sorted(extended_iglob(filepattern)), "'%s' doesn't match any files" % filepattern): 816 if os.path.abspath(file) in to_exclude: 817 continue 818 module_name = deps.fully_qualified_name(file) 819 if '*' in name: 820 if module_name in explicit_modules: 821 continue 822 elif name: 823 module_name = name 824 825 Utils.raise_error_if_module_name_forbidden(module_name) 826 827 if module_name not in seen: 828 try: 829 kwds = deps.distutils_info(file, aliases, base).values 830 except Exception: 831 if exclude_failures: 832 continue 833 raise 834 if base is not None: 835 for key, value in base.values.items(): 836 if key not in kwds: 837 kwds[key] = value 838 839 kwds['name'] = module_name 840 841 sources = [file] + [m for m in template.sources if m != filepattern] 842 if 'sources' in kwds: 843 # allow users to add .c files etc. 844 for source in kwds['sources']: 845 source = encode_filename_in_py2(source) 846 if source not in sources: 847 sources.append(source) 848 kwds['sources'] = sources 849 850 if ext_language and 'language' not in kwds: 851 kwds['language'] = ext_language 852 853 np_pythran = kwds.pop('np_pythran', False) 854 855 # Create the new extension 856 m, metadata = create_extension(template, kwds) 857 m.np_pythran = np_pythran or getattr(m, 'np_pythran', False) 858 if m.np_pythran: 859 update_pythran_extension(m) 860 module_list.append(m) 861 862 # Store metadata (this will be written as JSON in the 863 # generated C file but otherwise has no purpose) 864 module_metadata[module_name] = metadata 865 866 if file not in m.sources: 867 # Old setuptools unconditionally replaces .pyx with .c/.cpp 868 target_file = os.path.splitext(file)[0] + ('.cpp' if m.language == 'c++' else '.c') 869 try: 870 m.sources.remove(target_file) 871 except ValueError: 872 # never seen this in the wild, but probably better to warn about this unexpected case 873 print("Warning: Cython source file not found in sources list, adding %s" % file) 874 m.sources.insert(0, file) 875 seen.add(name) 876 return module_list, module_metadata 877 878 879# This is the user-exposed entry point. 880def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, force=False, language=None, 881 exclude_failures=False, **options): 882 """ 883 Compile a set of source modules into C/C++ files and return a list of distutils 884 Extension objects for them. 885 886 :param module_list: As module list, pass either a glob pattern, a list of glob 887 patterns or a list of Extension objects. The latter 888 allows you to configure the extensions separately 889 through the normal distutils options. 890 You can also pass Extension objects that have 891 glob patterns as their sources. Then, cythonize 892 will resolve the pattern and create a 893 copy of the Extension for every matching file. 894 895 :param exclude: When passing glob patterns as ``module_list``, you can exclude certain 896 module names explicitly by passing them into the ``exclude`` option. 897 898 :param nthreads: The number of concurrent builds for parallel compilation 899 (requires the ``multiprocessing`` module). 900 901 :param aliases: If you want to use compiler directives like ``# distutils: ...`` but 902 can only know at compile time (when running the ``setup.py``) which values 903 to use, you can use aliases and pass a dictionary mapping those aliases 904 to Python strings when calling :func:`cythonize`. As an example, say you 905 want to use the compiler 906 directive ``# distutils: include_dirs = ../static_libs/include/`` 907 but this path isn't always fixed and you want to find it when running 908 the ``setup.py``. You can then do ``# distutils: include_dirs = MY_HEADERS``, 909 find the value of ``MY_HEADERS`` in the ``setup.py``, put it in a python 910 variable called ``foo`` as a string, and then call 911 ``cythonize(..., aliases={'MY_HEADERS': foo})``. 912 913 :param quiet: If True, Cython won't print error, warning, or status messages during the 914 compilation. 915 916 :param force: Forces the recompilation of the Cython modules, even if the timestamps 917 don't indicate that a recompilation is necessary. 918 919 :param language: To globally enable C++ mode, you can pass ``language='c++'``. Otherwise, this 920 will be determined at a per-file level based on compiler directives. This 921 affects only modules found based on file names. Extension instances passed 922 into :func:`cythonize` will not be changed. It is recommended to rather 923 use the compiler directive ``# distutils: language = c++`` than this option. 924 925 :param exclude_failures: For a broad 'try to compile' mode that ignores compilation 926 failures and simply excludes the failed extensions, 927 pass ``exclude_failures=True``. Note that this only 928 really makes sense for compiling ``.py`` files which can also 929 be used without compilation. 930 931 :param annotate: If ``True``, will produce a HTML file for each of the ``.pyx`` or ``.py`` 932 files compiled. The HTML file gives an indication 933 of how much Python interaction there is in 934 each of the source code lines, compared to plain C code. 935 It also allows you to see the C/C++ code 936 generated for each line of Cython code. This report is invaluable when 937 optimizing a function for speed, 938 and for determining when to :ref:`release the GIL <nogil>`: 939 in general, a ``nogil`` block may contain only "white" code. 940 See examples in :ref:`determining_where_to_add_types` or 941 :ref:`primes`. 942 943 :param compiler_directives: Allow to set compiler directives in the ``setup.py`` like this: 944 ``compiler_directives={'embedsignature': True}``. 945 See :ref:`compiler-directives`. 946 """ 947 if exclude is None: 948 exclude = [] 949 if 'include_path' not in options: 950 options['include_path'] = ['.'] 951 if 'common_utility_include_dir' in options: 952 safe_makedirs(options['common_utility_include_dir']) 953 954 if pythran is None: 955 pythran_options = None 956 else: 957 pythran_options = CompilationOptions(**options) 958 pythran_options.cplus = True 959 pythran_options.np_pythran = True 960 961 c_options = CompilationOptions(**options) 962 cpp_options = CompilationOptions(**options); cpp_options.cplus = True 963 ctx = c_options.create_context() 964 options = c_options 965 module_list, module_metadata = create_extension_list( 966 module_list, 967 exclude=exclude, 968 ctx=ctx, 969 quiet=quiet, 970 exclude_failures=exclude_failures, 971 language=language, 972 aliases=aliases) 973 deps = create_dependency_tree(ctx, quiet=quiet) 974 build_dir = getattr(options, 'build_dir', None) 975 976 def copy_to_build_dir(filepath, root=os.getcwd()): 977 filepath_abs = os.path.abspath(filepath) 978 if os.path.isabs(filepath): 979 filepath = filepath_abs 980 if filepath_abs.startswith(root): 981 # distutil extension depends are relative to cwd 982 mod_dir = join_path(build_dir, 983 os.path.dirname(_relpath(filepath, root))) 984 copy_once_if_newer(filepath_abs, mod_dir) 985 986 modules_by_cfile = collections.defaultdict(list) 987 to_compile = [] 988 for m in module_list: 989 if build_dir: 990 for dep in m.depends: 991 copy_to_build_dir(dep) 992 993 cy_sources = [ 994 source for source in m.sources 995 if os.path.splitext(source)[1] in ('.pyx', '.py')] 996 if len(cy_sources) == 1: 997 # normal "special" case: believe the Extension module name to allow user overrides 998 full_module_name = m.name 999 else: 1000 # infer FQMN from source files 1001 full_module_name = None 1002 1003 new_sources = [] 1004 for source in m.sources: 1005 base, ext = os.path.splitext(source) 1006 if ext in ('.pyx', '.py'): 1007 if m.np_pythran: 1008 c_file = base + '.cpp' 1009 options = pythran_options 1010 elif m.language == 'c++': 1011 c_file = base + '.cpp' 1012 options = cpp_options 1013 else: 1014 c_file = base + '.c' 1015 options = c_options 1016 1017 # setup for out of place build directory if enabled 1018 if build_dir: 1019 if os.path.isabs(c_file): 1020 warnings.warn("build_dir has no effect for absolute source paths") 1021 c_file = os.path.join(build_dir, c_file) 1022 dir = os.path.dirname(c_file) 1023 safe_makedirs_once(dir) 1024 1025 if os.path.exists(c_file): 1026 c_timestamp = os.path.getmtime(c_file) 1027 else: 1028 c_timestamp = -1 1029 1030 # Priority goes first to modified files, second to direct 1031 # dependents, and finally to indirect dependents. 1032 if c_timestamp < deps.timestamp(source): 1033 dep_timestamp, dep = deps.timestamp(source), source 1034 priority = 0 1035 else: 1036 dep_timestamp, dep = deps.newest_dependency(source) 1037 priority = 2 - (dep in deps.immediate_dependencies(source)) 1038 if force or c_timestamp < dep_timestamp: 1039 if not quiet and not force: 1040 if source == dep: 1041 print("Compiling %s because it changed." % source) 1042 else: 1043 print("Compiling %s because it depends on %s." % (source, dep)) 1044 if not force and options.cache: 1045 fingerprint = deps.transitive_fingerprint(source, m, options) 1046 else: 1047 fingerprint = None 1048 to_compile.append(( 1049 priority, source, c_file, fingerprint, quiet, 1050 options, not exclude_failures, module_metadata.get(m.name), 1051 full_module_name)) 1052 new_sources.append(c_file) 1053 modules_by_cfile[c_file].append(m) 1054 else: 1055 new_sources.append(source) 1056 if build_dir: 1057 copy_to_build_dir(source) 1058 m.sources = new_sources 1059 1060 if options.cache: 1061 if not os.path.exists(options.cache): 1062 os.makedirs(options.cache) 1063 to_compile.sort() 1064 # Drop "priority" component of "to_compile" entries and add a 1065 # simple progress indicator. 1066 N = len(to_compile) 1067 progress_fmt = "[{0:%d}/{1}] " % len(str(N)) 1068 for i in range(N): 1069 progress = progress_fmt.format(i+1, N) 1070 to_compile[i] = to_compile[i][1:] + (progress,) 1071 1072 if N <= 1: 1073 nthreads = 0 1074 if nthreads: 1075 # Requires multiprocessing (or Python >= 2.6) 1076 try: 1077 import multiprocessing 1078 pool = multiprocessing.Pool( 1079 nthreads, initializer=_init_multiprocessing_helper) 1080 except (ImportError, OSError): 1081 print("multiprocessing required for parallel cythonization") 1082 nthreads = 0 1083 else: 1084 # This is a bit more involved than it should be, because KeyboardInterrupts 1085 # break the multiprocessing workers when using a normal pool.map(). 1086 # See, for example: 1087 # http://noswap.com/blog/python-multiprocessing-keyboardinterrupt 1088 try: 1089 result = pool.map_async(cythonize_one_helper, to_compile, chunksize=1) 1090 pool.close() 1091 while not result.ready(): 1092 try: 1093 result.get(99999) # seconds 1094 except multiprocessing.TimeoutError: 1095 pass 1096 except KeyboardInterrupt: 1097 pool.terminate() 1098 raise 1099 pool.join() 1100 if not nthreads: 1101 for args in to_compile: 1102 cythonize_one(*args) 1103 1104 if exclude_failures: 1105 failed_modules = set() 1106 for c_file, modules in modules_by_cfile.items(): 1107 if not os.path.exists(c_file): 1108 failed_modules.update(modules) 1109 elif os.path.getsize(c_file) < 200: 1110 f = io_open(c_file, 'r', encoding='iso8859-1') 1111 try: 1112 if f.read(len('#error ')) == '#error ': 1113 # dead compilation result 1114 failed_modules.update(modules) 1115 finally: 1116 f.close() 1117 if failed_modules: 1118 for module in failed_modules: 1119 module_list.remove(module) 1120 print("Failed compilations: %s" % ', '.join(sorted([ 1121 module.name for module in failed_modules]))) 1122 1123 if options.cache: 1124 cleanup_cache(options.cache, getattr(options, 'cache_size', 1024 * 1024 * 100)) 1125 # cythonize() is often followed by the (non-Python-buffered) 1126 # compiler output, flush now to avoid interleaving output. 1127 sys.stdout.flush() 1128 return module_list 1129 1130 1131if os.environ.get('XML_RESULTS'): 1132 compile_result_dir = os.environ['XML_RESULTS'] 1133 def record_results(func): 1134 def with_record(*args): 1135 t = time.time() 1136 success = True 1137 try: 1138 try: 1139 func(*args) 1140 except: 1141 success = False 1142 finally: 1143 t = time.time() - t 1144 module = fully_qualified_name(args[0]) 1145 name = "cythonize." + module 1146 failures = 1 - success 1147 if success: 1148 failure_item = "" 1149 else: 1150 failure_item = "failure" 1151 output = open(os.path.join(compile_result_dir, name + ".xml"), "w") 1152 output.write(""" 1153 <?xml version="1.0" ?> 1154 <testsuite name="%(name)s" errors="0" failures="%(failures)s" tests="1" time="%(t)s"> 1155 <testcase classname="%(name)s" name="cythonize"> 1156 %(failure_item)s 1157 </testcase> 1158 </testsuite> 1159 """.strip() % locals()) 1160 output.close() 1161 return with_record 1162else: 1163 def record_results(func): 1164 return func 1165 1166 1167# TODO: Share context? Issue: pyx processing leaks into pxd module 1168@record_results 1169def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, 1170 raise_on_failure=True, embedded_metadata=None, full_module_name=None, 1171 progress=""): 1172 from ..Compiler.Main import compile_single, default_options 1173 from ..Compiler.Errors import CompileError, PyrexError 1174 1175 if fingerprint: 1176 if not os.path.exists(options.cache): 1177 safe_makedirs(options.cache) 1178 # Cython-generated c files are highly compressible. 1179 # (E.g. a compression ratio of about 10 for Sage). 1180 fingerprint_file_base = join_path( 1181 options.cache, "%s-%s" % (os.path.basename(c_file), fingerprint)) 1182 gz_fingerprint_file = fingerprint_file_base + gzip_ext 1183 zip_fingerprint_file = fingerprint_file_base + '.zip' 1184 if os.path.exists(gz_fingerprint_file) or os.path.exists(zip_fingerprint_file): 1185 if not quiet: 1186 print("%sFound compiled %s in cache" % (progress, pyx_file)) 1187 if os.path.exists(gz_fingerprint_file): 1188 os.utime(gz_fingerprint_file, None) 1189 with contextlib.closing(gzip_open(gz_fingerprint_file, 'rb')) as g: 1190 with contextlib.closing(open(c_file, 'wb')) as f: 1191 shutil.copyfileobj(g, f) 1192 else: 1193 os.utime(zip_fingerprint_file, None) 1194 dirname = os.path.dirname(c_file) 1195 with contextlib.closing(zipfile.ZipFile(zip_fingerprint_file)) as z: 1196 for artifact in z.namelist(): 1197 z.extract(artifact, os.path.join(dirname, artifact)) 1198 return 1199 if not quiet: 1200 print("%sCythonizing %s" % (progress, pyx_file)) 1201 if options is None: 1202 options = CompilationOptions(default_options) 1203 options.output_file = c_file 1204 options.embedded_metadata = embedded_metadata 1205 1206 any_failures = 0 1207 try: 1208 result = compile_single(pyx_file, options, full_module_name=full_module_name) 1209 if result.num_errors > 0: 1210 any_failures = 1 1211 except (EnvironmentError, PyrexError) as e: 1212 sys.stderr.write('%s\n' % e) 1213 any_failures = 1 1214 # XXX 1215 import traceback 1216 traceback.print_exc() 1217 except Exception: 1218 if raise_on_failure: 1219 raise 1220 import traceback 1221 traceback.print_exc() 1222 any_failures = 1 1223 if any_failures: 1224 if raise_on_failure: 1225 raise CompileError(None, pyx_file) 1226 elif os.path.exists(c_file): 1227 os.remove(c_file) 1228 elif fingerprint: 1229 artifacts = list(filter(None, [ 1230 getattr(result, attr, None) 1231 for attr in ('c_file', 'h_file', 'api_file', 'i_file')])) 1232 if len(artifacts) == 1: 1233 fingerprint_file = gz_fingerprint_file 1234 with contextlib.closing(open(c_file, 'rb')) as f: 1235 with contextlib.closing(gzip_open(fingerprint_file + '.tmp', 'wb')) as g: 1236 shutil.copyfileobj(f, g) 1237 else: 1238 fingerprint_file = zip_fingerprint_file 1239 with contextlib.closing(zipfile.ZipFile( 1240 fingerprint_file + '.tmp', 'w', zipfile_compression_mode)) as zip: 1241 for artifact in artifacts: 1242 zip.write(artifact, os.path.basename(artifact)) 1243 os.rename(fingerprint_file + '.tmp', fingerprint_file) 1244 1245 1246def cythonize_one_helper(m): 1247 import traceback 1248 try: 1249 return cythonize_one(*m) 1250 except Exception: 1251 traceback.print_exc() 1252 raise 1253 1254 1255def _init_multiprocessing_helper(): 1256 # KeyboardInterrupt kills workers, so don't let them get it 1257 import signal 1258 signal.signal(signal.SIGINT, signal.SIG_IGN) 1259 1260 1261def cleanup_cache(cache, target_size, ratio=.85): 1262 try: 1263 p = subprocess.Popen(['du', '-s', '-k', os.path.abspath(cache)], stdout=subprocess.PIPE) 1264 res = p.wait() 1265 if res == 0: 1266 total_size = 1024 * int(p.stdout.read().strip().split()[0]) 1267 if total_size < target_size: 1268 return 1269 except (OSError, ValueError): 1270 pass 1271 total_size = 0 1272 all = [] 1273 for file in os.listdir(cache): 1274 path = join_path(cache, file) 1275 s = os.stat(path) 1276 total_size += s.st_size 1277 all.append((s.st_atime, s.st_size, path)) 1278 if total_size > target_size: 1279 for time, size, file in reversed(sorted(all)): 1280 os.unlink(file) 1281 total_size -= size 1282 if total_size < target_size * ratio: 1283 break 1284