1""" 2Generate and compile C modules for Python. 3 4""" 5from __future__ import absolute_import, print_function, division 6 7import atexit 8import textwrap 9import six.moves.cPickle as pickle 10import logging 11import os 12import re 13import shutil 14import stat 15import subprocess 16import sys 17import tempfile 18import time 19import platform 20import distutils.sysconfig 21import warnings 22 23import numpy.distutils 24 25import theano 26from theano.compat import PY3, decode, decode_iter 27from six import b, BytesIO, StringIO, string_types, iteritems 28from six.moves import xrange 29from theano.gof.utils import flatten 30from theano import config 31from theano.gof.utils import hash_from_code 32from theano.misc.windows import (subprocess_Popen, 33 output_subprocess_Popen) 34 35# we will abuse the lockfile mechanism when reading and writing the registry 36from theano.gof import compilelock 37from theano.configdefaults import gcc_version_str, local_bitwidth 38 39importlib = None 40try: 41 import importlib 42except ImportError: 43 pass 44 45_logger = logging.getLogger("theano.gof.cmodule") 46 47METH_VARARGS = "METH_VARARGS" 48METH_NOARGS = "METH_NOARGS" 49# global variable that represent the total time spent in importing module. 50import_time = 0 51 52 53class MissingGXX(Exception): 54 """ 55 This error is raised when we try to generate c code, 56 but g++ is not available. 57 58 """ 59 60 pass 61 62 63def debug_counter(name, every=1): 64 """ 65 Debug counter to know how often we go through some piece of code. 66 67 This is a utility function one may use when debugging. 68 69 Example 70 ------- 71 debug_counter('I want to know how often I run this line') 72 73 """ 74 setattr(debug_counter, name, getattr(debug_counter, name, 0) + 1) 75 n = getattr(debug_counter, name) 76 if n % every == 0: 77 print("debug_counter [%s]: %s" % (name, n), file=sys.stderr) 78 79 80class ExtFunction(object): 81 """ 82 A C function to put into a DynamicModule. 83 84 """ 85 86 name = "" 87 """ 88 str - function's name. 89 90 """ 91 code_block = "" 92 """ 93 str - the entire code for the function. 94 95 Has the form ``static PyObject* <name>([...]){ ... } 96 97 See Python's C API Reference for how to write c functions for python 98 modules. 99 100 """ 101 method = "" 102 """ 103 str - calling method for this function (i.e. 'METH_VARARGS', 'METH_NOARGS'). 104 105 """ 106 doc = "" 107 """ 108 str - documentation string for this function. 109 110 """ 111 112 def __init__(self, name, code_block, method, doc="undocumented"): 113 self.name = name 114 self.code_block = code_block 115 self.method = method 116 self.doc = doc 117 118 def method_decl(self): 119 """ 120 Returns the signature for this function. 121 122 It goes into the DynamicModule's method table. 123 124 """ 125 return '\t{"%s", %s, %s, "%s"}' % ( 126 self.name, self.name, self.method, self.doc) 127 128 129class DynamicModule(object): 130 131 def __init__(self, name=None): 132 assert name is None, ( 133 "The 'name' parameter of DynamicModule" 134 " cannot be specified anymore. Instead, 'code_hash'" 135 " will be automatically computed and can be used as" 136 " the module's name.") 137 # While the module is not finalized, we can call add_... 138 # when it is finalized, a hash is computed and used instead of 139 # the placeholder, and as module name. 140 self.finalized = False 141 self.code_hash = None 142 self.hash_placeholder = '<<<<HASH_PLACEHOLDER>>>>' 143 144 self.support_code = [] 145 self.functions = [] 146 self.includes = ["<Python.h>", "<iostream>", '"theano_mod_helper.h"'] 147 self.init_blocks = [] 148 149 def print_methoddef(self, stream): 150 print("static PyMethodDef MyMethods[] = {", file=stream) 151 for f in self.functions: 152 print(f.method_decl(), ',', file=stream) 153 print("\t{NULL, NULL, 0, NULL}", file=stream) 154 print("};", file=stream) 155 156 def print_init(self, stream): 157 if PY3: 158 print("""\ 159static struct PyModuleDef moduledef = {{ 160 PyModuleDef_HEAD_INIT, 161 "{name}", 162 NULL, 163 -1, 164 MyMethods, 165}}; 166""".format(name=self.hash_placeholder), file=stream) 167 print(("PyMODINIT_FUNC PyInit_%s(void) {" % 168 self.hash_placeholder), file=stream) 169 for block in self.init_blocks: 170 print(' ', block, file=stream) 171 print(" PyObject *m = PyModule_Create(&moduledef);", file=stream) 172 print(" return m;", file=stream) 173 else: 174 print(("PyMODINIT_FUNC init%s(void){" % 175 self.hash_placeholder), file=stream) 176 for block in self.init_blocks: 177 print(' ', block, file=stream) 178 print(' ', ('(void) Py_InitModule("%s", MyMethods);' 179 % self.hash_placeholder), file=stream) 180 print("}", file=stream) 181 182 def add_include(self, str): 183 assert not self.finalized 184 self.includes.append(str) 185 186 def add_init_code(self, code): 187 assert not self.finalized 188 self.init_blocks.append(code) 189 190 def add_support_code(self, code): 191 assert not self.finalized 192 if code and code not in self.support_code: # TODO: KLUDGE 193 self.support_code.append(code) 194 195 def add_function(self, fn): 196 assert not self.finalized 197 self.functions.append(fn) 198 199 def code(self): 200 sio = StringIO() 201 for inc in self.includes: 202 if not inc: 203 continue 204 if inc[0] == '<' or inc[0] == '"': 205 print("#include", inc, file=sio) 206 else: 207 print('#include "%s"' % inc, file=sio) 208 209 print("//////////////////////", file=sio) 210 print("//// Support Code", file=sio) 211 print("//////////////////////", file=sio) 212 for sc in self.support_code: 213 print(sc, file=sio) 214 215 print("//////////////////////", file=sio) 216 print("//// Functions", file=sio) 217 print("//////////////////////", file=sio) 218 for f in self.functions: 219 print(f.code_block, file=sio) 220 221 print("//////////////////////", file=sio) 222 print("//// Module init", file=sio) 223 print("//////////////////////", file=sio) 224 self.print_methoddef(sio) 225 self.print_init(sio) 226 227 rval = sio.getvalue() 228 # Make sure the hash of the code hasn't changed 229 h = hash_from_code(rval) 230 assert self.code_hash is None or self.code_hash == h 231 self.code_hash = h 232 rval = re.sub(self.hash_placeholder, self.code_hash, rval) 233 # Finalize the Module, so no support code or function 234 # can be added 235 self.finalized = True 236 237 return rval 238 239 def list_code(self, ofile=sys.stdout): 240 """ 241 Print out the code with line numbers to `ofile`. 242 243 """ 244 for i, line in enumerate(self.code().split('\n')): 245 print(('%4i' % (i + 1)), line, file=ofile) 246 ofile.flush() 247 248 # TODO: add_type 249 250 251def _get_ext_suffix(): 252 """Get the suffix for compiled extensions""" 253 dist_suffix = distutils.sysconfig.get_config_var("EXT_SUFFIX") 254 if dist_suffix is None: 255 dist_suffix = distutils.sysconfig.get_config_var("SO") 256 return dist_suffix 257 258 259def dlimport(fullpath, suffix=None): 260 """ 261 Dynamically load a .so, .pyd, .dll, or .py file. 262 263 Parameters 264 ---------- 265 fullpath : str 266 A fully-qualified path do a compiled python module. 267 suffix : str 268 A suffix to strip from the end of fullpath to get the 269 import name. 270 271 Returns 272 ------- 273 object 274 The dynamically loaded module (from __import__). 275 276 """ 277 if not os.path.isabs(fullpath): 278 raise ValueError('`fullpath` must be an absolute path', fullpath) 279 if suffix is None: 280 suffix = '' 281 282 dist_suffix = _get_ext_suffix() 283 if dist_suffix is not None and dist_suffix != '': 284 if fullpath.endswith(dist_suffix): 285 suffix = dist_suffix 286 287 if suffix == '': 288 if fullpath.endswith('.so'): 289 suffix = '.so' 290 elif fullpath.endswith('.pyd'): 291 suffix = '.pyd' 292 elif fullpath.endswith('.dll'): 293 suffix = '.dll' 294 elif fullpath.endswith('.py'): 295 suffix = '.py' 296 297 rval = None 298 if fullpath.endswith(suffix): 299 module_name = '.'.join(fullpath.split(os.path.sep)[-2:])[:-len(suffix)] 300 else: 301 raise ValueError('path has wrong suffix', (fullpath, suffix)) 302 workdir = fullpath[:-len(module_name) - 1 - len(suffix)] 303 304 _logger.debug("WORKDIR %s", workdir) 305 _logger.debug("module_name %s", module_name) 306 307 sys.path[0:0] = [workdir] # insert workdir at beginning (temporarily) 308 global import_time 309 try: 310 if importlib is not None: 311 if hasattr(importlib, "invalidate_caches"): 312 importlib.invalidate_caches() 313 t0 = time.time() 314 with warnings.catch_warnings(): 315 warnings.filterwarnings("ignore", 316 message="numpy.ndarray size changed") 317 rval = __import__(module_name, {}, {}, [module_name]) 318 t1 = time.time() 319 import_time += t1 - t0 320 if not rval: 321 raise Exception('__import__ failed', fullpath) 322 finally: 323 del sys.path[0] 324 325 assert fullpath.startswith(rval.__file__) 326 return rval 327 328 329def dlimport_workdir(basedir): 330 """ 331 Return a directory where you should put your .so file for dlimport 332 to be able to load it, given a basedir which should normally be 333 config.compiledir. 334 335 """ 336 return tempfile.mkdtemp(dir=basedir) 337 338 339def last_access_time(path): 340 """ 341 Return the number of seconds since the epoch of the last access of a 342 given file. 343 344 """ 345 return os.stat(path)[stat.ST_ATIME] 346 347 348def module_name_from_dir(dirname, err=True, files=None): 349 """ 350 Scan the contents of a cache directory and return full path of the 351 dynamic lib in it. 352 353 """ 354 if files is None: 355 try: 356 files = os.listdir(dirname) 357 except OSError as e: 358 if e.errno == 2 and not err: # No such file or directory 359 return None 360 names = [file for file in files 361 if file.endswith('.so') or file.endswith('.pyd')] 362 if len(names) == 0 and not err: 363 return None 364 elif len(names) == 1: 365 return os.path.join(dirname, names[0]) 366 else: 367 raise ValueError("More than 1 compiled module in this directory:" + 368 dirname) 369 370 371def is_same_entry(entry_1, entry_2): 372 """ 373 Return True iff both paths can be considered to point to the same module. 374 375 This is the case if and only if at least one of these conditions holds: 376 - They are equal. 377 - Their real paths are equal. 378 - They share the same temporary work directory and module file name. 379 380 """ 381 if entry_1 == entry_2: 382 return True 383 if os.path.realpath(entry_1) == os.path.realpath(entry_2): 384 return True 385 if (os.path.basename(entry_1) == os.path.basename(entry_2) and 386 (os.path.basename(os.path.dirname(entry_1)) == 387 os.path.basename(os.path.dirname(entry_2))) and 388 os.path.basename(os.path.dirname(entry_1)).startswith('tmp')): 389 return True 390 return False 391 392 393def get_module_hash(src_code, key): 394 """ 395 Return a SHA256 hash that uniquely identifies a module. 396 397 This hash takes into account: 398 1. The C source code of the module (`src_code`). 399 2. The version part of the key. 400 3. The compiler options defined in `key` (command line parameters and 401 libraries to link against). 402 4. The NumPy ABI version. 403 404 """ 405 # `to_hash` will contain any element such that we know for sure that if 406 # it changes, then the module hash should be different. 407 # We start with the source code itself (stripping blanks might avoid 408 # recompiling after a basic indentation fix for instance). 409 to_hash = [l.strip() for l in src_code.split('\n')] 410 # Get the version part of the key (ignore if unversioned). 411 if key[0]: 412 to_hash += list(map(str, key[0])) 413 c_link_key = key[1] 414 # Currently, in order to catch potential bugs early, we are very 415 # convervative about the structure of the key and raise an exception 416 # if it does not match exactly what we expect. In the future we may 417 # modify this behavior to be less strict and be able to accommodate 418 # changes to the key in an automatic way. 419 # Note that if the key structure changes, the `get_safe_part` function 420 # below may also need to be modified. 421 error_msg = ("This should not happen unless someone modified the code " 422 "that defines the CLinker key, in which case you should " 423 "ensure this piece of code is still valid (and this " 424 "AssertionError may be removed or modified to accommodate " 425 "this change)") 426 assert c_link_key[0] == 'CLinker.cmodule_key', error_msg 427 for key_element in c_link_key[1:]: 428 if isinstance(key_element, tuple): 429 # This should be the C++ compilation command line parameters or the 430 # libraries to link against. 431 to_hash += list(key_element) 432 elif isinstance(key_element, string_types): 433 if (key_element.startswith('md5:') or 434 key_element.startswith('hash:')): 435 # This is actually a sha256 hash of the config options. 436 # Currently, we still keep md5 to don't break old Theano. 437 # We add 'hash:' so that when we change it in 438 # the futur, it won't break this version of Theano. 439 break 440 elif (key_element.startswith('NPY_ABI_VERSION=0x') or 441 key_element.startswith('c_compiler_str=')): 442 to_hash.append(key_element) 443 else: 444 raise AssertionError(error_msg) 445 else: 446 raise AssertionError(error_msg) 447 return hash_from_code('\n'.join(to_hash)) 448 449 450def get_safe_part(key): 451 """ 452 Return a tuple containing a subset of `key`, to be used to find equal keys. 453 454 This tuple should only contain objects whose __eq__ and __hash__ methods 455 can be trusted (currently: the version part of the key, as well as the 456 SHA256 hash of the config options). 457 It is used to reduce the amount of key comparisons one has to go through 458 in order to find broken keys (i.e. keys with bad implementations of __eq__ 459 or __hash__). 460 461 462 """ 463 version = key[0] 464 # This function should only be called on versioned keys. 465 assert version 466 467 # Find the hash part. This is actually a sha256 hash of the config 468 # options. Currently, we still keep md5 to don't break old 469 # Theano. We add 'hash:' so that when we change it 470 # in the futur, it won't break this version of Theano. 471 c_link_key = key[1] 472 # In case in the future, we don't have an md5 part and we have 473 # such stuff in the cache. In that case, we can set None, and the 474 # rest of the cache mechanism will just skip that key. 475 hash = None 476 for key_element in c_link_key[1:]: 477 if isinstance(key_element, string_types): 478 if key_element.startswith('md5:'): 479 hash = key_element[4:] 480 break 481 elif key_element.startswith('hash:'): 482 hash = key_element[5:] 483 break 484 485 return key[0] + (hash, ) 486 487 488class KeyData(object): 489 """ 490 Used to store the key information in the cache. 491 492 Parameters 493 ---------- 494 keys 495 Set of keys that are associated to the exact same module. 496 module_hash 497 Hash identifying the module (it should hash both the code and the 498 compilation options). 499 key_pkl 500 Path to the file in which this KeyData object should be 501 pickled. 502 503 """ 504 505 def __init__(self, keys, module_hash, key_pkl, entry): 506 self.keys = keys 507 self.module_hash = module_hash 508 self.key_pkl = key_pkl 509 self.entry = entry 510 511 def add_key(self, key, save_pkl=True): 512 """ 513 Add a key to self.keys, and update pickled file if asked to. 514 515 """ 516 assert key not in self.keys 517 self.keys.add(key) 518 if save_pkl: 519 self.save_pkl() 520 521 def remove_key(self, key, save_pkl=True): 522 """ 523 Remove a key from self.keys, and update pickled file if asked to. 524 525 """ 526 self.keys.remove(key) 527 if save_pkl: 528 self.save_pkl() 529 530 def save_pkl(self): 531 """ 532 Dump this object into its `key_pkl` file. 533 534 May raise a cPickle.PicklingError if such an exception is raised at 535 pickle time (in which case a warning is also displayed). 536 537 """ 538 # Note that writing in binary mode is important under Windows. 539 try: 540 with open(self.key_pkl, 'wb') as f: 541 pickle.dump(self, f, protocol=pickle.HIGHEST_PROTOCOL) 542 except pickle.PicklingError: 543 _logger.warning("Cache leak due to unpickle-able key data %s", 544 self.keys) 545 os.remove(self.key_pkl) 546 raise 547 548 def get_entry(self): 549 """ 550 Return path to the module file. 551 552 """ 553 # TODO This method may be removed in the future (e.g. in 0.5) since 554 # its only purpose is to make sure that old KeyData objects created 555 # before the 'entry' field was added are properly handled. 556 if not hasattr(self, 'entry'): 557 self.entry = module_name_from_dir(os.path.dirname(self.key_pkl)) 558 return self.entry 559 560 def delete_keys_from(self, entry_from_key, do_manual_check=True): 561 """ 562 Delete from entry_from_key all keys associated to this KeyData object. 563 564 Note that broken keys will not appear in the keys field, so we also 565 manually look for keys associated to the same entry, unless 566 do_manual_check is False. 567 568 """ 569 entry = self.get_entry() 570 for key in self.keys: 571 try: 572 del entry_from_key[key] 573 except KeyError: 574 # This happen if the compiledir was deleted during 575 # this process execution. 576 pass 577 if do_manual_check: 578 to_del = [] 579 for key, key_entry in iteritems(entry_from_key): 580 if key_entry == entry: 581 to_del.append(key) 582 for key in to_del: 583 try: 584 del entry_from_key[key] 585 except KeyError: 586 # This happen if the compiledir was deleted during 587 # this process execution. 588 pass 589 590 591class ModuleCache(object): 592 """ 593 Interface to the cache of dynamically compiled modules on disk. 594 595 Note that this interface does not assume exclusive use of the cache 596 directory. It is built to handle the case where multiple programs are also 597 using instances of this class to manage the same directory. 598 599 The cache works on the basis of keys. Each key is mapped to only one 600 dynamic module, but multiple keys may be mapped to the same module (see 601 below for details). Each module is a dynamic library file, that Python 602 can import. 603 604 The cache contains one directory for each module, containing: 605 - the dynamic library file itself (.so/.pyd), 606 - an empty __init__.py file, so Python can import it, 607 - a file containing the source code for the module (mod.cpp/mod.cu), 608 - a key.pkl file, containing a KeyData object with all the keys 609 associated with that module, 610 - possibly a delete.me file, meaning this directory has been marked 611 for deletion. 612 613 Keys should be tuples of length 2: (version, rest). The 614 ``rest`` can be anything hashable and picklable, that uniquely 615 identifies the computation in the module. The key is returned by 616 ``CLinker.cmodule_key_``. 617 618 The ``version`` should be a hierarchy of tuples of integers. 619 If the ``version`` is either 0 or (), then the key is unversioned, and its 620 corresponding module will be marked for deletion in an atexit() handler. 621 If the ``version`` is neither 0 nor (), then the module will be kept in the 622 cache between processes. 623 624 An unversioned module is not always deleted by the process that 625 creates it. Deleting such modules may not work on NFS filesystems 626 because the tmpdir in which the library resides is in use until the 627 end of the process' lifetime. In this case, unversioned modules 628 are left in their tmpdirs without corresponding .pkl files. These 629 modules and their directories are erased by subsequent processes' 630 refresh() functions. 631 632 Two different keys are mapped to the same module when all conditions below 633 are met: 634 - They have the same version. 635 - They share the same compilation options in their ``rest`` part (see 636 ``CLinker.cmodule_key_`` for how this part is built). 637 - They share the same C code. 638 These three elements uniquely identify a module, and are summarized 639 in a single "module hash". 640 641 Parameters 642 ---------- 643 check_for_broken_eq 644 A bad __eq__ implementation can break this cache mechanism. 645 This option turns on a not-too-expensive sanity check every 646 time a new key is added to the cache. 647 648 do_refresh : bool 649 If True, then the ``refresh`` method will be called 650 in the constructor. 651 652 """ 653 654 dirname = "" 655 """ 656 The working directory that is managed by this interface. 657 658 """ 659 module_from_name = {} 660 """ 661 Maps a module filename to the loaded module object. 662 663 """ 664 entry_from_key = {} 665 """ 666 Maps keys to the filename of a .so/.pyd. 667 668 """ 669 similar_keys = {} 670 """ 671 Maps a part-of-key to all keys that share this same part. 672 673 """ 674 module_hash_to_key_data = {} 675 """ 676 Maps a module hash to its corresponding KeyData object. 677 678 """ 679 stats = [] 680 """ 681 A list with counters for the number of hits, loads, compiles issued by 682 module_from_key(). 683 684 """ 685 loaded_key_pkl = set() 686 """ 687 Set of all key.pkl files that have been loaded. 688 689 """ 690 691 def __init__(self, dirname, check_for_broken_eq=True, do_refresh=True): 692 self.dirname = dirname 693 self.module_from_name = dict(self.module_from_name) 694 self.entry_from_key = dict(self.entry_from_key) 695 self.module_hash_to_key_data = dict(self.module_hash_to_key_data) 696 self.similar_keys = dict(self.similar_keys) 697 self.stats = [0, 0, 0] 698 self.check_for_broken_eq = check_for_broken_eq 699 self.loaded_key_pkl = set() 700 self.time_spent_in_check_key = 0 701 702 if do_refresh: 703 self.refresh() 704 705 age_thresh_use = config.cmodule.age_thresh_use # default 24 days 706 """ 707 The default age threshold (in seconds) for cache files we want to use. 708 709 Older modules will be deleted in ``clear_old``. 710 711 """ 712 713 def _get_module(self, name): 714 """ 715 Fetch a compiled module from the loaded cache or the disk. 716 717 """ 718 if name not in self.module_from_name: 719 _logger.debug('loading name %s', name) 720 self.module_from_name[name] = dlimport(name) 721 self.stats[1] += 1 722 else: 723 _logger.debug('returning compiled module from cache %s', name) 724 self.stats[0] += 1 725 return self.module_from_name[name] 726 727 def refresh(self, age_thresh_use=None, delete_if_problem=False, 728 cleanup=True): 729 """ 730 Update cache data by walking the cache directory structure. 731 732 Load key.pkl files that have not been loaded yet. 733 Remove entries which have been removed from the filesystem. 734 Also, remove malformed cache directories. 735 736 Parameters 737 ---------- 738 age_thresh_use 739 Do not use modules other than this. Defaults to self.age_thresh_use. 740 delete_if_problem : bool 741 If True, cache entries that meet one of those two conditions are 742 deleted: 743 - Those for which unpickling the KeyData file fails with 744 an unknown exception. 745 - Duplicated modules, regardless of their age. 746 cleanup : bool 747 Do a cleanup of the cache removing expired and broken modules. 748 749 Returns 750 ------- 751 list 752 A list of modules of age higher than age_thresh_use. 753 754 """ 755 if age_thresh_use is None: 756 age_thresh_use = self.age_thresh_use 757 start_time = time.time() 758 too_old_to_use = [] 759 760 to_delete = [] 761 to_delete_empty = [] 762 763 def rmtree(*args, **kwargs): 764 if cleanup: 765 to_delete.append((args, kwargs)) 766 767 def rmtree_empty(*args, **kwargs): 768 if cleanup: 769 to_delete_empty.append((args, kwargs)) 770 771 # add entries that are not in the entry_from_key dictionary 772 time_now = time.time() 773 # Go through directories in alphabetical order to ensure consistent 774 # behavior. 775 try: 776 subdirs = sorted(os.listdir(self.dirname)) 777 except OSError: 778 # This can happen if the dir don't exist. 779 subdirs = [] 780 files, root = None, None # To make sure the "del" below works 781 for subdirs_elem in subdirs: 782 # Never clean/remove lock_dir 783 if subdirs_elem == 'lock_dir': 784 continue 785 root = os.path.join(self.dirname, subdirs_elem) 786 # Don't delete the gpuarray kernel cache 787 if root == config.gpuarray.cache_path: 788 continue 789 key_pkl = os.path.join(root, 'key.pkl') 790 if key_pkl in self.loaded_key_pkl: 791 continue 792 if not os.path.isdir(root): 793 continue 794 files = os.listdir(root) 795 if not files: 796 rmtree_empty(root, ignore_nocleanup=True, 797 msg="empty dir") 798 continue 799 if 'delete.me' in files: 800 rmtree(root, ignore_nocleanup=True, 801 msg="delete.me found in dir") 802 continue 803 elif 'key.pkl' in files: 804 try: 805 entry = module_name_from_dir(root, files=files) 806 except ValueError: # there is a key but no dll! 807 if not root.startswith("/tmp"): 808 # Under /tmp, file are removed periodically by the 809 # os. So it is normal that this happens from time 810 # to time. 811 _logger.warning("ModuleCache.refresh() Found key " 812 "without dll in cache, deleting it. %s", 813 key_pkl) 814 rmtree(root, ignore_nocleanup=True, 815 msg="missing module file", level=logging.INFO) 816 continue 817 if (time_now - last_access_time(entry)) < age_thresh_use: 818 _logger.debug('refresh adding %s', key_pkl) 819 820 def unpickle_failure(): 821 _logger.info("ModuleCache.refresh() Failed to " 822 "unpickle cache file %s", key_pkl) 823 824 try: 825 with open(key_pkl, 'rb') as f: 826 key_data = pickle.load(f) 827 except EOFError: 828 # Happened once... not sure why (would be worth 829 # investigating if it ever happens again). 830 unpickle_failure() 831 rmtree(root, ignore_nocleanup=True, 832 msg='broken cache directory [EOF]', 833 level=logging.WARNING) 834 continue 835 except Exception: 836 unpickle_failure() 837 if delete_if_problem: 838 rmtree(root, ignore_nocleanup=True, 839 msg='broken cache directory', 840 level=logging.INFO) 841 else: 842 # This exception is often triggered by keys 843 # that contain references to classes that have 844 # not yet been imported (e.g. when running two 845 # different Theano-based scripts). They are not 846 # necessarily broken, but we cannot load them 847 # now. They will be loaded later if needed. 848 pass 849 continue 850 851 if not isinstance(key_data, KeyData): 852 # This is some old cache data, that does not fit 853 # the new cache format. It would be possible to 854 # update it, but it is not entirely safe since we 855 # do not know the config options that were used. 856 # As a result, we delete it instead (which is also 857 # simpler to implement). 858 rmtree(root, ignore_nocleanup=True, 859 msg=( 860 'invalid cache entry format -- this ' 861 'should not happen unless your cache ' 862 'was really old'), 863 level=logging.WARN) 864 continue 865 866 # Check the path to the module stored in the KeyData 867 # object matches the path to `entry`. There may be 868 # a mismatch e.g. due to symlinks, or some directory 869 # being renamed since last time cache was created. 870 kd_entry = key_data.get_entry() 871 if kd_entry != entry: 872 if is_same_entry(entry, kd_entry): 873 # Update KeyData object. Note that we also need 874 # to update the key_pkl field, because it is 875 # likely to be incorrect if the entry itself 876 # was wrong. 877 key_data.entry = entry 878 key_data.key_pkl = key_pkl 879 else: 880 # This is suspicious. Better get rid of it. 881 rmtree(root, ignore_nocleanup=True, 882 msg='module file path mismatch', 883 level=logging.INFO) 884 continue 885 886 # Find unversioned keys from other processes. 887 # TODO: check if this can happen at all 888 to_del = [key for key in key_data.keys if not key[0]] 889 if to_del: 890 _logger.warning( 891 "ModuleCache.refresh() Found unversioned " 892 "key in cache, removing it. %s", key_pkl) 893 # Since the version is in the module hash, all 894 # keys should be unversioned. 895 if len(to_del) != len(key_data.keys): 896 _logger.warning( 897 'Found a mix of unversioned and ' 898 'versioned keys for the same ' 899 'module %s', key_pkl) 900 rmtree(root, ignore_nocleanup=True, 901 msg="unversioned key(s) in cache", 902 level=logging.INFO) 903 continue 904 905 mod_hash = key_data.module_hash 906 if mod_hash in self.module_hash_to_key_data: 907 # This may happen when two processes running 908 # simultaneously compiled the same module, one 909 # after the other. We delete one once it is old 910 # enough (to be confident there is no other process 911 # using it), or if `delete_if_problem` is True. 912 # Note that it is important to walk through 913 # directories in alphabetical order so as to make 914 # sure all new processes only use the first one. 915 if cleanup: 916 age = time.time() - last_access_time(entry) 917 if delete_if_problem or age > self.age_thresh_del: 918 rmtree(root, ignore_nocleanup=True, 919 msg='duplicated module', 920 level=logging.DEBUG) 921 else: 922 _logger.debug('Found duplicated module not ' 923 'old enough yet to be deleted ' 924 '(age: %s): %s', 925 age, entry) 926 continue 927 928 # Remember the map from a module's hash to the KeyData 929 # object associated with it. 930 self.module_hash_to_key_data[mod_hash] = key_data 931 932 for key in key_data.keys: 933 if key not in self.entry_from_key: 934 self.entry_from_key[key] = entry 935 # Assert that we have not already got this 936 # entry somehow. 937 assert entry not in self.module_from_name 938 # Store safe part of versioned keys. 939 if key[0]: 940 self.similar_keys.setdefault( 941 get_safe_part(key), 942 []).append(key) 943 else: 944 dir1 = os.path.dirname(self.entry_from_key[key]) 945 dir2 = os.path.dirname(entry) 946 _logger.warning( 947 "The same cache key is associated to " 948 "different modules (%s and %s). This " 949 "is not supposed to happen! You may " 950 "need to manually delete your cache " 951 "directory to fix this.", 952 dir1, dir2) 953 # Clean up the name space to prevent bug. 954 if key_data.keys: 955 del key 956 self.loaded_key_pkl.add(key_pkl) 957 else: 958 too_old_to_use.append(entry) 959 960 # If the compilation failed, no key.pkl is in that 961 # directory, but a mod.* should be there. 962 # We do nothing here. 963 964 # Clean up the name space to prevent bug. 965 del root, files, subdirs 966 967 # Remove entries that are not in the filesystem. 968 items_copy = list(self.module_hash_to_key_data.items()) 969 for module_hash, key_data in items_copy: 970 entry = key_data.get_entry() 971 try: 972 # Test to see that the file is [present and] readable. 973 open(entry).close() 974 gone = False 975 except IOError: 976 gone = True 977 978 if gone: 979 # Assert that we did not have one of the deleted files 980 # loaded up and in use. 981 # If so, it should not have been deleted. This should be 982 # considered a failure of the OTHER process, that deleted 983 # it. 984 if entry in self.module_from_name: 985 _logger.warning("A module that was loaded by this " 986 "ModuleCache can no longer be read from file " 987 "%s... this could lead to problems.", 988 entry) 989 del self.module_from_name[entry] 990 991 _logger.info("deleting ModuleCache entry %s", entry) 992 key_data.delete_keys_from(self.entry_from_key) 993 del self.module_hash_to_key_data[module_hash] 994 if key_data.keys and list(key_data.keys)[0][0]: 995 # this is a versioned entry, so should have been on 996 # disk. Something weird happened to cause this, so we 997 # are responding by printing a warning, removing 998 # evidence that we ever saw this mystery key. 999 pkl_file_to_remove = key_data.key_pkl 1000 if not key_data.key_pkl.startswith("/tmp"): 1001 # Under /tmp, file are removed periodically by the 1002 # os. So it is normal that this happen from time to 1003 # time. 1004 _logger.warning("Removing key file %s because the " 1005 "corresponding module is gone from the " 1006 "file system.", 1007 pkl_file_to_remove) 1008 self.loaded_key_pkl.remove(pkl_file_to_remove) 1009 1010 if to_delete or to_delete_empty: 1011 with compilelock.lock_ctx(): 1012 for a, kw in to_delete: 1013 _rmtree(*a, **kw) 1014 for a, kw in to_delete_empty: 1015 files = os.listdir(a[0]) 1016 if not files: 1017 _rmtree(*a, **kw) 1018 1019 _logger.debug('Time needed to refresh cache: %s', 1020 (time.time() - start_time)) 1021 1022 return too_old_to_use 1023 1024 def _get_from_key(self, key, key_data=None): 1025 """ 1026 Returns a module if the passed-in key is found in the cache 1027 and None otherwise. 1028 1029 May raise ValueError if the key is malformed. 1030 1031 """ 1032 name = None 1033 if key is not None: 1034 assert key_data is None 1035 try: 1036 _version, _rest = key 1037 except (TypeError, ValueError): 1038 raise ValueError( 1039 "Invalid key. key must have form (version, rest)", key) 1040 if key in self.entry_from_key: 1041 name = self.entry_from_key[key] 1042 else: 1043 assert key_data is not None 1044 name = key_data.get_entry() 1045 if name is None: 1046 return None 1047 return self._get_module(name) 1048 1049 def _get_from_hash(self, module_hash, key, keep_lock=False): 1050 if module_hash in self.module_hash_to_key_data: 1051 key_data = self.module_hash_to_key_data[module_hash] 1052 module = self._get_from_key(None, key_data) 1053 with compilelock.lock_ctx(keep_lock=keep_lock): 1054 try: 1055 key_data.add_key(key, save_pkl=bool(key[0])) 1056 key_broken = False 1057 except pickle.PicklingError: 1058 key_data.remove_key(key) 1059 key_broken = True 1060 # We need the lock while we check in case of parallel 1061 # process that could be changing the file at the same 1062 # time. 1063 if (key[0] and not key_broken and 1064 self.check_for_broken_eq): 1065 self.check_key(key, key_data.key_pkl) 1066 self._update_mappings(key, key_data, module.__file__, check_in_keys=not key_broken) 1067 return module 1068 else: 1069 return None 1070 1071 def _update_mappings(self, key, key_data, name, check_in_keys): 1072 all_keys = key_data.keys 1073 if not all_keys: 1074 all_keys = [key] 1075 if check_in_keys: 1076 assert key in all_keys 1077 for k in all_keys: 1078 if k in self.entry_from_key: 1079 assert self.entry_from_key[k] == name, (self.entry_from_key[k], name) 1080 else: 1081 self.entry_from_key[k] = name 1082 if key[0]: 1083 self.similar_keys.setdefault(get_safe_part(k), 1084 []).append(key) 1085 1086 def _add_to_cache(self, module, key, module_hash): 1087 """ 1088 This function expects the compile lock to be held. 1089 1090 """ 1091 name = module.__file__ 1092 _logger.debug("Adding module to cache %s %s", 1093 key, name) 1094 # Changing the hash of the key is not allowed during 1095 # compilation. That is the only cause found that makes 1096 # the following assert fail. 1097 assert key not in self.entry_from_key 1098 1099 location = os.path.dirname(name) 1100 key_pkl = os.path.join(location, 'key.pkl') 1101 assert not os.path.exists(key_pkl) 1102 key_data = KeyData( 1103 keys=set([key]), 1104 module_hash=module_hash, 1105 key_pkl=key_pkl, 1106 entry=name) 1107 1108 key_broken = False 1109 if key[0]: 1110 try: 1111 key_data.save_pkl() 1112 except pickle.PicklingError: 1113 key_broken = True 1114 key_data.remove_key(key) 1115 key_data.save_pkl() 1116 if not key_broken and self.check_for_broken_eq: 1117 self.check_key(key, key_pkl) 1118 self.loaded_key_pkl.add(key_pkl) 1119 elif config.cmodule.warn_no_version: 1120 key_flat = flatten(key) 1121 ops = [k for k in key_flat if isinstance(k, theano.Op)] 1122 _logger.warning("not all the" 1123 " following op(s) implement" 1124 " c_code_cache_version(). This makes them" 1125 " recompiled for each process." + str(ops)) 1126 self._update_mappings(key, key_data, module.__file__, not key_broken) 1127 return key_data 1128 1129 def module_from_key(self, key, lnk=None, keep_lock=False): 1130 """ 1131 Return a module from the cache, compiling it if necessary. 1132 1133 Parameters 1134 ---------- 1135 key 1136 The key object associated with the module. If this hits a match, 1137 we avoid compilation. 1138 lnk 1139 Usually a CLinker instance, but it can be any object that defines 1140 the `get_src_code()` and `compile_cmodule(location)` functions. The 1141 first one returns the source code of the module to load/compile and 1142 the second performs the actual compilation. 1143 keep_lock : bool 1144 If True, the compilation lock will not be released if taken. 1145 1146 """ 1147 # Is the module in the cache? 1148 module = self._get_from_key(key) 1149 if module is not None: 1150 return module 1151 1152 src_code = lnk.get_src_code() 1153 # Is the source code already in the cache? 1154 module_hash = get_module_hash(src_code, key) 1155 module = self._get_from_hash(module_hash, key, keep_lock=keep_lock) 1156 if module is not None: 1157 return module 1158 1159 with compilelock.lock_ctx(keep_lock=keep_lock): 1160 # 1) Maybe somebody else compiled it for us while we 1161 # where waiting for the lock. Try to load it again. 1162 # 2) If other repo that import Theano have Theano ops defined, 1163 # we need to refresh the cache here. Otherwise, there are import 1164 # order problems. 1165 # When device=gpu, we compile during Theano 1166 # import. This triggers the loading of the cache. But 1167 # unpickling the cache asks that the external Ops are 1168 # completly loaded, which isn't always the case! 1169 # If a module isn't completly loaded and its unpickling 1170 # fails, it means it is safe for this function 1171 # compilation to skip them, but not for future 1172 # compilations. So reloading the cache here 1173 # compilation fixes this problem. (we could do that only once) 1174 self.refresh(cleanup=False) 1175 1176 module = self._get_from_key(key) 1177 if module is not None: 1178 return module 1179 1180 module = self._get_from_hash(module_hash, key) 1181 if module is not None: 1182 return module 1183 1184 hash_key = hash(key) 1185 1186 nocleanup = False 1187 try: 1188 location = dlimport_workdir(self.dirname) 1189 module = lnk.compile_cmodule(location) 1190 name = module.__file__ 1191 assert name.startswith(location) 1192 assert name not in self.module_from_name 1193 self.module_from_name[name] = module 1194 nocleanup = True 1195 except OSError as e: 1196 _logger.error(e) 1197 if e.errno == 31: 1198 _logger.error('There are %i files in %s', 1199 len(os.listdir(config.compiledir)), 1200 config.compiledir) 1201 raise 1202 finally: 1203 if not nocleanup: 1204 _rmtree(location, ignore_if_missing=True, 1205 msg='exception during compilation') 1206 1207 # Changing the hash of the key is not allowed during 1208 # compilation. 1209 assert hash(key) == hash_key 1210 1211 key_data = self._add_to_cache(module, key, module_hash) 1212 self.module_hash_to_key_data[module_hash] = key_data 1213 1214 self.stats[2] += 1 1215 return module 1216 1217 def check_key(self, key, key_pkl): 1218 """ 1219 Perform checks to detect broken __eq__ / __hash__ implementations. 1220 1221 Parameters 1222 ---------- 1223 key 1224 The key to be checked. 1225 key_pkl 1226 Its associated pickled file containing a KeyData. 1227 1228 """ 1229 start_time = time.time() 1230 # Verify that when we reload the KeyData from the pickled file, the 1231 # same key can be found in it, and is not equal to more than one 1232 # other key. 1233 for i in range(3): 1234 try: 1235 with open(key_pkl, 'rb') as f: 1236 key_data = pickle.load(f) 1237 break 1238 except EOFError: 1239 # This file is probably getting written/updated at the 1240 # same time. This can happen as we read the cache 1241 # without taking the lock. 1242 if i == 2: 1243 with compilelock.lock_ctx(): 1244 with open(key_pkl, 'rb') as f: 1245 key_data = pickle.load(f) 1246 time.sleep(2) 1247 1248 found = sum(key == other_key for other_key in key_data.keys) 1249 msg = '' 1250 if found == 0: 1251 msg = 'Key not found in unpickled KeyData file' 1252 if key_data.keys: 1253 # This is to make debugging in pdb easier, by providing 1254 # the offending keys in the local context. 1255 # key_data_keys = list(key_data.keys) 1256 # import pdb; pdb.set_trace() 1257 pass 1258 elif found > 1: 1259 msg = 'Multiple equal keys found in unpickled KeyData file' 1260 if msg: 1261 raise AssertionError( 1262 "%s. Verify the __eq__ and __hash__ functions of your " 1263 "Ops. The file is: %s. The key is: %s" % (msg, key_pkl, key)) 1264 # Also verify that there exists no other loaded key that would be equal 1265 # to this key. In order to speed things up, we only compare to keys 1266 # with the same version part and config hash, since we can assume this 1267 # part of the key is not broken. 1268 for other in self.similar_keys.get(get_safe_part(key), []): 1269 if other is not key and other == key and hash(other) != hash(key): 1270 raise AssertionError( 1271 "Found two keys that are equal but have a different hash. " 1272 "Verify the __eq__ and __hash__ functions of your Ops. " 1273 "The keys are:\n %s\nand\n %s\n(found in %s)." % 1274 (other, key, key_pkl)) 1275 1276 self.time_spent_in_check_key += time.time() - start_time 1277 1278 # default 31 days 1279 age_thresh_del = config.cmodule.age_thresh_use + 60 * 60 * 24 * 7 1280 age_thresh_del_unversioned = 60 * 60 * 24 * 7 # 7 days 1281 """ 1282 The default age threshold for `clear_old` (in seconds). 1283 1284 """ 1285 1286 def clear_old(self, age_thresh_del=None, delete_if_problem=False): 1287 """Delete entries from the filesystem for cache entries that are too old. 1288 1289 This refreshes the content of the cache. Don't hold the lock 1290 while calling this method, this is useless. It will be taken 1291 if needed. 1292 1293 Parameters 1294 ---------- 1295 age_thresh_del 1296 Dynamic modules whose last access time is more than 1297 ``age_thresh_del`` seconds ago will be erased. 1298 Defaults to 31-day age if not provided. 1299 delete_if_problem 1300 See help of refresh() method. 1301 1302 """ 1303 if age_thresh_del is None: 1304 age_thresh_del = self.age_thresh_del 1305 1306 # Ensure that the too_old_to_use list return by refresh() will 1307 # contain all modules older than age_thresh_del. 1308 if age_thresh_del < self.age_thresh_use: 1309 if age_thresh_del > 0: 1310 _logger.warning("Clearing modules that were not deemed " 1311 "too old to use: age_thresh_del=%d, " 1312 "self.age_thresh_use=%d", 1313 age_thresh_del, 1314 self.age_thresh_use) 1315 else: 1316 _logger.info("Clearing all modules.") 1317 age_thresh_use = age_thresh_del 1318 else: 1319 age_thresh_use = None 1320 1321 too_old_to_use = self.refresh( 1322 age_thresh_use=age_thresh_use, 1323 delete_if_problem=delete_if_problem, 1324 # The clean up is done at init, no need to trigger it again 1325 cleanup=False) 1326 if not too_old_to_use: 1327 return 1328 with compilelock.lock_ctx(): 1329 # Update the age of modules that have been accessed by other 1330 # processes and get all module that are too old to use 1331 # (not loaded in self.entry_from_key). 1332 1333 for entry in too_old_to_use: 1334 # TODO: we are assuming that modules that haven't been 1335 # accessed in over age_thresh_del are not currently in 1336 # use by other processes, but that could be false for 1337 # long-running jobs, or if age_thresh_del < 0. 1338 assert entry not in self.module_from_name 1339 parent = os.path.dirname(entry) 1340 assert parent.startswith(os.path.join(self.dirname, 'tmp')) 1341 _rmtree(parent, msg='old cache directory', level=logging.INFO, 1342 ignore_nocleanup=True) 1343 1344 def clear(self, unversioned_min_age=None, clear_base_files=False, 1345 delete_if_problem=False): 1346 """ 1347 Clear all elements in the cache. 1348 1349 Parameters 1350 ---------- 1351 unversioned_min_age 1352 Forwarded to `clear_unversioned`. In particular, you can set it 1353 to -1 in order to delete all unversioned cached modules regardless 1354 of their age. 1355 clear_base_files : bool 1356 If True, then delete base directories 'cutils_ext', 1357 'lazylinker_ext' and 'scan_perform' if they are present. 1358 If False, those directories are left intact. 1359 delete_if_problem 1360 See help of refresh() method. 1361 1362 """ 1363 with compilelock.lock_ctx(): 1364 self.clear_old( 1365 age_thresh_del=-1.0, 1366 delete_if_problem=delete_if_problem) 1367 self.clear_unversioned(min_age=unversioned_min_age) 1368 if clear_base_files: 1369 self.clear_base_files() 1370 1371 def clear_base_files(self): 1372 """ 1373 Remove base directories 'cutils_ext', 'lazylinker_ext' and 1374 'scan_perform' if present. 1375 1376 Note that we do not delete them outright because it may not work on 1377 some systems due to these modules being currently in use. Instead we 1378 rename them with the '.delete.me' extension, to mark them to be deleted 1379 next time we clear the cache. 1380 1381 """ 1382 with compilelock.lock_ctx(): 1383 for base_dir in ('cutils_ext', 'lazylinker_ext', 'scan_perform'): 1384 to_delete = os.path.join(self.dirname, base_dir + '.delete.me') 1385 if os.path.isdir(to_delete): 1386 try: 1387 shutil.rmtree(to_delete) 1388 _logger.debug('Deleted: %s', to_delete) 1389 except Exception: 1390 _logger.warning('Could not delete %s', to_delete) 1391 continue 1392 to_rename = os.path.join(self.dirname, base_dir) 1393 if os.path.isdir(to_rename): 1394 try: 1395 shutil.move(to_rename, to_delete) 1396 except Exception: 1397 _logger.warning('Could not move %s to %s', 1398 to_rename, to_delete) 1399 1400 def clear_unversioned(self, min_age=None): 1401 """Delete unversioned dynamic modules. 1402 1403 They are deleted both from the internal dictionaries and from the 1404 filesystem. 1405 1406 No need to have the lock when calling this method. It does not 1407 take the lock as unversioned module aren't shared. 1408 1409 This method does not refresh the cache content, it just 1410 accesses the in-memory known module(s). 1411 1412 Parameters 1413 ---------- 1414 min_age 1415 Minimum age to be deleted, in seconds. Defaults to 1416 7-day age if not provided. 1417 1418 """ 1419 if min_age is None: 1420 min_age = self.age_thresh_del_unversioned 1421 1422 # As this delete object that we build and other don't use, we 1423 # don't need the lock. 1424 all_key_datas = list(self.module_hash_to_key_data.values()) 1425 for key_data in all_key_datas: 1426 if not key_data.keys: 1427 # May happen for broken versioned keys. 1428 continue 1429 for key_idx, key in enumerate(key_data.keys): 1430 version, rest = key 1431 if version: 1432 # Since the version is included in the module hash, 1433 # it should not be possible to mix versioned and 1434 # unversioned keys in the same KeyData object. 1435 assert key_idx == 0 1436 break 1437 if not version: 1438 # Note that unversioned keys cannot be broken, so we can 1439 # set do_manual_check to False to speed things up. 1440 key_data.delete_keys_from(self.entry_from_key, 1441 do_manual_check=False) 1442 entry = key_data.get_entry() 1443 # Entry is guaranteed to be in this dictionary, because 1444 # an unversioned entry should never have been loaded via 1445 # refresh. 1446 assert entry in self.module_from_name 1447 1448 del self.module_from_name[entry] 1449 del self.module_hash_to_key_data[key_data.module_hash] 1450 1451 parent = os.path.dirname(entry) 1452 assert parent.startswith(os.path.join(self.dirname, 'tmp')) 1453 _rmtree(parent, msg='unversioned', level=logging.INFO, 1454 ignore_nocleanup=True) 1455 1456 # Sanity check: all unversioned keys should have been removed at 1457 # this point. 1458 for key in self.entry_from_key: 1459 assert key[0] 1460 1461 to_del = [] 1462 time_now = time.time() 1463 for filename in os.listdir(self.dirname): 1464 if filename.startswith('tmp'): 1465 try: 1466 fname = os.path.join(self.dirname, filename, 'key.pkl') 1467 open(fname).close() 1468 has_key = True 1469 except IOError: 1470 has_key = False 1471 if not has_key: 1472 # Use the compiled file by default 1473 path = module_name_from_dir(os.path.join(self.dirname, 1474 filename), 1475 False) 1476 # If it don't exist, use any file in the directory. 1477 if path is None: 1478 path = os.path.join(self.dirname, filename) 1479 try: 1480 files = os.listdir(path) 1481 except OSError as e: 1482 if e.errno == 2: # No such file or directory 1483 # if it don't exist anymore, it mean 1484 # the clean up was already done by 1485 # someone else, so nothing to do about 1486 # it. 1487 continue 1488 if files: 1489 path = os.path.join(path, files[0]) 1490 else: 1491 # If the directory is empty skip it. 1492 # They are deleted elsewhere. 1493 continue 1494 age = time_now - last_access_time(path) 1495 1496 # In normal case, the processus that created this 1497 # directory will delete it. However, if this processus 1498 # crashes, it will not be cleaned up. 1499 # As we don't know if this directory is still used, 1500 # we wait one week and suppose that the processus 1501 # crashed, and we take care of the clean-up. 1502 if age > min_age: 1503 to_del.append(os.path.join(self.dirname, filename)) 1504 1505 # No need to take the lock as it isn't shared. 1506 for f in to_del: 1507 _rmtree(f, 1508 msg='old unversioned', level=logging.INFO, 1509 ignore_nocleanup=True) 1510 1511 def _on_atexit(self): 1512 # Note: no need to call refresh() since it is called by clear_old(). 1513 1514 # Note: no need to take the lock. For unversioned files, we 1515 # don't need it as they aren't shared. For old unversioned 1516 # files, this happen rarely, so we take the lock only when 1517 # this happen. 1518 1519 # Note: for clear_old(), as this happen unfrequently, we only 1520 # take the lock when it happen. 1521 self.clear_old() 1522 self.clear_unversioned() 1523 _logger.debug('Time spent checking keys: %s', 1524 self.time_spent_in_check_key) 1525 1526 1527def _rmtree(parent, ignore_nocleanup=False, msg='', level=logging.DEBUG, 1528 ignore_if_missing=False): 1529 """ 1530 On NFS filesystems, it is impossible to delete a directory with open 1531 files in it. 1532 1533 So instead, some commands in this file will respond to a 1534 failed rmtree() by touching a 'delete.me' file. This file is a message 1535 for a future process to try deleting the directory. 1536 1537 Parameters: 1538 ---------- 1539 parent 1540 Root node to start deleting from 1541 ignore_nocleanup 1542 Delete the tree if flag is TRUE 1543 level 1544 Python Logging level. Set to "DEBUG" by default 1545 ignore_if_missing 1546 If set to True, just return without any issue if parent is NULL 1547 """ 1548 if ignore_if_missing and not os.path.exists(parent): 1549 return 1550 try: 1551 if ignore_nocleanup or not config.nocleanup: 1552 log_msg = 'Deleting' 1553 if msg: 1554 log_msg += ' (%s)' % msg 1555 _logger.log(level, '%s: %s', log_msg, parent) 1556 shutil.rmtree(parent) 1557 except Exception as e: 1558 # If parent still exists, mark it for deletion by a future refresh() 1559 _logger.debug('In _rmtree, encountered exception: %s(%s)', 1560 type(e), e) 1561 if os.path.exists(parent): 1562 try: 1563 _logger.info('placing "delete.me" in %s', parent) 1564 open(os.path.join(parent, 'delete.me'), 'w').close() 1565 except Exception as ee: 1566 _logger.warning("Failed to remove or mark cache directory %s " 1567 "for removal %s", parent, ee) 1568 1569_module_cache = None 1570 1571 1572def get_module_cache(dirname, init_args=None): 1573 """ 1574 Create a new module_cache with the (k, v) pairs in this dictionary 1575 1576 Parameters 1577 ---------- 1578 init_args 1579 If not None, the (k, v) pairs in this dictionary will be forwarded to 1580 the ModuleCache constructor as keyword arguments. 1581 1582 """ 1583 global _module_cache 1584 if init_args is None: 1585 init_args = {} 1586 if _module_cache is None: 1587 _module_cache = ModuleCache(dirname, **init_args) 1588 atexit.register(_module_cache._on_atexit) 1589 elif init_args: 1590 _logger.warning('Ignoring init arguments for module cache because it ' 1591 'was created prior to this call') 1592 if _module_cache.dirname != dirname: 1593 _logger.warning("Returning module cache instance with different " 1594 "dirname (%s) than you requested (%s)", 1595 _module_cache.dirname, dirname) 1596 return _module_cache 1597 1598 1599def get_lib_extension(): 1600 """ 1601 Return the platform-dependent extension for compiled modules. 1602 1603 """ 1604 if sys.platform == 'win32': 1605 return 'pyd' 1606 elif sys.platform == 'cygwin': 1607 return 'dll' 1608 else: 1609 return 'so' 1610 1611 1612def get_gcc_shared_library_arg(): 1613 """ 1614 Return the platform-dependent GCC argument for shared libraries. 1615 1616 """ 1617 if sys.platform == 'darwin': 1618 return '-dynamiclib' 1619 else: 1620 return '-shared' 1621 1622 1623def std_include_dirs(): 1624 numpy_inc_dirs = numpy.distutils.misc_util.get_numpy_include_dirs() 1625 py_inc = distutils.sysconfig.get_python_inc() 1626 py_plat_spec_inc = distutils.sysconfig.get_python_inc(plat_specific=True) 1627 python_inc_dirs = ([py_inc] if py_inc == py_plat_spec_inc 1628 else [py_inc, py_plat_spec_inc]) 1629 gof_inc_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'c_code') 1630 return numpy_inc_dirs + python_inc_dirs + [gof_inc_dir] 1631 1632 1633def std_lib_dirs_and_libs(): 1634 # We cache the results as on Windows, this trigger file access and 1635 # this method is called many times. 1636 if std_lib_dirs_and_libs.data is not None: 1637 return std_lib_dirs_and_libs.data 1638 python_inc = distutils.sysconfig.get_python_inc() 1639 if sys.platform == 'win32': 1640 # Obtain the library name from the Python version instead of the 1641 # installation directory, in case the user defined a custom 1642 # installation directory. 1643 python_version = distutils.sysconfig.get_python_version() 1644 libname = 'python' + python_version.replace('.', '') 1645 # Also add directory containing the Python library to the library 1646 # directories. 1647 python_lib_dirs = [os.path.join(os.path.dirname(python_inc), 'libs')] 1648 if "Canopy" in python_lib_dirs[0]: 1649 # Canopy stores libpython27.a and libmsccr90.a in this directory. 1650 # For some reason, these files are needed when compiling Python 1651 # modules, even when libpython27.lib and python27.dll are 1652 # available, and the *.a files have to be found earlier than 1653 # the other ones. 1654 1655 # When Canopy is installed for the user: 1656 # sys.prefix:C:\Users\username\AppData\Local\Enthought\Canopy\User 1657 # sys.base_prefix:C:\Users\username\AppData\Local\Enthought\Canopy\App\appdata\canopy-1.1.0.1371.win-x86_64 1658 # When Canopy is installed for all users: 1659 # sys.base_prefix: C:\Program Files\Enthought\Canopy\App\appdata\canopy-1.1.0.1371.win-x86_64 1660 # sys.prefix: C:\Users\username\AppData\Local\Enthought\Canopy\User 1661 # So we need to use sys.prefix as it support both cases. 1662 # sys.base_prefix support only one case 1663 libdir = os.path.join(sys.prefix, 'libs') 1664 1665 for f, lib in [('libpython27.a', 'libpython 1.2')]: 1666 if not os.path.exists(os.path.join(libdir, f)): 1667 print(("Your Python version is from Canopy. " + 1668 "You need to install the package '" + lib + 1669 "' from Canopy package manager." 1670 )) 1671 libdirs = [ 1672 # Used in older Canopy 1673 os.path.join(sys.prefix, 'libs'), 1674 # Used in newer Canopy 1675 os.path.join(sys.prefix, 1676 r'EGG-INFO\mingw\usr\x86_64-w64-mingw32\lib')] 1677 for f, lib in [('libmsvcr90.a', 1678 'mingw 4.5.2 or 4.8.1-2 (newer could work)')]: 1679 if not any([os.path.exists(os.path.join(tmp_libdir, f)) 1680 for tmp_libdir in libdirs]): 1681 print(("Your Python version is from Canopy. " + 1682 "You need to install the package '" + lib + 1683 "' from Canopy package manager." 1684 )) 1685 python_lib_dirs.insert(0, libdir) 1686 std_lib_dirs_and_libs.data = [libname], python_lib_dirs 1687 1688 # Suppress -lpython2.x on OS X since the `-undefined dynamic_lookup` 1689 # makes it unnecessary. 1690 elif sys.platform == 'darwin': 1691 std_lib_dirs_and_libs.data = [], [] 1692 else: 1693 if platform.python_implementation() == 'PyPy': 1694 # Assume Linux (note: Ubuntu doesn't ship this .so) 1695 if sys.version_info < (3,): 1696 libname = "pypy-c" 1697 else: 1698 libname = "pypy3-c" 1699 # Unfortunately the only convention of this .so is that it appears 1700 # next to the location of the interpreter binary. 1701 libdir = os.path.dirname(os.path.realpath(sys.executable)) 1702 else: 1703 # Assume Linux 1704 # Typical include directory: /usr/include/python2.6 1705 1706 # get the name of the python library (shared object) 1707 1708 libname = distutils.sysconfig.get_config_var("LDLIBRARY") 1709 1710 if libname.startswith("lib"): 1711 libname = libname[3:] 1712 1713 # remove extension if present 1714 if libname.endswith(".so"): 1715 libname = libname[:-3] 1716 elif libname.endswith(".a"): 1717 libname = libname[:-2] 1718 1719 libdir = distutils.sysconfig.get_config_var("LIBDIR") 1720 1721 std_lib_dirs_and_libs.data = [libname], [libdir] 1722 1723 # sometimes, the linker cannot find -lpython so we need to tell it 1724 # explicitly where it is located this returns 1725 # somepath/lib/python2.x 1726 1727 python_lib = distutils.sysconfig.get_python_lib(plat_specific=1, 1728 standard_lib=1) 1729 python_lib = os.path.dirname(python_lib) 1730 if python_lib not in std_lib_dirs_and_libs.data[1]: 1731 std_lib_dirs_and_libs.data[1].append(python_lib) 1732 return std_lib_dirs_and_libs.data 1733std_lib_dirs_and_libs.data = None 1734 1735 1736def std_libs(): 1737 return std_lib_dirs_and_libs()[0] 1738 1739 1740def std_lib_dirs(): 1741 return std_lib_dirs_and_libs()[1] 1742 1743 1744def gcc_version(): 1745 return gcc_version_str 1746 1747 1748def gcc_llvm(): 1749 """ 1750 Detect if the g++ version used is the llvm one or not. 1751 1752 It don't support all g++ parameters even if it support many of them. 1753 1754 """ 1755 if gcc_llvm.is_llvm is None: 1756 try: 1757 p_out = output_subprocess_Popen([theano.config.cxx, '--version']) 1758 output = p_out[0] + p_out[1] 1759 except OSError: 1760 # Typically means g++ cannot be found. 1761 # So it is not an llvm compiler. 1762 1763 # Normally this should not happen as we should not try to 1764 # compile when g++ is not available. If this happen, it 1765 # will crash later so supposing it is not llvm is "safe". 1766 output = b('') 1767 gcc_llvm.is_llvm = b("llvm") in output 1768 return gcc_llvm.is_llvm 1769 1770gcc_llvm.is_llvm = None 1771 1772 1773class Compiler(object): 1774 """ 1775 Meta compiler that offer some generic function. 1776 1777 """ 1778 1779 @classmethod 1780 def _try_compile_tmp(cls, src_code, tmp_prefix='', flags=(), 1781 try_run=False, output=False, compiler=None, 1782 comp_args=True): 1783 """ 1784 Try to compile (and run) a test program. 1785 1786 This is useful in various occasions, to check if libraries 1787 or compilers are behaving as expected. 1788 1789 If try_run is True, the src_code is assumed to be executable, 1790 and will be run. 1791 1792 If try_run is False, returns the compilation status. 1793 If try_run is True, returns a (compile_status, run_status) pair. 1794 If output is there, we append the stdout and stderr to the output. 1795 1796 Compile arguments from the Compiler's compile_args() method are added 1797 if comp_args=True. 1798 """ 1799 if not compiler: 1800 return False 1801 flags = list(flags) 1802 # Get compile arguments from compiler method if required 1803 if comp_args: 1804 args = cls.compile_args() 1805 else: 1806 args = [] 1807 compilation_ok = True 1808 run_ok = False 1809 out, err = None, None 1810 try: 1811 fd, path = tempfile.mkstemp(suffix='.c', prefix=tmp_prefix) 1812 exe_path = path[:-2] 1813 if os.name == 'nt': 1814 path = "\"" + path + "\"" 1815 exe_path = "\"" + exe_path + "\"" 1816 try: 1817 # Python3 compatibility: try to cast Py3 strings as Py2 strings 1818 try: 1819 src_code = b(src_code) 1820 except Exception: 1821 pass 1822 os.write(fd, src_code) 1823 os.close(fd) 1824 fd = None 1825 out, err, p_ret = output_subprocess_Popen( 1826 [compiler] + args + [path, '-o', exe_path] + flags) 1827 if p_ret != 0: 1828 compilation_ok = False 1829 elif try_run: 1830 out, err, p_ret = output_subprocess_Popen([exe_path]) 1831 run_ok = (p_ret == 0) 1832 finally: 1833 try: 1834 if fd is not None: 1835 os.close(fd) 1836 finally: 1837 if os.path.exists(path): 1838 os.remove(path) 1839 if os.path.exists(exe_path): 1840 os.remove(exe_path) 1841 if os.path.exists(exe_path + ".exe"): 1842 os.remove(exe_path + ".exe") 1843 except OSError as e: 1844 if err is None: 1845 err = str(e) 1846 else: 1847 err = str(err) + "\n" + str(e) 1848 compilation_ok = False 1849 1850 if not try_run and not output: 1851 return compilation_ok 1852 elif not try_run and output: 1853 return (compilation_ok, out, err) 1854 elif not output: 1855 return (compilation_ok, run_ok) 1856 else: 1857 return (compilation_ok, run_ok, out, err) 1858 1859 @classmethod 1860 def _try_flags(cls, flag_list, preambule="", body="", 1861 try_run=False, output=False, compiler=None, 1862 comp_args=True): 1863 """ 1864 Try to compile a dummy file with these flags. 1865 1866 Returns True if compilation was successful, False if there 1867 were errors. 1868 1869 Compile arguments from the Compiler's compile_args() method are added 1870 if comp_args=True. 1871 1872 """ 1873 if not compiler: 1874 return False 1875 1876 code = b(""" 1877 %(preambule)s 1878 int main(int argc, char** argv) 1879 { 1880 %(body)s 1881 return 0; 1882 } 1883 """ % locals()) 1884 return cls._try_compile_tmp(code, tmp_prefix='try_flags_', 1885 flags=flag_list, try_run=try_run, 1886 output=output, compiler=compiler, 1887 comp_args=comp_args) 1888 1889 1890def try_march_flag(flags): 1891 """ 1892 Try to compile and run a simple C snippet using current flags. 1893 Return: compilation success (True/False), execution success (True/False) 1894 """ 1895 test_code = textwrap.dedent("""\ 1896 #include <cmath> 1897 using namespace std; 1898 int main(int argc, char** argv) 1899 { 1900 float Nx = -1.3787706641; 1901 float Sx = 25.0; 1902 double r = Nx + sqrt(Sx); 1903 if (abs(r - 3.621229) > 0.01) 1904 { 1905 return -1; 1906 } 1907 return 0; 1908 } 1909 """) 1910 1911 cflags = flags + ['-L' + d for d in theano.gof.cmodule.std_lib_dirs()] 1912 compilation_result, execution_result = GCC_compiler.try_compile_tmp( 1913 test_code, tmp_prefix='try_march_', 1914 flags=cflags, try_run=True) 1915 return compilation_result, execution_result 1916 1917 1918class GCC_compiler(Compiler): 1919 # The equivalent flags of --march=native used by g++. 1920 march_flags = None 1921 1922 supports_amdlibm = True 1923 1924 @staticmethod 1925 def version_str(): 1926 return theano.config.cxx + " " + gcc_version_str 1927 1928 @staticmethod 1929 def compile_args(march_flags=True): 1930 cxxflags = [flag for flag in config.gcc.cxxflags.split(' ') if flag] 1931 if "-fopenmp" in cxxflags: 1932 raise ValueError( 1933 "Do not use -fopenmp in Theano flag gcc.cxxflags." 1934 " To enable OpenMP, use the Theano flag openmp=True") 1935 # Add the equivalent of -march=native flag. We can't use 1936 # -march=native as when the compiledir is shared by multiple 1937 # computers (for example, if the home directory is on NFS), this 1938 # won't be optimum or cause crash depending if the file is compiled 1939 # on an older or more recent computer. 1940 # Those URL discuss how to find witch flags are used by -march=native. 1941 # http://en.gentoo-wiki.com/wiki/Safe_Cflags#-march.3Dnative 1942 # http://en.gentoo-wiki.com/wiki/Hardware_CFLAGS 1943 detect_march = GCC_compiler.march_flags is None and march_flags 1944 if detect_march: 1945 for f in cxxflags: 1946 # If the user give an -march=X parameter, don't add one ourself 1947 if ((f.startswith("--march=") or f.startswith("-march="))): 1948 detect_march = False 1949 GCC_compiler.march_flags = [] 1950 break 1951 1952 if ('g++' not in theano.config.cxx and 1953 'clang++' not in theano.config.cxx and 1954 'clang-omp++' not in theano.config.cxx and 1955 'icpc' not in theano.config.cxx): 1956 _logger.warn( 1957 "OPTIMIZATION WARNING: your Theano flag `cxx` seems not to be" 1958 " the g++ compiler. So we disable the compiler optimization" 1959 " specific to g++ that tell to compile for a specific CPU." 1960 " At worst, this could cause slow down.\n" 1961 " You can add those parameters to the compiler yourself" 1962 " via the Theano flag `gcc.cxxflags`." 1963 ) 1964 detect_march = False 1965 1966 if detect_march: 1967 GCC_compiler.march_flags = [] 1968 1969 def get_lines(cmd, parse=True): 1970 p = subprocess_Popen(cmd, 1971 stdout=subprocess.PIPE, 1972 stderr=subprocess.PIPE, 1973 stdin=subprocess.PIPE, 1974 shell=True) 1975 # For mingw64 with GCC >= 4.7, passing os.devnull 1976 # as stdin (which is the default) results in the process 1977 # waiting forever without returning. For that reason, 1978 # we use a pipe, and use the empty string as input. 1979 (stdout, stderr) = p.communicate(input=b('')) 1980 if p.returncode != 0: 1981 return None 1982 1983 lines = BytesIO(stdout + stderr).readlines() 1984 lines = decode_iter(lines) 1985 if parse: 1986 selected_lines = [] 1987 for line in lines: 1988 if ("COLLECT_GCC_OPTIONS=" in line or 1989 "CFLAGS=" in line or 1990 "CXXFLAGS=" in line or 1991 "-march=native" in line): 1992 continue 1993 for reg in ["-march=", "-mtune=", 1994 "-target-cpu", "-mabi="]: 1995 if reg in line: 1996 selected_lines.append(line.strip()) 1997 lines = list(set(selected_lines)) # to remove duplicate 1998 1999 return lines 2000 2001 # The '-' at the end is needed. Otherwise, g++ do not output 2002 # enough information. 2003 native_lines = get_lines("%s -march=native -E -v -" % theano.config.cxx) 2004 if native_lines is None: 2005 _logger.info("Call to 'g++ -march=native' failed," 2006 "not setting -march flag") 2007 detect_march = False 2008 else: 2009 _logger.info("g++ -march=native selected lines: %s", 2010 native_lines) 2011 2012 if detect_march: 2013 if len(native_lines) != 1: 2014 if len(native_lines) == 0: 2015 # That means we did not select the right lines, so 2016 # we have to report all the lines instead 2017 reported_lines = get_lines("%s -march=native -E -v -" % theano.config.cxx, 2018 parse=False) 2019 else: 2020 reported_lines = native_lines 2021 _logger.warn( 2022 "OPTIMIZATION WARNING: Theano was not able to find the" 2023 " g++ parameters that tune the compilation to your " 2024 " specific CPU. This can slow down the execution of Theano" 2025 " functions. Please submit the following lines to" 2026 " Theano's mailing list so that we can fix this" 2027 " problem:\n %s", 2028 reported_lines) 2029 else: 2030 default_lines = get_lines("%s -E -v -" % theano.config.cxx) 2031 _logger.info("g++ default lines: %s", default_lines) 2032 if len(default_lines) < 1: 2033 _logger.warn( 2034 "OPTIMIZATION WARNING: Theano was not able to find the" 2035 " default g++ parameters. This is needed to tune" 2036 " the compilation to your specific" 2037 " CPU. This can slow down the execution of Theano" 2038 " functions. Please submit the following lines to" 2039 " Theano's mailing list so that we can fix this" 2040 " problem:\n %s", 2041 get_lines("%s -E -v -" % theano.config.cxx, parse=False)) 2042 else: 2043 # Some options are actually given as "-option value", 2044 # we want to treat them as only one token when comparing 2045 # different command lines. 2046 # Heuristic: tokens not starting with a dash should be 2047 # joined with the previous one. 2048 def join_options(init_part): 2049 new_part = [] 2050 for i in range(len(init_part)): 2051 p = init_part[i] 2052 if p.startswith('-'): 2053 p_list = [p] 2054 while ((i + 1 < len(init_part)) and 2055 not init_part[i + 1].startswith('-')): 2056 # append that next part to p_list 2057 p_list.append(init_part[i + 1]) 2058 i += 1 2059 new_part.append(' '.join(p_list)) 2060 elif i == 0: 2061 # The first argument does not usually start 2062 # with "-", still add it 2063 new_part.append(p) 2064 # Else, skip it, as it was already included 2065 # with the previous part. 2066 return new_part 2067 2068 part = join_options(native_lines[0].split()) 2069 2070 for line in default_lines: 2071 if line.startswith(part[0]): 2072 part2 = [p for p in join_options(line.split()) 2073 if ('march' not in p and 2074 'mtune' not in p and 2075 'target-cpu' not in p)] 2076 if sys.platform == 'darwin': 2077 # We only use translated target-cpu on 2078 # mac since the other flags are not 2079 # supported as compiler flags for the 2080 # driver. 2081 new_flags = [p for p in part if 'target-cpu' in p] 2082 else: 2083 new_flags = [p for p in part if p not in part2] 2084 # Replace '-target-cpu value', which is an option 2085 # of clang, with '-march=value'. 2086 for i, p in enumerate(new_flags): 2087 if 'target-cpu' in p: 2088 opt = p.split() 2089 if len(opt) == 2: 2090 opt_name, opt_val = opt 2091 new_flags[i] = '-march=%s' % opt_val 2092 2093 # Some versions of GCC report the native arch 2094 # as "corei7-avx", but it generates illegal 2095 # instructions, and should be "corei7" instead. 2096 # Affected versions are: 2097 # - 4.6 before 4.6.4 2098 # - 4.7 before 4.7.3 2099 # - 4.8 before 4.8.1 2100 # Earlier versions did not have arch "corei7-avx" 2101 for i, p in enumerate(new_flags): 2102 if 'march' not in p: 2103 continue 2104 opt = p.split('=') 2105 if len(opt) != 2: 2106 # Inexpected, but do not crash 2107 continue 2108 opt_val = opt[1] 2109 if not opt_val.endswith('-avx'): 2110 # OK 2111 continue 2112 # Check the version of GCC 2113 version = gcc_version_str.split('.') 2114 if len(version) != 3: 2115 # Unexpected, but should not be a problem 2116 continue 2117 mj, mn, patch = [int(vp) for vp in version] 2118 if (((mj, mn) == (4, 6) and patch < 4) or 2119 ((mj, mn) == (4, 7) and patch <= 3) or 2120 ((mj, mn) == (4, 8) and patch < 1)): 2121 new_flags[i] = p.rstrip('-avx') 2122 2123 # Go back to split arguments, like 2124 # ["-option", "value"], 2125 # as this is the way g++ expects them split. 2126 split_flags = [] 2127 for p in new_flags: 2128 split_flags.extend(p.split()) 2129 2130 GCC_compiler.march_flags = split_flags 2131 break 2132 _logger.info("g++ -march=native equivalent flags: %s", 2133 GCC_compiler.march_flags) 2134 2135 # Find working march flag: 2136 # -- if current GCC_compiler.march_flags works, we're done. 2137 # -- else replace -march and -mtune with ['core-i7-avx', 'core-i7', 'core2'] 2138 # and retry with all other flags and arguments intact. 2139 # -- else remove all other flags and only try with -march = default + flags_to_try. 2140 # -- if none of that worked, set GCC_compiler.march_flags = [] (for x86). 2141 2142 default_compilation_result, default_execution_result = try_march_flag(GCC_compiler.march_flags) 2143 if not default_compilation_result or not default_execution_result: 2144 march_success = False 2145 march_ind = None 2146 mtune_ind = None 2147 default_detected_flag = [] 2148 march_flags_to_try = ['corei7-avx', 'corei7', 'core2'] 2149 2150 for m_ in xrange(len(GCC_compiler.march_flags)): 2151 march_flag = GCC_compiler.march_flags[m_] 2152 if 'march' in march_flag: 2153 march_ind = m_ 2154 default_detected_flag = [march_flag] 2155 elif 'mtune' in march_flag: 2156 mtune_ind = m_ 2157 2158 for march_flag in march_flags_to_try: 2159 if march_ind is not None: 2160 GCC_compiler.march_flags[march_ind] = '-march=' + march_flag 2161 if mtune_ind is not None: 2162 GCC_compiler.march_flags[mtune_ind] = '-mtune=' + march_flag 2163 2164 compilation_result, execution_result = try_march_flag(GCC_compiler.march_flags) 2165 2166 if compilation_result and execution_result: 2167 march_success = True 2168 break 2169 2170 if not march_success: 2171 # perhaps one of the other flags was problematic; try default flag in isolation again: 2172 march_flags_to_try = default_detected_flag + march_flags_to_try 2173 for march_flag in march_flags_to_try: 2174 compilation_result, execution_result = try_march_flag(['-march=' + march_flag]) 2175 if compilation_result and execution_result: 2176 march_success = True 2177 GCC_compiler.march_flags = ['-march=' + march_flag] 2178 break 2179 2180 if not march_success: 2181 GCC_compiler.march_flags = [] 2182 2183 # Add the detected -march=native equivalent flags 2184 if march_flags and GCC_compiler.march_flags: 2185 cxxflags.extend(GCC_compiler.march_flags) 2186 2187 # NumPy 1.7 Deprecate the old API. 2188 # The following macro asserts that we don't bring new code 2189 # that use the old API. 2190 cxxflags.append("-DNPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION") 2191 2192 # Platform-specific flags. 2193 # We put them here, rather than in compile_str(), so they en up 2194 # in the key of the compiled module, avoiding potential conflicts. 2195 2196 # Figure out whether the current Python executable is 32 2197 # or 64 bit and compile accordingly. This step is ignored for 2198 # ARM (32-bit and 64-bit) architectures in order to make 2199 # Theano compatible with the Raspberry Pi, Raspberry Pi 2, or 2200 # other systems with ARM processors. 2201 if (not any(['arm' in flag for flag in cxxflags]) and 2202 not any(arch in platform.machine() for arch in ['arm', 'aarch'])): 2203 n_bits = local_bitwidth() 2204 cxxflags.append('-m%d' % n_bits) 2205 _logger.debug("Compiling for %s bit architecture", n_bits) 2206 2207 if sys.platform != 'win32': 2208 # Under Windows it looks like fPIC is useless. Compiler warning: 2209 # '-fPIC ignored for target (all code is position independent)' 2210 cxxflags.append('-fPIC') 2211 2212 if sys.platform == 'win32' and local_bitwidth() == 64: 2213 # Under 64-bit Windows installation, sys.platform is 'win32'. 2214 # We need to define MS_WIN64 for the preprocessor to be able to 2215 # link with libpython. 2216 cxxflags.append('-DMS_WIN64') 2217 2218 if sys.platform == 'darwin': 2219 # Use the already-loaded python symbols. 2220 cxxflags.extend(['-undefined', 'dynamic_lookup']) 2221 2222 if sys.platform == 'win32': 2223 # Workaround for https://github.com/Theano/Theano/issues/4926. 2224 # https://github.com/python/cpython/pull/11283/ removed the "hypot" 2225 # redefinition for recent CPython versions (>=2.7.16 and >=3.7.3). 2226 # The following nullifies that redefinition, if it is found. 2227 python_version = sys.version_info[:3] 2228 if python_version < (2, 7, 16) or (3,) <= python_version < (3, 7, 3): 2229 config_h_filename = distutils.sysconfig.get_config_h_filename() 2230 try: 2231 with open(config_h_filename) as config_h: 2232 if any(line.startswith('#define hypot _hypot') for line in config_h): 2233 cxxflags.append('-D_hypot=hypot') 2234 except IOError: 2235 pass 2236 2237 return cxxflags 2238 2239 @classmethod 2240 def try_compile_tmp(cls, src_code, tmp_prefix='', flags=(), 2241 try_run=False, output=False, comp_args=True): 2242 return cls._try_compile_tmp(src_code, tmp_prefix, flags, 2243 try_run, output, theano.config.cxx, 2244 comp_args) 2245 2246 @classmethod 2247 def try_flags(cls, flag_list, preambule="", body="", 2248 try_run=False, output=False, comp_args=True): 2249 return cls._try_flags(flag_list, preambule, body, try_run, output, 2250 theano.config.cxx, comp_args) 2251 2252 @staticmethod 2253 def compile_str(module_name, src_code, location=None, 2254 include_dirs=None, lib_dirs=None, libs=None, 2255 preargs=None, py_module=True, hide_symbols=True): 2256 """ 2257 Parameters 2258 ---------- 2259 module_name : str 2260 This has been embedded in the src_code. 2261 src_code 2262 A complete c or c++ source listing for the module. 2263 location 2264 A pre-existing filesystem directory where the cpp file and .so will 2265 be written. 2266 include_dirs 2267 A list of include directory names (each gets prefixed with -I). 2268 lib_dirs 2269 A list of library search path directory names (each gets prefixed 2270 with -L). 2271 libs 2272 A list of libraries to link with (each gets prefixed with -l). 2273 preargs 2274 A list of extra compiler arguments. 2275 py_module 2276 If False, compile to a shared library, but do not import it as a 2277 Python module. 2278 hide_symbols 2279 If True (the default) all symbols will be hidden from the library 2280 symbol table (which means that other objects can't use them). 2281 2282 Returns 2283 ------- 2284 object 2285 Dynamically-imported python module of the compiled code (unless 2286 py_module is False, in that case returns None). 2287 2288 """ 2289 # TODO: Do not do the dlimport in this function 2290 2291 if not theano.config.cxx: 2292 raise MissingGXX("g++ not available! We can't compile c code.") 2293 2294 if include_dirs is None: 2295 include_dirs = [] 2296 if lib_dirs is None: 2297 lib_dirs = [] 2298 if libs is None: 2299 libs = [] 2300 if preargs is None: 2301 preargs = [] 2302 2303 # Remove empty string directory 2304 include_dirs = [d for d in include_dirs if d] 2305 lib_dirs = [d for d in lib_dirs if d] 2306 2307 include_dirs = include_dirs + std_include_dirs() 2308 libs = libs + std_libs() 2309 lib_dirs = lib_dirs + std_lib_dirs() 2310 2311 cppfilename = os.path.join(location, 'mod.cpp') 2312 with open(cppfilename, 'w') as cppfile: 2313 2314 _logger.debug('Writing module C++ code to %s', cppfilename) 2315 2316 cppfile.write(src_code) 2317 # Avoid gcc warning "no newline at end of file". 2318 if not src_code.endswith('\n'): 2319 cppfile.write('\n') 2320 2321 if platform.python_implementation() == 'PyPy': 2322 suffix = '.' + get_lib_extension() 2323 2324 dist_suffix = distutils.sysconfig.get_config_var("SO") 2325 if dist_suffix is not None and dist_suffix != '': 2326 suffix = dist_suffix 2327 2328 filepath = '%s%s' % (module_name, suffix) 2329 else: 2330 filepath = '%s.%s' % (module_name, get_lib_extension()) 2331 2332 lib_filename = os.path.join(location, filepath) 2333 2334 _logger.debug('Generating shared lib %s', lib_filename) 2335 cmd = [theano.config.cxx, get_gcc_shared_library_arg(), '-g'] 2336 2337 if config.cmodule.remove_gxx_opt: 2338 cmd.extend(p for p in preargs if not p.startswith('-O')) 2339 else: 2340 cmd.extend(preargs) 2341 # to support path that includes spaces, we need to wrap it with double quotes on Windows 2342 path_wrapper = "\"" if os.name == 'nt' else "" 2343 cmd.extend(['-I%s%s%s' % (path_wrapper, idir, path_wrapper) for idir in include_dirs]) 2344 cmd.extend(['-L%s%s%s' % (path_wrapper, ldir, path_wrapper) for ldir in lib_dirs]) 2345 if hide_symbols and sys.platform != 'win32': 2346 # This has been available since gcc 4.0 so we suppose it 2347 # is always available. We pass it here since it 2348 # significantly reduces the size of the symbol table for 2349 # the objects we want to share. This in turns leads to 2350 # improved loading times on most platforms (win32 is 2351 # different, as usual). 2352 cmd.append('-fvisibility=hidden') 2353 cmd.extend(['-o', '%s%s%s' % (path_wrapper, lib_filename, path_wrapper)]) 2354 cmd.append('%s%s%s' % (path_wrapper, cppfilename, path_wrapper)) 2355 cmd.extend(['-l%s' % l for l in libs]) 2356 # print >> sys.stderr, 'COMPILING W CMD', cmd 2357 _logger.debug('Running cmd: %s', ' '.join(cmd)) 2358 2359 def print_command_line_error(): 2360 # Print command line when a problem occurred. 2361 print(("Problem occurred during compilation with the " 2362 "command line below:"), file=sys.stderr) 2363 print(' '.join(cmd), file=sys.stderr) 2364 2365 try: 2366 p_out = output_subprocess_Popen(cmd) 2367 compile_stderr = decode(p_out[1]) 2368 except Exception: 2369 # An exception can occur e.g. if `g++` is not found. 2370 print_command_line_error() 2371 raise 2372 2373 status = p_out[2] 2374 2375 if status: 2376 tf = tempfile.NamedTemporaryFile( 2377 mode='w', 2378 prefix='theano_compilation_error_', 2379 delete=False 2380 ) 2381 # gcc put its messages to stderr, so we add ours now 2382 tf.write('===============================\n') 2383 for i, l in enumerate(src_code.split('\n')): 2384 tf.write('%05i\t%s\n' % (i + 1, l)) 2385 tf.write('===============================\n') 2386 tf.write("Problem occurred during compilation with the " 2387 "command line below:\n") 2388 tf.write(' '.join(cmd)) 2389 # Print errors just below the command line. 2390 tf.write(compile_stderr) 2391 tf.close() 2392 print('\nYou can find the C code in this temporary file: ' + tf.name) 2393 not_found_libraries = re.findall('-l["."-_a-zA-Z0-9]*', compile_stderr) 2394 for nf_lib in not_found_libraries: 2395 print('library ' + nf_lib[2:] + ' is not found.') 2396 if re.search('-lPYTHON[".0-9]*', nf_lib, re.IGNORECASE): 2397 py_string = re.search('-lpython[".0-9]*', nf_lib, re.IGNORECASE).group()[8:] 2398 if py_string != '': 2399 print( 2400 'Check if package python-dev ' + py_string + ' or python-devel ' + py_string + ' is installed.' 2401 ) 2402 else: 2403 print( 2404 'Check if package python-dev or python-devel is installed.' 2405 ) 2406 2407 # We replace '\n' by '. ' in the error message because when Python 2408 # prints the exception, having '\n' in the text makes it more 2409 # difficult to read. 2410 raise Exception('Compilation failed (return status=%s): %s' % 2411 (status, compile_stderr.replace('\n', '. '))) 2412 elif config.cmodule.compilation_warning and compile_stderr: 2413 # Print errors just below the command line. 2414 print(compile_stderr) 2415 2416 if py_module: 2417 # touch the __init__ file 2418 open(os.path.join(location, "__init__.py"), 'w').close() 2419 assert os.path.isfile(lib_filename) 2420 return dlimport(lib_filename) 2421 2422 2423def icc_module_compile_str(*args): 2424 raise NotImplementedError() 2425