1#!/usr/bin/env python 2# encoding: utf-8 3# Thomas Nagy, 2005-2018 (ita) 4 5""" 6Utilities and platform-specific fixes 7 8The portability fixes try to provide a consistent behavior of the Waf API 9through Python versions 2.5 to 3.X and across different platforms (win32, linux, etc) 10""" 11 12from __future__ import with_statement 13 14import atexit, os, sys, errno, inspect, re, datetime, platform, base64, signal, functools, time 15 16try: 17 import cPickle 18except ImportError: 19 import pickle as cPickle 20 21# leave this 22if os.name == 'posix' and sys.version_info[0] < 3: 23 try: 24 import subprocess32 as subprocess 25 except ImportError: 26 import subprocess 27else: 28 import subprocess 29 30try: 31 TimeoutExpired = subprocess.TimeoutExpired 32except AttributeError: 33 class TimeoutExpired(Exception): 34 pass 35 36from collections import deque, defaultdict 37 38try: 39 import _winreg as winreg 40except ImportError: 41 try: 42 import winreg 43 except ImportError: 44 winreg = None 45 46from waflib import Errors 47 48try: 49 from hashlib import md5 50except ImportError: 51 try: 52 from hashlib import sha1 as md5 53 except ImportError: 54 # never fail to enable potential fixes from another module 55 pass 56else: 57 try: 58 md5().digest() 59 except ValueError: 60 # Fips? #2213 61 from hashlib import sha1 as md5 62 63try: 64 import threading 65except ImportError: 66 if not 'JOBS' in os.environ: 67 # no threading :-( 68 os.environ['JOBS'] = '1' 69 70 class threading(object): 71 """ 72 A fake threading class for platforms lacking the threading module. 73 Use ``waf -j1`` on those platforms 74 """ 75 pass 76 class Lock(object): 77 """Fake Lock class""" 78 def acquire(self): 79 pass 80 def release(self): 81 pass 82 threading.Lock = threading.Thread = Lock 83 84SIG_NIL = 'SIG_NIL_SIG_NIL_'.encode() 85"""Arbitrary null value for hashes. Modify this value according to the hash function in use""" 86 87O644 = 420 88"""Constant representing the permissions for regular files (0644 raises a syntax error on python 3)""" 89 90O755 = 493 91"""Constant representing the permissions for executable files (0755 raises a syntax error on python 3)""" 92 93rot_chr = ['\\', '|', '/', '-'] 94"List of characters to use when displaying the throbber (progress bar)" 95 96rot_idx = 0 97"Index of the current throbber character (progress bar)" 98 99class ordered_iter_dict(dict): 100 """Ordered dictionary that provides iteration from the most recently inserted keys first""" 101 def __init__(self, *k, **kw): 102 self.lst = deque() 103 dict.__init__(self, *k, **kw) 104 def clear(self): 105 dict.clear(self) 106 self.lst = deque() 107 def __setitem__(self, key, value): 108 if key in dict.keys(self): 109 self.lst.remove(key) 110 dict.__setitem__(self, key, value) 111 self.lst.append(key) 112 def __delitem__(self, key): 113 dict.__delitem__(self, key) 114 try: 115 self.lst.remove(key) 116 except ValueError: 117 pass 118 def __iter__(self): 119 return reversed(self.lst) 120 def keys(self): 121 return reversed(self.lst) 122 123class lru_node(object): 124 """ 125 Used by :py:class:`waflib.Utils.lru_cache` 126 """ 127 __slots__ = ('next', 'prev', 'key', 'val') 128 def __init__(self): 129 self.next = self 130 self.prev = self 131 self.key = None 132 self.val = None 133 134class lru_cache(object): 135 """ 136 A simple least-recently used cache with lazy allocation 137 """ 138 __slots__ = ('maxlen', 'table', 'head') 139 def __init__(self, maxlen=100): 140 self.maxlen = maxlen 141 """ 142 Maximum amount of elements in the cache 143 """ 144 self.table = {} 145 """ 146 Mapping key-value 147 """ 148 self.head = lru_node() 149 self.head.next = self.head 150 self.head.prev = self.head 151 152 def __getitem__(self, key): 153 node = self.table[key] 154 # assert(key==node.key) 155 if node is self.head: 156 return node.val 157 158 # detach the node found 159 node.prev.next = node.next 160 node.next.prev = node.prev 161 162 # replace the head 163 node.next = self.head.next 164 node.prev = self.head 165 self.head = node.next.prev = node.prev.next = node 166 167 return node.val 168 169 def __setitem__(self, key, val): 170 if key in self.table: 171 # update the value for an existing key 172 node = self.table[key] 173 node.val = val 174 self.__getitem__(key) 175 else: 176 if len(self.table) < self.maxlen: 177 # the very first item is unused until the maximum is reached 178 node = lru_node() 179 node.prev = self.head 180 node.next = self.head.next 181 node.prev.next = node.next.prev = node 182 else: 183 node = self.head = self.head.next 184 try: 185 # that's another key 186 del self.table[node.key] 187 except KeyError: 188 pass 189 190 node.key = key 191 node.val = val 192 self.table[key] = node 193 194class lazy_generator(object): 195 def __init__(self, fun, params): 196 self.fun = fun 197 self.params = params 198 199 def __iter__(self): 200 return self 201 202 def __next__(self): 203 try: 204 it = self.it 205 except AttributeError: 206 it = self.it = self.fun(*self.params) 207 return next(it) 208 209 next = __next__ 210 211is_win32 = os.sep == '\\' or sys.platform == 'win32' or os.name == 'nt' # msys2 212""" 213Whether this system is a Windows series 214""" 215 216def readf(fname, m='r', encoding='latin-1'): 217 """ 218 Reads an entire file into a string. See also :py:meth:`waflib.Node.Node.readf`:: 219 220 def build(ctx): 221 from waflib import Utils 222 txt = Utils.readf(self.path.find_node('wscript').abspath()) 223 txt = ctx.path.find_node('wscript').read() 224 225 :type fname: string 226 :param fname: Path to file 227 :type m: string 228 :param m: Open mode 229 :type encoding: string 230 :param encoding: encoding value, only used for python 3 231 :rtype: string 232 :return: Content of the file 233 """ 234 235 if sys.hexversion > 0x3000000 and not 'b' in m: 236 m += 'b' 237 with open(fname, m) as f: 238 txt = f.read() 239 if encoding: 240 txt = txt.decode(encoding) 241 else: 242 txt = txt.decode() 243 else: 244 with open(fname, m) as f: 245 txt = f.read() 246 return txt 247 248def writef(fname, data, m='w', encoding='latin-1'): 249 """ 250 Writes an entire file from a string. 251 See also :py:meth:`waflib.Node.Node.writef`:: 252 253 def build(ctx): 254 from waflib import Utils 255 txt = Utils.writef(self.path.make_node('i_like_kittens').abspath(), 'some data') 256 self.path.make_node('i_like_kittens').write('some data') 257 258 :type fname: string 259 :param fname: Path to file 260 :type data: string 261 :param data: The contents to write to the file 262 :type m: string 263 :param m: Open mode 264 :type encoding: string 265 :param encoding: encoding value, only used for python 3 266 """ 267 if sys.hexversion > 0x3000000 and not 'b' in m: 268 data = data.encode(encoding) 269 m += 'b' 270 with open(fname, m) as f: 271 f.write(data) 272 273def h_file(fname): 274 """ 275 Computes a hash value for a file by using md5. Use the md5_tstamp 276 extension to get faster build hashes if necessary. 277 278 :type fname: string 279 :param fname: path to the file to hash 280 :return: hash of the file contents 281 :rtype: string or bytes 282 """ 283 m = md5() 284 with open(fname, 'rb') as f: 285 while fname: 286 fname = f.read(200000) 287 m.update(fname) 288 return m.digest() 289 290def readf_win32(f, m='r', encoding='latin-1'): 291 flags = os.O_NOINHERIT | os.O_RDONLY 292 if 'b' in m: 293 flags |= os.O_BINARY 294 if '+' in m: 295 flags |= os.O_RDWR 296 try: 297 fd = os.open(f, flags) 298 except OSError: 299 raise IOError('Cannot read from %r' % f) 300 301 if sys.hexversion > 0x3000000 and not 'b' in m: 302 m += 'b' 303 with os.fdopen(fd, m) as f: 304 txt = f.read() 305 if encoding: 306 txt = txt.decode(encoding) 307 else: 308 txt = txt.decode() 309 else: 310 with os.fdopen(fd, m) as f: 311 txt = f.read() 312 return txt 313 314def writef_win32(f, data, m='w', encoding='latin-1'): 315 if sys.hexversion > 0x3000000 and not 'b' in m: 316 data = data.encode(encoding) 317 m += 'b' 318 flags = os.O_CREAT | os.O_TRUNC | os.O_WRONLY | os.O_NOINHERIT 319 if 'b' in m: 320 flags |= os.O_BINARY 321 if '+' in m: 322 flags |= os.O_RDWR 323 try: 324 fd = os.open(f, flags) 325 except OSError: 326 raise OSError('Cannot write to %r' % f) 327 with os.fdopen(fd, m) as f: 328 f.write(data) 329 330def h_file_win32(fname): 331 try: 332 fd = os.open(fname, os.O_BINARY | os.O_RDONLY | os.O_NOINHERIT) 333 except OSError: 334 raise OSError('Cannot read from %r' % fname) 335 m = md5() 336 with os.fdopen(fd, 'rb') as f: 337 while fname: 338 fname = f.read(200000) 339 m.update(fname) 340 return m.digest() 341 342# always save these 343readf_unix = readf 344writef_unix = writef 345h_file_unix = h_file 346if hasattr(os, 'O_NOINHERIT') and sys.hexversion < 0x3040000: 347 # replace the default functions 348 readf = readf_win32 349 writef = writef_win32 350 h_file = h_file_win32 351 352try: 353 x = ''.encode('hex') 354except LookupError: 355 import binascii 356 def to_hex(s): 357 ret = binascii.hexlify(s) 358 if not isinstance(ret, str): 359 ret = ret.decode('utf-8') 360 return ret 361else: 362 def to_hex(s): 363 return s.encode('hex') 364 365to_hex.__doc__ = """ 366Return the hexadecimal representation of a string 367 368:param s: string to convert 369:type s: string 370""" 371 372def listdir_win32(s): 373 """ 374 Lists the contents of a folder in a portable manner. 375 On Win32, returns the list of drive letters: ['C:', 'X:', 'Z:'] when an empty string is given. 376 377 :type s: string 378 :param s: a string, which can be empty on Windows 379 """ 380 if not s: 381 try: 382 import ctypes 383 except ImportError: 384 # there is nothing much we can do 385 return [x + ':\\' for x in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'] 386 else: 387 dlen = 4 # length of "?:\\x00" 388 maxdrives = 26 389 buf = ctypes.create_string_buffer(maxdrives * dlen) 390 ndrives = ctypes.windll.kernel32.GetLogicalDriveStringsA(maxdrives*dlen, ctypes.byref(buf)) 391 return [ str(buf.raw[4*i:4*i+2].decode('ascii')) for i in range(int(ndrives/dlen)) ] 392 393 if len(s) == 2 and s[1] == ":": 394 s += os.sep 395 396 if not os.path.isdir(s): 397 e = OSError('%s is not a directory' % s) 398 e.errno = errno.ENOENT 399 raise e 400 return os.listdir(s) 401 402listdir = os.listdir 403if is_win32: 404 listdir = listdir_win32 405 406def num2ver(ver): 407 """ 408 Converts a string, tuple or version number into an integer. The number is supposed to have at most 4 digits:: 409 410 from waflib.Utils import num2ver 411 num2ver('1.3.2') == num2ver((1,3,2)) == num2ver((1,3,2,0)) 412 413 :type ver: string or tuple of numbers 414 :param ver: a version number 415 """ 416 if isinstance(ver, str): 417 ver = tuple(ver.split('.')) 418 if isinstance(ver, tuple): 419 ret = 0 420 for i in range(4): 421 if i < len(ver): 422 ret += 256**(3 - i) * int(ver[i]) 423 return ret 424 return ver 425 426def to_list(val): 427 """ 428 Converts a string argument to a list by splitting it by spaces. 429 Returns the object if not a string:: 430 431 from waflib.Utils import to_list 432 lst = to_list('a b c d') 433 434 :param val: list of string or space-separated string 435 :rtype: list 436 :return: Argument converted to list 437 """ 438 if isinstance(val, str): 439 return val.split() 440 else: 441 return val 442 443def console_encoding(): 444 try: 445 import ctypes 446 except ImportError: 447 pass 448 else: 449 try: 450 codepage = ctypes.windll.kernel32.GetConsoleCP() 451 except AttributeError: 452 pass 453 else: 454 if codepage: 455 return 'cp%d' % codepage 456 return sys.stdout.encoding or ('cp1252' if is_win32 else 'latin-1') 457 458def split_path_unix(path): 459 return path.split('/') 460 461def split_path_cygwin(path): 462 if path.startswith('//'): 463 ret = path.split('/')[2:] 464 ret[0] = '/' + ret[0] 465 return ret 466 return path.split('/') 467 468re_sp = re.compile('[/\\\\]+') 469def split_path_win32(path): 470 if path.startswith('\\\\'): 471 ret = re_sp.split(path)[1:] 472 ret[0] = '\\\\' + ret[0] 473 if ret[0] == '\\\\?': 474 return ret[1:] 475 return ret 476 return re_sp.split(path) 477 478msysroot = None 479def split_path_msys(path): 480 if path.startswith(('/', '\\')) and not path.startswith(('//', '\\\\')): 481 # msys paths can be in the form /usr/bin 482 global msysroot 483 if not msysroot: 484 # msys has python 2.7 or 3, so we can use this 485 msysroot = subprocess.check_output(['cygpath', '-w', '/']).decode(sys.stdout.encoding or 'latin-1') 486 msysroot = msysroot.strip() 487 path = os.path.normpath(msysroot + os.sep + path) 488 return split_path_win32(path) 489 490if sys.platform == 'cygwin': 491 split_path = split_path_cygwin 492elif is_win32: 493 # Consider this an MSYSTEM environment if $MSYSTEM is set and python 494 # reports is executable from a unix like path on a windows host. 495 if os.environ.get('MSYSTEM') and sys.executable.startswith('/'): 496 split_path = split_path_msys 497 else: 498 split_path = split_path_win32 499else: 500 split_path = split_path_unix 501 502split_path.__doc__ = """ 503Splits a path by / or \\; do not confuse this function with with ``os.path.split`` 504 505:type path: string 506:param path: path to split 507:return: list of string 508""" 509 510def check_dir(path): 511 """ 512 Ensures that a directory exists (similar to ``mkdir -p``). 513 514 :type path: string 515 :param path: Path to directory 516 :raises: :py:class:`waflib.Errors.WafError` if the folder cannot be added. 517 """ 518 if not os.path.isdir(path): 519 try: 520 os.makedirs(path) 521 except OSError as e: 522 if not os.path.isdir(path): 523 raise Errors.WafError('Cannot create the folder %r' % path, ex=e) 524 525def check_exe(name, env=None): 526 """ 527 Ensures that a program exists 528 529 :type name: string 530 :param name: path to the program 531 :param env: configuration object 532 :type env: :py:class:`waflib.ConfigSet.ConfigSet` 533 :return: path of the program or None 534 :raises: :py:class:`waflib.Errors.WafError` if the folder cannot be added. 535 """ 536 if not name: 537 raise ValueError('Cannot execute an empty string!') 538 def is_exe(fpath): 539 return os.path.isfile(fpath) and os.access(fpath, os.X_OK) 540 541 fpath, fname = os.path.split(name) 542 if fpath and is_exe(name): 543 return os.path.abspath(name) 544 else: 545 env = env or os.environ 546 for path in env['PATH'].split(os.pathsep): 547 path = path.strip('"') 548 exe_file = os.path.join(path, name) 549 if is_exe(exe_file): 550 return os.path.abspath(exe_file) 551 return None 552 553def def_attrs(cls, **kw): 554 """ 555 Sets default attributes on a class instance 556 557 :type cls: class 558 :param cls: the class to update the given attributes in. 559 :type kw: dict 560 :param kw: dictionary of attributes names and values. 561 """ 562 for k, v in kw.items(): 563 if not hasattr(cls, k): 564 setattr(cls, k, v) 565 566def quote_define_name(s): 567 """ 568 Converts a string into an identifier suitable for C defines. 569 570 :type s: string 571 :param s: String to convert 572 :rtype: string 573 :return: Identifier suitable for C defines 574 """ 575 fu = re.sub('[^a-zA-Z0-9]', '_', s) 576 fu = re.sub('_+', '_', fu) 577 fu = fu.upper() 578 return fu 579 580re_sh = re.compile('\\s|\'|"') 581""" 582Regexp used for shell_escape below 583""" 584 585def shell_escape(cmd): 586 """ 587 Escapes a command: 588 ['ls', '-l', 'arg space'] -> ls -l 'arg space' 589 """ 590 if isinstance(cmd, str): 591 return cmd 592 return ' '.join(repr(x) if re_sh.search(x) else x for x in cmd) 593 594def h_list(lst): 595 """ 596 Hashes lists of ordered data. 597 598 Using hash(tup) for tuples would be much more efficient, 599 but Python now enforces hash randomization 600 601 :param lst: list to hash 602 :type lst: list of strings 603 :return: hash of the list 604 """ 605 return md5(repr(lst).encode()).digest() 606 607if sys.hexversion < 0x3000000: 608 def h_list_python2(lst): 609 return md5(repr(lst)).digest() 610 h_list_python2.__doc__ = h_list.__doc__ 611 h_list = h_list_python2 612 613def h_fun(fun): 614 """ 615 Hash functions 616 617 :param fun: function to hash 618 :type fun: function 619 :return: hash of the function 620 :rtype: string or bytes 621 """ 622 try: 623 return fun.code 624 except AttributeError: 625 if isinstance(fun, functools.partial): 626 code = list(fun.args) 627 # The method items() provides a sequence of tuples where the first element 628 # represents an optional argument of the partial function application 629 # 630 # The sorting result outcome will be consistent because: 631 # 1. tuples are compared in order of their elements 632 # 2. optional argument namess are unique 633 code.extend(sorted(fun.keywords.items())) 634 code.append(h_fun(fun.func)) 635 fun.code = h_list(code) 636 return fun.code 637 try: 638 h = inspect.getsource(fun) 639 except EnvironmentError: 640 h = 'nocode' 641 try: 642 fun.code = h 643 except AttributeError: 644 pass 645 return h 646 647def h_cmd(ins): 648 """ 649 Hashes objects recursively 650 651 :param ins: input object 652 :type ins: string or list or tuple or function 653 :rtype: string or bytes 654 """ 655 # this function is not meant to be particularly fast 656 if isinstance(ins, str): 657 # a command is either a string 658 ret = ins 659 elif isinstance(ins, list) or isinstance(ins, tuple): 660 # or a list of functions/strings 661 ret = str([h_cmd(x) for x in ins]) 662 else: 663 # or just a python function 664 ret = str(h_fun(ins)) 665 if sys.hexversion > 0x3000000: 666 ret = ret.encode('latin-1', 'xmlcharrefreplace') 667 return ret 668 669reg_subst = re.compile(r"(\\\\)|(\$\$)|\$\{([^}]+)\}") 670def subst_vars(expr, params): 671 """ 672 Replaces ${VAR} with the value of VAR taken from a dict or a config set:: 673 674 from waflib import Utils 675 s = Utils.subst_vars('${PREFIX}/bin', env) 676 677 :type expr: string 678 :param expr: String to perform substitution on 679 :param params: Dictionary or config set to look up variable values. 680 """ 681 def repl_var(m): 682 if m.group(1): 683 return '\\' 684 if m.group(2): 685 return '$' 686 try: 687 # ConfigSet instances may contain lists 688 return params.get_flat(m.group(3)) 689 except AttributeError: 690 return params[m.group(3)] 691 # if you get a TypeError, it means that 'expr' is not a string... 692 # Utils.subst_vars(None, env) will not work 693 return reg_subst.sub(repl_var, expr) 694 695def destos_to_binfmt(key): 696 """ 697 Returns the binary format based on the unversioned platform name, 698 and defaults to ``elf`` if nothing is found. 699 700 :param key: platform name 701 :type key: string 702 :return: string representing the binary format 703 """ 704 if key == 'darwin': 705 return 'mac-o' 706 elif key in ('win32', 'cygwin', 'uwin', 'msys'): 707 return 'pe' 708 return 'elf' 709 710def unversioned_sys_platform(): 711 """ 712 Returns the unversioned platform name. 713 Some Python platform names contain versions, that depend on 714 the build environment, e.g. linux2, freebsd6, etc. 715 This returns the name without the version number. Exceptions are 716 os2 and win32, which are returned verbatim. 717 718 :rtype: string 719 :return: Unversioned platform name 720 """ 721 s = sys.platform 722 if s.startswith('java'): 723 # The real OS is hidden under the JVM. 724 from java.lang import System 725 s = System.getProperty('os.name') 726 # see http://lopica.sourceforge.net/os.html for a list of possible values 727 if s == 'Mac OS X': 728 return 'darwin' 729 elif s.startswith('Windows '): 730 return 'win32' 731 elif s == 'OS/2': 732 return 'os2' 733 elif s == 'HP-UX': 734 return 'hp-ux' 735 elif s in ('SunOS', 'Solaris'): 736 return 'sunos' 737 else: s = s.lower() 738 739 # powerpc == darwin for our purposes 740 if s == 'powerpc': 741 return 'darwin' 742 if s == 'win32' or s == 'os2': 743 return s 744 if s == 'cli' and os.name == 'nt': 745 # ironpython is only on windows as far as we know 746 return 'win32' 747 return re.split(r'\d+$', s)[0] 748 749def nada(*k, **kw): 750 """ 751 Does nothing 752 753 :return: None 754 """ 755 pass 756 757class Timer(object): 758 """ 759 Simple object for timing the execution of commands. 760 Its string representation is the duration:: 761 762 from waflib.Utils import Timer 763 timer = Timer() 764 a_few_operations() 765 s = str(timer) 766 """ 767 def __init__(self): 768 self.start_time = self.now() 769 770 def __str__(self): 771 delta = self.now() - self.start_time 772 if not isinstance(delta, datetime.timedelta): 773 delta = datetime.timedelta(seconds=delta) 774 days = delta.days 775 hours, rem = divmod(delta.seconds, 3600) 776 minutes, seconds = divmod(rem, 60) 777 seconds += delta.microseconds * 1e-6 778 result = '' 779 if days: 780 result += '%dd' % days 781 if days or hours: 782 result += '%dh' % hours 783 if days or hours or minutes: 784 result += '%dm' % minutes 785 return '%s%.3fs' % (result, seconds) 786 787 def now(self): 788 return datetime.datetime.utcnow() 789 790 if hasattr(time, 'perf_counter'): 791 def now(self): 792 return time.perf_counter() 793 794def read_la_file(path): 795 """ 796 Reads property files, used by msvc.py 797 798 :param path: file to read 799 :type path: string 800 """ 801 sp = re.compile(r'^([^=]+)=\'(.*)\'$') 802 dc = {} 803 for line in readf(path).splitlines(): 804 try: 805 _, left, right, _ = sp.split(line.strip()) 806 dc[left] = right 807 except ValueError: 808 pass 809 return dc 810 811def run_once(fun): 812 """ 813 Decorator: let a function cache its results, use like this:: 814 815 @run_once 816 def foo(k): 817 return 345*2343 818 819 .. note:: in practice this can cause memory leaks, prefer a :py:class:`waflib.Utils.lru_cache` 820 821 :param fun: function to execute 822 :type fun: function 823 :return: the return value of the function executed 824 """ 825 cache = {} 826 def wrap(*k): 827 try: 828 return cache[k] 829 except KeyError: 830 ret = fun(*k) 831 cache[k] = ret 832 return ret 833 wrap.__cache__ = cache 834 wrap.__name__ = fun.__name__ 835 return wrap 836 837def get_registry_app_path(key, filename): 838 """ 839 Returns the value of a registry key for an executable 840 841 :type key: string 842 :type filename: list of string 843 """ 844 if not winreg: 845 return None 846 try: 847 result = winreg.QueryValue(key, "Software\\Microsoft\\Windows\\CurrentVersion\\App Paths\\%s.exe" % filename[0]) 848 except OSError: 849 pass 850 else: 851 if os.path.isfile(result): 852 return result 853 854def lib64(): 855 """ 856 Guess the default ``/usr/lib`` extension for 64-bit applications 857 858 :return: '64' or '' 859 :rtype: string 860 """ 861 # default settings for /usr/lib 862 if os.sep == '/': 863 if platform.architecture()[0] == '64bit': 864 if os.path.exists('/usr/lib64') and not os.path.exists('/usr/lib32'): 865 return '64' 866 return '' 867 868def sane_path(p): 869 # private function for the time being! 870 return os.path.abspath(os.path.expanduser(p)) 871 872process_pool = [] 873""" 874List of processes started to execute sub-process commands 875""" 876 877def get_process(): 878 """ 879 Returns a process object that can execute commands as sub-processes 880 881 :rtype: subprocess.Popen 882 """ 883 try: 884 return process_pool.pop() 885 except IndexError: 886 filepath = os.path.dirname(os.path.abspath(__file__)) + os.sep + 'processor.py' 887 cmd = [sys.executable, '-c', readf(filepath)] 888 return subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, bufsize=0, close_fds=not is_win32) 889 890def run_prefork_process(cmd, kwargs, cargs): 891 """ 892 Delegates process execution to a pre-forked process instance. 893 """ 894 if not 'env' in kwargs: 895 kwargs['env'] = dict(os.environ) 896 try: 897 obj = base64.b64encode(cPickle.dumps([cmd, kwargs, cargs])) 898 except (TypeError, AttributeError): 899 return run_regular_process(cmd, kwargs, cargs) 900 901 proc = get_process() 902 if not proc: 903 return run_regular_process(cmd, kwargs, cargs) 904 905 proc.stdin.write(obj) 906 proc.stdin.write('\n'.encode()) 907 proc.stdin.flush() 908 obj = proc.stdout.readline() 909 if not obj: 910 raise OSError('Preforked sub-process %r died' % proc.pid) 911 912 process_pool.append(proc) 913 lst = cPickle.loads(base64.b64decode(obj)) 914 # Jython wrapper failures (bash/execvp) 915 assert len(lst) == 5 916 ret, out, err, ex, trace = lst 917 if ex: 918 if ex == 'OSError': 919 raise OSError(trace) 920 elif ex == 'ValueError': 921 raise ValueError(trace) 922 elif ex == 'TimeoutExpired': 923 exc = TimeoutExpired(cmd, timeout=cargs['timeout'], output=out) 924 exc.stderr = err 925 raise exc 926 else: 927 raise Exception(trace) 928 return ret, out, err 929 930def lchown(path, user=-1, group=-1): 931 """ 932 Change the owner/group of a path, raises an OSError if the 933 ownership change fails. 934 935 :param user: user to change 936 :type user: int or str 937 :param group: group to change 938 :type group: int or str 939 """ 940 if isinstance(user, str): 941 import pwd 942 entry = pwd.getpwnam(user) 943 if not entry: 944 raise OSError('Unknown user %r' % user) 945 user = entry[2] 946 if isinstance(group, str): 947 import grp 948 entry = grp.getgrnam(group) 949 if not entry: 950 raise OSError('Unknown group %r' % group) 951 group = entry[2] 952 return os.lchown(path, user, group) 953 954def run_regular_process(cmd, kwargs, cargs={}): 955 """ 956 Executes a subprocess command by using subprocess.Popen 957 """ 958 proc = subprocess.Popen(cmd, **kwargs) 959 if kwargs.get('stdout') or kwargs.get('stderr'): 960 try: 961 out, err = proc.communicate(**cargs) 962 except TimeoutExpired: 963 if kwargs.get('start_new_session') and hasattr(os, 'killpg'): 964 os.killpg(proc.pid, signal.SIGKILL) 965 else: 966 proc.kill() 967 out, err = proc.communicate() 968 exc = TimeoutExpired(proc.args, timeout=cargs['timeout'], output=out) 969 exc.stderr = err 970 raise exc 971 status = proc.returncode 972 else: 973 out, err = (None, None) 974 try: 975 status = proc.wait(**cargs) 976 except TimeoutExpired as e: 977 if kwargs.get('start_new_session') and hasattr(os, 'killpg'): 978 os.killpg(proc.pid, signal.SIGKILL) 979 else: 980 proc.kill() 981 proc.wait() 982 raise e 983 return status, out, err 984 985def run_process(cmd, kwargs, cargs={}): 986 """ 987 Executes a subprocess by using a pre-forked process when possible 988 or falling back to subprocess.Popen. See :py:func:`waflib.Utils.run_prefork_process` 989 and :py:func:`waflib.Utils.run_regular_process` 990 """ 991 if kwargs.get('stdout') and kwargs.get('stderr'): 992 return run_prefork_process(cmd, kwargs, cargs) 993 else: 994 return run_regular_process(cmd, kwargs, cargs) 995 996def alloc_process_pool(n, force=False): 997 """ 998 Allocates an amount of processes to the default pool so its size is at least *n*. 999 It is useful to call this function early so that the pre-forked 1000 processes use as little memory as possible. 1001 1002 :param n: pool size 1003 :type n: integer 1004 :param force: if True then *n* more processes are added to the existing pool 1005 :type force: bool 1006 """ 1007 # mandatory on python2, unnecessary on python >= 3.2 1008 global run_process, get_process, alloc_process_pool 1009 if not force: 1010 n = max(n - len(process_pool), 0) 1011 try: 1012 lst = [get_process() for x in range(n)] 1013 except OSError: 1014 run_process = run_regular_process 1015 get_process = alloc_process_pool = nada 1016 else: 1017 for x in lst: 1018 process_pool.append(x) 1019 1020def atexit_pool(): 1021 for k in process_pool: 1022 try: 1023 os.kill(k.pid, 9) 1024 except OSError: 1025 pass 1026 else: 1027 k.wait() 1028# see #1889 1029if (sys.hexversion<0x207000f and not is_win32) or sys.hexversion>=0x306000f: 1030 atexit.register(atexit_pool) 1031 1032if os.environ.get('WAF_NO_PREFORK') or sys.platform == 'cli' or not sys.executable: 1033 run_process = run_regular_process 1034 get_process = alloc_process_pool = nada 1035 1036