1#!/usr/bin/env python 2# encoding: utf-8 3# Thomas Nagy, 2005-2018 (ita) 4 5""" 6Utilities and platform-specific fixes 7 8The portability fixes try to provide a consistent behavior of the Waf API 9through Python versions 2.5 to 3.X and across different platforms (win32, linux, etc) 10""" 11 12from __future__ import with_statement 13 14import atexit, os, sys, errno, inspect, re, datetime, platform, base64, signal, functools, time 15 16try: 17 import cPickle 18except ImportError: 19 import pickle as cPickle 20 21# leave this 22if os.name == 'posix' and sys.version_info[0] < 3: 23 try: 24 import subprocess32 as subprocess 25 except ImportError: 26 import subprocess 27else: 28 import subprocess 29 30try: 31 TimeoutExpired = subprocess.TimeoutExpired 32except AttributeError: 33 class TimeoutExpired(Exception): 34 pass 35 36from collections import deque, defaultdict 37 38try: 39 import _winreg as winreg 40except ImportError: 41 try: 42 import winreg 43 except ImportError: 44 winreg = None 45 46from waflib import Errors 47 48try: 49 from hashlib import md5 50except ImportError: 51 try: 52 from md5 import md5 53 except ImportError: 54 # never fail to enable fixes from another module 55 pass 56 57try: 58 import threading 59except ImportError: 60 if not 'JOBS' in os.environ: 61 # no threading :-( 62 os.environ['JOBS'] = '1' 63 64 class threading(object): 65 """ 66 A fake threading class for platforms lacking the threading module. 67 Use ``waf -j1`` on those platforms 68 """ 69 pass 70 class Lock(object): 71 """Fake Lock class""" 72 def acquire(self): 73 pass 74 def release(self): 75 pass 76 threading.Lock = threading.Thread = Lock 77 78SIG_NIL = 'SIG_NIL_SIG_NIL_'.encode() 79"""Arbitrary null value for hashes. Modify this value according to the hash function in use""" 80 81O644 = 420 82"""Constant representing the permissions for regular files (0644 raises a syntax error on python 3)""" 83 84O755 = 493 85"""Constant representing the permissions for executable files (0755 raises a syntax error on python 3)""" 86 87rot_chr = ['\\', '|', '/', '-'] 88"List of characters to use when displaying the throbber (progress bar)" 89 90rot_idx = 0 91"Index of the current throbber character (progress bar)" 92 93class ordered_iter_dict(dict): 94 """Ordered dictionary that provides iteration from the most recently inserted keys first""" 95 def __init__(self, *k, **kw): 96 self.lst = deque() 97 dict.__init__(self, *k, **kw) 98 def clear(self): 99 dict.clear(self) 100 self.lst = deque() 101 def __setitem__(self, key, value): 102 if key in dict.keys(self): 103 self.lst.remove(key) 104 dict.__setitem__(self, key, value) 105 self.lst.append(key) 106 def __delitem__(self, key): 107 dict.__delitem__(self, key) 108 try: 109 self.lst.remove(key) 110 except ValueError: 111 pass 112 def __iter__(self): 113 return reversed(self.lst) 114 def keys(self): 115 return reversed(self.lst) 116 117class lru_node(object): 118 """ 119 Used by :py:class:`waflib.Utils.lru_cache` 120 """ 121 __slots__ = ('next', 'prev', 'key', 'val') 122 def __init__(self): 123 self.next = self 124 self.prev = self 125 self.key = None 126 self.val = None 127 128class lru_cache(object): 129 """ 130 A simple least-recently used cache with lazy allocation 131 """ 132 __slots__ = ('maxlen', 'table', 'head') 133 def __init__(self, maxlen=100): 134 self.maxlen = maxlen 135 """ 136 Maximum amount of elements in the cache 137 """ 138 self.table = {} 139 """ 140 Mapping key-value 141 """ 142 self.head = lru_node() 143 self.head.next = self.head 144 self.head.prev = self.head 145 146 def __getitem__(self, key): 147 node = self.table[key] 148 # assert(key==node.key) 149 if node is self.head: 150 return node.val 151 152 # detach the node found 153 node.prev.next = node.next 154 node.next.prev = node.prev 155 156 # replace the head 157 node.next = self.head.next 158 node.prev = self.head 159 self.head = node.next.prev = node.prev.next = node 160 161 return node.val 162 163 def __setitem__(self, key, val): 164 if key in self.table: 165 # update the value for an existing key 166 node = self.table[key] 167 node.val = val 168 self.__getitem__(key) 169 else: 170 if len(self.table) < self.maxlen: 171 # the very first item is unused until the maximum is reached 172 node = lru_node() 173 node.prev = self.head 174 node.next = self.head.next 175 node.prev.next = node.next.prev = node 176 else: 177 node = self.head = self.head.next 178 try: 179 # that's another key 180 del self.table[node.key] 181 except KeyError: 182 pass 183 184 node.key = key 185 node.val = val 186 self.table[key] = node 187 188class lazy_generator(object): 189 def __init__(self, fun, params): 190 self.fun = fun 191 self.params = params 192 193 def __iter__(self): 194 return self 195 196 def __next__(self): 197 try: 198 it = self.it 199 except AttributeError: 200 it = self.it = self.fun(*self.params) 201 return next(it) 202 203 next = __next__ 204 205is_win32 = os.sep == '\\' or sys.platform == 'win32' # msys2 206""" 207Whether this system is a Windows series 208""" 209 210def readf(fname, m='r', encoding='latin-1'): 211 """ 212 Reads an entire file into a string. See also :py:meth:`waflib.Node.Node.readf`:: 213 214 def build(ctx): 215 from waflib import Utils 216 txt = Utils.readf(self.path.find_node('wscript').abspath()) 217 txt = ctx.path.find_node('wscript').read() 218 219 :type fname: string 220 :param fname: Path to file 221 :type m: string 222 :param m: Open mode 223 :type encoding: string 224 :param encoding: encoding value, only used for python 3 225 :rtype: string 226 :return: Content of the file 227 """ 228 229 if sys.hexversion > 0x3000000 and not 'b' in m: 230 m += 'b' 231 with open(fname, m) as f: 232 txt = f.read() 233 if encoding: 234 txt = txt.decode(encoding) 235 else: 236 txt = txt.decode() 237 else: 238 with open(fname, m) as f: 239 txt = f.read() 240 return txt 241 242def writef(fname, data, m='w', encoding='latin-1'): 243 """ 244 Writes an entire file from a string. 245 See also :py:meth:`waflib.Node.Node.writef`:: 246 247 def build(ctx): 248 from waflib import Utils 249 txt = Utils.writef(self.path.make_node('i_like_kittens').abspath(), 'some data') 250 self.path.make_node('i_like_kittens').write('some data') 251 252 :type fname: string 253 :param fname: Path to file 254 :type data: string 255 :param data: The contents to write to the file 256 :type m: string 257 :param m: Open mode 258 :type encoding: string 259 :param encoding: encoding value, only used for python 3 260 """ 261 if sys.hexversion > 0x3000000 and not 'b' in m: 262 data = data.encode(encoding) 263 m += 'b' 264 with open(fname, m) as f: 265 f.write(data) 266 267def h_file(fname): 268 """ 269 Computes a hash value for a file by using md5. Use the md5_tstamp 270 extension to get faster build hashes if necessary. 271 272 :type fname: string 273 :param fname: path to the file to hash 274 :return: hash of the file contents 275 :rtype: string or bytes 276 """ 277 m = md5() 278 with open(fname, 'rb') as f: 279 while fname: 280 fname = f.read(200000) 281 m.update(fname) 282 return m.digest() 283 284def readf_win32(f, m='r', encoding='latin-1'): 285 flags = os.O_NOINHERIT | os.O_RDONLY 286 if 'b' in m: 287 flags |= os.O_BINARY 288 if '+' in m: 289 flags |= os.O_RDWR 290 try: 291 fd = os.open(f, flags) 292 except OSError: 293 raise IOError('Cannot read from %r' % f) 294 295 if sys.hexversion > 0x3000000 and not 'b' in m: 296 m += 'b' 297 with os.fdopen(fd, m) as f: 298 txt = f.read() 299 if encoding: 300 txt = txt.decode(encoding) 301 else: 302 txt = txt.decode() 303 else: 304 with os.fdopen(fd, m) as f: 305 txt = f.read() 306 return txt 307 308def writef_win32(f, data, m='w', encoding='latin-1'): 309 if sys.hexversion > 0x3000000 and not 'b' in m: 310 data = data.encode(encoding) 311 m += 'b' 312 flags = os.O_CREAT | os.O_TRUNC | os.O_WRONLY | os.O_NOINHERIT 313 if 'b' in m: 314 flags |= os.O_BINARY 315 if '+' in m: 316 flags |= os.O_RDWR 317 try: 318 fd = os.open(f, flags) 319 except OSError: 320 raise OSError('Cannot write to %r' % f) 321 with os.fdopen(fd, m) as f: 322 f.write(data) 323 324def h_file_win32(fname): 325 try: 326 fd = os.open(fname, os.O_BINARY | os.O_RDONLY | os.O_NOINHERIT) 327 except OSError: 328 raise OSError('Cannot read from %r' % fname) 329 m = md5() 330 with os.fdopen(fd, 'rb') as f: 331 while fname: 332 fname = f.read(200000) 333 m.update(fname) 334 return m.digest() 335 336# always save these 337readf_unix = readf 338writef_unix = writef 339h_file_unix = h_file 340if hasattr(os, 'O_NOINHERIT') and sys.hexversion < 0x3040000: 341 # replace the default functions 342 readf = readf_win32 343 writef = writef_win32 344 h_file = h_file_win32 345 346try: 347 x = ''.encode('hex') 348except LookupError: 349 import binascii 350 def to_hex(s): 351 ret = binascii.hexlify(s) 352 if not isinstance(ret, str): 353 ret = ret.decode('utf-8') 354 return ret 355else: 356 def to_hex(s): 357 return s.encode('hex') 358 359to_hex.__doc__ = """ 360Return the hexadecimal representation of a string 361 362:param s: string to convert 363:type s: string 364""" 365 366def listdir_win32(s): 367 """ 368 Lists the contents of a folder in a portable manner. 369 On Win32, returns the list of drive letters: ['C:', 'X:', 'Z:'] when an empty string is given. 370 371 :type s: string 372 :param s: a string, which can be empty on Windows 373 """ 374 if not s: 375 try: 376 import ctypes 377 except ImportError: 378 # there is nothing much we can do 379 return [x + ':\\' for x in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'] 380 else: 381 dlen = 4 # length of "?:\\x00" 382 maxdrives = 26 383 buf = ctypes.create_string_buffer(maxdrives * dlen) 384 ndrives = ctypes.windll.kernel32.GetLogicalDriveStringsA(maxdrives*dlen, ctypes.byref(buf)) 385 return [ str(buf.raw[4*i:4*i+2].decode('ascii')) for i in range(int(ndrives/dlen)) ] 386 387 if len(s) == 2 and s[1] == ":": 388 s += os.sep 389 390 if not os.path.isdir(s): 391 e = OSError('%s is not a directory' % s) 392 e.errno = errno.ENOENT 393 raise e 394 return os.listdir(s) 395 396listdir = os.listdir 397if is_win32: 398 listdir = listdir_win32 399 400def num2ver(ver): 401 """ 402 Converts a string, tuple or version number into an integer. The number is supposed to have at most 4 digits:: 403 404 from waflib.Utils import num2ver 405 num2ver('1.3.2') == num2ver((1,3,2)) == num2ver((1,3,2,0)) 406 407 :type ver: string or tuple of numbers 408 :param ver: a version number 409 """ 410 if isinstance(ver, str): 411 ver = tuple(ver.split('.')) 412 if isinstance(ver, tuple): 413 ret = 0 414 for i in range(4): 415 if i < len(ver): 416 ret += 256**(3 - i) * int(ver[i]) 417 return ret 418 return ver 419 420def to_list(val): 421 """ 422 Converts a string argument to a list by splitting it by spaces. 423 Returns the object if not a string:: 424 425 from waflib.Utils import to_list 426 lst = to_list('a b c d') 427 428 :param val: list of string or space-separated string 429 :rtype: list 430 :return: Argument converted to list 431 """ 432 if isinstance(val, str): 433 return val.split() 434 else: 435 return val 436 437def console_encoding(): 438 try: 439 import ctypes 440 except ImportError: 441 pass 442 else: 443 try: 444 codepage = ctypes.windll.kernel32.GetConsoleCP() 445 except AttributeError: 446 pass 447 else: 448 if codepage: 449 return 'cp%d' % codepage 450 return sys.stdout.encoding or ('cp1252' if is_win32 else 'latin-1') 451 452def split_path_unix(path): 453 return path.split('/') 454 455def split_path_cygwin(path): 456 if path.startswith('//'): 457 ret = path.split('/')[2:] 458 ret[0] = '/' + ret[0] 459 return ret 460 return path.split('/') 461 462re_sp = re.compile('[/\\\\]+') 463def split_path_win32(path): 464 if path.startswith('\\\\'): 465 ret = re_sp.split(path)[1:] 466 ret[0] = '\\\\' + ret[0] 467 if ret[0] == '\\\\?': 468 return ret[1:] 469 return ret 470 return re_sp.split(path) 471 472msysroot = None 473def split_path_msys(path): 474 if path.startswith(('/', '\\')) and not path.startswith(('//', '\\\\')): 475 # msys paths can be in the form /usr/bin 476 global msysroot 477 if not msysroot: 478 # msys has python 2.7 or 3, so we can use this 479 msysroot = subprocess.check_output(['cygpath', '-w', '/']).decode(sys.stdout.encoding or 'latin-1') 480 msysroot = msysroot.strip() 481 path = os.path.normpath(msysroot + os.sep + path) 482 return split_path_win32(path) 483 484if sys.platform == 'cygwin': 485 split_path = split_path_cygwin 486elif is_win32: 487 if os.environ.get('MSYSTEM'): 488 split_path = split_path_msys 489 else: 490 split_path = split_path_win32 491else: 492 split_path = split_path_unix 493 494split_path.__doc__ = """ 495Splits a path by / or \\; do not confuse this function with with ``os.path.split`` 496 497:type path: string 498:param path: path to split 499:return: list of string 500""" 501 502def check_dir(path): 503 """ 504 Ensures that a directory exists (similar to ``mkdir -p``). 505 506 :type path: string 507 :param path: Path to directory 508 :raises: :py:class:`waflib.Errors.WafError` if the folder cannot be added. 509 """ 510 if not os.path.isdir(path): 511 try: 512 os.makedirs(path) 513 except OSError as e: 514 if not os.path.isdir(path): 515 raise Errors.WafError('Cannot create the folder %r' % path, ex=e) 516 517def check_exe(name, env=None): 518 """ 519 Ensures that a program exists 520 521 :type name: string 522 :param name: path to the program 523 :param env: configuration object 524 :type env: :py:class:`waflib.ConfigSet.ConfigSet` 525 :return: path of the program or None 526 :raises: :py:class:`waflib.Errors.WafError` if the folder cannot be added. 527 """ 528 if not name: 529 raise ValueError('Cannot execute an empty string!') 530 def is_exe(fpath): 531 return os.path.isfile(fpath) and os.access(fpath, os.X_OK) 532 533 fpath, fname = os.path.split(name) 534 if fpath and is_exe(name): 535 return os.path.abspath(name) 536 else: 537 env = env or os.environ 538 for path in env['PATH'].split(os.pathsep): 539 path = path.strip('"') 540 exe_file = os.path.join(path, name) 541 if is_exe(exe_file): 542 return os.path.abspath(exe_file) 543 return None 544 545def def_attrs(cls, **kw): 546 """ 547 Sets default attributes on a class instance 548 549 :type cls: class 550 :param cls: the class to update the given attributes in. 551 :type kw: dict 552 :param kw: dictionary of attributes names and values. 553 """ 554 for k, v in kw.items(): 555 if not hasattr(cls, k): 556 setattr(cls, k, v) 557 558def quote_define_name(s): 559 """ 560 Converts a string into an identifier suitable for C defines. 561 562 :type s: string 563 :param s: String to convert 564 :rtype: string 565 :return: Identifier suitable for C defines 566 """ 567 fu = re.sub('[^a-zA-Z0-9]', '_', s) 568 fu = re.sub('_+', '_', fu) 569 fu = fu.upper() 570 return fu 571 572re_sh = re.compile('\\s|\'|"') 573""" 574Regexp used for shell_escape below 575""" 576 577def shell_escape(cmd): 578 """ 579 Escapes a command: 580 ['ls', '-l', 'arg space'] -> ls -l 'arg space' 581 """ 582 if isinstance(cmd, str): 583 return cmd 584 return ' '.join(repr(x) if re_sh.search(x) else x for x in cmd) 585 586def h_list(lst): 587 """ 588 Hashes lists of ordered data. 589 590 Using hash(tup) for tuples would be much more efficient, 591 but Python now enforces hash randomization 592 593 :param lst: list to hash 594 :type lst: list of strings 595 :return: hash of the list 596 """ 597 return md5(repr(lst).encode()).digest() 598 599def h_fun(fun): 600 """ 601 Hash functions 602 603 :param fun: function to hash 604 :type fun: function 605 :return: hash of the function 606 :rtype: string or bytes 607 """ 608 try: 609 return fun.code 610 except AttributeError: 611 if isinstance(fun, functools.partial): 612 code = list(fun.args) 613 # The method items() provides a sequence of tuples where the first element 614 # represents an optional argument of the partial function application 615 # 616 # The sorting result outcome will be consistent because: 617 # 1. tuples are compared in order of their elements 618 # 2. optional argument namess are unique 619 code.extend(sorted(fun.keywords.items())) 620 code.append(h_fun(fun.func)) 621 fun.code = h_list(code) 622 return fun.code 623 try: 624 h = inspect.getsource(fun) 625 except EnvironmentError: 626 h = 'nocode' 627 try: 628 fun.code = h 629 except AttributeError: 630 pass 631 return h 632 633def h_cmd(ins): 634 """ 635 Hashes objects recursively 636 637 :param ins: input object 638 :type ins: string or list or tuple or function 639 :rtype: string or bytes 640 """ 641 # this function is not meant to be particularly fast 642 if isinstance(ins, str): 643 # a command is either a string 644 ret = ins 645 elif isinstance(ins, list) or isinstance(ins, tuple): 646 # or a list of functions/strings 647 ret = str([h_cmd(x) for x in ins]) 648 else: 649 # or just a python function 650 ret = str(h_fun(ins)) 651 if sys.hexversion > 0x3000000: 652 ret = ret.encode('latin-1', 'xmlcharrefreplace') 653 return ret 654 655reg_subst = re.compile(r"(\\\\)|(\$\$)|\$\{([^}]+)\}") 656def subst_vars(expr, params): 657 """ 658 Replaces ${VAR} with the value of VAR taken from a dict or a config set:: 659 660 from waflib import Utils 661 s = Utils.subst_vars('${PREFIX}/bin', env) 662 663 :type expr: string 664 :param expr: String to perform substitution on 665 :param params: Dictionary or config set to look up variable values. 666 """ 667 def repl_var(m): 668 if m.group(1): 669 return '\\' 670 if m.group(2): 671 return '$' 672 try: 673 # ConfigSet instances may contain lists 674 return params.get_flat(m.group(3)) 675 except AttributeError: 676 return params[m.group(3)] 677 # if you get a TypeError, it means that 'expr' is not a string... 678 # Utils.subst_vars(None, env) will not work 679 return reg_subst.sub(repl_var, expr) 680 681def destos_to_binfmt(key): 682 """ 683 Returns the binary format based on the unversioned platform name, 684 and defaults to ``elf`` if nothing is found. 685 686 :param key: platform name 687 :type key: string 688 :return: string representing the binary format 689 """ 690 if key == 'darwin': 691 return 'mac-o' 692 elif key in ('win32', 'cygwin', 'uwin', 'msys'): 693 return 'pe' 694 return 'elf' 695 696def unversioned_sys_platform(): 697 """ 698 Returns the unversioned platform name. 699 Some Python platform names contain versions, that depend on 700 the build environment, e.g. linux2, freebsd6, etc. 701 This returns the name without the version number. Exceptions are 702 os2 and win32, which are returned verbatim. 703 704 :rtype: string 705 :return: Unversioned platform name 706 """ 707 s = sys.platform 708 if s.startswith('java'): 709 # The real OS is hidden under the JVM. 710 from java.lang import System 711 s = System.getProperty('os.name') 712 # see http://lopica.sourceforge.net/os.html for a list of possible values 713 if s == 'Mac OS X': 714 return 'darwin' 715 elif s.startswith('Windows '): 716 return 'win32' 717 elif s == 'OS/2': 718 return 'os2' 719 elif s == 'HP-UX': 720 return 'hp-ux' 721 elif s in ('SunOS', 'Solaris'): 722 return 'sunos' 723 else: s = s.lower() 724 725 # powerpc == darwin for our purposes 726 if s == 'powerpc': 727 return 'darwin' 728 if s == 'win32' or s == 'os2': 729 return s 730 if s == 'cli' and os.name == 'nt': 731 # ironpython is only on windows as far as we know 732 return 'win32' 733 return re.split('\d+$', s)[0] 734 735def nada(*k, **kw): 736 """ 737 Does nothing 738 739 :return: None 740 """ 741 pass 742 743class Timer(object): 744 """ 745 Simple object for timing the execution of commands. 746 Its string representation is the duration:: 747 748 from waflib.Utils import Timer 749 timer = Timer() 750 a_few_operations() 751 s = str(timer) 752 """ 753 def __init__(self): 754 self.start_time = self.now() 755 756 def __str__(self): 757 delta = self.now() - self.start_time 758 if not isinstance(delta, datetime.timedelta): 759 delta = datetime.timedelta(seconds=delta) 760 days = delta.days 761 hours, rem = divmod(delta.seconds, 3600) 762 minutes, seconds = divmod(rem, 60) 763 seconds += delta.microseconds * 1e-6 764 result = '' 765 if days: 766 result += '%dd' % days 767 if days or hours: 768 result += '%dh' % hours 769 if days or hours or minutes: 770 result += '%dm' % minutes 771 return '%s%.3fs' % (result, seconds) 772 773 def now(self): 774 return datetime.datetime.utcnow() 775 776 if hasattr(time, 'perf_counter'): 777 def now(self): 778 return time.perf_counter() 779 780def read_la_file(path): 781 """ 782 Reads property files, used by msvc.py 783 784 :param path: file to read 785 :type path: string 786 """ 787 sp = re.compile(r'^([^=]+)=\'(.*)\'$') 788 dc = {} 789 for line in readf(path).splitlines(): 790 try: 791 _, left, right, _ = sp.split(line.strip()) 792 dc[left] = right 793 except ValueError: 794 pass 795 return dc 796 797def run_once(fun): 798 """ 799 Decorator: let a function cache its results, use like this:: 800 801 @run_once 802 def foo(k): 803 return 345*2343 804 805 .. note:: in practice this can cause memory leaks, prefer a :py:class:`waflib.Utils.lru_cache` 806 807 :param fun: function to execute 808 :type fun: function 809 :return: the return value of the function executed 810 """ 811 cache = {} 812 def wrap(*k): 813 try: 814 return cache[k] 815 except KeyError: 816 ret = fun(*k) 817 cache[k] = ret 818 return ret 819 wrap.__cache__ = cache 820 wrap.__name__ = fun.__name__ 821 return wrap 822 823def get_registry_app_path(key, filename): 824 """ 825 Returns the value of a registry key for an executable 826 827 :type key: string 828 :type filename: list of string 829 """ 830 if not winreg: 831 return None 832 try: 833 result = winreg.QueryValue(key, "Software\\Microsoft\\Windows\\CurrentVersion\\App Paths\\%s.exe" % filename[0]) 834 except OSError: 835 pass 836 else: 837 if os.path.isfile(result): 838 return result 839 840def lib64(): 841 """ 842 Guess the default ``/usr/lib`` extension for 64-bit applications 843 844 :return: '64' or '' 845 :rtype: string 846 """ 847 # default settings for /usr/lib 848 if os.sep == '/': 849 if platform.architecture()[0] == '64bit': 850 if os.path.exists('/usr/lib64') and not os.path.exists('/usr/lib32'): 851 return '64' 852 return '' 853 854def sane_path(p): 855 # private function for the time being! 856 return os.path.abspath(os.path.expanduser(p)) 857 858process_pool = [] 859""" 860List of processes started to execute sub-process commands 861""" 862 863def get_process(): 864 """ 865 Returns a process object that can execute commands as sub-processes 866 867 :rtype: subprocess.Popen 868 """ 869 try: 870 return process_pool.pop() 871 except IndexError: 872 filepath = os.path.dirname(os.path.abspath(__file__)) + os.sep + 'processor.py' 873 cmd = [sys.executable, '-c', readf(filepath)] 874 return subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, bufsize=0) 875 876def run_prefork_process(cmd, kwargs, cargs): 877 """ 878 Delegates process execution to a pre-forked process instance. 879 """ 880 if not 'env' in kwargs: 881 kwargs['env'] = dict(os.environ) 882 try: 883 obj = base64.b64encode(cPickle.dumps([cmd, kwargs, cargs])) 884 except (TypeError, AttributeError): 885 return run_regular_process(cmd, kwargs, cargs) 886 887 proc = get_process() 888 if not proc: 889 return run_regular_process(cmd, kwargs, cargs) 890 891 proc.stdin.write(obj) 892 proc.stdin.write('\n'.encode()) 893 proc.stdin.flush() 894 obj = proc.stdout.readline() 895 if not obj: 896 raise OSError('Preforked sub-process %r died' % proc.pid) 897 898 process_pool.append(proc) 899 lst = cPickle.loads(base64.b64decode(obj)) 900 # Jython wrapper failures (bash/execvp) 901 assert len(lst) == 5 902 ret, out, err, ex, trace = lst 903 if ex: 904 if ex == 'OSError': 905 raise OSError(trace) 906 elif ex == 'ValueError': 907 raise ValueError(trace) 908 elif ex == 'TimeoutExpired': 909 exc = TimeoutExpired(cmd, timeout=cargs['timeout'], output=out) 910 exc.stderr = err 911 raise exc 912 else: 913 raise Exception(trace) 914 return ret, out, err 915 916def lchown(path, user=-1, group=-1): 917 """ 918 Change the owner/group of a path, raises an OSError if the 919 ownership change fails. 920 921 :param user: user to change 922 :type user: int or str 923 :param group: group to change 924 :type group: int or str 925 """ 926 if isinstance(user, str): 927 import pwd 928 entry = pwd.getpwnam(user) 929 if not entry: 930 raise OSError('Unknown user %r' % user) 931 user = entry[2] 932 if isinstance(group, str): 933 import grp 934 entry = grp.getgrnam(group) 935 if not entry: 936 raise OSError('Unknown group %r' % group) 937 group = entry[2] 938 return os.lchown(path, user, group) 939 940def run_regular_process(cmd, kwargs, cargs={}): 941 """ 942 Executes a subprocess command by using subprocess.Popen 943 """ 944 proc = subprocess.Popen(cmd, **kwargs) 945 if kwargs.get('stdout') or kwargs.get('stderr'): 946 try: 947 out, err = proc.communicate(**cargs) 948 except TimeoutExpired: 949 if kwargs.get('start_new_session') and hasattr(os, 'killpg'): 950 os.killpg(proc.pid, signal.SIGKILL) 951 else: 952 proc.kill() 953 out, err = proc.communicate() 954 exc = TimeoutExpired(proc.args, timeout=cargs['timeout'], output=out) 955 exc.stderr = err 956 raise exc 957 status = proc.returncode 958 else: 959 out, err = (None, None) 960 try: 961 status = proc.wait(**cargs) 962 except TimeoutExpired as e: 963 if kwargs.get('start_new_session') and hasattr(os, 'killpg'): 964 os.killpg(proc.pid, signal.SIGKILL) 965 else: 966 proc.kill() 967 proc.wait() 968 raise e 969 return status, out, err 970 971def run_process(cmd, kwargs, cargs={}): 972 """ 973 Executes a subprocess by using a pre-forked process when possible 974 or falling back to subprocess.Popen. See :py:func:`waflib.Utils.run_prefork_process` 975 and :py:func:`waflib.Utils.run_regular_process` 976 """ 977 if kwargs.get('stdout') and kwargs.get('stderr'): 978 return run_prefork_process(cmd, kwargs, cargs) 979 else: 980 return run_regular_process(cmd, kwargs, cargs) 981 982def alloc_process_pool(n, force=False): 983 """ 984 Allocates an amount of processes to the default pool so its size is at least *n*. 985 It is useful to call this function early so that the pre-forked 986 processes use as little memory as possible. 987 988 :param n: pool size 989 :type n: integer 990 :param force: if True then *n* more processes are added to the existing pool 991 :type force: bool 992 """ 993 # mandatory on python2, unnecessary on python >= 3.2 994 global run_process, get_process, alloc_process_pool 995 if not force: 996 n = max(n - len(process_pool), 0) 997 try: 998 lst = [get_process() for x in range(n)] 999 except OSError: 1000 run_process = run_regular_process 1001 get_process = alloc_process_pool = nada 1002 else: 1003 for x in lst: 1004 process_pool.append(x) 1005 1006def atexit_pool(): 1007 for k in process_pool: 1008 try: 1009 os.kill(k.pid, 9) 1010 except OSError: 1011 pass 1012 else: 1013 k.wait() 1014# see #1889 1015if (sys.hexversion<0x207000f and not is_win32) or sys.hexversion>=0x306000f: 1016 atexit.register(atexit_pool) 1017 1018if os.environ.get('WAF_NO_PREFORK') or sys.platform == 'cli' or not sys.executable: 1019 run_process = run_regular_process 1020 get_process = alloc_process_pool = nada 1021 1022