1#!/usr/bin/env python 2# encoding: utf-8 3# Thomas Nagy, 2005-2018 (ita) 4 5""" 6Tasks represent atomic operations such as processes. 7""" 8 9import os, re, sys, tempfile, traceback 10from waflib import Utils, Logs, Errors 11 12# task states 13NOT_RUN = 0 14"""The task was not executed yet""" 15 16MISSING = 1 17"""The task has been executed but the files have not been created""" 18 19CRASHED = 2 20"""The task execution returned a non-zero exit status""" 21 22EXCEPTION = 3 23"""An exception occurred in the task execution""" 24 25CANCELED = 4 26"""A dependency for the task is missing so it was cancelled""" 27 28SKIPPED = 8 29"""The task did not have to be executed""" 30 31SUCCESS = 9 32"""The task was successfully executed""" 33 34ASK_LATER = -1 35"""The task is not ready to be executed""" 36 37SKIP_ME = -2 38"""The task does not need to be executed""" 39 40RUN_ME = -3 41"""The task must be executed""" 42 43CANCEL_ME = -4 44"""The task cannot be executed because of a dependency problem""" 45 46COMPILE_TEMPLATE_SHELL = ''' 47def f(tsk): 48 env = tsk.env 49 gen = tsk.generator 50 bld = gen.bld 51 cwdx = tsk.get_cwd() 52 p = env.get_flat 53 def to_list(xx): 54 if isinstance(xx, str): return [xx] 55 return xx 56 tsk.last_cmd = cmd = \'\'\' %s \'\'\' % s 57 return tsk.exec_command(cmd, cwd=cwdx, env=env.env or None) 58''' 59 60COMPILE_TEMPLATE_NOSHELL = ''' 61def f(tsk): 62 env = tsk.env 63 gen = tsk.generator 64 bld = gen.bld 65 cwdx = tsk.get_cwd() 66 def to_list(xx): 67 if isinstance(xx, str): return [xx] 68 return xx 69 def merge(lst1, lst2): 70 if lst1 and lst2: 71 return lst1[:-1] + [lst1[-1] + lst2[0]] + lst2[1:] 72 return lst1 + lst2 73 lst = [] 74 %s 75 if '' in lst: 76 lst = [x for x in lst if x] 77 tsk.last_cmd = lst 78 return tsk.exec_command(lst, cwd=cwdx, env=env.env or None) 79''' 80 81COMPILE_TEMPLATE_SIG_VARS = ''' 82def f(tsk): 83 sig = tsk.generator.bld.hash_env_vars(tsk.env, tsk.vars) 84 tsk.m.update(sig) 85 env = tsk.env 86 gen = tsk.generator 87 bld = gen.bld 88 cwdx = tsk.get_cwd() 89 p = env.get_flat 90 buf = [] 91 %s 92 tsk.m.update(repr(buf).encode()) 93''' 94 95classes = {} 96""" 97The metaclass :py:class:`waflib.Task.store_task_type` stores all class tasks 98created by user scripts or Waf tools to this dict. It maps class names to class objects. 99""" 100 101class store_task_type(type): 102 """ 103 Metaclass: store the task classes into the dict pointed by the 104 class attribute 'register' which defaults to :py:const:`waflib.Task.classes`, 105 106 The attribute 'run_str' is compiled into a method 'run' bound to the task class. 107 """ 108 def __init__(cls, name, bases, dict): 109 super(store_task_type, cls).__init__(name, bases, dict) 110 name = cls.__name__ 111 112 if name != 'evil' and name != 'Task': 113 if getattr(cls, 'run_str', None): 114 # if a string is provided, convert it to a method 115 (f, dvars) = compile_fun(cls.run_str, cls.shell) 116 cls.hcode = Utils.h_cmd(cls.run_str) 117 cls.orig_run_str = cls.run_str 118 # change the name of run_str or it is impossible to subclass with a function 119 cls.run_str = None 120 cls.run = f 121 # process variables 122 cls.vars = list(set(cls.vars + dvars)) 123 cls.vars.sort() 124 if cls.vars: 125 fun = compile_sig_vars(cls.vars) 126 if fun: 127 cls.sig_vars = fun 128 elif getattr(cls, 'run', None) and not 'hcode' in cls.__dict__: 129 # getattr(cls, 'hcode') would look in the upper classes 130 cls.hcode = Utils.h_cmd(cls.run) 131 132 # be creative 133 getattr(cls, 'register', classes)[name] = cls 134 135evil = store_task_type('evil', (object,), {}) 136"Base class provided to avoid writing a metaclass, so the code can run in python 2.6 and 3.x unmodified" 137 138class Task(evil): 139 """ 140 Task objects represents actions to perform such as commands to execute by calling the `run` method. 141 142 Detecting when to execute a task occurs in the method :py:meth:`waflib.Task.Task.runnable_status`. 143 144 Detecting which tasks to execute is performed through a hash value returned by 145 :py:meth:`waflib.Task.Task.signature`. The task signature is persistent from build to build. 146 """ 147 vars = [] 148 """ConfigSet variables that should trigger a rebuild (class attribute used for :py:meth:`waflib.Task.Task.sig_vars`)""" 149 150 always_run = False 151 """Specify whether task instances must always be executed or not (class attribute)""" 152 153 shell = False 154 """Execute the command with the shell (class attribute)""" 155 156 color = 'GREEN' 157 """Color for the console display, see :py:const:`waflib.Logs.colors_lst`""" 158 159 ext_in = [] 160 """File extensions that objects of this task class may use""" 161 162 ext_out = [] 163 """File extensions that objects of this task class may create""" 164 165 before = [] 166 """List of task class names to execute before instances of this class""" 167 168 after = [] 169 """List of task class names to execute after instances of this class""" 170 171 hcode = Utils.SIG_NIL 172 """String representing an additional hash for the class representation""" 173 174 keep_last_cmd = False 175 """Whether to keep the last command executed on the instance after execution. 176 This may be useful for certain extensions but it can a lot of memory. 177 """ 178 179 weight = 0 180 """Optional weight to tune the priority for task instances. 181 The higher, the earlier. The weight only applies to single task objects.""" 182 183 tree_weight = 0 184 """Optional weight to tune the priority of task instances and whole subtrees. 185 The higher, the earlier.""" 186 187 prio_order = 0 188 """Priority order set by the scheduler on instances during the build phase. 189 You most likely do not need to set it. 190 """ 191 192 __slots__ = ('hasrun', 'generator', 'env', 'inputs', 'outputs', 'dep_nodes', 'run_after') 193 194 def __init__(self, *k, **kw): 195 self.hasrun = NOT_RUN 196 try: 197 self.generator = kw['generator'] 198 except KeyError: 199 self.generator = self 200 201 self.env = kw['env'] 202 """:py:class:`waflib.ConfigSet.ConfigSet` object (make sure to provide one)""" 203 204 self.inputs = [] 205 """List of input nodes, which represent the files used by the task instance""" 206 207 self.outputs = [] 208 """List of output nodes, which represent the files created by the task instance""" 209 210 self.dep_nodes = [] 211 """List of additional nodes to depend on""" 212 213 self.run_after = set() 214 """Set of tasks that must be executed before this one""" 215 216 def __lt__(self, other): 217 return self.priority() > other.priority() 218 def __le__(self, other): 219 return self.priority() >= other.priority() 220 def __gt__(self, other): 221 return self.priority() < other.priority() 222 def __ge__(self, other): 223 return self.priority() <= other.priority() 224 225 def get_cwd(self): 226 """ 227 :return: current working directory 228 :rtype: :py:class:`waflib.Node.Node` 229 """ 230 bld = self.generator.bld 231 ret = getattr(self, 'cwd', None) or getattr(bld, 'cwd', bld.bldnode) 232 if isinstance(ret, str): 233 if os.path.isabs(ret): 234 ret = bld.root.make_node(ret) 235 else: 236 ret = self.generator.path.make_node(ret) 237 return ret 238 239 def quote_flag(self, x): 240 """ 241 Surround a process argument by quotes so that a list of arguments can be written to a file 242 243 :param x: flag 244 :type x: string 245 :return: quoted flag 246 :rtype: string 247 """ 248 old = x 249 if '\\' in x: 250 x = x.replace('\\', '\\\\') 251 if '"' in x: 252 x = x.replace('"', '\\"') 253 if old != x or ' ' in x or '\t' in x or "'" in x: 254 x = '"%s"' % x 255 return x 256 257 def priority(self): 258 """ 259 Priority of execution; the higher, the earlier 260 261 :return: the priority value 262 :rtype: a tuple of numeric values 263 """ 264 return (self.weight + self.prio_order, - getattr(self.generator, 'tg_idx_count', 0)) 265 266 def split_argfile(self, cmd): 267 """ 268 Splits a list of process commands into the executable part and its list of arguments 269 270 :return: a tuple containing the executable first and then the rest of arguments 271 :rtype: tuple 272 """ 273 return ([cmd[0]], [self.quote_flag(x) for x in cmd[1:]]) 274 275 def exec_command(self, cmd, **kw): 276 """ 277 Wrapper for :py:meth:`waflib.Context.Context.exec_command`. 278 This version set the current working directory (``build.variant_dir``), 279 applies PATH settings (if self.env.PATH is provided), and can run long 280 commands through a temporary ``@argfile``. 281 282 :param cmd: process command to execute 283 :type cmd: list of string (best) or string (process will use a shell) 284 :return: the return code 285 :rtype: int 286 287 Optional parameters: 288 289 #. cwd: current working directory (Node or string) 290 #. stdout: set to None to prevent waf from capturing the process standard output 291 #. stderr: set to None to prevent waf from capturing the process standard error 292 #. timeout: timeout value (Python 3) 293 """ 294 if not 'cwd' in kw: 295 kw['cwd'] = self.get_cwd() 296 297 if hasattr(self, 'timeout'): 298 kw['timeout'] = self.timeout 299 300 if self.env.PATH: 301 env = kw['env'] = dict(kw.get('env') or self.env.env or os.environ) 302 env['PATH'] = self.env.PATH if isinstance(self.env.PATH, str) else os.pathsep.join(self.env.PATH) 303 304 if hasattr(self, 'stdout'): 305 kw['stdout'] = self.stdout 306 if hasattr(self, 'stderr'): 307 kw['stderr'] = self.stderr 308 309 # workaround for command line length limit: 310 # http://support.microsoft.com/kb/830473 311 if not isinstance(cmd, str) and (len(repr(cmd)) >= 8192 if Utils.is_win32 else len(cmd) > 200000): 312 cmd, args = self.split_argfile(cmd) 313 try: 314 (fd, tmp) = tempfile.mkstemp() 315 os.write(fd, '\r\n'.join(args).encode()) 316 os.close(fd) 317 if Logs.verbose: 318 Logs.debug('argfile: @%r -> %r', tmp, args) 319 return self.generator.bld.exec_command(cmd + ['@' + tmp], **kw) 320 finally: 321 try: 322 os.remove(tmp) 323 except OSError: 324 # anti-virus and indexers can keep files open -_- 325 pass 326 else: 327 return self.generator.bld.exec_command(cmd, **kw) 328 329 def process(self): 330 """ 331 Runs the task and handles errors 332 333 :return: 0 or None if everything is fine 334 :rtype: integer 335 """ 336 # remove the task signature immediately before it is executed 337 # so that the task will be executed again in case of failure 338 try: 339 del self.generator.bld.task_sigs[self.uid()] 340 except KeyError: 341 pass 342 343 try: 344 ret = self.run() 345 except Exception: 346 self.err_msg = traceback.format_exc() 347 self.hasrun = EXCEPTION 348 else: 349 if ret: 350 self.err_code = ret 351 self.hasrun = CRASHED 352 else: 353 try: 354 self.post_run() 355 except Errors.WafError: 356 pass 357 except Exception: 358 self.err_msg = traceback.format_exc() 359 self.hasrun = EXCEPTION 360 else: 361 self.hasrun = SUCCESS 362 363 if self.hasrun != SUCCESS and self.scan: 364 # rescan dependencies on next run 365 try: 366 del self.generator.bld.imp_sigs[self.uid()] 367 except KeyError: 368 pass 369 370 def log_display(self, bld): 371 "Writes the execution status on the context logger" 372 if self.generator.bld.progress_bar == 3: 373 return 374 375 s = self.display() 376 if s: 377 if bld.logger: 378 logger = bld.logger 379 else: 380 logger = Logs 381 382 if self.generator.bld.progress_bar == 1: 383 c1 = Logs.colors.cursor_off 384 c2 = Logs.colors.cursor_on 385 logger.info(s, extra={'stream': sys.stderr, 'terminator':'', 'c1': c1, 'c2' : c2}) 386 else: 387 logger.info(s, extra={'terminator':'', 'c1': '', 'c2' : ''}) 388 389 def display(self): 390 """ 391 Returns an execution status for the console, the progress bar, or the IDE output. 392 393 :rtype: string 394 """ 395 col1 = Logs.colors(self.color) 396 col2 = Logs.colors.NORMAL 397 master = self.generator.bld.producer 398 399 def cur(): 400 # the current task position, computed as late as possible 401 return master.processed - master.ready.qsize() 402 403 if self.generator.bld.progress_bar == 1: 404 return self.generator.bld.progress_line(cur(), master.total, col1, col2) 405 406 if self.generator.bld.progress_bar == 2: 407 ela = str(self.generator.bld.timer) 408 try: 409 ins = ','.join([n.name for n in self.inputs]) 410 except AttributeError: 411 ins = '' 412 try: 413 outs = ','.join([n.name for n in self.outputs]) 414 except AttributeError: 415 outs = '' 416 return '|Total %s|Current %s|Inputs %s|Outputs %s|Time %s|\n' % (master.total, cur(), ins, outs, ela) 417 418 s = str(self) 419 if not s: 420 return None 421 422 total = master.total 423 n = len(str(total)) 424 fs = '[%%%dd/%%%dd] %%s%%s%%s%%s\n' % (n, n) 425 kw = self.keyword() 426 if kw: 427 kw += ' ' 428 return fs % (cur(), total, kw, col1, s, col2) 429 430 def hash_constraints(self): 431 """ 432 Identifies a task type for all the constraints relevant for the scheduler: precedence, file production 433 434 :return: a hash value 435 :rtype: string 436 """ 437 return (tuple(self.before), tuple(self.after), tuple(self.ext_in), tuple(self.ext_out), self.__class__.__name__, self.hcode) 438 439 def format_error(self): 440 """ 441 Returns an error message to display the build failure reasons 442 443 :rtype: string 444 """ 445 if Logs.verbose: 446 msg = ': %r\n%r' % (self, getattr(self, 'last_cmd', '')) 447 else: 448 msg = ' (run with -v to display more information)' 449 name = getattr(self.generator, 'name', '') 450 if getattr(self, "err_msg", None): 451 return self.err_msg 452 elif not self.hasrun: 453 return 'task in %r was not executed for some reason: %r' % (name, self) 454 elif self.hasrun == CRASHED: 455 try: 456 return ' -> task in %r failed with exit status %r%s' % (name, self.err_code, msg) 457 except AttributeError: 458 return ' -> task in %r failed%s' % (name, msg) 459 elif self.hasrun == MISSING: 460 return ' -> missing files in %r%s' % (name, msg) 461 elif self.hasrun == CANCELED: 462 return ' -> %r canceled because of missing dependencies' % name 463 else: 464 return 'invalid status for task in %r: %r' % (name, self.hasrun) 465 466 def colon(self, var1, var2): 467 """ 468 Enable scriptlet expressions of the form ${FOO_ST:FOO} 469 If the first variable (FOO_ST) is empty, then an empty list is returned 470 471 The results will be slightly different if FOO_ST is a list, for example:: 472 473 env.FOO = ['p1', 'p2'] 474 env.FOO_ST = '-I%s' 475 # ${FOO_ST:FOO} returns 476 ['-Ip1', '-Ip2'] 477 478 env.FOO_ST = ['-a', '-b'] 479 # ${FOO_ST:FOO} returns 480 ['-a', '-b', 'p1', '-a', '-b', 'p2'] 481 """ 482 tmp = self.env[var1] 483 if not tmp: 484 return [] 485 486 if isinstance(var2, str): 487 it = self.env[var2] 488 else: 489 it = var2 490 if isinstance(tmp, str): 491 return [tmp % x for x in it] 492 else: 493 lst = [] 494 for y in it: 495 lst.extend(tmp) 496 lst.append(y) 497 return lst 498 499 def __str__(self): 500 "string to display to the user" 501 name = self.__class__.__name__ 502 if self.outputs: 503 if name.endswith(('lib', 'program')) or not self.inputs: 504 node = self.outputs[0] 505 return node.path_from(node.ctx.launch_node()) 506 if not (self.inputs or self.outputs): 507 return self.__class__.__name__ 508 if len(self.inputs) == 1: 509 node = self.inputs[0] 510 return node.path_from(node.ctx.launch_node()) 511 512 src_str = ' '.join([a.path_from(a.ctx.launch_node()) for a in self.inputs]) 513 tgt_str = ' '.join([a.path_from(a.ctx.launch_node()) for a in self.outputs]) 514 if self.outputs: 515 sep = ' -> ' 516 else: 517 sep = '' 518 return '%s: %s%s%s' % (self.__class__.__name__, src_str, sep, tgt_str) 519 520 def keyword(self): 521 "Display keyword used to prettify the console outputs" 522 name = self.__class__.__name__ 523 if name.endswith(('lib', 'program')): 524 return 'Linking' 525 if len(self.inputs) == 1 and len(self.outputs) == 1: 526 return 'Compiling' 527 if not self.inputs: 528 if self.outputs: 529 return 'Creating' 530 else: 531 return 'Running' 532 return 'Processing' 533 534 def __repr__(self): 535 "for debugging purposes" 536 try: 537 ins = ",".join([x.name for x in self.inputs]) 538 outs = ",".join([x.name for x in self.outputs]) 539 except AttributeError: 540 ins = ",".join([str(x) for x in self.inputs]) 541 outs = ",".join([str(x) for x in self.outputs]) 542 return "".join(['\n\t{task %r: ' % id(self), self.__class__.__name__, " ", ins, " -> ", outs, '}']) 543 544 def uid(self): 545 """ 546 Returns an identifier used to determine if tasks are up-to-date. Since the 547 identifier will be stored between executions, it must be: 548 549 - unique for a task: no two tasks return the same value (for a given build context) 550 - the same for a given task instance 551 552 By default, the node paths, the class name, and the function are used 553 as inputs to compute a hash. 554 555 The pointer to the object (python built-in 'id') will change between build executions, 556 and must be avoided in such hashes. 557 558 :return: hash value 559 :rtype: string 560 """ 561 try: 562 return self.uid_ 563 except AttributeError: 564 m = Utils.md5(self.__class__.__name__) 565 up = m.update 566 for x in self.inputs + self.outputs: 567 up(x.abspath()) 568 self.uid_ = m.digest() 569 return self.uid_ 570 571 def set_inputs(self, inp): 572 """ 573 Appends the nodes to the *inputs* list 574 575 :param inp: input nodes 576 :type inp: node or list of nodes 577 """ 578 if isinstance(inp, list): 579 self.inputs += inp 580 else: 581 self.inputs.append(inp) 582 583 def set_outputs(self, out): 584 """ 585 Appends the nodes to the *outputs* list 586 587 :param out: output nodes 588 :type out: node or list of nodes 589 """ 590 if isinstance(out, list): 591 self.outputs += out 592 else: 593 self.outputs.append(out) 594 595 def set_run_after(self, task): 596 """ 597 Run this task only after the given *task*. 598 599 Calling this method from :py:meth:`waflib.Task.Task.runnable_status` may cause 600 build deadlocks; see :py:meth:`waflib.Tools.fc.fc.runnable_status` for details. 601 602 :param task: task 603 :type task: :py:class:`waflib.Task.Task` 604 """ 605 assert isinstance(task, Task) 606 self.run_after.add(task) 607 608 def signature(self): 609 """ 610 Task signatures are stored between build executions, they are use to track the changes 611 made to the input nodes (not to the outputs!). The signature hashes data from various sources: 612 613 * explicit dependencies: files listed in the inputs (list of node objects) :py:meth:`waflib.Task.Task.sig_explicit_deps` 614 * implicit dependencies: list of nodes returned by scanner methods (when present) :py:meth:`waflib.Task.Task.sig_implicit_deps` 615 * hashed data: variables/values read from task.vars/task.env :py:meth:`waflib.Task.Task.sig_vars` 616 617 If the signature is expected to give a different result, clear the cache kept in ``self.cache_sig``:: 618 619 from waflib import Task 620 class cls(Task.Task): 621 def signature(self): 622 sig = super(Task.Task, self).signature() 623 delattr(self, 'cache_sig') 624 return super(Task.Task, self).signature() 625 626 :return: the signature value 627 :rtype: string or bytes 628 """ 629 try: 630 return self.cache_sig 631 except AttributeError: 632 pass 633 634 self.m = Utils.md5(self.hcode) 635 636 # explicit deps 637 self.sig_explicit_deps() 638 639 # env vars 640 self.sig_vars() 641 642 # implicit deps / scanner results 643 if self.scan: 644 try: 645 self.sig_implicit_deps() 646 except Errors.TaskRescan: 647 return self.signature() 648 649 ret = self.cache_sig = self.m.digest() 650 return ret 651 652 def runnable_status(self): 653 """ 654 Returns the Task status 655 656 :return: a task state in :py:const:`waflib.Task.RUN_ME`, 657 :py:const:`waflib.Task.SKIP_ME`, :py:const:`waflib.Task.CANCEL_ME` or :py:const:`waflib.Task.ASK_LATER`. 658 :rtype: int 659 """ 660 bld = self.generator.bld 661 if bld.is_install < 0: 662 return SKIP_ME 663 664 for t in self.run_after: 665 if not t.hasrun: 666 return ASK_LATER 667 elif t.hasrun < SKIPPED: 668 # a dependency has an error 669 return CANCEL_ME 670 671 # first compute the signature 672 try: 673 new_sig = self.signature() 674 except Errors.TaskNotReady: 675 return ASK_LATER 676 677 # compare the signature to a signature computed previously 678 key = self.uid() 679 try: 680 prev_sig = bld.task_sigs[key] 681 except KeyError: 682 Logs.debug('task: task %r must run: it was never run before or the task code changed', self) 683 return RUN_ME 684 685 if new_sig != prev_sig: 686 Logs.debug('task: task %r must run: the task signature changed', self) 687 return RUN_ME 688 689 # compare the signatures of the outputs 690 for node in self.outputs: 691 sig = bld.node_sigs.get(node) 692 if not sig: 693 Logs.debug('task: task %r must run: an output node has no signature', self) 694 return RUN_ME 695 if sig != key: 696 Logs.debug('task: task %r must run: an output node was produced by another task', self) 697 return RUN_ME 698 if not node.exists(): 699 Logs.debug('task: task %r must run: an output node does not exist', self) 700 return RUN_ME 701 702 return (self.always_run and RUN_ME) or SKIP_ME 703 704 def post_run(self): 705 """ 706 Called after successful execution to record that the task has run by 707 updating the entry in :py:attr:`waflib.Build.BuildContext.task_sigs`. 708 """ 709 bld = self.generator.bld 710 for node in self.outputs: 711 if not node.exists(): 712 self.hasrun = MISSING 713 self.err_msg = '-> missing file: %r' % node.abspath() 714 raise Errors.WafError(self.err_msg) 715 bld.node_sigs[node] = self.uid() # make sure this task produced the files in question 716 bld.task_sigs[self.uid()] = self.signature() 717 if not self.keep_last_cmd: 718 try: 719 del self.last_cmd 720 except AttributeError: 721 pass 722 723 def sig_explicit_deps(self): 724 """ 725 Used by :py:meth:`waflib.Task.Task.signature`; it hashes :py:attr:`waflib.Task.Task.inputs` 726 and :py:attr:`waflib.Task.Task.dep_nodes` signatures. 727 """ 728 bld = self.generator.bld 729 upd = self.m.update 730 731 # the inputs 732 for x in self.inputs + self.dep_nodes: 733 upd(x.get_bld_sig()) 734 735 # manual dependencies, they can slow down the builds 736 if bld.deps_man: 737 additional_deps = bld.deps_man 738 for x in self.inputs + self.outputs: 739 try: 740 d = additional_deps[x] 741 except KeyError: 742 continue 743 744 for v in d: 745 try: 746 v = v.get_bld_sig() 747 except AttributeError: 748 if hasattr(v, '__call__'): 749 v = v() # dependency is a function, call it 750 upd(v) 751 752 def sig_deep_inputs(self): 753 """ 754 Enable rebuilds on input files task signatures. Not used by default. 755 756 Example: hashes of output programs can be unchanged after being re-linked, 757 despite the libraries being different. This method can thus prevent stale unit test 758 results (waf_unit_test.py). 759 760 Hashing input file timestamps is another possibility for the implementation. 761 This may cause unnecessary rebuilds when input tasks are frequently executed. 762 Here is an implementation example:: 763 764 lst = [] 765 for node in self.inputs + self.dep_nodes: 766 st = os.stat(node.abspath()) 767 lst.append(st.st_mtime) 768 lst.append(st.st_size) 769 self.m.update(Utils.h_list(lst)) 770 771 The downside of the implementation is that it absolutely requires all build directory 772 files to be declared within the current build. 773 """ 774 bld = self.generator.bld 775 lst = [bld.task_sigs[bld.node_sigs[node]] for node in (self.inputs + self.dep_nodes) if node.is_bld()] 776 self.m.update(Utils.h_list(lst)) 777 778 def sig_vars(self): 779 """ 780 Used by :py:meth:`waflib.Task.Task.signature`; it hashes :py:attr:`waflib.Task.Task.env` variables/values 781 When overriding this method, and if scriptlet expressions are used, make sure to follow 782 the code in :py:meth:`waflib.Task.Task.compile_sig_vars` to enable dependencies on scriptlet results. 783 784 This method may be replaced on subclasses by the metaclass to force dependencies on scriptlet code. 785 """ 786 sig = self.generator.bld.hash_env_vars(self.env, self.vars) 787 self.m.update(sig) 788 789 scan = None 790 """ 791 This method, when provided, returns a tuple containing: 792 793 * a list of nodes corresponding to real files 794 * a list of names for files not found in path_lst 795 796 For example:: 797 798 from waflib.Task import Task 799 class mytask(Task): 800 def scan(self, node): 801 return ([], []) 802 803 The first and second lists in the tuple are stored in :py:attr:`waflib.Build.BuildContext.node_deps` and 804 :py:attr:`waflib.Build.BuildContext.raw_deps` respectively. 805 """ 806 807 def sig_implicit_deps(self): 808 """ 809 Used by :py:meth:`waflib.Task.Task.signature`; it hashes node signatures 810 obtained by scanning for dependencies (:py:meth:`waflib.Task.Task.scan`). 811 812 The exception :py:class:`waflib.Errors.TaskRescan` is thrown 813 when a file has changed. In this case, the method :py:meth:`waflib.Task.Task.signature` is called 814 once again, and return here to call :py:meth:`waflib.Task.Task.scan` and searching for dependencies. 815 """ 816 bld = self.generator.bld 817 818 # get the task signatures from previous runs 819 key = self.uid() 820 prev = bld.imp_sigs.get(key, []) 821 822 # for issue #379 823 if prev: 824 try: 825 if prev == self.compute_sig_implicit_deps(): 826 return prev 827 except Errors.TaskNotReady: 828 raise 829 except EnvironmentError: 830 # when a file was renamed, remove the stale nodes (headers in folders without source files) 831 # this will break the order calculation for headers created during the build in the source directory (should be uncommon) 832 # the behaviour will differ when top != out 833 for x in bld.node_deps.get(self.uid(), []): 834 if not x.is_bld() and not x.exists(): 835 try: 836 del x.parent.children[x.name] 837 except KeyError: 838 pass 839 del bld.imp_sigs[key] 840 raise Errors.TaskRescan('rescan') 841 842 # no previous run or the signature of the dependencies has changed, rescan the dependencies 843 (bld.node_deps[key], bld.raw_deps[key]) = self.scan() 844 if Logs.verbose: 845 Logs.debug('deps: scanner for %s: %r; unresolved: %r', self, bld.node_deps[key], bld.raw_deps[key]) 846 847 # recompute the signature and return it 848 try: 849 bld.imp_sigs[key] = self.compute_sig_implicit_deps() 850 except EnvironmentError: 851 for k in bld.node_deps.get(self.uid(), []): 852 if not k.exists(): 853 Logs.warn('Dependency %r for %r is missing: check the task declaration and the build order!', k, self) 854 raise 855 856 def compute_sig_implicit_deps(self): 857 """ 858 Used by :py:meth:`waflib.Task.Task.sig_implicit_deps` for computing the actual hash of the 859 :py:class:`waflib.Node.Node` returned by the scanner. 860 861 :return: a hash value for the implicit dependencies 862 :rtype: string or bytes 863 """ 864 upd = self.m.update 865 self.are_implicit_nodes_ready() 866 867 # scanner returns a node that does not have a signature 868 # just *ignore* the error and let them figure out from the compiler output 869 # waf -k behaviour 870 for k in self.generator.bld.node_deps.get(self.uid(), []): 871 upd(k.get_bld_sig()) 872 return self.m.digest() 873 874 def are_implicit_nodes_ready(self): 875 """ 876 For each node returned by the scanner, see if there is a task that creates it, 877 and infer the build order 878 879 This has a low performance impact on null builds (1.86s->1.66s) thanks to caching (28s->1.86s) 880 """ 881 bld = self.generator.bld 882 try: 883 cache = bld.dct_implicit_nodes 884 except AttributeError: 885 bld.dct_implicit_nodes = cache = {} 886 887 # one cache per build group 888 try: 889 dct = cache[bld.current_group] 890 except KeyError: 891 dct = cache[bld.current_group] = {} 892 for tsk in bld.cur_tasks: 893 for x in tsk.outputs: 894 dct[x] = tsk 895 896 modified = False 897 for x in bld.node_deps.get(self.uid(), []): 898 if x in dct: 899 self.run_after.add(dct[x]) 900 modified = True 901 902 if modified: 903 for tsk in self.run_after: 904 if not tsk.hasrun: 905 #print "task is not ready..." 906 raise Errors.TaskNotReady('not ready') 907if sys.hexversion > 0x3000000: 908 def uid(self): 909 try: 910 return self.uid_ 911 except AttributeError: 912 m = Utils.md5(self.__class__.__name__.encode('latin-1', 'xmlcharrefreplace')) 913 up = m.update 914 for x in self.inputs + self.outputs: 915 up(x.abspath().encode('latin-1', 'xmlcharrefreplace')) 916 self.uid_ = m.digest() 917 return self.uid_ 918 uid.__doc__ = Task.uid.__doc__ 919 Task.uid = uid 920 921def is_before(t1, t2): 922 """ 923 Returns a non-zero value if task t1 is to be executed before task t2:: 924 925 t1.ext_out = '.h' 926 t2.ext_in = '.h' 927 t2.after = ['t1'] 928 t1.before = ['t2'] 929 waflib.Task.is_before(t1, t2) # True 930 931 :param t1: Task object 932 :type t1: :py:class:`waflib.Task.Task` 933 :param t2: Task object 934 :type t2: :py:class:`waflib.Task.Task` 935 """ 936 to_list = Utils.to_list 937 for k in to_list(t2.ext_in): 938 if k in to_list(t1.ext_out): 939 return 1 940 941 if t1.__class__.__name__ in to_list(t2.after): 942 return 1 943 944 if t2.__class__.__name__ in to_list(t1.before): 945 return 1 946 947 return 0 948 949def set_file_constraints(tasks): 950 """ 951 Updates the ``run_after`` attribute of all tasks based on the task inputs and outputs 952 953 :param tasks: tasks 954 :type tasks: list of :py:class:`waflib.Task.Task` 955 """ 956 ins = Utils.defaultdict(set) 957 outs = Utils.defaultdict(set) 958 for x in tasks: 959 for a in x.inputs: 960 ins[a].add(x) 961 for a in x.dep_nodes: 962 ins[a].add(x) 963 for a in x.outputs: 964 outs[a].add(x) 965 966 links = set(ins.keys()).intersection(outs.keys()) 967 for k in links: 968 for a in ins[k]: 969 a.run_after.update(outs[k]) 970 971 972class TaskGroup(object): 973 """ 974 Wrap nxm task order constraints into a single object 975 to prevent the creation of large list/set objects 976 977 This is an optimization 978 """ 979 def __init__(self, prev, next): 980 self.prev = prev 981 self.next = next 982 self.done = False 983 984 def get_hasrun(self): 985 for k in self.prev: 986 if not k.hasrun: 987 return NOT_RUN 988 return SUCCESS 989 990 hasrun = property(get_hasrun, None) 991 992def set_precedence_constraints(tasks): 993 """ 994 Updates the ``run_after`` attribute of all tasks based on the after/before/ext_out/ext_in attributes 995 996 :param tasks: tasks 997 :type tasks: list of :py:class:`waflib.Task.Task` 998 """ 999 cstr_groups = Utils.defaultdict(list) 1000 for x in tasks: 1001 h = x.hash_constraints() 1002 cstr_groups[h].append(x) 1003 1004 keys = list(cstr_groups.keys()) 1005 maxi = len(keys) 1006 1007 # this list should be short 1008 for i in range(maxi): 1009 t1 = cstr_groups[keys[i]][0] 1010 for j in range(i + 1, maxi): 1011 t2 = cstr_groups[keys[j]][0] 1012 1013 # add the constraints based on the comparisons 1014 if is_before(t1, t2): 1015 a = i 1016 b = j 1017 elif is_before(t2, t1): 1018 a = j 1019 b = i 1020 else: 1021 continue 1022 1023 a = cstr_groups[keys[a]] 1024 b = cstr_groups[keys[b]] 1025 1026 if len(a) < 2 or len(b) < 2: 1027 for x in b: 1028 x.run_after.update(a) 1029 else: 1030 group = TaskGroup(set(a), set(b)) 1031 for x in b: 1032 x.run_after.add(group) 1033 1034def funex(c): 1035 """ 1036 Compiles a scriptlet expression into a Python function 1037 1038 :param c: function to compile 1039 :type c: string 1040 :return: the function 'f' declared in the input string 1041 :rtype: function 1042 """ 1043 dc = {} 1044 exec(c, dc) 1045 return dc['f'] 1046 1047re_cond = re.compile('(?P<var>\w+)|(?P<or>\|)|(?P<and>&)') 1048re_novar = re.compile(r'^(SRC|TGT)\W+.*?$') 1049reg_act = re.compile(r'(?P<backslash>\\)|(?P<dollar>\$\$)|(?P<subst>\$\{(?P<var>\w+)(?P<code>.*?)\})', re.M) 1050def compile_fun_shell(line): 1051 """ 1052 Creates a compiled function to execute a process through a sub-shell 1053 """ 1054 extr = [] 1055 def repl(match): 1056 g = match.group 1057 if g('dollar'): 1058 return "$" 1059 elif g('backslash'): 1060 return '\\\\' 1061 elif g('subst'): 1062 extr.append((g('var'), g('code'))) 1063 return "%s" 1064 return None 1065 line = reg_act.sub(repl, line) or line 1066 dvars = [] 1067 def add_dvar(x): 1068 if x not in dvars: 1069 dvars.append(x) 1070 1071 def replc(m): 1072 # performs substitutions and populates dvars 1073 if m.group('and'): 1074 return ' and ' 1075 elif m.group('or'): 1076 return ' or ' 1077 else: 1078 x = m.group('var') 1079 add_dvar(x) 1080 return 'env[%r]' % x 1081 1082 parm = [] 1083 app = parm.append 1084 for (var, meth) in extr: 1085 if var == 'SRC': 1086 if meth: 1087 app('tsk.inputs%s' % meth) 1088 else: 1089 app('" ".join([a.path_from(cwdx) for a in tsk.inputs])') 1090 elif var == 'TGT': 1091 if meth: 1092 app('tsk.outputs%s' % meth) 1093 else: 1094 app('" ".join([a.path_from(cwdx) for a in tsk.outputs])') 1095 elif meth: 1096 if meth.startswith(':'): 1097 add_dvar(var) 1098 m = meth[1:] 1099 if m == 'SRC': 1100 m = '[a.path_from(cwdx) for a in tsk.inputs]' 1101 elif m == 'TGT': 1102 m = '[a.path_from(cwdx) for a in tsk.outputs]' 1103 elif re_novar.match(m): 1104 m = '[tsk.inputs%s]' % m[3:] 1105 elif re_novar.match(m): 1106 m = '[tsk.outputs%s]' % m[3:] 1107 else: 1108 add_dvar(m) 1109 if m[:3] not in ('tsk', 'gen', 'bld'): 1110 m = '%r' % m 1111 app('" ".join(tsk.colon(%r, %s))' % (var, m)) 1112 elif meth.startswith('?'): 1113 # In A?B|C output env.A if one of env.B or env.C is non-empty 1114 expr = re_cond.sub(replc, meth[1:]) 1115 app('p(%r) if (%s) else ""' % (var, expr)) 1116 else: 1117 call = '%s%s' % (var, meth) 1118 add_dvar(call) 1119 app(call) 1120 else: 1121 add_dvar(var) 1122 app("p('%s')" % var) 1123 if parm: 1124 parm = "%% (%s) " % (',\n\t\t'.join(parm)) 1125 else: 1126 parm = '' 1127 1128 c = COMPILE_TEMPLATE_SHELL % (line, parm) 1129 Logs.debug('action: %s', c.strip().splitlines()) 1130 return (funex(c), dvars) 1131 1132reg_act_noshell = re.compile(r"(?P<space>\s+)|(?P<subst>\$\{(?P<var>\w+)(?P<code>.*?)\})|(?P<text>([^$ \t\n\r\f\v]|\$\$)+)", re.M) 1133def compile_fun_noshell(line): 1134 """ 1135 Creates a compiled function to execute a process without a sub-shell 1136 """ 1137 buf = [] 1138 dvars = [] 1139 merge = False 1140 app = buf.append 1141 1142 def add_dvar(x): 1143 if x not in dvars: 1144 dvars.append(x) 1145 1146 def replc(m): 1147 # performs substitutions and populates dvars 1148 if m.group('and'): 1149 return ' and ' 1150 elif m.group('or'): 1151 return ' or ' 1152 else: 1153 x = m.group('var') 1154 add_dvar(x) 1155 return 'env[%r]' % x 1156 1157 for m in reg_act_noshell.finditer(line): 1158 if m.group('space'): 1159 merge = False 1160 continue 1161 elif m.group('text'): 1162 app('[%r]' % m.group('text').replace('$$', '$')) 1163 elif m.group('subst'): 1164 var = m.group('var') 1165 code = m.group('code') 1166 if var == 'SRC': 1167 if code: 1168 app('[tsk.inputs%s]' % code) 1169 else: 1170 app('[a.path_from(cwdx) for a in tsk.inputs]') 1171 elif var == 'TGT': 1172 if code: 1173 app('[tsk.outputs%s]' % code) 1174 else: 1175 app('[a.path_from(cwdx) for a in tsk.outputs]') 1176 elif code: 1177 if code.startswith(':'): 1178 # a composed variable ${FOO:OUT} 1179 add_dvar(var) 1180 m = code[1:] 1181 if m == 'SRC': 1182 m = '[a.path_from(cwdx) for a in tsk.inputs]' 1183 elif m == 'TGT': 1184 m = '[a.path_from(cwdx) for a in tsk.outputs]' 1185 elif re_novar.match(m): 1186 m = '[tsk.inputs%s]' % m[3:] 1187 elif re_novar.match(m): 1188 m = '[tsk.outputs%s]' % m[3:] 1189 else: 1190 add_dvar(m) 1191 if m[:3] not in ('tsk', 'gen', 'bld'): 1192 m = '%r' % m 1193 app('tsk.colon(%r, %s)' % (var, m)) 1194 elif code.startswith('?'): 1195 # In A?B|C output env.A if one of env.B or env.C is non-empty 1196 expr = re_cond.sub(replc, code[1:]) 1197 app('to_list(env[%r] if (%s) else [])' % (var, expr)) 1198 else: 1199 # plain code such as ${tsk.inputs[0].abspath()} 1200 call = '%s%s' % (var, code) 1201 add_dvar(call) 1202 app('to_list(%s)' % call) 1203 else: 1204 # a plain variable such as # a plain variable like ${AR} 1205 app('to_list(env[%r])' % var) 1206 add_dvar(var) 1207 if merge: 1208 tmp = 'merge(%s, %s)' % (buf[-2], buf[-1]) 1209 del buf[-1] 1210 buf[-1] = tmp 1211 merge = True # next turn 1212 1213 buf = ['lst.extend(%s)' % x for x in buf] 1214 fun = COMPILE_TEMPLATE_NOSHELL % "\n\t".join(buf) 1215 Logs.debug('action: %s', fun.strip().splitlines()) 1216 return (funex(fun), dvars) 1217 1218def compile_fun(line, shell=False): 1219 """ 1220 Parses a string expression such as '${CC} ${SRC} -o ${TGT}' and returns a pair containing: 1221 1222 * The function created (compiled) for use as :py:meth:`waflib.Task.Task.run` 1223 * The list of variables that must cause rebuilds when *env* data is modified 1224 1225 for example:: 1226 1227 from waflib.Task import compile_fun 1228 compile_fun('cxx', '${CXX} -o ${TGT[0]} ${SRC} -I ${SRC[0].parent.bldpath()}') 1229 1230 def build(bld): 1231 bld(source='wscript', rule='echo "foo\\${SRC[0].name}\\bar"') 1232 1233 The env variables (CXX, ..) on the task must not hold dicts so as to preserve a consistent order. 1234 The reserved keywords ``TGT`` and ``SRC`` represent the task input and output nodes 1235 1236 """ 1237 if isinstance(line, str): 1238 if line.find('<') > 0 or line.find('>') > 0 or line.find('&&') > 0: 1239 shell = True 1240 else: 1241 dvars_lst = [] 1242 funs_lst = [] 1243 for x in line: 1244 if isinstance(x, str): 1245 fun, dvars = compile_fun(x, shell) 1246 dvars_lst += dvars 1247 funs_lst.append(fun) 1248 else: 1249 # assume a function to let through 1250 funs_lst.append(x) 1251 def composed_fun(task): 1252 for x in funs_lst: 1253 ret = x(task) 1254 if ret: 1255 return ret 1256 return None 1257 return composed_fun, dvars_lst 1258 if shell: 1259 return compile_fun_shell(line) 1260 else: 1261 return compile_fun_noshell(line) 1262 1263def compile_sig_vars(vars): 1264 """ 1265 This method produces a sig_vars method suitable for subclasses that provide 1266 scriptlet code in their run_str code. 1267 If no such method can be created, this method returns None. 1268 1269 The purpose of the sig_vars method returned is to ensures 1270 that rebuilds occur whenever the contents of the expression changes. 1271 This is the case B below:: 1272 1273 import time 1274 # case A: regular variables 1275 tg = bld(rule='echo ${FOO}') 1276 tg.env.FOO = '%s' % time.time() 1277 # case B 1278 bld(rule='echo ${gen.foo}', foo='%s' % time.time()) 1279 1280 :param vars: env variables such as CXXFLAGS or gen.foo 1281 :type vars: list of string 1282 :return: A sig_vars method relevant for dependencies if adequate, else None 1283 :rtype: A function, or None in most cases 1284 """ 1285 buf = [] 1286 for x in sorted(vars): 1287 if x[:3] in ('tsk', 'gen', 'bld'): 1288 buf.append('buf.append(%s)' % x) 1289 if buf: 1290 return funex(COMPILE_TEMPLATE_SIG_VARS % '\n\t'.join(buf)) 1291 return None 1292 1293def task_factory(name, func=None, vars=None, color='GREEN', ext_in=[], ext_out=[], before=[], after=[], shell=False, scan=None): 1294 """ 1295 Returns a new task subclass with the function ``run`` compiled from the line given. 1296 1297 :param func: method run 1298 :type func: string or function 1299 :param vars: list of variables to hash 1300 :type vars: list of string 1301 :param color: color to use 1302 :type color: string 1303 :param shell: when *func* is a string, enable/disable the use of the shell 1304 :type shell: bool 1305 :param scan: method scan 1306 :type scan: function 1307 :rtype: :py:class:`waflib.Task.Task` 1308 """ 1309 1310 params = { 1311 'vars': vars or [], # function arguments are static, and this one may be modified by the class 1312 'color': color, 1313 'name': name, 1314 'shell': shell, 1315 'scan': scan, 1316 } 1317 1318 if isinstance(func, str) or isinstance(func, tuple): 1319 params['run_str'] = func 1320 else: 1321 params['run'] = func 1322 1323 cls = type(Task)(name, (Task,), params) 1324 classes[name] = cls 1325 1326 if ext_in: 1327 cls.ext_in = Utils.to_list(ext_in) 1328 if ext_out: 1329 cls.ext_out = Utils.to_list(ext_out) 1330 if before: 1331 cls.before = Utils.to_list(before) 1332 if after: 1333 cls.after = Utils.to_list(after) 1334 1335 return cls 1336 1337def deep_inputs(cls): 1338 """ 1339 Task class decorator to enable rebuilds on input files task signatures 1340 """ 1341 def sig_explicit_deps(self): 1342 Task.sig_explicit_deps(self) 1343 Task.sig_deep_inputs(self) 1344 cls.sig_explicit_deps = sig_explicit_deps 1345 return cls 1346 1347TaskBase = Task 1348"Provided for compatibility reasons, TaskBase should not be used" 1349 1350class TaskSemaphore(object): 1351 """ 1352 Task semaphores provide a simple and efficient way of throttling the amount of 1353 a particular task to run concurrently. The throttling value is capped 1354 by the amount of maximum jobs, so for example, a `TaskSemaphore(10)` 1355 has no effect in a `-j2` build. 1356 1357 Task semaphores are typically specified on the task class level:: 1358 1359 class compile(waflib.Task.Task): 1360 semaphore = waflib.Task.TaskSemaphore(2) 1361 run_str = 'touch ${TGT}' 1362 1363 Task semaphores are meant to be used by the build scheduler in the main 1364 thread, so there are no guarantees of thread safety. 1365 """ 1366 def __init__(self, num): 1367 """ 1368 :param num: maximum value of concurrent tasks 1369 :type num: int 1370 """ 1371 self.num = num 1372 self.locking = set() 1373 self.waiting = set() 1374 1375 def is_locked(self): 1376 """Returns True if this semaphore cannot be acquired by more tasks""" 1377 return len(self.locking) >= self.num 1378 1379 def acquire(self, tsk): 1380 """ 1381 Mark the semaphore as used by the given task (not re-entrant). 1382 1383 :param tsk: task object 1384 :type tsk: :py:class:`waflib.Task.Task` 1385 :raises: :py:class:`IndexError` in case the resource is already acquired 1386 """ 1387 if self.is_locked(): 1388 raise IndexError('Cannot lock more %r' % self.locking) 1389 self.locking.add(tsk) 1390 1391 def release(self, tsk): 1392 """ 1393 Mark the semaphore as unused by the given task. 1394 1395 :param tsk: task object 1396 :type tsk: :py:class:`waflib.Task.Task` 1397 :raises: :py:class:`KeyError` in case the resource is not acquired by the task 1398 """ 1399 self.locking.remove(tsk) 1400 1401