1"""SCons.Executor 2 3A module for executing actions with specific lists of target and source 4Nodes. 5 6""" 7 8# 9# Copyright (c) 2001 - 2014 The SCons Foundation 10# 11# Permission is hereby granted, free of charge, to any person obtaining 12# a copy of this software and associated documentation files (the 13# "Software"), to deal in the Software without restriction, including 14# without limitation the rights to use, copy, modify, merge, publish, 15# distribute, sublicense, and/or sell copies of the Software, and to 16# permit persons to whom the Software is furnished to do so, subject to 17# the following conditions: 18# 19# The above copyright notice and this permission notice shall be included 20# in all copies or substantial portions of the Software. 21# 22# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY 23# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 24# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 25# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 26# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 27# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 28# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 29 30__revision__ = "src/engine/SCons/Executor.py 2014/08/24 12:12:31 garyo" 31 32import collections 33 34import SCons.Debug 35from SCons.Debug import logInstanceCreation 36import SCons.Errors 37import SCons.Memoize 38 39 40class Batch(object): 41 """Remembers exact association between targets 42 and sources of executor.""" 43 def __init__(self, targets=[], sources=[]): 44 self.targets = targets 45 self.sources = sources 46 47 48 49class TSList(collections.UserList): 50 """A class that implements $TARGETS or $SOURCES expansions by wrapping 51 an executor Method. This class is used in the Executor.lvars() 52 to delay creation of NodeList objects until they're needed. 53 54 Note that we subclass collections.UserList purely so that the 55 is_Sequence() function will identify an object of this class as 56 a list during variable expansion. We're not really using any 57 collections.UserList methods in practice. 58 """ 59 def __init__(self, func): 60 self.func = func 61 def __getattr__(self, attr): 62 nl = self.func() 63 return getattr(nl, attr) 64 def __getitem__(self, i): 65 nl = self.func() 66 return nl[i] 67 def __getslice__(self, i, j): 68 nl = self.func() 69 i = max(i, 0); j = max(j, 0) 70 return nl[i:j] 71 def __str__(self): 72 nl = self.func() 73 return str(nl) 74 def __repr__(self): 75 nl = self.func() 76 return repr(nl) 77 78class TSObject(object): 79 """A class that implements $TARGET or $SOURCE expansions by wrapping 80 an Executor method. 81 """ 82 def __init__(self, func): 83 self.func = func 84 def __getattr__(self, attr): 85 n = self.func() 86 return getattr(n, attr) 87 def __str__(self): 88 n = self.func() 89 if n: 90 return str(n) 91 return '' 92 def __repr__(self): 93 n = self.func() 94 if n: 95 return repr(n) 96 return '' 97 98def rfile(node): 99 """ 100 A function to return the results of a Node's rfile() method, 101 if it exists, and the Node itself otherwise (if it's a Value 102 Node, e.g.). 103 """ 104 try: 105 rfile = node.rfile 106 except AttributeError: 107 return node 108 else: 109 return rfile() 110 111 112class Executor(object): 113 """A class for controlling instances of executing an action. 114 115 This largely exists to hold a single association of an action, 116 environment, list of environment override dictionaries, targets 117 and sources for later processing as needed. 118 """ 119 120 if SCons.Memoize.use_memoizer: 121 __metaclass__ = SCons.Memoize.Memoized_Metaclass 122 123 memoizer_counters = [] 124 125 def __init__(self, action, env=None, overridelist=[{}], 126 targets=[], sources=[], builder_kw={}): 127 if SCons.Debug.track_instances: logInstanceCreation(self, 'Executor.Executor') 128 self.set_action_list(action) 129 self.pre_actions = [] 130 self.post_actions = [] 131 self.env = env 132 self.overridelist = overridelist 133 if targets or sources: 134 self.batches = [Batch(targets[:], sources[:])] 135 else: 136 self.batches = [] 137 self.builder_kw = builder_kw 138 self._memo = {} 139 140 def get_lvars(self): 141 try: 142 return self.lvars 143 except AttributeError: 144 self.lvars = { 145 'CHANGED_SOURCES' : TSList(self._get_changed_sources), 146 'CHANGED_TARGETS' : TSList(self._get_changed_targets), 147 'SOURCE' : TSObject(self._get_source), 148 'SOURCES' : TSList(self._get_sources), 149 'TARGET' : TSObject(self._get_target), 150 'TARGETS' : TSList(self._get_targets), 151 'UNCHANGED_SOURCES' : TSList(self._get_unchanged_sources), 152 'UNCHANGED_TARGETS' : TSList(self._get_unchanged_targets), 153 } 154 return self.lvars 155 156 def _get_changes(self): 157 cs = [] 158 ct = [] 159 us = [] 160 ut = [] 161 for b in self.batches: 162 if b.targets[0].is_up_to_date(): 163 us.extend(list(map(rfile, b.sources))) 164 ut.extend(b.targets) 165 else: 166 cs.extend(list(map(rfile, b.sources))) 167 ct.extend(b.targets) 168 self._changed_sources_list = SCons.Util.NodeList(cs) 169 self._changed_targets_list = SCons.Util.NodeList(ct) 170 self._unchanged_sources_list = SCons.Util.NodeList(us) 171 self._unchanged_targets_list = SCons.Util.NodeList(ut) 172 173 def _get_changed_sources(self, *args, **kw): 174 try: 175 return self._changed_sources_list 176 except AttributeError: 177 self._get_changes() 178 return self._changed_sources_list 179 180 def _get_changed_targets(self, *args, **kw): 181 try: 182 return self._changed_targets_list 183 except AttributeError: 184 self._get_changes() 185 return self._changed_targets_list 186 187 def _get_source(self, *args, **kw): 188 #return SCons.Util.NodeList([rfile(self.batches[0].sources[0]).get_subst_proxy()]) 189 return rfile(self.batches[0].sources[0]).get_subst_proxy() 190 191 def _get_sources(self, *args, **kw): 192 return SCons.Util.NodeList([rfile(n).get_subst_proxy() for n in self.get_all_sources()]) 193 194 def _get_target(self, *args, **kw): 195 #return SCons.Util.NodeList([self.batches[0].targets[0].get_subst_proxy()]) 196 return self.batches[0].targets[0].get_subst_proxy() 197 198 def _get_targets(self, *args, **kw): 199 return SCons.Util.NodeList([n.get_subst_proxy() for n in self.get_all_targets()]) 200 201 def _get_unchanged_sources(self, *args, **kw): 202 try: 203 return self._unchanged_sources_list 204 except AttributeError: 205 self._get_changes() 206 return self._unchanged_sources_list 207 208 def _get_unchanged_targets(self, *args, **kw): 209 try: 210 return self._unchanged_targets_list 211 except AttributeError: 212 self._get_changes() 213 return self._unchanged_targets_list 214 215 def get_action_targets(self): 216 if not self.action_list: 217 return [] 218 targets_string = self.action_list[0].get_targets(self.env, self) 219 if targets_string[0] == '$': 220 targets_string = targets_string[1:] 221 return self.get_lvars()[targets_string] 222 223 def set_action_list(self, action): 224 import SCons.Util 225 if not SCons.Util.is_List(action): 226 if not action: 227 import SCons.Errors 228 raise SCons.Errors.UserError("Executor must have an action.") 229 action = [action] 230 self.action_list = action 231 232 def get_action_list(self): 233 if self.action_list is None: 234 return [] 235 return self.pre_actions + self.action_list + self.post_actions 236 237 def get_all_targets(self): 238 """Returns all targets for all batches of this Executor.""" 239 result = [] 240 for batch in self.batches: 241 result.extend(batch.targets) 242 return result 243 244 def get_all_sources(self): 245 """Returns all sources for all batches of this Executor.""" 246 result = [] 247 for batch in self.batches: 248 result.extend(batch.sources) 249 return result 250 251 def get_all_children(self): 252 """Returns all unique children (dependencies) for all batches 253 of this Executor. 254 255 The Taskmaster can recognize when it's already evaluated a 256 Node, so we don't have to make this list unique for its intended 257 canonical use case, but we expect there to be a lot of redundancy 258 (long lists of batched .cc files #including the same .h files 259 over and over), so removing the duplicates once up front should 260 save the Taskmaster a lot of work. 261 """ 262 result = SCons.Util.UniqueList([]) 263 for target in self.get_all_targets(): 264 result.extend(target.children()) 265 return result 266 267 def get_all_prerequisites(self): 268 """Returns all unique (order-only) prerequisites for all batches 269 of this Executor. 270 """ 271 result = SCons.Util.UniqueList([]) 272 for target in self.get_all_targets(): 273 if target.prerequisites is not None: 274 result.extend(target.prerequisites) 275 return result 276 277 def get_action_side_effects(self): 278 279 """Returns all side effects for all batches of this 280 Executor used by the underlying Action. 281 """ 282 result = SCons.Util.UniqueList([]) 283 for target in self.get_action_targets(): 284 result.extend(target.side_effects) 285 return result 286 287 memoizer_counters.append(SCons.Memoize.CountValue('get_build_env')) 288 289 def get_build_env(self): 290 """Fetch or create the appropriate build Environment 291 for this Executor. 292 """ 293 try: 294 return self._memo['get_build_env'] 295 except KeyError: 296 pass 297 298 # Create the build environment instance with appropriate 299 # overrides. These get evaluated against the current 300 # environment's construction variables so that users can 301 # add to existing values by referencing the variable in 302 # the expansion. 303 overrides = {} 304 for odict in self.overridelist: 305 overrides.update(odict) 306 307 import SCons.Defaults 308 env = self.env or SCons.Defaults.DefaultEnvironment() 309 build_env = env.Override(overrides) 310 311 self._memo['get_build_env'] = build_env 312 313 return build_env 314 315 def get_build_scanner_path(self, scanner): 316 """Fetch the scanner path for this executor's targets and sources. 317 """ 318 env = self.get_build_env() 319 try: 320 cwd = self.batches[0].targets[0].cwd 321 except (IndexError, AttributeError): 322 cwd = None 323 return scanner.path(env, cwd, 324 self.get_all_targets(), 325 self.get_all_sources()) 326 327 def get_kw(self, kw={}): 328 result = self.builder_kw.copy() 329 result.update(kw) 330 result['executor'] = self 331 return result 332 333 def do_nothing(self, target, kw): 334 return 0 335 336 def do_execute(self, target, kw): 337 """Actually execute the action list.""" 338 env = self.get_build_env() 339 kw = self.get_kw(kw) 340 status = 0 341 for act in self.get_action_list(): 342 #args = (self.get_all_targets(), self.get_all_sources(), env) 343 args = ([], [], env) 344 status = act(*args, **kw) 345 if isinstance(status, SCons.Errors.BuildError): 346 status.executor = self 347 raise status 348 elif status: 349 msg = "Error %s" % status 350 raise SCons.Errors.BuildError( 351 errstr=msg, 352 node=self.batches[0].targets, 353 executor=self, 354 action=act) 355 return status 356 357 # use extra indirection because with new-style objects (Python 2.2 358 # and above) we can't override special methods, and nullify() needs 359 # to be able to do this. 360 361 def __call__(self, target, **kw): 362 return self.do_execute(target, kw) 363 364 def cleanup(self): 365 self._memo = {} 366 367 def add_sources(self, sources): 368 """Add source files to this Executor's list. This is necessary 369 for "multi" Builders that can be called repeatedly to build up 370 a source file list for a given target.""" 371 # TODO(batch): extend to multiple batches 372 assert (len(self.batches) == 1) 373 # TODO(batch): remove duplicates? 374 sources = [x for x in sources if x not in self.batches[0].sources] 375 self.batches[0].sources.extend(sources) 376 377 def get_sources(self): 378 return self.batches[0].sources 379 380 def add_batch(self, targets, sources): 381 """Add pair of associated target and source to this Executor's list. 382 This is necessary for "batch" Builders that can be called repeatedly 383 to build up a list of matching target and source files that will be 384 used in order to update multiple target files at once from multiple 385 corresponding source files, for tools like MSVC that support it.""" 386 self.batches.append(Batch(targets, sources)) 387 388 def prepare(self): 389 """ 390 Preparatory checks for whether this Executor can go ahead 391 and (try to) build its targets. 392 """ 393 for s in self.get_all_sources(): 394 if s.missing(): 395 msg = "Source `%s' not found, needed by target `%s'." 396 raise SCons.Errors.StopError(msg % (s, self.batches[0].targets[0])) 397 398 def add_pre_action(self, action): 399 self.pre_actions.append(action) 400 401 def add_post_action(self, action): 402 self.post_actions.append(action) 403 404 # another extra indirection for new-style objects and nullify... 405 406 def my_str(self): 407 env = self.get_build_env() 408 return "\n".join([action.genstring(self.get_all_targets(), 409 self.get_all_sources(), 410 env) 411 for action in self.get_action_list()]) 412 413 414 def __str__(self): 415 return self.my_str() 416 417 def nullify(self): 418 self.cleanup() 419 self.do_execute = self.do_nothing 420 self.my_str = lambda: '' 421 422 memoizer_counters.append(SCons.Memoize.CountValue('get_contents')) 423 424 def get_contents(self): 425 """Fetch the signature contents. This is the main reason this 426 class exists, so we can compute this once and cache it regardless 427 of how many target or source Nodes there are. 428 """ 429 try: 430 return self._memo['get_contents'] 431 except KeyError: 432 pass 433 env = self.get_build_env() 434 result = "".join([action.get_contents(self.get_all_targets(), 435 self.get_all_sources(), 436 env) 437 for action in self.get_action_list()]) 438 self._memo['get_contents'] = result 439 return result 440 441 def get_timestamp(self): 442 """Fetch a time stamp for this Executor. We don't have one, of 443 course (only files do), but this is the interface used by the 444 timestamp module. 445 """ 446 return 0 447 448 def scan_targets(self, scanner): 449 # TODO(batch): scan by batches 450 self.scan(scanner, self.get_all_targets()) 451 452 def scan_sources(self, scanner): 453 # TODO(batch): scan by batches 454 if self.batches[0].sources: 455 self.scan(scanner, self.get_all_sources()) 456 457 def scan(self, scanner, node_list): 458 """Scan a list of this Executor's files (targets or sources) for 459 implicit dependencies and update all of the targets with them. 460 This essentially short-circuits an N*M scan of the sources for 461 each individual target, which is a hell of a lot more efficient. 462 """ 463 env = self.get_build_env() 464 465 # TODO(batch): scan by batches) 466 deps = [] 467 if scanner: 468 for node in node_list: 469 node.disambiguate() 470 s = scanner.select(node) 471 if not s: 472 continue 473 path = self.get_build_scanner_path(s) 474 deps.extend(node.get_implicit_deps(env, s, path)) 475 else: 476 kw = self.get_kw() 477 for node in node_list: 478 node.disambiguate() 479 scanner = node.get_env_scanner(env, kw) 480 if not scanner: 481 continue 482 scanner = scanner.select(node) 483 if not scanner: 484 continue 485 path = self.get_build_scanner_path(scanner) 486 deps.extend(node.get_implicit_deps(env, scanner, path)) 487 488 deps.extend(self.get_implicit_deps()) 489 490 for tgt in self.get_all_targets(): 491 tgt.add_to_implicit(deps) 492 493 def _get_unignored_sources_key(self, node, ignore=()): 494 return (node,) + tuple(ignore) 495 496 memoizer_counters.append(SCons.Memoize.CountDict('get_unignored_sources', _get_unignored_sources_key)) 497 498 def get_unignored_sources(self, node, ignore=()): 499 key = (node,) + tuple(ignore) 500 try: 501 memo_dict = self._memo['get_unignored_sources'] 502 except KeyError: 503 memo_dict = {} 504 self._memo['get_unignored_sources'] = memo_dict 505 else: 506 try: 507 return memo_dict[key] 508 except KeyError: 509 pass 510 511 if node: 512 # TODO: better way to do this (it's a linear search, 513 # but it may not be critical path)? 514 sourcelist = [] 515 for b in self.batches: 516 if node in b.targets: 517 sourcelist = b.sources 518 break 519 else: 520 sourcelist = self.get_all_sources() 521 if ignore: 522 idict = {} 523 for i in ignore: 524 idict[i] = 1 525 sourcelist = [s for s in sourcelist if s not in idict] 526 527 memo_dict[key] = sourcelist 528 529 return sourcelist 530 531 def get_implicit_deps(self): 532 """Return the executor's implicit dependencies, i.e. the nodes of 533 the commands to be executed.""" 534 result = [] 535 build_env = self.get_build_env() 536 for act in self.get_action_list(): 537 deps = act.get_implicit_deps(self.get_all_targets(), 538 self.get_all_sources(), 539 build_env) 540 result.extend(deps) 541 return result 542 543 544 545_batch_executors = {} 546 547def GetBatchExecutor(key): 548 return _batch_executors[key] 549 550def AddBatchExecutor(key, executor): 551 assert key not in _batch_executors 552 _batch_executors[key] = executor 553 554nullenv = None 555 556 557def get_NullEnvironment(): 558 """Use singleton pattern for Null Environments.""" 559 global nullenv 560 561 import SCons.Util 562 class NullEnvironment(SCons.Util.Null): 563 import SCons.CacheDir 564 _CacheDir_path = None 565 _CacheDir = SCons.CacheDir.CacheDir(None) 566 def get_CacheDir(self): 567 return self._CacheDir 568 569 if not nullenv: 570 nullenv = NullEnvironment() 571 return nullenv 572 573class Null(object): 574 """A null Executor, with a null build Environment, that does 575 nothing when the rest of the methods call it. 576 577 This might be able to disappear when we refactor things to 578 disassociate Builders from Nodes entirely, so we're not 579 going to worry about unit tests for this--at least for now. 580 """ 581 def __init__(self, *args, **kw): 582 if SCons.Debug.track_instances: logInstanceCreation(self, 'Executor.Null') 583 self.batches = [Batch(kw['targets'][:], [])] 584 def get_build_env(self): 585 return get_NullEnvironment() 586 def get_build_scanner_path(self): 587 return None 588 def cleanup(self): 589 pass 590 def prepare(self): 591 pass 592 def get_unignored_sources(self, *args, **kw): 593 return tuple(()) 594 def get_action_targets(self): 595 return [] 596 def get_action_list(self): 597 return [] 598 def get_all_targets(self): 599 return self.batches[0].targets 600 def get_all_sources(self): 601 return self.batches[0].targets[0].sources 602 def get_all_children(self): 603 return self.batches[0].targets[0].children() 604 def get_all_prerequisites(self): 605 return [] 606 def get_action_side_effects(self): 607 return [] 608 def __call__(self, *args, **kw): 609 return 0 610 def get_contents(self): 611 return '' 612 def _morph(self): 613 """Morph this Null executor to a real Executor object.""" 614 batches = self.batches 615 self.__class__ = Executor 616 self.__init__([]) 617 self.batches = batches 618 619 # The following methods require morphing this Null Executor to a 620 # real Executor object. 621 622 def add_pre_action(self, action): 623 self._morph() 624 self.add_pre_action(action) 625 def add_post_action(self, action): 626 self._morph() 627 self.add_post_action(action) 628 def set_action_list(self, action): 629 self._morph() 630 self.set_action_list(action) 631 632# Local Variables: 633# tab-width:4 634# indent-tabs-mode:nil 635# End: 636# vim: set expandtab tabstop=4 shiftwidth=4: 637