1# Status: being ported by Vladimir Prus 2# Base revision: 48649 3# TODO: replace the logging with dout 4 5# Copyright Vladimir Prus 2002. 6# Copyright Rene Rivera 2006. 7# 8# Distributed under the Boost Software License, Version 1.0. 9# (See accompanying file LICENSE_1_0.txt or copy at 10# http://www.boost.org/LICENSE_1_0.txt) 11 12# Manages 'generators' --- objects which can do transformation between different 13# target types and contain algorithm for finding transformation from sources 14# to targets. 15# 16# The main entry point to this module is generators.construct rule. It is given 17# a list of source targets, desired target type and a set of properties. 18# It starts by selecting 'viable generators', which have any chances of producing 19# the desired target type with the required properties. Generators are ranked and 20# a set of most specific ones is selected. 21# 22# The most specific generators have their 'run' methods called, with the properties 23# and list of sources. Each one selects target which can be directly consumed, and 24# tries to convert the remaining ones to the types it can consume. This is done 25# by recursively calling 'construct' with all consumable types. 26# 27# If the generator has collected all the targets it needs, it creates targets 28# corresponding to result, and returns it. When all generators have been run, 29# results of one of them are selected and returned as result. 30# 31# It's quite possible that 'construct' returns more targets that it was asked for. 32# For example, it was asked to target type EXE, but the only found generators produces 33# both EXE and TDS (file with debug) information. The extra target will be returned. 34# 35# Likewise, when generator tries to convert sources to consumable types, it can get 36# more targets that it was asked for. The question is what to do with extra targets. 37# Boost.Build attempts to convert them to requested types, and attempts as early as 38# possible. Specifically, this is done after invoking each generator. (Later I'll 39# document the rationale for trying extra target conversion at that point). 40# 41# That early conversion is not always desirable. Suppose a generator got a source of 42# type Y and must consume one target of type X_1 and one target of type X_2. 43# When converting Y to X_1 extra target of type Y_2 is created. We should not try to 44# convert it to type X_1, because if we do so, the generator will get two targets 45# of type X_1, and will be at loss as to which one to use. Because of that, the 46# 'construct' rule has a parameter, telling if multiple targets can be returned. If 47# the parameter is false, conversion of extra targets is not performed. 48 49 50import re 51import cStringIO 52import os.path 53 54from virtual_target import Subvariant 55from . import virtual_target, type, property_set, property 56from b2.exceptions import BaseBoostBuildException 57from b2.util.logger import * 58from b2.util.utility import * 59from b2.util import set as set_, is_iterable_typed, is_iterable, bjam_signature 60from b2.util.sequence import unique 61import b2.util.sequence as sequence 62from b2.manager import get_manager 63import b2.build.type 64 65def reset (): 66 """ Clear the module state. This is mainly for testing purposes. 67 """ 68 global __generators, __type_to_generators, __generators_for_toolset, __construct_stack 69 global __overrides, __active_generators 70 global __viable_generators_cache, __viable_source_types_cache 71 global __vstg_cached_generators, __vst_cached_types 72 73 __generators = {} 74 __type_to_generators = {} 75 __generators_for_toolset = {} 76 __overrides = {} 77 78 # TODO: can these be global? 79 __construct_stack = [] 80 __viable_generators_cache = {} 81 __viable_source_types_cache = {} 82 __active_generators = [] 83 84 __vstg_cached_generators = [] 85 __vst_cached_types = [] 86 87reset () 88 89_re_separate_types_prefix_and_postfix = re.compile ('([^\\(]*)(\\((.*)%(.*)\\))?') 90_re_match_type = re.compile('([^\\(]*)(\\(.*\\))?') 91 92 93__debug = None 94__indent = "" 95 96def debug(): 97 global __debug 98 if __debug is None: 99 __debug = "--debug-generators" in bjam.variable("ARGV") 100 return __debug 101 102def increase_indent(): 103 global __indent 104 __indent += " " 105 106def decrease_indent(): 107 global __indent 108 __indent = __indent[0:-4] 109 110 111# Updated cached viable source target type information as needed after a new 112# derived target type gets added. This is needed because if a target type is a 113# viable source target type for some generator then all of the target type's 114# derived target types are automatically viable as source target types for the 115# same generator. Does nothing if a non-derived target type is passed to it. 116# 117def update_cached_information_with_a_new_type(type): 118 assert isinstance(type, basestring) 119 base_type = b2.build.type.base(type) 120 121 if base_type: 122 for g in __vstg_cached_generators: 123 if base_type in __viable_source_types_cache.get(g, []): 124 __viable_source_types_cache[g].append(type) 125 126 for t in __vst_cached_types: 127 if base_type in __viable_source_types_cache.get(t, []): 128 __viable_source_types_cache[t].append(type) 129 130# Clears cached viable source target type information except for target types 131# and generators with all source types listed as viable. Should be called when 132# something invalidates those cached values by possibly causing some new source 133# types to become viable. 134# 135def invalidate_extendable_viable_source_target_type_cache(): 136 137 global __vstg_cached_generators 138 generators_with_cached_source_types = __vstg_cached_generators 139 __vstg_cached_generators = [] 140 141 for g in generators_with_cached_source_types: 142 if g in __viable_source_types_cache: 143 if __viable_source_types_cache[g] == ["*"]: 144 __vstg_cached_generators.append(g) 145 else: 146 del __viable_source_types_cache[g] 147 148 global __vst_cached_types 149 types_with_cached_sources_types = __vst_cached_types 150 __vst_cached_types = [] 151 for t in types_with_cached_sources_types: 152 if t in __viable_source_types_cache: 153 if __viable_source_types_cache[t] == ["*"]: 154 __vst_cached_types.append(t) 155 else: 156 del __viable_source_types_cache[t] 157 158def dout(message): 159 if debug(): 160 print __indent + message 161 162 163class InvalidTargetSource(BaseBoostBuildException): 164 """ 165 Should be raised when a target contains a source that is invalid. 166 """ 167 168 169class Generator: 170 """ Creates a generator. 171 manager: the build manager. 172 id: identifies the generator 173 174 rule: the rule which sets up build actions. 175 176 composing: whether generator processes each source target in 177 turn, converting it to required types. 178 Ordinary generators pass all sources together to 179 recusrive generators.construct_types call. 180 181 source_types (optional): types that this generator can handle 182 183 target_types_and_names: types the generator will create and, optionally, names for 184 created targets. Each element should have the form 185 type["(" name-pattern ")"] 186 for example, obj(%_x). Name of generated target will be found 187 by replacing % with the name of source, provided explicit name 188 was not specified. 189 190 requirements (optional) 191 192 NOTE: all subclasses must have a similar signature for clone to work! 193 """ 194 def __init__ (self, id, composing, source_types, target_types_and_names, requirements = []): 195 assert isinstance(id, basestring) 196 assert isinstance(composing, bool) 197 assert is_iterable_typed(source_types, basestring) 198 assert is_iterable_typed(target_types_and_names, basestring) 199 assert is_iterable_typed(requirements, basestring) 200 self.id_ = id 201 self.composing_ = composing 202 self.source_types_ = source_types 203 self.target_types_and_names_ = target_types_and_names 204 self.requirements_ = requirements 205 206 self.target_types_ = [] 207 self.name_prefix_ = [] 208 self.name_postfix_ = [] 209 210 for e in target_types_and_names: 211 # Create three parallel lists: one with the list of target types, 212 # and two other with prefixes and postfixes to be added to target 213 # name. We use parallel lists for prefix and postfix (as opposed 214 # to mapping), because given target type might occur several times, 215 # for example "H H(%_symbols)". 216 m = _re_separate_types_prefix_and_postfix.match (e) 217 218 if not m: 219 raise BaseException ("Invalid type and name '%s' in declaration of type '%s'" % (e, id)) 220 221 target_type = m.group (1) 222 if not target_type: target_type = '' 223 prefix = m.group (3) 224 if not prefix: prefix = '' 225 postfix = m.group (4) 226 if not postfix: postfix = '' 227 228 self.target_types_.append (target_type) 229 self.name_prefix_.append (prefix) 230 self.name_postfix_.append (postfix) 231 232 for x in self.source_types_: 233 type.validate (x) 234 235 for x in self.target_types_: 236 type.validate (x) 237 238 def clone (self, new_id, new_toolset_properties): 239 """ Returns another generator which differers from $(self) in 240 - id 241 - value to <toolset> feature in properties 242 """ 243 assert isinstance(new_id, basestring) 244 assert is_iterable_typed(new_toolset_properties, basestring) 245 return self.__class__ (new_id, 246 self.composing_, 247 self.source_types_, 248 self.target_types_and_names_, 249 # Note: this does not remove any subfeatures of <toolset> 250 # which might cause problems 251 property.change (self.requirements_, '<toolset>') + new_toolset_properties) 252 253 def clone_and_change_target_type(self, base, type): 254 """Creates another generator that is the same as $(self), except that 255 if 'base' is in target types of $(self), 'type' will in target types 256 of the new generator.""" 257 assert isinstance(base, basestring) 258 assert isinstance(type, basestring) 259 target_types = [] 260 for t in self.target_types_and_names_: 261 m = _re_match_type.match(t) 262 assert m 263 264 if m.group(1) == base: 265 if m.group(2): 266 target_types.append(type + m.group(2)) 267 else: 268 target_types.append(type) 269 else: 270 target_types.append(t) 271 272 return self.__class__(self.id_, self.composing_, 273 self.source_types_, 274 target_types, 275 self.requirements_) 276 277 278 def id(self): 279 return self.id_ 280 281 def source_types (self): 282 """ Returns the list of target type the generator accepts. 283 """ 284 return self.source_types_ 285 286 def target_types (self): 287 """ Returns the list of target types that this generator produces. 288 It is assumed to be always the same -- i.e. it cannot change depending 289 list of sources. 290 """ 291 return self.target_types_ 292 293 def requirements (self): 294 """ Returns the required properties for this generator. Properties 295 in returned set must be present in build properties if this 296 generator is to be used. If result has grist-only element, 297 that build properties must include some value of that feature. 298 """ 299 return self.requirements_ 300 301 def match_rank (self, ps): 302 """ Returns true if the generator can be run with the specified 303 properties. 304 """ 305 # See if generator's requirements are satisfied by 306 # 'properties'. Treat a feature name in requirements 307 # (i.e. grist-only element), as matching any value of the 308 # feature. 309 assert isinstance(ps, property_set.PropertySet) 310 all_requirements = self.requirements () 311 312 property_requirements = [] 313 feature_requirements = [] 314 # This uses strings because genenator requirements allow 315 # the '<feature>' syntax without value and regular validation 316 # is not happy about that. 317 for r in all_requirements: 318 if get_value (r): 319 property_requirements.append (r) 320 321 else: 322 feature_requirements.append (r) 323 324 return all(ps.get(get_grist(s)) == [get_value(s)] for s in property_requirements) \ 325 and all(ps.get(get_grist(s)) for s in feature_requirements) 326 327 def run (self, project, name, prop_set, sources): 328 """ Tries to invoke this generator on the given sources. Returns a 329 list of generated targets (instances of 'virtual-target'). 330 331 project: Project for which the targets are generated. 332 333 name: Determines the name of 'name' attribute for 334 all generated targets. See 'generated_targets' method. 335 336 prop_set: Desired properties for generated targets. 337 338 sources: Source targets. 339 """ 340 if __debug__: 341 from .targets import ProjectTarget 342 assert isinstance(project, ProjectTarget) 343 # intermediary targets don't have names, so None is possible 344 assert isinstance(name, basestring) or name is None 345 assert isinstance(prop_set, property_set.PropertySet) 346 assert is_iterable_typed(sources, virtual_target.VirtualTarget) 347 if project.manager ().logger ().on (): 348 project.manager ().logger ().log (__name__, " generator '%s'" % self.id_) 349 project.manager ().logger ().log (__name__, " composing: '%s'" % self.composing_) 350 351 if not sources: 352 s = 'An empty source list was passed in to the "{}" generator'.format(self.id_) 353 if name: 354 s += ' for target "{}"'.format(name) 355 raise InvalidTargetSource(s) 356 357 if not self.composing_ and len (sources) > 1 and len (self.source_types_) > 1: 358 raise BaseException ("Unsupported source/source_type combination") 359 360 # We don't run composing generators if no name is specified. The reason 361 # is that composing generator combines several targets, which can have 362 # different names, and it cannot decide which name to give for produced 363 # target. Therefore, the name must be passed. 364 # 365 # This in effect, means that composing generators are runnable only 366 # at top-level of transofrmation graph, or if name is passed explicitly. 367 # Thus, we dissallow composing generators in the middle. For example, the 368 # transofrmation CPP -> OBJ -> STATIC_LIB -> RSP -> EXE won't be allowed 369 # (the OBJ -> STATIC_LIB generator is composing) 370 if not self.composing_ or name: 371 return self.run_really (project, name, prop_set, sources) 372 else: 373 return [] 374 375 def run_really (self, project, name, prop_set, sources): 376 if __debug__: 377 from .targets import ProjectTarget 378 assert isinstance(project, ProjectTarget) 379 # intermediary targets don't have names, so None is possible 380 assert isinstance(name, basestring) or name is None 381 assert isinstance(prop_set, property_set.PropertySet) 382 assert is_iterable_typed(sources, virtual_target.VirtualTarget) 383 # consumed: Targets that this generator will consume directly. 384 385 if self.composing_: 386 consumed = self.convert_multiple_sources_to_consumable_types (project, prop_set, sources) 387 else: 388 consumed = self.convert_to_consumable_types (project, name, prop_set, sources) 389 390 result = [] 391 if consumed: 392 result = self.construct_result (consumed, project, name, prop_set) 393 394 if result: 395 if project.manager ().logger ().on (): 396 project.manager ().logger ().log (__name__, " SUCCESS: ", result) 397 398 else: 399 project.manager ().logger ().log (__name__, " FAILURE") 400 401 return result 402 403 def construct_result (self, consumed, project, name, prop_set): 404 """ Constructs the dependency graph that will be returned by this 405 generator. 406 consumed: Already prepared list of consumable targets 407 If generator requires several source files will contain 408 exactly len $(self.source_types_) targets with matching types 409 Otherwise, might contain several targets with the type of 410 self.source_types_ [0] 411 project: 412 name: 413 prop_set: Properties to be used for all actions create here 414 """ 415 if __debug__: 416 from .targets import ProjectTarget 417 assert is_iterable_typed(consumed, virtual_target.VirtualTarget) 418 assert isinstance(project, ProjectTarget) 419 assert isinstance(name, basestring) or name is None 420 assert isinstance(prop_set, property_set.PropertySet) 421 result = [] 422 # If this is 1->1 transformation, apply it to all consumed targets in order. 423 if len (self.source_types_) < 2 and not self.composing_: 424 425 for r in consumed: 426 result.extend(self.generated_targets([r], prop_set, project, name)) 427 elif consumed: 428 result.extend(self.generated_targets(consumed, prop_set, project, name)) 429 430 return result 431 432 def determine_target_name(self, fullname): 433 assert isinstance(fullname, basestring) 434 # Determine target name from fullname (maybe including path components) 435 # Place optional prefix and postfix around basename 436 437 dir = os.path.dirname(fullname) 438 name = os.path.basename(fullname) 439 idx = name.find(".") 440 if idx != -1: 441 name = name[:idx] 442 443 if dir and not ".." in dir and not os.path.isabs(dir): 444 # Relative path is always relative to the source 445 # directory. Retain it, so that users can have files 446 # with the same in two different subdirectories. 447 name = dir + "/" + name 448 449 return name 450 451 def determine_output_name(self, sources): 452 """Determine the name of the produced target from the 453 names of the sources.""" 454 assert is_iterable_typed(sources, virtual_target.VirtualTarget) 455 456 # The simple case if when a name 457 # of source has single dot. Then, we take the part before 458 # dot. Several dots can be caused by: 459 # - Using source file like a.host.cpp 460 # - A type which suffix has a dot. Say, we can 461 # type 'host_cpp' with extension 'host.cpp'. 462 # In the first case, we want to take the part till the last 463 # dot. In the second case -- no sure, but for now take 464 # the part till the last dot too. 465 name = os.path.splitext(sources[0].name())[0] 466 467 for s in sources[1:]: 468 n2 = os.path.splitext(s.name()) 469 if n2 != name: 470 get_manager().errors()( 471 "%s: source targets have different names: cannot determine target name" 472 % (self.id_)) 473 474 # Names of sources might include directory. We should strip it. 475 return self.determine_target_name(sources[0].name()) 476 477 478 def generated_targets (self, sources, prop_set, project, name): 479 """ Constructs targets that are created after consuming 'sources'. 480 The result will be the list of virtual-target, which the same length 481 as 'target_types' attribute and with corresponding types. 482 483 When 'name' is empty, all source targets must have the same value of 484 the 'name' attribute, which will be used instead of the 'name' argument. 485 486 The value of 'name' attribute for each generated target will be equal to 487 the 'name' parameter if there's no name pattern for this type. Otherwise, 488 the '%' symbol in the name pattern will be replaced with the 'name' parameter 489 to obtain the 'name' attribute. 490 491 For example, if targets types are T1 and T2(with name pattern "%_x"), suffixes 492 for T1 and T2 are .t1 and t2, and source if foo.z, then created files would 493 be "foo.t1" and "foo_x.t2". The 'name' attribute actually determined the 494 basename of a file. 495 496 Note that this pattern mechanism has nothing to do with implicit patterns 497 in make. It's a way to produce target which name is different for name of 498 source. 499 """ 500 if __debug__: 501 from .targets import ProjectTarget 502 assert is_iterable_typed(sources, virtual_target.VirtualTarget) 503 assert isinstance(prop_set, property_set.PropertySet) 504 assert isinstance(project, ProjectTarget) 505 assert isinstance(name, basestring) or name is None 506 if not name: 507 name = self.determine_output_name(sources) 508 509 # Assign an action for each target 510 action = self.action_class() 511 a = action(project.manager(), sources, self.id_, prop_set) 512 513 # Create generated target for each target type. 514 targets = [] 515 pre = self.name_prefix_ 516 post = self.name_postfix_ 517 for t in self.target_types_: 518 basename = os.path.basename(name) 519 generated_name = pre[0] + basename + post[0] 520 generated_name = os.path.join(os.path.dirname(name), generated_name) 521 pre = pre[1:] 522 post = post[1:] 523 524 targets.append(virtual_target.FileTarget(generated_name, t, project, a)) 525 526 return [ project.manager().virtual_targets().register(t) for t in targets ] 527 528 def convert_to_consumable_types (self, project, name, prop_set, sources, only_one=False): 529 """ Attempts to convert 'source' to the types that this generator can 530 handle. The intention is to produce the set of targets can should be 531 used when generator is run. 532 only_one: convert 'source' to only one of source types 533 if there's more that one possibility, report an 534 error. 535 536 Returns a pair: 537 consumed: all targets that can be consumed. 538 """ 539 if __debug__: 540 from .targets import ProjectTarget 541 assert isinstance(name, basestring) or name is None 542 assert isinstance(project, ProjectTarget) 543 assert isinstance(prop_set, property_set.PropertySet) 544 assert is_iterable_typed(sources, virtual_target.VirtualTarget) 545 assert isinstance(only_one, bool) 546 consumed = [] 547 missing_types = [] 548 549 if len (sources) > 1: 550 # Don't know how to handle several sources yet. Just try 551 # to pass the request to other generator 552 missing_types = self.source_types_ 553 554 else: 555 (c, m) = self.consume_directly (sources [0]) 556 consumed += c 557 missing_types += m 558 559 # No need to search for transformation if 560 # some source type has consumed source and 561 # no more source types are needed. 562 if only_one and consumed: 563 missing_types = [] 564 565 #TODO: we should check that only one source type 566 #if create of 'only_one' is true. 567 # TODO: consider if consuned/bypassed separation should 568 # be done by 'construct_types'. 569 570 if missing_types: 571 transformed = construct_types (project, name, missing_types, prop_set, sources) 572 573 # Add targets of right type to 'consumed'. Add others to 574 # 'bypassed'. The 'generators.construct' rule has done 575 # its best to convert everything to the required type. 576 # There's no need to rerun it on targets of different types. 577 578 # NOTE: ignoring usage requirements 579 for t in transformed[1]: 580 if t.type() in missing_types: 581 consumed.append(t) 582 583 consumed = unique(consumed) 584 585 return consumed 586 587 588 def convert_multiple_sources_to_consumable_types (self, project, prop_set, sources): 589 """ Converts several files to consumable types. 590 """ 591 if __debug__: 592 from .targets import ProjectTarget 593 594 assert isinstance(project, ProjectTarget) 595 assert isinstance(prop_set, property_set.PropertySet) 596 assert is_iterable_typed(sources, virtual_target.VirtualTarget) 597 if not self.source_types_: 598 return list(sources) 599 600 acceptable_types = set() 601 for t in self.source_types_: 602 acceptable_types.update(type.all_derived(t)) 603 604 result = [] 605 for source in sources: 606 if source.type() not in acceptable_types: 607 transformed = construct_types( 608 project, None,self.source_types_, prop_set, [source]) 609 # construct_types returns [prop_set, [targets]] 610 for t in transformed[1]: 611 if t.type() in self.source_types_: 612 result.append(t) 613 if not transformed: 614 project.manager().logger().log(__name__, " failed to convert ", source) 615 else: 616 result.append(source) 617 618 result = sequence.unique(result, stable=True) 619 return result 620 621 622 623 def consume_directly (self, source): 624 assert isinstance(source, virtual_target.VirtualTarget) 625 real_source_type = source.type () 626 627 # If there are no source types, we can consume anything 628 source_types = self.source_types() 629 if not source_types: 630 source_types = [real_source_type] 631 632 consumed = [] 633 missing_types = [] 634 for st in source_types: 635 # The 'source' if of right type already) 636 if real_source_type == st or type.is_derived (real_source_type, st): 637 consumed = [source] 638 639 else: 640 missing_types.append (st) 641 642 return (consumed, missing_types) 643 644 def action_class (self): 645 """ Returns the class to be used to actions. Default implementation 646 returns "action". 647 """ 648 return virtual_target.Action 649 650 651def find (id): 652 """ Finds the generator with id. Returns None if not found. 653 """ 654 assert isinstance(id, basestring) 655 return __generators.get (id, None) 656 657def register (g): 658 """ Registers new generator instance 'g'. 659 """ 660 assert isinstance(g, Generator) 661 id = g.id() 662 663 __generators [id] = g 664 665 # A generator can produce several targets of the 666 # same type. We want unique occurrence of that generator 667 # in .generators.$(t) in that case, otherwise, it will 668 # be tried twice and we'll get false ambiguity. 669 for t in sequence.unique(g.target_types()): 670 __type_to_generators.setdefault(t, []).append(g) 671 672 # Update the set of generators for toolset 673 674 # TODO: should we check that generator with this id 675 # is not already registered. For example, the fop.jam 676 # module intentionally declared two generators with the 677 # same id, so such check will break it. 678 679 # Some generators have multiple periods in their name, so the 680 # normal $(id:S=) won't generate the right toolset name. 681 # e.g. if id = gcc.compile.c++, then 682 # .generators-for-toolset.$(id:S=) will append to 683 # .generators-for-toolset.gcc.compile, which is a separate 684 # value from .generators-for-toolset.gcc. Correcting this 685 # makes generator inheritance work properly. 686 # See also inherit-generators in module toolset 687 base = id.split ('.', 100) [0] 688 689 __generators_for_toolset.setdefault(base, []).append(g) 690 691 # After adding a new generator that can construct new target types, we need 692 # to clear the related cached viable source target type information for 693 # constructing a specific target type or using a specific generator. Cached 694 # viable source target type lists affected by this are those containing any 695 # of the target types constructed by the new generator or any of their base 696 # target types. 697 # 698 # A more advanced alternative to clearing that cached viable source target 699 # type information would be to expand it with additional source types or 700 # even better - mark it as needing to be expanded on next use. 701 # 702 # For now we just clear all the cached viable source target type information 703 # that does not simply state 'all types' and may implement a more detailed 704 # algorithm later on if it becomes needed. 705 706 invalidate_extendable_viable_source_target_type_cache() 707 708 709def check_register_types(fn): 710 def wrapper(id, source_types, target_types, requirements=[]): 711 assert isinstance(id, basestring) 712 assert is_iterable_typed(source_types, basestring) 713 assert is_iterable_typed(target_types, basestring) 714 assert is_iterable_typed(requirements, basestring) 715 return fn(id, source_types, target_types, requirements=requirements) 716 wrapper.__name__ = fn.__name__ 717 wrapper.__doc__ = fn.__doc__ 718 return wrapper 719 720 721@bjam_signature([['id'], ['source_types', '*'], ['target_types', '*'], ['requirements', '*']]) 722@check_register_types 723def register_standard (id, source_types, target_types, requirements = []): 724 """ Creates new instance of the 'generator' class and registers it. 725 Returns the creates instance. 726 Rationale: the instance is returned so that it's possible to first register 727 a generator and then call 'run' method on that generator, bypassing all 728 generator selection. 729 """ 730 g = Generator (id, False, source_types, target_types, requirements) 731 register (g) 732 return g 733 734 735@check_register_types 736def register_composing (id, source_types, target_types, requirements = []): 737 g = Generator (id, True, source_types, target_types, requirements) 738 register (g) 739 return g 740 741def generators_for_toolset (toolset): 742 """ Returns all generators which belong to 'toolset'. 743 """ 744 assert isinstance(toolset, basestring) 745 return __generators_for_toolset.get(toolset, []) 746 747def override (overrider_id, overridee_id): 748 """Make generator 'overrider-id' be preferred to 749 'overridee-id'. If, when searching for generators 750 that could produce a target of certain type, 751 both those generators are among viable generators, 752 the overridden generator is immediately discarded. 753 754 The overridden generators are discarded immediately 755 after computing the list of viable generators, before 756 running any of them.""" 757 assert isinstance(overrider_id, basestring) 758 assert isinstance(overridee_id, basestring) 759 760 __overrides.setdefault(overrider_id, []).append(overridee_id) 761 762def __viable_source_types_real (target_type): 763 """ Returns a list of source type which can possibly be converted 764 to 'target_type' by some chain of generator invocation. 765 766 More formally, takes all generators for 'target_type' and 767 returns union of source types for those generators and result 768 of calling itself recusrively on source types. 769 """ 770 assert isinstance(target_type, basestring) 771 generators = [] 772 773 # 't0' is the initial list of target types we need to process to get a list 774 # of their viable source target types. New target types will not be added to 775 # this list. 776 t0 = type.all_bases (target_type) 777 778 779 # 't' is the list of target types which have not yet been processed to get a 780 # list of their viable source target types. This list will get expanded as 781 # we locate more target types to process. 782 t = t0 783 784 result = [] 785 while t: 786 # Find all generators for current type. 787 # Unlike 'find_viable_generators' we don't care about prop_set. 788 generators = __type_to_generators.get (t [0], []) 789 t = t[1:] 790 791 for g in generators: 792 if not g.source_types(): 793 # Empty source types -- everything can be accepted 794 result = "*" 795 # This will terminate outer loop. 796 t = None 797 break 798 799 for source_type in g.source_types (): 800 if not source_type in result: 801 # If generator accepts 'source_type' it 802 # will happily accept any type derived from it 803 all = type.all_derived (source_type) 804 for n in all: 805 if not n in result: 806 807 # Here there is no point in adding target types to 808 # the list of types to process in case they are or 809 # have already been on that list. We optimize this 810 # check by realizing that we only need to avoid the 811 # original target type's base types. Other target 812 # types that are or have been on the list of target 813 # types to process have been added to the 'result' 814 # list as well and have thus already been eliminated 815 # by the previous if. 816 if not n in t0: 817 t.append (n) 818 result.append (n) 819 820 return result 821 822 823def viable_source_types (target_type): 824 """ Helper rule, caches the result of '__viable_source_types_real'. 825 """ 826 assert isinstance(target_type, basestring) 827 if target_type not in __viable_source_types_cache: 828 __vst_cached_types.append(target_type) 829 __viable_source_types_cache [target_type] = __viable_source_types_real (target_type) 830 return __viable_source_types_cache [target_type] 831 832def viable_source_types_for_generator_real (generator): 833 """ Returns the list of source types, which, when passed to 'run' 834 method of 'generator', has some change of being eventually used 835 (probably after conversion by other generators) 836 """ 837 assert isinstance(generator, Generator) 838 source_types = generator.source_types () 839 840 if not source_types: 841 # If generator does not specify any source types, 842 # it might be special generator like builtin.lib-generator 843 # which just relays to other generators. Return '*' to 844 # indicate that any source type is possibly OK, since we don't 845 # know for sure. 846 return ['*'] 847 848 else: 849 result = [] 850 for s in source_types: 851 viable_sources = viable_source_types(s) 852 if viable_sources == "*": 853 result = ["*"] 854 break 855 else: 856 result.extend(type.all_derived(s) + viable_sources) 857 return unique(result) 858 859def viable_source_types_for_generator (generator): 860 """ Caches the result of 'viable_source_types_for_generator'. 861 """ 862 assert isinstance(generator, Generator) 863 if generator not in __viable_source_types_cache: 864 __vstg_cached_generators.append(generator) 865 __viable_source_types_cache[generator] = viable_source_types_for_generator_real (generator) 866 867 return __viable_source_types_cache[generator] 868 869def try_one_generator_really (project, name, generator, target_type, properties, sources): 870 """ Returns usage requirements + list of created targets. 871 """ 872 if __debug__: 873 from .targets import ProjectTarget 874 assert isinstance(project, ProjectTarget) 875 assert isinstance(name, basestring) or name is None 876 assert isinstance(generator, Generator) 877 assert isinstance(target_type, basestring) 878 assert isinstance(properties, property_set.PropertySet) 879 assert is_iterable_typed(sources, virtual_target.VirtualTarget) 880 targets = generator.run (project, name, properties, sources) 881 882 usage_requirements = [] 883 success = False 884 885 dout("returned " + str(targets)) 886 887 if targets: 888 success = True; 889 890 if isinstance (targets[0], property_set.PropertySet): 891 usage_requirements = targets [0] 892 targets = targets [1] 893 894 else: 895 usage_requirements = property_set.empty () 896 897 dout( " generator" + generator.id() + " spawned ") 898 # generators.dout [ indent ] " " $(targets) ; 899# if $(usage-requirements) 900# { 901# generators.dout [ indent ] " with usage requirements:" $(x) ; 902# } 903 904 if success: 905 return (usage_requirements, targets) 906 else: 907 return None 908 909def try_one_generator (project, name, generator, target_type, properties, sources): 910 """ Checks if generator invocation can be pruned, because it's guaranteed 911 to fail. If so, quickly returns empty list. Otherwise, calls 912 try_one_generator_really. 913 """ 914 if __debug__: 915 from .targets import ProjectTarget 916 assert isinstance(project, ProjectTarget) 917 assert isinstance(name, basestring) or name is None 918 assert isinstance(generator, Generator) 919 assert isinstance(target_type, basestring) 920 assert isinstance(properties, property_set.PropertySet) 921 assert is_iterable_typed(sources, virtual_target.VirtualTarget) 922 source_types = [] 923 924 for s in sources: 925 source_types.append (s.type ()) 926 927 viable_source_types = viable_source_types_for_generator (generator) 928 929 if source_types and viable_source_types != ['*'] and\ 930 not set_.intersection (source_types, viable_source_types): 931 if project.manager ().logger ().on (): 932 id = generator.id () 933 project.manager ().logger ().log (__name__, "generator '%s' pruned" % id) 934 project.manager ().logger ().log (__name__, "source_types" '%s' % source_types) 935 project.manager ().logger ().log (__name__, "viable_source_types '%s'" % viable_source_types) 936 937 return [] 938 939 else: 940 return try_one_generator_really (project, name, generator, target_type, properties, sources) 941 942 943def construct_types (project, name, target_types, prop_set, sources): 944 945 if __debug__: 946 from .targets import ProjectTarget 947 assert isinstance(project, ProjectTarget) 948 assert isinstance(name, basestring) or name is None 949 assert is_iterable_typed(target_types, basestring) 950 assert isinstance(prop_set, property_set.PropertySet) 951 assert is_iterable_typed(sources, virtual_target.VirtualTarget) 952 953 result = [] 954 usage_requirements = property_set.empty() 955 956 for t in target_types: 957 r = construct (project, name, t, prop_set, sources) 958 959 if r: 960 (ur, targets) = r 961 usage_requirements = usage_requirements.add(ur) 962 result.extend(targets) 963 964 # TODO: have to introduce parameter controlling if 965 # several types can be matched and add appropriate 966 # checks 967 968 # TODO: need to review the documentation for 969 # 'construct' to see if it should return $(source) even 970 # if nothing can be done with it. Currents docs seem to 971 # imply that, contrary to the behaviour. 972 if result: 973 return (usage_requirements, result) 974 975 else: 976 return (usage_requirements, sources) 977 978def __ensure_type (targets): 979 """ Ensures all 'targets' have types. If this is not so, exists with 980 error. 981 """ 982 assert is_iterable_typed(targets, virtual_target.VirtualTarget) 983 for t in targets: 984 if not t.type (): 985 get_manager().errors()("target '%s' has no type" % str (t)) 986 987def find_viable_generators_aux (target_type, prop_set): 988 """ Returns generators which can be used to construct target of specified type 989 with specified properties. Uses the following algorithm: 990 - iterates over requested target_type and all it's bases (in the order returned bt 991 type.all-bases. 992 - for each type find all generators that generate that type and which requirements 993 are satisfied by properties. 994 - if the set of generators is not empty, returns that set. 995 996 Note: this algorithm explicitly ignores generators for base classes if there's 997 at least one generator for requested target_type. 998 """ 999 assert isinstance(target_type, basestring) 1000 assert isinstance(prop_set, property_set.PropertySet) 1001 # Select generators that can create the required target type. 1002 viable_generators = [] 1003 initial_generators = [] 1004 1005 from . import type 1006 1007 # Try all-type generators first. Assume they have 1008 # quite specific requirements. 1009 all_bases = type.all_bases(target_type) 1010 1011 for t in all_bases: 1012 1013 initial_generators = __type_to_generators.get(t, []) 1014 1015 if initial_generators: 1016 dout("there are generators for this type") 1017 if t != target_type: 1018 # We're here, when no generators for target-type are found, 1019 # but there are some generators for a base type. 1020 # We'll try to use them, but they will produce targets of 1021 # base type, not of 'target-type'. So, we clone the generators 1022 # and modify the list of target types. 1023 generators2 = [] 1024 for g in initial_generators[:]: 1025 # generators.register adds generator to the list of generators 1026 # for toolsets, which is a bit strange, but should work. 1027 # That list is only used when inheriting toolset, which 1028 # should have being done before generators are run. 1029 ng = g.clone_and_change_target_type(t, target_type) 1030 generators2.append(ng) 1031 register(ng) 1032 1033 initial_generators = generators2 1034 break 1035 1036 for g in initial_generators: 1037 dout("trying generator " + g.id() 1038 + "(" + str(g.source_types()) + "->" + str(g.target_types()) + ")") 1039 1040 m = g.match_rank(prop_set) 1041 if m: 1042 dout(" is viable") 1043 viable_generators.append(g) 1044 1045 return viable_generators 1046 1047def find_viable_generators (target_type, prop_set): 1048 assert isinstance(target_type, basestring) 1049 assert isinstance(prop_set, property_set.PropertySet) 1050 key = target_type + '.' + str (prop_set) 1051 1052 l = __viable_generators_cache.get (key, None) 1053 if not l: 1054 l = [] 1055 1056 if not l: 1057 l = find_viable_generators_aux (target_type, prop_set) 1058 1059 __viable_generators_cache [key] = l 1060 1061 viable_generators = [] 1062 for g in l: 1063 # Avoid trying the same generator twice on different levels. 1064 # TODO: is this really used? 1065 if not g in __active_generators: 1066 viable_generators.append (g) 1067 else: 1068 dout(" generator %s is active, discarding" % g.id()) 1069 1070 # Generators which override 'all'. 1071 all_overrides = [] 1072 1073 # Generators which are overridden 1074 overriden_ids = [] 1075 1076 for g in viable_generators: 1077 id = g.id () 1078 1079 this_overrides = __overrides.get (id, []) 1080 1081 if this_overrides: 1082 overriden_ids.extend (this_overrides) 1083 if 'all' in this_overrides: 1084 all_overrides.append (g) 1085 1086 if all_overrides: 1087 viable_generators = all_overrides 1088 1089 return [g for g in viable_generators if not g.id() in overriden_ids] 1090 1091def __construct_really (project, name, target_type, prop_set, sources): 1092 """ Attempts to construct target by finding viable generators, running them 1093 and selecting the dependency graph. 1094 """ 1095 if __debug__: 1096 from .targets import ProjectTarget 1097 assert isinstance(project, ProjectTarget) 1098 assert isinstance(name, basestring) or name is None 1099 assert isinstance(target_type, basestring) 1100 assert isinstance(prop_set, property_set.PropertySet) 1101 assert is_iterable_typed(sources, virtual_target.VirtualTarget) 1102 viable_generators = find_viable_generators (target_type, prop_set) 1103 1104 result = [] 1105 1106 dout(" *** %d viable generators" % len (viable_generators)) 1107 1108 generators_that_succeeded = [] 1109 1110 for g in viable_generators: 1111 __active_generators.append(g) 1112 r = try_one_generator (project, name, g, target_type, prop_set, sources) 1113 del __active_generators[-1] 1114 1115 if r: 1116 generators_that_succeeded.append(g) 1117 if result: 1118 output = cStringIO.StringIO() 1119 print >>output, "ambiguity found when searching for best transformation" 1120 print >>output, "Trying to produce type '%s' from: " % (target_type) 1121 for s in sources: 1122 print >>output, " - " + s.str() 1123 print >>output, "Generators that succeeded:" 1124 for g in generators_that_succeeded: 1125 print >>output, " - " + g.id() 1126 print >>output, "First generator produced: " 1127 for t in result[1:]: 1128 print >>output, " - " + str(t) 1129 print >>output, "Second generator produced:" 1130 for t in r[1:]: 1131 print >>output, " - " + str(t) 1132 get_manager().errors()(output.getvalue()) 1133 else: 1134 result = r; 1135 1136 return result; 1137 1138 1139def construct (project, name, target_type, prop_set, sources, top_level=False): 1140 """ Attempts to create target of 'target-type' with 'properties' 1141 from 'sources'. The 'sources' are treated as a collection of 1142 *possible* ingridients -- i.e. it is not required to consume 1143 them all. If 'multiple' is true, the rule is allowed to return 1144 several targets of 'target-type'. 1145 1146 Returns a list of target. When this invocation is first instance of 1147 'construct' in stack, returns only targets of requested 'target-type', 1148 otherwise, returns also unused sources and additionally generated 1149 targets. 1150 1151 If 'top-level' is set, does not suppress generators that are already 1152 used in the stack. This may be useful in cases where a generator 1153 has to build a metatarget -- for example a target corresponding to 1154 built tool. 1155 """ 1156 if __debug__: 1157 from .targets import ProjectTarget 1158 assert isinstance(project, ProjectTarget) 1159 assert isinstance(name, basestring) or name is None 1160 assert isinstance(target_type, basestring) 1161 assert isinstance(prop_set, property_set.PropertySet) 1162 assert is_iterable_typed(sources, virtual_target.VirtualTarget) 1163 assert isinstance(top_level, bool) 1164 global __active_generators 1165 if top_level: 1166 saved_active = __active_generators 1167 __active_generators = [] 1168 1169 global __construct_stack 1170 if not __construct_stack: 1171 __ensure_type (sources) 1172 1173 __construct_stack.append (1) 1174 1175 increase_indent () 1176 1177 if project.manager().logger().on(): 1178 dout( "*** construct " + target_type) 1179 1180 for s in sources: 1181 dout(" from " + str(s)) 1182 1183 project.manager().logger().log (__name__, " properties: ", prop_set.raw ()) 1184 1185 result = __construct_really(project, name, target_type, prop_set, sources) 1186 1187 decrease_indent() 1188 1189 __construct_stack = __construct_stack [1:] 1190 1191 if top_level: 1192 __active_generators = saved_active 1193 1194 return result 1195 1196def add_usage_requirements (result, raw_properties): 1197 if result: 1198 if isinstance (result[0], property_set.PropertySet): 1199 return (result[0].add_raw(raw_properties), result[1]) 1200 else: 1201 return (property_set.create(raw_properties), result) 1202 #if [ class.is-a $(result[1]) : property-set ] 1203 #{ 1204 # return [ $(result[1]).add-raw $(raw-properties) ] $(result[2-]) ; 1205 #} 1206 #else 1207 #{ 1208 # return [ property-set.create $(raw-properties) ] $(result) ; 1209 #} 1210