1# Samba automatic dependency handling and project rules 2 3import os, sys, re 4 5from waflib import Build, Options, Logs, Utils, Errors 6from waflib.Logs import debug 7from waflib.Configure import conf 8from waflib import ConfigSet 9 10from samba_bundled import BUILTIN_LIBRARY 11from samba_utils import LOCAL_CACHE, TO_LIST, get_tgt_list, unique_list 12from samba_autoconf import library_flags 13 14@conf 15def ADD_GLOBAL_DEPENDENCY(ctx, dep): 16 '''add a dependency for all binaries and libraries''' 17 if not 'GLOBAL_DEPENDENCIES' in ctx.env: 18 ctx.env.GLOBAL_DEPENDENCIES = [] 19 ctx.env.GLOBAL_DEPENDENCIES.append(dep) 20 21 22@conf 23def BREAK_CIRCULAR_LIBRARY_DEPENDENCIES(ctx): 24 '''indicate that circular dependencies between libraries should be broken.''' 25 ctx.env.ALLOW_CIRCULAR_LIB_DEPENDENCIES = True 26 27 28@conf 29def SET_SYSLIB_DEPS(conf, target, deps): 30 '''setup some implied dependencies for a SYSLIB''' 31 cache = LOCAL_CACHE(conf, 'SYSLIB_DEPS') 32 cache[target] = deps 33 34 35def expand_subsystem_deps(bld): 36 '''expand the reverse dependencies resulting from subsystem 37 attributes of modules. This is walking over the complete list 38 of declared subsystems, and expands the samba_deps_extended list for any 39 module<->subsystem dependencies''' 40 41 subsystem_list = LOCAL_CACHE(bld, 'INIT_FUNCTIONS') 42 targets = LOCAL_CACHE(bld, 'TARGET_TYPE') 43 44 for subsystem_name in subsystem_list: 45 bld.ASSERT(subsystem_name in targets, "Subsystem target %s not declared" % subsystem_name) 46 type = targets[subsystem_name] 47 if type == 'DISABLED' or type == 'EMPTY': 48 continue 49 50 # for example, 51 # subsystem_name = dcerpc_server (a subsystem) 52 # subsystem = dcerpc_server (a subsystem object) 53 # module_name = rpc_epmapper (a module within the dcerpc_server subsystem) 54 # module = rpc_epmapper (a module object within the dcerpc_server subsystem) 55 56 subsystem = bld.get_tgen_by_name(subsystem_name) 57 bld.ASSERT(subsystem is not None, "Unable to find subsystem %s" % subsystem_name) 58 for d in subsystem_list[subsystem_name]: 59 module_name = d['TARGET'] 60 module_type = targets[module_name] 61 if module_type in ['DISABLED', 'EMPTY']: 62 continue 63 bld.ASSERT(subsystem is not None, 64 "Subsystem target %s for %s (%s) not found" % (subsystem_name, module_name, module_type)) 65 if module_type in ['SUBSYSTEM']: 66 # if a module is a plain object type (not a library) then the 67 # subsystem it is part of needs to have it as a dependency, so targets 68 # that depend on this subsystem get the modules of that subsystem 69 subsystem.samba_deps_extended.append(module_name) 70 subsystem.samba_deps_extended = unique_list(subsystem.samba_deps_extended) 71 72 73 74def build_dependencies(self): 75 '''This builds the dependency list for a target. It runs after all the targets are declared 76 77 The reason this is not just done in the SAMBA_*() rules is that we have no way of knowing 78 the full dependency list for a target until we have all of the targets declared. 79 ''' 80 81 if self.samba_type in ['LIBRARY', 'BINARY', 'PYTHON']: 82 self.uselib = list(self.final_syslibs) 83 self.uselib_local = list(self.final_libs) 84 self.add_objects = list(self.final_objects) 85 86 # extra link flags from pkg_config 87 libs = self.final_syslibs.copy() 88 89 (cflags, ldflags, cpppath) = library_flags(self, list(libs)) 90 new_ldflags = getattr(self, 'samba_ldflags', [])[:] 91 new_ldflags.extend(ldflags) 92 self.ldflags = new_ldflags 93 94 if getattr(self, 'allow_undefined_symbols', False) and self.env.undefined_ldflags: 95 for f in self.env.undefined_ldflags: 96 self.ldflags.remove(f) 97 98 if getattr(self, 'allow_undefined_symbols', False) and self.env.undefined_ignore_ldflags: 99 for f in self.env.undefined_ignore_ldflags: 100 self.ldflags.append(f) 101 102 debug('deps: computed dependencies for target %s: uselib=%s uselib_local=%s add_objects=%s', 103 self.sname, self.uselib, self.uselib_local, self.add_objects) 104 105 if self.samba_type in ['SUBSYSTEM']: 106 # this is needed for the cflags of libs that come from pkg_config 107 self.uselib = list(self.final_syslibs) 108 self.uselib.extend(list(self.direct_syslibs)) 109 for lib in self.final_libs: 110 t = self.bld.get_tgen_by_name(lib) 111 self.uselib.extend(list(t.final_syslibs)) 112 self.uselib = unique_list(self.uselib) 113 114 if getattr(self, 'uselib', None): 115 up_list = [] 116 for l in self.uselib: 117 up_list.append(l.upper()) 118 self.uselib = up_list 119 120 121def build_includes(self): 122 '''This builds the right set of includes for a target. 123 124 One tricky part of this is that the includes= attribute for a 125 target needs to use paths which are relative to that targets 126 declaration directory (which we can get at via t.path). 127 128 The way this works is the includes list gets added as 129 samba_includes in the main build task declaration. Then this 130 function runs after all of the tasks are declared, and it 131 processes the samba_includes attribute to produce a includes= 132 attribute 133 ''' 134 135 if getattr(self, 'samba_includes', None) is None: 136 return 137 138 bld = self.bld 139 140 inc_deps = includes_objects(bld, self, set(), {}) 141 142 includes = [] 143 144 # maybe add local includes 145 if getattr(self, 'local_include', True) and getattr(self, 'local_include_first', True): 146 includes.append('.') 147 148 includes.extend(self.samba_includes_extended) 149 150 if 'EXTRA_INCLUDES' in bld.env and getattr(self, 'global_include', True): 151 includes.extend(bld.env['EXTRA_INCLUDES']) 152 153 includes.append('#') 154 155 inc_set = set() 156 inc_abs = [] 157 158 for d in inc_deps: 159 t = bld.get_tgen_by_name(d) 160 bld.ASSERT(t is not None, "Unable to find dependency %s for %s" % (d, self.sname)) 161 inclist = getattr(t, 'samba_includes_extended', [])[:] 162 if getattr(t, 'local_include', True): 163 inclist.append('.') 164 if inclist == []: 165 continue 166 tpath = t.samba_abspath 167 for inc in inclist: 168 npath = tpath + '/' + inc 169 if not npath in inc_set: 170 inc_abs.append(npath) 171 inc_set.add(npath) 172 173 mypath = self.path.abspath(bld.env) 174 for inc in inc_abs: 175 relpath = os.path.relpath(inc, mypath) 176 includes.append(relpath) 177 178 if getattr(self, 'local_include', True) and not getattr(self, 'local_include_first', True): 179 includes.append('.') 180 181 # now transform the includes list to be relative to the top directory 182 # which is represented by '#' in waf. This allows waf to cache the 183 # includes lists more efficiently 184 includes_top = [] 185 for i in includes: 186 if i[0] == '#': 187 # some are already top based 188 includes_top.append(i) 189 continue 190 absinc = os.path.join(self.path.abspath(), i) 191 relinc = os.path.relpath(absinc, self.bld.srcnode.abspath()) 192 includes_top.append('#' + relinc) 193 194 self.includes = unique_list(includes_top) 195 debug('deps: includes for target %s: includes=%s', 196 self.sname, self.includes) 197 198 199def add_init_functions(self): 200 '''This builds the right set of init functions''' 201 202 bld = self.bld 203 204 subsystems = LOCAL_CACHE(bld, 'INIT_FUNCTIONS') 205 206 # cope with the separated object lists from BINARY and LIBRARY targets 207 sname = self.sname 208 if sname.endswith('.objlist'): 209 sname = sname[0:-8] 210 211 modules = [] 212 if sname in subsystems: 213 modules.append(sname) 214 215 m = getattr(self, 'samba_modules', None) 216 if m is not None: 217 modules.extend(TO_LIST(m)) 218 219 m = getattr(self, 'samba_subsystem', None) 220 if m is not None: 221 modules.append(m) 222 223 if 'pyembed' in self.features: 224 return 225 226 sentinel = getattr(self, 'init_function_sentinel', 'NULL') 227 228 targets = LOCAL_CACHE(bld, 'TARGET_TYPE') 229 cflags = getattr(self, 'samba_cflags', [])[:] 230 231 if modules == []: 232 sname = sname.replace('-','_') 233 sname = sname.replace('.','_') 234 sname = sname.replace('/','_') 235 cflags.append('-DSTATIC_%s_MODULES=%s' % (sname, sentinel)) 236 if sentinel == 'NULL': 237 proto = "extern void __%s_dummy_module_proto(void)" % (sname) 238 cflags.append('-DSTATIC_%s_MODULES_PROTO=%s' % (sname, proto)) 239 self.cflags = cflags 240 return 241 242 for m in modules: 243 bld.ASSERT(m in subsystems, 244 "No init_function defined for module '%s' in target '%s'" % (m, self.sname)) 245 init_fn_list = [] 246 for d in subsystems[m]: 247 if targets[d['TARGET']] != 'DISABLED': 248 init_fn_list.append(d['INIT_FUNCTION']) 249 if init_fn_list == []: 250 cflags.append('-DSTATIC_%s_MODULES=%s' % (m, sentinel)) 251 if sentinel == 'NULL': 252 proto = "extern void __%s_dummy_module_proto(void)" % (m) 253 cflags.append('-DSTATIC_%s_MODULES_PROTO=%s' % (m, proto)) 254 else: 255 cflags.append('-DSTATIC_%s_MODULES=%s' % (m, ','.join(init_fn_list) + ',' + sentinel)) 256 proto = "".join('_MODULE_PROTO(%s)' % f for f in init_fn_list) +\ 257 "extern void __%s_dummy_module_proto(void)" % (m) 258 cflags.append('-DSTATIC_%s_MODULES_PROTO=%s' % (m, proto)) 259 self.cflags = cflags 260 261 262def check_duplicate_sources(bld, tgt_list): 263 '''see if we are compiling the same source file more than once''' 264 265 debug('deps: checking for duplicate sources') 266 targets = LOCAL_CACHE(bld, 'TARGET_TYPE') 267 268 for t in tgt_list: 269 source_list = TO_LIST(getattr(t, 'source', '')) 270 tpath = os.path.normpath(os.path.relpath(t.path.abspath(bld.env), t.env.BUILD_DIRECTORY + '/default')) 271 obj_sources = set() 272 for s in source_list: 273 if not isinstance(s, str): 274 print('strange path in check_duplicate_sources %r' % s) 275 s = s.abspath() 276 p = os.path.normpath(os.path.join(tpath, s)) 277 if p in obj_sources: 278 Logs.error("ERROR: source %s appears twice in target '%s'" % (p, t.sname)) 279 sys.exit(1) 280 obj_sources.add(p) 281 t.samba_source_set = obj_sources 282 283 subsystems = {} 284 285 # build a list of targets that each source file is part of 286 for t in tgt_list: 287 if not targets[t.sname] in [ 'LIBRARY', 'BINARY', 'PYTHON' ]: 288 continue 289 for obj in t.add_objects: 290 t2 = t.bld.get_tgen_by_name(obj) 291 source_set = getattr(t2, 'samba_source_set', set()) 292 for s in source_set: 293 if not s in subsystems: 294 subsystems[s] = {} 295 if not t.sname in subsystems[s]: 296 subsystems[s][t.sname] = [] 297 subsystems[s][t.sname].append(t2.sname) 298 299 for s in subsystems: 300 if len(subsystems[s]) > 1 and Options.options.SHOW_DUPLICATES: 301 Logs.warn("WARNING: source %s is in more than one target: %s" % (s, subsystems[s].keys())) 302 for tname in subsystems[s]: 303 if len(subsystems[s][tname]) > 1: 304 raise Errors.WafError("ERROR: source %s is in more than one subsystem of target '%s': %s" % (s, tname, subsystems[s][tname])) 305 306 return True 307 308def check_group_ordering(bld, tgt_list): 309 '''see if we have any dependencies that violate the group ordering 310 311 It is an error for a target to depend on a target from a later 312 build group 313 ''' 314 315 def group_name(g): 316 tm = bld.task_manager 317 return [x for x in tm.groups_names if id(tm.groups_names[x]) == id(g)][0] 318 319 for g in bld.task_manager.groups: 320 gname = group_name(g) 321 for t in g.tasks_gen: 322 t.samba_group = gname 323 324 grp_map = {} 325 idx = 0 326 for g in bld.task_manager.groups: 327 name = group_name(g) 328 grp_map[name] = idx 329 idx += 1 330 331 targets = LOCAL_CACHE(bld, 'TARGET_TYPE') 332 333 ret = True 334 for t in tgt_list: 335 tdeps = getattr(t, 'add_objects', []) + getattr(t, 'uselib_local', []) 336 for d in tdeps: 337 t2 = bld.get_tgen_by_name(d) 338 if t2 is None: 339 continue 340 map1 = grp_map[t.samba_group] 341 map2 = grp_map[t2.samba_group] 342 343 if map2 > map1: 344 Logs.error("Target %r in build group %r depends on target %r from later build group %r" % ( 345 t.sname, t.samba_group, t2.sname, t2.samba_group)) 346 ret = False 347 348 return ret 349Build.BuildContext.check_group_ordering = check_group_ordering 350 351def show_final_deps(bld, tgt_list): 352 '''show the final dependencies for all targets''' 353 354 targets = LOCAL_CACHE(bld, 'TARGET_TYPE') 355 356 for t in tgt_list: 357 if not targets[t.sname] in ['LIBRARY', 'BINARY', 'PYTHON', 'SUBSYSTEM']: 358 continue 359 debug('deps: final dependencies for target %s: uselib=%s uselib_local=%s add_objects=%s', 360 t.sname, t.uselib, getattr(t, 'uselib_local', []), getattr(t, 'add_objects', [])) 361 362 363def add_samba_attributes(bld, tgt_list): 364 '''ensure a target has a the required samba attributes''' 365 366 targets = LOCAL_CACHE(bld, 'TARGET_TYPE') 367 368 for t in tgt_list: 369 if t.name != '': 370 t.sname = t.name 371 else: 372 t.sname = t.target 373 t.samba_type = targets[t.sname] 374 t.samba_abspath = t.path.abspath(bld.env) 375 t.samba_deps_extended = t.samba_deps[:] 376 t.samba_includes_extended = TO_LIST(t.samba_includes)[:] 377 t.cflags = getattr(t, 'samba_cflags', '') 378 379def replace_grouping_libraries(bld, tgt_list): 380 '''replace dependencies based on grouping libraries 381 382 If a library is marked as a grouping library, then any target that 383 depends on a subsystem that is part of that grouping library gets 384 that dependency replaced with a dependency on the grouping library 385 ''' 386 387 targets = LOCAL_CACHE(bld, 'TARGET_TYPE') 388 389 grouping = {} 390 391 # find our list of grouping libraries, mapped from the subsystems they depend on 392 for t in tgt_list: 393 if not getattr(t, 'grouping_library', False): 394 continue 395 for dep in t.samba_deps_extended: 396 bld.ASSERT(dep in targets, "grouping library target %s not declared in %s" % (dep, t.sname)) 397 if targets[dep] == 'SUBSYSTEM': 398 grouping[dep] = t.sname 399 400 # now replace any dependencies on elements of grouping libraries 401 for t in tgt_list: 402 for i in range(len(t.samba_deps_extended)): 403 dep = t.samba_deps_extended[i] 404 if dep in grouping: 405 if t.sname != grouping[dep]: 406 debug("deps: target %s: replacing dependency %s with grouping library %s" % (t.sname, dep, grouping[dep])) 407 t.samba_deps_extended[i] = grouping[dep] 408 409 410 411def build_direct_deps(bld, tgt_list): 412 '''build the direct_objects and direct_libs sets for each target''' 413 414 targets = LOCAL_CACHE(bld, 'TARGET_TYPE') 415 syslib_deps = LOCAL_CACHE(bld, 'SYSLIB_DEPS') 416 417 global_deps = bld.env.GLOBAL_DEPENDENCIES 418 global_deps_exclude = set() 419 for dep in global_deps: 420 t = bld.get_tgen_by_name(dep) 421 for d in t.samba_deps: 422 # prevent loops from the global dependencies list 423 global_deps_exclude.add(d) 424 global_deps_exclude.add(d + '.objlist') 425 426 for t in tgt_list: 427 t.direct_objects = set() 428 t.direct_libs = set() 429 t.direct_syslibs = set() 430 deps = t.samba_deps_extended[:] 431 if getattr(t, 'samba_use_global_deps', False) and not t.sname in global_deps_exclude: 432 deps.extend(global_deps) 433 for d in deps: 434 if d == t.sname: continue 435 if not d in targets: 436 Logs.error("Unknown dependency '%s' in '%s'" % (d, t.sname)) 437 sys.exit(1) 438 if targets[d] in [ 'EMPTY', 'DISABLED' ]: 439 continue 440 if targets[d] == 'PYTHON' and targets[t.sname] != 'PYTHON' and t.sname.find('.objlist') == -1: 441 # this check should be more restrictive, but for now we have pidl-generated python 442 # code that directly depends on other python modules 443 Logs.error('ERROR: Target %s has dependency on python module %s' % (t.sname, d)) 444 sys.exit(1) 445 if targets[d] == 'SYSLIB': 446 t.direct_syslibs.add(d) 447 if d in syslib_deps: 448 for implied in TO_LIST(syslib_deps[d]): 449 if BUILTIN_LIBRARY(bld, implied): 450 t.direct_objects.add(implied) 451 elif targets[implied] == 'SYSLIB': 452 t.direct_syslibs.add(implied) 453 elif targets[implied] in ['LIBRARY', 'MODULE']: 454 t.direct_libs.add(implied) 455 else: 456 Logs.error('Implied dependency %s in %s is of type %s' % ( 457 implied, t.sname, targets[implied])) 458 sys.exit(1) 459 continue 460 t2 = bld.get_tgen_by_name(d) 461 if t2 is None: 462 Logs.error("no task %s of type %s in %s" % (d, targets[d], t.sname)) 463 sys.exit(1) 464 if t2.samba_type in [ 'LIBRARY', 'MODULE' ]: 465 t.direct_libs.add(d) 466 elif t2.samba_type in [ 'SUBSYSTEM', 'ASN1', 'PYTHON' ]: 467 t.direct_objects.add(d) 468 debug('deps: built direct dependencies') 469 470 471def dependency_loop(loops, t, target): 472 '''add a dependency loop to the loops dictionary''' 473 if t.sname == target: 474 return 475 if not target in loops: 476 loops[target] = set() 477 if not t.sname in loops[target]: 478 loops[target].add(t.sname) 479 480 481def indirect_libs(bld, t, chain, loops): 482 '''recursively calculate the indirect library dependencies for a target 483 484 An indirect library is a library that results from a dependency on 485 a subsystem 486 ''' 487 488 ret = getattr(t, 'indirect_libs', None) 489 if ret is not None: 490 return ret 491 492 ret = set() 493 for obj in t.direct_objects: 494 if obj in chain: 495 dependency_loop(loops, t, obj) 496 continue 497 chain.add(obj) 498 t2 = bld.get_tgen_by_name(obj) 499 r2 = indirect_libs(bld, t2, chain, loops) 500 chain.remove(obj) 501 ret = ret.union(t2.direct_libs) 502 ret = ret.union(r2) 503 504 for obj in indirect_objects(bld, t, set(), loops): 505 if obj in chain: 506 dependency_loop(loops, t, obj) 507 continue 508 chain.add(obj) 509 t2 = bld.get_tgen_by_name(obj) 510 r2 = indirect_libs(bld, t2, chain, loops) 511 chain.remove(obj) 512 ret = ret.union(t2.direct_libs) 513 ret = ret.union(r2) 514 515 t.indirect_libs = ret 516 517 return ret 518 519 520def indirect_objects(bld, t, chain, loops): 521 '''recursively calculate the indirect object dependencies for a target 522 523 indirect objects are the set of objects from expanding the 524 subsystem dependencies 525 ''' 526 527 ret = getattr(t, 'indirect_objects', None) 528 if ret is not None: return ret 529 530 ret = set() 531 for lib in t.direct_objects: 532 if lib in chain: 533 dependency_loop(loops, t, lib) 534 continue 535 chain.add(lib) 536 t2 = bld.get_tgen_by_name(lib) 537 r2 = indirect_objects(bld, t2, chain, loops) 538 chain.remove(lib) 539 ret = ret.union(t2.direct_objects) 540 ret = ret.union(r2) 541 542 t.indirect_objects = ret 543 return ret 544 545 546def extended_objects(bld, t, chain): 547 '''recursively calculate the extended object dependencies for a target 548 549 extended objects are the union of: 550 - direct objects 551 - indirect objects 552 - direct and indirect objects of all direct and indirect libraries 553 ''' 554 555 ret = getattr(t, 'extended_objects', None) 556 if ret is not None: return ret 557 558 ret = set() 559 ret = ret.union(t.final_objects) 560 561 for lib in t.final_libs: 562 if lib in chain: 563 continue 564 t2 = bld.get_tgen_by_name(lib) 565 chain.add(lib) 566 r2 = extended_objects(bld, t2, chain) 567 chain.remove(lib) 568 ret = ret.union(t2.final_objects) 569 ret = ret.union(r2) 570 571 t.extended_objects = ret 572 return ret 573 574 575def includes_objects(bld, t, chain, inc_loops): 576 '''recursively calculate the includes object dependencies for a target 577 578 includes dependencies come from either library or object dependencies 579 ''' 580 ret = getattr(t, 'includes_objects', None) 581 if ret is not None: 582 return ret 583 584 ret = t.direct_objects.copy() 585 ret = ret.union(t.direct_libs) 586 587 for obj in t.direct_objects: 588 if obj in chain: 589 dependency_loop(inc_loops, t, obj) 590 continue 591 chain.add(obj) 592 t2 = bld.get_tgen_by_name(obj) 593 r2 = includes_objects(bld, t2, chain, inc_loops) 594 chain.remove(obj) 595 ret = ret.union(t2.direct_objects) 596 ret = ret.union(r2) 597 598 for lib in t.direct_libs: 599 if lib in chain: 600 dependency_loop(inc_loops, t, lib) 601 continue 602 chain.add(lib) 603 t2 = bld.get_tgen_by_name(lib) 604 if t2 is None: 605 targets = LOCAL_CACHE(bld, 'TARGET_TYPE') 606 Logs.error('Target %s of type %s not found in direct_libs for %s' % ( 607 lib, targets[lib], t.sname)) 608 sys.exit(1) 609 r2 = includes_objects(bld, t2, chain, inc_loops) 610 chain.remove(lib) 611 ret = ret.union(t2.direct_objects) 612 ret = ret.union(r2) 613 614 t.includes_objects = ret 615 return ret 616 617 618def break_dependency_loops(bld, tgt_list): 619 '''find and break dependency loops''' 620 loops = {} 621 inc_loops = {} 622 623 # build up the list of loops 624 for t in tgt_list: 625 indirect_objects(bld, t, set(), loops) 626 indirect_libs(bld, t, set(), loops) 627 includes_objects(bld, t, set(), inc_loops) 628 629 # break the loops 630 for t in tgt_list: 631 if t.sname in loops: 632 for attr in ['direct_objects', 'indirect_objects', 'direct_libs', 'indirect_libs']: 633 objs = getattr(t, attr, set()) 634 setattr(t, attr, objs.difference(loops[t.sname])) 635 636 for loop in loops: 637 debug('deps: Found dependency loops for target %s : %s', loop, loops[loop]) 638 639 for loop in inc_loops: 640 debug('deps: Found include loops for target %s : %s', loop, inc_loops[loop]) 641 642 # expand the loops mapping by one level 643 for loop in loops.copy(): 644 for tgt in loops[loop]: 645 if tgt in loops: 646 loops[loop] = loops[loop].union(loops[tgt]) 647 648 for loop in inc_loops.copy(): 649 for tgt in inc_loops[loop]: 650 if tgt in inc_loops: 651 inc_loops[loop] = inc_loops[loop].union(inc_loops[tgt]) 652 653 654 # expand indirect subsystem and library loops 655 for loop in loops.copy(): 656 t = bld.get_tgen_by_name(loop) 657 if t.samba_type in ['SUBSYSTEM']: 658 loops[loop] = loops[loop].union(t.indirect_objects) 659 loops[loop] = loops[loop].union(t.direct_objects) 660 if t.samba_type in ['LIBRARY','PYTHON']: 661 loops[loop] = loops[loop].union(t.indirect_libs) 662 loops[loop] = loops[loop].union(t.direct_libs) 663 if loop in loops[loop]: 664 loops[loop].remove(loop) 665 666 # expand indirect includes loops 667 for loop in inc_loops.copy(): 668 t = bld.get_tgen_by_name(loop) 669 inc_loops[loop] = inc_loops[loop].union(t.includes_objects) 670 if loop in inc_loops[loop]: 671 inc_loops[loop].remove(loop) 672 673 # add in the replacement dependencies 674 for t in tgt_list: 675 for loop in loops: 676 for attr in ['indirect_objects', 'indirect_libs']: 677 objs = getattr(t, attr, set()) 678 if loop in objs: 679 diff = loops[loop].difference(objs) 680 if t.sname in diff: 681 diff.remove(t.sname) 682 if diff: 683 debug('deps: Expanded target %s of type %s from loop %s by %s', t.sname, t.samba_type, loop, diff) 684 objs = objs.union(diff) 685 setattr(t, attr, objs) 686 687 for loop in inc_loops: 688 objs = getattr(t, 'includes_objects', set()) 689 if loop in objs: 690 diff = inc_loops[loop].difference(objs) 691 if t.sname in diff: 692 diff.remove(t.sname) 693 if diff: 694 debug('deps: Expanded target %s includes of type %s from loop %s by %s', t.sname, t.samba_type, loop, diff) 695 objs = objs.union(diff) 696 setattr(t, 'includes_objects', objs) 697 698 699def reduce_objects(bld, tgt_list): 700 '''reduce objects by looking for indirect object dependencies''' 701 rely_on = {} 702 703 for t in tgt_list: 704 t.extended_objects = None 705 706 changed = False 707 708 for type in ['BINARY', 'PYTHON', 'LIBRARY']: 709 for t in tgt_list: 710 if t.samba_type != type: continue 711 # if we will indirectly link to a target then we don't need it 712 new = t.final_objects.copy() 713 for l in t.final_libs: 714 t2 = bld.get_tgen_by_name(l) 715 t2_obj = extended_objects(bld, t2, set()) 716 dup = new.intersection(t2_obj) 717 if t.sname in rely_on: 718 dup = dup.difference(rely_on[t.sname]) 719 if dup: 720 # Do not remove duplicates of BUILTINS 721 d = next(iter(dup)) 722 if BUILTIN_LIBRARY(bld, d): 723 continue 724 725 debug('deps: removing dups from %s of type %s: %s also in %s %s', 726 t.sname, t.samba_type, dup, t2.samba_type, l) 727 new = new.difference(dup) 728 changed = True 729 if not l in rely_on: 730 rely_on[l] = set() 731 rely_on[l] = rely_on[l].union(dup) 732 t.final_objects = new 733 734 if not changed: 735 return False 736 737 # add back in any objects that were relied upon by the reduction rules 738 for r in rely_on: 739 t = bld.get_tgen_by_name(r) 740 t.final_objects = t.final_objects.union(rely_on[r]) 741 742 return True 743 744 745def show_library_loop(bld, lib1, lib2, path, seen): 746 '''show the detailed path of a library loop between lib1 and lib2''' 747 748 t = bld.get_tgen_by_name(lib1) 749 if not lib2 in getattr(t, 'final_libs', set()): 750 return 751 752 for d in t.samba_deps_extended: 753 if d in seen: 754 continue 755 seen.add(d) 756 path2 = path + '=>' + d 757 if d == lib2: 758 Logs.warn('library loop path: ' + path2) 759 return 760 show_library_loop(bld, d, lib2, path2, seen) 761 seen.remove(d) 762 763 764def calculate_final_deps(bld, tgt_list, loops): 765 '''calculate the final library and object dependencies''' 766 for t in tgt_list: 767 # start with the maximum possible list 768 t.final_libs = t.direct_libs.union(indirect_libs(bld, t, set(), loops)) 769 t.final_objects = t.direct_objects.union(indirect_objects(bld, t, set(), loops)) 770 771 for t in tgt_list: 772 # don't depend on ourselves 773 if t.sname in t.final_libs: 774 t.final_libs.remove(t.sname) 775 if t.sname in t.final_objects: 776 t.final_objects.remove(t.sname) 777 778 # handle any non-shared binaries 779 for t in tgt_list: 780 if t.samba_type == 'BINARY' and bld.NONSHARED_BINARY(t.sname): 781 subsystem_list = LOCAL_CACHE(bld, 'INIT_FUNCTIONS') 782 targets = LOCAL_CACHE(bld, 'TARGET_TYPE') 783 784 # replace lib deps with objlist deps 785 for l in t.final_libs: 786 objname = l + '.objlist' 787 t2 = bld.get_tgen_by_name(objname) 788 if t2 is None: 789 Logs.error('ERROR: subsystem %s not found' % objname) 790 sys.exit(1) 791 t.final_objects.add(objname) 792 t.final_objects = t.final_objects.union(extended_objects(bld, t2, set())) 793 if l in subsystem_list: 794 # its a subsystem - we also need the contents of any modules 795 for d in subsystem_list[l]: 796 module_name = d['TARGET'] 797 if targets[module_name] == 'LIBRARY': 798 objname = module_name + '.objlist' 799 elif targets[module_name] == 'SUBSYSTEM': 800 objname = module_name 801 else: 802 continue 803 t2 = bld.get_tgen_by_name(objname) 804 if t2 is None: 805 Logs.error('ERROR: subsystem %s not found' % objname) 806 sys.exit(1) 807 t.final_objects.add(objname) 808 t.final_objects = t.final_objects.union(extended_objects(bld, t2, set())) 809 t.final_libs = set() 810 811 # find any library loops 812 for t in tgt_list: 813 if t.samba_type in ['LIBRARY', 'PYTHON']: 814 for l in t.final_libs.copy(): 815 t2 = bld.get_tgen_by_name(l) 816 if t.sname in t2.final_libs: 817 if getattr(bld.env, "ALLOW_CIRCULAR_LIB_DEPENDENCIES", False): 818 # we could break this in either direction. If one of the libraries 819 # has a version number, and will this be distributed publicly, then 820 # we should make it the lower level library in the DAG 821 Logs.warn('deps: removing library loop %s from %s' % (t.sname, t2.sname)) 822 dependency_loop(loops, t, t2.sname) 823 t2.final_libs.remove(t.sname) 824 else: 825 Logs.error('ERROR: circular library dependency between %s and %s' 826 % (t.sname, t2.sname)) 827 show_library_loop(bld, t.sname, t2.sname, t.sname, set()) 828 show_library_loop(bld, t2.sname, t.sname, t2.sname, set()) 829 sys.exit(1) 830 831 for loop in loops: 832 debug('deps: Found dependency loops for target %s : %s', loop, loops[loop]) 833 834 # we now need to make corrections for any library loops we broke up 835 # any target that depended on the target of the loop and doesn't 836 # depend on the source of the loop needs to get the loop source added 837 for type in ['BINARY','PYTHON','LIBRARY','BINARY']: 838 for t in tgt_list: 839 if t.samba_type != type: continue 840 for loop in loops: 841 if loop in t.final_libs: 842 diff = loops[loop].difference(t.final_libs) 843 if t.sname in diff: 844 diff.remove(t.sname) 845 if t.sname in diff: 846 diff.remove(t.sname) 847 # make sure we don't recreate the loop again! 848 for d in diff.copy(): 849 t2 = bld.get_tgen_by_name(d) 850 if t2.samba_type == 'LIBRARY': 851 if t.sname in t2.final_libs: 852 debug('deps: removing expansion %s from %s', d, t.sname) 853 diff.remove(d) 854 if diff: 855 debug('deps: Expanded target %s by loop %s libraries (loop %s) %s', t.sname, loop, 856 loops[loop], diff) 857 t.final_libs = t.final_libs.union(diff) 858 859 # remove objects that are also available in linked libs 860 count = 0 861 while reduce_objects(bld, tgt_list): 862 count += 1 863 if count > 100: 864 Logs.warn("WARNING: Unable to remove all inter-target object duplicates") 865 break 866 debug('deps: Object reduction took %u iterations', count) 867 868 # add in any syslib dependencies 869 for t in tgt_list: 870 if not t.samba_type in ['BINARY','PYTHON','LIBRARY','SUBSYSTEM']: 871 continue 872 syslibs = set() 873 for d in t.final_objects: 874 t2 = bld.get_tgen_by_name(d) 875 syslibs = syslibs.union(t2.direct_syslibs) 876 # this adds the indirect syslibs as well, which may not be needed 877 # depending on the linker flags 878 for d in t.final_libs: 879 t2 = bld.get_tgen_by_name(d) 880 syslibs = syslibs.union(t2.direct_syslibs) 881 t.final_syslibs = syslibs 882 883 884 # find any unresolved library loops 885 lib_loop_error = False 886 for t in tgt_list: 887 if t.samba_type in ['LIBRARY', 'PYTHON']: 888 for l in t.final_libs.copy(): 889 t2 = bld.get_tgen_by_name(l) 890 if t.sname in t2.final_libs: 891 Logs.error('ERROR: Unresolved library loop %s from %s' % (t.sname, t2.sname)) 892 lib_loop_error = True 893 if lib_loop_error: 894 sys.exit(1) 895 896 debug('deps: removed duplicate dependencies') 897 898 899def show_dependencies(bld, target, seen): 900 '''recursively show the dependencies of target''' 901 902 if target in seen: 903 return 904 905 t = bld.get_tgen_by_name(target) 906 if t is None: 907 Logs.error("ERROR: Unable to find target '%s'" % target) 908 sys.exit(1) 909 910 Logs.info('%s(OBJECTS): %s' % (target, t.direct_objects)) 911 Logs.info('%s(LIBS): %s' % (target, t.direct_libs)) 912 Logs.info('%s(SYSLIBS): %s' % (target, t.direct_syslibs)) 913 914 seen.add(target) 915 916 for t2 in t.direct_objects: 917 show_dependencies(bld, t2, seen) 918 919 920def show_object_duplicates(bld, tgt_list): 921 '''show a list of object files that are included in more than 922 one library or binary''' 923 924 targets = LOCAL_CACHE(bld, 'TARGET_TYPE') 925 926 used_by = {} 927 928 Logs.info("showing duplicate objects") 929 930 for t in tgt_list: 931 if not targets[t.sname] in [ 'LIBRARY', 'PYTHON' ]: 932 continue 933 for n in getattr(t, 'final_objects', set()): 934 t2 = bld.get_tgen_by_name(n) 935 if not n in used_by: 936 used_by[n] = set() 937 used_by[n].add(t.sname) 938 939 for n in used_by: 940 if len(used_by[n]) > 1: 941 Logs.info("target '%s' is used by %s" % (n, used_by[n])) 942 943 Logs.info("showing indirect dependency counts (sorted by count)") 944 945 def indirect_count(t1, t2): 946 return len(t2.indirect_objects) - len(t1.indirect_objects) 947 948 sorted_list = sorted(tgt_list, cmp=indirect_count) 949 for t in sorted_list: 950 if len(t.indirect_objects) > 1: 951 Logs.info("%s depends on %u indirect objects" % (t.sname, len(t.indirect_objects))) 952 953 954###################################################################### 955# this provides a way to save our dependency calculations between runs 956savedeps_version = 3 957savedeps_inputs = ['samba_deps', 'samba_includes', 'local_include', 'local_include_first', 'samba_cflags', 958 'source', 'grouping_library', 'samba_ldflags', 'allow_undefined_symbols', 959 'use_global_deps', 'global_include' ] 960savedeps_outputs = ['uselib', 'uselib_local', 'add_objects', 'includes', 961 'cflags', 'ldflags', 'samba_deps_extended', 'final_libs'] 962savedeps_outenv = ['INC_PATHS'] 963savedeps_envvars = ['NONSHARED_BINARIES', 'GLOBAL_DEPENDENCIES', 'EXTRA_CFLAGS', 'EXTRA_LDFLAGS', 'EXTRA_INCLUDES' ] 964savedeps_caches = ['GLOBAL_DEPENDENCIES', 'TARGET_TYPE', 'INIT_FUNCTIONS', 'SYSLIB_DEPS'] 965savedeps_files = ['buildtools/wafsamba/samba_deps.py'] 966 967def save_samba_deps(bld, tgt_list): 968 '''save the dependency calculations between builds, to make 969 further builds faster''' 970 denv = ConfigSet.ConfigSet() 971 972 denv.version = savedeps_version 973 denv.savedeps_inputs = savedeps_inputs 974 denv.savedeps_outputs = savedeps_outputs 975 denv.input = {} 976 denv.output = {} 977 denv.outenv = {} 978 denv.caches = {} 979 denv.envvar = {} 980 denv.files = {} 981 982 for f in savedeps_files: 983 denv.files[f] = os.stat(os.path.join(bld.srcnode.abspath(), f)).st_mtime 984 985 for c in savedeps_caches: 986 denv.caches[c] = LOCAL_CACHE(bld, c) 987 988 for e in savedeps_envvars: 989 denv.envvar[e] = bld.env[e] 990 991 for t in tgt_list: 992 # save all the input attributes for each target 993 tdeps = {} 994 for attr in savedeps_inputs: 995 v = getattr(t, attr, None) 996 if v is not None: 997 tdeps[attr] = v 998 if tdeps != {}: 999 denv.input[t.sname] = tdeps 1000 1001 # save all the output attributes for each target 1002 tdeps = {} 1003 for attr in savedeps_outputs: 1004 v = getattr(t, attr, None) 1005 if v is not None: 1006 tdeps[attr] = v 1007 if tdeps != {}: 1008 denv.output[t.sname] = tdeps 1009 1010 tdeps = {} 1011 for attr in savedeps_outenv: 1012 if attr in t.env: 1013 tdeps[attr] = t.env[attr] 1014 if tdeps != {}: 1015 denv.outenv[t.sname] = tdeps 1016 1017 depsfile = os.path.join(bld.cache_dir, "sambadeps") 1018 denv.store_fast(depsfile) 1019 1020 1021 1022def load_samba_deps(bld, tgt_list): 1023 '''load a previous set of build dependencies if possible''' 1024 depsfile = os.path.join(bld.cache_dir, "sambadeps") 1025 denv = ConfigSet.ConfigSet() 1026 try: 1027 debug('deps: checking saved dependencies') 1028 denv.load_fast(depsfile) 1029 if (denv.version != savedeps_version or 1030 denv.savedeps_inputs != savedeps_inputs or 1031 denv.savedeps_outputs != savedeps_outputs): 1032 return False 1033 except Exception: 1034 return False 1035 1036 # check if critical files have changed 1037 for f in savedeps_files: 1038 if f not in denv.files: 1039 return False 1040 if denv.files[f] != os.stat(os.path.join(bld.srcnode.abspath(), f)).st_mtime: 1041 return False 1042 1043 # check if caches are the same 1044 for c in savedeps_caches: 1045 if c not in denv.caches or denv.caches[c] != LOCAL_CACHE(bld, c): 1046 return False 1047 1048 # check if caches are the same 1049 for e in savedeps_envvars: 1050 if e not in denv.envvar or denv.envvar[e] != bld.env[e]: 1051 return False 1052 1053 # check inputs are the same 1054 for t in tgt_list: 1055 tdeps = {} 1056 for attr in savedeps_inputs: 1057 v = getattr(t, attr, None) 1058 if v is not None: 1059 tdeps[attr] = v 1060 if t.sname in denv.input: 1061 olddeps = denv.input[t.sname] 1062 else: 1063 olddeps = {} 1064 if tdeps != olddeps: 1065 #print '%s: \ntdeps=%s \nodeps=%s' % (t.sname, tdeps, olddeps) 1066 return False 1067 1068 # put outputs in place 1069 for t in tgt_list: 1070 if not t.sname in denv.output: continue 1071 tdeps = denv.output[t.sname] 1072 for a in tdeps: 1073 setattr(t, a, tdeps[a]) 1074 1075 # put output env vars in place 1076 for t in tgt_list: 1077 if not t.sname in denv.outenv: continue 1078 tdeps = denv.outenv[t.sname] 1079 for a in tdeps: 1080 t.env[a] = tdeps[a] 1081 1082 debug('deps: loaded saved dependencies') 1083 return True 1084 1085 1086 1087def check_project_rules(bld): 1088 '''check the project rules - ensuring the targets are sane''' 1089 1090 loops = {} 1091 inc_loops = {} 1092 1093 tgt_list = get_tgt_list(bld) 1094 1095 add_samba_attributes(bld, tgt_list) 1096 1097 force_project_rules = (Options.options.SHOWDEPS or 1098 Options.options.SHOW_DUPLICATES) 1099 1100 if not force_project_rules and load_samba_deps(bld, tgt_list): 1101 return 1102 1103 timer = Utils.Timer() 1104 1105 bld.new_rules = True 1106 Logs.info("Checking project rules ...") 1107 1108 debug('deps: project rules checking started') 1109 1110 expand_subsystem_deps(bld) 1111 1112 debug("deps: expand_subsystem_deps: %s" % str(timer)) 1113 1114 replace_grouping_libraries(bld, tgt_list) 1115 1116 debug("deps: replace_grouping_libraries: %s" % str(timer)) 1117 1118 build_direct_deps(bld, tgt_list) 1119 1120 debug("deps: build_direct_deps: %s" % str(timer)) 1121 1122 break_dependency_loops(bld, tgt_list) 1123 1124 debug("deps: break_dependency_loops: %s" % str(timer)) 1125 1126 if Options.options.SHOWDEPS: 1127 show_dependencies(bld, Options.options.SHOWDEPS, set()) 1128 1129 calculate_final_deps(bld, tgt_list, loops) 1130 1131 debug("deps: calculate_final_deps: %s" % str(timer)) 1132 1133 if Options.options.SHOW_DUPLICATES: 1134 show_object_duplicates(bld, tgt_list) 1135 1136 # run the various attribute generators 1137 for f in [ build_dependencies, build_includes, add_init_functions ]: 1138 debug('deps: project rules checking %s', f) 1139 for t in tgt_list: f(t) 1140 debug("deps: %s: %s" % (f, str(timer))) 1141 1142 debug('deps: project rules stage1 completed') 1143 1144 if not check_duplicate_sources(bld, tgt_list): 1145 Logs.error("Duplicate sources present - aborting") 1146 sys.exit(1) 1147 1148 debug("deps: check_duplicate_sources: %s" % str(timer)) 1149 1150 if not bld.check_group_ordering(tgt_list): 1151 Logs.error("Bad group ordering - aborting") 1152 sys.exit(1) 1153 1154 debug("deps: check_group_ordering: %s" % str(timer)) 1155 1156 show_final_deps(bld, tgt_list) 1157 1158 debug("deps: show_final_deps: %s" % str(timer)) 1159 1160 debug('deps: project rules checking completed - %u targets checked', 1161 len(tgt_list)) 1162 1163 if not bld.is_install: 1164 save_samba_deps(bld, tgt_list) 1165 1166 debug("deps: save_samba_deps: %s" % str(timer)) 1167 1168 Logs.info("Project rules pass") 1169 1170 1171def CHECK_PROJECT_RULES(bld): 1172 '''enable checking of project targets for sanity''' 1173 if bld.env.added_project_rules: 1174 return 1175 bld.env.added_project_rules = True 1176 bld.add_pre_fun(check_project_rules) 1177Build.BuildContext.CHECK_PROJECT_RULES = CHECK_PROJECT_RULES 1178 1179 1180