1#! /usr/bin/python2 2import os.path 3import sys 4import shlex 5import re 6import tempfile 7import copy 8 9from headerutils import * 10 11requires = { } 12provides = { } 13 14no_remove = [ "system.h", "coretypes.h", "config.h" , "bconfig.h", "backend.h" ] 15 16# These targets are the ones which provide "coverage". Typically, if any 17# target is going to fail compilation, it's one of these. This was determined 18# during the initial runs of reduce-headers... On a full set of target builds, 19# every failure which occured was triggered by one of these. 20# This list is used during target-list construction simply to put any of these 21# *first* in the candidate list, increasing the probability that a failure is 22# found quickly. 23target_priority = [ 24 "aarch64-linux-gnu", 25 "arm-netbsdelf", 26 "c6x-elf", 27 "epiphany-elf", 28 "hppa2.0-hpux10.1", 29 "i686-mingw32crt", 30 "i686-pc-msdosdjgpp", 31 "mipsel-elf", 32 "powerpc-eabisimaltivec", 33 "rs6000-ibm-aix5.1.0", 34 "sh-superh-elf", 35 "sparc64-elf", 36 "spu-elf" 37] 38 39 40target_dir = "" 41build_dir = "" 42ignore_list = list() 43target_builds = list() 44 45target_dict = { } 46header_dict = { } 47search_path = [ ".", "../include", "../libcpp/include" ] 48 49remove_count = { } 50 51 52# Given a header name, normalize it. ie. cp/cp-tree.h could be in gcc, while 53# the same header could be referenced from within the cp subdirectory as 54# just cp-tree.h 55# for now, just assume basenames are unique 56 57def normalize_header (header): 58 return os.path.basename (header) 59 60 61# Adds a header file and its sub-includes to the global dictionary if they 62# aren't already there. Specify s_path since different build directories may 63# append themselves on demand to the global list. 64# return entry for the specified header, knowing all sub entries are completed 65 66def get_header_info (header, s_path): 67 global header_dict 68 global empty_iinfo 69 process_list = list () 70 location = "" 71 bname = "" 72 bname_iinfo = empty_iinfo 73 for path in s_path: 74 if os.path.exists (path + "/" + header): 75 location = path + "/" + header 76 break 77 78 if location: 79 bname = normalize_header (location) 80 if header_dict.get (bname): 81 bname_iinfo = header_dict[bname] 82 loc2 = ii_path (bname_iinfo)+ "/" + bname 83 if loc2[:2] == "./": 84 loc2 = loc2[2:] 85 if location[:2] == "./": 86 location = location[2:] 87 if loc2 != location: 88 # Don't use the cache if it isnt the right one. 89 bname_iinfo = process_ii_macro (location) 90 return bname_iinfo 91 92 bname_iinfo = process_ii_macro (location) 93 header_dict[bname] = bname_iinfo 94 # now decend into the include tree 95 for i in ii_include_list (bname_iinfo): 96 get_header_info (i, s_path) 97 else: 98 # if the file isnt in the source directories, look in the build and target 99 # directories. If it is here, then aggregate all the versions. 100 location = build_dir + "/gcc/" + header 101 build_inc = target_inc = False 102 if os.path.exists (location): 103 build_inc = True 104 for x in target_dict: 105 location = target_dict[x] + "/gcc/" + header 106 if os.path.exists (location): 107 target_inc = True 108 break 109 110 if (build_inc or target_inc): 111 bname = normalize_header(header) 112 defines = set() 113 consumes = set() 114 incl = set() 115 if build_inc: 116 iinfo = process_ii_macro (build_dir + "/gcc/" + header) 117 defines = set (ii_macro_define (iinfo)) 118 consumes = set (ii_macro_consume (iinfo)) 119 incl = set (ii_include_list (iinfo)) 120 121 if (target_inc): 122 for x in target_dict: 123 location = target_dict[x] + "/gcc/" + header 124 if os.path.exists (location): 125 iinfo = process_ii_macro (location) 126 defines.update (ii_macro_define (iinfo)) 127 consumes.update (ii_macro_consume (iinfo)) 128 incl.update (ii_include_list (iinfo)) 129 130 bname_iinfo = (header, "build", list(incl), list(), list(consumes), list(defines), list(), list()) 131 132 header_dict[bname] = bname_iinfo 133 for i in incl: 134 get_header_info (i, s_path) 135 136 return bname_iinfo 137 138 139# return a list of all headers brought in by this header 140def all_headers (fname): 141 global header_dict 142 headers_stack = list() 143 headers_list = list() 144 if header_dict.get (fname) == None: 145 return list () 146 for y in ii_include_list (header_dict[fname]): 147 headers_stack.append (y) 148 149 while headers_stack: 150 h = headers_stack.pop () 151 hn = normalize_header (h) 152 if hn not in headers_list: 153 headers_list.append (hn) 154 if header_dict.get(hn): 155 for y in ii_include_list (header_dict[hn]): 156 if normalize_header (y) not in headers_list: 157 headers_stack.append (y) 158 159 return headers_list 160 161 162 163 164# Search bld_dir for all target tuples, confirm that they have a build path with 165# bld_dir/target-tuple/gcc, and build a dictionary of build paths indexed by 166# target tuple.. 167 168def build_target_dict (bld_dir, just_these): 169 global target_dict 170 target_doct = { } 171 error = False 172 if os.path.exists (bld_dir): 173 if just_these: 174 ls = just_these 175 else: 176 ls = os.listdir(bld_dir) 177 for t in ls: 178 if t.find("-") != -1: 179 target = t.strip() 180 tpath = bld_dir + "/" + target 181 if not os.path.exists (tpath + "/gcc"): 182 print "Error: gcc build directory for target " + t + " Does not exist: " + tpath + "/gcc" 183 error = True 184 else: 185 target_dict[target] = tpath 186 187 if error: 188 target_dict = { } 189 190def get_obj_name (src_file): 191 if src_file[-2:] == ".c": 192 return src_file.replace (".c", ".o") 193 elif src_file[-3:] == ".cc": 194 return src_file.replace (".cc", ".o") 195 return "" 196 197def target_obj_exists (target, obj_name): 198 global target_dict 199 # look in a subdir if src has a subdir, then check gcc base directory. 200 if target_dict.get(target): 201 obj = target_dict[target] + "/gcc/" + obj_name 202 if not os.path.exists (obj): 203 obj = target_dict[target] + "/gcc/" + os.path.basename(obj_name) 204 if os.path.exists (obj): 205 return True 206 return False 207 208# Given a src file, return a list of targets which may build this file. 209def find_targets (src_file): 210 global target_dict 211 targ_list = list() 212 obj_name = get_obj_name (src_file) 213 if not obj_name: 214 print "Error: " + src_file + " - Cannot determine object name." 215 return list() 216 217 # Put the high priority targets which tend to trigger failures first 218 for target in target_priority: 219 if target_obj_exists (target, obj_name): 220 targ_list.append ((target, target_dict[target])) 221 222 for target in target_dict: 223 if target not in target_priority and target_obj_exists (target, obj_name): 224 targ_list.append ((target, target_dict[target])) 225 226 return targ_list 227 228 229def try_to_remove (src_file, h_list, verbose): 230 global target_dict 231 global header_dict 232 global build_dir 233 234 # build from scratch each time 235 header_dict = { } 236 summary = "" 237 rmcount = 0 238 239 because = { } 240 src_info = process_ii_macro_src (src_file) 241 src_data = ii_src (src_info) 242 if src_data: 243 inclist = ii_include_list_non_cond (src_info) 244 # work is done if there are no includes to check 245 if not inclist: 246 return src_file + ": No include files to attempt to remove" 247 248 # work on the include list in reverse. 249 inclist.reverse() 250 251 # Get the target list 252 targ_list = list() 253 targ_list = find_targets (src_file) 254 255 spath = search_path 256 if os.path.dirname (src_file): 257 spath.append (os.path.dirname (src_file)) 258 259 hostbuild = True 260 if src_file.find("config/") != -1: 261 # config files dont usually build on the host 262 hostbuild = False 263 obn = get_obj_name (os.path.basename (src_file)) 264 if obn and os.path.exists (build_dir + "/gcc/" + obn): 265 hostbuild = True 266 if not target_dict: 267 summary = src_file + ": Target builds are required for config files. None found." 268 print summary 269 return summary 270 if not targ_list: 271 summary =src_file + ": Cannot find any targets which build this file." 272 print summary 273 return summary 274 275 if hostbuild: 276 # confirm it actually builds before we do anything 277 print "Confirming source file builds" 278 res = get_make_output (build_dir + "/gcc", "all") 279 if res[0] != 0: 280 message = "Error: " + src_file + " does not build currently." 281 summary = src_file + " does not build on host." 282 print message 283 print res[1] 284 if verbose: 285 verbose.write (message + "\n") 286 verbose.write (res[1]+ "\n") 287 return summary 288 289 src_requires = set (ii_macro_consume (src_info)) 290 for macro in src_requires: 291 because[macro] = src_file 292 header_seen = list () 293 294 os.rename (src_file, src_file + ".bak") 295 src_orig = copy.deepcopy (src_data) 296 src_tmp = copy.deepcopy (src_data) 297 298 try: 299 # process the includes from bottom to top. This is because we know that 300 # later includes have are known to be needed, so any dependency from this 301 # header is a true dependency 302 for inc_file in inclist: 303 inc_file_norm = normalize_header (inc_file) 304 305 if inc_file in no_remove: 306 continue 307 if len (h_list) != 0 and inc_file_norm not in h_list: 308 continue 309 if inc_file_norm[0:3] == "gt-": 310 continue 311 if inc_file_norm[0:6] == "gtype-": 312 continue 313 if inc_file_norm.replace(".h",".c") == os.path.basename(src_file): 314 continue 315 316 lookfor = ii_src_line(src_info)[inc_file] 317 src_tmp.remove (lookfor) 318 message = "Trying " + src_file + " without " + inc_file 319 print message 320 if verbose: 321 verbose.write (message + "\n") 322 out = open(src_file, "w") 323 for line in src_tmp: 324 out.write (line) 325 out.close() 326 327 keep = False 328 if hostbuild: 329 res = get_make_output (build_dir + "/gcc", "all") 330 else: 331 res = (0, "") 332 333 rc = res[0] 334 message = "Passed Host build" 335 if (rc != 0): 336 # host build failed 337 message = "Compilation failed:\n"; 338 keep = True 339 else: 340 if targ_list: 341 objfile = get_obj_name (src_file) 342 t1 = targ_list[0] 343 if objfile and os.path.exists(t1[1] +"/gcc/"+objfile): 344 res = get_make_output_parallel (targ_list, objfile, 0) 345 else: 346 res = get_make_output_parallel (targ_list, "all-gcc", 0) 347 rc = res[0] 348 if rc != 0: 349 message = "Compilation failed on TARGET : " + res[2] 350 keep = True 351 else: 352 message = "Passed host and target builds" 353 354 if keep: 355 print message + "\n" 356 357 if (rc != 0): 358 if verbose: 359 verbose.write (message + "\n"); 360 verbose.write (res[1]) 361 verbose.write ("\n"); 362 if os.path.exists (inc_file): 363 ilog = open(inc_file+".log","a") 364 ilog.write (message + " for " + src_file + ":\n\n"); 365 ilog.write ("============================================\n"); 366 ilog.write (res[1]) 367 ilog.write ("\n"); 368 ilog.close() 369 if os.path.exists (src_file): 370 ilog = open(src_file+".log","a") 371 ilog.write (message + " for " +inc_file + ":\n\n"); 372 ilog.write ("============================================\n"); 373 ilog.write (res[1]) 374 ilog.write ("\n"); 375 ilog.close() 376 377 # Given a sequence where : 378 # #include "tm.h" 379 # #include "target.h" // includes tm.h 380 381 # target.h was required, and when attempting to remove tm.h we'd see that 382 # all the macro defintions are "required" since they all look like: 383 # #ifndef HAVE_blah 384 # #define HAVE_blah 385 # endif 386 387 # when target.h was found to be required, tm.h will be tagged as included. 388 # so when we get this far, we know we dont have to check the macros for 389 # tm.h since we know it is already been included. 390 391 if inc_file_norm not in header_seen: 392 iinfo = get_header_info (inc_file, spath) 393 newlist = all_headers (inc_file_norm) 394 if ii_path(iinfo) == "build" and not target_dict: 395 keep = True 396 text = message + " : Will not remove a build file without some targets." 397 print text 398 ilog = open(src_file+".log","a") 399 ilog.write (text +"\n") 400 ilog.write ("============================================\n"); 401 ilog.close() 402 ilog = open("reduce-headers-kept.log","a") 403 ilog.write (src_file + " " + text +"\n") 404 ilog.close() 405 else: 406 newlist = list() 407 if not keep and inc_file_norm not in header_seen: 408 # now look for any macro requirements. 409 for h in newlist: 410 if not h in header_seen: 411 if header_dict.get(h): 412 defined = ii_macro_define (header_dict[h]) 413 for dep in defined: 414 if dep in src_requires and dep not in ignore_list: 415 keep = True; 416 text = message + ", but must keep " + inc_file + " because it provides " + dep 417 if because.get(dep) != None: 418 text = text + " Possibly required by " + because[dep] 419 print text 420 ilog = open(inc_file+".log","a") 421 ilog.write (because[dep]+": Requires [dep] in "+src_file+"\n") 422 ilog.write ("============================================\n"); 423 ilog.close() 424 ilog = open(src_file+".log","a") 425 ilog.write (text +"\n") 426 ilog.write ("============================================\n"); 427 ilog.close() 428 ilog = open("reduce-headers-kept.log","a") 429 ilog.write (src_file + " " + text +"\n") 430 ilog.close() 431 if verbose: 432 verbose.write (text + "\n") 433 434 if keep: 435 # add all headers 'consumes' to src_requires list, and mark as seen 436 for h in newlist: 437 if not h in header_seen: 438 header_seen.append (h) 439 if header_dict.get(h): 440 consume = ii_macro_consume (header_dict[h]) 441 for dep in consume: 442 if dep not in src_requires: 443 src_requires.add (dep) 444 if because.get(dep) == None: 445 because[dep] = inc_file 446 447 src_tmp = copy.deepcopy (src_data) 448 else: 449 print message + " --> removing " + inc_file + "\n" 450 rmcount += 1 451 if verbose: 452 verbose.write (message + " --> removing " + inc_file + "\n") 453 if remove_count.get(inc_file) == None: 454 remove_count[inc_file] = 1 455 else: 456 remove_count[inc_file] += 1 457 src_data = copy.deepcopy (src_tmp) 458 except: 459 print "Interuption: restoring original file" 460 out = open(src_file, "w") 461 for line in src_orig: 462 out.write (line) 463 out.close() 464 raise 465 466 # copy current version, since it is the "right" one now. 467 out = open(src_file, "w") 468 for line in src_data: 469 out.write (line) 470 out.close() 471 472 # Try a final host bootstrap build to make sure everything is kosher. 473 if hostbuild: 474 res = get_make_output (build_dir, "all") 475 rc = res[0] 476 if (rc != 0): 477 # host build failed! return to original version 478 print "Error: " + src_file + " Failed to bootstrap at end!!! restoring." 479 print " Bad version at " + src_file + ".bad" 480 os.rename (src_file, src_file + ".bad") 481 out = open(src_file, "w") 482 for line in src_orig: 483 out.write (line) 484 out.close() 485 return src_file + ": failed to build after reduction. Restored original" 486 487 if src_data == src_orig: 488 summary = src_file + ": No change." 489 else: 490 summary = src_file + ": Reduction performed, "+str(rmcount)+" includes removed." 491 print summary 492 return summary 493 494only_h = list () 495ignore_cond = False 496 497usage = False 498src = list() 499only_targs = list () 500for x in sys.argv[1:]: 501 if x[0:2] == "-b": 502 build_dir = x[2:] 503 elif x[0:2] == "-f": 504 fn = normalize_header (x[2:]) 505 if fn not in only_h: 506 only_h.append (fn) 507 elif x[0:2] == "-h": 508 usage = True 509 elif x[0:2] == "-d": 510 ignore_cond = True 511 elif x[0:2] == "-D": 512 ignore_list.append(x[2:]) 513 elif x[0:2] == "-T": 514 only_targs.append(x[2:]) 515 elif x[0:2] == "-t": 516 target_dir = x[2:] 517 elif x[0] == "-": 518 print "Error: Unrecognized option " + x 519 usgae = True 520 else: 521 if not os.path.exists (x): 522 print "Error: specified file " + x + " does not exist." 523 usage = True 524 else: 525 src.append (x) 526 527if target_dir: 528 build_target_dict (target_dir, only_targs) 529 530if build_dir == "" and target_dir == "": 531 print "Error: Must specify a build directory, and/or a target directory." 532 usage = True 533 534if build_dir and not os.path.exists (build_dir): 535 print "Error: specified build directory does not exist : " + build_dir 536 usage = True 537 538if target_dir and not os.path.exists (target_dir): 539 print "Error: specified target directory does not exist : " + target_dir 540 usage = True 541 542if usage: 543 print "Attempts to remove extraneous include files from source files." 544 print " " 545 print "Should be run from the main gcc source directory, and works on a target" 546 print "directory, as we attempt to make the 'all' target." 547 print " " 548 print "By default, gcc-reorder-includes is run on each file before attempting" 549 print "to remove includes. this removes duplicates and puts some headers in a" 550 print "canonical ordering" 551 print " " 552 print "The build directory should be ready to compile via make. Time is saved" 553 print "if the build is already complete, so that only changes need to be built." 554 print " " 555 print "Usage: [options] file1.c [file2.c] ... [filen.c]" 556 print " -bdir : the root build directory to attempt buiding .o files." 557 print " -tdir : the target build directory" 558 print " -d : Ignore conditional macro dependencies." 559 print " " 560 print " -Dmacro : Ignore a specific macro for dependencies" 561 print " -Ttarget : Only consider target in target directory." 562 print " -fheader : Specifies a specific .h file to be considered." 563 print " " 564 print " -D, -T, and -f can be specified mulitple times and are aggregated." 565 print " " 566 print " The original file will be in filen.bak" 567 print " " 568 sys.exit (0) 569 570if only_h: 571 print "Attempting to remove only these files:" 572 for x in only_h: 573 print x 574 print " " 575 576logfile = open("reduce-headers.log","w") 577 578for x in src: 579 msg = try_to_remove (x, only_h, logfile) 580 ilog = open("reduce-headers.sum","a") 581 ilog.write (msg + "\n") 582 ilog.close() 583 584ilog = open("reduce-headers.sum","a") 585ilog.write ("===============================================================\n") 586for x in remove_count: 587 msg = x + ": Removed " + str(remove_count[x]) + " times." 588 print msg 589 logfile.write (msg + "\n") 590 ilog.write (msg + "\n") 591 592 593 594 595 596