1#! /usr/bin/python2
2import os.path
3import sys
4import shlex
5import re
6import tempfile
7import copy
8
9from headerutils import *
10
11requires = { }
12provides = { }
13
14no_remove = [ "system.h", "coretypes.h", "config.h" , "bconfig.h", "backend.h" ]
15
16# These targets are the ones which provide "coverage".  Typically, if any
17# target is going to fail compilation, it's one of these.  This was determined
18# during the initial runs of reduce-headers... On a full set of target builds,
19# every failure which occured was triggered by one of these.
20# This list is used during target-list construction simply to put any of these
21# *first* in the candidate list, increasing the probability that a failure is
22# found quickly.
23target_priority = [
24    "aarch64-linux-gnu",
25    "arm-netbsdelf",
26    "c6x-elf",
27    "epiphany-elf",
28    "hppa2.0-hpux10.1",
29    "i686-mingw32crt",
30    "i686-pc-msdosdjgpp",
31    "mipsel-elf",
32    "powerpc-eabisimaltivec",
33    "rs6000-ibm-aix5.1.0",
34    "sh-superh-elf",
35    "sparc64-elf",
36    "spu-elf"
37]
38
39
40target_dir = ""
41build_dir = ""
42ignore_list = list()
43target_builds = list()
44
45target_dict = { }
46header_dict = { }
47search_path = [ ".", "../include", "../libcpp/include" ]
48
49remove_count = { }
50
51
52# Given a header name, normalize it.  ie.  cp/cp-tree.h could be in gcc, while
53# the same header could be referenced from within the cp subdirectory as
54# just cp-tree.h
55# for now, just assume basenames are unique
56
57def normalize_header (header):
58  return os.path.basename (header)
59
60
61# Adds a header file and its sub-includes to the global dictionary if they
62# aren't already there.  Specify s_path since different build directories may
63# append themselves on demand to the global list.
64# return entry for the specified header, knowing all sub entries are completed
65
66def get_header_info (header, s_path):
67  global header_dict
68  global empty_iinfo
69  process_list = list ()
70  location = ""
71  bname = ""
72  bname_iinfo = empty_iinfo
73  for path in s_path:
74    if os.path.exists (path + "/" + header):
75      location = path + "/" + header
76      break
77
78  if location:
79    bname = normalize_header (location)
80    if header_dict.get (bname):
81      bname_iinfo = header_dict[bname]
82      loc2 = ii_path (bname_iinfo)+ "/" + bname
83      if loc2[:2] == "./":
84        loc2 = loc2[2:]
85      if location[:2] == "./":
86        location = location[2:]
87      if loc2 != location:
88        # Don't use the cache if it isnt the right one.
89        bname_iinfo = process_ii_macro (location)
90      return bname_iinfo
91
92    bname_iinfo = process_ii_macro (location)
93    header_dict[bname] = bname_iinfo
94    # now decend into the include tree
95    for i in ii_include_list (bname_iinfo):
96      get_header_info (i, s_path)
97  else:
98    # if the file isnt in the source directories, look in the build and target
99    # directories. If it is here, then aggregate all the versions.
100    location = build_dir + "/gcc/" + header
101    build_inc = target_inc = False
102    if os.path.exists (location):
103      build_inc = True
104    for x in target_dict:
105      location = target_dict[x] + "/gcc/" + header
106      if os.path.exists (location):
107        target_inc = True
108        break
109
110    if (build_inc or target_inc):
111      bname = normalize_header(header)
112      defines = set()
113      consumes = set()
114      incl = set()
115      if build_inc:
116        iinfo = process_ii_macro (build_dir + "/gcc/" + header)
117        defines = set (ii_macro_define (iinfo))
118        consumes = set (ii_macro_consume (iinfo))
119        incl = set (ii_include_list (iinfo))
120
121      if (target_inc):
122        for x in target_dict:
123          location = target_dict[x] + "/gcc/" + header
124          if os.path.exists (location):
125            iinfo = process_ii_macro (location)
126            defines.update (ii_macro_define (iinfo))
127            consumes.update (ii_macro_consume (iinfo))
128            incl.update (ii_include_list (iinfo))
129
130      bname_iinfo = (header, "build", list(incl), list(), list(consumes), list(defines), list(), list())
131
132      header_dict[bname] = bname_iinfo
133      for i in incl:
134        get_header_info (i, s_path)
135
136  return bname_iinfo
137
138
139# return a list of all headers brought in by this header
140def all_headers (fname):
141  global header_dict
142  headers_stack = list()
143  headers_list = list()
144  if header_dict.get (fname) == None:
145    return list ()
146  for y in ii_include_list (header_dict[fname]):
147    headers_stack.append (y)
148
149  while headers_stack:
150    h = headers_stack.pop ()
151    hn = normalize_header (h)
152    if hn not in headers_list:
153      headers_list.append (hn)
154      if header_dict.get(hn):
155        for y in ii_include_list (header_dict[hn]):
156          if normalize_header (y) not in headers_list:
157            headers_stack.append (y)
158
159  return headers_list
160
161
162
163
164# Search bld_dir for all target tuples, confirm that they have a build path with
165# bld_dir/target-tuple/gcc, and build a dictionary of build paths indexed by
166# target tuple..
167
168def build_target_dict (bld_dir, just_these):
169  global target_dict
170  target_doct = { }
171  error = False
172  if os.path.exists (bld_dir):
173    if just_these:
174      ls = just_these
175    else:
176      ls = os.listdir(bld_dir)
177    for t in ls:
178      if t.find("-") != -1:
179        target = t.strip()
180        tpath = bld_dir + "/" + target
181        if not os.path.exists (tpath + "/gcc"):
182          print "Error: gcc build directory for target " + t + " Does not exist: " + tpath + "/gcc"
183          error = True
184        else:
185          target_dict[target] = tpath
186
187  if error:
188    target_dict = { }
189
190def get_obj_name (src_file):
191  if src_file[-2:] == ".c":
192    return src_file.replace (".c", ".o")
193  elif src_file[-3:] == ".cc":
194    return src_file.replace (".cc", ".o")
195  return ""
196
197def target_obj_exists (target, obj_name):
198  global target_dict
199  # look in a subdir if src has a subdir, then check gcc base directory.
200  if target_dict.get(target):
201    obj = target_dict[target] + "/gcc/" + obj_name
202    if not os.path.exists (obj):
203      obj = target_dict[target] + "/gcc/" + os.path.basename(obj_name)
204    if os.path.exists (obj):
205      return True
206  return False
207
208# Given a src file, return a list of targets which may build this file.
209def find_targets (src_file):
210  global target_dict
211  targ_list = list()
212  obj_name = get_obj_name (src_file)
213  if not obj_name:
214    print "Error: " + src_file + " - Cannot determine object name."
215    return list()
216
217  # Put the high priority targets which tend to trigger failures first
218  for target in target_priority:
219    if target_obj_exists (target, obj_name):
220      targ_list.append ((target, target_dict[target]))
221
222  for target in target_dict:
223    if target not in target_priority and target_obj_exists (target, obj_name):
224      targ_list.append ((target, target_dict[target]))
225
226  return targ_list
227
228
229def try_to_remove (src_file, h_list, verbose):
230  global target_dict
231  global header_dict
232  global build_dir
233
234  # build from scratch each time
235  header_dict = { }
236  summary = ""
237  rmcount = 0
238
239  because = { }
240  src_info = process_ii_macro_src (src_file)
241  src_data = ii_src (src_info)
242  if src_data:
243    inclist = ii_include_list_non_cond (src_info)
244    # work is done if there are no includes to check
245    if not inclist:
246      return src_file + ": No include files to attempt to remove"
247
248    # work on the include list in reverse.
249    inclist.reverse()
250
251    # Get the target list
252    targ_list = list()
253    targ_list = find_targets (src_file)
254
255    spath = search_path
256    if os.path.dirname (src_file):
257      spath.append (os.path.dirname (src_file))
258
259    hostbuild = True
260    if src_file.find("config/") != -1:
261      # config files dont usually build on the host
262      hostbuild = False
263      obn = get_obj_name (os.path.basename (src_file))
264      if obn and os.path.exists (build_dir + "/gcc/" + obn):
265        hostbuild = True
266      if not target_dict:
267        summary = src_file + ": Target builds are required for config files.  None found."
268        print summary
269        return summary
270      if not targ_list:
271        summary =src_file + ": Cannot find any targets which build this file."
272        print summary
273        return summary
274
275    if hostbuild:
276      # confirm it actually builds before we do anything
277      print "Confirming source file builds"
278      res = get_make_output (build_dir + "/gcc", "all")
279      if res[0] != 0:
280        message = "Error: " + src_file + " does not build currently."
281        summary = src_file + " does not build on host."
282        print message
283        print res[1]
284        if verbose:
285          verbose.write (message + "\n")
286          verbose.write (res[1]+ "\n")
287        return summary
288
289    src_requires = set (ii_macro_consume (src_info))
290    for macro in src_requires:
291      because[macro] = src_file
292    header_seen = list ()
293
294    os.rename (src_file, src_file + ".bak")
295    src_orig = copy.deepcopy (src_data)
296    src_tmp = copy.deepcopy (src_data)
297
298    try:
299      # process the includes from bottom to top.  This is because we know that
300      # later includes have are known to be needed, so any dependency from this
301      # header is a true dependency
302      for inc_file in inclist:
303        inc_file_norm = normalize_header (inc_file)
304
305        if inc_file in no_remove:
306          continue
307        if len (h_list) != 0 and inc_file_norm not in h_list:
308          continue
309        if inc_file_norm[0:3] == "gt-":
310          continue
311        if inc_file_norm[0:6] == "gtype-":
312          continue
313        if inc_file_norm.replace(".h",".c") == os.path.basename(src_file):
314          continue
315
316        lookfor = ii_src_line(src_info)[inc_file]
317        src_tmp.remove (lookfor)
318        message = "Trying " + src_file + " without " + inc_file
319        print message
320        if verbose:
321          verbose.write (message + "\n")
322        out = open(src_file, "w")
323        for line in src_tmp:
324          out.write (line)
325        out.close()
326
327        keep = False
328        if hostbuild:
329          res = get_make_output (build_dir + "/gcc", "all")
330        else:
331          res = (0, "")
332
333        rc = res[0]
334        message = "Passed Host build"
335        if (rc != 0):
336          # host build failed
337          message  = "Compilation failed:\n";
338          keep = True
339        else:
340          if targ_list:
341            objfile = get_obj_name (src_file)
342            t1 = targ_list[0]
343            if objfile and os.path.exists(t1[1] +"/gcc/"+objfile):
344              res = get_make_output_parallel (targ_list, objfile, 0)
345            else:
346              res = get_make_output_parallel (targ_list, "all-gcc", 0)
347            rc = res[0]
348            if rc != 0:
349              message = "Compilation failed on TARGET : " + res[2]
350              keep = True
351            else:
352              message = "Passed host and target builds"
353
354        if keep:
355          print message + "\n"
356
357        if (rc != 0):
358          if verbose:
359            verbose.write (message + "\n");
360            verbose.write (res[1])
361            verbose.write ("\n");
362            if os.path.exists (inc_file):
363              ilog = open(inc_file+".log","a")
364              ilog.write (message + " for " + src_file + ":\n\n");
365              ilog.write ("============================================\n");
366              ilog.write (res[1])
367              ilog.write ("\n");
368              ilog.close()
369            if os.path.exists (src_file):
370              ilog = open(src_file+".log","a")
371              ilog.write (message + " for " +inc_file + ":\n\n");
372              ilog.write ("============================================\n");
373              ilog.write (res[1])
374              ilog.write ("\n");
375              ilog.close()
376
377        # Given a sequence where :
378        # #include "tm.h"
379        # #include "target.h"  // includes tm.h
380
381        # target.h was required, and when attempting to remove tm.h we'd see that
382        # all the macro defintions are "required" since they all look like:
383        # #ifndef HAVE_blah
384        # #define HAVE_blah
385        # endif
386
387        # when target.h was found to be required, tm.h will be tagged as included.
388        # so when we get this far, we know we dont have to check the macros for
389        # tm.h since we know it is already been included.
390
391        if inc_file_norm not in header_seen:
392          iinfo = get_header_info (inc_file, spath)
393          newlist = all_headers (inc_file_norm)
394          if ii_path(iinfo) == "build" and not target_dict:
395            keep = True
396            text = message + " : Will not remove a build file without some targets."
397            print text
398            ilog = open(src_file+".log","a")
399            ilog.write (text +"\n")
400            ilog.write ("============================================\n");
401            ilog.close()
402            ilog = open("reduce-headers-kept.log","a")
403            ilog.write (src_file + " " + text +"\n")
404            ilog.close()
405        else:
406          newlist = list()
407        if not keep and inc_file_norm not in header_seen:
408          # now look for any macro requirements.
409          for h in newlist:
410            if not h in header_seen:
411              if header_dict.get(h):
412                defined = ii_macro_define (header_dict[h])
413                for dep in defined:
414                  if dep in src_requires and dep not in ignore_list:
415                    keep = True;
416                    text = message + ", but must keep " + inc_file + " because it provides " + dep
417                    if because.get(dep) != None:
418                      text = text + " Possibly required by " + because[dep]
419                    print text
420                    ilog = open(inc_file+".log","a")
421                    ilog.write (because[dep]+": Requires [dep] in "+src_file+"\n")
422                    ilog.write ("============================================\n");
423                    ilog.close()
424                    ilog = open(src_file+".log","a")
425                    ilog.write (text +"\n")
426                    ilog.write ("============================================\n");
427                    ilog.close()
428                    ilog = open("reduce-headers-kept.log","a")
429                    ilog.write (src_file + " " + text +"\n")
430                    ilog.close()
431                    if verbose:
432                      verbose.write (text + "\n")
433
434        if keep:
435          # add all headers 'consumes' to src_requires list, and mark as seen
436          for h in newlist:
437            if not h in header_seen:
438              header_seen.append (h)
439              if header_dict.get(h):
440                consume = ii_macro_consume (header_dict[h])
441                for dep in consume:
442                  if dep not in src_requires:
443                    src_requires.add (dep)
444                    if because.get(dep) == None:
445                      because[dep] = inc_file
446
447          src_tmp = copy.deepcopy (src_data)
448        else:
449          print message + "  --> removing " + inc_file + "\n"
450          rmcount += 1
451          if verbose:
452            verbose.write (message + "  --> removing " + inc_file + "\n")
453          if remove_count.get(inc_file) == None:
454            remove_count[inc_file] = 1
455          else:
456            remove_count[inc_file] += 1
457          src_data = copy.deepcopy (src_tmp)
458    except:
459      print "Interuption: restoring original file"
460      out = open(src_file, "w")
461      for line in src_orig:
462        out.write (line)
463      out.close()
464      raise
465
466    # copy current version, since it is the "right" one now.
467    out = open(src_file, "w")
468    for line in src_data:
469      out.write (line)
470    out.close()
471
472    # Try a final host bootstrap build to make sure everything is kosher.
473    if hostbuild:
474      res = get_make_output (build_dir, "all")
475      rc = res[0]
476      if (rc != 0):
477        # host build failed! return to original version
478        print "Error: " + src_file + " Failed to bootstrap at end!!! restoring."
479        print "        Bad version at " + src_file + ".bad"
480        os.rename (src_file, src_file + ".bad")
481        out = open(src_file, "w")
482        for line in src_orig:
483          out.write (line)
484        out.close()
485        return src_file + ": failed to build after reduction.  Restored original"
486
487    if src_data == src_orig:
488      summary = src_file + ": No change."
489    else:
490      summary = src_file + ": Reduction performed, "+str(rmcount)+" includes removed."
491  print summary
492  return summary
493
494only_h = list ()
495ignore_cond = False
496
497usage = False
498src = list()
499only_targs = list ()
500for x in sys.argv[1:]:
501  if x[0:2] == "-b":
502    build_dir = x[2:]
503  elif x[0:2] == "-f":
504    fn = normalize_header (x[2:])
505    if fn not in only_h:
506      only_h.append (fn)
507  elif x[0:2] == "-h":
508    usage = True
509  elif x[0:2] == "-d":
510    ignore_cond = True
511  elif x[0:2] == "-D":
512    ignore_list.append(x[2:])
513  elif x[0:2] == "-T":
514    only_targs.append(x[2:])
515  elif x[0:2] == "-t":
516    target_dir = x[2:]
517  elif x[0] == "-":
518    print "Error:  Unrecognized option " + x
519    usgae = True
520  else:
521    if not os.path.exists (x):
522      print "Error: specified file " + x + " does not exist."
523      usage = True
524    else:
525      src.append (x)
526
527if target_dir:
528  build_target_dict (target_dir, only_targs)
529
530if build_dir == "" and target_dir == "":
531  print "Error: Must specify a build directory, and/or a target directory."
532  usage = True
533
534if build_dir and not os.path.exists (build_dir):
535    print "Error: specified build directory does not exist : " + build_dir
536    usage = True
537
538if target_dir and not os.path.exists (target_dir):
539    print "Error: specified target directory does not exist : " + target_dir
540    usage = True
541
542if usage:
543  print "Attempts to remove extraneous include files from source files."
544  print " "
545  print "Should be run from the main gcc source directory, and works on a target"
546  print "directory, as we attempt to make the 'all' target."
547  print " "
548  print "By default, gcc-reorder-includes is run on each file before attempting"
549  print "to remove includes. this removes duplicates and puts some headers in a"
550  print "canonical ordering"
551  print " "
552  print "The build directory should be ready to compile via make. Time is saved"
553  print "if the build is already complete, so that only changes need to be built."
554  print " "
555  print "Usage: [options] file1.c [file2.c] ... [filen.c]"
556  print "      -bdir    : the root build directory to attempt buiding .o files."
557  print "      -tdir    : the target build directory"
558  print "      -d       : Ignore conditional macro dependencies."
559  print " "
560  print "      -Dmacro  : Ignore a specific macro for dependencies"
561  print "      -Ttarget : Only consider target in target directory."
562  print "      -fheader : Specifies a specific .h file to be considered."
563  print " "
564  print "      -D, -T, and -f can be specified mulitple times and are aggregated."
565  print " "
566  print "  The original file will be in filen.bak"
567  print " "
568  sys.exit (0)
569
570if only_h:
571  print "Attempting to remove only these files:"
572  for x in only_h:
573    print x
574  print " "
575
576logfile = open("reduce-headers.log","w")
577
578for x in src:
579  msg = try_to_remove (x, only_h, logfile)
580  ilog = open("reduce-headers.sum","a")
581  ilog.write (msg + "\n")
582  ilog.close()
583
584ilog = open("reduce-headers.sum","a")
585ilog.write ("===============================================================\n")
586for x in remove_count:
587  msg = x + ": Removed " + str(remove_count[x]) + " times."
588  print msg
589  logfile.write (msg + "\n")
590  ilog.write (msg + "\n")
591
592
593
594
595
596