1#! /usr/bin/python2
2import os.path
3import sys
4import shlex
5import re
6import subprocess
7import shutil
8import pickle
9
10import multiprocessing
11
12def find_pound_include (line, use_outside, use_slash):
13  inc = re.findall (ur"^\s*#\s*include\s*\"(.+?)\"", line)
14  if len(inc) == 1:
15    nm = inc[0]
16    if use_outside or os.path.exists (nm):
17      if use_slash or '/' not in nm:
18        return nm
19  return ""
20
21def find_system_include (line):
22  inc = re.findall (ur"^\s*#\s*include\s*<(.+?)>", line)
23  if len(inc) == 1:
24    return inc[0]
25  return ""
26
27def find_pound_define (line):
28  inc = re.findall (ur"^\s*#\s*define ([A-Za-z0-9_]+)", line)
29  if len(inc) != 0:
30    if len(inc) > 1:
31      print "What? more than 1 match in #define??"
32      print inc
33      sys.exit(5)
34    return inc[0];
35  return ""
36
37def is_pound_if (line):
38  inc = re.findall ("^\s*#\s*if\s", line)
39  if not inc:
40    inc = re.findall ("^\s*#\s*if[n]?def\s", line)
41  if inc:
42    return True
43  return False
44
45def is_pound_endif (line):
46  inc = re.findall ("^\s*#\s*endif", line)
47  if inc:
48    return True
49  return False
50
51def find_pound_if (line):
52  inc = re.findall (ur"^\s*#\s*if\s+(.*)", line)
53  if len(inc) == 0:
54    inc = re.findall (ur"^\s*#\s*elif\s+(.*)", line)
55  if len(inc) > 0:
56    inc2 = re.findall (ur"defined\s*\((.+?)\)", inc[0])
57    inc3 = re.findall (ur"defined\s+([a-zA-Z0-9_]+)", inc[0])
58    for yy in inc3:
59      inc2.append (yy)
60    return inc2
61  else:
62    inc = re.findall (ur"^\s*#\s*ifdef\s(.*)", line)
63    if len(inc) == 0:
64      inc = re.findall (ur"^\s*#\s*ifndef\s(.*)", line)
65    if len(inc) > 0:
66      inc2 = re.findall ("[A-Za-z_][A-Za-z_0-9]*", inc[0])
67      return inc2
68  if len(inc) == 0:
69    return list ()
70  print "WTF. more than one line returned for find_pound_if"
71  print inc
72  sys.exit(5)
73
74
75# IINFO - this is a vector of include information. It consists of 7 elements.
76# [0] - base name of the file
77# [1] - path leading to this file.
78# [2] - orderd list of all headers directly included by this file.
79# [3] - Ordered list of any headers included within condionally compiled code.
80#       headers files are expected to have all includes one level deep due to
81#       the omnipresent guards at the top of the file.
82# [4] - List of all macros which are consumed (used) within this file.
83# [5] - list of all macros which may be defined in this file.
84# [6] - The source code for this file, if cached.
85# [7] - line number info for any headers in the source file.  Indexed by base
86#       name, returning the line the include is on.
87
88empty_iinfo =  ("", "", list(), list(), list(), list(), list())
89
90# This function will process a file and extract interesting information.
91# DO_MACROS indicates whether macros defined and used should be recorded.
92# KEEP_SRC indicates the source for the file should be cached.
93def process_include_info (filen, do_macros, keep_src):
94  header = False
95  if not os.path.exists (filen):
96    return empty_iinfo
97
98  sfile = open (filen, "r");
99  data = sfile.readlines()
100  sfile.close()
101
102  # Ignore the initial #ifdef HEADER_H in header files
103  if filen[-2:] == ".h":
104    nest = -1
105    header = True
106  else:
107    nest = 0
108
109  macout = list ()
110  macin = list()
111  incl = list()
112  cond_incl = list()
113  src_line = { }
114  guard = ""
115
116  for line in (data):
117    if is_pound_if (line):
118      nest += 1
119    elif is_pound_endif (line):
120      nest -= 1
121
122    nm = find_pound_include (line, True, True)
123    if nm != "" and nm not in incl and nm[-2:] == ".h":
124      incl.append (nm)
125      if nest > 0:
126        cond_incl.append (nm)
127      if keep_src:
128        src_line[nm] = line
129      continue
130
131    if do_macros:
132      d = find_pound_define (line)
133      if d:
134        if d not in macout:
135          macout.append (d);
136          continue
137
138      d = find_pound_if (line)
139      if d:
140        # The first #if in a header file should be the guard
141        if header and len (d) == 1 and guard == "":
142          if d[0][-2:] == "_H":
143            guard = d
144          else:
145            guard = "Guess there was no guard..."
146        else:
147          for mac in d:
148            if mac != "defined" and mac not in macin:
149              macin.append (mac);
150
151  if not keep_src:
152    data = list()
153
154  return (os.path.basename (filen), os.path.dirname (filen), incl, cond_incl,
155          macin, macout, data, src_line)
156
157# Extract header info, but no macros or source code.
158def process_ii (filen):
159  return process_include_info (filen, False, False)
160
161# Extract header information, and collect macro information.
162def process_ii_macro (filen):
163  return process_include_info (filen, True, False)
164
165# Extract header information, cache the source lines.
166def process_ii_src (filen):
167  return process_include_info (filen, False, True)
168
169# Extract header information, coolewc macro info and cache the source lines.
170def process_ii_macro_src (filen):
171  return process_include_info (filen, True, True)
172
173
174def ii_base (iinfo):
175  return iinfo[0]
176
177def ii_path (iinfo):
178  return iinfo[1]
179
180def ii_include_list (iinfo):
181  return iinfo[2]
182
183def ii_include_list_cond (iinfo):
184  return iinfo[3]
185
186def ii_include_list_non_cond (iinfo):
187  l = ii_include_list (iinfo)
188  for n in ii_include_list_cond (iinfo):
189    l.remove (n)
190  return l
191
192def ii_macro_consume (iinfo):
193  return iinfo[4]
194
195def ii_macro_define (iinfo):
196  return iinfo[5]
197
198def ii_src (iinfo):
199  return iinfo[6]
200
201def ii_src_line (iinfo):
202  return iinfo[7]
203
204def ii_read (fname):
205  f = open (fname, 'rb')
206  incl = pickle.load (f)
207  consumes = pickle.load (f)
208  defines = pickle.load (f)
209  obj = (fname,fname,incl,list(), list(), consumes, defines, list(), list())
210  return obj
211
212def ii_write (fname, obj):
213  f = open (fname, 'wb')
214  pickle.dump (obj[2], f)
215  pickle.dump (obj[4], f)
216  pickle.dump (obj[5], f)
217  f.close ()
218
219# execute a system command which returns file names
220def execute_command (command):
221  files = list()
222  f = os.popen (command)
223  for x in f:
224    if x[0:2] == "./":
225      fn = x.rstrip()[2:]
226    else:
227      fn = x.rstrip()
228    files.append(fn)
229  return files
230
231# Try to locate a build directory from PATH
232def find_gcc_bld_dir (path):
233  blddir = ""
234  # Look for blddir/gcc/tm.h
235  command = "find " + path + " -mindepth 2 -maxdepth 3 -name tm.h"
236  files = execute_command (command)
237  for y in files:
238    p = os.path.dirname (y)
239    if os.path.basename (p) == "gcc":
240      blddir = p
241      break
242  # If not found, try looking a bit deeper
243  # Dont look this deep initially because a lot of cross target builds may show
244  # up in the list before a native build... but those are better than nothing.
245  if not blddir:
246    command = "find " + path + " -mindepth 3 -maxdepth 5 -name tm.h"
247    files = execute_command (command)
248    for y in files:
249      p = os.path.dirname (y)
250      if os.path.basename (p) == "gcc":
251	blddir = p
252	break
253
254  return blddir
255
256
257# Find files matching pattern NAME, return in a list.
258# CURRENT is True if you want to include the current directory
259# DEEPER is True if you want to search 3 levels below the current directory
260# any files with testsuite diurectories are ignored
261
262def find_gcc_files (name, current, deeper):
263  files = list()
264  command = ""
265  if current:
266    if not deeper:
267      command = "find -maxdepth 1 -name " + name + " -not -path \"./testsuite/*\""
268    else:
269      command = "find -maxdepth 4 -name " + name + " -not -path \"./testsuite/*\""
270  else:
271    if deeper:
272      command = "find -maxdepth 4 -mindepth 2 -name " + name + " -not -path \"./testsuite/*\""
273
274  if command != "":
275    files = execute_command (command)
276
277  return files
278
279# find the list of unique include names found in a file.
280def find_unique_include_list_src (data):
281  found = list ()
282  for line in data:
283    d = find_pound_include (line, True, True)
284    if d and d not in found and d[-2:] == ".h":
285      found.append (d)
286  return found
287
288# find the list of unique include names found in a file.
289def find_unique_include_list (filen):
290  data = open (filen).read().splitlines()
291  return find_unique_include_list_src (data)
292
293
294# Create the macin, macout, and incl vectors for a file FILEN.
295# macin are the macros that are used in #if* conditional expressions
296# macout are the macros which are #defined
297# incl is the list of incluide files encountered
298# returned as a tuple of the filename followed by the triplet of lists
299# (filen, macin, macout, incl)
300
301def create_macro_in_out (filen):
302  sfile = open (filen, "r");
303  data = sfile.readlines()
304  sfile.close()
305
306  macout = list ()
307  macin = list()
308  incl = list()
309
310  for line in (data):
311    d = find_pound_define (line)
312    if d != "":
313      if d not in macout:
314        macout.append (d);
315      continue
316
317    d = find_pound_if (line)
318    if len(d) != 0:
319      for mac in d:
320        if mac != "defined" and mac not in macin:
321          macin.append (mac);
322      continue
323
324    nm = find_pound_include (line, True, True)
325    if nm != "" and nm not in incl:
326      incl.append (nm)
327
328  return (filen, macin, macout, incl)
329
330# create the macro information for filen, and create .macin, .macout, and .incl
331# files.  Return the created macro tuple.
332def create_include_data_files (filen):
333
334  macros = create_macro_in_out (filen)
335  depends = macros[1]
336  defines = macros[2]
337  incls = macros[3]
338
339  disp_message = filen
340  if len (defines) > 0:
341    disp_message = disp_message + " " + str(len (defines)) + " #defines"
342  dfile = open (filen + ".macout", "w")
343  for x in defines:
344    dfile.write (x + "\n")
345  dfile.close ()
346
347  if len (depends) > 0:
348    disp_message = disp_message + " " + str(len (depends)) + " #if dependencies"
349  dfile = open (filen + ".macin", "w")
350  for x in depends:
351    dfile.write (x + "\n")
352  dfile.close ()
353
354  if len (incls) > 0:
355    disp_message = disp_message + " " + str(len (incls)) + " #includes"
356  dfile = open (filen + ".incl", "w")
357  for x in incls:
358    dfile.write (x + "\n")
359  dfile.close ()
360
361  return macros
362
363
364
365# extract data for include file name_h and enter it into the dictionary.
366# this does not change once read in.  use_requires is True if you want to
367# prime the values with already created .requires and .provides files.
368def get_include_data (name_h, use_requires):
369  macin = list()
370  macout = list()
371  incl = list ()
372  if use_requires and os.path.exists (name_h + ".requires"):
373    macin = open (name_h + ".requires").read().splitlines()
374  elif os.path.exists (name_h + ".macin"):
375    macin = open (name_h + ".macin").read().splitlines()
376
377  if use_requires and os.path.exists (name_h + ".provides"):
378    macout  = open (name_h + ".provides").read().splitlines()
379  elif os.path.exists (name_h + ".macout"):
380    macout  = open (name_h + ".macout").read().splitlines()
381
382  if os.path.exists (name_h + ".incl"):
383    incl = open (name_h + ".incl").read().splitlines()
384
385  if len(macin) == 0 and len(macout) == 0 and len(incl) == 0:
386    return ()
387  data = ( name_h, macin, macout, incl )
388  return data
389
390# find FIND in src, and replace it with the list of headers in REPLACE.
391# Remove any duplicates of FIND in REPLACE, and if some of the REPLACE
392# headers occur earlier in the include chain, leave them.
393# Return the new SRC only if anything changed.
394def find_replace_include (find, replace, src):
395  res = list()
396  seen = { }
397  anything = False
398  for line in src:
399    inc = find_pound_include (line, True, True)
400    if inc == find:
401      for y in replace:
402        if seen.get(y) == None:
403          res.append("#include \""+y+"\"\n")
404          seen[y] = True
405          if y != find:
406            anything = True
407# if find isnt in the replacement list, then we are deleting FIND, so changes.
408      if find not in replace:
409        anything = True
410    else:
411      if inc in replace:
412        if seen.get(inc) == None:
413          res.append (line)
414          seen[inc] = True
415      else:
416        res.append (line)
417
418  if (anything):
419    return res
420  else:
421    return list()
422
423
424# pass in a require and provide dictionary to be read in.
425def read_require_provides (require, provide):
426  if not os.path.exists ("require-provide.master"):
427    print "require-provide.master file is not available. please run data collection."
428    sys.exit(1)
429  incl_list = open("require-provide.master").read().splitlines()
430  for f in incl_list:
431    if os.path.exists (f+".requires"):
432      require[os.path.basename (f)] = open (f + ".requires").read().splitlines()
433    else:
434      require[os.path.basename (f)] = list ()
435    if os.path.exists (f+".provides"):
436      provide[os.path.basename (f)] = open (f + ".provides").read().splitlines()
437    else:
438      provide [os.path.basename (f)] = list ()
439
440
441def build_include_list (filen):
442  include_files = list()
443  sfile = open (filen, "r")
444  data = sfile.readlines()
445  sfile.close()
446  for line in data:
447    nm = find_pound_include (line, False, False)
448    if nm != "" and nm[-2:] == ".h":
449      if nm not in include_files:
450        include_files.append(nm)
451  return include_files
452
453def build_reverse_include_list (filen):
454  include_files = list()
455  sfile = open (filen, "r")
456  data = sfile.readlines()
457  sfile.close()
458  for line in reversed(data):
459    nm = find_pound_include (line, False, False)
460    if nm != "":
461      if nm not in include_files:
462        include_files.append(nm)
463  return include_files
464
465# Get compilation return code, and compensate for a warning that we want to
466# consider an error when it comes to inlined templates.
467def get_make_rc (rc, output):
468  rc = rc % 1280
469  if rc == 0:
470    # This is not considered an error during compilation of an individual file,
471    # but it will cause an error during link if it isn't defined.  If this
472    # warning is seen during compiling a file, make it a build error so we
473    # don't remove the header.
474    h = re.findall ("warning: inline function.*used but never defined", output)
475    if len(h) != 0:
476      rc = 1
477  return rc;
478
479def get_make_output (build_dir, make_opt):
480  devnull = open('/dev/null', 'w')
481  at_a_time = multiprocessing.cpu_count() * 2
482  make = "make -j"+str(at_a_time)+ " "
483  if build_dir != "":
484    command = "cd " + build_dir +"; " + make + make_opt
485  else:
486    command = make + make_opt
487  process = subprocess.Popen(command, stdout=devnull, stderr=subprocess.PIPE, shell=True)
488  output = process.communicate();
489  rc = get_make_rc (process.returncode, output[1])
490  return (rc , output[1])
491
492def spawn_makes (command_list):
493  devnull = open('/dev/null', 'w')
494  rc = (0,"", "")
495  proc_res = list()
496  text = "  Trying target builds : "
497  for command_pair in command_list:
498    tname = command_pair[0]
499    command = command_pair[1]
500    text += tname + ", "
501    c = subprocess.Popen(command, bufsize=-1, stdout=devnull, stderr=subprocess.PIPE, shell=True)
502    proc_res.append ((c, tname))
503
504  print text[:-2]
505
506  for p in proc_res:
507    output = p[0].communicate()
508    ret = (get_make_rc (p[0].returncode, output[1]), output[1], p[1])
509    if (ret[0] != 0):
510      # Just record the first one.
511      if rc[0] == 0:
512        rc = ret;
513  return rc
514
515def get_make_output_parallel (targ_list, make_opt, at_a_time):
516  command = list()
517  targname = list()
518  if at_a_time == 0:
519    at_a_time = multiprocessing.cpu_count() * 2
520  proc_res = [0] * at_a_time
521  for x in targ_list:
522    if make_opt[-2:] == ".o":
523      s = "cd " + x[1] + "/gcc/; make " + make_opt
524    else:
525      s = "cd " + x[1] +"; make " + make_opt
526    command.append ((x[0],s))
527
528  num = len(command)
529  rc = (0,"", "")
530  loops = num // at_a_time
531
532  if (loops > 0):
533    for idx in range (loops):
534      ret = spawn_makes (command[idx*at_a_time:(idx+1)*at_a_time])
535      if ret[0] != 0:
536        rc = ret
537        break
538
539  if (rc[0] == 0):
540    leftover = num % at_a_time
541    if (leftover > 0):
542      ret = spawn_makes (command[-leftover:])
543      if ret[0] != 0:
544        rc = ret
545
546  return rc
547
548
549def readwholefile (src_file):
550  sfile = open (src_file, "r")
551  src_data = sfile.readlines()
552  sfile.close()
553  return src_data
554
555