1#!/usr/bin/env python
2
3from __future__ import print_function
4from collections import OrderedDict
5from shutil import copyfile
6import argparse
7import json
8import os
9import pprint
10import re
11import subprocess
12import sys
13import tempfile
14
15
16def normalize(dict_var):
17    for k, v in dict_var.items():
18        if isinstance(v, OrderedDict):
19            normalize(v)
20        elif isinstance(v, list):
21            for e in v:
22                if isinstance(e, OrderedDict):
23                    normalize(e)
24        elif type(v) is unicode:
25            st = v.encode('utf-8')
26            if v != "0x0" and re.match(r"0x[0-9A-Fa-f]+", v):
27                dict_var[k] = u'0x{{.*}}'
28            elif os.path.isfile(v):
29                dict_var[k] = u'{{.*}}'
30            else:
31                splits = (v.split(u' '))
32                out_splits = []
33                for split in splits:
34                    inner_splits = split.rsplit(u':',2)
35                    if os.path.isfile(inner_splits[0]):
36                        out_splits.append(
37                            u'{{.*}}:%s:%s'
38                            %(inner_splits[1],
39                              inner_splits[2]))
40                        continue
41                    out_splits.append(split)
42
43                dict_var[k] = ' '.join(out_splits)
44
45def filter_json(dict_var, filters, out):
46    for k, v in dict_var.items():
47        if type(v) is unicode:
48            st = v.encode('utf-8')
49            if st in filters:
50                out.append(dict_var)
51                break
52        elif isinstance(v, OrderedDict):
53            filter_json(v, filters, out)
54        elif isinstance(v, list):
55            for e in v:
56                if isinstance(e, OrderedDict):
57                    filter_json(e, filters, out)
58
59
60def default_clang_path():
61    guessed_clang = os.path.join(os.path.dirname(__file__), "clang")
62    if os.path.isfile(guessed_clang):
63        return guessed_clang
64    return None
65
66
67def main():
68    parser = argparse.ArgumentParser()
69    parser.add_argument("--clang", help="The clang binary (could be a relative or absolute path)",
70                        action="store", default=default_clang_path())
71    parser.add_argument("--source", help="the source file(s). Without --update, the command used to generate the JSON "
72                                         "will be of the format <clang> -cc1 -ast-dump=json <opts> <source>",
73                        action="store", nargs=argparse.ONE_OR_MORE, required=True)
74    parser.add_argument("--filters", help="comma separated list of AST filters. Ex: --filters=TypedefDecl,BuiltinType",
75                        action="store", default='')
76    update_or_generate_group = parser.add_mutually_exclusive_group()
77    update_or_generate_group.add_argument("--update", help="Update the file in-place", action="store_true")
78    update_or_generate_group.add_argument("--opts", help="other options",
79                                          action="store", default='', type=str)
80    parser.add_argument("--update-manual", help="When using --update, also update files that do not have the "
81                                                "autogenerated disclaimer", action="store_true")
82    args = parser.parse_args()
83
84    if not args.source:
85        sys.exit("Specify the source file to give to clang.")
86
87    clang_binary = os.path.abspath(args.clang)
88    if not os.path.isfile(clang_binary):
89        sys.exit("clang binary specified not present.")
90
91    for src in args.source:
92        process_file(src, clang_binary, cmdline_filters=args.filters,
93                     cmdline_opts=args.opts, do_update=args.update,
94                     force_update=args.update_manual)
95
96
97def process_file(source_file, clang_binary, cmdline_filters, cmdline_opts,
98                 do_update, force_update):
99    note_firstline = "// NOTE: CHECK lines have been autogenerated by " \
100                     "gen_ast_dump_json_test.py"
101    filters_line_prefix = "// using --filters="
102    note = note_firstline
103
104    cmd = [clang_binary, "-cc1"]
105    if do_update:
106        # When updating the first line of the test must be a RUN: line
107        with open(source_file, "r") as srcf:
108            first_line = srcf.readline()
109            found_autogenerated_line = False
110            filters_line = None
111            for i, line in enumerate(srcf.readlines()):
112                if found_autogenerated_line:
113                    # print("Filters line: '", line.rstrip(), "'", sep="")
114                    if line.startswith(filters_line_prefix):
115                        filters_line = line[len(filters_line_prefix):].rstrip()
116                    break
117                if line.startswith(note_firstline):
118                    found_autogenerated_line = True
119                    # print("Found autogenerated disclaimer at line", i + 1)
120        if not found_autogenerated_line and not force_update:
121            print("Not updating", source_file, "since it is not autogenerated.",
122                  file=sys.stderr)
123            return
124        if not cmdline_filters and filters_line:
125            cmdline_filters = filters_line
126            print("Inferred filters as '" + cmdline_filters + "'")
127
128        if "RUN: %clang_cc1 " not in first_line:
129            sys.exit("When using --update the first line of the input file must contain RUN: %clang_cc1")
130        clang_start = first_line.find("%clang_cc1") + len("%clang_cc1")
131        file_check_idx = first_line.rfind("| FileCheck")
132        if file_check_idx:
133            dump_cmd = first_line[clang_start:file_check_idx]
134        else:
135            dump_cmd = first_line[clang_start:]
136        print("Inferred run arguments as '", dump_cmd, "'", sep="")
137        options = dump_cmd.split()
138        if "-ast-dump=json" not in options:
139            sys.exit("ERROR: RUN: line does not contain -ast-dump=json")
140        if "%s" not in options:
141            sys.exit("ERROR: RUN: line does not contain %s")
142        options.remove("%s")
143    else:
144        options = cmdline_opts.split()
145        options.append("-ast-dump=json")
146    cmd.extend(options)
147    using_ast_dump_filter = any('ast-dump-filter' in arg for arg in cmd)
148    cmd.append(source_file)
149    print("Will run", cmd)
150    filters = set()
151    if cmdline_filters:
152        note += "\n" + filters_line_prefix + cmdline_filters
153        filters = set(cmdline_filters.split(','))
154    print("Will use the following filters:", filters)
155
156    try:
157        json_str = subprocess.check_output(cmd)
158    except Exception as ex:
159        print("The clang command failed with %s" % ex)
160        return -1
161
162    out_asts = []
163    if using_ast_dump_filter:
164        splits = re.split('Dumping .*:\n', json_str)
165        if len(splits) > 1:
166            for split in splits[1:]:
167                j = json.loads(split.decode('utf-8'), object_pairs_hook=OrderedDict)
168                normalize(j)
169                out_asts.append(j)
170    else:
171        j = json.loads(json_str.decode('utf-8'), object_pairs_hook=OrderedDict)
172        normalize(j)
173
174        if len(filters) == 0:
175            out_asts.append(j)
176        else:
177            #assert using_ast_dump_filter is False,\
178            #    "Does not support using compiler's ast-dump-filter "\
179            #    "and the tool's filter option at the same time yet."
180
181            filter_json(j, filters, out_asts)
182
183    with tempfile.NamedTemporaryFile("wb", delete=False) as f:
184        with open(source_file, "r") as srcf:
185            for line in srcf.readlines():
186                # copy up to the note:
187                if line.rstrip() == note_firstline:
188                    break
189                f.write(line)
190        f.write(note + "\n")
191        for out_ast in out_asts:
192            append_str = json.dumps(out_ast, indent=1, ensure_ascii=False)
193            out_str = '\n\n'
194            index = 0
195            for append_line in append_str.splitlines()[2:]:
196                if index == 0:
197                    out_str += '// CHECK: %s\n' %(append_line.rstrip())
198                    index += 1
199                else:
200                    out_str += '// CHECK-NEXT: %s\n' %(append_line.rstrip())
201
202            f.write(out_str)
203        f.flush()
204        f.close()
205        if do_update:
206            print("Updating json appended source file to %s." % source_file)
207            copyfile(f.name, source_file)
208        else:
209            partition = source_file.rpartition('.')
210            dest_path = '%s-json%s%s' % (partition[0], partition[1], partition[2])
211            print("Writing json appended source file to %s." % dest_path)
212            copyfile(f.name, dest_path)
213        os.remove(f.name)
214    return 0
215
216
217if __name__ == '__main__':
218    main()
219