1#!/usr/bin/env python3
2
3"""A script to generate FileCheck statements for 'opt' analysis tests.
4
5This script is a utility to update LLVM opt analysis test cases with new
6FileCheck patterns. It can either update all of the tests in the file or
7a single test function.
8
9Example usage:
10$ update_analyze_test_checks.py --opt=../bin/opt test/foo.ll
11
12Workflow:
131. Make a compiler patch that requires updating some number of FileCheck lines
14   in regression test files.
152. Save the patch and revert it from your local work area.
163. Update the RUN-lines in the affected regression tests to look canonical.
17   Example: "; RUN: opt < %s -analyze -cost-model -S | FileCheck %s"
184. Refresh the FileCheck lines for either the entire file or select functions by
19   running this script.
205. Commit the fresh baseline of checks.
216. Apply your patch from step 1 and rebuild your local binaries.
227. Re-run this script on affected regression tests.
238. Check the diffs to ensure the script has done something reasonable.
249. Submit a patch including the regression test diffs for review.
25
26A common pattern is to have the script insert complete checking of every
27instruction. Then, edit it down to only check the relevant instructions.
28The script is designed to make adding checks to a test case fast, it is *not*
29designed to be authoratitive about what constitutes a good test!
30"""
31
32from __future__ import print_function
33
34import argparse
35import glob
36import itertools
37import os         # Used to advertise this file's name ("autogenerated_note").
38import string
39import subprocess
40import sys
41import tempfile
42import re
43
44from UpdateTestChecks import common
45
46ADVERT = '; NOTE: Assertions have been autogenerated by '
47
48def main():
49  from argparse import RawTextHelpFormatter
50  parser = argparse.ArgumentParser(description=__doc__, formatter_class=RawTextHelpFormatter)
51  parser.add_argument('--opt-binary', default='opt',
52                      help='The opt binary used to generate the test case')
53  parser.add_argument(
54      '--function', help='The function in the test file to update')
55  parser.add_argument('tests', nargs='+')
56  args = common.parse_commandline_args(parser)
57
58  script_name = os.path.basename(__file__)
59  autogenerated_note = (ADVERT + 'utils/' + script_name)
60
61  opt_basename = os.path.basename(args.opt_binary)
62  if (opt_basename != "opt"):
63    common.error('Unexpected opt name: ' + opt_basename)
64    sys.exit(1)
65
66  test_paths = [test for pattern in args.tests for test in glob.glob(pattern)]
67  for test in test_paths:
68    with open(test) as f:
69      input_lines = [l.rstrip() for l in f]
70
71    first_line = input_lines[0] if input_lines else ""
72    if 'autogenerated' in first_line and script_name not in first_line:
73      common.warn("Skipping test which wasn't autogenerated by " + script_name + ": " + test)
74      continue
75
76    if args.update_only:
77      if not first_line or 'autogenerated' not in first_line:
78        common.warn("Skipping test which isn't autogenerated: " + test)
79        continue
80
81    run_lines = common.find_run_lines(test, input_lines)
82    prefix_list = []
83    for l in run_lines:
84      if '|' not in l:
85        common.warn('Skipping unparseable RUN line: ' + l)
86        continue
87
88      (tool_cmd, filecheck_cmd) = tuple([cmd.strip() for cmd in l.split('|', 1)])
89      common.verify_filecheck_prefixes(filecheck_cmd)
90
91      if not tool_cmd.startswith(opt_basename + ' '):
92        common.warn('WSkipping non-%s RUN line: %s' % (opt_basename, l))
93        continue
94
95      if not filecheck_cmd.startswith('FileCheck '):
96        common.warn('Skipping non-FileChecked RUN line: ' + l)
97        continue
98
99      tool_cmd_args = tool_cmd[len(opt_basename):].strip()
100      tool_cmd_args = tool_cmd_args.replace('< %s', '').replace('%s', '').strip()
101
102      check_prefixes = [item for m in common.CHECK_PREFIX_RE.finditer(filecheck_cmd)
103                               for item in m.group(1).split(',')]
104      if not check_prefixes:
105        check_prefixes = ['CHECK']
106
107      # FIXME: We should use multiple check prefixes to common check lines. For
108      # now, we just ignore all but the last.
109      prefix_list.append((check_prefixes, tool_cmd_args))
110
111    builder = common.FunctionTestBuilder(
112      run_list = prefix_list,
113      flags = type('', (object,), {
114            'verbose': args.verbose,
115            'function_signature': False,
116            'check_attributes': False,
117            'replace_value_regex': []}),
118      scrubber_args = [],
119      path=test)
120
121    for prefixes, opt_args in prefix_list:
122      common.debug('Extracted opt cmd:', opt_basename, opt_args, file=sys.stderr)
123      common.debug('Extracted FileCheck prefixes:', str(prefixes), file=sys.stderr)
124
125      raw_tool_outputs = common.invoke_tool(args.opt_binary, opt_args, test)
126
127      # Split analysis outputs by "Printing analysis " declarations.
128      for raw_tool_output in re.split(r'Printing analysis ', raw_tool_outputs):
129        builder.process_run_line(common.ANALYZE_FUNCTION_RE, common.scrub_body,
130                                 raw_tool_output, prefixes)
131
132    func_dict = builder.finish_and_get_func_dict()
133    is_in_function = False
134    is_in_function_start = False
135    prefix_set = set([prefix for prefixes, _ in prefix_list for prefix in prefixes])
136    common.debug('Rewriting FileCheck prefixes:', str(prefix_set), file=sys.stderr)
137    output_lines = []
138    output_lines.append(autogenerated_note)
139
140    for input_line in input_lines:
141      if is_in_function_start:
142        if input_line == '':
143          continue
144        if input_line.lstrip().startswith(';'):
145          m = common.CHECK_RE.match(input_line)
146          if not m or m.group(1) not in prefix_set:
147            output_lines.append(input_line)
148            continue
149
150        # Print out the various check lines here.
151        common.add_analyze_checks(output_lines, ';', prefix_list, func_dict, func_name)
152        is_in_function_start = False
153
154      if is_in_function:
155        if common.should_add_line_to_output(input_line, prefix_set):
156          # This input line of the function body will go as-is into the output.
157          # Except make leading whitespace uniform: 2 spaces.
158          input_line = common.SCRUB_LEADING_WHITESPACE_RE.sub(r'  ', input_line)
159          output_lines.append(input_line)
160        else:
161          continue
162        if input_line.strip() == '}':
163          is_in_function = False
164        continue
165
166      # Discard any previous script advertising.
167      if input_line.startswith(ADVERT):
168        continue
169
170      # If it's outside a function, it just gets copied to the output.
171      output_lines.append(input_line)
172
173      m = common.IR_FUNCTION_RE.match(input_line)
174      if not m:
175        continue
176      func_name = m.group(1)
177      if args.function is not None and func_name != args.function:
178        # When filtering on a specific function, skip all others.
179        continue
180      is_in_function = is_in_function_start = True
181
182    common.debug('Writing %d lines to %s...' % (len(output_lines), test))
183
184    with open(test, 'wb') as f:
185      f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])
186
187
188if __name__ == '__main__':
189  main()
190