1#!/usr/bin/env python3
2
3"""A script to generate FileCheck statements for 'opt' regression tests.
4
5This script is a utility to update LLVM opt test cases with new
6FileCheck patterns. It can either update all of the tests in the file or
7a single test function.
8
9Example usage:
10$ update_test_checks.py --opt=../bin/opt test/foo.ll
11
12Workflow:
131. Make a compiler patch that requires updating some number of FileCheck lines
14   in regression test files.
152. Save the patch and revert it from your local work area.
163. Update the RUN-lines in the affected regression tests to look canonical.
17   Example: "; RUN: opt < %s -instcombine -S | FileCheck %s"
184. Refresh the FileCheck lines for either the entire file or select functions by
19   running this script.
205. Commit the fresh baseline of checks.
216. Apply your patch from step 1 and rebuild your local binaries.
227. Re-run this script on affected regression tests.
238. Check the diffs to ensure the script has done something reasonable.
249. Submit a patch including the regression test diffs for review.
25
26A common pattern is to have the script insert complete checking of every
27instruction. Then, edit it down to only check the relevant instructions.
28The script is designed to make adding checks to a test case fast, it is *not*
29designed to be authoratitive about what constitutes a good test!
30"""
31
32from __future__ import print_function
33
34import argparse
35import glob
36import itertools
37import os         # Used to advertise this file's name ("autogenerated_note").
38import string
39import subprocess
40import sys
41import tempfile
42import re
43
44from UpdateTestChecks import common
45
46ADVERT = '; NOTE: Assertions have been autogenerated by '
47
48# RegEx: this is where the magic happens.
49
50IR_FUNCTION_RE = re.compile('^\s*define\s+(?:internal\s+)?[^@]*@([\w-]+)\s*\(')
51
52
53
54
55
56def main():
57  from argparse import RawTextHelpFormatter
58  parser = argparse.ArgumentParser(description=__doc__, formatter_class=RawTextHelpFormatter)
59  parser.add_argument('--opt-binary', default='opt',
60                      help='The opt binary used to generate the test case')
61  parser.add_argument(
62      '--function', help='The function in the test file to update')
63  parser.add_argument('-p', '--preserve-names', action='store_true',
64                      help='Do not scrub IR names')
65  parser.add_argument('--function-signature', action='store_true',
66                      help='Keep function signature information around for the check line')
67  parser.add_argument('--scrub-attributes', action='store_true',
68                      help='Remove attribute annotations (#0) from the end of check line')
69  parser.add_argument('tests', nargs='+')
70  args = common.parse_commandline_args(parser)
71
72  script_name = os.path.basename(__file__)
73  autogenerated_note = (ADVERT + 'utils/' + script_name)
74
75  opt_basename = os.path.basename(args.opt_binary)
76  if not re.match(r'^opt(-\d+)?$', opt_basename):
77    common.error('Unexpected opt name: ' + opt_basename)
78    sys.exit(1)
79  opt_basename = 'opt'
80
81  for test in args.tests:
82    if not glob.glob(test):
83      common.warn("Test file pattern '%s' was not found. Ignoring it." % (test,))
84      continue
85
86  # On Windows we must expand the patterns ourselves.
87  test_paths = [test for pattern in args.tests for test in glob.glob(pattern)]
88  for test in test_paths:
89    with open(test) as f:
90      input_lines = [l.rstrip() for l in f]
91
92    first_line = input_lines[0] if input_lines else ""
93    if 'autogenerated' in first_line and script_name not in first_line:
94      common.warn("Skipping test which wasn't autogenerated by " + script_name, test)
95      continue
96
97    if args.update_only:
98      if not first_line or 'autogenerated' not in first_line:
99        common.warn("Skipping test which isn't autogenerated: " + test)
100        continue
101
102    run_lines = common.find_run_lines(test, input_lines)
103
104    # If requested we scrub trailing attribute annotations, e.g., '#0', together with whitespaces
105    if args.scrub_attributes:
106      common.SCRUB_TRAILING_WHITESPACE_TEST_RE = common.SCRUB_TRAILING_WHITESPACE_AND_ATTRIBUTES_RE
107    else:
108      common.SCRUB_TRAILING_WHITESPACE_TEST_RE = common.SCRUB_TRAILING_WHITESPACE_RE
109
110    prefix_list = []
111    for l in run_lines:
112      if '|' not in l:
113        common.warn('Skipping unparseable RUN line: ' + l)
114        continue
115
116      (tool_cmd, filecheck_cmd) = tuple([cmd.strip() for cmd in l.split('|', 1)])
117      common.verify_filecheck_prefixes(filecheck_cmd)
118      if not tool_cmd.startswith(opt_basename + ' '):
119        common.warn('Skipping non-%s RUN line: %s' % (opt_basename, l))
120        continue
121
122      if not filecheck_cmd.startswith('FileCheck '):
123        common.warn('Skipping non-FileChecked RUN line: ' + l)
124        continue
125
126      tool_cmd_args = tool_cmd[len(opt_basename):].strip()
127      tool_cmd_args = tool_cmd_args.replace('< %s', '').replace('%s', '').strip()
128
129      check_prefixes = [item for m in common.CHECK_PREFIX_RE.finditer(filecheck_cmd)
130                               for item in m.group(1).split(',')]
131      if not check_prefixes:
132        check_prefixes = ['CHECK']
133
134      # FIXME: We should use multiple check prefixes to common check lines. For
135      # now, we just ignore all but the last.
136      prefix_list.append((check_prefixes, tool_cmd_args))
137
138    func_dict = {}
139    for prefixes, _ in prefix_list:
140      for prefix in prefixes:
141        func_dict.update({prefix: dict()})
142    for prefixes, opt_args in prefix_list:
143      common.debug('Extracted opt cmd: ' + opt_basename + ' ' + opt_args)
144      common.debug('Extracted FileCheck prefixes: ' + str(prefixes))
145
146      raw_tool_output = common.invoke_tool(args.opt_binary, opt_args, test)
147      common.build_function_body_dictionary(
148              common.OPT_FUNCTION_RE, common.scrub_body, [],
149              raw_tool_output, prefixes, func_dict, args.verbose,
150              args.function_signature)
151
152    is_in_function = False
153    is_in_function_start = False
154    prefix_set = set([prefix for prefixes, _ in prefix_list for prefix in prefixes])
155    common.debug('Rewriting FileCheck prefixes:', str(prefix_set))
156    output_lines = []
157    output_lines.append(autogenerated_note)
158
159    for input_line in input_lines:
160      if is_in_function_start:
161        if input_line == '':
162          continue
163        if input_line.lstrip().startswith(';'):
164          m = common.CHECK_RE.match(input_line)
165          if not m or m.group(1) not in prefix_set:
166            output_lines.append(input_line)
167            continue
168
169        # Print out the various check lines here.
170        common.add_ir_checks(output_lines, ';', prefix_list, func_dict,
171                             func_name, args.preserve_names, args.function_signature)
172        is_in_function_start = False
173
174      if is_in_function:
175        if common.should_add_line_to_output(input_line, prefix_set):
176          # This input line of the function body will go as-is into the output.
177          # Except make leading whitespace uniform: 2 spaces.
178          input_line = common.SCRUB_LEADING_WHITESPACE_RE.sub(r'  ', input_line)
179          output_lines.append(input_line)
180        else:
181          continue
182        if input_line.strip() == '}':
183          is_in_function = False
184        continue
185
186      # Discard any previous script advertising.
187      if input_line.startswith(ADVERT):
188        continue
189
190      # If it's outside a function, it just gets copied to the output.
191      output_lines.append(input_line)
192
193      m = IR_FUNCTION_RE.match(input_line)
194      if not m:
195        continue
196      func_name = m.group(1)
197      if args.function is not None and func_name != args.function:
198        # When filtering on a specific function, skip all others.
199        continue
200      is_in_function = is_in_function_start = True
201
202    common.debug('Writing %d lines to %s...' % (len(output_lines), test))
203
204    with open(test, 'wb') as f:
205      f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])
206
207
208if __name__ == '__main__':
209  main()
210