1#!/usr/bin/env python3
2# -*- coding: UTF-8 -*-
3
4# Polly/LLVM update_check.py
5# Update lit FileCheck files by replacing the 'CHECK:' lines by the actual output of the 'RUN:' command.
6
7import argparse
8import os
9import subprocess
10import shlex
11import re
12
13
14polly_src_dir = '''@POLLY_SOURCE_DIR@'''
15polly_lib_dir = '''@POLLY_LIB_DIR@'''
16shlibext = '''@LLVM_SHLIBEXT@'''
17llvm_tools_dir = '''@LLVM_TOOLS_DIR@'''
18llvm_polly_link_into_tools = not '''@LLVM_POLLY_LINK_INTO_TOOLS@'''.lower() in {'','0','n','no','off','false','notfound','llvm_polly_link_into_tools-notfound'}
19
20runre = re.compile(r'\s*\;\s*RUN\s*\:(?P<tool>.*)')
21filecheckre = re.compile(r'\s*(?P<tool>.*)\|\s*(?P<filecheck>FileCheck\s[^|]*)')
22emptyline = re.compile(r'\s*(\;\s*)?')
23commentline = re.compile(r'\s*(\;.*)?')
24
25
26def ltrim_emptylines(lines,meta=None):
27    while len(lines) and emptyline.fullmatch(lines[0]):
28        del lines[0]
29        if meta is not None:
30            del meta[0]
31
32
33def rtrim_emptylines(lines):
34    while len(lines) and emptyline.fullmatch(lines[-1]):
35        del lines[-1]
36
37
38def trim_emptylines(lines):
39    ltrim_emptylines(lines)
40    rtrim_emptylines(lines)
41
42
43def complete_exename(path, filename):
44    complpath = os.path.join(path, filename)
45    if os.path.isfile(complpath):
46        return complpath
47    elif os.path.isfile(complpath + '.exe'):
48        return complpath + '.exe'
49    return filename
50
51
52def indention(line):
53    for i,c in enumerate(line):
54        if c != ' ' and c != '\t':
55            return i
56    return None
57
58
59def common_indent(lines):
60    indentions = (indention(line) for line in lines)
61    indentions = (indent for indent in indentions if indent is not None)
62    return min(indentions,default=0)
63
64
65funcre = re.compile(r'^    Function: \S*$')
66regionre = re.compile(r'^    Region: \S*$')
67depthre = re.compile(r'^    Max Loop Depth: .*')
68paramre = re.compile(r'    [0-9a-z-A-Z_]+\: .*')
69
70def classyfier1(lines):
71    i = iter(lines)
72    line = i.__next__()
73    while True:
74        if line.startswith("Printing analysis 'Polly - Calculate dependences' for region: "):
75            yield {'PrintingDependenceInfo'}
76        elif line.startswith("remark: "):
77            yield {'Remark'}
78        elif funcre.fullmatch(line):
79            yield {'Function'}
80        elif regionre.fullmatch(line):
81            yield  { 'Region'}
82        elif depthre.fullmatch(line):
83            yield  {'MaxLoopDepth'}
84        elif line == '    Invariant Accesses: {':
85            while True:
86                yield { 'InvariantAccesses'}
87                if line == '    }':
88                    break
89                line = i.__next__()
90        elif line == '    Context:':
91            yield  {'Context'}
92            line = i.__next__()
93            yield  {'Context'}
94        elif line == '    Assumed Context:':
95            yield  {'AssumedContext'}
96            line = i.__next__()
97            yield  {'AssumedContext'}
98        elif line == '    Invalid Context:':
99            yield  {'InvalidContext'}
100            line = i.__next__()
101            yield  {'InvalidContext'}
102        elif line == '    Boundary Context:':
103            yield  {'BoundaryContext'}
104            line = i.__next__()
105            yield  {'BoundaryContext'}
106            line = i.__next__()
107            while paramre.fullmatch(line):
108                yield  {'Param'}
109                line = i.__next__()
110            continue
111        elif line == '    Arrays {':
112            while True:
113                yield  {'Arrays'}
114                if line == '    }':
115                    break
116                line = i.__next__()
117        elif line == '    Arrays (Bounds as pw_affs) {':
118            while True:
119                yield  {'PwAffArrays'}
120                if line == '    }':
121                    break
122                line = i.__next__()
123        elif line.startswith('    Alias Groups ('):
124            while True:
125                yield  {'AliasGroups'}
126                line = i.__next__()
127                if not line.startswith('        '):
128                    break
129            continue
130        elif line == '    Statements {':
131            while True:
132                yield  {'Statements'}
133                if line == '    }':
134                    break
135                line = i.__next__()
136        elif line == '    RAW dependences:':
137            yield {'RAWDep','BasicDep','Dep','DepInfo'}
138            line = i.__next__()
139            while line.startswith('        '):
140                yield  {'RAWDep','BasicDep','Dep','DepInfo'}
141                line = i.__next__()
142            continue
143        elif line == '    WAR dependences:':
144            yield {'WARDep','BasicDep','Dep','DepInfo'}
145            line = i.__next__()
146            while line.startswith('        '):
147                yield  {'WARDep','BasicDep','Dep','DepInfo'}
148                line = i.__next__()
149            continue
150        elif line == '    WAW dependences:':
151            yield {'WAWDep','BasicDep','Dep','DepInfo'}
152            line = i.__next__()
153            while line.startswith('        '):
154                yield  {'WAWDep','BasicDep','Dep','DepInfo'}
155                line = i.__next__()
156            continue
157        elif line == '    Reduction dependences:':
158            yield {'RedDep','Dep','DepInfo'}
159            line = i.__next__()
160            while line.startswith('        '):
161                yield  {'RedDep','Dep','DepInfo'}
162                line = i.__next__()
163            continue
164        elif line == '    Transitive closure of reduction dependences:':
165            yield {'TransitiveClosureDep','DepInfo'}
166            line = i.__next__()
167            while line.startswith('        '):
168                yield  {'TransitiveClosureDep','DepInfo'}
169                line = i.__next__()
170            continue
171        elif line.startswith("New access function '"):
172            yield {'NewAccessFunction'}
173        elif line == 'Schedule before flattening {':
174            while True:
175                yield  {'ScheduleBeforeFlattening'}
176                if line == '}':
177                    break
178                line = i.__next__()
179        elif line == 'Schedule after flattening {':
180            while True:
181                yield  {'ScheduleAfterFlattening'}
182                if line == '}':
183                    break
184                line = i.__next__()
185        else:
186            yield set()
187        line = i.__next__()
188
189
190def classyfier2(lines):
191    i = iter(lines)
192    line = i.__next__()
193    while True:
194        if funcre.fullmatch(line):
195            while line.startswith('    '):
196                yield  {'FunctionDetail'}
197                line = i.__next__()
198            continue
199        elif line.startswith("Printing analysis 'Polly - Generate an AST from the SCoP (isl)' for region: "):
200            yield {'PrintingIslAst'}
201            line = i.__next__()
202            while not line.startswith('Printing analysis'):
203                yield  {'AstDetail'}
204                line = i.__next__()
205            continue
206        else:
207            yield set()
208        line = i.__next__()
209
210
211replrepl = {'{{':'{{[{][{]}}','}}': '{{[}][}]}}', '[[':'{{\[\[}}',']]': '{{\]\]}}'}
212replre = re.compile('|'.join(re.escape(k) for k in replrepl.keys()))
213
214def main():
215    parser = argparse.ArgumentParser(description="Update CHECK lines")
216    parser.add_argument('testfile',help="File to update (absolute or relative to --testdir)")
217    parser.add_argument('--check-style',choices=['CHECK','CHECK-NEXT'],default='CHECK-NEXT',help="What kind of checks lines to generate")
218    parser.add_argument('--check-position',choices=['end','before-content','autodetect'],default='autodetect',help="Where to add the CHECK lines into the file; 'autodetect' searches for the first 'CHECK' line ind inserts it there")
219    parser.add_argument('--check-include',action='append',default=[], help="What parts of the output lines to check; use syntax 'CHECK=include' to apply to one CHECK-prefix only (by default, everything)")
220    parser.add_argument('--check-label-include',action='append',default=[],help="Use CHECK-LABEL for these includes")
221    parser.add_argument('--check-part-newline',action='store_true',help="Add empty line between different check parts")
222    parser.add_argument('--prefix-only',action='append',default=None,help="Update only these prefixes (default: all)")
223    parser.add_argument('--bindir',help="Location of the opt program")
224    parser.add_argument('--testdir',help="Root dir for unit tests")
225    parser.add_argument('--inplace','-i',action='store_true',help="Replace input file")
226    parser.add_argument('--output','-o',help="Write changed input to this file")
227    known = parser.parse_args()
228
229    if not known.inplace and known.output is None:
230        print("Must specify what to do with output (--output or --inplace)")
231        exit(1)
232    if known.inplace and known.output is not None:
233        print("--inplace and --output are mutually exclusive")
234        exit(1)
235
236    outfile = known.output
237
238    filecheckparser = argparse.ArgumentParser(add_help=False)
239    filecheckparser.add_argument('-check-prefix','--check-prefix',default='CHECK')
240
241    filename = known.testfile
242    for dir in ['.', known.testdir, os.path.join(polly_src_dir,'test'), polly_src_dir]:
243        if not dir:
244            continue
245        testfilename = os.path.join(dir,filename)
246        if os.path.isfile(testfilename):
247            filename = testfilename
248            break
249
250    if known.inplace:
251        outfile = filename
252
253    allchecklines = []
254    checkprefixes = []
255
256    with open(filename, 'r') as file:
257        oldlines = [line.rstrip('\r\n') for line in file.readlines()]
258
259    runlines = []
260    for line in oldlines:
261        m = runre.match(line)
262        if m:
263            runlines.append(m.group('tool'))
264
265    continuation = ''
266    newrunlines = []
267    for line in runlines:
268        if line.endswith('\\'):
269            continuation += line[:-2] + ' '
270        else:
271            newrunlines.append(continuation + line)
272            continuation = ''
273    if continuation:
274        newrunlines.append(continuation)
275
276    for line in newrunlines:
277        m = filecheckre.match(line)
278        if not m:
279            continue
280
281        tool, filecheck = m.group('tool', 'filecheck')
282        filecheck = shlex.split(filecheck)
283        tool = shlex.split(tool)
284        if known.bindir is not None:
285            tool[0] = complete_exename(known.bindir, tool[0])
286        if os.path.isdir(llvm_tools_dir):
287            tool[0] = complete_exename(llvm_tools_dir, tool[0])
288        check_prefix = filecheckparser.parse_known_args(filecheck)[0].check_prefix
289        if known.prefix_only is not None and not check_prefix in known.prefix_only:
290            continue
291        if check_prefix in checkprefixes:
292            continue
293        checkprefixes.append(check_prefix)
294
295        newtool = []
296        optstderr = None
297        for toolarg in tool:
298            toolarg = toolarg.replace('%s', filename)
299            toolarg = toolarg.replace('%S', os.path.dirname(filename))
300            if toolarg == '%loadPolly':
301                if not llvm_polly_link_into_tools:
302                    newtool += ['-load',os.path.join(polly_lib_dir,'LLVMPolly' + shlibext)]
303                newtool.append('-polly-process-unprofitable')
304                newtool.append('-polly-remarks-minimal')
305            elif toolarg == '2>&1':
306                optstderr = subprocess.STDOUT
307            else:
308                newtool.append(toolarg)
309        tool = newtool
310
311        inpfile = None
312        i = 1
313        while i <  len(tool):
314            if tool[i] == '<':
315                inpfile = tool[i + 1]
316                del tool[i:i + 2]
317                continue
318            i += 1
319        if inpfile:
320            with open(inpfile) as inp:
321                retlines = subprocess.check_output(tool,universal_newlines=True,stdin=inp,stderr=optstderr)
322        else:
323            retlines = subprocess.check_output(tool,universal_newlines=True,stderr=optstderr)
324        retlines = [line.replace('\t', '    ') for line in retlines.splitlines()]
325        check_include = []
326        for checkme in known.check_include + known.check_label_include:
327            parts = checkme.split('=')
328            if len(parts) == 2:
329                if parts[0] == check_prefix:
330                    check_include.append(parts[1])
331            else:
332                check_include.append(checkme)
333
334        if check_include:
335            filtered_retlines = []
336            classified_retlines = []
337            lastmatch = None
338            for line,kind in ((line,class1.union(class2)) for line,class1,class2 in zip(retlines,classyfier1(retlines), classyfier2(retlines))):
339                match = kind.intersection(check_include)
340                if match:
341                    if lastmatch != match:
342                        filtered_retlines.append('')
343                        classified_retlines.append({'Separator'})
344                    filtered_retlines.append(line)
345                    classified_retlines.append(kind)
346                lastmatch = match
347
348            retlines = filtered_retlines
349        else:
350            classified_retlines = (set() for line in retlines)
351
352        rtrim_emptylines(retlines)
353        ltrim_emptylines(retlines,classified_retlines)
354        retlines = [replre.sub(lambda m: replrepl[m.group(0)], line) for line in retlines]
355        indent = common_indent(retlines)
356        retlines = [line[indent:] for line in retlines]
357        checklines = []
358        previous_was_empty = True
359        for line,kind in zip(retlines,classified_retlines):
360            if line:
361                if known.check_style == 'CHECK' and known.check_label_include:
362                    if not kind.isdisjoint(known.check_label_include):
363                        checklines.append('; ' + check_prefix + '-LABEL: ' + line)
364                    else:
365                        checklines.append('; ' + check_prefix + ':       ' + line)
366                elif known.check_style == 'CHECK':
367                    checklines.append('; ' + check_prefix + ': ' + line)
368                elif known.check_label_include and known.check_label_include:
369                    if not kind.isdisjoint(known.check_label_include):
370                        checklines.append('; ' + check_prefix + '-LABEL: ' + line)
371                    elif previous_was_empty:
372                        checklines.append('; ' + check_prefix + ':       ' + line)
373                    else:
374                        checklines.append('; ' + check_prefix + '-NEXT:  ' + line)
375                else:
376                    if previous_was_empty:
377                        checklines.append('; ' + check_prefix + ':      ' + line)
378                    else:
379                        checklines.append('; ' + check_prefix + '-NEXT: ' + line)
380                previous_was_empty = False
381            else:
382                if not 'Separator' in kind or known.check_part_newline:
383                    checklines.append(';')
384                previous_was_empty = True
385        allchecklines.append(checklines)
386
387    if not checkprefixes:
388        return
389
390    checkre = re.compile(r'^\s*\;\s*(' + '|'.join([re.escape(s) for s in checkprefixes]) + ')(\-NEXT|\-DAG|\-NOT|\-LABEL|\-SAME)?\s*\:')
391    firstcheckline = None
392    firstnoncommentline = None
393    headerlines = []
394    newlines = []
395    uptonowlines = []
396    emptylines = []
397    lastwascheck = False
398    for line in oldlines:
399        if checkre.match(line):
400            if firstcheckline is None:
401                firstcheckline = len(newlines) + len(emptylines)
402            if not lastwascheck:
403                uptonowlines += emptylines
404            emptylines = []
405            lastwascheck = True
406        elif emptyline.fullmatch(line):
407            emptylines.append(line)
408        else:
409            newlines += uptonowlines
410            newlines += emptylines
411            newlines.append(line)
412            emptylines = []
413            uptonowlines = []
414            lastwascheck = False
415
416    for i,line in enumerate(newlines):
417        if not commentline.fullmatch(line):
418            firstnoncommentline = i
419            break
420
421    with open(outfile,'w',newline='') as file:
422        def writelines(lines):
423            for line in lines:
424                file.write(line)
425                file.write('\n')
426
427        if firstcheckline is not None and known.check_position == 'autodetect':
428            writelines(newlines[:firstcheckline])
429            writelines(uptonowlines)
430            for i,checklines in enumerate(allchecklines):
431                if i != 0:
432                    file.write('\n')
433                writelines(checklines)
434            writelines(newlines[firstcheckline:])
435            writelines(emptylines)
436        elif firstnoncommentline is not None and known.check_position == 'before-content':
437            headerlines = newlines[:firstnoncommentline]
438            rtrim_emptylines(headerlines)
439            contentlines = newlines[firstnoncommentline:]
440            ltrim_emptylines(contentlines)
441
442            writelines(headerlines)
443            for checklines in allchecklines:
444                file.write('\n')
445                writelines(checklines)
446            file.write('\n')
447            writelines(contentlines)
448            writelines(uptonowlines)
449            writelines(emptylines)
450        else:
451            writelines(newlines)
452            rtrim_emptylines(newlines)
453            for checklines in allchecklines:
454                file.write('\n\n')
455                writelines(checklines)
456
457
458if __name__ == '__main__':
459    main()
460