1#!/usr/bin/env python
2
3from  __future__ import print_function
4
5import os
6import sys
7import subprocess
8import re
9import difflib
10
11import scriptCommon
12from scriptCommon import catchPath
13
14rootPath = os.path.join(catchPath, 'projects/SelfTest/Baselines')
15
16
17filelocParser = re.compile(r'''
18    .*/
19    (.+\.[ch]pp)  # filename
20    (?::|\()      # : is starting separator between filename and line number on Linux, ( on Windows
21    ([0-9]*)      # line number
22    \)?           # Windows also has an ending separator, )
23''', re.VERBOSE)
24lineNumberParser = re.compile(r' line="[0-9]*"')
25hexParser = re.compile(r'\b(0[xX][0-9a-fA-F]+)\b')
26durationsParser = re.compile(r' time="[0-9]*\.[0-9]*"')
27timestampsParser = re.compile(r' timestamp="\d{4}-\d{2}-\d{2}T\d{2}\:\d{2}\:\d{2}Z"')
28versionParser = re.compile(r'Catch v[0-9]+\.[0-9]+\.[0-9]+(-develop\.[0-9]+)?')
29nullParser = re.compile(r'\b(__null|nullptr)\b')
30exeNameParser = re.compile(r'''
31    \b
32    (CatchSelfTest|SelfTest)  # Expected executable name
33    (?:.exe)?                 # Executable name contains .exe on Windows.
34    \b
35''', re.VERBOSE)
36# This is a hack until something more reasonable is figured out
37specialCaseParser = re.compile(r'file\((\d+)\)')
38
39# errno macro expands into various names depending on platform, so we need to fix them up as well
40errnoParser = re.compile(r'''
41    \(\*__errno_location\ \(\)\)
42    |
43    \(\*__error\(\)\)
44''', re.VERBOSE)
45
46if len(sys.argv) == 2:
47    cmdPath = sys.argv[1]
48else:
49    cmdPath = os.path.join(catchPath, scriptCommon.getBuildExecutable())
50
51overallResult = 0
52
53def diffFiles(fileA, fileB):
54    with open(fileA, 'r') as file:
55        aLines = [line.rstrip() for line in file.readlines()]
56    with open(fileB, 'r') as file:
57        bLines = [line.rstrip() for line in file.readlines()]
58
59    shortenedFilenameA = fileA.rsplit(os.sep, 1)[-1]
60    shortenedFilenameB = fileB.rsplit(os.sep, 1)[-1]
61
62    diff = difflib.unified_diff(aLines, bLines, fromfile=shortenedFilenameA, tofile=shortenedFilenameB, n=0)
63    return [line for line in diff if line[0] in ('+', '-')]
64
65
66def filterLine(line):
67    if catchPath in line:
68        # make paths relative to Catch root
69        line = line.replace(catchPath + os.sep, '')
70        # go from \ in windows paths to /
71        line = line.replace('\\', '/')
72
73
74    # strip source line numbers
75    m = filelocParser.match(line)
76    if m:
77        # note that this also strips directories, leaving only the filename
78        filename, lnum = m.groups()
79        lnum = ":<line number>" if lnum else ""
80        line = filename + lnum + line[m.end():]
81    else:
82        line = lineNumberParser.sub(" ", line)
83
84    # strip Catch version number
85    line = versionParser.sub("<version>", line)
86
87    # replace *null* with 0
88    line = nullParser.sub("0", line)
89
90    # strip executable name
91    line = exeNameParser.sub("<exe-name>", line)
92
93    # strip hexadecimal numbers (presumably pointer values)
94    line = hexParser.sub("0x<hex digits>", line)
95
96    # strip durations and timestamps
97    line = durationsParser.sub(' time="{duration}"', line)
98    line = timestampsParser.sub(' timestamp="{iso8601-timestamp}"', line)
99    line = specialCaseParser.sub('file:\g<1>', line)
100    line = errnoParser.sub('errno', line)
101    return line
102
103
104def approve(baseName, args):
105    global overallResult
106    args[0:0] = [cmdPath]
107    if not os.path.exists(cmdPath):
108        raise Exception("Executable doesn't exist at " + cmdPath)
109    baselinesPath = os.path.join(rootPath, '{0}.approved.txt'.format(baseName))
110    rawResultsPath = os.path.join(rootPath, '_{0}.tmp'.format(baseName))
111    filteredResultsPath = os.path.join(rootPath, '{0}.unapproved.txt'.format(baseName))
112
113    f = open(rawResultsPath, 'w')
114    subprocess.call(args, stdout=f, stderr=f)
115    f.close()
116
117    rawFile = open(rawResultsPath, 'r')
118    filteredFile = open(filteredResultsPath, 'w')
119    for line in rawFile:
120        filteredFile.write(filterLine(line).rstrip() + "\n")
121    filteredFile.close()
122    rawFile.close()
123
124    os.remove(rawResultsPath)
125    print()
126    print(baseName + ":")
127    if os.path.exists(baselinesPath):
128        diffResult = diffFiles(baselinesPath, filteredResultsPath)
129        if diffResult:
130            print('\n'.join(diffResult))
131            print("  \n****************************\n  \033[91mResults differed")
132            if len(diffResult) > overallResult:
133                overallResult = len(diffResult)
134        else:
135            os.remove(filteredResultsPath)
136            print("  \033[92mResults matched")
137        print("\033[0m")
138    else:
139        print("  first approval")
140        if overallResult == 0:
141            overallResult = 1
142
143
144print("Running approvals against executable:")
145print("  " + cmdPath)
146
147# Standard console reporter
148approve("console.std", ["~[c++11]~[!nonportable]", "--order", "lex"])
149# console reporter, include passes, warn about No Assertions
150approve("console.sw", ["~[c++11]~[!nonportable]", "-s", "-w", "NoAssertions", "--order", "lex"])
151# console reporter, include passes, warn about No Assertions, limit failures to first 4
152approve("console.swa4", ["~[c++11]~[!nonportable]", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"])
153# junit reporter, include passes, warn about No Assertions
154approve("junit.sw", ["~[c++11]~[!nonportable]", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"])
155# xml reporter, include passes, warn about No Assertions
156approve("xml.sw", ["~[c++11]~[!nonportable]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"])
157
158if overallResult != 0:
159    print("If these differences are expected, run approve.py to approve new baselines.")
160exit(overallResult)
161