1#!/usr/bin/env python3
2# Copyright 2014 BitPay Inc.
3# Copyright 2016-2017 The Bitcoin Core developers
4# Distributed under the MIT software license, see the accompanying
5# file COPYING or http://www.opensource.org/licenses/mit-license.php.
6"""Test framework for bitcoin utils.
7
8Runs automatically during `make check`.
9
10Can also be run manually."""
11
12import argparse
13import binascii
14import configparser
15import difflib
16import json
17import logging
18import os
19import pprint
20import subprocess
21import sys
22
23def main():
24    config = configparser.ConfigParser()
25    config.optionxform = str
26    config.read_file(open(os.path.join(os.path.dirname(__file__), "../config.ini"), encoding="utf8"))
27    env_conf = dict(config.items('environment'))
28
29    parser = argparse.ArgumentParser(description=__doc__)
30    parser.add_argument('-v', '--verbose', action='store_true')
31    args = parser.parse_args()
32    verbose = args.verbose
33
34    if verbose:
35        level = logging.DEBUG
36    else:
37        level = logging.ERROR
38    formatter = '%(asctime)s - %(levelname)s - %(message)s'
39    # Add the format/level to the logger
40    logging.basicConfig(format=formatter, level=level)
41
42    bctester(os.path.join(env_conf["SRCDIR"], "test", "util", "data"), "bitcoin-util-test.json", env_conf)
43
44def bctester(testDir, input_basename, buildenv):
45    """ Loads and parses the input file, runs all tests and reports results"""
46    input_filename = os.path.join(testDir, input_basename)
47    raw_data = open(input_filename, encoding="utf8").read()
48    input_data = json.loads(raw_data)
49
50    failed_testcases = []
51
52    for testObj in input_data:
53        try:
54            bctest(testDir, testObj, buildenv)
55            logging.info("PASSED: " + testObj["description"])
56        except:
57            logging.info("FAILED: " + testObj["description"])
58            failed_testcases.append(testObj["description"])
59
60    if failed_testcases:
61        error_message = "FAILED_TESTCASES:\n"
62        error_message += pprint.pformat(failed_testcases, width=400)
63        logging.error(error_message)
64        sys.exit(1)
65    else:
66        sys.exit(0)
67
68def bctest(testDir, testObj, buildenv):
69    """Runs a single test, comparing output and RC to expected output and RC.
70
71    Raises an error if input can't be read, executable fails, or output/RC
72    are not as expected. Error is caught by bctester() and reported.
73    """
74    # Get the exec names and arguments
75    execprog = os.path.join(buildenv["BUILDDIR"], "src", testObj["exec"] + buildenv["EXEEXT"])
76    execargs = testObj['args']
77    execrun = [execprog] + execargs
78
79    # Read the input data (if there is any)
80    stdinCfg = None
81    inputData = None
82    if "input" in testObj:
83        filename = os.path.join(testDir, testObj["input"])
84        inputData = open(filename, encoding="utf8").read()
85        stdinCfg = subprocess.PIPE
86
87    # Read the expected output data (if there is any)
88    outputFn = None
89    outputData = None
90    outputType = None
91    if "output_cmp" in testObj:
92        outputFn = testObj['output_cmp']
93        outputType = os.path.splitext(outputFn)[1][1:]  # output type from file extension (determines how to compare)
94        try:
95            outputData = open(os.path.join(testDir, outputFn), encoding="utf8").read()
96        except:
97            logging.error("Output file " + outputFn + " can not be opened")
98            raise
99        if not outputData:
100            logging.error("Output data missing for " + outputFn)
101            raise Exception
102        if not outputType:
103            logging.error("Output file %s does not have a file extension" % outputFn)
104            raise Exception
105
106    # Run the test
107    proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
108    try:
109        outs = proc.communicate(input=inputData)
110    except OSError:
111        logging.error("OSError, Failed to execute " + execprog)
112        raise
113
114    if outputData:
115        data_mismatch, formatting_mismatch = False, False
116        # Parse command output and expected output
117        try:
118            a_parsed = parse_output(outs[0], outputType)
119        except Exception as e:
120            logging.error('Error parsing command output as %s: %s' % (outputType, e))
121            raise
122        try:
123            b_parsed = parse_output(outputData, outputType)
124        except Exception as e:
125            logging.error('Error parsing expected output %s as %s: %s' % (outputFn, outputType, e))
126            raise
127        # Compare data
128        if a_parsed != b_parsed:
129            logging.error("Output data mismatch for " + outputFn + " (format " + outputType + ")")
130            data_mismatch = True
131        # Compare formatting
132        if outs[0] != outputData:
133            error_message = "Output formatting mismatch for " + outputFn + ":\n"
134            error_message += "".join(difflib.context_diff(outputData.splitlines(True),
135                                                          outs[0].splitlines(True),
136                                                          fromfile=outputFn,
137                                                          tofile="returned"))
138            logging.error(error_message)
139            formatting_mismatch = True
140
141        assert not data_mismatch and not formatting_mismatch
142
143    # Compare the return code to the expected return code
144    wantRC = 0
145    if "return_code" in testObj:
146        wantRC = testObj['return_code']
147    if proc.returncode != wantRC:
148        logging.error("Return code mismatch for " + outputFn)
149        raise Exception
150
151    if "error_txt" in testObj:
152        want_error = testObj["error_txt"]
153        # Compare error text
154        # TODO: ideally, we'd compare the strings exactly and also assert
155        # That stderr is empty if no errors are expected. However, bitcoin-tx
156        # emits DISPLAY errors when running as a windows application on
157        # linux through wine. Just assert that the expected error text appears
158        # somewhere in stderr.
159        if want_error not in outs[1]:
160            logging.error("Error mismatch:\n" + "Expected: " + want_error + "\nReceived: " + outs[1].rstrip())
161            raise Exception
162
163def parse_output(a, fmt):
164    """Parse the output according to specified format.
165
166    Raise an error if the output can't be parsed."""
167    if fmt == 'json':  # json: compare parsed data
168        return json.loads(a)
169    elif fmt == 'hex':  # hex: parse and compare binary data
170        return binascii.a2b_hex(a.strip())
171    else:
172        raise NotImplementedError("Don't know how to compare %s" % fmt)
173
174if __name__ == '__main__':
175    main()
176