1#!/usr/bin/env python3 2# Copyright 2014 BitPay Inc. 3# Copyright 2016-2017 The Bitcoin Core developers 4# Distributed under the MIT software license, see the accompanying 5# file COPYING or http://www.opensource.org/licenses/mit-license.php. 6"""Test framework for bitcoin utils. 7 8Runs automatically during `make check`. 9 10Can also be run manually.""" 11 12import argparse 13import binascii 14import configparser 15import difflib 16import json 17import logging 18import os 19import pprint 20import subprocess 21import sys 22 23def main(): 24 config = configparser.ConfigParser() 25 config.optionxform = str 26 config.read_file(open(os.path.join(os.path.dirname(__file__), "../config.ini"), encoding="utf8")) 27 env_conf = dict(config.items('environment')) 28 29 parser = argparse.ArgumentParser(description=__doc__) 30 parser.add_argument('-v', '--verbose', action='store_true') 31 args = parser.parse_args() 32 verbose = args.verbose 33 34 if verbose: 35 level = logging.DEBUG 36 else: 37 level = logging.ERROR 38 formatter = '%(asctime)s - %(levelname)s - %(message)s' 39 # Add the format/level to the logger 40 logging.basicConfig(format=formatter, level=level) 41 42 bctester(os.path.join(env_conf["SRCDIR"], "test", "util", "data"), "bitcoin-util-test.json", env_conf) 43 44def bctester(testDir, input_basename, buildenv): 45 """ Loads and parses the input file, runs all tests and reports results""" 46 input_filename = os.path.join(testDir, input_basename) 47 raw_data = open(input_filename, encoding="utf8").read() 48 input_data = json.loads(raw_data) 49 50 failed_testcases = [] 51 52 for testObj in input_data: 53 try: 54 bctest(testDir, testObj, buildenv) 55 logging.info("PASSED: " + testObj["description"]) 56 except: 57 logging.info("FAILED: " + testObj["description"]) 58 failed_testcases.append(testObj["description"]) 59 60 if failed_testcases: 61 error_message = "FAILED_TESTCASES:\n" 62 error_message += pprint.pformat(failed_testcases, width=400) 63 logging.error(error_message) 64 sys.exit(1) 65 else: 66 sys.exit(0) 67 68def bctest(testDir, testObj, buildenv): 69 """Runs a single test, comparing output and RC to expected output and RC. 70 71 Raises an error if input can't be read, executable fails, or output/RC 72 are not as expected. Error is caught by bctester() and reported. 73 """ 74 # Get the exec names and arguments 75 execprog = os.path.join(buildenv["BUILDDIR"], "src", testObj["exec"] + buildenv["EXEEXT"]) 76 execargs = testObj['args'] 77 execrun = [execprog] + execargs 78 79 # Read the input data (if there is any) 80 stdinCfg = None 81 inputData = None 82 if "input" in testObj: 83 filename = os.path.join(testDir, testObj["input"]) 84 inputData = open(filename, encoding="utf8").read() 85 stdinCfg = subprocess.PIPE 86 87 # Read the expected output data (if there is any) 88 outputFn = None 89 outputData = None 90 outputType = None 91 if "output_cmp" in testObj: 92 outputFn = testObj['output_cmp'] 93 outputType = os.path.splitext(outputFn)[1][1:] # output type from file extension (determines how to compare) 94 try: 95 outputData = open(os.path.join(testDir, outputFn), encoding="utf8").read() 96 except: 97 logging.error("Output file " + outputFn + " can not be opened") 98 raise 99 if not outputData: 100 logging.error("Output data missing for " + outputFn) 101 raise Exception 102 if not outputType: 103 logging.error("Output file %s does not have a file extension" % outputFn) 104 raise Exception 105 106 # Run the test 107 proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) 108 try: 109 outs = proc.communicate(input=inputData) 110 except OSError: 111 logging.error("OSError, Failed to execute " + execprog) 112 raise 113 114 if outputData: 115 data_mismatch, formatting_mismatch = False, False 116 # Parse command output and expected output 117 try: 118 a_parsed = parse_output(outs[0], outputType) 119 except Exception as e: 120 logging.error('Error parsing command output as %s: %s' % (outputType, e)) 121 raise 122 try: 123 b_parsed = parse_output(outputData, outputType) 124 except Exception as e: 125 logging.error('Error parsing expected output %s as %s: %s' % (outputFn, outputType, e)) 126 raise 127 # Compare data 128 if a_parsed != b_parsed: 129 logging.error("Output data mismatch for " + outputFn + " (format " + outputType + ")") 130 data_mismatch = True 131 # Compare formatting 132 if outs[0] != outputData: 133 error_message = "Output formatting mismatch for " + outputFn + ":\n" 134 error_message += "".join(difflib.context_diff(outputData.splitlines(True), 135 outs[0].splitlines(True), 136 fromfile=outputFn, 137 tofile="returned")) 138 logging.error(error_message) 139 formatting_mismatch = True 140 141 assert not data_mismatch and not formatting_mismatch 142 143 # Compare the return code to the expected return code 144 wantRC = 0 145 if "return_code" in testObj: 146 wantRC = testObj['return_code'] 147 if proc.returncode != wantRC: 148 logging.error("Return code mismatch for " + outputFn) 149 raise Exception 150 151 if "error_txt" in testObj: 152 want_error = testObj["error_txt"] 153 # Compare error text 154 # TODO: ideally, we'd compare the strings exactly and also assert 155 # That stderr is empty if no errors are expected. However, bitcoin-tx 156 # emits DISPLAY errors when running as a windows application on 157 # linux through wine. Just assert that the expected error text appears 158 # somewhere in stderr. 159 if want_error not in outs[1]: 160 logging.error("Error mismatch:\n" + "Expected: " + want_error + "\nReceived: " + outs[1].rstrip()) 161 raise Exception 162 163def parse_output(a, fmt): 164 """Parse the output according to specified format. 165 166 Raise an error if the output can't be parsed.""" 167 if fmt == 'json': # json: compare parsed data 168 return json.loads(a) 169 elif fmt == 'hex': # hex: parse and compare binary data 170 return binascii.a2b_hex(a.strip()) 171 else: 172 raise NotImplementedError("Don't know how to compare %s" % fmt) 173 174if __name__ == '__main__': 175 main() 176