1#!/usr/bin/env python3 2# Copyright (c) 2014-2016 The Bitcoin Core developers 3# Distributed under the MIT software license, see the accompanying 4# file COPYING or http://www.opensource.org/licenses/mit-license.php. 5 6""" 7Run Regression Test Suite 8 9This module calls down into individual test cases via subprocess. It will 10forward all unrecognized arguments onto the individual test scripts, other 11than: 12 13 - `-extended`: run the "extended" test suite in addition to the basic one. 14 - `-win`: signal that this is running in a Windows environment, and we 15 should run the tests. 16 - `--coverage`: this generates a basic coverage report for the RPC 17 interface. 18 19For a description of arguments recognized by test scripts, see 20`qa/pull-tester/test_framework/test_framework.py:BitcoinTestFramework.main`. 21 22""" 23 24import os 25import time 26import shutil 27import sys 28import subprocess 29import tempfile 30import re 31 32sys.path.append("qa/pull-tester/") 33from tests_config import * 34 35BOLD = ("","") 36if os.name == 'posix': 37 # primitive formatting on supported 38 # terminal via ANSI escape sequences: 39 BOLD = ('\033[0m', '\033[1m') 40 41RPC_TESTS_DIR = SRCDIR + '/qa/rpc-tests/' 42 43#If imported values are not defined then set to zero (or disabled) 44if 'ENABLE_WALLET' not in vars(): 45 ENABLE_WALLET=0 46if 'ENABLE_BITCOIND' not in vars(): 47 ENABLE_BITCOIND=0 48if 'ENABLE_UTILS' not in vars(): 49 ENABLE_UTILS=0 50if 'ENABLE_ZMQ' not in vars(): 51 ENABLE_ZMQ=0 52 53ENABLE_COVERAGE=0 54 55#Create a set to store arguments and create the passon string 56opts = set() 57passon_args = [] 58PASSON_REGEX = re.compile("^--") 59PARALLEL_REGEX = re.compile('^-parallel=') 60 61print_help = False 62run_parallel = 4 63 64for arg in sys.argv[1:]: 65 if arg == "--help" or arg == "-h" or arg == "-?": 66 print_help = True 67 break 68 if arg == '--coverage': 69 ENABLE_COVERAGE = 1 70 elif PASSON_REGEX.match(arg): 71 passon_args.append(arg) 72 elif PARALLEL_REGEX.match(arg): 73 run_parallel = int(arg.split(sep='=', maxsplit=1)[1]) 74 else: 75 opts.add(arg) 76 77#Set env vars 78if "BITCOIND" not in os.environ: 79 os.environ["BITCOIND"] = BUILDDIR + '/src/bitcoind' + EXEEXT 80if "BITCOINCLI" not in os.environ: 81 os.environ["BITCOINCLI"] = BUILDDIR + '/src/bitcoin-cli' + EXEEXT 82 83if EXEEXT == ".exe" and "-win" not in opts: 84 # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9 85 # https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964 86 print("Win tests currently disabled by default. Use -win option to enable") 87 sys.exit(0) 88 89if not (ENABLE_WALLET == 1 and ENABLE_UTILS == 1 and ENABLE_BITCOIND == 1): 90 print("No rpc tests to run. Wallet, utils, and bitcoind must all be enabled") 91 sys.exit(0) 92 93# python3-zmq may not be installed. Handle this gracefully and with some helpful info 94if ENABLE_ZMQ: 95 try: 96 import zmq 97 except ImportError: 98 print("ERROR: \"import zmq\" failed. Set ENABLE_ZMQ=0 or " 99 "to run zmq tests, see dependency info in /qa/README.md.") 100 # ENABLE_ZMQ=0 101 raise 102 103testScripts = [ 104 # longest test should go first, to favor running tests in parallel 105 'p2p-fullblocktest.py', 106 'walletbackup.py', 107 'bip68-112-113-p2p.py', 108 'wallet.py', 109 'wallet-hd.py', 110 'wallet-dump.py', 111 'listtransactions.py', 112 'receivedby.py', 113 'mempool_resurrect_test.py', 114 'txn_doublespend.py --mineblock', 115 'txn_clone.py', 116 'getchaintips.py', 117 'rawtransactions.py', 118 'rest.py', 119 'mempool_spendcoinbase.py', 120 'mempool_reorg.py', 121 'mempool_limit.py', 122 'httpbasics.py', 123 'multi_rpc.py', 124 'zapwallettxes.py', 125 'proxy_test.py', 126 'merkle_blocks.py', 127 'fundrawtransaction.py', 128 'signrawtransactions.py', 129 'nodehandling.py', 130 'reindex.py', 131 'decodescript.py', 132 'blockchain.py', 133 'disablewallet.py', 134 'sendheaders.py', 135 'keypool.py', 136 'prioritise_transaction.py', 137 'invalidblockrequest.py', 138 'invalidtxrequest.py', 139 'abandonconflict.py', 140 'p2p-versionbits-warning.py', 141 'p2p-segwit.py', 142 'segwit.py', 143 'importprunedfunds.py', 144 'signmessages.py', 145 'p2p-compactblocks.py', 146 'nulldummy.py', 147] 148if ENABLE_ZMQ: 149 testScripts.append('zmq_test.py') 150 151testScriptsExt = [ 152 'bip9-softforks.py', 153 'bip65-cltv.py', 154 'bip65-cltv-p2p.py', 155 'bip68-sequence.py', 156 'bipdersig-p2p.py', 157 'bipdersig.py', 158 'getblocktemplate_longpoll.py', 159 'getblocktemplate_proposals.py', 160 'txn_doublespend.py', 161 'txn_clone.py --mineblock', 162 'forknotify.py', 163 'invalidateblock.py', 164 'rpcbind_test.py', 165 'smartfees.py', 166 'maxblocksinflight.py', 167 'p2p-acceptblock.py', 168 'mempool_packages.py', 169 'maxuploadtarget.py', 170 'replace-by-fee.py', 171 'p2p-feefilter.py', 172 'pruning.py', # leave pruning last as it takes a REALLY long time 173] 174 175 176def runtests(): 177 test_list = [] 178 if '-extended' in opts: 179 test_list = testScripts + testScriptsExt 180 elif len(opts) == 0 or (len(opts) == 1 and "-win" in opts): 181 test_list = testScripts 182 else: 183 for t in testScripts + testScriptsExt: 184 if t in opts or re.sub(".py$", "", t) in opts: 185 test_list.append(t) 186 187 if print_help: 188 # Only print help of the first script and exit 189 subprocess.check_call((RPC_TESTS_DIR + test_list[0]).split() + ['-h']) 190 sys.exit(0) 191 192 coverage = None 193 194 if ENABLE_COVERAGE: 195 coverage = RPCCoverage() 196 print("Initializing coverage directory at %s\n" % coverage.dir) 197 flags = ["--srcdir=%s/src" % BUILDDIR] + passon_args 198 if coverage: 199 flags.append(coverage.flag) 200 201 if len(test_list) > 1 and run_parallel > 1: 202 # Populate cache 203 subprocess.check_output([RPC_TESTS_DIR + 'create_cache.py'] + flags) 204 205 #Run Tests 206 max_len_name = len(max(test_list, key=len)) 207 time_sum = 0 208 time0 = time.time() 209 job_queue = RPCTestHandler(run_parallel, test_list, flags) 210 results = BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "PASSED", "DURATION") + BOLD[0] 211 all_passed = True 212 for _ in range(len(test_list)): 213 (name, stdout, stderr, passed, duration) = job_queue.get_next() 214 all_passed = all_passed and passed 215 time_sum += duration 216 217 print('\n' + BOLD[1] + name + BOLD[0] + ":") 218 print(stdout) 219 print('stderr:\n' if not stderr == '' else '', stderr) 220 results += "%s | %s | %s s\n" % (name.ljust(max_len_name), str(passed).ljust(6), duration) 221 print("Pass: %s%s%s, Duration: %s s\n" % (BOLD[1], passed, BOLD[0], duration)) 222 results += BOLD[1] + "\n%s | %s | %s s (accumulated)" % ("ALL".ljust(max_len_name), str(all_passed).ljust(6), time_sum) + BOLD[0] 223 print(results) 224 print("\nRuntime: %s s" % (int(time.time() - time0))) 225 226 if coverage: 227 coverage.report_rpc_coverage() 228 229 print("Cleaning up coverage data") 230 coverage.cleanup() 231 232 sys.exit(not all_passed) 233 234 235class RPCTestHandler: 236 """ 237 Trigger the testscrips passed in via the list. 238 """ 239 240 def __init__(self, num_tests_parallel, test_list=None, flags=None): 241 assert(num_tests_parallel >= 1) 242 self.num_jobs = num_tests_parallel 243 self.test_list = test_list 244 self.flags = flags 245 self.num_running = 0 246 # In case there is a graveyard of zombie bitcoinds, we can apply a 247 # pseudorandom offset to hopefully jump over them. 248 # (625 is PORT_RANGE/MAX_NODES) 249 self.portseed_offset = int(time.time() * 1000) % 625 250 self.jobs = [] 251 252 def get_next(self): 253 while self.num_running < self.num_jobs and self.test_list: 254 # Add tests 255 self.num_running += 1 256 t = self.test_list.pop(0) 257 port_seed = ["--portseed={}".format(len(self.test_list) + self.portseed_offset)] 258 log_stdout = tempfile.SpooledTemporaryFile(max_size=2**16) 259 log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16) 260 self.jobs.append((t, 261 time.time(), 262 subprocess.Popen((RPC_TESTS_DIR + t).split() + self.flags + port_seed, 263 universal_newlines=True, 264 stdout=log_stdout, 265 stderr=log_stderr), 266 log_stdout, 267 log_stderr)) 268 if not self.jobs: 269 raise IndexError('pop from empty list') 270 while True: 271 # Return first proc that finishes 272 time.sleep(.5) 273 for j in self.jobs: 274 (name, time0, proc, log_out, log_err) = j 275 if proc.poll() is not None: 276 log_out.seek(0), log_err.seek(0) 277 [stdout, stderr] = [l.read().decode('utf-8') for l in (log_out, log_err)] 278 log_out.close(), log_err.close() 279 passed = stderr == "" and proc.returncode == 0 280 self.num_running -= 1 281 self.jobs.remove(j) 282 return name, stdout, stderr, passed, int(time.time() - time0) 283 print('.', end='', flush=True) 284 285 286class RPCCoverage(object): 287 """ 288 Coverage reporting utilities for pull-tester. 289 290 Coverage calculation works by having each test script subprocess write 291 coverage files into a particular directory. These files contain the RPC 292 commands invoked during testing, as well as a complete listing of RPC 293 commands per `bitcoin-cli help` (`rpc_interface.txt`). 294 295 After all tests complete, the commands run are combined and diff'd against 296 the complete list to calculate uncovered RPC commands. 297 298 See also: qa/rpc-tests/test_framework/coverage.py 299 300 """ 301 def __init__(self): 302 self.dir = tempfile.mkdtemp(prefix="coverage") 303 self.flag = '--coveragedir=%s' % self.dir 304 305 def report_rpc_coverage(self): 306 """ 307 Print out RPC commands that were unexercised by tests. 308 309 """ 310 uncovered = self._get_uncovered_rpc_commands() 311 312 if uncovered: 313 print("Uncovered RPC commands:") 314 print("".join((" - %s\n" % i) for i in sorted(uncovered))) 315 else: 316 print("All RPC commands covered.") 317 318 def cleanup(self): 319 return shutil.rmtree(self.dir) 320 321 def _get_uncovered_rpc_commands(self): 322 """ 323 Return a set of currently untested RPC commands. 324 325 """ 326 # This is shared from `qa/rpc-tests/test-framework/coverage.py` 327 REFERENCE_FILENAME = 'rpc_interface.txt' 328 COVERAGE_FILE_PREFIX = 'coverage.' 329 330 coverage_ref_filename = os.path.join(self.dir, REFERENCE_FILENAME) 331 coverage_filenames = set() 332 all_cmds = set() 333 covered_cmds = set() 334 335 if not os.path.isfile(coverage_ref_filename): 336 raise RuntimeError("No coverage reference found") 337 338 with open(coverage_ref_filename, 'r') as f: 339 all_cmds.update([i.strip() for i in f.readlines()]) 340 341 for root, dirs, files in os.walk(self.dir): 342 for filename in files: 343 if filename.startswith(COVERAGE_FILE_PREFIX): 344 coverage_filenames.add(os.path.join(root, filename)) 345 346 for filename in coverage_filenames: 347 with open(filename, 'r') as f: 348 covered_cmds.update([i.strip() for i in f.readlines()]) 349 350 return all_cmds - covered_cmds 351 352 353if __name__ == '__main__': 354 runtests() 355