1#!/usr/bin/env python 2# 3# Simple benchmarking framework 4# 5# Copyright (c) 2019 Virtuozzo International GmbH. 6# 7# This program is free software; you can redistribute it and/or modify 8# it under the terms of the GNU General Public License as published by 9# the Free Software Foundation; either version 2 of the License, or 10# (at your option) any later version. 11# 12# This program is distributed in the hope that it will be useful, 13# but WITHOUT ANY WARRANTY; without even the implied warranty of 14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15# GNU General Public License for more details. 16# 17# You should have received a copy of the GNU General Public License 18# along with this program. If not, see <http://www.gnu.org/licenses/>. 19# 20 21 22def bench_one(test_func, test_env, test_case, count=5, initial_run=True): 23 """Benchmark one test-case 24 25 test_func -- benchmarking function with prototype 26 test_func(env, case), which takes test_env and test_case 27 arguments and returns {'seconds': int} (which is benchmark 28 result) on success and {'error': str} on error. Returned 29 dict may contain any other additional fields. 30 test_env -- test environment - opaque first argument for test_func 31 test_case -- test case - opaque second argument for test_func 32 count -- how many times to call test_func, to calculate average 33 initial_run -- do initial run of test_func, which don't get into result 34 35 Returns dict with the following fields: 36 'runs': list of test_func results 37 'average': average seconds per run (exists only if at least one run 38 succeeded) 39 'delta': maximum delta between test_func result and the average 40 (exists only if at least one run succeeded) 41 'n-failed': number of failed runs (exists only if at least one run 42 failed) 43 """ 44 if initial_run: 45 print(' #initial run:') 46 print(' ', test_func(test_env, test_case)) 47 48 runs = [] 49 for i in range(count): 50 print(' #run {}'.format(i+1)) 51 res = test_func(test_env, test_case) 52 print(' ', res) 53 runs.append(res) 54 55 result = {'runs': runs} 56 57 successed = [r for r in runs if ('seconds' in r)] 58 if successed: 59 avg = sum(r['seconds'] for r in successed) / len(successed) 60 result['average'] = avg 61 result['delta'] = max(abs(r['seconds'] - avg) for r in successed) 62 63 if len(successed) < count: 64 result['n-failed'] = count - len(successed) 65 66 return result 67 68 69def ascii_one(result): 70 """Return ASCII representation of bench_one() returned dict.""" 71 if 'average' in result: 72 s = '{:.2f} +- {:.2f}'.format(result['average'], result['delta']) 73 if 'n-failed' in result: 74 s += '\n({} failed)'.format(result['n-failed']) 75 return s 76 else: 77 return 'FAILED' 78 79 80def bench(test_func, test_envs, test_cases, *args, **vargs): 81 """Fill benchmark table 82 83 test_func -- benchmarking function, see bench_one for description 84 test_envs -- list of test environments, see bench_one 85 test_cases -- list of test cases, see bench_one 86 args, vargs -- additional arguments for bench_one 87 88 Returns dict with the following fields: 89 'envs': test_envs 90 'cases': test_cases 91 'tab': filled 2D array, where cell [i][j] is bench_one result for 92 test_cases[i] for test_envs[j] (i.e., rows are test cases and 93 columns are test environments) 94 """ 95 tab = {} 96 results = { 97 'envs': test_envs, 98 'cases': test_cases, 99 'tab': tab 100 } 101 n = 1 102 n_tests = len(test_envs) * len(test_cases) 103 for env in test_envs: 104 for case in test_cases: 105 print('Testing {}/{}: {} :: {}'.format(n, n_tests, 106 env['id'], case['id'])) 107 if case['id'] not in tab: 108 tab[case['id']] = {} 109 tab[case['id']][env['id']] = bench_one(test_func, env, case, 110 *args, **vargs) 111 n += 1 112 113 print('Done') 114 return results 115 116 117def ascii(results): 118 """Return ASCII representation of bench() returned dict.""" 119 from tabulate import tabulate 120 121 tab = [[""] + [c['id'] for c in results['envs']]] 122 for case in results['cases']: 123 row = [case['id']] 124 for env in results['envs']: 125 row.append(ascii_one(results['tab'][case['id']][env['id']])) 126 tab.append(row) 127 128 return tabulate(tab) 129