1#!/usr/bin/env python 2# 3# Simple benchmarking framework 4# 5# Copyright (c) 2019 Virtuozzo International GmbH. 6# 7# This program is free software; you can redistribute it and/or modify 8# it under the terms of the GNU General Public License as published by 9# the Free Software Foundation; either version 2 of the License, or 10# (at your option) any later version. 11# 12# This program is distributed in the hope that it will be useful, 13# but WITHOUT ANY WARRANTY; without even the implied warranty of 14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15# GNU General Public License for more details. 16# 17# You should have received a copy of the GNU General Public License 18# along with this program. If not, see <http://www.gnu.org/licenses/>. 19# 20 21import statistics 22 23 24def bench_one(test_func, test_env, test_case, count=5, initial_run=True): 25 """Benchmark one test-case 26 27 test_func -- benchmarking function with prototype 28 test_func(env, case), which takes test_env and test_case 29 arguments and on success returns dict with 'seconds' or 30 'iops' (or both) fields, specifying the benchmark result. 31 If both 'iops' and 'seconds' provided, the 'iops' is 32 considered the main, and 'seconds' is just an additional 33 info. On failure test_func should return {'error': str}. 34 Returned dict may contain any other additional fields. 35 test_env -- test environment - opaque first argument for test_func 36 test_case -- test case - opaque second argument for test_func 37 count -- how many times to call test_func, to calculate average 38 initial_run -- do initial run of test_func, which don't get into result 39 40 Returns dict with the following fields: 41 'runs': list of test_func results 42 'dimension': dimension of results, may be 'seconds' or 'iops' 43 'average': average value (iops or seconds) per run (exists only if at 44 least one run succeeded) 45 'stdev': standard deviation of results 46 (exists only if at least one run succeeded) 47 'n-failed': number of failed runs (exists only if at least one run 48 failed) 49 """ 50 if initial_run: 51 print(' #initial run:') 52 print(' ', test_func(test_env, test_case)) 53 54 runs = [] 55 for i in range(count): 56 print(' #run {}'.format(i+1)) 57 res = test_func(test_env, test_case) 58 print(' ', res) 59 runs.append(res) 60 61 result = {'runs': runs} 62 63 succeeded = [r for r in runs if ('seconds' in r or 'iops' in r)] 64 if succeeded: 65 if 'iops' in succeeded[0]: 66 assert all('iops' in r for r in succeeded) 67 dim = 'iops' 68 else: 69 assert all('seconds' in r for r in succeeded) 70 assert all('iops' not in r for r in succeeded) 71 dim = 'seconds' 72 result['dimension'] = dim 73 result['average'] = statistics.mean(r[dim] for r in succeeded) 74 result['stdev'] = statistics.stdev(r[dim] for r in succeeded) 75 76 if len(succeeded) < count: 77 result['n-failed'] = count - len(succeeded) 78 79 return result 80 81 82def bench(test_func, test_envs, test_cases, *args, **vargs): 83 """Fill benchmark table 84 85 test_func -- benchmarking function, see bench_one for description 86 test_envs -- list of test environments, see bench_one 87 test_cases -- list of test cases, see bench_one 88 args, vargs -- additional arguments for bench_one 89 90 Returns dict with the following fields: 91 'envs': test_envs 92 'cases': test_cases 93 'tab': filled 2D array, where cell [i][j] is bench_one result for 94 test_cases[i] for test_envs[j] (i.e., rows are test cases and 95 columns are test environments) 96 """ 97 tab = {} 98 results = { 99 'envs': test_envs, 100 'cases': test_cases, 101 'tab': tab 102 } 103 n = 1 104 n_tests = len(test_envs) * len(test_cases) 105 for env in test_envs: 106 for case in test_cases: 107 print('Testing {}/{}: {} :: {}'.format(n, n_tests, 108 env['id'], case['id'])) 109 if case['id'] not in tab: 110 tab[case['id']] = {} 111 tab[case['id']][env['id']] = bench_one(test_func, env, case, 112 *args, **vargs) 113 n += 1 114 115 print('Done') 116 return results 117