1*181f60c8SVladimir Sementsov-Ogievskiy#!/usr/bin/env python3 2*181f60c8SVladimir Sementsov-Ogievskiy# 38e979febSVladimir Sementsov-Ogievskiy# Simple benchmarking framework 48e979febSVladimir Sementsov-Ogievskiy# 58e979febSVladimir Sementsov-Ogievskiy# Copyright (c) 2019 Virtuozzo International GmbH. 68e979febSVladimir Sementsov-Ogievskiy# 78e979febSVladimir Sementsov-Ogievskiy# This program is free software; you can redistribute it and/or modify 88e979febSVladimir Sementsov-Ogievskiy# it under the terms of the GNU General Public License as published by 98e979febSVladimir Sementsov-Ogievskiy# the Free Software Foundation; either version 2 of the License, or 108e979febSVladimir Sementsov-Ogievskiy# (at your option) any later version. 118e979febSVladimir Sementsov-Ogievskiy# 128e979febSVladimir Sementsov-Ogievskiy# This program is distributed in the hope that it will be useful, 138e979febSVladimir Sementsov-Ogievskiy# but WITHOUT ANY WARRANTY; without even the implied warranty of 148e979febSVladimir Sementsov-Ogievskiy# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 158e979febSVladimir Sementsov-Ogievskiy# GNU General Public License for more details. 168e979febSVladimir Sementsov-Ogievskiy# 178e979febSVladimir Sementsov-Ogievskiy# You should have received a copy of the GNU General Public License 188e979febSVladimir Sementsov-Ogievskiy# along with this program. If not, see <http://www.gnu.org/licenses/>. 198e979febSVladimir Sementsov-Ogievskiy# 208e979febSVladimir Sementsov-Ogievskiy 2196be1aeeSVladimir Sementsov-Ogievskiyimport math 22aa362403SVladimir Sementsov-Ogievskiyimport tabulate 23aa362403SVladimir Sementsov-Ogievskiy 24aa362403SVladimir Sementsov-Ogievskiy# We want leading whitespace for difference row cells (see below) 25aa362403SVladimir Sementsov-Ogievskiytabulate.PRESERVE_WHITESPACE = True 2696be1aeeSVladimir Sementsov-Ogievskiy 2796be1aeeSVladimir Sementsov-Ogievskiy 2896be1aeeSVladimir Sementsov-Ogievskiydef format_value(x, stdev): 2996be1aeeSVladimir Sementsov-Ogievskiy stdev_pr = stdev / x * 100 3096be1aeeSVladimir Sementsov-Ogievskiy if stdev_pr < 1.5: 3196be1aeeSVladimir Sementsov-Ogievskiy # don't care too much 3296be1aeeSVladimir Sementsov-Ogievskiy return f'{x:.2g}' 3396be1aeeSVladimir Sementsov-Ogievskiy else: 3496be1aeeSVladimir Sementsov-Ogievskiy return f'{x:.2g} ± {math.ceil(stdev_pr)}%' 3596be1aeeSVladimir Sementsov-Ogievskiy 368e979febSVladimir Sementsov-Ogievskiy 378e979febSVladimir Sementsov-Ogievskiydef result_to_text(result): 388e979febSVladimir Sementsov-Ogievskiy """Return text representation of bench_one() returned dict.""" 398e979febSVladimir Sementsov-Ogievskiy if 'average' in result: 4096be1aeeSVladimir Sementsov-Ogievskiy s = format_value(result['average'], result['stdev']) 418e979febSVladimir Sementsov-Ogievskiy if 'n-failed' in result: 428e979febSVladimir Sementsov-Ogievskiy s += '\n({} failed)'.format(result['n-failed']) 438e979febSVladimir Sementsov-Ogievskiy return s 448e979febSVladimir Sementsov-Ogievskiy else: 458e979febSVladimir Sementsov-Ogievskiy return 'FAILED' 468e979febSVladimir Sementsov-Ogievskiy 478e979febSVladimir Sementsov-Ogievskiy 48aa362403SVladimir Sementsov-Ogievskiydef results_dimension(results): 498e979febSVladimir Sementsov-Ogievskiy dim = None 508e979febSVladimir Sementsov-Ogievskiy for case in results['cases']: 518e979febSVladimir Sementsov-Ogievskiy for env in results['envs']: 528e979febSVladimir Sementsov-Ogievskiy res = results['tab'][case['id']][env['id']] 538e979febSVladimir Sementsov-Ogievskiy if dim is None: 548e979febSVladimir Sementsov-Ogievskiy dim = res['dimension'] 558e979febSVladimir Sementsov-Ogievskiy else: 568e979febSVladimir Sementsov-Ogievskiy assert dim == res['dimension'] 57aa362403SVladimir Sementsov-Ogievskiy 58aa362403SVladimir Sementsov-Ogievskiy assert dim in ('iops', 'seconds') 59aa362403SVladimir Sementsov-Ogievskiy 60aa362403SVladimir Sementsov-Ogievskiy return dim 61aa362403SVladimir Sementsov-Ogievskiy 62aa362403SVladimir Sementsov-Ogievskiy 63aa362403SVladimir Sementsov-Ogievskiydef results_to_text(results): 64aa362403SVladimir Sementsov-Ogievskiy """Return text representation of bench() returned dict.""" 65aa362403SVladimir Sementsov-Ogievskiy n_columns = len(results['envs']) 66aa362403SVladimir Sementsov-Ogievskiy named_columns = n_columns > 2 67aa362403SVladimir Sementsov-Ogievskiy dim = results_dimension(results) 68aa362403SVladimir Sementsov-Ogievskiy tab = [] 69aa362403SVladimir Sementsov-Ogievskiy 70aa362403SVladimir Sementsov-Ogievskiy if named_columns: 71aa362403SVladimir Sementsov-Ogievskiy # Environment columns are named A, B, ... 72aa362403SVladimir Sementsov-Ogievskiy tab.append([''] + [chr(ord('A') + i) for i in range(n_columns)]) 73aa362403SVladimir Sementsov-Ogievskiy 74aa362403SVladimir Sementsov-Ogievskiy tab.append([''] + [c['id'] for c in results['envs']]) 75aa362403SVladimir Sementsov-Ogievskiy 76aa362403SVladimir Sementsov-Ogievskiy for case in results['cases']: 77aa362403SVladimir Sementsov-Ogievskiy row = [case['id']] 78aa362403SVladimir Sementsov-Ogievskiy case_results = results['tab'][case['id']] 79aa362403SVladimir Sementsov-Ogievskiy for env in results['envs']: 80aa362403SVladimir Sementsov-Ogievskiy res = case_results[env['id']] 818e979febSVladimir Sementsov-Ogievskiy row.append(result_to_text(res)) 828e979febSVladimir Sementsov-Ogievskiy tab.append(row) 838e979febSVladimir Sementsov-Ogievskiy 84aa362403SVladimir Sementsov-Ogievskiy # Add row of difference between columns. For each column starting from 85aa362403SVladimir Sementsov-Ogievskiy # B we calculate difference with all previous columns. 86aa362403SVladimir Sementsov-Ogievskiy row = ['', ''] # case name and first column 87aa362403SVladimir Sementsov-Ogievskiy for i in range(1, n_columns): 88aa362403SVladimir Sementsov-Ogievskiy cell = '' 89aa362403SVladimir Sementsov-Ogievskiy env = results['envs'][i] 90aa362403SVladimir Sementsov-Ogievskiy res = case_results[env['id']] 91aa362403SVladimir Sementsov-Ogievskiy 92aa362403SVladimir Sementsov-Ogievskiy if 'average' not in res: 93aa362403SVladimir Sementsov-Ogievskiy # Failed result 94aa362403SVladimir Sementsov-Ogievskiy row.append(cell) 95aa362403SVladimir Sementsov-Ogievskiy continue 96aa362403SVladimir Sementsov-Ogievskiy 97aa362403SVladimir Sementsov-Ogievskiy for j in range(0, i): 98aa362403SVladimir Sementsov-Ogievskiy env_j = results['envs'][j] 99aa362403SVladimir Sementsov-Ogievskiy res_j = case_results[env_j['id']] 100aa362403SVladimir Sementsov-Ogievskiy cell += ' ' 101aa362403SVladimir Sementsov-Ogievskiy 102aa362403SVladimir Sementsov-Ogievskiy if 'average' not in res_j: 103aa362403SVladimir Sementsov-Ogievskiy # Failed result 104aa362403SVladimir Sementsov-Ogievskiy cell += '--' 105aa362403SVladimir Sementsov-Ogievskiy continue 106aa362403SVladimir Sementsov-Ogievskiy 107aa362403SVladimir Sementsov-Ogievskiy col_j = tab[0][j + 1] if named_columns else '' 108aa362403SVladimir Sementsov-Ogievskiy diff_pr = round((res['average'] - res_j['average']) / 109aa362403SVladimir Sementsov-Ogievskiy res_j['average'] * 100) 110aa362403SVladimir Sementsov-Ogievskiy cell += f' {col_j}{diff_pr:+}%' 111aa362403SVladimir Sementsov-Ogievskiy row.append(cell) 112aa362403SVladimir Sementsov-Ogievskiy tab.append(row) 113aa362403SVladimir Sementsov-Ogievskiy 114aa362403SVladimir Sementsov-Ogievskiy return f'All results are in {dim}\n\n' + tabulate.tabulate(tab) 115*181f60c8SVladimir Sementsov-Ogievskiy 116*181f60c8SVladimir Sementsov-Ogievskiy 117*181f60c8SVladimir Sementsov-Ogievskiyif __name__ == '__main__': 118*181f60c8SVladimir Sementsov-Ogievskiy import sys 119*181f60c8SVladimir Sementsov-Ogievskiy import json 120*181f60c8SVladimir Sementsov-Ogievskiy 121*181f60c8SVladimir Sementsov-Ogievskiy if len(sys.argv) < 2: 122*181f60c8SVladimir Sementsov-Ogievskiy print(f'USAGE: {sys.argv[0]} results.json') 123*181f60c8SVladimir Sementsov-Ogievskiy exit(1) 124*181f60c8SVladimir Sementsov-Ogievskiy 125*181f60c8SVladimir Sementsov-Ogievskiy with open(sys.argv[1]) as f: 126*181f60c8SVladimir Sementsov-Ogievskiy print(results_to_text(json.load(f))) 127