1#!/usr/bin/env python 2# -*- coding: utf-8 -*- 3# 4# Copyright (c) 2017, IntoPIX SA 5# Contact: support@intopix.com 6# Author: Even Rouault 7# 8# Redistribution and use in source and binary forms, with or without 9# modification, are permitted provided that the following conditions 10# are met: 11# 1. Redistributions of source code must retain the above copyright 12# notice, this list of conditions and the following disclaimer. 13# 2. Redistributions in binary form must reproduce the above copyright 14# notice, this list of conditions and the following disclaimer in the 15# documentation and/or other materials provided with the distribution. 16# 17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS `AS IS' 18# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27# POSSIBILITY OF SUCH DAMAGE. 28# 29 30import sys 31 32 33def Usage(): 34 print('Usage: compare_perfs.py [-noise_threshold val_in_pct]') 35 print(' [-warning_threshold val_in_pct]') 36 print(' [-error_threshold val_in_pct]') 37 print(' [-global_error_threshold val_in_pct]') 38 print(' ref.csv new.csv') 39 sys.exit(1) 40 41ref_filename = None 42new_filename = None 43noise_threshold = 2 44warning_threshold = 4 45error_threshold = 6 46global_error_threshold = 2 47i = 1 48while i < len(sys.argv): 49 if sys.argv[i] == '-noise_threshold' and i + 1 < len(sys.argv): 50 i += 1 51 noise_threshold = int(sys.argv[i]) 52 elif sys.argv[i] == '-warning_threshold' and i + 1 < len(sys.argv): 53 i += 1 54 warning_threshold = int(sys.argv[i]) 55 elif sys.argv[i] == '-error_threshold' and i + 1 < len(sys.argv): 56 i += 1 57 error_threshold = int(sys.argv[i]) 58 elif sys.argv[i] == '-global_error_threshold' and i + 1 < len(sys.argv): 59 i += 1 60 global_error_threshold = int(sys.argv[i]) 61 elif sys.argv[i][0] == '-': 62 Usage() 63 elif ref_filename is None: 64 ref_filename = sys.argv[i] 65 elif new_filename is None: 66 new_filename = sys.argv[i] 67 else: 68 Usage() 69 i += 1 70if ref_filename is None or new_filename is None: 71 Usage() 72 73assert noise_threshold < warning_threshold 74assert warning_threshold < error_threshold 75assert global_error_threshold >= noise_threshold 76assert global_error_threshold <= error_threshold 77 78ref_lines = open(ref_filename, 'rt').readlines()[1:] 79new_lines = open(new_filename, 'rt').readlines()[1:] 80if len(ref_lines) != len(new_lines): 81 raise Exception('files are not comparable') 82 83ret_code = 0 84for i in range(len(ref_lines)): 85 line = ref_lines[i].replace('\n', '') 86 filename_ref, num_iterations_ref, num_threads_ref, command_ref, \ 87 _, time_ms_ref = line.split(',') 88 line = new_lines[i].replace('\n', '') 89 filename_new, num_iterations_new, num_threads_new, command_new, \ 90 _, time_ms_new = line.split(',') 91 assert filename_ref == filename_new 92 assert num_iterations_ref == num_iterations_new 93 assert num_threads_ref == num_threads_new 94 assert command_ref == command_new 95 time_ms_ref = int(time_ms_ref) 96 time_ms_new = int(time_ms_new) 97 if filename_ref == 'TOTAL': 98 display = 'TOTAL' 99 else: 100 display = '%s, %s iterations, %s threads, %s' % \ 101 (filename_ref, num_iterations_ref, num_threads_ref, command_ref) 102 display += ': ref_time %d ms, new_time %d ms' % (time_ms_ref, time_ms_new) 103 var_pct = 100.0 * (time_ms_new - time_ms_ref) / time_ms_ref 104 if abs(var_pct) <= noise_threshold: 105 display += ', (stable) %0.1f %%' % var_pct 106 elif var_pct < 0: 107 display += ', (improvement) %0.1f %%' % var_pct 108 else: 109 display += ', (regression) %0.1f %%' % var_pct 110 if filename_ref == 'TOTAL' and var_pct > global_error_threshold: 111 display += ', ERROR_THRESHOLD' 112 ret_code = 1 113 elif var_pct > error_threshold: 114 display += ', ERROR_THRESHOLD' 115 ret_code = 1 116 elif var_pct > warning_threshold: 117 display += ', WARNING_THRESHOLD' 118 print(display) 119 120sys.exit(ret_code) 121