1#!/usr/bin/env python3 2 3import csv 4import collections 5 6#---------------------------------------------------------------- 7# This is a specially modified chart generator that takes many 8# repetions of the same test and displays them all on each chart. 9#---------------------------------------------------------------- 10 11# 12#---------------------------------------------------------------- 13# The files this generator acts upon are the CSV files output 14# from one or more runs of test_measure_perf.py. 15# 16# Data file naming convention: prefix_id_rep.csv 17# 18# This script takes no arguments. When it's run, it will 19# produce an HTML file called "test_perf_charts.html". 20# 21# Open that file in your browser to see the charts. 22#---------------------------------------------------------------- 23 24#---------------------------------------------------------------- 25# The following variables will all need to be edited to match 26# the tests that you are charting 27#---------------------------------------------------------------- 28# 29# vpx is now vp8 30# x264 is now h264 31# 32# TODO 33# -- Move the legend into the chart title area 34# -- Abbreviate the app names so they fit on the chart 35# -- Update the documentation to describe how to use multiple unique 36# directories containing files with same key, instead of one directory 37# and files with unique keys. 38# 39 40# Location of the data files 41base_dir = "/home/nickc/xtests/logs" 42 43# Data file prefix 44prefix_all = "all_tests_40" 45prefix_glx = "h264_glx" 46 47# Result subdirectories 48subs_h = ["hv", "h1", "h2", "h3"] 49subs_8 = ["0.15.0", "8585_1", "8585_2", "8585_3"] 50subs_9 = ["0.15.0", "9612_1", "9612_2", "9612_3"] 51 52# id is the actual id string used in the data file name 53# dir is an optional subdirectory within the base_dir where the data is stored 54# display is how that parameter should be displayed in the charts 55params = [ 56 {"id": "8585", "dir": subs_8[0], "display": "8585_all_0", "prefix": prefix_all}, 57 {"id": "8585", "dir": subs_8[1], "display": "8585_all_1", "prefix": prefix_all}, 58 {"id": "8585", "dir": subs_8[2], "display": "8585_all_2", "prefix": prefix_all}, 59 {"id": "8585", "dir": subs_8[3], "display": "8585_all_3", "prefix": prefix_all}, 60 {"id": "8585", "dir": subs_h[0], "display": "8585_glx_0", "prefix": prefix_glx}, 61 {"id": "8585", "dir": subs_h[1], "display": "8585_glx_1", "prefix": prefix_glx}, 62 {"id": "8585", "dir": subs_h[2], "display": "8585_glx_2", "prefix": prefix_glx}, 63 {"id": "8585", "dir": subs_h[3], "display": "8585_glx_3", "prefix": prefix_glx}, 64 {"id": "9612", "dir": subs_9[0], "display": "9612_all_0", "prefix": prefix_all}, 65 {"id": "9612", "dir": subs_9[1], "display": "9612_all_1", "prefix": prefix_all}, 66 {"id": "9612", "dir": subs_9[2], "display": "9612_all_2", "prefix": prefix_all}, 67 {"id": "9612", "dir": subs_9[3], "display": "9612_all_3", "prefix": prefix_all}, 68 {"id": "9612", "dir": subs_h[0], "display": "9612_glx_0", "prefix": prefix_glx}, 69 {"id": "9612", "dir": subs_h[1], "display": "9612_glx_1", "prefix": prefix_glx}, 70 {"id": "9612", "dir": subs_h[2], "display": "9612_glx_2", "prefix": prefix_glx}, 71 {"id": "9612", "dir": subs_h[3], "display": "9612_glx_3", "prefix": prefix_glx} 72] 73#params = [ 74# {"id": "8585", "display": "8585"}, 75# {"id": "9612", "display": "9612"} 76#] 77 78# The description will be shown on the output page 79description = 'Results of every test run. The red bars are from the original 2 comparisons.' 80 81# Each file name's 'rep' value is the sequence number of that 82# data file, when results of multiple files should be averaged 83reps = 9 # Number of data files in each set 84 85# Options 86needReallyUniqueId = True 87showLegend = "false" 88 89#---------------------------------------------------------------- 90# Set any of the values in the following lists to 1 in order to 91# include that test app, or metric column in the chart page. 92# 93apps = {"glxgears": 0, 94 "glxspheres": 1, 95 "moebiusgears": 0, 96 "polytopes": 0, 97 "x11perf": 0, 98 "xterm": 0, 99 "gtkperf": 0} 100 101metrics = {"Regions/s": 1, 102 "Pixels/s Sent": 1, 103 "Encoding Pixels/s": 1, 104 "Decoding Pixels/s": 1, 105 "Application packets in/s": 1, 106 "Application bytes in/s": 1, 107 "Application packets out/s": 1, 108 "Application bytes out/s": 1, 109 "client user cpu_pct": 1, 110 "client system cpu pct": 1, 111 "client number of threads": 1, 112 "client vsize (MB)": 1, 113 "client rss (MB)": 1, 114 "server user cpu_pct": 1, 115 "server system cpu pct": 1, 116 "server number of threads": 1, 117 "server vsize (MB)": 1, 118 "server rss (MB)": 1, 119 "Min Batch Delay (ms)": 1, 120 "Avg Batch Delay (ms)": 1, 121 "Max Batch Delay (ms)": 1, 122 "Min Damage Latency (ms)": 1, 123 "Avg Damage Latency (ms)": 1, 124 "Max Damage Latency (ms)": 1, 125 "Min Quality": 1, 126 "Avg Quality": 1, 127 "Max Quality": 1, 128 "Min Speed": 0, 129 "Avg Speed": 0, 130 "Max Speed": 0} 131 132encodings = {"png": 0, 133 "rgb24": 0, 134 "jpeg": 0, 135 "h264": 1, 136 "vp8": 0, 137 "vp9": 0, 138 "mmap": 0} 139 140header_dupes = [] 141headers = {} 142titles = [] 143param_ids = [] 144param_names = [] 145 146ENCODING_RGB24 = "rgb24" 147 148def tree(): 149 return collections.defaultdict(tree) 150tests = tree() 151 152def ftree(): 153 return collections.defaultdict(float) 154 155# Create test map -- schema: 156# {metric: {encoding: {id: {app: {rep: avg_value}}}}} 157def accumulate_values(file_name, rep, param, uniqueId): 158 rownum = 0 159 rgb_count = 0 160 rgb_values = None 161 162 ifile = open(file_name, "rb") 163 for row in csv.reader(ifile, skipinitialspace=True): 164 if (rownum == 0): 165 if (len(headers) == 0): 166 get_headers(row) 167 else: 168 app = get_value(row, "Test Command") 169 if (apps[app] == 1): 170 encoding = get_value(row, "Encoding") 171 # x264 is now h264 172 if (encoding == 'x264'): 173 encoding = 'h264' 174 # vpx is now vp8 175 if (encoding == 'vpx'): 176 encoding = 'vp8' 177 178 if (encodings[encoding] == 1): 179 if (encoding == ENCODING_RGB24): 180 if (rgb_values is None): 181 rgb_values = ftree() 182 rgb_count = 0 183 184 for metric in metrics: 185 if (metrics[metric] == 1): 186 row_value = float(get_metric(row, metric)) 187 if (encoding == ENCODING_RGB24): 188 if (metric in rgb_values.keys()): 189 rgb_values[metric] += row_value 190 else: 191 rgb_values[metric] = row_value 192 else: 193 tests[metric][encoding][uniqueId][app][rep] = row_value 194 195 if (encoding == ENCODING_RGB24): 196 rgb_count += 1 197 if (rgb_count == 3): 198 for metric in metrics: 199 if (metrics[metric] == 1): 200 tests[metric][encoding][uniqueId][app][rep] = rgb_values[metric] / 3 201 rgb_count = 0 202 rgb_values = None 203 rownum += 1 204 ifile.close() 205 206def write_html(): 207 app_count = 0 208 for app in apps.keys(): 209 if (apps[app] == 1): 210 app_count += 1 211 212 chart_count = 0 213 for encoding in encodings.keys(): 214 if (encodings[encoding] == 1): 215 chart_count += 1 216 box_height = 0 217 if (chart_count < 4): 218 box_height = 400 219 elif (chart_count < 7): 220 box_height = 800 221 elif (chart_count < 10): 222 box_height = 1200 223 224 ofile = open("charts.html", "w") 225 ofile.write('<!DOCTYPE html>\n') 226 ofile.write('<html>\n') 227 ofile.write('<head>\n') 228 ofile.write(' <meta http-equiv="Content-Type" content="text/html; charset=utf-8">\n') 229 ofile.write(' <title>Xpra Performance Results</title>\n') 230 ofile.write(' <link href="css/xpra.css" rel="stylesheet" type="text/css">\n') 231 ofile.write(' <script language="javascript" type="text/javascript" src="js/jquery.js"></script>\n') 232 ofile.write(' <script language="javascript" type="text/javascript" src="js/jquery.flot.js"></script>\n') 233 ofile.write(' <script language="javascript" type="text/javascript" src="js/jquery.flot.categories.js"></script>\n') 234 ofile.write(' <script language="javascript" type="text/javascript" src="js/jquery.flot.orderbars_mod.js"></script>\n') 235 ofile.write(' <script language="javascript" type="text/javascript" src="js/xpra.js"></script>\n') 236 ofile.write(' <script language="javascript" type="text/javascript">\n') 237 ofile.write(' var options = {canvas:true, grid: {margin: {top:50}, hoverable: true}, series: {bars: {show: true, barWidth: 0.5}}, legend: {show: ' + showLegend + '}, ' 238 #' xaxis: {mode: "categories", tickLength: 0, min: -0.3, max: ' + str(app_count) +'}, ' 239 ' xaxis: {mode: "categories", tickLength: 0, min: -6, max: 6}, ' 240 ' colors: ["#cc0000", "#787A40", "#9FBF8C", "#C8AB65", "#cc0000", "#787A40", "#9FBF8C", "#C8AB65", "#cc0000", "#787A40", "#9FBF8C", "#C8AB65", "#cc0000", "#787A40", "#9FBF8C", "#C8AB65"]};\n') 241 242 m_index = 0 243 m_names = [] 244 for metric in sorted(tests.keys()): 245 m_names.append(metric) 246 e_names = [] 247 for encoding in sorted(tests[metric].keys()): 248 e_names.append(encoding) 249 titles.append(metric + ' ( ' + encoding + ' )') 250 for param in sorted(tests[metric][encoding].keys()): 251 ofile.write(' var e' + str(m_index) + '_' + encoding + '_' + param + ' = [') 252 for app in sorted(tests[metric][encoding][param].keys()): 253 value = 0 254 actual_reps = 0 255 for rep in sorted(tests[metric][encoding][param][app].keys()): 256 value += float(tests[metric][encoding][param][app][rep]) 257 actual_reps += 1 258 value = value / actual_reps 259 ofile.write('["' + app + '", ' + str(value) + '], ') 260 ofile.write('];' + '\n') 261 varStr = ' var d' + str(m_index) + '_' + encoding + ' = [' 262 for i in range(0, len(param_ids)): 263 if (i > 0): 264 varStr += ',' 265 varStr += '{label: "' + param_names[i] + '", data: e' + str(m_index) + '_' + encoding + '_' + param_ids[i] + ', bars:{order:' + str(i) + '}}' 266 varStr += '];\n' 267 ofile.write(varStr) 268 m_index += 1 269 270 chart_index = 0 271 m_index = 0 272 ofile.write(' $(function() {\n') 273 for metric in sorted(tests.keys()): 274 e_index = 0 275 for encoding in sorted(tests[metric].keys()): 276 ofile.write(' var plot' +str(chart_index)+ ' = $.plot($("#placeholder_' + str(m_index) + '_' + str(e_index) + '"), d' + str(m_index) + '_' + e_names[e_index] + ', options);\n') 277 e_index += 1 278 chart_index += 1 279 m_index += 1 280 title_index = 0 281 282 for metric in sorted(tests.keys()): 283 for encoding in sorted(tests[metric].keys()): 284 ofile.write(' set_title(' + str(title_index) + ', "' + titles[title_index] + '");\n') 285 title_index += 1 286 287 for mx in range(0, m_index): 288 ofile.write('$("#metric_link_'+str(mx)+'").click(function() {$("#metric_list").scrollTop(' + str(box_height) + '*'+str(mx)+');});') 289 ofile.write(' });\n') 290 ofile.write(' </script>\n') 291 ofile.write(' <style>.metric_box {height: ' + str(box_height) + 'px}</style>\n') 292 ofile.write('</head>\n') 293 ofile.write('<body>\n') 294 ofile.write(' <div id="page">\n') 295 ofile.write(' <div id="header_box">\n') 296 ofile.write(' <div id="header">\n') 297 ofile.write(' <h2>Xpra Performance Results</h2>\n') 298 ofile.write(' <h3>' + description + '</h3>\n') 299 ofile.write(' <div id="help_text">Click a metric to locate it in the results.</div>\n') 300 ofile.write(' </div>\n') 301 302 ofile.write(' <div id="select_box">\n') 303 m_index = 0 304 for metric in sorted(tests.keys()): 305 ofile.write(' <div id="metric_link_' + str(m_index) + '" style="float:left;height:20px;width:200px"><a href="#">' + metric + '</a></div>\n') 306 m_index += 1 307 ofile.write(' </div>\n') 308 ofile.write(' </div>\n') 309 310 ofile.write(' <div style="clear:both"></div>\n') 311 ofile.write(' <div id="metric_list">\n') 312 m_index = 0 313 for metric in sorted(tests.keys()): 314 ofile.write(' <div class="metric_box" id="metric_box_' + str(m_index) + '">\n') 315 ofile.write(' <div class="metric_label">' + metric + '</div>\n') 316 e_index = 0 317 for encoding in sorted(tests[metric].keys()): 318 ofile.write(' <div class="container">\n') 319 ofile.write(' <div id="placeholder_' + str(m_index) + '_' + str(e_index) + '" class="placeholder"></div>\n') 320 ofile.write(' </div>\n') 321 e_index += 1 322 323 ofile.write(' </div>\n') 324 m_index += 1 325 ofile.write(' <div class="metric_box"></div>\n') 326 ofile.write(' </div>\n') 327 ofile.write(' </div>\n') 328 ofile.write('</body>\n') 329 ofile.write('</html>\n') 330 ofile.close() 331 332def col_index(label): 333 return headers[label] 334 335def get_value(row, label): 336 return row[col_index(label)].strip() 337 338def get_metric(row, label): 339 cell = row[col_index(label)] 340 if cell is None or cell is '': 341 cell = '0' 342 return cell.strip() 343 344def sanitize(dirName): 345 # Make the directory name valid as a javascript variable 346 newName = dirName.replace('.', '_') 347 return newName 348 349def get_headers(row): 350 index = 0 351 for column in row: 352 col = column.strip() 353 if col in headers: 354 header_dupes.append(col) 355 headers[col] = index 356 index += 1 357 358def print_headers(): 359 for entry in headers: 360 print(entry + " " + str(headers[entry])) 361 for entry in header_dupes: 362 print("Found dupe: %s" % entry) 363 364def main(): 365 for param in params: 366 param_id = param_name = param['id'] 367 if ('dir' in param.keys()): 368 param_id = sanitize(param['dir']) 369 if (needReallyUniqueId): 370 param_id = param['id'] + '_' + sanitize(param['dir']); 371 372 if ('display' in param.keys()): 373 param_name = param['display'] 374 param_ids.append(param_id) 375 param_names.append(param_name) 376 377 for param in params: 378 uniqueId = param['id'] 379 if ('dir' in param.keys()): 380 uniqueId = sanitize(param['dir']) 381 382 if (needReallyUniqueId): 383 uniqueId = param['id'] + '_' + sanitize(param['dir']); 384 385 #file_name = base_dir + '/' + param['dir'] + '/' + param['prefix'] + '_' + param['id'] + '_x.csv' 386 #print file_name 387 388 for rep in range(0, reps): 389 if ('dir' in param.keys()): 390 file_name = base_dir + '/' + param['dir'] + '/' + param['prefix'] + '_' + param['id'] + '_' + str(rep+1) + '.csv' 391 else: 392 file_name = base_dir + '/' + param['prefix'] + '_' + param['id'] + '_' + str(rep+1) + '.csv' 393 accumulate_values(file_name, rep, param, uniqueId) 394 write_html() 395 print('\nCreated: charts.html\n') 396 397if __name__ == "__main__": 398 main() 399 400