1#!/usr/bin/env python3 2 3import csv 4import collections, math 5 6#---------------------------------------------------------------- 7# The files this generator acts upon are the CSV files output 8# from one or more runs of test_measure_perf.py. 9# 10# Data file naming convention: prefix_id_rep.csv 11# 12# This script takes no arguments. When it's run, it will 13# produce an HTML file called "test_perf_charts.html". 14# 15# Open that file in your browser to see the charts. 16#---------------------------------------------------------------- 17 18#---------------------------------------------------------------- 19# The following variables will all need to be edited to match 20# the tests that you are charting 21#---------------------------------------------------------------- 22# 23# vpx is now vp8 24# x264 is now h264 25# 26# TODO 27# -- Move the legend into the chart title area 28# -- Abbreviate the app names so they fit on the chart 29# -- Update the documentation to describe how to use multiple unique 30# directories containing files with same key, instead of one directory 31# and files with unique keys. 32# 33 34# Location of the data files 35#base_dir = "/home/nickc/xtests/logs/0.15.0" 36base_dir = "/home/nickc/xtests/logs/smo" 37 38# Data file prefix 39prefix = "smo_test" 40#prefix = "h264_glx" 41#prefix = "all_tests_40" 42 43# Result subdirectories 44#subs = ["0.15.0", "8585_1", "8585_2", "8585_3"] 45#subs = ["hv", "h1", "h2", "h3"] 46#subs = ["8585_2", "9612_2"] 47#subs = [] 48 49# id is the actual id string used in the data file name 50# dir is an optional subdirectory within the base_dir where the data is stored 51# display is how that parameter should be displayed in the charts 52#params = [ 53# {"id": "9612", "dir": subs[0], "display": "0"}, 54# {"id": "9612", "dir": subs[1], "display": "1"}, 55# {"id": "9612", "dir": subs[2], "display": "2"}, 56# {"id": "9612", "dir": subs[3], "display": "3"} 57#] 58 59#params = [ 60# {"id": "8585", "display": "8585"}, 61# {"id": "9612", "display": "9612"} 62#] 63 64params = [ 65 {"id": "15r10784", "display": "15.6"}, 66 {"id": "16r10655", "display": "16"} 67] 68 69# The description will be shown on the output page 70description = 'Comparison of v15 and v16.' 71 72# Each file name's 'rep' value is the sequence number of that 73# data file, when results of multiple files should be averaged 74reps = 5 # Number of data files in each set 75 76#---------------------------------------------------------------- 77# Set any of the values in the following lists to 1 in order to 78# include that test app, or metric column in the chart page. 79# 80apps = {"glxgears": 0, 81 "glxspheres": 0, 82 "glxspheres64": 0, 83 "moebiusgears": 0, 84 "polytopes": 0, 85 86 "x11perf": 0, # Not reliable 87 "xterm": 0, 88 "gtkperf": 0, 89 "memscroller": 0, 90 "deluxe": 0, 91 92 "eruption": 1, 93 "vlc sound visual": 1, 94 "vlc video": 1, 95 "xonotic-glx": 1} 96 97metrics = {"Regions/s": 1, 98 "Pixels/s Sent": 1, 99 "Encoding Pixels/s": 1, 100 "Decoding Pixels/s": 1, 101 "Application packets in/s": 1, 102 "Application bytes in/s": 1, 103 "Application packets out/s": 1, 104 "Application bytes out/s": 1, 105 "Frame Total Latency": 1, 106 "Client Frame Latency": 1, 107 "client user cpu_pct": 1, 108 "client system cpu pct": 1, 109 "client number of threads": 1, 110 "client vsize (MB)": 1, 111 "client rss (MB)": 1, 112 "server user cpu_pct": 1, 113 "server system cpu pct": 1, 114 "server number of threads": 1, 115 "server vsize (MB)": 1, 116 "server rss (MB)": 1, 117 "Min Batch Delay (ms)": 1, 118 "Avg Batch Delay (ms)": 1, 119 "Max Batch Delay (ms)": 1, 120 "Min Damage Latency (ms)": 1, 121 "Avg Damage Latency (ms)": 1, 122 "Max Damage Latency (ms)": 1, 123 "Min Quality": 1, 124 "Avg Quality": 1, 125 "Max Quality": 1, 126 "Min Speed": 0, 127 "Avg Speed": 0, 128 "Max Speed": 0} 129 130encodings = {"png": 1, 131 "rgb": 1, 132 "rgb24": 0, 133 "h264": 1, 134 "jpeg": 1, 135 "vp8": 1, 136 "vp9": 1, 137 "mmap": 1} 138 139header_dupes = [] 140headers = {} 141titles = [] 142param_ids = [] 143param_names = [] 144displayed_encodings = {} 145 146ENCODING_RGB24 = "rgb24" 147 148def tree(): 149 return collections.defaultdict(tree) 150tests = tree() 151 152def ftree(): 153 return collections.defaultdict(float) 154 155# Create test map -- schema: 156# {metric: {encoding: {id: {app: {rep: avg_value}}}}} 157def accumulate_values(file_name, rep, param, uniqueId): 158 rownum = 0 159 rgb_count = 0 160 rgb_values = None 161 #print "uniqueid ", uniqueId 162 163 ifile = open(file_name, "rb") 164 for row in csv.reader(ifile, skipinitialspace=True): 165 if (rownum == 0): 166 if (len(headers) == 0): 167 get_headers(row) 168 else: 169 app = get_value(row, "Test Command") 170 if (not app in apps): 171 print("Application: " + app + " not defined.") 172 exit() 173 174 if (apps[app] == 1): 175 encoding = get_value(row, "Encoding") 176 # x264 is now h264 177 if (encoding == 'x264'): 178 encoding = 'h264' 179 # vpx is now vp8 180 if (encoding == 'vpx'): 181 encoding = 'vp8' 182 183 if (not encoding in encodings): 184 print("Encoding: " + encoding + " not defined.") 185 exit() 186 187 if (encodings[encoding] == 1): 188 displayed_encodings[encoding] = encoding; 189 if (encoding == ENCODING_RGB24): 190 if (rgb_values is None): 191 rgb_values = ftree() 192 rgb_count = 0 193 194 for metric in metrics: 195 if (metrics[metric] == 1): 196 row_value = float(get_metric(row, metric)) 197 if (encoding == ENCODING_RGB24): 198 if (metric in rgb_values.keys()): 199 rgb_values[metric] += row_value 200 else: 201 rgb_values[metric] = row_value 202 else: 203 tests[metric][encoding][uniqueId][app][rep] = row_value 204 205 if (encoding == ENCODING_RGB24): 206 rgb_count += 1 207 if (rgb_count == 3): 208 for metric in metrics: 209 if (metrics[metric] == 1): 210 tests[metric][encoding][uniqueId][app][rep] = rgb_values[metric] / 3 211 rgb_count = 0 212 rgb_values = None 213 rownum += 1 214 ifile.close() 215 216def write_html(): 217 app_count = 0 218 for app in apps.keys(): 219 if (apps[app] == 1): 220 app_count += 1 221 222 chart_count = 0 223 for encoding in displayed_encodings.keys(): 224 if (encodings[encoding] == 1): 225 chart_count += 1 226 row_count = math.ceil(chart_count / 2.0) 227 box_height = row_count * 400 228 229 ofile = open("charts.html", "w") 230 ofile.write('<!DOCTYPE html>\n') 231 ofile.write('<html>\n') 232 ofile.write('<head>\n') 233 ofile.write(' <meta http-equiv="Content-Type" content="text/html; charset=utf-8">\n') 234 ofile.write(' <title>Xpra Performance Results</title>\n') 235 ofile.write(' <link href="css/xpra.css" rel="stylesheet" type="text/css">\n') 236 ofile.write(' <script language="javascript" type="text/javascript" src="js/jquery.js"></script>\n') 237 ofile.write(' <script language="javascript" type="text/javascript" src="js/jquery.flot.js"></script>\n') 238 ofile.write(' <script language="javascript" type="text/javascript" src="js/jquery.flot.categories.js"></script>\n') 239 ofile.write(' <script language="javascript" type="text/javascript" src="js/jquery.flot.orderbars_mod.js"></script>\n') 240 ofile.write(' <script language="javascript" type="text/javascript" src="js/xpra.js"></script>\n') 241 ofile.write(' <script language="javascript" type="text/javascript">\n') 242 ofile.write(' var options = {canvas:true, grid: {margin: {top:50}, hoverable: true}, series: {bars: {show: true, barWidth: 0.08}}, ' 243 #' xaxis: {mode: "categories", tickLength: 0, min: -0.3, max: ' + str(app_count) +'}, colors: ["#cc0000", "#787A40", "#9FBF8C", "#C8AB65", "#D4CBC3"]};\n') 244 ' xaxis: {mode: "categories", tickLength: 0, min: -0.3, max: ' + str(app_count) +'}, colors: ["#688b8a", "#a57c65"]};\n') 245 246 m_index = 0 247 m_names = [] 248 for metric in sorted(tests.keys()): 249 m_names.append(metric) 250 e_names = [] 251 for encoding in sorted(tests[metric].keys()): 252 e_names.append(encoding) 253 titles.append(metric + ' ( ' + encoding + ' )') 254 for param in sorted(tests[metric][encoding].keys()): 255 ofile.write(' var e' + str(m_index) + '_' + encoding + '_' + param + ' = [') 256 for app in sorted(tests[metric][encoding][param].keys()): 257 value = 0 258 actual_reps = 0 259 for rep in sorted(tests[metric][encoding][param][app].keys()): 260 value += float(tests[metric][encoding][param][app][rep]) 261 actual_reps += 1 262 value = value / actual_reps 263 ofile.write('["' + app + '", ' + str(value) + '], ') 264 ofile.write('];' + '\n') 265 varStr = ' var d' + str(m_index) + '_' + encoding + ' = [' 266 for i in range(0, len(param_ids)): 267 if (i > 0): 268 varStr += ',' 269 varStr += '{label: "' + param_names[i] + '", data: e' + str(m_index) + '_' + encoding + '_' + param_ids[i] + ', bars:{order:' + str(i) + '}}' 270 varStr += '];\n' 271 ofile.write(varStr) 272 m_index += 1 273 274 chart_index = 0 275 m_index = 0 276 ofile.write(' $(function() {\n') 277 for metric in sorted(tests.keys()): 278 e_index = 0 279 for encoding in sorted(tests[metric].keys()): 280 ofile.write(' var plot' +str(chart_index)+ ' = $.plot($("#placeholder_' + str(m_index) + '_' + str(e_index) + '"), d' + str(m_index) + '_' + e_names[e_index] + ', options);\n') 281 e_index += 1 282 chart_index += 1 283 m_index += 1 284 title_index = 0 285 286 for metric in sorted(tests.keys()): 287 for encoding in sorted(tests[metric].keys()): 288 ofile.write(' set_title(' + str(title_index) + ', "' + titles[title_index] + '");\n') 289 title_index += 1 290 291 for mx in range(0, m_index): 292 ofile.write('$("#metric_link_'+str(mx)+'").click(function() {$("#metric_list").scrollTop(' + str(box_height) + '*'+str(mx)+');});') 293 ofile.write(' });\n') 294 ofile.write(' </script>\n') 295 ofile.write(' <style>.metric_box {height: ' + str(box_height) + 'px}</style>\n') 296 ofile.write('</head>\n') 297 ofile.write('<body>\n') 298 ofile.write(' <div id="page">\n') 299 ofile.write(' <div id="header_box">\n') 300 ofile.write(' <div id="header">\n') 301 ofile.write(' <h2>Xpra Performance Results</h2>\n') 302 ofile.write(' <h3>' + description + '</h3>\n') 303 ofile.write(' <div id="help_text">Click a metric to locate it in the results.</div>\n') 304 ofile.write(' </div>\n') 305 306 ofile.write(' <div id="select_box">\n') 307 m_index = 0 308 for metric in sorted(tests.keys()): 309 ofile.write(' <div id="metric_link_' + str(m_index) + '" style="float:left;height:20px;width:200px"><a href="#">' + metric + '</a></div>\n') 310 m_index += 1 311 ofile.write(' </div>\n') 312 ofile.write(' </div>\n') 313 314 ofile.write(' <div style="clear:both"></div>\n') 315 ofile.write(' <div id="metric_list">\n') 316 m_index = 0 317 for metric in sorted(tests.keys()): 318 ofile.write(' <div class="metric_box" id="metric_box_' + str(m_index) + '">\n') 319 ofile.write(' <div class="metric_label">' + metric + '</div>\n') 320 e_index = 0 321 for encoding in sorted(tests[metric].keys()): 322 ofile.write(' <div class="container">\n') 323 ofile.write(' <div id="placeholder_' + str(m_index) + '_' + str(e_index) + '" class="placeholder"></div>\n') 324 ofile.write(' </div>\n') 325 e_index += 1 326 327 ofile.write(' </div>\n') 328 m_index += 1 329 ofile.write(' <div class="metric_box"></div>\n') 330 ofile.write(' </div>\n') 331 ofile.write(' </div>\n') 332 ofile.write('</body>\n') 333 ofile.write('</html>\n') 334 ofile.close() 335 336def col_index(label): 337 return headers[label] 338 339def get_value(row, label): 340 return row[col_index(label)].strip() 341 342def get_metric(row, label): 343 cell = row[col_index(label)] 344 if cell is None or cell is '': 345 cell = '0' 346 return cell.strip() 347 348def sanitize(dirName): 349 # Make the directory name valid as a javascript variable 350 newName = dirName.replace('.', '_') 351 return newName 352 353def get_headers(row): 354 index = 0 355 for column in row: 356 col = column.strip() 357 if col in headers: 358 header_dupes.append(col) 359 headers[col] = index 360 index += 1 361 362def print_headers(): 363 for entry in headers: 364 print(entry + " " + str(headers[entry])) 365 for entry in header_dupes: 366 print("Found dupe: %s" % entry) 367 368def main(): 369 for param in params: 370 param_id = param_name = param['id'] 371 if ('dir' in param.keys()): 372 param_id = sanitize(param['dir']) 373 if ('display' in param.keys()): 374 param_name = param['display'] 375 param_ids.append(param_id) 376 param_names.append(param_name) 377 378 for param in params: 379 uniqueId = param['id'] 380 if ('dir' in param.keys()): 381 uniqueId = sanitize(param['dir']) 382 383 for rep in range(0, reps): 384 if ('dir' in param.keys()): 385 file_name = base_dir + '/' + param['dir'] + '/' + prefix + '_' + param['id'] + '_' + str(rep+1) + '.csv' 386 else: 387 file_name = base_dir + '/' + prefix + '_' + param['id'] + '_' + str(rep+1) + '.csv' 388 print "Processing: ", file_name 389 accumulate_values(file_name, rep, param, uniqueId) 390 write_html() 391 print('\nCreated: charts.html\n') 392 393if __name__ == "__main__": 394 main() 395 396