1#!/usr/bin/env python3
2
3import csv
4import collections
5
6#----------------------------------------------------------------
7# This is a specially modified chart generator that takes many
8# repetions of the same test and displays them all on each chart.
9#----------------------------------------------------------------
10
11#
12#----------------------------------------------------------------
13# The files this generator acts upon are the CSV files output
14# from one or more runs of test_measure_perf.py.
15#
16# Data file naming convention: prefix_id_rep.csv
17#
18# This script takes no arguments. When it's run, it will
19# produce an HTML file called "test_perf_charts.html".
20#
21# Open that file in your browser to see the charts.
22#----------------------------------------------------------------
23
24#----------------------------------------------------------------
25# The following variables will all need to be edited to match
26# the tests that you are charting
27#----------------------------------------------------------------
28#
29# vpx is now vp8
30# x264 is now h264
31#
32# TODO
33# -- Move the legend into the chart title area
34# -- Abbreviate the app names so they fit on the chart
35# -- Update the documentation to describe how to use multiple unique
36#    directories containing files with same key, instead of one directory
37#    and files with unique keys.
38#
39
40# Location of the data files
41base_dir = "/home/nickc/xtests/logs"
42
43# Data file prefix
44prefix_all = "all_tests_40"
45prefix_glx = "h264_glx"
46
47# Result subdirectories
48subs_h = ["hv", "h1", "h2", "h3"]
49subs_8 = ["0.15.0", "8585_1", "8585_2", "8585_3"]
50subs_9 = ["0.15.0", "9612_1", "9612_2", "9612_3"]
51
52# id is the actual id string used in the data file name
53# dir is an optional subdirectory within the base_dir where the data is stored
54# display is how that parameter should be displayed in the charts
55params = [
56    {"id": "8585", "dir": subs_8[0], "display": "8585_all_0", "prefix": prefix_all},
57    {"id": "8585", "dir": subs_8[1], "display": "8585_all_1", "prefix": prefix_all},
58    {"id": "8585", "dir": subs_8[2], "display": "8585_all_2", "prefix": prefix_all},
59    {"id": "8585", "dir": subs_8[3], "display": "8585_all_3", "prefix": prefix_all},
60    {"id": "8585", "dir": subs_h[0], "display": "8585_glx_0", "prefix": prefix_glx},
61    {"id": "8585", "dir": subs_h[1], "display": "8585_glx_1", "prefix": prefix_glx},
62    {"id": "8585", "dir": subs_h[2], "display": "8585_glx_2", "prefix": prefix_glx},
63    {"id": "8585", "dir": subs_h[3], "display": "8585_glx_3", "prefix": prefix_glx},
64    {"id": "9612", "dir": subs_9[0], "display": "9612_all_0", "prefix": prefix_all},
65    {"id": "9612", "dir": subs_9[1], "display": "9612_all_1", "prefix": prefix_all},
66    {"id": "9612", "dir": subs_9[2], "display": "9612_all_2", "prefix": prefix_all},
67    {"id": "9612", "dir": subs_9[3], "display": "9612_all_3", "prefix": prefix_all},
68    {"id": "9612", "dir": subs_h[0], "display": "9612_glx_0", "prefix": prefix_glx},
69    {"id": "9612", "dir": subs_h[1], "display": "9612_glx_1", "prefix": prefix_glx},
70    {"id": "9612", "dir": subs_h[2], "display": "9612_glx_2", "prefix": prefix_glx},
71    {"id": "9612", "dir": subs_h[3], "display": "9612_glx_3", "prefix": prefix_glx}
72]
73#params = [
74#    {"id": "8585", "display": "8585"},
75#    {"id": "9612", "display": "9612"}
76#]
77
78# The description will be shown on the output page
79description = 'Results of every test run. The red bars are from the original 2 comparisons.'
80
81# Each file name's 'rep' value is the sequence number of that
82# data file, when results of multiple files should be averaged
83reps = 9     # Number of data files in each set
84
85# Options
86needReallyUniqueId = True
87showLegend = "false"
88
89#----------------------------------------------------------------
90# Set any of the values in the following lists to 1 in order to
91# include that test app, or metric column in the chart page.
92#
93apps = {"glxgears": 0,
94        "glxspheres": 1,
95        "moebiusgears": 0,
96        "polytopes": 0,
97        "x11perf": 0,
98        "xterm": 0,
99        "gtkperf": 0}
100
101metrics = {"Regions/s": 1,
102           "Pixels/s Sent": 1,
103           "Encoding Pixels/s": 1,
104           "Decoding Pixels/s": 1,
105           "Application packets in/s": 1,
106           "Application bytes in/s": 1,
107           "Application packets out/s": 1,
108           "Application bytes out/s": 1,
109           "client user cpu_pct": 1,
110           "client system cpu pct": 1,
111           "client number of threads": 1,
112           "client vsize (MB)": 1,
113           "client rss (MB)": 1,
114           "server user cpu_pct": 1,
115           "server system cpu pct": 1,
116           "server number of threads": 1,
117           "server vsize (MB)": 1,
118           "server rss (MB)": 1,
119           "Min Batch Delay (ms)": 1,
120           "Avg Batch Delay (ms)": 1,
121           "Max Batch Delay (ms)": 1,
122           "Min Damage Latency (ms)": 1,
123           "Avg Damage Latency (ms)": 1,
124           "Max Damage Latency (ms)": 1,
125           "Min Quality": 1,
126           "Avg Quality": 1,
127           "Max Quality": 1,
128           "Min Speed": 0,
129           "Avg Speed": 0,
130           "Max Speed": 0}
131
132encodings = {"png": 0,
133             "rgb24": 0,
134             "jpeg": 0,
135             "h264": 1,
136             "vp8": 0,
137             "vp9": 0,
138             "mmap": 0}
139
140header_dupes = []
141headers = {}
142titles = []
143param_ids = []
144param_names = []
145
146ENCODING_RGB24 = "rgb24"
147
148def tree():
149    return collections.defaultdict(tree)
150tests = tree()
151
152def ftree():
153    return collections.defaultdict(float)
154
155# Create test map -- schema:
156# {metric: {encoding: {id: {app: {rep: avg_value}}}}}
157def accumulate_values(file_name, rep, param, uniqueId):
158    rownum = 0
159    rgb_count = 0
160    rgb_values = None
161
162    ifile = open(file_name, "rb")
163    for row in csv.reader(ifile, skipinitialspace=True):
164        if (rownum == 0):
165            if (len(headers) == 0):
166                get_headers(row)
167        else:
168            app = get_value(row, "Test Command")
169            if (apps[app] == 1):
170                encoding = get_value(row, "Encoding")
171                # x264 is now h264
172                if (encoding == 'x264'):
173                    encoding = 'h264'
174                # vpx is now vp8
175                if (encoding == 'vpx'):
176                    encoding = 'vp8'
177
178                if (encodings[encoding] == 1):
179                    if (encoding == ENCODING_RGB24):
180                        if (rgb_values is None):
181                            rgb_values = ftree()
182                            rgb_count = 0
183
184                    for metric in metrics:
185                        if (metrics[metric] == 1):
186                            row_value = float(get_metric(row, metric))
187                            if (encoding == ENCODING_RGB24):
188                                if (metric in rgb_values.keys()):
189                                    rgb_values[metric] += row_value
190                                else:
191                                    rgb_values[metric] = row_value
192                            else:
193                                tests[metric][encoding][uniqueId][app][rep] = row_value
194
195                    if (encoding == ENCODING_RGB24):
196                        rgb_count += 1
197                        if (rgb_count == 3):
198                            for metric in metrics:
199                                if (metrics[metric] == 1):
200                                    tests[metric][encoding][uniqueId][app][rep] = rgb_values[metric] / 3
201                            rgb_count = 0
202                            rgb_values = None
203        rownum += 1
204    ifile.close()
205
206def write_html():
207    app_count = 0
208    for app in apps.keys():
209        if (apps[app] == 1):
210            app_count += 1
211
212    chart_count = 0
213    for encoding in encodings.keys():
214        if (encodings[encoding] == 1):
215            chart_count += 1
216    box_height = 0
217    if (chart_count < 4):
218        box_height = 400
219    elif (chart_count < 7):
220        box_height = 800
221    elif (chart_count < 10):
222        box_height = 1200
223
224    ofile = open("charts.html", "w")
225    ofile.write('<!DOCTYPE html>\n')
226    ofile.write('<html>\n')
227    ofile.write('<head>\n')
228    ofile.write('  <meta http-equiv="Content-Type" content="text/html; charset=utf-8">\n')
229    ofile.write('  <title>Xpra Performance Results</title>\n')
230    ofile.write('  <link href="css/xpra.css" rel="stylesheet" type="text/css">\n')
231    ofile.write('  <script language="javascript" type="text/javascript" src="js/jquery.js"></script>\n')
232    ofile.write('  <script language="javascript" type="text/javascript" src="js/jquery.flot.js"></script>\n')
233    ofile.write('  <script language="javascript" type="text/javascript" src="js/jquery.flot.categories.js"></script>\n')
234    ofile.write('  <script language="javascript" type="text/javascript" src="js/jquery.flot.orderbars_mod.js"></script>\n')
235    ofile.write('  <script language="javascript" type="text/javascript" src="js/xpra.js"></script>\n')
236    ofile.write('  <script language="javascript" type="text/javascript">\n')
237    ofile.write('    var options = {canvas:true, grid: {margin: {top:50}, hoverable: true}, series: {bars: {show: true, barWidth: 0.5}}, legend: {show: ' + showLegend + '}, '
238                #' xaxis: {mode: "categories", tickLength: 0, min: -0.3, max: ' + str(app_count) +'}, '
239                ' xaxis: {mode: "categories", tickLength: 0, min: -6, max: 6}, '
240                ' colors: ["#cc0000", "#787A40", "#9FBF8C", "#C8AB65", "#cc0000", "#787A40", "#9FBF8C", "#C8AB65", "#cc0000", "#787A40", "#9FBF8C", "#C8AB65", "#cc0000", "#787A40", "#9FBF8C", "#C8AB65"]};\n')
241
242    m_index = 0
243    m_names = []
244    for metric in sorted(tests.keys()):
245        m_names.append(metric)
246        e_names = []
247        for encoding in sorted(tests[metric].keys()):
248            e_names.append(encoding)
249            titles.append(metric + ' ( ' + encoding + ' )')
250            for param in sorted(tests[metric][encoding].keys()):
251                ofile.write('    var e' + str(m_index) + '_' + encoding + '_' + param + ' = [')
252                for app in sorted(tests[metric][encoding][param].keys()):
253                    value = 0
254                    actual_reps = 0
255                    for rep in sorted(tests[metric][encoding][param][app].keys()):
256                        value += float(tests[metric][encoding][param][app][rep])
257                        actual_reps += 1
258                    value = value / actual_reps
259                    ofile.write('["' + app + '", ' + str(value) + '], ')
260                ofile.write('];' + '\n')
261            varStr = '    var d' + str(m_index) + '_' + encoding + ' = ['
262            for i in range(0, len(param_ids)):
263                if (i > 0):
264                    varStr += ','
265                varStr += '{label: "' + param_names[i] + '", data: e' + str(m_index) + '_' + encoding + '_' + param_ids[i] + ', bars:{order:' + str(i) + '}}'
266            varStr += '];\n'
267            ofile.write(varStr)
268        m_index += 1
269
270    chart_index = 0
271    m_index = 0
272    ofile.write('    $(function() {\n')
273    for metric in sorted(tests.keys()):
274        e_index = 0
275        for encoding in sorted(tests[metric].keys()):
276            ofile.write('        var plot' +str(chart_index)+ ' = $.plot($("#placeholder_' + str(m_index) + '_' + str(e_index) + '"), d' + str(m_index) + '_' + e_names[e_index] + ', options);\n')
277            e_index += 1
278            chart_index += 1
279        m_index += 1
280    title_index = 0
281
282    for metric in sorted(tests.keys()):
283        for encoding in sorted(tests[metric].keys()):
284            ofile.write('        set_title(' + str(title_index) + ', "' + titles[title_index] + '");\n')
285            title_index += 1
286
287    for mx in range(0, m_index):
288        ofile.write('$("#metric_link_'+str(mx)+'").click(function() {$("#metric_list").scrollTop(' + str(box_height) + '*'+str(mx)+');});')
289    ofile.write('    });\n')
290    ofile.write('  </script>\n')
291    ofile.write('  <style>.metric_box {height: ' + str(box_height) + 'px}</style>\n')
292    ofile.write('</head>\n')
293    ofile.write('<body>\n')
294    ofile.write('  <div id="page">\n')
295    ofile.write('    <div id="header_box">\n')
296    ofile.write('      <div id="header">\n')
297    ofile.write('        <h2>Xpra Performance Results</h2>\n')
298    ofile.write('        <h3>' + description + '</h3>\n')
299    ofile.write('        <div id="help_text">Click a metric to locate it in the results.</div>\n')
300    ofile.write('      </div>\n')
301
302    ofile.write('      <div id="select_box">\n')
303    m_index = 0
304    for metric in sorted(tests.keys()):
305        ofile.write('        <div id="metric_link_' + str(m_index) + '" style="float:left;height:20px;width:200px"><a href="#">' + metric + '</a></div>\n')
306        m_index += 1
307    ofile.write('      </div>\n')
308    ofile.write('    </div>\n')
309
310    ofile.write('    <div style="clear:both"></div>\n')
311    ofile.write('    <div id="metric_list">\n')
312    m_index = 0
313    for metric in sorted(tests.keys()):
314        ofile.write('      <div class="metric_box" id="metric_box_' + str(m_index) + '">\n')
315        ofile.write('        <div class="metric_label">' + metric + '</div>\n')
316        e_index = 0
317        for encoding in sorted(tests[metric].keys()):
318            ofile.write('        <div class="container">\n')
319            ofile.write('          <div id="placeholder_' + str(m_index) + '_' + str(e_index) + '" class="placeholder"></div>\n')
320            ofile.write('        </div>\n')
321            e_index += 1
322
323        ofile.write('      </div>\n')
324        m_index += 1
325    ofile.write('      <div class="metric_box"></div>\n')
326    ofile.write('    </div>\n')
327    ofile.write('  </div>\n')
328    ofile.write('</body>\n')
329    ofile.write('</html>\n')
330    ofile.close()
331
332def col_index(label):
333    return headers[label]
334
335def get_value(row, label):
336    return row[col_index(label)].strip()
337
338def get_metric(row, label):
339    cell = row[col_index(label)]
340    if cell is None or cell is '':
341        cell = '0'
342    return cell.strip()
343
344def sanitize(dirName):
345    # Make the directory name valid as a javascript variable
346    newName = dirName.replace('.', '_')
347    return newName
348
349def get_headers(row):
350    index = 0
351    for column in row:
352        col = column.strip()
353        if col in headers:
354            header_dupes.append(col)
355        headers[col] = index
356        index += 1
357
358def print_headers():
359    for entry in headers:
360        print(entry + " " + str(headers[entry]))
361    for entry in header_dupes:
362        print("Found dupe: %s" % entry)
363
364def main():
365    for param in params:
366        param_id = param_name = param['id']
367        if ('dir' in param.keys()):
368            param_id = sanitize(param['dir'])
369        if (needReallyUniqueId):
370            param_id = param['id'] + '_' + sanitize(param['dir']);
371
372        if ('display' in param.keys()):
373            param_name = param['display']
374        param_ids.append(param_id)
375        param_names.append(param_name)
376
377    for param in params:
378        uniqueId = param['id']
379        if ('dir' in param.keys()):
380            uniqueId = sanitize(param['dir'])
381
382        if (needReallyUniqueId):
383            uniqueId = param['id'] + '_' + sanitize(param['dir']);
384
385        #file_name = base_dir + '/' + param['dir'] + '/' + param['prefix'] + '_' + param['id'] + '_x.csv'
386        #print file_name
387
388        for rep in range(0, reps):
389            if ('dir' in param.keys()):
390                file_name = base_dir + '/' + param['dir'] + '/' + param['prefix'] + '_' + param['id'] + '_' + str(rep+1) + '.csv'
391            else:
392                file_name = base_dir + '/' + param['prefix'] + '_' + param['id'] + '_' + str(rep+1) + '.csv'
393            accumulate_values(file_name, rep, param, uniqueId)
394    write_html()
395    print('\nCreated: charts.html\n')
396
397if __name__ == "__main__":
398    main()
399
400