1#!/usr/bin/env python3
2
3import csv
4import collections, math
5
6#----------------------------------------------------------------
7# The files this generator acts upon are the CSV files output
8# from one or more runs of test_measure_perf.py.
9#
10# Data file naming convention: prefix_id_rep.csv
11#
12# This script takes no arguments. When it's run, it will
13# produce an HTML file called "test_perf_charts.html".
14#
15# Open that file in your browser to see the charts.
16#----------------------------------------------------------------
17
18#----------------------------------------------------------------
19# The following variables will all need to be edited to match
20# the tests that you are charting
21#----------------------------------------------------------------
22#
23# vpx is now vp8
24# x264 is now h264
25#
26# TODO
27# -- Move the legend into the chart title area
28# -- Abbreviate the app names so they fit on the chart
29# -- Update the documentation to describe how to use multiple unique
30#    directories containing files with same key, instead of one directory
31#    and files with unique keys.
32#
33
34# Location of the data files
35#base_dir = "/home/nickc/xtests/logs/0.15.0"
36base_dir = "/home/nickc/xtests/logs/smo"
37
38# Data file prefix
39prefix = "smo_test"
40#prefix = "h264_glx"
41#prefix = "all_tests_40"
42
43# Result subdirectories
44#subs = ["0.15.0", "8585_1", "8585_2", "8585_3"]
45#subs = ["hv", "h1", "h2", "h3"]
46#subs = ["8585_2", "9612_2"]
47#subs = []
48
49# id is the actual id string used in the data file name
50# dir is an optional subdirectory within the base_dir where the data is stored
51# display is how that parameter should be displayed in the charts
52#params = [
53#    {"id": "9612", "dir": subs[0], "display": "0"},
54#    {"id": "9612", "dir": subs[1], "display": "1"},
55#    {"id": "9612", "dir": subs[2], "display": "2"},
56#    {"id": "9612", "dir": subs[3], "display": "3"}
57#]
58
59#params = [
60#    {"id": "8585", "display": "8585"},
61#    {"id": "9612", "display": "9612"}
62#]
63
64params = [
65    {"id": "15r10784", "display": "15.6"},
66    {"id": "16r10655", "display": "16"}
67]
68
69# The description will be shown on the output page
70description = 'Comparison of v15 and v16.'
71
72# Each file name's 'rep' value is the sequence number of that
73# data file, when results of multiple files should be averaged
74reps = 5     # Number of data files in each set
75
76#----------------------------------------------------------------
77# Set any of the values in the following lists to 1 in order to
78# include that test app, or metric column in the chart page.
79#
80apps = {"glxgears": 0,
81        "glxspheres": 0,
82        "glxspheres64": 0,
83        "moebiusgears": 0,
84        "polytopes": 0,
85
86        "x11perf": 0, # Not reliable
87        "xterm": 0,
88        "gtkperf": 0,
89        "memscroller": 0,
90        "deluxe": 0,
91
92        "eruption": 1,
93        "vlc sound visual": 1,
94        "vlc video": 1,
95        "xonotic-glx": 1}
96
97metrics = {"Regions/s": 1,
98           "Pixels/s Sent": 1,
99           "Encoding Pixels/s": 1,
100           "Decoding Pixels/s": 1,
101           "Application packets in/s": 1,
102           "Application bytes in/s": 1,
103           "Application packets out/s": 1,
104           "Application bytes out/s": 1,
105           "Frame Total Latency": 1,
106           "Client Frame Latency": 1,
107           "client user cpu_pct": 1,
108           "client system cpu pct": 1,
109           "client number of threads": 1,
110           "client vsize (MB)": 1,
111           "client rss (MB)": 1,
112           "server user cpu_pct": 1,
113           "server system cpu pct": 1,
114           "server number of threads": 1,
115           "server vsize (MB)": 1,
116           "server rss (MB)": 1,
117           "Min Batch Delay (ms)": 1,
118           "Avg Batch Delay (ms)": 1,
119           "Max Batch Delay (ms)": 1,
120           "Min Damage Latency (ms)": 1,
121           "Avg Damage Latency (ms)": 1,
122           "Max Damage Latency (ms)": 1,
123           "Min Quality": 1,
124           "Avg Quality": 1,
125           "Max Quality": 1,
126           "Min Speed": 0,
127           "Avg Speed": 0,
128           "Max Speed": 0}
129
130encodings = {"png": 1,
131             "rgb": 1,
132             "rgb24": 0,
133             "h264": 1,
134             "jpeg": 1,
135             "vp8": 1,
136             "vp9": 1,
137             "mmap": 1}
138
139header_dupes = []
140headers = {}
141titles = []
142param_ids = []
143param_names = []
144displayed_encodings = {}
145
146ENCODING_RGB24 = "rgb24"
147
148def tree():
149    return collections.defaultdict(tree)
150tests = tree()
151
152def ftree():
153    return collections.defaultdict(float)
154
155# Create test map -- schema:
156# {metric: {encoding: {id: {app: {rep: avg_value}}}}}
157def accumulate_values(file_name, rep, param, uniqueId):
158    rownum = 0
159    rgb_count = 0
160    rgb_values = None
161    #print "uniqueid ", uniqueId
162
163    ifile = open(file_name, "rb")
164    for row in csv.reader(ifile, skipinitialspace=True):
165        if (rownum == 0):
166            if (len(headers) == 0):
167                get_headers(row)
168        else:
169            app = get_value(row, "Test Command")
170            if (not app in apps):
171                print("Application: " + app + " not defined.")
172                exit()
173
174            if (apps[app] == 1):
175                encoding = get_value(row, "Encoding")
176                # x264 is now h264
177                if (encoding == 'x264'):
178                    encoding = 'h264'
179                # vpx is now vp8
180                if (encoding == 'vpx'):
181                    encoding = 'vp8'
182
183                if (not encoding in encodings):
184                    print("Encoding: " + encoding + " not defined.")
185                    exit()
186
187                if (encodings[encoding] == 1):
188                    displayed_encodings[encoding] = encoding;
189                    if (encoding == ENCODING_RGB24):
190                        if (rgb_values is None):
191                            rgb_values = ftree()
192                            rgb_count = 0
193
194                    for metric in metrics:
195                        if (metrics[metric] == 1):
196                            row_value = float(get_metric(row, metric))
197                            if (encoding == ENCODING_RGB24):
198                                if (metric in rgb_values.keys()):
199                                    rgb_values[metric] += row_value
200                                else:
201                                    rgb_values[metric] = row_value
202                            else:
203                                tests[metric][encoding][uniqueId][app][rep] = row_value
204
205                    if (encoding == ENCODING_RGB24):
206                        rgb_count += 1
207                        if (rgb_count == 3):
208                            for metric in metrics:
209                                if (metrics[metric] == 1):
210                                    tests[metric][encoding][uniqueId][app][rep] = rgb_values[metric] / 3
211                            rgb_count = 0
212                            rgb_values = None
213        rownum += 1
214    ifile.close()
215
216def write_html():
217    app_count = 0
218    for app in apps.keys():
219        if (apps[app] == 1):
220            app_count += 1
221
222    chart_count = 0
223    for encoding in displayed_encodings.keys():
224        if (encodings[encoding] == 1):
225            chart_count += 1
226    row_count = math.ceil(chart_count / 2.0)
227    box_height = row_count * 400
228
229    ofile = open("charts.html", "w")
230    ofile.write('<!DOCTYPE html>\n')
231    ofile.write('<html>\n')
232    ofile.write('<head>\n')
233    ofile.write('  <meta http-equiv="Content-Type" content="text/html; charset=utf-8">\n')
234    ofile.write('  <title>Xpra Performance Results</title>\n')
235    ofile.write('  <link href="css/xpra.css" rel="stylesheet" type="text/css">\n')
236    ofile.write('  <script language="javascript" type="text/javascript" src="js/jquery.js"></script>\n')
237    ofile.write('  <script language="javascript" type="text/javascript" src="js/jquery.flot.js"></script>\n')
238    ofile.write('  <script language="javascript" type="text/javascript" src="js/jquery.flot.categories.js"></script>\n')
239    ofile.write('  <script language="javascript" type="text/javascript" src="js/jquery.flot.orderbars_mod.js"></script>\n')
240    ofile.write('  <script language="javascript" type="text/javascript" src="js/xpra.js"></script>\n')
241    ofile.write('  <script language="javascript" type="text/javascript">\n')
242    ofile.write('    var options = {canvas:true, grid: {margin: {top:50}, hoverable: true}, series: {bars: {show: true, barWidth: 0.08}}, '
243                #' xaxis: {mode: "categories", tickLength: 0, min: -0.3, max: ' + str(app_count) +'}, colors: ["#cc0000", "#787A40", "#9FBF8C", "#C8AB65", "#D4CBC3"]};\n')
244                ' xaxis: {mode: "categories", tickLength: 0, min: -0.3, max: ' + str(app_count) +'}, colors: ["#688b8a", "#a57c65"]};\n')
245
246    m_index = 0
247    m_names = []
248    for metric in sorted(tests.keys()):
249        m_names.append(metric)
250        e_names = []
251        for encoding in sorted(tests[metric].keys()):
252            e_names.append(encoding)
253            titles.append(metric + ' ( ' + encoding + ' )')
254            for param in sorted(tests[metric][encoding].keys()):
255                ofile.write('    var e' + str(m_index) + '_' + encoding + '_' + param + ' = [')
256                for app in sorted(tests[metric][encoding][param].keys()):
257                    value = 0
258                    actual_reps = 0
259                    for rep in sorted(tests[metric][encoding][param][app].keys()):
260                        value += float(tests[metric][encoding][param][app][rep])
261                        actual_reps += 1
262                    value = value / actual_reps
263                    ofile.write('["' + app + '", ' + str(value) + '], ')
264                ofile.write('];' + '\n')
265            varStr = '    var d' + str(m_index) + '_' + encoding + ' = ['
266            for i in range(0, len(param_ids)):
267                if (i > 0):
268                    varStr += ','
269                varStr += '{label: "' + param_names[i] + '", data: e' + str(m_index) + '_' + encoding + '_' + param_ids[i] + ', bars:{order:' + str(i) + '}}'
270            varStr += '];\n'
271            ofile.write(varStr)
272        m_index += 1
273
274    chart_index = 0
275    m_index = 0
276    ofile.write('    $(function() {\n')
277    for metric in sorted(tests.keys()):
278        e_index = 0
279        for encoding in sorted(tests[metric].keys()):
280            ofile.write('        var plot' +str(chart_index)+ ' = $.plot($("#placeholder_' + str(m_index) + '_' + str(e_index) + '"), d' + str(m_index) + '_' + e_names[e_index] + ', options);\n')
281            e_index += 1
282            chart_index += 1
283        m_index += 1
284    title_index = 0
285
286    for metric in sorted(tests.keys()):
287        for encoding in sorted(tests[metric].keys()):
288            ofile.write('        set_title(' + str(title_index) + ', "' + titles[title_index] + '");\n')
289            title_index += 1
290
291    for mx in range(0, m_index):
292        ofile.write('$("#metric_link_'+str(mx)+'").click(function() {$("#metric_list").scrollTop(' + str(box_height) + '*'+str(mx)+');});')
293    ofile.write('    });\n')
294    ofile.write('  </script>\n')
295    ofile.write('  <style>.metric_box {height: ' + str(box_height) + 'px}</style>\n')
296    ofile.write('</head>\n')
297    ofile.write('<body>\n')
298    ofile.write('  <div id="page">\n')
299    ofile.write('    <div id="header_box">\n')
300    ofile.write('      <div id="header">\n')
301    ofile.write('        <h2>Xpra Performance Results</h2>\n')
302    ofile.write('        <h3>' + description + '</h3>\n')
303    ofile.write('        <div id="help_text">Click a metric to locate it in the results.</div>\n')
304    ofile.write('      </div>\n')
305
306    ofile.write('      <div id="select_box">\n')
307    m_index = 0
308    for metric in sorted(tests.keys()):
309        ofile.write('        <div id="metric_link_' + str(m_index) + '" style="float:left;height:20px;width:200px"><a href="#">' + metric + '</a></div>\n')
310        m_index += 1
311    ofile.write('      </div>\n')
312    ofile.write('    </div>\n')
313
314    ofile.write('    <div style="clear:both"></div>\n')
315    ofile.write('    <div id="metric_list">\n')
316    m_index = 0
317    for metric in sorted(tests.keys()):
318        ofile.write('      <div class="metric_box" id="metric_box_' + str(m_index) + '">\n')
319        ofile.write('        <div class="metric_label">' + metric + '</div>\n')
320        e_index = 0
321        for encoding in sorted(tests[metric].keys()):
322            ofile.write('        <div class="container">\n')
323            ofile.write('          <div id="placeholder_' + str(m_index) + '_' + str(e_index) + '" class="placeholder"></div>\n')
324            ofile.write('        </div>\n')
325            e_index += 1
326
327        ofile.write('      </div>\n')
328        m_index += 1
329    ofile.write('      <div class="metric_box"></div>\n')
330    ofile.write('    </div>\n')
331    ofile.write('  </div>\n')
332    ofile.write('</body>\n')
333    ofile.write('</html>\n')
334    ofile.close()
335
336def col_index(label):
337    return headers[label]
338
339def get_value(row, label):
340    return row[col_index(label)].strip()
341
342def get_metric(row, label):
343    cell = row[col_index(label)]
344    if cell is None or cell is '':
345        cell = '0'
346    return cell.strip()
347
348def sanitize(dirName):
349    # Make the directory name valid as a javascript variable
350    newName = dirName.replace('.', '_')
351    return newName
352
353def get_headers(row):
354    index = 0
355    for column in row:
356        col = column.strip()
357        if col in headers:
358            header_dupes.append(col)
359        headers[col] = index
360        index += 1
361
362def print_headers():
363    for entry in headers:
364        print(entry + " " + str(headers[entry]))
365    for entry in header_dupes:
366        print("Found dupe: %s" % entry)
367
368def main():
369    for param in params:
370        param_id = param_name = param['id']
371        if ('dir' in param.keys()):
372            param_id = sanitize(param['dir'])
373        if ('display' in param.keys()):
374            param_name = param['display']
375        param_ids.append(param_id)
376        param_names.append(param_name)
377
378    for param in params:
379        uniqueId = param['id']
380        if ('dir' in param.keys()):
381            uniqueId = sanitize(param['dir'])
382
383        for rep in range(0, reps):
384            if ('dir' in param.keys()):
385                file_name = base_dir + '/' + param['dir'] + '/' + prefix + '_' + param['id'] + '_' + str(rep+1) + '.csv'
386            else:
387                file_name = base_dir + '/' + prefix + '_' + param['id'] + '_' + str(rep+1) + '.csv'
388            print "Processing: ", file_name
389            accumulate_values(file_name, rep, param, uniqueId)
390    write_html()
391    print('\nCreated: charts.html\n')
392
393if __name__ == "__main__":
394    main()
395
396