1#  Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
2#
3#  Use of this source code is governed by a BSD-style license
4#  that can be found in the LICENSE file in the root of the source
5#  tree. An additional intellectual property rights grant can be found
6#  in the file PATENTS.  All contributing project authors may
7#  be found in the AUTHORS file in the root of the source tree.
8
9"""Plots statistics from WebRTC integration test logs.
10
11Usage: $ python plot_webrtc_test_logs.py filename.txt
12"""
13
14import numpy
15import sys
16import re
17
18import matplotlib.pyplot as plt
19
20# Log events.
21EVENT_START = 'RUN      ] CodecSettings/VideoCodecTestParameterized.'
22EVENT_END = 'OK ] CodecSettings/VideoCodecTestParameterized.'
23
24# Metrics to plot, tuple: (name to parse in file, label to use when plotting).
25WIDTH = ('width', 'width')
26HEIGHT = ('height', 'height')
27FILENAME = ('filename', 'clip')
28CODEC_TYPE = ('codec_type', 'Codec')
29ENCODER_IMPLEMENTATION_NAME = ('enc_impl_name', 'enc name')
30DECODER_IMPLEMENTATION_NAME = ('dec_impl_name', 'dec name')
31CODEC_IMPLEMENTATION_NAME = ('codec_impl_name', 'codec name')
32CORES = ('num_cores', 'CPU cores used')
33DENOISING = ('denoising', 'denoising')
34RESILIENCE = ('resilience', 'resilience')
35ERROR_CONCEALMENT = ('error_concealment', 'error concealment')
36CPU_USAGE = ('cpu_usage_percent', 'CPU usage (%)')
37BITRATE = ('target_bitrate_kbps', 'target bitrate (kbps)')
38FRAMERATE = ('input_framerate_fps', 'fps')
39QP = ('avg_qp', 'QP avg')
40PSNR = ('avg_psnr', 'PSNR (dB)')
41SSIM = ('avg_ssim', 'SSIM')
42ENC_BITRATE = ('bitrate_kbps', 'encoded bitrate (kbps)')
43NUM_FRAMES = ('num_input_frames', 'num frames')
44NUM_DROPPED_FRAMES = ('num_dropped_frames', 'num dropped frames')
45TIME_TO_TARGET = ('time_to_reach_target_bitrate_sec',
46                  'time to reach target rate (sec)')
47ENCODE_SPEED_FPS = ('enc_speed_fps', 'encode speed (fps)')
48DECODE_SPEED_FPS = ('dec_speed_fps', 'decode speed (fps)')
49AVG_KEY_FRAME_SIZE = ('avg_key_frame_size_bytes', 'avg key frame size (bytes)')
50AVG_DELTA_FRAME_SIZE = ('avg_delta_frame_size_bytes',
51                        'avg delta frame size (bytes)')
52
53# Settings.
54SETTINGS = [
55  WIDTH,
56  HEIGHT,
57  FILENAME,
58  NUM_FRAMES,
59]
60
61# Settings, options for x-axis.
62X_SETTINGS = [
63  CORES,
64  FRAMERATE,
65  DENOISING,
66  RESILIENCE,
67  ERROR_CONCEALMENT,
68  BITRATE,  # TODO(asapersson): Needs to be last.
69]
70
71# Settings, options for subplots.
72SUBPLOT_SETTINGS = [
73  CODEC_TYPE,
74  ENCODER_IMPLEMENTATION_NAME,
75  DECODER_IMPLEMENTATION_NAME,
76  CODEC_IMPLEMENTATION_NAME,
77] + X_SETTINGS
78
79# Results.
80RESULTS = [
81  PSNR,
82  SSIM,
83  ENC_BITRATE,
84  NUM_DROPPED_FRAMES,
85  TIME_TO_TARGET,
86  ENCODE_SPEED_FPS,
87  DECODE_SPEED_FPS,
88  QP,
89  CPU_USAGE,
90  AVG_KEY_FRAME_SIZE,
91  AVG_DELTA_FRAME_SIZE,
92]
93
94METRICS_TO_PARSE = SETTINGS + SUBPLOT_SETTINGS + RESULTS
95
96Y_METRICS = [res[1] for res in RESULTS]
97
98# Parameters for plotting.
99FIG_SIZE_SCALE_FACTOR_X = 1.6
100FIG_SIZE_SCALE_FACTOR_Y = 1.8
101GRID_COLOR = [0.45, 0.45, 0.45]
102
103
104def ParseSetting(filename, setting):
105  """Parses setting from file.
106
107  Args:
108    filename: The name of the file.
109    setting: Name of setting to parse (e.g. width).
110
111  Returns:
112    A list holding parsed settings, e.g. ['width: 128.0', 'width: 160.0'] """
113
114  settings = []
115
116  settings_file = open(filename)
117  while True:
118    line = settings_file.readline()
119    if not line:
120      break
121    if re.search(r'%s' % EVENT_START, line):
122      # Parse event.
123      parsed = {}
124      while True:
125        line = settings_file.readline()
126        if not line:
127          break
128        if re.search(r'%s' % EVENT_END, line):
129          # Add parsed setting to list.
130          if setting in parsed:
131            s = setting + ': ' + str(parsed[setting])
132            if s not in settings:
133              settings.append(s)
134          break
135
136        TryFindMetric(parsed, line)
137
138  settings_file.close()
139  return settings
140
141
142def ParseMetrics(filename, setting1, setting2):
143  """Parses metrics from file.
144
145  Args:
146    filename: The name of the file.
147    setting1: First setting for sorting metrics (e.g. width).
148    setting2: Second setting for sorting metrics (e.g. CPU cores used).
149
150  Returns:
151    A dictionary holding parsed metrics.
152
153  For example:
154    metrics[key1][key2][measurement]
155
156  metrics = {
157  "width: 352": {
158    "CPU cores used: 1.0": {
159      "encode time (us)": [0.718005, 0.806925, 0.909726, 0.931835, 0.953642],
160      "PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551],
161      "bitrate (kbps)": [50, 100, 300, 500, 1000]
162    },
163    "CPU cores used: 2.0": {
164      "encode time (us)": [0.718005, 0.806925, 0.909726, 0.931835, 0.953642],
165      "PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551],
166      "bitrate (kbps)": [50, 100, 300, 500, 1000]
167    },
168  },
169  "width: 176": {
170    "CPU cores used: 1.0": {
171      "encode time (us)": [0.857897, 0.91608, 0.959173, 0.971116, 0.980961],
172      "PSNR (dB)": [30.243646, 33.375592, 37.574387, 39.42184, 41.437897],
173      "bitrate (kbps)": [50, 100, 300, 500, 1000]
174    },
175  }
176  } """
177
178  metrics = {}
179
180  # Parse events.
181  settings_file = open(filename)
182  while True:
183    line = settings_file.readline()
184    if not line:
185      break
186    if re.search(r'%s' % EVENT_START, line):
187      # Parse event.
188      parsed = {}
189      while True:
190        line = settings_file.readline()
191        if not line:
192          break
193        if re.search(r'%s' % EVENT_END, line):
194          # Add parsed values to metrics.
195          key1 = setting1 + ': ' + str(parsed[setting1])
196          key2 = setting2 + ': ' + str(parsed[setting2])
197          if key1 not in metrics:
198            metrics[key1] = {}
199          if key2 not in metrics[key1]:
200            metrics[key1][key2] = {}
201
202          for label in parsed:
203            if label not in metrics[key1][key2]:
204              metrics[key1][key2][label] = []
205            metrics[key1][key2][label].append(parsed[label])
206
207          break
208
209        TryFindMetric(parsed, line)
210
211  settings_file.close()
212  return metrics
213
214
215def TryFindMetric(parsed, line):
216  for metric in METRICS_TO_PARSE:
217    name = metric[0]
218    label = metric[1]
219    if re.search(r'%s' % name, line):
220      found, value = GetMetric(name, line)
221      if found:
222        parsed[label] = value
223      return
224
225
226def GetMetric(name, string):
227  # Float (e.g. bitrate = 98.8253).
228  pattern = r'%s\s*[:=]\s*([+-]?\d+\.*\d*)' % name
229  m = re.search(r'%s' % pattern, string)
230  if m is not None:
231    return StringToFloat(m.group(1))
232
233  # Alphanumeric characters (e.g. codec type : VP8).
234  pattern = r'%s\s*[:=]\s*(\w+)' % name
235  m = re.search(r'%s' % pattern, string)
236  if m is not None:
237    return True, m.group(1)
238
239  return False, -1
240
241
242def StringToFloat(value):
243  try:
244    value = float(value)
245  except ValueError:
246    print "Not a float, skipped %s" % value
247    return False, -1
248
249  return True, value
250
251
252def Plot(y_metric, x_metric, metrics):
253  """Plots y_metric vs x_metric per key in metrics.
254
255  For example:
256    y_metric = 'PSNR (dB)'
257    x_metric = 'bitrate (kbps)'
258    metrics = {
259      "CPU cores used: 1.0": {
260        "PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551],
261        "bitrate (kbps)": [50, 100, 300, 500, 1000]
262      },
263      "CPU cores used: 2.0": {
264        "PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551],
265        "bitrate (kbps)": [50, 100, 300, 500, 1000]
266      },
267    }
268    """
269  for key in sorted(metrics):
270    data = metrics[key]
271    if y_metric not in data:
272      print "Failed to find metric: %s" % y_metric
273      continue
274
275    y = numpy.array(data[y_metric])
276    x = numpy.array(data[x_metric])
277    if len(y) != len(x):
278      print "Length mismatch for %s, %s" % (y, x)
279      continue
280
281    label = y_metric + ' - ' + str(key)
282
283    plt.plot(x, y, label=label, linewidth=1.5, marker='o', markersize=5,
284             markeredgewidth=0.0)
285
286
287def PlotFigure(settings, y_metrics, x_metric, metrics, title):
288  """Plots metrics in y_metrics list. One figure is plotted and each entry
289  in the list is plotted in a subplot (and sorted per settings).
290
291  For example:
292    settings = ['width: 128.0', 'width: 160.0']. Sort subplot per setting.
293    y_metrics = ['PSNR (dB)', 'PSNR (dB)']. Metric to plot per subplot.
294    x_metric = 'bitrate (kbps)'
295
296  """
297
298  plt.figure()
299  plt.suptitle(title, fontsize='large', fontweight='bold')
300  settings.sort()
301  rows = len(settings)
302  cols = 1
303  pos = 1
304  while pos <= rows:
305    plt.rc('grid', color=GRID_COLOR)
306    ax = plt.subplot(rows, cols, pos)
307    plt.grid()
308    plt.setp(ax.get_xticklabels(), visible=(pos == rows), fontsize='large')
309    plt.setp(ax.get_yticklabels(), fontsize='large')
310    setting = settings[pos - 1]
311    Plot(y_metrics[pos - 1], x_metric, metrics[setting])
312    if setting.startswith(WIDTH[1]):
313      plt.title(setting, fontsize='medium')
314    plt.legend(fontsize='large', loc='best')
315    pos += 1
316
317  plt.xlabel(x_metric, fontsize='large')
318  plt.subplots_adjust(left=0.06, right=0.98, bottom=0.05, top=0.94, hspace=0.08)
319
320
321def GetTitle(filename, setting):
322  title = ''
323  if setting != CODEC_IMPLEMENTATION_NAME[1] and setting != CODEC_TYPE[1]:
324    codec_types = ParseSetting(filename, CODEC_TYPE[1])
325    for i in range(0, len(codec_types)):
326      title += codec_types[i] + ', '
327
328  if setting != CORES[1]:
329    cores = ParseSetting(filename, CORES[1])
330    for i in range(0, len(cores)):
331      title += cores[i].split('.')[0] + ', '
332
333  if setting != FRAMERATE[1]:
334    framerate = ParseSetting(filename, FRAMERATE[1])
335    for i in range(0, len(framerate)):
336      title += framerate[i].split('.')[0] + ', '
337
338  if (setting != CODEC_IMPLEMENTATION_NAME[1] and
339      setting != ENCODER_IMPLEMENTATION_NAME[1]):
340    enc_names = ParseSetting(filename, ENCODER_IMPLEMENTATION_NAME[1])
341    for i in range(0, len(enc_names)):
342      title += enc_names[i] + ', '
343
344  if (setting != CODEC_IMPLEMENTATION_NAME[1] and
345      setting != DECODER_IMPLEMENTATION_NAME[1]):
346    dec_names = ParseSetting(filename, DECODER_IMPLEMENTATION_NAME[1])
347    for i in range(0, len(dec_names)):
348      title += dec_names[i] + ', '
349
350  filenames = ParseSetting(filename, FILENAME[1])
351  title += filenames[0].split('_')[0]
352
353  num_frames = ParseSetting(filename, NUM_FRAMES[1])
354  for i in range(0, len(num_frames)):
355    title += ' (' + num_frames[i].split('.')[0] + ')'
356
357  return title
358
359
360def ToString(input_list):
361  return ToStringWithoutMetric(input_list, ('', ''))
362
363
364def ToStringWithoutMetric(input_list, metric):
365  i = 1
366  output_str = ""
367  for m in input_list:
368    if m != metric:
369      output_str = output_str + ("%s. %s\n" % (i, m[1]))
370      i += 1
371  return output_str
372
373
374def GetIdx(text_list):
375  return int(raw_input(text_list)) - 1
376
377
378def main():
379  filename = sys.argv[1]
380
381  # Setup.
382  idx_metric = GetIdx("Choose metric:\n0. All\n%s" % ToString(RESULTS))
383  if idx_metric == -1:
384    # Plot all metrics. One subplot for each metric.
385    # Per subplot: metric vs bitrate (per resolution).
386    cores = ParseSetting(filename, CORES[1])
387    setting1 = CORES[1]
388    setting2 = WIDTH[1]
389    sub_keys = [cores[0]] * len(Y_METRICS)
390    y_metrics = Y_METRICS
391    x_metric = BITRATE[1]
392  else:
393    resolutions = ParseSetting(filename, WIDTH[1])
394    idx = GetIdx("Select metric for x-axis:\n%s" % ToString(X_SETTINGS))
395    if X_SETTINGS[idx] == BITRATE:
396      idx = GetIdx("Plot per:\n%s" % ToStringWithoutMetric(SUBPLOT_SETTINGS,
397                                                           BITRATE))
398      idx_setting = METRICS_TO_PARSE.index(SUBPLOT_SETTINGS[idx])
399      # Plot one metric. One subplot for each resolution.
400      # Per subplot: metric vs bitrate (per setting).
401      setting1 = WIDTH[1]
402      setting2 = METRICS_TO_PARSE[idx_setting][1]
403      sub_keys = resolutions
404      y_metrics = [RESULTS[idx_metric][1]] * len(sub_keys)
405      x_metric = BITRATE[1]
406    else:
407      # Plot one metric. One subplot for each resolution.
408      # Per subplot: metric vs setting (per bitrate).
409      setting1 = WIDTH[1]
410      setting2 = BITRATE[1]
411      sub_keys = resolutions
412      y_metrics = [RESULTS[idx_metric][1]] * len(sub_keys)
413      x_metric = X_SETTINGS[idx][1]
414
415  metrics = ParseMetrics(filename, setting1, setting2)
416
417  # Stretch fig size.
418  figsize = plt.rcParams["figure.figsize"]
419  figsize[0] *= FIG_SIZE_SCALE_FACTOR_X
420  figsize[1] *= FIG_SIZE_SCALE_FACTOR_Y
421  plt.rcParams["figure.figsize"] = figsize
422
423  PlotFigure(sub_keys, y_metrics, x_metric, metrics,
424             GetTitle(filename, setting2))
425
426  plt.show()
427
428
429if __name__ == '__main__':
430  main()
431