1#  Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
2#
3#  Use of this source code is governed by a BSD-style license
4#  that can be found in the LICENSE file in the root of the source
5#  tree. An additional intellectual property rights grant can be found
6#  in the file PATENTS.  All contributing project authors may
7#  be found in the AUTHORS file in the root of the source tree.
8
9"""Plots statistics from WebRTC integration test logs.
10
11Usage: $ python plot_webrtc_test_logs.py filename.txt
12"""
13
14import numpy
15import sys
16import re
17
18import matplotlib.pyplot as plt
19
20# Log events.
21EVENT_START = \
22  'RUN      ] CodecSettings/VideoProcessorIntegrationTestParameterized.'
23EVENT_END = 'OK ] CodecSettings/VideoProcessorIntegrationTestParameterized.'
24
25# Metrics to plot, tuple: (name to parse in file, label to use when plotting).
26BITRATE = ('Target bitrate', 'target bitrate (kbps)')
27WIDTH = ('Width', 'width')
28HEIGHT = ('Height', 'height')
29FILENAME = ('Filename', 'clip')
30CODEC_TYPE = ('Codec type', 'Codec')
31ENCODER_IMPLEMENTATION_NAME = ('Encoder implementation name', 'enc name')
32DECODER_IMPLEMENTATION_NAME = ('Decoder implementation name', 'dec name')
33CODEC_IMPLEMENTATION_NAME = ('Codec implementation name', 'codec name')
34CORES = ('# CPU cores used', 'CPU cores used')
35DENOISING = ('Denoising', 'denoising')
36RESILIENCE = ('Resilience', 'resilience')
37ERROR_CONCEALMENT = ('Error concealment', 'error concealment')
38QP = ('Average QP', 'avg QP')
39CPU_USAGE = ('CPU usage %', 'CPU usage (%)')
40PSNR = ('PSNR avg', 'PSNR (dB)')
41SSIM = ('SSIM avg', 'SSIM')
42ENC_BITRATE = ('Encoded bitrate', 'encoded bitrate (kbps)')
43FRAMERATE = ('Frame rate', 'fps')
44NUM_FRAMES = ('# processed frames', 'num frames')
45NUM_DROPPED_FRAMES = ('# dropped frames', 'num dropped frames')
46NUM_FRAMES_TO_TARGET = ('# frames to convergence',
47                        'frames to reach target rate')
48ENCODE_TIME = ('Encoding time', 'encode time (us)')
49ENCODE_TIME_AVG = ('Encoding time', 'encode time (us) avg')
50DECODE_TIME = ('Decoding time', 'decode time (us)')
51DECODE_TIME_AVG = ('Decoding time', 'decode time (us) avg')
52FRAME_SIZE = ('Frame sizes', 'frame size (bytes)')
53FRAME_SIZE_AVG = ('Frame sizes', 'frame size (bytes) avg')
54AVG_KEY_FRAME_SIZE = ('Average key frame size', 'avg key frame size (bytes)')
55AVG_NON_KEY_FRAME_SIZE = ('Average non-key frame size',
56                          'avg non-key frame size (bytes)')
57
58# Settings.
59SETTINGS = [
60  WIDTH,
61  HEIGHT,
62  FILENAME,
63  NUM_FRAMES,
64  ENCODE_TIME,
65  DECODE_TIME,
66  FRAME_SIZE,
67]
68
69# Settings, options for x-axis.
70X_SETTINGS = [
71  CORES,
72  FRAMERATE,
73  DENOISING,
74  RESILIENCE,
75  ERROR_CONCEALMENT,
76  BITRATE,  # TODO(asapersson): Needs to be last.
77]
78
79# Settings, options for subplots.
80SUBPLOT_SETTINGS = [
81  CODEC_TYPE,
82  ENCODER_IMPLEMENTATION_NAME,
83  DECODER_IMPLEMENTATION_NAME,
84  CODEC_IMPLEMENTATION_NAME,
85] + X_SETTINGS
86
87# Results.
88RESULTS = [
89  PSNR,
90  SSIM,
91  ENC_BITRATE,
92  NUM_DROPPED_FRAMES,
93  NUM_FRAMES_TO_TARGET,
94  ENCODE_TIME_AVG,
95  DECODE_TIME_AVG,
96  QP,
97  CPU_USAGE,
98  AVG_KEY_FRAME_SIZE,
99  AVG_NON_KEY_FRAME_SIZE,
100]
101
102METRICS_TO_PARSE = SETTINGS + SUBPLOT_SETTINGS + RESULTS
103
104Y_METRICS = [res[1] for res in RESULTS]
105
106# Parameters for plotting.
107FIG_SIZE_SCALE_FACTOR_X = 1.6
108FIG_SIZE_SCALE_FACTOR_Y = 1.8
109GRID_COLOR = [0.45, 0.45, 0.45]
110
111
112def ParseSetting(filename, setting):
113  """Parses setting from file.
114
115  Args:
116    filename: The name of the file.
117    setting: Name of setting to parse (e.g. width).
118
119  Returns:
120    A list holding parsed settings, e.g. ['width: 128.0', 'width: 160.0'] """
121
122  settings = []
123
124  settings_file = open(filename)
125  while True:
126    line = settings_file.readline()
127    if not line:
128      break
129    if re.search(r'%s' % EVENT_START, line):
130      # Parse event.
131      parsed = {}
132      while True:
133        line = settings_file.readline()
134        if not line:
135          break
136        if re.search(r'%s' % EVENT_END, line):
137          # Add parsed setting to list.
138          if setting in parsed:
139            s = setting + ': ' + str(parsed[setting])
140            if s not in settings:
141              settings.append(s)
142          break
143
144        TryFindMetric(parsed, line, settings_file)
145
146  settings_file.close()
147  return settings
148
149
150def ParseMetrics(filename, setting1, setting2):
151  """Parses metrics from file.
152
153  Args:
154    filename: The name of the file.
155    setting1: First setting for sorting metrics (e.g. width).
156    setting2: Second setting for sorting metrics (e.g. CPU cores used).
157
158  Returns:
159    A dictionary holding parsed metrics.
160
161  For example:
162    metrics[key1][key2][measurement]
163
164  metrics = {
165  "width: 352": {
166    "CPU cores used: 1.0": {
167      "encode time (us)": [0.718005, 0.806925, 0.909726, 0.931835, 0.953642],
168      "PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551],
169      "bitrate (kbps)": [50, 100, 300, 500, 1000]
170    },
171    "CPU cores used: 2.0": {
172      "encode time (us)": [0.718005, 0.806925, 0.909726, 0.931835, 0.953642],
173      "PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551],
174      "bitrate (kbps)": [50, 100, 300, 500, 1000]
175    },
176  },
177  "width: 176": {
178    "CPU cores used: 1.0": {
179      "encode time (us)": [0.857897, 0.91608, 0.959173, 0.971116, 0.980961],
180      "PSNR (dB)": [30.243646, 33.375592, 37.574387, 39.42184, 41.437897],
181      "bitrate (kbps)": [50, 100, 300, 500, 1000]
182    },
183  }
184  } """
185
186  metrics = {}
187
188  # Parse events.
189  settings_file = open(filename)
190  while True:
191    line = settings_file.readline()
192    if not line:
193      break
194    if re.search(r'%s' % EVENT_START, line):
195      # Parse event.
196      parsed = {}
197      while True:
198        line = settings_file.readline()
199        if not line:
200          break
201        if re.search(r'%s' % EVENT_END, line):
202          # Add parsed values to metrics.
203          key1 = setting1 + ': ' + str(parsed[setting1])
204          key2 = setting2 + ': ' + str(parsed[setting2])
205          if key1 not in metrics:
206            metrics[key1] = {}
207          if key2 not in metrics[key1]:
208            metrics[key1][key2] = {}
209
210          for label in parsed:
211            if label not in metrics[key1][key2]:
212              metrics[key1][key2][label] = []
213            metrics[key1][key2][label].append(parsed[label])
214
215          break
216
217        TryFindMetric(parsed, line, settings_file)
218
219  settings_file.close()
220  return metrics
221
222
223def TryFindMetric(parsed, line, settings_file):
224  for metric in METRICS_TO_PARSE:
225    name = metric[0]
226    label = metric[1]
227    if re.search(r'%s' % name, line):
228      found, value = GetMetric(name, line)
229      if not found:
230        # TODO(asapersson): Change format.
231        # Try find min, max, average stats.
232        found, minimum = GetMetric("Min", settings_file.readline())
233        if not found:
234          return
235        found, maximum = GetMetric("Max", settings_file.readline())
236        if not found:
237          return
238        found, average = GetMetric("Average", settings_file.readline())
239        if not found:
240          return
241
242        parsed[label + ' min'] = minimum
243        parsed[label + ' max'] = maximum
244        parsed[label + ' avg'] = average
245
246      parsed[label] = value
247      return
248
249
250def GetMetric(name, string):
251  # Float (e.g. bitrate = 98.8253).
252  pattern = r'%s\s*[:=]\s*([+-]?\d+\.*\d*)' % name
253  m = re.search(r'%s' % pattern, string)
254  if m is not None:
255    return StringToFloat(m.group(1))
256
257  # Alphanumeric characters (e.g. codec type : VP8).
258  pattern = r'%s\s*[:=]\s*(\w+)' % name
259  m = re.search(r'%s' % pattern, string)
260  if m is not None:
261    return True, m.group(1)
262
263  return False, -1
264
265
266def StringToFloat(value):
267  try:
268    value = float(value)
269  except ValueError:
270    print "Not a float, skipped %s" % value
271    return False, -1
272
273  return True, value
274
275
276def Plot(y_metric, x_metric, metrics):
277  """Plots y_metric vs x_metric per key in metrics.
278
279  For example:
280    y_metric = 'PSNR (dB)'
281    x_metric = 'bitrate (kbps)'
282    metrics = {
283      "CPU cores used: 1.0": {
284        "PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551],
285        "bitrate (kbps)": [50, 100, 300, 500, 1000]
286      },
287      "CPU cores used: 2.0": {
288        "PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551],
289        "bitrate (kbps)": [50, 100, 300, 500, 1000]
290      },
291    }
292    """
293  for key in sorted(metrics):
294    data = metrics[key]
295    if y_metric not in data:
296      print "Failed to find metric: %s" % y_metric
297      continue
298
299    y = numpy.array(data[y_metric])
300    x = numpy.array(data[x_metric])
301    if len(y) != len(x):
302      print "Length mismatch for %s, %s" % (y, x)
303      continue
304
305    label = y_metric + ' - ' + str(key)
306
307    plt.plot(x, y, label=label, linewidth=1.5, marker='o', markersize=5,
308             markeredgewidth=0.0)
309
310
311def PlotFigure(settings, y_metrics, x_metric, metrics, title):
312  """Plots metrics in y_metrics list. One figure is plotted and each entry
313  in the list is plotted in a subplot (and sorted per settings).
314
315  For example:
316    settings = ['width: 128.0', 'width: 160.0']. Sort subplot per setting.
317    y_metrics = ['PSNR (dB)', 'PSNR (dB)']. Metric to plot per subplot.
318    x_metric = 'bitrate (kbps)'
319
320  """
321
322  plt.figure()
323  plt.suptitle(title, fontsize='large', fontweight='bold')
324  settings.sort()
325  rows = len(settings)
326  cols = 1
327  pos = 1
328  while pos <= rows:
329    plt.rc('grid', color=GRID_COLOR)
330    ax = plt.subplot(rows, cols, pos)
331    plt.grid()
332    plt.setp(ax.get_xticklabels(), visible=(pos == rows), fontsize='large')
333    plt.setp(ax.get_yticklabels(), fontsize='large')
334    setting = settings[pos - 1]
335    Plot(y_metrics[pos - 1], x_metric, metrics[setting])
336    if setting.startswith(WIDTH[1]):
337      plt.title(setting, fontsize='medium')
338    plt.legend(fontsize='large', loc='best')
339    pos += 1
340
341  plt.xlabel(x_metric, fontsize='large')
342  plt.subplots_adjust(left=0.06, right=0.98, bottom=0.05, top=0.94, hspace=0.08)
343
344
345def GetTitle(filename, setting):
346  title = ''
347  if setting != CODEC_IMPLEMENTATION_NAME[1] and setting != CODEC_TYPE[1]:
348    codec_types = ParseSetting(filename, CODEC_TYPE[1])
349    for i in range(0, len(codec_types)):
350      title += codec_types[i] + ', '
351
352  if setting != CORES[1]:
353    cores = ParseSetting(filename, CORES[1])
354    for i in range(0, len(cores)):
355      title += cores[i].split('.')[0] + ', '
356
357  if setting != FRAMERATE[1]:
358    framerate = ParseSetting(filename, FRAMERATE[1])
359    for i in range(0, len(framerate)):
360      title += framerate[i].split('.')[0] + ', '
361
362  if (setting != CODEC_IMPLEMENTATION_NAME[1] and
363      setting != ENCODER_IMPLEMENTATION_NAME[1]):
364    enc_names = ParseSetting(filename, ENCODER_IMPLEMENTATION_NAME[1])
365    for i in range(0, len(enc_names)):
366      title += enc_names[i] + ', '
367
368  if (setting != CODEC_IMPLEMENTATION_NAME[1] and
369      setting != DECODER_IMPLEMENTATION_NAME[1]):
370    dec_names = ParseSetting(filename, DECODER_IMPLEMENTATION_NAME[1])
371    for i in range(0, len(dec_names)):
372      title += dec_names[i] + ', '
373
374  filenames = ParseSetting(filename, FILENAME[1])
375  title += filenames[0].split('_')[0]
376
377  num_frames = ParseSetting(filename, NUM_FRAMES[1])
378  for i in range(0, len(num_frames)):
379    title += ' (' + num_frames[i].split('.')[0] + ')'
380
381  return title
382
383
384def ToString(input_list):
385  return ToStringWithoutMetric(input_list, ('', ''))
386
387
388def ToStringWithoutMetric(input_list, metric):
389  i = 1
390  output_str = ""
391  for m in input_list:
392    if m != metric:
393      output_str = output_str + ("%s. %s\n" % (i, m[1]))
394      i += 1
395  return output_str
396
397
398def GetIdx(text_list):
399  return int(raw_input(text_list)) - 1
400
401
402def main():
403  filename = sys.argv[1]
404
405  # Setup.
406  idx_metric = GetIdx("Choose metric:\n0. All\n%s" % ToString(RESULTS))
407  if idx_metric == -1:
408    # Plot all metrics. One subplot for each metric.
409    # Per subplot: metric vs bitrate (per resolution).
410    cores = ParseSetting(filename, CORES[1])
411    setting1 = CORES[1]
412    setting2 = WIDTH[1]
413    sub_keys = [cores[0]] * len(Y_METRICS)
414    y_metrics = Y_METRICS
415    x_metric = BITRATE[1]
416  else:
417    resolutions = ParseSetting(filename, WIDTH[1])
418    idx = GetIdx("Select metric for x-axis:\n%s" % ToString(X_SETTINGS))
419    if X_SETTINGS[idx] == BITRATE:
420      idx = GetIdx("Plot per:\n%s" % ToStringWithoutMetric(SUBPLOT_SETTINGS,
421                                                           BITRATE))
422      idx_setting = METRICS_TO_PARSE.index(SUBPLOT_SETTINGS[idx])
423      # Plot one metric. One subplot for each resolution.
424      # Per subplot: metric vs bitrate (per setting).
425      setting1 = WIDTH[1]
426      setting2 = METRICS_TO_PARSE[idx_setting][1]
427      sub_keys = resolutions
428      y_metrics = [RESULTS[idx_metric][1]] * len(sub_keys)
429      x_metric = BITRATE[1]
430    else:
431      # Plot one metric. One subplot for each resolution.
432      # Per subplot: metric vs setting (per bitrate).
433      setting1 = WIDTH[1]
434      setting2 = BITRATE[1]
435      sub_keys = resolutions
436      y_metrics = [RESULTS[idx_metric][1]] * len(sub_keys)
437      x_metric = X_SETTINGS[idx][1]
438
439  metrics = ParseMetrics(filename, setting1, setting2)
440
441  # Stretch fig size.
442  figsize = plt.rcParams["figure.figsize"]
443  figsize[0] *= FIG_SIZE_SCALE_FACTOR_X
444  figsize[1] *= FIG_SIZE_SCALE_FACTOR_Y
445  plt.rcParams["figure.figsize"] = figsize
446
447  PlotFigure(sub_keys, y_metrics, x_metric, metrics,
448             GetTitle(filename, setting2))
449
450  plt.show()
451
452
453if __name__ == '__main__':
454  main()
455