1#!/usr/bin/env python
2# Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3#
4# Use of this source code is governed by a BSD-style license
5# that can be found in the LICENSE file in the root of the source
6# tree. An additional intellectual property rights grant can be found
7# in the file PATENTS.  All contributing project authors may
8# be found in the AUTHORS file in the root of the source tree.
9
10"""Generate graphs for data generated by loopback tests.
11
12Usage examples:
13  Show end to end time for a single full stack test.
14  ./full_stack_tests_plot.py -df end_to_end -o 600 --frames 1000 vp9_data.txt
15
16  Show simultaneously PSNR and encoded frame size for two different runs of
17  full stack test. Averaged over a cycle of 200 frames. Used e.g. for
18  screenshare slide test.
19  ./full_stack_tests_plot.py -c 200 -df psnr -drf encoded_frame_size \\
20                             before.txt after.txt
21
22  Similar to the previous test, but multiple graphs.
23  ./full_stack_tests_plot.py -c 200 -df psnr vp8.txt vp9.txt --next \\
24                             -c 200 -df sender_time vp8.txt vp9.txt --next \\
25                             -c 200 -df end_to_end vp8.txt vp9.txt
26"""
27
28import argparse
29from collections import defaultdict
30import itertools
31import sys
32import matplotlib.pyplot as plt
33import numpy
34
35# Fields
36DROPPED = 0
37INPUT_TIME = 1              # ms (timestamp)
38SEND_TIME = 2               # ms (timestamp)
39RECV_TIME = 3               # ms (timestamp)
40RENDER_TIME = 4             # ms (timestamp)
41ENCODED_FRAME_SIZE = 5      # bytes
42PSNR = 6
43SSIM = 7
44ENCODE_TIME = 8             # ms (time interval)
45
46TOTAL_RAW_FIELDS = 9
47
48SENDER_TIME = TOTAL_RAW_FIELDS + 0
49RECEIVER_TIME = TOTAL_RAW_FIELDS + 1
50END_TO_END = TOTAL_RAW_FIELDS + 2
51RENDERED_DELTA = TOTAL_RAW_FIELDS + 3
52
53FIELD_MASK = 255
54
55# Options
56HIDE_DROPPED = 256
57RIGHT_Y_AXIS = 512
58
59# internal field id, field name, title
60_FIELDS = [
61    # Raw
62    (DROPPED, "dropped", "dropped"),
63    (INPUT_TIME, "input_time_ms", "input time"),
64    (SEND_TIME, "send_time_ms", "send time"),
65    (RECV_TIME, "recv_time_ms", "recv time"),
66    (ENCODED_FRAME_SIZE, "encoded_frame_size", "encoded frame size"),
67    (PSNR, "psnr", "PSNR"),
68    (SSIM, "ssim", "SSIM"),
69    (RENDER_TIME, "render_time_ms", "render time"),
70    (ENCODE_TIME, "encode_time_ms", "encode time"),
71    # Auto-generated
72    (SENDER_TIME, "sender_time", "sender time"),
73    (RECEIVER_TIME, "receiver_time", "receiver time"),
74    (END_TO_END, "end_to_end", "end to end"),
75    (RENDERED_DELTA, "rendered_delta", "rendered delta"),
76]
77
78NAME_TO_ID = {field[1]: field[0] for field in _FIELDS}
79ID_TO_TITLE = {field[0]: field[2] for field in _FIELDS}
80
81def FieldArgToId(arg):
82  if arg == "none":
83    return None
84  if arg in NAME_TO_ID:
85    return NAME_TO_ID[arg]
86  if arg + "_ms" in NAME_TO_ID:
87    return NAME_TO_ID[arg + "_ms"]
88  raise Exception("Unrecognized field name \"{}\"".format(arg))
89
90
91class PlotLine(object):
92  """Data for a single graph line."""
93
94  def __init__(self, label, values, flags):
95    self.label = label
96    self.values = values
97    self.flags = flags
98
99
100class Data(object):
101  """Object representing one full stack test."""
102
103  def __init__(self, filename):
104    self.title = ""
105    self.length = 0
106    self.samples = defaultdict(list)
107
108    self._ReadSamples(filename)
109
110  def _ReadSamples(self, filename):
111    """Reads graph data from the given file."""
112    f = open(filename)
113    it = iter(f)
114
115    self.title = it.next().strip()
116    self.length = int(it.next())
117    field_names = [name.strip() for name in it.next().split()]
118    field_ids = [NAME_TO_ID[name] for name in field_names]
119
120    for field_id in field_ids:
121      self.samples[field_id] = [0.0] * self.length
122
123    for sample_id in xrange(self.length):
124      for col, value in enumerate(it.next().split()):
125        self.samples[field_ids[col]][sample_id] = float(value)
126
127    self._SubtractFirstInputTime()
128    self._GenerateAdditionalData()
129
130    f.close()
131
132  def _SubtractFirstInputTime(self):
133    offset = self.samples[INPUT_TIME][0]
134    for field in [INPUT_TIME, SEND_TIME, RECV_TIME, RENDER_TIME]:
135      if field in self.samples:
136        self.samples[field] = [x - offset for x in self.samples[field]]
137
138  def _GenerateAdditionalData(self):
139    """Calculates sender time, receiver time etc. from the raw data."""
140    s = self.samples
141    last_render_time = 0
142    for field_id in [SENDER_TIME, RECEIVER_TIME, END_TO_END, RENDERED_DELTA]:
143      s[field_id] = [0] * self.length
144
145    for k in range(self.length):
146      s[SENDER_TIME][k] = s[SEND_TIME][k] - s[INPUT_TIME][k]
147
148      decoded_time = s[RENDER_TIME][k]
149      s[RECEIVER_TIME][k] = decoded_time - s[RECV_TIME][k]
150      s[END_TO_END][k] = decoded_time - s[INPUT_TIME][k]
151      if not s[DROPPED][k]:
152        if k > 0:
153          s[RENDERED_DELTA][k] = decoded_time - last_render_time
154        last_render_time = decoded_time
155
156  def _Hide(self, values):
157    """
158    Replaces values for dropped frames with None.
159    These values are then skipped by the Plot() method.
160    """
161
162    return [None if self.samples[DROPPED][k] else values[k]
163            for k in range(len(values))]
164
165  def AddSamples(self, config, target_lines_list):
166    """Creates graph lines from the current data set with given config."""
167    for field in config.fields:
168      # field is None means the user wants just to skip the color.
169      if field is None:
170        target_lines_list.append(None)
171        continue
172
173      field_id = field & FIELD_MASK
174      values = self.samples[field_id]
175
176      if field & HIDE_DROPPED:
177        values = self._Hide(values)
178
179      target_lines_list.append(PlotLine(
180          self.title + " " + ID_TO_TITLE[field_id],
181          values, field & ~FIELD_MASK))
182
183
184def AverageOverCycle(values, length):
185  """
186  Returns the list:
187    [
188        avg(values[0], values[length], ...),
189        avg(values[1], values[length + 1], ...),
190        ...
191        avg(values[length - 1], values[2 * length - 1], ...),
192    ]
193
194  Skips None values when calculating the average value.
195  """
196
197  total = [0.0] * length
198  count = [0] * length
199  for k, val in enumerate(values):
200    if val is not None:
201      total[k % length] += val
202      count[k % length] += 1
203
204  result = [0.0] * length
205  for k in range(length):
206    result[k] = total[k] / count[k] if count[k] else None
207  return result
208
209
210class PlotConfig(object):
211  """Object representing a single graph."""
212
213  def __init__(self, fields, data_list, cycle_length=None, frames=None,
214               offset=0, output_filename=None, title="Graph"):
215    self.fields = fields
216    self.data_list = data_list
217    self.cycle_length = cycle_length
218    self.frames = frames
219    self.offset = offset
220    self.output_filename = output_filename
221    self.title = title
222
223  def Plot(self, ax1):
224    lines = []
225    for data in self.data_list:
226      if not data:
227        # Add None lines to skip the colors.
228        lines.extend([None] * len(self.fields))
229      else:
230        data.AddSamples(self, lines)
231
232    def _SliceValues(values):
233      if self.offset:
234        values = values[self.offset:]
235      if self.frames:
236        values = values[:self.frames]
237      return values
238
239    length = None
240    for line in lines:
241      if line is None:
242        continue
243
244      line.values = _SliceValues(line.values)
245      if self.cycle_length:
246        line.values = AverageOverCycle(line.values, self.cycle_length)
247
248      if length is None:
249        length = len(line.values)
250      elif length != len(line.values):
251        raise Exception("All arrays should have the same length!")
252
253    ax1.set_xlabel("Frame", fontsize="large")
254    if any(line.flags & RIGHT_Y_AXIS for line in lines if line):
255      ax2 = ax1.twinx()
256      ax2.set_xlabel("Frame", fontsize="large")
257    else:
258      ax2 = None
259
260    # Have to implement color_cycle manually, due to two scales in a graph.
261    color_cycle = ["b", "r", "g", "c", "m", "y", "k"]
262    color_iter = itertools.cycle(color_cycle)
263
264    for line in lines:
265      if not line:
266        color_iter.next()
267        continue
268
269      if self.cycle_length:
270        x = numpy.array(range(self.cycle_length))
271      else:
272        x = numpy.array(range(self.offset, self.offset + len(line.values)))
273      y = numpy.array(line.values)
274      ax = ax2 if line.flags & RIGHT_Y_AXIS else ax1
275      ax.Plot(x, y, "o-", label=line.label, markersize=3.0, linewidth=1.0,
276              color=color_iter.next())
277
278    ax1.grid(True)
279    if ax2:
280      ax1.legend(loc="upper left", shadow=True, fontsize="large")
281      ax2.legend(loc="upper right", shadow=True, fontsize="large")
282    else:
283      ax1.legend(loc="best", shadow=True, fontsize="large")
284
285
286def LoadFiles(filenames):
287  result = []
288  for filename in filenames:
289    if filename in LoadFiles.cache:
290      result.append(LoadFiles.cache[filename])
291    else:
292      data = Data(filename)
293      LoadFiles.cache[filename] = data
294      result.append(data)
295  return result
296LoadFiles.cache = {}
297
298
299def GetParser():
300  class CustomAction(argparse.Action):
301    def __call__(self, parser, namespace, values, option_string=None):
302      if "ordered_args" not in namespace:
303        namespace.ordered_args = []
304      namespace.ordered_args.append((self.dest, values))
305
306  parser = argparse.ArgumentParser(
307      description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
308
309  parser.add_argument(
310      "-c", "--cycle_length", nargs=1, action=CustomAction,
311      type=int, help="Cycle length over which to average the values.")
312  parser.add_argument(
313      "-f", "--field", nargs=1, action=CustomAction,
314      help="Name of the field to show. Use 'none' to skip a color.")
315  parser.add_argument("-r", "--right", nargs=0, action=CustomAction,
316                      help="Use right Y axis for given field.")
317  parser.add_argument("-d", "--drop", nargs=0, action=CustomAction,
318                      help="Hide values for dropped frames.")
319  parser.add_argument("-o", "--offset", nargs=1, action=CustomAction, type=int,
320                      help="Frame offset.")
321  parser.add_argument("-n", "--next", nargs=0, action=CustomAction,
322                      help="Separator for multiple graphs.")
323  parser.add_argument(
324      "--frames", nargs=1, action=CustomAction, type=int,
325      help="Frame count to show or take into account while averaging.")
326  parser.add_argument("-t", "--title", nargs=1, action=CustomAction,
327                      help="Title of the graph.")
328  parser.add_argument(
329      "-O", "--output_filename", nargs=1, action=CustomAction,
330      help="Use to save the graph into a file. "
331           "Otherwise, a window will be shown.")
332  parser.add_argument(
333      "files", nargs="+", action=CustomAction,
334      help="List of text-based files generated by loopback tests.")
335  return parser
336
337
338def _PlotConfigFromArgs(args, graph_num):
339  # Pylint complains about using kwargs, so have to do it this way.
340  cycle_length = None
341  frames = None
342  offset = 0
343  output_filename = None
344  title = "Graph"
345
346  fields = []
347  files = []
348  mask = 0
349  for key, values in args:
350    if key == "cycle_length":
351      cycle_length = values[0]
352    elif key == "frames":
353      frames = values[0]
354    elif key == "offset":
355      offset = values[0]
356    elif key == "output_filename":
357      output_filename = values[0]
358    elif key == "title":
359      title = values[0]
360    elif key == "drop":
361      mask |= HIDE_DROPPED
362    elif key == "right":
363      mask |= RIGHT_Y_AXIS
364    elif key == "field":
365      field_id = FieldArgToId(values[0])
366      fields.append(field_id | mask if field_id is not None else None)
367      mask = 0  # Reset mask after the field argument.
368    elif key == "files":
369      files.extend(values)
370
371  if not files:
372    raise Exception("Missing file argument(s) for graph #{}".format(graph_num))
373  if not fields:
374    raise Exception("Missing field argument(s) for graph #{}".format(graph_num))
375
376  return PlotConfig(fields, LoadFiles(files), cycle_length=cycle_length,
377      frames=frames, offset=offset, output_filename=output_filename,
378      title=title)
379
380
381def PlotConfigsFromArgs(args):
382  """Generates plot configs for given command line arguments."""
383  # The way it works:
384  #   First we detect separators -n/--next and split arguments into groups, one
385  #   for each plot. For each group, we partially parse it with
386  #   argparse.ArgumentParser, modified to remember the order of arguments.
387  #   Then we traverse the argument list and fill the PlotConfig.
388  args = itertools.groupby(args, lambda x: x in ["-n", "--next"])
389  prep_args = list(list(group) for match, group in args if not match)
390
391  parser = GetParser()
392  plot_configs = []
393  for index, raw_args in enumerate(prep_args):
394    graph_args = parser.parse_args(raw_args).ordered_args
395    plot_configs.append(_PlotConfigFromArgs(graph_args, index))
396  return plot_configs
397
398
399def ShowOrSavePlots(plot_configs):
400  for config in plot_configs:
401    fig = plt.figure(figsize=(14.0, 10.0))
402    ax = fig.add_subPlot(1, 1, 1)
403
404    plt.title(config.title)
405    config.Plot(ax)
406    if config.output_filename:
407      print "Saving to", config.output_filename
408      fig.savefig(config.output_filename)
409      plt.close(fig)
410
411  plt.show()
412
413if __name__ == "__main__":
414  ShowOrSavePlots(PlotConfigsFromArgs(sys.argv[1:]))
415