1# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. 2# 3# Use of this source code is governed by a BSD-style license 4# that can be found in the LICENSE file in the root of the source 5# tree. An additional intellectual property rights grant can be found 6# in the file PATENTS. All contributing project authors may 7# be found in the AUTHORS file in the root of the source tree. 8 9"""Plots statistics from WebRTC integration test logs. 10 11Usage: $ python plot_webrtc_test_logs.py filename.txt 12""" 13 14import numpy 15import sys 16import re 17 18import matplotlib.pyplot as plt 19 20# Log events. 21EVENT_START = \ 22 'RUN ] CodecSettings/VideoProcessorIntegrationTestParameterized.' 23EVENT_END = 'OK ] CodecSettings/VideoProcessorIntegrationTestParameterized.' 24 25# Metrics to plot, tuple: (name to parse in file, label to use when plotting). 26BITRATE = ('Target bitrate', 'target bitrate (kbps)') 27WIDTH = ('Width', 'width') 28HEIGHT = ('Height', 'height') 29FILENAME = ('Filename', 'clip') 30CODEC_TYPE = ('Codec type', 'Codec') 31ENCODER_IMPLEMENTATION_NAME = ('Encoder implementation name', 'enc name') 32DECODER_IMPLEMENTATION_NAME = ('Decoder implementation name', 'dec name') 33CODEC_IMPLEMENTATION_NAME = ('Codec implementation name', 'codec name') 34CORES = ('# CPU cores used', 'CPU cores used') 35DENOISING = ('Denoising', 'denoising') 36RESILIENCE = ('Resilience', 'resilience') 37ERROR_CONCEALMENT = ('Error concealment', 'error concealment') 38QP = ('Average QP', 'avg QP') 39CPU_USAGE = ('CPU usage %', 'CPU usage (%)') 40PSNR = ('PSNR avg', 'PSNR (dB)') 41SSIM = ('SSIM avg', 'SSIM') 42ENC_BITRATE = ('Encoded bitrate', 'encoded bitrate (kbps)') 43FRAMERATE = ('Frame rate', 'fps') 44NUM_FRAMES = ('# processed frames', 'num frames') 45NUM_DROPPED_FRAMES = ('# dropped frames', 'num dropped frames') 46NUM_FRAMES_TO_TARGET = ('# frames to convergence', 47 'frames to reach target rate') 48ENCODE_TIME = ('Encoding time', 'encode time (us)') 49ENCODE_TIME_AVG = ('Encoding time', 'encode time (us) avg') 50DECODE_TIME = ('Decoding time', 'decode time (us)') 51DECODE_TIME_AVG = ('Decoding time', 'decode time (us) avg') 52FRAME_SIZE = ('Frame sizes', 'frame size (bytes)') 53FRAME_SIZE_AVG = ('Frame sizes', 'frame size (bytes) avg') 54AVG_KEY_FRAME_SIZE = ('Average key frame size', 'avg key frame size (bytes)') 55AVG_NON_KEY_FRAME_SIZE = ('Average non-key frame size', 56 'avg non-key frame size (bytes)') 57 58# Settings. 59SETTINGS = [ 60 WIDTH, 61 HEIGHT, 62 FILENAME, 63 NUM_FRAMES, 64 ENCODE_TIME, 65 DECODE_TIME, 66 FRAME_SIZE, 67] 68 69# Settings, options for x-axis. 70X_SETTINGS = [ 71 CORES, 72 FRAMERATE, 73 DENOISING, 74 RESILIENCE, 75 ERROR_CONCEALMENT, 76 BITRATE, # TODO(asapersson): Needs to be last. 77] 78 79# Settings, options for subplots. 80SUBPLOT_SETTINGS = [ 81 CODEC_TYPE, 82 ENCODER_IMPLEMENTATION_NAME, 83 DECODER_IMPLEMENTATION_NAME, 84 CODEC_IMPLEMENTATION_NAME, 85] + X_SETTINGS 86 87# Results. 88RESULTS = [ 89 PSNR, 90 SSIM, 91 ENC_BITRATE, 92 NUM_DROPPED_FRAMES, 93 NUM_FRAMES_TO_TARGET, 94 ENCODE_TIME_AVG, 95 DECODE_TIME_AVG, 96 QP, 97 CPU_USAGE, 98 AVG_KEY_FRAME_SIZE, 99 AVG_NON_KEY_FRAME_SIZE, 100] 101 102METRICS_TO_PARSE = SETTINGS + SUBPLOT_SETTINGS + RESULTS 103 104Y_METRICS = [res[1] for res in RESULTS] 105 106# Parameters for plotting. 107FIG_SIZE_SCALE_FACTOR_X = 1.6 108FIG_SIZE_SCALE_FACTOR_Y = 1.8 109GRID_COLOR = [0.45, 0.45, 0.45] 110 111 112def ParseSetting(filename, setting): 113 """Parses setting from file. 114 115 Args: 116 filename: The name of the file. 117 setting: Name of setting to parse (e.g. width). 118 119 Returns: 120 A list holding parsed settings, e.g. ['width: 128.0', 'width: 160.0'] """ 121 122 settings = [] 123 124 settings_file = open(filename) 125 while True: 126 line = settings_file.readline() 127 if not line: 128 break 129 if re.search(r'%s' % EVENT_START, line): 130 # Parse event. 131 parsed = {} 132 while True: 133 line = settings_file.readline() 134 if not line: 135 break 136 if re.search(r'%s' % EVENT_END, line): 137 # Add parsed setting to list. 138 if setting in parsed: 139 s = setting + ': ' + str(parsed[setting]) 140 if s not in settings: 141 settings.append(s) 142 break 143 144 TryFindMetric(parsed, line, settings_file) 145 146 settings_file.close() 147 return settings 148 149 150def ParseMetrics(filename, setting1, setting2): 151 """Parses metrics from file. 152 153 Args: 154 filename: The name of the file. 155 setting1: First setting for sorting metrics (e.g. width). 156 setting2: Second setting for sorting metrics (e.g. CPU cores used). 157 158 Returns: 159 A dictionary holding parsed metrics. 160 161 For example: 162 metrics[key1][key2][measurement] 163 164 metrics = { 165 "width: 352": { 166 "CPU cores used: 1.0": { 167 "encode time (us)": [0.718005, 0.806925, 0.909726, 0.931835, 0.953642], 168 "PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551], 169 "bitrate (kbps)": [50, 100, 300, 500, 1000] 170 }, 171 "CPU cores used: 2.0": { 172 "encode time (us)": [0.718005, 0.806925, 0.909726, 0.931835, 0.953642], 173 "PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551], 174 "bitrate (kbps)": [50, 100, 300, 500, 1000] 175 }, 176 }, 177 "width: 176": { 178 "CPU cores used: 1.0": { 179 "encode time (us)": [0.857897, 0.91608, 0.959173, 0.971116, 0.980961], 180 "PSNR (dB)": [30.243646, 33.375592, 37.574387, 39.42184, 41.437897], 181 "bitrate (kbps)": [50, 100, 300, 500, 1000] 182 }, 183 } 184 } """ 185 186 metrics = {} 187 188 # Parse events. 189 settings_file = open(filename) 190 while True: 191 line = settings_file.readline() 192 if not line: 193 break 194 if re.search(r'%s' % EVENT_START, line): 195 # Parse event. 196 parsed = {} 197 while True: 198 line = settings_file.readline() 199 if not line: 200 break 201 if re.search(r'%s' % EVENT_END, line): 202 # Add parsed values to metrics. 203 key1 = setting1 + ': ' + str(parsed[setting1]) 204 key2 = setting2 + ': ' + str(parsed[setting2]) 205 if key1 not in metrics: 206 metrics[key1] = {} 207 if key2 not in metrics[key1]: 208 metrics[key1][key2] = {} 209 210 for label in parsed: 211 if label not in metrics[key1][key2]: 212 metrics[key1][key2][label] = [] 213 metrics[key1][key2][label].append(parsed[label]) 214 215 break 216 217 TryFindMetric(parsed, line, settings_file) 218 219 settings_file.close() 220 return metrics 221 222 223def TryFindMetric(parsed, line, settings_file): 224 for metric in METRICS_TO_PARSE: 225 name = metric[0] 226 label = metric[1] 227 if re.search(r'%s' % name, line): 228 found, value = GetMetric(name, line) 229 if not found: 230 # TODO(asapersson): Change format. 231 # Try find min, max, average stats. 232 found, minimum = GetMetric("Min", settings_file.readline()) 233 if not found: 234 return 235 found, maximum = GetMetric("Max", settings_file.readline()) 236 if not found: 237 return 238 found, average = GetMetric("Average", settings_file.readline()) 239 if not found: 240 return 241 242 parsed[label + ' min'] = minimum 243 parsed[label + ' max'] = maximum 244 parsed[label + ' avg'] = average 245 246 parsed[label] = value 247 return 248 249 250def GetMetric(name, string): 251 # Float (e.g. bitrate = 98.8253). 252 pattern = r'%s\s*[:=]\s*([+-]?\d+\.*\d*)' % name 253 m = re.search(r'%s' % pattern, string) 254 if m is not None: 255 return StringToFloat(m.group(1)) 256 257 # Alphanumeric characters (e.g. codec type : VP8). 258 pattern = r'%s\s*[:=]\s*(\w+)' % name 259 m = re.search(r'%s' % pattern, string) 260 if m is not None: 261 return True, m.group(1) 262 263 return False, -1 264 265 266def StringToFloat(value): 267 try: 268 value = float(value) 269 except ValueError: 270 print "Not a float, skipped %s" % value 271 return False, -1 272 273 return True, value 274 275 276def Plot(y_metric, x_metric, metrics): 277 """Plots y_metric vs x_metric per key in metrics. 278 279 For example: 280 y_metric = 'PSNR (dB)' 281 x_metric = 'bitrate (kbps)' 282 metrics = { 283 "CPU cores used: 1.0": { 284 "PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551], 285 "bitrate (kbps)": [50, 100, 300, 500, 1000] 286 }, 287 "CPU cores used: 2.0": { 288 "PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551], 289 "bitrate (kbps)": [50, 100, 300, 500, 1000] 290 }, 291 } 292 """ 293 for key in sorted(metrics): 294 data = metrics[key] 295 if y_metric not in data: 296 print "Failed to find metric: %s" % y_metric 297 continue 298 299 y = numpy.array(data[y_metric]) 300 x = numpy.array(data[x_metric]) 301 if len(y) != len(x): 302 print "Length mismatch for %s, %s" % (y, x) 303 continue 304 305 label = y_metric + ' - ' + str(key) 306 307 plt.plot(x, y, label=label, linewidth=1.5, marker='o', markersize=5, 308 markeredgewidth=0.0) 309 310 311def PlotFigure(settings, y_metrics, x_metric, metrics, title): 312 """Plots metrics in y_metrics list. One figure is plotted and each entry 313 in the list is plotted in a subplot (and sorted per settings). 314 315 For example: 316 settings = ['width: 128.0', 'width: 160.0']. Sort subplot per setting. 317 y_metrics = ['PSNR (dB)', 'PSNR (dB)']. Metric to plot per subplot. 318 x_metric = 'bitrate (kbps)' 319 320 """ 321 322 plt.figure() 323 plt.suptitle(title, fontsize='large', fontweight='bold') 324 settings.sort() 325 rows = len(settings) 326 cols = 1 327 pos = 1 328 while pos <= rows: 329 plt.rc('grid', color=GRID_COLOR) 330 ax = plt.subplot(rows, cols, pos) 331 plt.grid() 332 plt.setp(ax.get_xticklabels(), visible=(pos == rows), fontsize='large') 333 plt.setp(ax.get_yticklabels(), fontsize='large') 334 setting = settings[pos - 1] 335 Plot(y_metrics[pos - 1], x_metric, metrics[setting]) 336 if setting.startswith(WIDTH[1]): 337 plt.title(setting, fontsize='medium') 338 plt.legend(fontsize='large', loc='best') 339 pos += 1 340 341 plt.xlabel(x_metric, fontsize='large') 342 plt.subplots_adjust(left=0.06, right=0.98, bottom=0.05, top=0.94, hspace=0.08) 343 344 345def GetTitle(filename, setting): 346 title = '' 347 if setting != CODEC_IMPLEMENTATION_NAME[1] and setting != CODEC_TYPE[1]: 348 codec_types = ParseSetting(filename, CODEC_TYPE[1]) 349 for i in range(0, len(codec_types)): 350 title += codec_types[i] + ', ' 351 352 if setting != CORES[1]: 353 cores = ParseSetting(filename, CORES[1]) 354 for i in range(0, len(cores)): 355 title += cores[i].split('.')[0] + ', ' 356 357 if setting != FRAMERATE[1]: 358 framerate = ParseSetting(filename, FRAMERATE[1]) 359 for i in range(0, len(framerate)): 360 title += framerate[i].split('.')[0] + ', ' 361 362 if (setting != CODEC_IMPLEMENTATION_NAME[1] and 363 setting != ENCODER_IMPLEMENTATION_NAME[1]): 364 enc_names = ParseSetting(filename, ENCODER_IMPLEMENTATION_NAME[1]) 365 for i in range(0, len(enc_names)): 366 title += enc_names[i] + ', ' 367 368 if (setting != CODEC_IMPLEMENTATION_NAME[1] and 369 setting != DECODER_IMPLEMENTATION_NAME[1]): 370 dec_names = ParseSetting(filename, DECODER_IMPLEMENTATION_NAME[1]) 371 for i in range(0, len(dec_names)): 372 title += dec_names[i] + ', ' 373 374 filenames = ParseSetting(filename, FILENAME[1]) 375 title += filenames[0].split('_')[0] 376 377 num_frames = ParseSetting(filename, NUM_FRAMES[1]) 378 for i in range(0, len(num_frames)): 379 title += ' (' + num_frames[i].split('.')[0] + ')' 380 381 return title 382 383 384def ToString(input_list): 385 return ToStringWithoutMetric(input_list, ('', '')) 386 387 388def ToStringWithoutMetric(input_list, metric): 389 i = 1 390 output_str = "" 391 for m in input_list: 392 if m != metric: 393 output_str = output_str + ("%s. %s\n" % (i, m[1])) 394 i += 1 395 return output_str 396 397 398def GetIdx(text_list): 399 return int(raw_input(text_list)) - 1 400 401 402def main(): 403 filename = sys.argv[1] 404 405 # Setup. 406 idx_metric = GetIdx("Choose metric:\n0. All\n%s" % ToString(RESULTS)) 407 if idx_metric == -1: 408 # Plot all metrics. One subplot for each metric. 409 # Per subplot: metric vs bitrate (per resolution). 410 cores = ParseSetting(filename, CORES[1]) 411 setting1 = CORES[1] 412 setting2 = WIDTH[1] 413 sub_keys = [cores[0]] * len(Y_METRICS) 414 y_metrics = Y_METRICS 415 x_metric = BITRATE[1] 416 else: 417 resolutions = ParseSetting(filename, WIDTH[1]) 418 idx = GetIdx("Select metric for x-axis:\n%s" % ToString(X_SETTINGS)) 419 if X_SETTINGS[idx] == BITRATE: 420 idx = GetIdx("Plot per:\n%s" % ToStringWithoutMetric(SUBPLOT_SETTINGS, 421 BITRATE)) 422 idx_setting = METRICS_TO_PARSE.index(SUBPLOT_SETTINGS[idx]) 423 # Plot one metric. One subplot for each resolution. 424 # Per subplot: metric vs bitrate (per setting). 425 setting1 = WIDTH[1] 426 setting2 = METRICS_TO_PARSE[idx_setting][1] 427 sub_keys = resolutions 428 y_metrics = [RESULTS[idx_metric][1]] * len(sub_keys) 429 x_metric = BITRATE[1] 430 else: 431 # Plot one metric. One subplot for each resolution. 432 # Per subplot: metric vs setting (per bitrate). 433 setting1 = WIDTH[1] 434 setting2 = BITRATE[1] 435 sub_keys = resolutions 436 y_metrics = [RESULTS[idx_metric][1]] * len(sub_keys) 437 x_metric = X_SETTINGS[idx][1] 438 439 metrics = ParseMetrics(filename, setting1, setting2) 440 441 # Stretch fig size. 442 figsize = plt.rcParams["figure.figsize"] 443 figsize[0] *= FIG_SIZE_SCALE_FACTOR_X 444 figsize[1] *= FIG_SIZE_SCALE_FACTOR_Y 445 plt.rcParams["figure.figsize"] = figsize 446 447 PlotFigure(sub_keys, y_metrics, x_metric, metrics, 448 GetTitle(filename, setting2)) 449 450 plt.show() 451 452 453if __name__ == '__main__': 454 main() 455