1# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. 2# 3# Use of this source code is governed by a BSD-style license 4# that can be found in the LICENSE file in the root of the source 5# tree. An additional intellectual property rights grant can be found 6# in the file PATENTS. All contributing project authors may 7# be found in the AUTHORS file in the root of the source tree. 8 9"""Plots statistics from WebRTC integration test logs. 10 11Usage: $ python plot_webrtc_test_logs.py filename.txt 12""" 13 14import numpy 15import sys 16import re 17 18import matplotlib.pyplot as plt 19 20# Log events. 21EVENT_START = 'RUN ] CodecSettings/VideoCodecTestParameterized.' 22EVENT_END = 'OK ] CodecSettings/VideoCodecTestParameterized.' 23 24# Metrics to plot, tuple: (name to parse in file, label to use when plotting). 25WIDTH = ('width', 'width') 26HEIGHT = ('height', 'height') 27FILENAME = ('filename', 'clip') 28CODEC_TYPE = ('codec_type', 'Codec') 29ENCODER_IMPLEMENTATION_NAME = ('enc_impl_name', 'enc name') 30DECODER_IMPLEMENTATION_NAME = ('dec_impl_name', 'dec name') 31CODEC_IMPLEMENTATION_NAME = ('codec_impl_name', 'codec name') 32CORES = ('num_cores', 'CPU cores used') 33DENOISING = ('denoising', 'denoising') 34RESILIENCE = ('resilience', 'resilience') 35ERROR_CONCEALMENT = ('error_concealment', 'error concealment') 36CPU_USAGE = ('cpu_usage_percent', 'CPU usage (%)') 37BITRATE = ('target_bitrate_kbps', 'target bitrate (kbps)') 38FRAMERATE = ('input_framerate_fps', 'fps') 39QP = ('avg_qp', 'QP avg') 40PSNR = ('avg_psnr', 'PSNR (dB)') 41SSIM = ('avg_ssim', 'SSIM') 42ENC_BITRATE = ('bitrate_kbps', 'encoded bitrate (kbps)') 43NUM_FRAMES = ('num_input_frames', 'num frames') 44NUM_DROPPED_FRAMES = ('num_dropped_frames', 'num dropped frames') 45TIME_TO_TARGET = ('time_to_reach_target_bitrate_sec', 46 'time to reach target rate (sec)') 47ENCODE_SPEED_FPS = ('enc_speed_fps', 'encode speed (fps)') 48DECODE_SPEED_FPS = ('dec_speed_fps', 'decode speed (fps)') 49AVG_KEY_FRAME_SIZE = ('avg_key_frame_size_bytes', 'avg key frame size (bytes)') 50AVG_DELTA_FRAME_SIZE = ('avg_delta_frame_size_bytes', 51 'avg delta frame size (bytes)') 52 53# Settings. 54SETTINGS = [ 55 WIDTH, 56 HEIGHT, 57 FILENAME, 58 NUM_FRAMES, 59] 60 61# Settings, options for x-axis. 62X_SETTINGS = [ 63 CORES, 64 FRAMERATE, 65 DENOISING, 66 RESILIENCE, 67 ERROR_CONCEALMENT, 68 BITRATE, # TODO(asapersson): Needs to be last. 69] 70 71# Settings, options for subplots. 72SUBPLOT_SETTINGS = [ 73 CODEC_TYPE, 74 ENCODER_IMPLEMENTATION_NAME, 75 DECODER_IMPLEMENTATION_NAME, 76 CODEC_IMPLEMENTATION_NAME, 77] + X_SETTINGS 78 79# Results. 80RESULTS = [ 81 PSNR, 82 SSIM, 83 ENC_BITRATE, 84 NUM_DROPPED_FRAMES, 85 TIME_TO_TARGET, 86 ENCODE_SPEED_FPS, 87 DECODE_SPEED_FPS, 88 QP, 89 CPU_USAGE, 90 AVG_KEY_FRAME_SIZE, 91 AVG_DELTA_FRAME_SIZE, 92] 93 94METRICS_TO_PARSE = SETTINGS + SUBPLOT_SETTINGS + RESULTS 95 96Y_METRICS = [res[1] for res in RESULTS] 97 98# Parameters for plotting. 99FIG_SIZE_SCALE_FACTOR_X = 1.6 100FIG_SIZE_SCALE_FACTOR_Y = 1.8 101GRID_COLOR = [0.45, 0.45, 0.45] 102 103 104def ParseSetting(filename, setting): 105 """Parses setting from file. 106 107 Args: 108 filename: The name of the file. 109 setting: Name of setting to parse (e.g. width). 110 111 Returns: 112 A list holding parsed settings, e.g. ['width: 128.0', 'width: 160.0'] """ 113 114 settings = [] 115 116 settings_file = open(filename) 117 while True: 118 line = settings_file.readline() 119 if not line: 120 break 121 if re.search(r'%s' % EVENT_START, line): 122 # Parse event. 123 parsed = {} 124 while True: 125 line = settings_file.readline() 126 if not line: 127 break 128 if re.search(r'%s' % EVENT_END, line): 129 # Add parsed setting to list. 130 if setting in parsed: 131 s = setting + ': ' + str(parsed[setting]) 132 if s not in settings: 133 settings.append(s) 134 break 135 136 TryFindMetric(parsed, line) 137 138 settings_file.close() 139 return settings 140 141 142def ParseMetrics(filename, setting1, setting2): 143 """Parses metrics from file. 144 145 Args: 146 filename: The name of the file. 147 setting1: First setting for sorting metrics (e.g. width). 148 setting2: Second setting for sorting metrics (e.g. CPU cores used). 149 150 Returns: 151 A dictionary holding parsed metrics. 152 153 For example: 154 metrics[key1][key2][measurement] 155 156 metrics = { 157 "width: 352": { 158 "CPU cores used: 1.0": { 159 "encode time (us)": [0.718005, 0.806925, 0.909726, 0.931835, 0.953642], 160 "PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551], 161 "bitrate (kbps)": [50, 100, 300, 500, 1000] 162 }, 163 "CPU cores used: 2.0": { 164 "encode time (us)": [0.718005, 0.806925, 0.909726, 0.931835, 0.953642], 165 "PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551], 166 "bitrate (kbps)": [50, 100, 300, 500, 1000] 167 }, 168 }, 169 "width: 176": { 170 "CPU cores used: 1.0": { 171 "encode time (us)": [0.857897, 0.91608, 0.959173, 0.971116, 0.980961], 172 "PSNR (dB)": [30.243646, 33.375592, 37.574387, 39.42184, 41.437897], 173 "bitrate (kbps)": [50, 100, 300, 500, 1000] 174 }, 175 } 176 } """ 177 178 metrics = {} 179 180 # Parse events. 181 settings_file = open(filename) 182 while True: 183 line = settings_file.readline() 184 if not line: 185 break 186 if re.search(r'%s' % EVENT_START, line): 187 # Parse event. 188 parsed = {} 189 while True: 190 line = settings_file.readline() 191 if not line: 192 break 193 if re.search(r'%s' % EVENT_END, line): 194 # Add parsed values to metrics. 195 key1 = setting1 + ': ' + str(parsed[setting1]) 196 key2 = setting2 + ': ' + str(parsed[setting2]) 197 if key1 not in metrics: 198 metrics[key1] = {} 199 if key2 not in metrics[key1]: 200 metrics[key1][key2] = {} 201 202 for label in parsed: 203 if label not in metrics[key1][key2]: 204 metrics[key1][key2][label] = [] 205 metrics[key1][key2][label].append(parsed[label]) 206 207 break 208 209 TryFindMetric(parsed, line) 210 211 settings_file.close() 212 return metrics 213 214 215def TryFindMetric(parsed, line): 216 for metric in METRICS_TO_PARSE: 217 name = metric[0] 218 label = metric[1] 219 if re.search(r'%s' % name, line): 220 found, value = GetMetric(name, line) 221 if found: 222 parsed[label] = value 223 return 224 225 226def GetMetric(name, string): 227 # Float (e.g. bitrate = 98.8253). 228 pattern = r'%s\s*[:=]\s*([+-]?\d+\.*\d*)' % name 229 m = re.search(r'%s' % pattern, string) 230 if m is not None: 231 return StringToFloat(m.group(1)) 232 233 # Alphanumeric characters (e.g. codec type : VP8). 234 pattern = r'%s\s*[:=]\s*(\w+)' % name 235 m = re.search(r'%s' % pattern, string) 236 if m is not None: 237 return True, m.group(1) 238 239 return False, -1 240 241 242def StringToFloat(value): 243 try: 244 value = float(value) 245 except ValueError: 246 print "Not a float, skipped %s" % value 247 return False, -1 248 249 return True, value 250 251 252def Plot(y_metric, x_metric, metrics): 253 """Plots y_metric vs x_metric per key in metrics. 254 255 For example: 256 y_metric = 'PSNR (dB)' 257 x_metric = 'bitrate (kbps)' 258 metrics = { 259 "CPU cores used: 1.0": { 260 "PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551], 261 "bitrate (kbps)": [50, 100, 300, 500, 1000] 262 }, 263 "CPU cores used: 2.0": { 264 "PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551], 265 "bitrate (kbps)": [50, 100, 300, 500, 1000] 266 }, 267 } 268 """ 269 for key in sorted(metrics): 270 data = metrics[key] 271 if y_metric not in data: 272 print "Failed to find metric: %s" % y_metric 273 continue 274 275 y = numpy.array(data[y_metric]) 276 x = numpy.array(data[x_metric]) 277 if len(y) != len(x): 278 print "Length mismatch for %s, %s" % (y, x) 279 continue 280 281 label = y_metric + ' - ' + str(key) 282 283 plt.plot(x, y, label=label, linewidth=1.5, marker='o', markersize=5, 284 markeredgewidth=0.0) 285 286 287def PlotFigure(settings, y_metrics, x_metric, metrics, title): 288 """Plots metrics in y_metrics list. One figure is plotted and each entry 289 in the list is plotted in a subplot (and sorted per settings). 290 291 For example: 292 settings = ['width: 128.0', 'width: 160.0']. Sort subplot per setting. 293 y_metrics = ['PSNR (dB)', 'PSNR (dB)']. Metric to plot per subplot. 294 x_metric = 'bitrate (kbps)' 295 296 """ 297 298 plt.figure() 299 plt.suptitle(title, fontsize='large', fontweight='bold') 300 settings.sort() 301 rows = len(settings) 302 cols = 1 303 pos = 1 304 while pos <= rows: 305 plt.rc('grid', color=GRID_COLOR) 306 ax = plt.subplot(rows, cols, pos) 307 plt.grid() 308 plt.setp(ax.get_xticklabels(), visible=(pos == rows), fontsize='large') 309 plt.setp(ax.get_yticklabels(), fontsize='large') 310 setting = settings[pos - 1] 311 Plot(y_metrics[pos - 1], x_metric, metrics[setting]) 312 if setting.startswith(WIDTH[1]): 313 plt.title(setting, fontsize='medium') 314 plt.legend(fontsize='large', loc='best') 315 pos += 1 316 317 plt.xlabel(x_metric, fontsize='large') 318 plt.subplots_adjust(left=0.06, right=0.98, bottom=0.05, top=0.94, hspace=0.08) 319 320 321def GetTitle(filename, setting): 322 title = '' 323 if setting != CODEC_IMPLEMENTATION_NAME[1] and setting != CODEC_TYPE[1]: 324 codec_types = ParseSetting(filename, CODEC_TYPE[1]) 325 for i in range(0, len(codec_types)): 326 title += codec_types[i] + ', ' 327 328 if setting != CORES[1]: 329 cores = ParseSetting(filename, CORES[1]) 330 for i in range(0, len(cores)): 331 title += cores[i].split('.')[0] + ', ' 332 333 if setting != FRAMERATE[1]: 334 framerate = ParseSetting(filename, FRAMERATE[1]) 335 for i in range(0, len(framerate)): 336 title += framerate[i].split('.')[0] + ', ' 337 338 if (setting != CODEC_IMPLEMENTATION_NAME[1] and 339 setting != ENCODER_IMPLEMENTATION_NAME[1]): 340 enc_names = ParseSetting(filename, ENCODER_IMPLEMENTATION_NAME[1]) 341 for i in range(0, len(enc_names)): 342 title += enc_names[i] + ', ' 343 344 if (setting != CODEC_IMPLEMENTATION_NAME[1] and 345 setting != DECODER_IMPLEMENTATION_NAME[1]): 346 dec_names = ParseSetting(filename, DECODER_IMPLEMENTATION_NAME[1]) 347 for i in range(0, len(dec_names)): 348 title += dec_names[i] + ', ' 349 350 filenames = ParseSetting(filename, FILENAME[1]) 351 title += filenames[0].split('_')[0] 352 353 num_frames = ParseSetting(filename, NUM_FRAMES[1]) 354 for i in range(0, len(num_frames)): 355 title += ' (' + num_frames[i].split('.')[0] + ')' 356 357 return title 358 359 360def ToString(input_list): 361 return ToStringWithoutMetric(input_list, ('', '')) 362 363 364def ToStringWithoutMetric(input_list, metric): 365 i = 1 366 output_str = "" 367 for m in input_list: 368 if m != metric: 369 output_str = output_str + ("%s. %s\n" % (i, m[1])) 370 i += 1 371 return output_str 372 373 374def GetIdx(text_list): 375 return int(raw_input(text_list)) - 1 376 377 378def main(): 379 filename = sys.argv[1] 380 381 # Setup. 382 idx_metric = GetIdx("Choose metric:\n0. All\n%s" % ToString(RESULTS)) 383 if idx_metric == -1: 384 # Plot all metrics. One subplot for each metric. 385 # Per subplot: metric vs bitrate (per resolution). 386 cores = ParseSetting(filename, CORES[1]) 387 setting1 = CORES[1] 388 setting2 = WIDTH[1] 389 sub_keys = [cores[0]] * len(Y_METRICS) 390 y_metrics = Y_METRICS 391 x_metric = BITRATE[1] 392 else: 393 resolutions = ParseSetting(filename, WIDTH[1]) 394 idx = GetIdx("Select metric for x-axis:\n%s" % ToString(X_SETTINGS)) 395 if X_SETTINGS[idx] == BITRATE: 396 idx = GetIdx("Plot per:\n%s" % ToStringWithoutMetric(SUBPLOT_SETTINGS, 397 BITRATE)) 398 idx_setting = METRICS_TO_PARSE.index(SUBPLOT_SETTINGS[idx]) 399 # Plot one metric. One subplot for each resolution. 400 # Per subplot: metric vs bitrate (per setting). 401 setting1 = WIDTH[1] 402 setting2 = METRICS_TO_PARSE[idx_setting][1] 403 sub_keys = resolutions 404 y_metrics = [RESULTS[idx_metric][1]] * len(sub_keys) 405 x_metric = BITRATE[1] 406 else: 407 # Plot one metric. One subplot for each resolution. 408 # Per subplot: metric vs setting (per bitrate). 409 setting1 = WIDTH[1] 410 setting2 = BITRATE[1] 411 sub_keys = resolutions 412 y_metrics = [RESULTS[idx_metric][1]] * len(sub_keys) 413 x_metric = X_SETTINGS[idx][1] 414 415 metrics = ParseMetrics(filename, setting1, setting2) 416 417 # Stretch fig size. 418 figsize = plt.rcParams["figure.figsize"] 419 figsize[0] *= FIG_SIZE_SCALE_FACTOR_X 420 figsize[1] *= FIG_SIZE_SCALE_FACTOR_Y 421 plt.rcParams["figure.figsize"] = figsize 422 423 PlotFigure(sub_keys, y_metrics, x_metric, metrics, 424 GetTitle(filename, setting2)) 425 426 plt.show() 427 428 429if __name__ == '__main__': 430 main() 431