1#!/usr/bin/env python 2# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. 3# 4# Use of this source code is governed by a BSD-style license 5# that can be found in the LICENSE file in the root of the source 6# tree. An additional intellectual property rights grant can be found 7# in the file PATENTS. All contributing project authors may 8# be found in the AUTHORS file in the root of the source tree. 9 10"""Shows boxplots of given score for different values of selected 11parameters. Can be used to compare scores by audioproc_f flag. 12 13Usage: apm_quality_assessment_boxplot.py -o /path/to/output 14 -v polqa 15 -n /path/to/dir/with/apm_configs 16 -z audioproc_f_arg1 [arg2 ...] 17 18Arguments --config_names, --render_names, --echo_simulator_names, 19--test_data_generators, --eval_scores can be used to filter the data 20used for plotting. 21""" 22 23import collections 24import logging 25import matplotlib.pyplot as plt 26import os 27 28import quality_assessment.data_access as data_access 29import quality_assessment.collect_data as collect_data 30 31 32def InstanceArgumentsParser(): 33 """Arguments parser factory. 34 """ 35 parser = collect_data.InstanceArgumentsParser() 36 parser.description = ( 37 'Shows boxplot of given score for different values of selected' 38 'parameters. Can be used to compare scores by audioproc_f flag') 39 40 parser.add_argument('-v', '--eval_score', required=True, 41 help=('Score name for constructing boxplots')) 42 43 parser.add_argument('-n', '--config_dir', required=False, 44 help=('path to the folder with the configuration files'), 45 default='apm_configs') 46 47 parser.add_argument('-z', '--params_to_plot', required=True, 48 nargs='+', help=('audioproc_f parameter values' 49 'by which to group scores (no leading dash)')) 50 51 return parser 52 53 54def FilterScoresByParams(data_frame, filter_params, score_name, config_dir): 55 """Filters data on the values of one or more parameters. 56 57 Args: 58 data_frame: pandas.DataFrame of all used input data. 59 60 filter_params: each config of the input data is assumed to have 61 exactly one parameter from `filter_params` defined. Every value 62 of the parameters in `filter_params` is a key in the returned 63 dict; the associated value is all cells of the data with that 64 value of the parameter. 65 66 score_name: Name of score which value is boxplotted. Currently cannot do 67 more than one value. 68 69 config_dir: path to dir with APM configs. 70 71 Returns: dictionary, key is a param value, result is all scores for 72 that param value (see `filter_params` for explanation). 73 """ 74 results = collections.defaultdict(dict) 75 config_names = data_frame['apm_config'].drop_duplicates().values.tolist() 76 77 for config_name in config_names: 78 config_json = data_access.AudioProcConfigFile.Load( 79 os.path.join(config_dir, config_name + '.json')) 80 data_with_config = data_frame[data_frame.apm_config == config_name] 81 data_cell_scores = data_with_config[data_with_config.eval_score_name == 82 score_name] 83 84 # Exactly one of |params_to_plot| must match: 85 (matching_param, ) = [x for x in filter_params if '-' + x in config_json] 86 87 # Add scores for every track to the result. 88 for capture_name in data_cell_scores.capture: 89 result_score = float(data_cell_scores[data_cell_scores.capture == 90 capture_name].score) 91 config_dict = results[config_json['-' + matching_param]] 92 if capture_name not in config_dict: 93 config_dict[capture_name] = {} 94 95 config_dict[capture_name][matching_param] = result_score 96 97 return results 98 99 100def _FlattenToScoresList(config_param_score_dict): 101 """Extracts a list of scores from input data structure. 102 103 Args: 104 config_param_score_dict: of the form {'capture_name': 105 {'param_name' : score_value,.. } ..} 106 107 Returns: Plain list of all score value present in input data 108 structure 109 """ 110 result = [] 111 for capture_name in config_param_score_dict: 112 result += list(config_param_score_dict[capture_name].values()) 113 return result 114 115 116def main(): 117 # Init. 118 # TODO(alessiob): INFO once debugged. 119 logging.basicConfig(level=logging.DEBUG) 120 parser = InstanceArgumentsParser() 121 args = parser.parse_args() 122 123 # Get the scores. 124 src_path = collect_data.ConstructSrcPath(args) 125 logging.debug(src_path) 126 scores_data_frame = collect_data.FindScores(src_path, args) 127 128 # Filter the data by `args.params_to_plot` 129 scores_filtered = FilterScoresByParams(scores_data_frame, 130 args.params_to_plot, 131 args.eval_score, 132 args.config_dir) 133 134 data_list = sorted(scores_filtered.items()) 135 data_values = [_FlattenToScoresList(x) for (_, x) in data_list] 136 data_labels = [x for (x, _) in data_list] 137 138 _, axes = plt.subplots(nrows=1, ncols=1, figsize=(6, 6)) 139 axes.boxplot(data_values, labels=data_labels) 140 axes.set_ylabel(args.eval_score) 141 axes.set_xlabel('/'.join(args.params_to_plot)) 142 plt.show() 143 144 145if __name__ == "__main__": 146 main() 147