1#!/usr/bin/env python
2# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
3#
4# Use of this source code is governed by a BSD-style license
5# that can be found in the LICENSE file in the root of the source
6# tree. An additional intellectual property rights grant can be found
7# in the file PATENTS.  All contributing project authors may
8# be found in the AUTHORS file in the root of the source tree.
9
10"""Shows boxplots of given score for different values of selected
11parameters. Can be used to compare scores by audioproc_f flag.
12
13Usage: apm_quality_assessment_boxplot.py -o /path/to/output
14                                         -v polqa
15                                         -n /path/to/dir/with/apm_configs
16                                         -z audioproc_f_arg1 [arg2 ...]
17
18Arguments --config_names, --render_names, --echo_simulator_names,
19--test_data_generators, --eval_scores can be used to filter the data
20used for plotting.
21"""
22
23import collections
24import logging
25import matplotlib.pyplot as plt
26import os
27
28import quality_assessment.data_access as data_access
29import quality_assessment.collect_data as collect_data
30
31
32def InstanceArgumentsParser():
33  """Arguments parser factory.
34  """
35  parser = collect_data.InstanceArgumentsParser()
36  parser.description = (
37      'Shows boxplot of given score for different values of selected'
38      'parameters. Can be used to compare scores by audioproc_f flag')
39
40  parser.add_argument('-v', '--eval_score', required=True,
41                      help=('Score name for constructing boxplots'))
42
43  parser.add_argument('-n', '--config_dir', required=False,
44                      help=('path to the folder with the configuration files'),
45                      default='apm_configs')
46
47  parser.add_argument('-z', '--params_to_plot', required=True,
48                      nargs='+', help=('audioproc_f parameter values'
49                      'by which to group scores (no leading dash)'))
50
51  return parser
52
53
54def FilterScoresByParams(data_frame, filter_params, score_name, config_dir):
55  """Filters data on the values of one or more parameters.
56
57  Args:
58    data_frame: pandas.DataFrame of all used input data.
59
60    filter_params: each config of the input data is assumed to have
61      exactly one parameter from `filter_params` defined. Every value
62      of the parameters in `filter_params` is a key in the returned
63      dict; the associated value is all cells of the data with that
64      value of the parameter.
65
66    score_name: Name of score which value is boxplotted. Currently cannot do
67      more than one value.
68
69    config_dir: path to dir with APM configs.
70
71  Returns: dictionary, key is a param value, result is all scores for
72    that param value (see `filter_params` for explanation).
73  """
74  results = collections.defaultdict(dict)
75  config_names = data_frame['apm_config'].drop_duplicates().values.tolist()
76
77  for config_name in config_names:
78    config_json = data_access.AudioProcConfigFile.Load(
79        os.path.join(config_dir, config_name + '.json'))
80    data_with_config = data_frame[data_frame.apm_config == config_name]
81    data_cell_scores = data_with_config[data_with_config.eval_score_name ==
82                                        score_name]
83
84    # Exactly one of |params_to_plot| must match:
85    (matching_param, ) = [x for x in filter_params if '-' + x in config_json]
86
87    # Add scores for every track to the result.
88    for capture_name in data_cell_scores.capture:
89      result_score = float(data_cell_scores[data_cell_scores.capture ==
90                                            capture_name].score)
91      config_dict = results[config_json['-' + matching_param]]
92      if capture_name not in config_dict:
93        config_dict[capture_name] = {}
94
95      config_dict[capture_name][matching_param] = result_score
96
97  return results
98
99
100def _FlattenToScoresList(config_param_score_dict):
101  """Extracts a list of scores from input data structure.
102
103  Args:
104    config_param_score_dict: of the form {'capture_name':
105    {'param_name' : score_value,.. } ..}
106
107  Returns: Plain list of all score value present in input data
108    structure
109  """
110  result = []
111  for capture_name in config_param_score_dict:
112    result += list(config_param_score_dict[capture_name].values())
113  return result
114
115
116def main():
117  # Init.
118  # TODO(alessiob): INFO once debugged.
119  logging.basicConfig(level=logging.DEBUG)
120  parser = InstanceArgumentsParser()
121  args = parser.parse_args()
122
123  # Get the scores.
124  src_path = collect_data.ConstructSrcPath(args)
125  logging.debug(src_path)
126  scores_data_frame = collect_data.FindScores(src_path, args)
127
128  # Filter the data by `args.params_to_plot`
129  scores_filtered = FilterScoresByParams(scores_data_frame,
130                                         args.params_to_plot,
131                                         args.eval_score,
132                                         args.config_dir)
133
134  data_list = sorted(scores_filtered.items())
135  data_values = [_FlattenToScoresList(x) for (_, x) in data_list]
136  data_labels = [x for (x, _) in data_list]
137
138  _, axes = plt.subplots(nrows=1, ncols=1, figsize=(6, 6))
139  axes.boxplot(data_values, labels=data_labels)
140  axes.set_ylabel(args.eval_score)
141  axes.set_xlabel('/'.join(args.params_to_plot))
142  plt.show()
143
144
145if __name__ == "__main__":
146  main()
147