1# This Source Code Form is subject to the terms of the Mozilla Public
2# License, v. 2.0. If a copy of the MPL was not distributed with this
3# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4import json
5import jsonschema
6import os
7import pathlib
8import statistics
9
10from mozperftest.layers import Layer
11from mozperftest.metrics.exceptions import PerfherderValidDataError
12from mozperftest.metrics.common import filtered_metrics
13from mozperftest.metrics.utils import write_json
14
15
16PERFHERDER_SCHEMA = pathlib.Path(
17    "testing", "mozharness", "external_tools", "performance-artifact-schema.json"
18)
19
20
21class Perfherder(Layer):
22    """Output data in the perfherder format.
23    """
24
25    name = "perfherder"
26    activated = False
27
28    arguments = {
29        "prefix": {
30            "type": str,
31            "default": "",
32            "help": "Prefix the output files with this string.",
33        },
34        "app": {
35            "type": str,
36            "default": "firefox",
37            "choices": [
38                "firefox",
39                "chrome-m",
40                "chrome",
41                "chromium",
42                "fennec",
43                "geckoview",
44                "fenix",
45                "refbrow",
46            ],
47            "help": (
48                "Shorthand name of application that is "
49                "being tested (used in perfherder data)."
50            ),
51        },
52        "metrics": {
53            "nargs": "*",
54            "default": [],
55            "help": "The metrics that should be retrieved from the data.",
56        },
57        "stats": {
58            "action": "store_true",
59            "default": False,
60            "help": "If set, browsertime statistics will be reported.",
61        },
62    }
63
64    def __call__(self, metadata):
65        """Processes the given results into a perfherder-formatted data blob.
66
67        If the `--perfherder` flag isn't provided, then the
68        results won't be processed into a perfherder-data blob. If the
69        flavor is unknown to us, then we assume that it comes from
70        browsertime.
71
72        XXX If needed, make a way to do flavor-specific processing
73
74        :param results list/dict/str: Results to process.
75        :param perfherder bool: True if results should be processed
76            into a perfherder-data blob.
77        :param flavor str: The flavor that is being processed.
78        """
79        prefix = self.get_arg("prefix")
80        output = self.get_arg("output")
81
82        # XXX Make an arugment for exclusions from metrics
83        # (or go directly to regex's for metrics)
84        exclusions = None
85        if not self.get_arg("stats"):
86            exclusions = ["statistics."]
87
88        # Get filtered metrics
89        results, fullsettings = filtered_metrics(
90            metadata,
91            output,
92            prefix,
93            metrics=self.get_arg("metrics"),
94            settings=True,
95            exclude=exclusions,
96        )
97
98        if not any([results[name] for name in results]):
99            self.warning("No results left after filtering")
100            return metadata
101
102        # XXX Add version info into this data
103        app_info = {"name": self.get_arg("app", default="firefox")}
104
105        all_perfherder_data = None
106        for name, res in results.items():
107            settings = fullsettings[name]
108
109            # XXX Instead of just passing replicates here, we should build
110            # up a partial perfherder data blob (with options) and subtest
111            # overall values.
112            subtests = {}
113            for r in res:
114                vals = [
115                    v["value"]
116                    for v in r["data"]
117                    if isinstance(v["value"], (int, float))
118                ]
119                if vals:
120                    subtests[r["subtest"]] = vals
121
122            perfherder_data = self._build_blob(
123                subtests,
124                name=name,
125                extra_options=settings.get("extraOptions"),
126                should_alert=settings.get("shouldAlert", False),
127                application=app_info,
128                alert_threshold=settings.get("alertThreshold", 2.0),
129                lower_is_better=settings.get("lowerIsBetter", True),
130                unit=settings.get("unit", "ms"),
131                summary=settings.get("value"),
132            )
133
134            if all_perfherder_data is None:
135                all_perfherder_data = perfherder_data
136            else:
137                all_perfherder_data["suites"].extend(perfherder_data["suites"])
138
139        # Validate the final perfherder data blob
140        with pathlib.Path(metadata._mach_cmd.topsrcdir, PERFHERDER_SCHEMA).open() as f:
141            schema = json.load(f)
142        jsonschema.validate(all_perfherder_data, schema)
143
144        file = "perfherder-data.json"
145        if prefix:
146            file = "{}-{}".format(prefix, file)
147        self.info("Writing perfherder results to {}".format(os.path.join(output, file)))
148
149        # XXX "suites" key error occurs when using self.info so a print
150        # is being done for now.
151        print("PERFHERDER_DATA: " + json.dumps(all_perfherder_data))
152        metadata.set_output(write_json(all_perfherder_data, output, file))
153        return metadata
154
155    def _build_blob(
156        self,
157        subtests,
158        name="browsertime",
159        test_type="pageload",
160        extra_options=None,
161        should_alert=False,
162        subtest_should_alert=None,
163        suiteshould_alert=False,
164        framework=None,
165        application=None,
166        alert_threshold=2.0,
167        lower_is_better=True,
168        unit="ms",
169        summary=None,
170    ):
171        """Build a PerfHerder data blob from the given subtests.
172
173        NOTE: This is a WIP, see the many TODOs across this file.
174
175        Given a dictionary of subtests, and the values. Build up a
176        perfherder data blob. Note that the naming convention for
177        these arguments is different then the rest of the scripts
178        to make it easier to see where they are going to in the perfherder
179        data.
180
181        For the `should_alert` field, if should_alert is True but `subtest_should_alert`
182        is empty, then all subtests along with the suite will generate alerts.
183        Otherwise, if the subtest_should_alert contains subtests to alert on, then
184        only those will alert and nothing else (including the suite). If the
185        suite value should alert, then set `suiteshould_alert` to True.
186
187        :param subtests dict: A dictionary of subtests and the values.
188            XXX TODO items for subtests:
189                (1) Allow it to contain replicates and individual settings
190                    for each of the subtests.
191                (2) The geomean of the replicates will be taken for now,
192                    but it should be made more flexible in some way.
193                (3) We need some way to handle making multiple suites.
194        :param name str: Name to give to the suite.
195        :param test_type str: The type of test that was run.
196        :param extra_options list: A list of extra options to store.
197        :param should_alert bool: Whether all values in the suite should
198            generate alerts or not.
199        :param subtest_should_alert list: A list of subtests to alert on. If this
200            is not empty, then it will disable the suite-level alerts.
201        :param suiteshould_alert bool: Used if `subtest_should_alert` is not
202            empty, and if True, then the suite-level value will generate
203            alerts.
204        :param framework dict: Information about the framework that
205            is being tested.
206        :param application dict: Information about the application that
207            is being tested. Must include name, and optionally a version.
208        :param alert_threshold float: The change in percentage this
209            metric must undergo to to generate an alert.
210        :param lower_is_better bool: If True, then lower values are better
211            than higher ones.
212        :param unit str: The unit of the data.
213        :param summary float: The summary value to use in the perfherder
214            data blob. By default, the mean of all the subtests will be
215            used.
216
217        :return dict: The PerfHerder data blob.
218        """
219        if extra_options is None:
220            extra_options = []
221        if subtest_should_alert is None:
222            subtest_should_alert = []
223        if framework is None:
224            framework = {"name": "browsertime"}
225        if application is None:
226            application = {"name": "firefox", "version": "9000"}
227
228        perf_subtests = []
229        suite = {
230            "name": name,
231            "type": test_type,
232            "value": None,
233            "unit": unit,
234            "extraOptions": extra_options,
235            "lowerIsBetter": lower_is_better,
236            "alertThreshold": alert_threshold,
237            "shouldAlert": (should_alert and not subtest_should_alert)
238            or suiteshould_alert,
239            "subtests": perf_subtests,
240        }
241
242        perfherder = {
243            "suites": [suite],
244            "framework": framework,
245            "application": application,
246        }
247
248        allvals = []
249        for measurement in subtests:
250            reps = subtests[measurement]
251            allvals.extend(reps)
252
253            if len(reps) == 0:
254                self.warning("No replicates found for {}, skipping".format(measurement))
255                continue
256
257            perf_subtests.append(
258                {
259                    "name": measurement,
260                    "replicates": reps,
261                    "lowerIsBetter": lower_is_better,
262                    "value": statistics.mean(reps),
263                    "unit": unit,
264                    "shouldAlert": should_alert or measurement in subtest_should_alert,
265                }
266            )
267
268        if len(allvals) == 0:
269            raise PerfherderValidDataError(
270                "Could not build perfherder data blob because no valid data was provided, "
271                + "only int/float data is accepted."
272            )
273
274        suite["value"] = statistics.mean(allvals)
275        return perfherder
276