talos/talos/results.py

#!/usr/bin/env python

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

"""
objects and methods for parsing and serializing Talos results
see https://wiki.mozilla.org/Buildbot/Talos/DataFormat
"""

import json
import os
import re
import csv
from talos import output, utils, filter


class TalosResults(object):
    """Container class for Talos results"""

    def __init__(self):
        self.results = []
        self.extra_options = []

    def add(self, test_results):
        self.results.append(test_results)

    def add_extra_option(self, extra_option):
        self.extra_options.append(extra_option)

    def output(self, output_formats):
        """
        output all results to appropriate URLs
        - output_formats: a dict mapping formats to a list of URLs
        """

        tbpl_output = {}
        try:

            for key, urls in output_formats.items():
                _output = output.Output(self)
                results = _output()
                for url in urls:
                    _output.output(results, url, tbpl_output)

        except utils.TalosError as e:
            # print to results.out
            try:
                _output = output.GraphserverOutput(self)
                results = _output()
                _output.output(
                    'file://%s' % os.path.join(os.getcwd(), 'results.out'),
                    results
                )
            except:
                pass
            print('\nFAIL: %s' % str(e).replace('\n', '\nRETURN:'))
            raise e

        if tbpl_output:
            print("TinderboxPrint: TalosResult: %s" % json.dumps(tbpl_output))


class TestResults(object):
    """container object for all test results across cycles"""

    def __init__(self, test_config, global_counters=None, framework=None):
        self.results = []
        self.test_config = test_config
        self.format = None
        self.global_counters = global_counters or {}
        self.all_counter_results = []
        self.framework = framework
        self.using_xperf = False

    def name(self):
        return self.test_config['name']

    def mainthread(self):
        return self.test_config['mainthread']

    def add(self, results, counter_results=None):
        """
        accumulate one cycle of results
        - results : TalosResults instance or path to browser log
        - counter_results : counters accumulated for this cycle
        """

        # convert to a results class via parsing the browser log
        browserLog = BrowserLogResults(
            results,
            counter_results=counter_results,
            global_counters=self.global_counters
        )
        results = browserLog.results()

        self.using_xperf = browserLog.using_xperf
        # ensure the results format matches previous results
        if self.results:
            if not results.format == self.results[0].format:
                raise utils.TalosError("Conflicting formats for results")
        else:
            self.format = results.format

        self.results.append(results)

        if counter_results:
            self.all_counter_results.append(counter_results)


class Results(object):
    def filter(self, testname, filters):
        """
        filter the results set;
        applies each of the filters in order to the results data
        filters should be callables that take a list
        the last filter should return a scalar (float or int)
        returns a list of [[data, page], ...]
        """
        retval = []
        for result in self.results:
            page = result['page']
            data = result['runs']
            remaining_filters = []

            # ignore* functions return a filtered set of data
            for f in filters:
                if f.func.__name__.startswith('ignore'):
                    data = f.apply(data)
                else:
                    remaining_filters.append(f)

            # apply the summarization filters
            for f in remaining_filters:
                if f.func.__name__ == "v8_subtest":
                    # for v8_subtest we need to page for reference data
                    data = filter.v8_subtest(data, page)
                else:
                    data = f.apply(data)

            summary = {
                'filtered': data,  # backward compatibility with perfherder
                'value': data
            }

            retval.append([summary, page])

        return retval

    def raw_values(self):
        return [(result['page'], result['runs']) for result in self.results]

    def values(self, testname, filters):
        """return filtered (value, page) for each value"""
        return [[val, page] for val, page in self.filter(testname, filters)
                if val['filtered'] > -1]


class TsResults(Results):
    """
    results for Ts tests
    """

    format = 'tsformat'

    def __init__(self, string, counter_results=None):
        self.counter_results = counter_results

        string = string.strip()
        lines = string.splitlines()

        # gather the data
        self.results = []
        index = 0

        # Handle the case where we support a pagename in the results
        # (new format)
        for line in lines:
            result = {}
            r = line.strip().split(',')
            r = [i for i in r if i]
            if len(r) <= 1:
                continue
            result['index'] = index
            result['page'] = r[0]
            # note: if we have len(r) >1, then we have pagename,raw_results
            result['runs'] = [float(i) for i in r[1:]]
            self.results.append(result)
            index += 1

        # The original case where we just have numbers and no pagename
        if not self.results:
            result = {}
            result['index'] = index
            result['page'] = 'NULL'
            result['runs'] = [float(val) for val in string.split('|')]
            self.results.append(result)


class PageloaderResults(Results):
    """
    results from a browser_dump snippet
    https://wiki.mozilla.org/Buildbot/Talos/DataFormat#browser_output.txt
    """

    format = 'tpformat'

    def __init__(self, string, counter_results=None):
        """
        - string : string of relevent part of browser dump
        - counter_results : counter results dictionary
        """

        self.counter_results = counter_results

        string = string.strip()
        lines = string.splitlines()

        # currently we ignore the metadata on top of the output (e.g.):
        # _x_x_mozilla_page_load
        # _x_x_mozilla_page_load_details
        # |i|pagename|runs|
        lines = [line for line in lines if ';' in line]

        # gather the data
        self.results = []
        for line in lines:
            result = {}
            r = line.strip('|').split(';')
            r = [i for i in r if i]
            if len(r) <= 2:
                continue
            result['index'] = int(r[0])
            result['page'] = r[1]
            result['runs'] = [float(i) for i in r[2:]]

            # fix up page
            result['page'] = self.format_pagename(result['page'])

            self.results.append(result)

    def format_pagename(self, page):
        """
        fix up the page for reporting
        """
        page = page.rstrip('/')
        if '/' in page:
            page = page.split('/')[0]
        return page


class BrowserLogResults(object):
    """parse the results from the browser log output"""

    # tokens for the report types
    report_tokens = [
        ('tsformat', ('__start_report', '__end_report')),
        ('tpformat', ('__start_tp_report', '__end_tp_report'))
    ]

    # tokens for timestamps, in order (attribute, (start_delimeter,
    # end_delimter))
    time_tokens = [
        ('startTime', ('__startTimestamp', '__endTimestamp')),
        ('beforeLaunchTime', ('__startBeforeLaunchTimestamp',
                              '__endBeforeLaunchTimestamp')),
        ('endTime', ('__startAfterTerminationTimestamp',
                     '__endAfterTerminationTimestamp'))
    ]

    # regular expression for failure case if we can't parse the tokens
    RESULTS_REGEX_FAIL = re.compile('__FAIL(.*?)__FAIL',
                                    re.DOTALL | re.MULTILINE)

    # regular expression for RSS results
    RSS_REGEX = re.compile('RSS:\s+([a-zA-Z0-9]+):\s+([0-9]+)$')

    # regular expression for responsiveness results
    RESULTS_RESPONSIVENESS_REGEX = re.compile(
        'MOZ_EVENT_TRACE\ssample\s\d*?\s(\d*\.?\d*)$',
        re.DOTALL | re.MULTILINE
    )

    # classes for results types
    classes = {'tsformat': TsResults,
               'tpformat': PageloaderResults}

    # If we are using xperf, we do not upload the regular results, only
    # xperf counters
    using_xperf = False

    def __init__(self, results_raw, counter_results=None,
                 global_counters=None):
        """
        - shutdown : whether to record shutdown results or not
        """

        self.counter_results = counter_results
        self.global_counters = global_counters

        self.results_raw = results_raw

        # parse the results
        try:
            match = self.RESULTS_REGEX_FAIL.search(self.results_raw)
            if match:
                self.error(match.group(1))
                raise utils.TalosError(match.group(1))

            self.parse()
        except utils.TalosError:
            # TODO: consider investigating this further or adding additional
            # information
            raise  # reraise failing exception

        # accumulate counter results
        self.counters(self.counter_results, self.global_counters)

    def error(self, message):
        """raise a TalosError for bad parsing of the browser log"""
        raise utils.TalosError(message)

    def parse(self):
        position = -1

        # parse the report
        for format, tokens in self.report_tokens:
            report, position = self.get_single_token(*tokens)
            if report is None:
                continue
            self.browser_results = report
            self.format = format
            previous_tokens = tokens
            break
        else:
            self.error("Could not find report in browser output: %s"
                       % self.report_tokens)

        # parse the timestamps
        for attr, tokens in self.time_tokens:

            # parse the token contents
            value, _last_token = self.get_single_token(*tokens)

            # check for errors
            if not value:
                self.error("Could not find %s in browser output: (tokens: %s)"
                           % (attr, tokens))
            try:
                value = int(value)
            except ValueError:
                self.error("Could not cast %s to an integer: %s"
                           % (attr, value))
            if _last_token < position:
                self.error("%s [character position: %s] found before %s"
                           " [character position: %s]"
                           % (tokens, _last_token, previous_tokens, position))

            # process
            setattr(self, attr, value)
            position = _last_token
            previous_tokens = tokens

    def get_single_token(self, start_token, end_token):
        """browser logs should only have a single instance of token pairs"""
        try:
            parts, last_token = utils.tokenize(self.results_raw,
                                               start_token, end_token)
        except AssertionError as e:
            self.error(str(e))
        if not parts:
            return None, -1  # no match
        if len(parts) != 1:
            self.error("Multiple matches for %s,%s" % (start_token, end_token))
        return parts[0], last_token

    def results(self):
        """return results instance appropriate to the format detected"""

        if self.format not in self.classes:
            raise utils.TalosError(
                "Unable to find a results class for format: %s"
                % repr(self.format)
            )

        return self.classes[self.format](self.browser_results)

    # methods for counters

    def counters(self, counter_results=None, global_counters=None):
        """accumulate all counters"""

        if counter_results is not None:
            self.rss(counter_results)

        if global_counters is not None:
            if 'shutdown' in global_counters:
                self.shutdown(global_counters)
            if 'responsiveness' in global_counters:
                global_counters['responsiveness'].extend(self.responsiveness())
            self.xperf(global_counters)

    def xperf(self, counter_results):
        """record xperf counters in counter_results dictionary"""

        counters = ['main_startup_fileio',
                    'main_startup_netio',
                    'main_normal_fileio',
                    'main_normal_netio',
                    'nonmain_startup_fileio',
                    'nonmain_normal_fileio',
                    'nonmain_normal_netio']

        mainthread_counter_keys = ['readcount', 'readbytes', 'writecount',
                                   'writebytes']
        mainthread_counters = ['_'.join(['mainthread', counter_key])
                               for counter_key in mainthread_counter_keys]

        self.mainthread_io(counter_results)

        if not set(counters).union(set(mainthread_counters))\
                .intersection(counter_results.keys()):
            # no xperf counters to accumulate
            return

        filename = 'etl_output_thread_stats.csv'
        if not os.path.exists(filename):
            print("Warning: we are looking for xperf results file %s, and"
                  " didn't find it" % filename)
            return

        contents = open(filename).read()
        lines = contents.splitlines()
        reader = csv.reader(lines)
        header = None
        for row in reader:
            # Read CSV
            row = [i.strip() for i in row]
            if not header:
                # We are assuming the first row is the header and all other
                # data is counters
                header = row
                continue
            values = dict(zip(header, row))

            # Format for talos
            thread = values['thread']
            counter = values['counter'].rsplit('_io_bytes', 1)[0]
            counter_name = '%s_%s_%sio' % (thread, values['stage'], counter)
            value = float(values['value'])

            # Accrue counter
            if counter_name in counter_results:
                counter_results.setdefault(counter_name, []).append(value)
                self.using_xperf = True

        if (set(mainthread_counters).intersection(counter_results.keys())):
            filename = 'etl_output.csv'
            if not os.path.exists(filename):
                print("Warning: we are looking for xperf results file"
                      " %s, and didn't find it" % filename)
                return

            contents = open(filename).read()
            lines = contents.splitlines()
            reader = csv.reader(lines)
            header = None
            for row in reader:
                row = [i.strip() for i in row]
                if not header:
                    # We are assuming the first row is the header and all
                    # other data is counters
                    header = row
                    continue
                values = dict(zip(header, row))
                for i, mainthread_counter in enumerate(mainthread_counters):
                    if int(values[mainthread_counter_keys[i]]) > 0:
                        counter_results.setdefault(mainthread_counter, [])\
                            .append([int(values[mainthread_counter_keys[i]]),
                                     values['filename']])

    def rss(self, counter_results):
        """record rss counters in counter_results dictionary"""

        counters = ['Main', 'Content']
        if not set(['%s_RSS' % i for i in counters])\
                .intersection(counter_results.keys()):
            # no RSS counters to accumulate
            return
        for line in self.results_raw.split('\n'):
            rssmatch = self.RSS_REGEX.search(line)
            if rssmatch:
                (type, value) = (rssmatch.group(1), rssmatch.group(2))
                # type will be 'Main' or 'Content'
                counter_name = '%s_RSS' % type
                if counter_name in counter_results:
                    counter_results[counter_name].append(value)

    def mainthread_io(self, counter_results):
        """record mainthread IO counters in counter_results dictionary"""

        # we want to measure mtio on xperf runs.
        # this will be shoved into the xperf results as we ignore those
        SCRIPT_DIR = \
            os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
        filename = os.path.join(SCRIPT_DIR, 'mainthread_io.json')
        try:
            contents = open(filename).read()
            counter_results.setdefault('mainthreadio', []).append(contents)
            self.using_xperf = True
        except:
            # silent failure is fine here as we will only see this on tp5n runs
            pass

    def shutdown(self, counter_results):
        """record shutdown time in counter_results dictionary"""
        counter_results.setdefault('shutdown', [])\
            .append(int(self.endTime - self.startTime))

    def responsiveness(self):
        return self.RESULTS_RESPONSIVENESS_REGEX.findall(self.results_raw)