1#!/usr/bin/env python
2# Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
3#
4# Use of this source code is governed by a BSD-style license
5# that can be found in the LICENSE file in the root of the source
6# tree. An additional intellectual property rights grant can be found
7# in the file PATENTS.  All contributing project authors may
8# be found in the AUTHORS file in the root of the source tree.
9
10import datetime
11import httplib2
12import json
13import subprocess
14import time
15import zlib
16
17from tracing.value import histogram
18from tracing.value import histogram_set
19from tracing.value.diagnostics import generic_set
20from tracing.value.diagnostics import reserved_infos
21
22
23def _GenerateOauthToken():
24    args = ['luci-auth', 'token']
25    p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
26    if p.wait() == 0:
27        output = p.stdout.read()
28        return output.strip()
29    else:
30        raise RuntimeError(
31            'Error generating authentication token.\nStdout: %s\nStderr:%s' %
32            (p.stdout.read(), p.stderr.read()))
33
34
35def _SendHistogramSet(url, histograms, oauth_token):
36    """Make a HTTP POST with the given JSON to the Performance Dashboard.
37
38    Args:
39      url: URL of Performance Dashboard instance, e.g.
40          "https://chromeperf.appspot.com".
41      histograms: a histogram set object that contains the data to be sent.
42      oauth_token: An oauth token to use for authorization.
43    """
44    headers = {'Authorization': 'Bearer %s' % oauth_token}
45
46    serialized = json.dumps(_ApplyHacks(histograms.AsDicts()), indent=4)
47
48    if url.startswith('http://localhost'):
49        # The catapult server turns off compression in developer mode.
50        data = serialized
51    else:
52        data = zlib.compress(serialized)
53
54    print 'Sending %d bytes to %s.' % (len(data), url + '/add_histograms')
55
56    http = httplib2.Http()
57    response, content = http.request(url + '/add_histograms',
58                                     method='POST',
59                                     body=data,
60                                     headers=headers)
61    return response, content
62
63
64def _WaitForUploadConfirmation(url, oauth_token, upload_token, wait_timeout,
65                               wait_polling_period):
66    """Make a HTTP GET requests to the Performance Dashboard untill upload
67    status is known or the time is out.
68
69    Args:
70      url: URL of Performance Dashboard instance, e.g.
71          "https://chromeperf.appspot.com".
72      oauth_token: An oauth token to use for authorization.
73      upload_token: String that identifies Performance Dashboard and can be used
74        for the status check.
75      wait_timeout: (datetime.timedelta) Maximum time to wait for the
76        confirmation.
77      wait_polling_period: (datetime.timedelta) Performance Dashboard will be
78        polled every wait_polling_period amount of time.
79    """
80    assert wait_polling_period <= wait_timeout
81
82    headers = {'Authorization': 'Bearer %s' % oauth_token}
83    http = httplib2.Http()
84
85    response = None
86    resp_json = None
87    current_time = datetime.datetime.now()
88    end_time = current_time + wait_timeout
89    next_poll_time = current_time + wait_polling_period
90    while datetime.datetime.now() < end_time:
91        current_time = datetime.datetime.now()
92        if next_poll_time > current_time:
93            time.sleep((next_poll_time - current_time).total_seconds())
94        next_poll_time = datetime.datetime.now() + wait_polling_period
95
96        response, content = http.request(url + '/uploads' + upload_token,
97                                         method='GET', headers=headers)
98        resp_json = json.loads(content)
99
100        print 'Upload state polled. Response: %s.' % content
101
102        if (response.status != 200 or
103            resp_json['state'] == 'COMPLETED' or
104            resp_json['state'] == 'FAILED'):
105            break
106
107    return response, resp_json
108
109
110# TODO(https://crbug.com/1029452): HACKHACK
111# Remove once we have doubles in the proto and handle -infinity correctly.
112def _ApplyHacks(dicts):
113    for d in dicts:
114        if 'running' in d:
115
116            def _NoInf(value):
117                if value == float('inf'):
118                    return histogram.JS_MAX_VALUE
119                if value == float('-inf'):
120                    return -histogram.JS_MAX_VALUE
121                return value
122
123            d['running'] = [_NoInf(value) for value in d['running']]
124
125    return dicts
126
127
128def _LoadHistogramSetFromProto(options):
129    hs = histogram_set.HistogramSet()
130    with options.input_results_file as f:
131        hs.ImportProto(f.read())
132
133    return hs
134
135
136def _AddBuildInfo(histograms, options):
137    common_diagnostics = {
138        reserved_infos.MASTERS: options.perf_dashboard_machine_group,
139        reserved_infos.BOTS: options.bot,
140        reserved_infos.POINT_ID: options.commit_position,
141        reserved_infos.BENCHMARKS: options.test_suite,
142        reserved_infos.WEBRTC_REVISIONS: str(options.webrtc_git_hash),
143        reserved_infos.BUILD_URLS: options.build_page_url,
144    }
145
146    for k, v in common_diagnostics.items():
147        histograms.AddSharedDiagnosticToAllHistograms(
148            k.name, generic_set.GenericSet([v]))
149
150
151def _DumpOutput(histograms, output_file):
152    with output_file:
153        json.dump(_ApplyHacks(histograms.AsDicts()), output_file, indent=4)
154
155
156def UploadToDashboard(options):
157    histograms = _LoadHistogramSetFromProto(options)
158    _AddBuildInfo(histograms, options)
159
160    if options.output_json_file:
161        _DumpOutput(histograms, options.output_json_file)
162
163    oauth_token = _GenerateOauthToken()
164    response, content = _SendHistogramSet(
165        options.dashboard_url, histograms, oauth_token)
166
167    upload_token = json.loads(content).get('token')
168    if not options.wait_for_upload or not upload_token:
169        print 'Not waiting for upload status confirmation.'
170        if response.status == 200:
171            print 'Received 200 from dashboard.'
172            return 0
173        else:
174            print('Upload failed with %d: %s\n\n%s' % (response.status,
175                                                      response.reason, content))
176            return 1
177
178    response, resp_json = _WaitForUploadConfirmation(
179        options.dashboard_url,
180        oauth_token,
181        upload_token,
182        datetime.timedelta(seconds=options.wait_timeout_sec),
183        datetime.timedelta(seconds=options.wait_polling_period_sec))
184
185    if response.status != 200 or resp_json['state'] == 'FAILED':
186        print('Upload failed with %d: %s\n\n%s' % (response.status,
187                                                  response.reason,
188                                                  str(resp_json)))
189        return 1
190
191    if resp_json['state'] == 'COMPLETED':
192        print 'Upload completed.'
193        return 0
194
195    print('Upload wasn\'t completed in a given time: %d.', options.wait_timeout)
196    return 1
197