1#!/usr/bin/env python 2# Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. 3# 4# Use of this source code is governed by a BSD-style license 5# that can be found in the LICENSE file in the root of the source 6# tree. An additional intellectual property rights grant can be found 7# in the file PATENTS. All contributing project authors may 8# be found in the AUTHORS file in the root of the source tree. 9 10import datetime 11import httplib2 12import json 13import subprocess 14import time 15import zlib 16 17from tracing.value import histogram 18from tracing.value import histogram_set 19from tracing.value.diagnostics import generic_set 20from tracing.value.diagnostics import reserved_infos 21 22 23def _GenerateOauthToken(): 24 args = ['luci-auth', 'token'] 25 p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 26 if p.wait() == 0: 27 output = p.stdout.read() 28 return output.strip() 29 else: 30 raise RuntimeError( 31 'Error generating authentication token.\nStdout: %s\nStderr:%s' % 32 (p.stdout.read(), p.stderr.read())) 33 34 35def _SendHistogramSet(url, histograms, oauth_token): 36 """Make a HTTP POST with the given JSON to the Performance Dashboard. 37 38 Args: 39 url: URL of Performance Dashboard instance, e.g. 40 "https://chromeperf.appspot.com". 41 histograms: a histogram set object that contains the data to be sent. 42 oauth_token: An oauth token to use for authorization. 43 """ 44 headers = {'Authorization': 'Bearer %s' % oauth_token} 45 46 serialized = json.dumps(_ApplyHacks(histograms.AsDicts()), indent=4) 47 48 if url.startswith('http://localhost'): 49 # The catapult server turns off compression in developer mode. 50 data = serialized 51 else: 52 data = zlib.compress(serialized) 53 54 print 'Sending %d bytes to %s.' % (len(data), url + '/add_histograms') 55 56 http = httplib2.Http() 57 response, content = http.request(url + '/add_histograms', 58 method='POST', 59 body=data, 60 headers=headers) 61 return response, content 62 63 64def _WaitForUploadConfirmation(url, oauth_token, upload_token, wait_timeout, 65 wait_polling_period): 66 """Make a HTTP GET requests to the Performance Dashboard untill upload 67 status is known or the time is out. 68 69 Args: 70 url: URL of Performance Dashboard instance, e.g. 71 "https://chromeperf.appspot.com". 72 oauth_token: An oauth token to use for authorization. 73 upload_token: String that identifies Performance Dashboard and can be used 74 for the status check. 75 wait_timeout: (datetime.timedelta) Maximum time to wait for the 76 confirmation. 77 wait_polling_period: (datetime.timedelta) Performance Dashboard will be 78 polled every wait_polling_period amount of time. 79 """ 80 assert wait_polling_period <= wait_timeout 81 82 headers = {'Authorization': 'Bearer %s' % oauth_token} 83 http = httplib2.Http() 84 85 response = None 86 resp_json = None 87 current_time = datetime.datetime.now() 88 end_time = current_time + wait_timeout 89 next_poll_time = current_time + wait_polling_period 90 while datetime.datetime.now() < end_time: 91 current_time = datetime.datetime.now() 92 if next_poll_time > current_time: 93 time.sleep((next_poll_time - current_time).total_seconds()) 94 next_poll_time = datetime.datetime.now() + wait_polling_period 95 96 response, content = http.request(url + '/uploads' + upload_token, 97 method='GET', headers=headers) 98 resp_json = json.loads(content) 99 100 print 'Upload state polled. Response: %s.' % content 101 102 if (response.status != 200 or 103 resp_json['state'] == 'COMPLETED' or 104 resp_json['state'] == 'FAILED'): 105 break 106 107 return response, resp_json 108 109 110# TODO(https://crbug.com/1029452): HACKHACK 111# Remove once we have doubles in the proto and handle -infinity correctly. 112def _ApplyHacks(dicts): 113 for d in dicts: 114 if 'running' in d: 115 116 def _NoInf(value): 117 if value == float('inf'): 118 return histogram.JS_MAX_VALUE 119 if value == float('-inf'): 120 return -histogram.JS_MAX_VALUE 121 return value 122 123 d['running'] = [_NoInf(value) for value in d['running']] 124 125 return dicts 126 127 128def _LoadHistogramSetFromProto(options): 129 hs = histogram_set.HistogramSet() 130 with options.input_results_file as f: 131 hs.ImportProto(f.read()) 132 133 return hs 134 135 136def _AddBuildInfo(histograms, options): 137 common_diagnostics = { 138 reserved_infos.MASTERS: options.perf_dashboard_machine_group, 139 reserved_infos.BOTS: options.bot, 140 reserved_infos.POINT_ID: options.commit_position, 141 reserved_infos.BENCHMARKS: options.test_suite, 142 reserved_infos.WEBRTC_REVISIONS: str(options.webrtc_git_hash), 143 reserved_infos.BUILD_URLS: options.build_page_url, 144 } 145 146 for k, v in common_diagnostics.items(): 147 histograms.AddSharedDiagnosticToAllHistograms( 148 k.name, generic_set.GenericSet([v])) 149 150 151def _DumpOutput(histograms, output_file): 152 with output_file: 153 json.dump(_ApplyHacks(histograms.AsDicts()), output_file, indent=4) 154 155 156def UploadToDashboard(options): 157 histograms = _LoadHistogramSetFromProto(options) 158 _AddBuildInfo(histograms, options) 159 160 if options.output_json_file: 161 _DumpOutput(histograms, options.output_json_file) 162 163 oauth_token = _GenerateOauthToken() 164 response, content = _SendHistogramSet( 165 options.dashboard_url, histograms, oauth_token) 166 167 upload_token = json.loads(content).get('token') 168 if not options.wait_for_upload or not upload_token: 169 print 'Not waiting for upload status confirmation.' 170 if response.status == 200: 171 print 'Received 200 from dashboard.' 172 return 0 173 else: 174 print('Upload failed with %d: %s\n\n%s' % (response.status, 175 response.reason, content)) 176 return 1 177 178 response, resp_json = _WaitForUploadConfirmation( 179 options.dashboard_url, 180 oauth_token, 181 upload_token, 182 datetime.timedelta(seconds=options.wait_timeout_sec), 183 datetime.timedelta(seconds=options.wait_polling_period_sec)) 184 185 if response.status != 200 or resp_json['state'] == 'FAILED': 186 print('Upload failed with %d: %s\n\n%s' % (response.status, 187 response.reason, 188 str(resp_json))) 189 return 1 190 191 if resp_json['state'] == 'COMPLETED': 192 print 'Upload completed.' 193 return 0 194 195 print('Upload wasn\'t completed in a given time: %d.', options.wait_timeout) 196 return 1 197