1#!/usr/bin/env python 2# Copyright (c) 2018 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6from __future__ import print_function 7 8import re 9import scm 10import subprocess2 11import sys 12 13try: 14 import urlparse 15except ImportError: # For Py3 compatibility 16 import urllib.parse as urlparse 17 18 19# Current version of metrics recording. 20# When we add new metrics, the version number will be increased, we display the 21# user what has changed, and ask the user to agree again. 22CURRENT_VERSION = 1 23 24APP_URL = 'https://cit-cli-metrics.appspot.com' 25 26def get_notice_countdown_header(countdown): 27 if countdown == 0: 28 yield ' METRICS COLLECTION IS TAKING PLACE' 29 else: 30 yield ' METRICS COLLECTION WILL START IN %d EXECUTIONS' % countdown 31 32def get_notice_version_change_header(): 33 yield ' WE ARE COLLECTING ADDITIONAL METRICS' 34 yield '' 35 yield ' Please review the changes and opt-in again.' 36 37def get_notice_footer(): 38 yield 'To suppress this message opt in or out using:' 39 yield '$ gclient metrics [--opt-in] [--opt-out]' 40 yield 'For more information please see metrics.README.md' 41 yield 'in your depot_tools checkout or visit' 42 yield 'https://goo.gl/yNpRDV.' 43 44def get_change_notice(version): 45 if version == 0: 46 pass # No changes for version 0 47 elif version == 1: 48 yield 'We want to collect the Git version.' 49 yield 'We want to collect information about the HTTP' 50 yield 'requests that depot_tools makes, and the git and' 51 yield 'cipd commands it executes.' 52 yield '' 53 yield 'We only collect known strings to make sure we' 54 yield 'don\'t record PII.' 55 56 57KNOWN_PROJECT_URLS = { 58 'https://chrome-internal.googlesource.com/chrome/ios_internal', 59 'https://chrome-internal.googlesource.com/infra/infra_internal', 60 'https://chromium.googlesource.com/breakpad/breakpad', 61 'https://chromium.googlesource.com/chromium/src', 62 'https://chromium.googlesource.com/chromium/tools/depot_tools', 63 'https://chromium.googlesource.com/crashpad/crashpad', 64 'https://chromium.googlesource.com/external/gyp', 65 'https://chromium.googlesource.com/external/naclports', 66 'https://chromium.googlesource.com/infra/goma/client', 67 'https://chromium.googlesource.com/infra/infra', 68 'https://chromium.googlesource.com/native_client/', 69 'https://chromium.googlesource.com/syzygy', 70 'https://chromium.googlesource.com/v8/v8', 71 'https://dart.googlesource.com/sdk', 72 'https://pdfium.googlesource.com/pdfium', 73 'https://skia.googlesource.com/buildbot', 74 'https://skia.googlesource.com/skia', 75 'https://webrtc.googlesource.com/src', 76} 77 78KNOWN_HTTP_HOSTS = { 79 'chrome-internal-review.googlesource.com', 80 'chromium-review.googlesource.com', 81 'dart-review.googlesource.com', 82 'eu1-mirror-chromium-review.googlesource.com', 83 'pdfium-review.googlesource.com', 84 'skia-review.googlesource.com', 85 'us1-mirror-chromium-review.googlesource.com', 86 'us2-mirror-chromium-review.googlesource.com', 87 'us3-mirror-chromium-review.googlesource.com', 88 'webrtc-review.googlesource.com', 89} 90 91KNOWN_HTTP_METHODS = { 92 'DELETE', 93 'GET', 94 'PATCH', 95 'POST', 96 'PUT', 97} 98 99KNOWN_HTTP_PATHS = { 100 'accounts': 101 re.compile(r'(/a)?/accounts/.*'), 102 'changes': 103 re.compile(r'(/a)?/changes/([^/]+)?$'), 104 'changes/abandon': 105 re.compile(r'(/a)?/changes/.*/abandon'), 106 'changes/comments': 107 re.compile(r'(/a)?/changes/.*/comments'), 108 'changes/detail': 109 re.compile(r'(/a)?/changes/.*/detail'), 110 'changes/edit': 111 re.compile(r'(/a)?/changes/.*/edit'), 112 'changes/message': 113 re.compile(r'(/a)?/changes/.*/message'), 114 'changes/restore': 115 re.compile(r'(/a)?/changes/.*/restore'), 116 'changes/reviewers': 117 re.compile(r'(/a)?/changes/.*/reviewers/.*'), 118 'changes/revisions/commit': 119 re.compile(r'(/a)?/changes/.*/revisions/.*/commit'), 120 'changes/revisions/review': 121 re.compile(r'(/a)?/changes/.*/revisions/.*/review'), 122 'changes/submit': 123 re.compile(r'(/a)?/changes/.*/submit'), 124 'projects/branches': 125 re.compile(r'(/a)?/projects/.*/branches/.*'), 126} 127 128KNOWN_HTTP_ARGS = { 129 'ALL_REVISIONS', 130 'CURRENT_COMMIT', 131 'CURRENT_REVISION', 132 'DETAILED_ACCOUNTS', 133 'LABELS', 134} 135 136GIT_VERSION_RE = re.compile( 137 r'git version (\d)\.(\d{0,2})\.(\d{0,2})' 138) 139 140KNOWN_SUBCOMMAND_ARGS = { 141 'cc', 142 'hashtag', 143 'l=Auto-Submit+1', 144 'l=Code-Review+1', 145 'l=Code-Review+2', 146 'l=Commit-Queue+1', 147 'l=Commit-Queue+2', 148 'label', 149 'm', 150 'notify=ALL', 151 'notify=NONE', 152 'private', 153 'r', 154 'ready', 155 'topic', 156 'wip' 157} 158 159 160def get_python_version(): 161 """Return the python version in the major.minor.micro format.""" 162 return '{v.major}.{v.minor}.{v.micro}'.format(v=sys.version_info) 163 164 165def get_git_version(): 166 """Return the Git version in the major.minor.micro format.""" 167 p = subprocess2.Popen( 168 ['git', '--version'], 169 stdout=subprocess2.PIPE, stderr=subprocess2.PIPE) 170 stdout, _ = p.communicate() 171 match = GIT_VERSION_RE.match(stdout.decode('utf-8')) 172 if not match: 173 return None 174 return '%s.%s.%s' % match.groups() 175 176 177def return_code_from_exception(exception): 178 """Returns the exit code that would result of raising the exception.""" 179 if exception is None: 180 return 0 181 if isinstance(exception[1], SystemExit): 182 return exception[1].code 183 return 1 184 185 186def extract_known_subcommand_args(args): 187 """Extract the known arguments from the passed list of args.""" 188 known_args = [] 189 for arg in args: 190 if arg in KNOWN_SUBCOMMAND_ARGS: 191 known_args.append(arg) 192 else: 193 arg = arg.split('=')[0] 194 if arg in KNOWN_SUBCOMMAND_ARGS: 195 known_args.append(arg) 196 return sorted(known_args) 197 198 199def extract_http_metrics(request_uri, method, status, response_time): 200 """Extract metrics from the request URI. 201 202 Extracts the host, path, and arguments from the request URI, and returns them 203 along with the method, status and response time. 204 205 The host, method, path and arguments must be in the KNOWN_HTTP_* constants 206 defined above. 207 208 Arguments are the values of the o= url parameter. In Gerrit, additional fields 209 can be obtained by adding o parameters, each option requires more database 210 lookups and slows down the query response time to the client, so we make an 211 effort to collect them. 212 213 The regex defined in KNOWN_HTTP_PATH_RES are checked against the path, and 214 those that match will be returned. 215 """ 216 http_metrics = { 217 'status': status, 218 'response_time': response_time, 219 } 220 221 if method in KNOWN_HTTP_METHODS: 222 http_metrics['method'] = method 223 224 parsed_url = urlparse.urlparse(request_uri) 225 226 if parsed_url.netloc in KNOWN_HTTP_HOSTS: 227 http_metrics['host'] = parsed_url.netloc 228 229 for name, path_re in KNOWN_HTTP_PATHS.items(): 230 if path_re.match(parsed_url.path): 231 http_metrics['path'] = name 232 break 233 234 parsed_query = urlparse.parse_qs(parsed_url.query) 235 236 # Collect o-parameters from the request. 237 args = [ 238 arg for arg in parsed_query.get('o', []) 239 if arg in KNOWN_HTTP_ARGS 240 ] 241 if args: 242 http_metrics['arguments'] = args 243 244 return http_metrics 245 246 247def get_repo_timestamp(path_to_repo): 248 """Get an approximate timestamp for the upstream of |path_to_repo|. 249 250 Returns the top two bits of the timestamp of the HEAD for the upstream of the 251 branch path_to_repo is checked out at. 252 """ 253 # Get the upstream for the current branch. If we're not in a branch, fallback 254 # to HEAD. 255 try: 256 upstream = scm.GIT.GetUpstreamBranch(path_to_repo) or 'HEAD' 257 except subprocess2.CalledProcessError: 258 upstream = 'HEAD' 259 260 # Get the timestamp of the HEAD for the upstream of the current branch. 261 p = subprocess2.Popen( 262 ['git', '-C', path_to_repo, 'log', '-n1', upstream, '--format=%at'], 263 stdout=subprocess2.PIPE, stderr=subprocess2.PIPE) 264 stdout, _ = p.communicate() 265 266 # If there was an error, give up. 267 if p.returncode != 0: 268 return None 269 270 return stdout.strip() 271 272def print_boxed_text(out, min_width, lines): 273 [EW, NS, SE, SW, NE, NW] = list('=|++++') 274 width = max(min_width, max(len(line) for line in lines)) 275 out(SE + EW * (width + 2) + SW + '\n') 276 for line in lines: 277 out('%s %-*s %s\n' % (NS, width, line, NS)) 278 out(NE + EW * (width + 2) + NW + '\n') 279 280def print_notice(countdown): 281 """Print a notice to let the user know the status of metrics collection.""" 282 lines = list(get_notice_countdown_header(countdown)) 283 lines.append('') 284 lines += list(get_notice_footer()) 285 print_boxed_text(sys.stderr.write, 49, lines) 286 287def print_version_change(config_version): 288 """Print a notice to let the user know we are collecting more metrics.""" 289 lines = list(get_notice_version_change_header()) 290 for version in range(config_version + 1, CURRENT_VERSION + 1): 291 lines.append('') 292 lines += list(get_change_notice(version)) 293 print_boxed_text(sys.stderr.write, 49, lines) 294