1#!/usr/bin/env python
2# Copyright (c) 2018 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6from __future__ import print_function
7
8import re
9import scm
10import subprocess2
11import sys
12
13try:
14  import urlparse
15except ImportError:  # For Py3 compatibility
16  import urllib.parse as urlparse
17
18
19# Current version of metrics recording.
20# When we add new metrics, the version number will be increased, we display the
21# user what has changed, and ask the user to agree again.
22CURRENT_VERSION = 1
23
24APP_URL = 'https://cit-cli-metrics.appspot.com'
25
26def get_notice_countdown_header(countdown):
27  if countdown == 0:
28    yield '     METRICS COLLECTION IS TAKING PLACE'
29  else:
30    yield '  METRICS COLLECTION WILL START IN %d EXECUTIONS' % countdown
31
32def get_notice_version_change_header():
33  yield '       WE ARE COLLECTING ADDITIONAL METRICS'
34  yield ''
35  yield ' Please review the changes and opt-in again.'
36
37def get_notice_footer():
38  yield 'To suppress this message opt in or out using:'
39  yield '$ gclient metrics [--opt-in] [--opt-out]'
40  yield 'For more information please see metrics.README.md'
41  yield 'in your depot_tools checkout or visit'
42  yield 'https://goo.gl/yNpRDV.'
43
44def get_change_notice(version):
45  if version == 0:
46    pass # No changes for version 0
47  elif version == 1:
48    yield 'We want to collect the Git version.'
49    yield 'We want to collect information about the HTTP'
50    yield 'requests that depot_tools makes, and the git and'
51    yield 'cipd commands it executes.'
52    yield ''
53    yield 'We only collect known strings to make sure we'
54    yield 'don\'t record PII.'
55
56
57KNOWN_PROJECT_URLS = {
58  'https://chrome-internal.googlesource.com/chrome/ios_internal',
59  'https://chrome-internal.googlesource.com/infra/infra_internal',
60  'https://chromium.googlesource.com/breakpad/breakpad',
61  'https://chromium.googlesource.com/chromium/src',
62  'https://chromium.googlesource.com/chromium/tools/depot_tools',
63  'https://chromium.googlesource.com/crashpad/crashpad',
64  'https://chromium.googlesource.com/external/gyp',
65  'https://chromium.googlesource.com/external/naclports',
66  'https://chromium.googlesource.com/infra/goma/client',
67  'https://chromium.googlesource.com/infra/infra',
68  'https://chromium.googlesource.com/native_client/',
69  'https://chromium.googlesource.com/syzygy',
70  'https://chromium.googlesource.com/v8/v8',
71  'https://dart.googlesource.com/sdk',
72  'https://pdfium.googlesource.com/pdfium',
73  'https://skia.googlesource.com/buildbot',
74  'https://skia.googlesource.com/skia',
75  'https://webrtc.googlesource.com/src',
76}
77
78KNOWN_HTTP_HOSTS = {
79  'chrome-internal-review.googlesource.com',
80  'chromium-review.googlesource.com',
81  'dart-review.googlesource.com',
82  'eu1-mirror-chromium-review.googlesource.com',
83  'pdfium-review.googlesource.com',
84  'skia-review.googlesource.com',
85  'us1-mirror-chromium-review.googlesource.com',
86  'us2-mirror-chromium-review.googlesource.com',
87  'us3-mirror-chromium-review.googlesource.com',
88  'webrtc-review.googlesource.com',
89}
90
91KNOWN_HTTP_METHODS = {
92  'DELETE',
93  'GET',
94  'PATCH',
95  'POST',
96  'PUT',
97}
98
99KNOWN_HTTP_PATHS = {
100  'accounts':
101      re.compile(r'(/a)?/accounts/.*'),
102  'changes':
103      re.compile(r'(/a)?/changes/([^/]+)?$'),
104  'changes/abandon':
105      re.compile(r'(/a)?/changes/.*/abandon'),
106  'changes/comments':
107      re.compile(r'(/a)?/changes/.*/comments'),
108  'changes/detail':
109      re.compile(r'(/a)?/changes/.*/detail'),
110  'changes/edit':
111      re.compile(r'(/a)?/changes/.*/edit'),
112  'changes/message':
113      re.compile(r'(/a)?/changes/.*/message'),
114  'changes/restore':
115      re.compile(r'(/a)?/changes/.*/restore'),
116  'changes/reviewers':
117      re.compile(r'(/a)?/changes/.*/reviewers/.*'),
118  'changes/revisions/commit':
119      re.compile(r'(/a)?/changes/.*/revisions/.*/commit'),
120  'changes/revisions/review':
121      re.compile(r'(/a)?/changes/.*/revisions/.*/review'),
122  'changes/submit':
123      re.compile(r'(/a)?/changes/.*/submit'),
124  'projects/branches':
125      re.compile(r'(/a)?/projects/.*/branches/.*'),
126}
127
128KNOWN_HTTP_ARGS = {
129  'ALL_REVISIONS',
130  'CURRENT_COMMIT',
131  'CURRENT_REVISION',
132  'DETAILED_ACCOUNTS',
133  'LABELS',
134}
135
136GIT_VERSION_RE = re.compile(
137  r'git version (\d)\.(\d{0,2})\.(\d{0,2})'
138)
139
140KNOWN_SUBCOMMAND_ARGS = {
141  'cc',
142  'hashtag',
143  'l=Auto-Submit+1',
144  'l=Code-Review+1',
145  'l=Code-Review+2',
146  'l=Commit-Queue+1',
147  'l=Commit-Queue+2',
148  'label',
149  'm',
150  'notify=ALL',
151  'notify=NONE',
152  'private',
153  'r',
154  'ready',
155  'topic',
156  'wip'
157}
158
159
160def get_python_version():
161  """Return the python version in the major.minor.micro format."""
162  return '{v.major}.{v.minor}.{v.micro}'.format(v=sys.version_info)
163
164
165def get_git_version():
166  """Return the Git version in the major.minor.micro format."""
167  p = subprocess2.Popen(
168      ['git', '--version'],
169      stdout=subprocess2.PIPE, stderr=subprocess2.PIPE)
170  stdout, _ = p.communicate()
171  match = GIT_VERSION_RE.match(stdout.decode('utf-8'))
172  if not match:
173    return None
174  return '%s.%s.%s' % match.groups()
175
176
177def return_code_from_exception(exception):
178  """Returns the exit code that would result of raising the exception."""
179  if exception is None:
180    return 0
181  if isinstance(exception[1], SystemExit):
182    return exception[1].code
183  return 1
184
185
186def extract_known_subcommand_args(args):
187  """Extract the known arguments from the passed list of args."""
188  known_args = []
189  for arg in args:
190    if arg in KNOWN_SUBCOMMAND_ARGS:
191      known_args.append(arg)
192    else:
193      arg = arg.split('=')[0]
194      if arg in KNOWN_SUBCOMMAND_ARGS:
195        known_args.append(arg)
196  return sorted(known_args)
197
198
199def extract_http_metrics(request_uri, method, status, response_time):
200  """Extract metrics from the request URI.
201
202  Extracts the host, path, and arguments from the request URI, and returns them
203  along with the method, status and response time.
204
205  The host, method, path and arguments must be in the KNOWN_HTTP_* constants
206  defined above.
207
208  Arguments are the values of the o= url parameter. In Gerrit, additional fields
209  can be obtained by adding o parameters, each option requires more database
210  lookups and slows down the query response time to the client, so we make an
211  effort to collect them.
212
213  The regex defined in KNOWN_HTTP_PATH_RES are checked against the path, and
214  those that match will be returned.
215  """
216  http_metrics = {
217    'status': status,
218    'response_time': response_time,
219  }
220
221  if method in KNOWN_HTTP_METHODS:
222    http_metrics['method'] = method
223
224  parsed_url = urlparse.urlparse(request_uri)
225
226  if parsed_url.netloc in KNOWN_HTTP_HOSTS:
227    http_metrics['host'] = parsed_url.netloc
228
229  for name, path_re in KNOWN_HTTP_PATHS.items():
230    if path_re.match(parsed_url.path):
231      http_metrics['path'] = name
232      break
233
234  parsed_query = urlparse.parse_qs(parsed_url.query)
235
236  # Collect o-parameters from the request.
237  args = [
238    arg for arg in parsed_query.get('o', [])
239    if arg in KNOWN_HTTP_ARGS
240  ]
241  if args:
242    http_metrics['arguments'] = args
243
244  return http_metrics
245
246
247def get_repo_timestamp(path_to_repo):
248  """Get an approximate timestamp for the upstream of |path_to_repo|.
249
250  Returns the top two bits of the timestamp of the HEAD for the upstream of the
251  branch path_to_repo is checked out at.
252  """
253  # Get the upstream for the current branch. If we're not in a branch, fallback
254  # to HEAD.
255  try:
256    upstream = scm.GIT.GetUpstreamBranch(path_to_repo) or 'HEAD'
257  except subprocess2.CalledProcessError:
258    upstream = 'HEAD'
259
260  # Get the timestamp of the HEAD for the upstream of the current branch.
261  p = subprocess2.Popen(
262      ['git', '-C', path_to_repo, 'log', '-n1', upstream, '--format=%at'],
263      stdout=subprocess2.PIPE, stderr=subprocess2.PIPE)
264  stdout, _ = p.communicate()
265
266  # If there was an error, give up.
267  if p.returncode != 0:
268    return None
269
270  return stdout.strip()
271
272def print_boxed_text(out, min_width, lines):
273  [EW, NS, SE, SW, NE, NW] = list('=|++++')
274  width = max(min_width, max(len(line) for line in lines))
275  out(SE + EW * (width + 2) + SW + '\n')
276  for line in lines:
277    out('%s %-*s %s\n' % (NS, width, line, NS))
278  out(NE + EW * (width + 2) + NW + '\n')
279
280def print_notice(countdown):
281  """Print a notice to let the user know the status of metrics collection."""
282  lines = list(get_notice_countdown_header(countdown))
283  lines.append('')
284  lines += list(get_notice_footer())
285  print_boxed_text(sys.stderr.write, 49, lines)
286
287def print_version_change(config_version):
288  """Print a notice to let the user know we are collecting more metrics."""
289  lines = list(get_notice_version_change_header())
290  for version in range(config_version + 1, CURRENT_VERSION + 1):
291    lines.append('')
292    lines += list(get_change_notice(version))
293  print_boxed_text(sys.stderr.write, 49, lines)
294