1# Copyright 2015 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4"""Provides a web interface for loading graph data from the production server.
5
6This is meant to be used on a dev server only.
7"""
8from __future__ import print_function
9from __future__ import division
10from __future__ import absolute_import
11
12import base64
13import json
14import os
15import urllib
16
17from google.appengine.api import app_identity
18from google.appengine.api import urlfetch
19from google.appengine.ext import ndb
20from google.appengine.ext.ndb import model
21
22from dashboard import update_test_suites
23from dashboard.common import datastore_hooks
24from dashboard.common import request_handler
25
26_DEV_APP_ID = 'dev~' + app_identity.get_application_id()
27
28_PROD_DUMP_GRAPH_JSON_URL = 'https://chromeperf.appspot.com/dump_graph_json'
29
30
31class LoadFromProdHandler(request_handler.RequestHandler):
32  """Debugging handler to load data from the production instance."""
33
34  def get(self):
35    if 'Development' not in os.environ['SERVER_SOFTWARE']:
36      self.RenderHtml('result.html', {
37          'errors':
38              ['This should not be run in production, only on dev server.']
39      })
40      return
41    self.RenderHtml('load_from_prod.html', {})
42
43  def post(self):
44    """Loads the requested data from the production server."""
45    if 'Development' not in os.environ['SERVER_SOFTWARE']:
46      self.RenderHtml('result.html', {
47          'errors':
48              ['This should not be run in production, only on dev server.']
49      })
50      return
51
52    sheriff = self.request.get('sheriff')
53    test_path = self.request.get('test_path')
54    raw_json = self.request.get('raw_json')
55    protos = None
56    if test_path:
57      num_points = self.request.get('num_points')
58      end_rev = self.request.get('end_rev')
59      url = ('%s?test_path=%s&num_points=%s' %
60             (_PROD_DUMP_GRAPH_JSON_URL, urllib.quote(test_path), num_points))
61      if end_rev:
62        url += '&end_rev=%s' % end_rev
63    elif sheriff:
64      sheriff_name = self.request.get('sheriff')
65      num_alerts = self.request.get('num_alerts')
66      num_points = self.request.get('num_points')
67      url = ('%s?sheriff=%s&num_alerts=%s&num_points=%s' %
68             (_PROD_DUMP_GRAPH_JSON_URL, urllib.quote(sheriff_name), num_alerts,
69              num_points))
70    elif raw_json:
71      protos = json.loads(raw_json)
72    else:
73      self.RenderHtml('result.html', {
74          'errors': ['Need to specify a test_path, sheriff or json data file.']
75      })
76      return
77
78    if not protos:
79      # This takes a while.
80      response = urlfetch.fetch(url, deadline=60)
81      if response.status_code != 200:
82        msg_template = 'Could not fetch %s (Status: %s)'
83        err_msg = msg_template % (url, response.status_code)
84        self.RenderHtml('result.html', {'errors': [err_msg]})
85        return
86      protos = json.loads(response.content)
87
88    kinds = ['Master', 'Bot', 'TestMetadata', 'Row', 'Sheriff', 'Anomaly']
89    entities = {k: [] for k in kinds}
90    for proto in protos:
91      pb = model.entity_pb.EntityProto(base64.b64decode(proto))
92      # App ID is set in the key and all the ReferenceProperty keys to
93      # 's~chromeperf'. It won't be found in queries unless we use the
94      # devserver app ID.
95      key = pb.mutable_key()
96      key.set_app(_DEV_APP_ID)
97      for prop in pb.property_list():
98        val = prop.mutable_value()
99        if val.has_referencevalue():
100          ref = val.mutable_referencevalue()
101          ref.set_app(_DEV_APP_ID)
102      entity = ndb.ModelAdapter().pb_to_entity(pb)
103      entities[entity.key.kind()].append(entity)
104
105    for kind in kinds:
106      ndb.put_multi(entities[kind])
107
108    update_test_suites.UpdateTestSuites(datastore_hooks.INTERNAL)
109    update_test_suites.UpdateTestSuites(datastore_hooks.EXTERNAL)
110
111    num_entities = sum(len(entities[kind]) for kind in kinds)
112    self.RenderHtml('result.html', {
113        'results': [{
114            'name': 'Added data',
115            'value': '%d entities' % num_entities
116        }]
117    })
118