1# Copyright 2015 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4"""Provides a web interface for loading graph data from the production server. 5 6This is meant to be used on a dev server only. 7""" 8from __future__ import print_function 9from __future__ import division 10from __future__ import absolute_import 11 12import base64 13import json 14import os 15import urllib 16 17from google.appengine.api import app_identity 18from google.appengine.api import urlfetch 19from google.appengine.ext import ndb 20from google.appengine.ext.ndb import model 21 22from dashboard import update_test_suites 23from dashboard.common import datastore_hooks 24from dashboard.common import request_handler 25 26_DEV_APP_ID = 'dev~' + app_identity.get_application_id() 27 28_PROD_DUMP_GRAPH_JSON_URL = 'https://chromeperf.appspot.com/dump_graph_json' 29 30 31class LoadFromProdHandler(request_handler.RequestHandler): 32 """Debugging handler to load data from the production instance.""" 33 34 def get(self): 35 if 'Development' not in os.environ['SERVER_SOFTWARE']: 36 self.RenderHtml('result.html', { 37 'errors': 38 ['This should not be run in production, only on dev server.'] 39 }) 40 return 41 self.RenderHtml('load_from_prod.html', {}) 42 43 def post(self): 44 """Loads the requested data from the production server.""" 45 if 'Development' not in os.environ['SERVER_SOFTWARE']: 46 self.RenderHtml('result.html', { 47 'errors': 48 ['This should not be run in production, only on dev server.'] 49 }) 50 return 51 52 sheriff = self.request.get('sheriff') 53 test_path = self.request.get('test_path') 54 raw_json = self.request.get('raw_json') 55 protos = None 56 if test_path: 57 num_points = self.request.get('num_points') 58 end_rev = self.request.get('end_rev') 59 url = ('%s?test_path=%s&num_points=%s' % 60 (_PROD_DUMP_GRAPH_JSON_URL, urllib.quote(test_path), num_points)) 61 if end_rev: 62 url += '&end_rev=%s' % end_rev 63 elif sheriff: 64 sheriff_name = self.request.get('sheriff') 65 num_alerts = self.request.get('num_alerts') 66 num_points = self.request.get('num_points') 67 url = ('%s?sheriff=%s&num_alerts=%s&num_points=%s' % 68 (_PROD_DUMP_GRAPH_JSON_URL, urllib.quote(sheriff_name), num_alerts, 69 num_points)) 70 elif raw_json: 71 protos = json.loads(raw_json) 72 else: 73 self.RenderHtml('result.html', { 74 'errors': ['Need to specify a test_path, sheriff or json data file.'] 75 }) 76 return 77 78 if not protos: 79 # This takes a while. 80 response = urlfetch.fetch(url, deadline=60) 81 if response.status_code != 200: 82 msg_template = 'Could not fetch %s (Status: %s)' 83 err_msg = msg_template % (url, response.status_code) 84 self.RenderHtml('result.html', {'errors': [err_msg]}) 85 return 86 protos = json.loads(response.content) 87 88 kinds = ['Master', 'Bot', 'TestMetadata', 'Row', 'Sheriff', 'Anomaly'] 89 entities = {k: [] for k in kinds} 90 for proto in protos: 91 pb = model.entity_pb.EntityProto(base64.b64decode(proto)) 92 # App ID is set in the key and all the ReferenceProperty keys to 93 # 's~chromeperf'. It won't be found in queries unless we use the 94 # devserver app ID. 95 key = pb.mutable_key() 96 key.set_app(_DEV_APP_ID) 97 for prop in pb.property_list(): 98 val = prop.mutable_value() 99 if val.has_referencevalue(): 100 ref = val.mutable_referencevalue() 101 ref.set_app(_DEV_APP_ID) 102 entity = ndb.ModelAdapter().pb_to_entity(pb) 103 entities[entity.key.kind()].append(entity) 104 105 for kind in kinds: 106 ndb.put_multi(entities[kind]) 107 108 update_test_suites.UpdateTestSuites(datastore_hooks.INTERNAL) 109 update_test_suites.UpdateTestSuites(datastore_hooks.EXTERNAL) 110 111 num_entities = sum(len(entities[kind]) for kind in kinds) 112 self.RenderHtml('result.html', { 113 'results': [{ 114 'name': 'Added data', 115 'value': '%d entities' % num_entities 116 }] 117 }) 118