1#!/usr/bin/env python3 2# Copyright 2017 gRPC authors. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16# Utilities for manipulating JSON data that represents microbenchmark results. 17 18import os 19 20# template arguments and dynamic arguments of individual benchmark types 21# Example benchmark name: "BM_UnaryPingPong<TCP, NoOpMutator, NoOpMutator>/0/0" 22_BM_SPECS = { 23 'BM_UnaryPingPong': { 24 'tpl': ['fixture', 'client_mutator', 'server_mutator'], 25 'dyn': ['request_size', 'response_size'], 26 }, 27 'BM_PumpStreamClientToServer': { 28 'tpl': ['fixture'], 29 'dyn': ['request_size'], 30 }, 31 'BM_PumpStreamServerToClient': { 32 'tpl': ['fixture'], 33 'dyn': ['request_size'], 34 }, 35 'BM_StreamingPingPong': { 36 'tpl': ['fixture', 'client_mutator', 'server_mutator'], 37 'dyn': ['request_size', 'request_count'], 38 }, 39 'BM_StreamingPingPongMsgs': { 40 'tpl': ['fixture', 'client_mutator', 'server_mutator'], 41 'dyn': ['request_size'], 42 }, 43 'BM_PumpStreamServerToClient_Trickle': { 44 'tpl': [], 45 'dyn': ['request_size', 'bandwidth_kilobits'], 46 }, 47 'BM_PumpUnbalancedUnary_Trickle': { 48 'tpl': [], 49 'dyn': ['cli_req_size', 'svr_req_size', 'bandwidth_kilobits'], 50 }, 51 'BM_ErrorStringOnNewError': { 52 'tpl': ['fixture'], 53 'dyn': [], 54 }, 55 'BM_ErrorStringRepeatedly': { 56 'tpl': ['fixture'], 57 'dyn': [], 58 }, 59 'BM_ErrorGetStatus': { 60 'tpl': ['fixture'], 61 'dyn': [], 62 }, 63 'BM_ErrorGetStatusCode': { 64 'tpl': ['fixture'], 65 'dyn': [], 66 }, 67 'BM_ErrorHttpError': { 68 'tpl': ['fixture'], 69 'dyn': [], 70 }, 71 'BM_HasClearGrpcStatus': { 72 'tpl': ['fixture'], 73 'dyn': [], 74 }, 75 'BM_IsolatedFilter': { 76 'tpl': ['fixture', 'client_mutator'], 77 'dyn': [], 78 }, 79 'BM_HpackEncoderEncodeHeader': { 80 'tpl': ['fixture'], 81 'dyn': ['end_of_stream', 'request_size'], 82 }, 83 'BM_HpackParserParseHeader': { 84 'tpl': ['fixture'], 85 'dyn': [], 86 }, 87 'BM_CallCreateDestroy': { 88 'tpl': ['fixture'], 89 'dyn': [], 90 }, 91 'BM_Zalloc': { 92 'tpl': [], 93 'dyn': ['request_size'], 94 }, 95 'BM_PollEmptyPollset_SpeedOfLight': { 96 'tpl': [], 97 'dyn': ['request_size', 'request_count'], 98 }, 99 'BM_StreamCreateSendInitialMetadataDestroy': { 100 'tpl': ['fixture'], 101 'dyn': [], 102 }, 103 'BM_TransportStreamSend': { 104 'tpl': [], 105 'dyn': ['request_size'], 106 }, 107 'BM_TransportStreamRecv': { 108 'tpl': [], 109 'dyn': ['request_size'], 110 }, 111 'BM_StreamingPingPongWithCoalescingApi': { 112 'tpl': ['fixture', 'client_mutator', 'server_mutator'], 113 'dyn': ['request_size', 'request_count', 'end_of_stream'], 114 }, 115 'BM_Base16SomeStuff': { 116 'tpl': [], 117 'dyn': ['request_size'], 118 } 119} 120 121 122def numericalize(s): 123 """Convert abbreviations like '100M' or '10k' to a number.""" 124 if not s: 125 return '' 126 if s[-1] == 'k': 127 return float(s[:-1]) * 1024 128 if s[-1] == 'M': 129 return float(s[:-1]) * 1024 * 1024 130 if 0 <= (ord(s[-1]) - ord('0')) <= 9: 131 return float(s) 132 assert 'not a number: %s' % s 133 134 135def parse_name(name): 136 cpp_name = name 137 if '<' not in name and '/' not in name and name not in _BM_SPECS: 138 return {'name': name, 'cpp_name': name} 139 rest = name 140 out = {} 141 tpl_args = [] 142 dyn_args = [] 143 if '<' in rest: 144 tpl_bit = rest[rest.find('<') + 1:rest.rfind('>')] 145 arg = '' 146 nesting = 0 147 for c in tpl_bit: 148 if c == '<': 149 nesting += 1 150 arg += c 151 elif c == '>': 152 nesting -= 1 153 arg += c 154 elif c == ',': 155 if nesting == 0: 156 tpl_args.append(arg.strip()) 157 arg = '' 158 else: 159 arg += c 160 else: 161 arg += c 162 tpl_args.append(arg.strip()) 163 rest = rest[:rest.find('<')] + rest[rest.rfind('>') + 1:] 164 if '/' in rest: 165 s = rest.split('/') 166 rest = s[0] 167 dyn_args = s[1:] 168 name = rest 169 assert name in _BM_SPECS, '_BM_SPECS needs to be expanded for %s' % name 170 assert len(dyn_args) == len(_BM_SPECS[name]['dyn']) 171 assert len(tpl_args) == len(_BM_SPECS[name]['tpl']) 172 out['name'] = name 173 out['cpp_name'] = cpp_name 174 out.update( 175 dict((k, numericalize(v)) 176 for k, v in zip(_BM_SPECS[name]['dyn'], dyn_args))) 177 out.update(dict(zip(_BM_SPECS[name]['tpl'], tpl_args))) 178 return out 179 180 181def expand_json(js, js2=None): 182 if not js and not js2: 183 raise StopIteration() 184 if not js: 185 js = js2 186 for bm in js['benchmarks']: 187 if bm['name'].endswith('_stddev') or bm['name'].endswith('_mean'): 188 continue 189 context = js['context'] 190 if 'label' in bm: 191 labels_list = [ 192 s.split(':') 193 for s in bm['label'].strip().split(' ') 194 if len(s) and s[0] != '#' 195 ] 196 for el in labels_list: 197 el[0] = el[0].replace('/iter', '_per_iteration') 198 labels = dict(labels_list) 199 else: 200 labels = {} 201 row = { 202 'jenkins_build': os.environ.get('BUILD_NUMBER', ''), 203 'jenkins_job': os.environ.get('JOB_NAME', ''), 204 } 205 row.update(context) 206 row.update(bm) 207 row.update(parse_name(row['name'])) 208 row.update(labels) 209 if js2: 210 for bm2 in js2['benchmarks']: 211 if bm['name'] == bm2['name'] and 'already_used' not in bm2: 212 row['cpu_time'] = bm2['cpu_time'] 213 row['real_time'] = bm2['real_time'] 214 row['iterations'] = bm2['iterations'] 215 bm2['already_used'] = True 216 break 217 yield row 218