1#!/usr/bin/env python3
2# Copyright 2017 gRPC authors.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#     http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16# Utilities for manipulating JSON data that represents microbenchmark results.
17
18import os
19
20# template arguments and dynamic arguments of individual benchmark types
21# Example benchmark name: "BM_UnaryPingPong<TCP, NoOpMutator, NoOpMutator>/0/0"
22_BM_SPECS = {
23    'BM_UnaryPingPong': {
24        'tpl': ['fixture', 'client_mutator', 'server_mutator'],
25        'dyn': ['request_size', 'response_size'],
26    },
27    'BM_PumpStreamClientToServer': {
28        'tpl': ['fixture'],
29        'dyn': ['request_size'],
30    },
31    'BM_PumpStreamServerToClient': {
32        'tpl': ['fixture'],
33        'dyn': ['request_size'],
34    },
35    'BM_StreamingPingPong': {
36        'tpl': ['fixture', 'client_mutator', 'server_mutator'],
37        'dyn': ['request_size', 'request_count'],
38    },
39    'BM_StreamingPingPongMsgs': {
40        'tpl': ['fixture', 'client_mutator', 'server_mutator'],
41        'dyn': ['request_size'],
42    },
43    'BM_PumpStreamServerToClient_Trickle': {
44        'tpl': [],
45        'dyn': ['request_size', 'bandwidth_kilobits'],
46    },
47    'BM_PumpUnbalancedUnary_Trickle': {
48        'tpl': [],
49        'dyn': ['cli_req_size', 'svr_req_size', 'bandwidth_kilobits'],
50    },
51    'BM_ErrorStringOnNewError': {
52        'tpl': ['fixture'],
53        'dyn': [],
54    },
55    'BM_ErrorStringRepeatedly': {
56        'tpl': ['fixture'],
57        'dyn': [],
58    },
59    'BM_ErrorGetStatus': {
60        'tpl': ['fixture'],
61        'dyn': [],
62    },
63    'BM_ErrorGetStatusCode': {
64        'tpl': ['fixture'],
65        'dyn': [],
66    },
67    'BM_ErrorHttpError': {
68        'tpl': ['fixture'],
69        'dyn': [],
70    },
71    'BM_HasClearGrpcStatus': {
72        'tpl': ['fixture'],
73        'dyn': [],
74    },
75    'BM_IsolatedFilter': {
76        'tpl': ['fixture', 'client_mutator'],
77        'dyn': [],
78    },
79    'BM_HpackEncoderEncodeHeader': {
80        'tpl': ['fixture'],
81        'dyn': ['end_of_stream', 'request_size'],
82    },
83    'BM_HpackParserParseHeader': {
84        'tpl': ['fixture'],
85        'dyn': [],
86    },
87    'BM_CallCreateDestroy': {
88        'tpl': ['fixture'],
89        'dyn': [],
90    },
91    'BM_Zalloc': {
92        'tpl': [],
93        'dyn': ['request_size'],
94    },
95    'BM_PollEmptyPollset_SpeedOfLight': {
96        'tpl': [],
97        'dyn': ['request_size', 'request_count'],
98    },
99    'BM_StreamCreateSendInitialMetadataDestroy': {
100        'tpl': ['fixture'],
101        'dyn': [],
102    },
103    'BM_TransportStreamSend': {
104        'tpl': [],
105        'dyn': ['request_size'],
106    },
107    'BM_TransportStreamRecv': {
108        'tpl': [],
109        'dyn': ['request_size'],
110    },
111    'BM_StreamingPingPongWithCoalescingApi': {
112        'tpl': ['fixture', 'client_mutator', 'server_mutator'],
113        'dyn': ['request_size', 'request_count', 'end_of_stream'],
114    },
115    'BM_Base16SomeStuff': {
116        'tpl': [],
117        'dyn': ['request_size'],
118    }
119}
120
121
122def numericalize(s):
123    """Convert abbreviations like '100M' or '10k' to a number."""
124    if not s:
125        return ''
126    if s[-1] == 'k':
127        return float(s[:-1]) * 1024
128    if s[-1] == 'M':
129        return float(s[:-1]) * 1024 * 1024
130    if 0 <= (ord(s[-1]) - ord('0')) <= 9:
131        return float(s)
132    assert 'not a number: %s' % s
133
134
135def parse_name(name):
136    cpp_name = name
137    if '<' not in name and '/' not in name and name not in _BM_SPECS:
138        return {'name': name, 'cpp_name': name}
139    rest = name
140    out = {}
141    tpl_args = []
142    dyn_args = []
143    if '<' in rest:
144        tpl_bit = rest[rest.find('<') + 1:rest.rfind('>')]
145        arg = ''
146        nesting = 0
147        for c in tpl_bit:
148            if c == '<':
149                nesting += 1
150                arg += c
151            elif c == '>':
152                nesting -= 1
153                arg += c
154            elif c == ',':
155                if nesting == 0:
156                    tpl_args.append(arg.strip())
157                    arg = ''
158                else:
159                    arg += c
160            else:
161                arg += c
162        tpl_args.append(arg.strip())
163        rest = rest[:rest.find('<')] + rest[rest.rfind('>') + 1:]
164    if '/' in rest:
165        s = rest.split('/')
166        rest = s[0]
167        dyn_args = s[1:]
168    name = rest
169    assert name in _BM_SPECS, '_BM_SPECS needs to be expanded for %s' % name
170    assert len(dyn_args) == len(_BM_SPECS[name]['dyn'])
171    assert len(tpl_args) == len(_BM_SPECS[name]['tpl'])
172    out['name'] = name
173    out['cpp_name'] = cpp_name
174    out.update(
175        dict((k, numericalize(v))
176             for k, v in zip(_BM_SPECS[name]['dyn'], dyn_args)))
177    out.update(dict(zip(_BM_SPECS[name]['tpl'], tpl_args)))
178    return out
179
180
181def expand_json(js, js2=None):
182    if not js and not js2:
183        raise StopIteration()
184    if not js:
185        js = js2
186    for bm in js['benchmarks']:
187        if bm['name'].endswith('_stddev') or bm['name'].endswith('_mean'):
188            continue
189        context = js['context']
190        if 'label' in bm:
191            labels_list = [
192                s.split(':')
193                for s in bm['label'].strip().split(' ')
194                if len(s) and s[0] != '#'
195            ]
196            for el in labels_list:
197                el[0] = el[0].replace('/iter', '_per_iteration')
198            labels = dict(labels_list)
199        else:
200            labels = {}
201        row = {
202            'jenkins_build': os.environ.get('BUILD_NUMBER', ''),
203            'jenkins_job': os.environ.get('JOB_NAME', ''),
204        }
205        row.update(context)
206        row.update(bm)
207        row.update(parse_name(row['name']))
208        row.update(labels)
209        if js2:
210            for bm2 in js2['benchmarks']:
211                if bm['name'] == bm2['name'] and 'already_used' not in bm2:
212                    row['cpu_time'] = bm2['cpu_time']
213                    row['real_time'] = bm2['real_time']
214                    row['iterations'] = bm2['iterations']
215                    bm2['already_used'] = True
216                    break
217        yield row
218