1#!/usr/bin/env python3
2#
3# Script to analyze results of our branch prediction heuristics
4#
5# This file is part of GCC.
6#
7# GCC is free software; you can redistribute it and/or modify it under
8# the terms of the GNU General Public License as published by the Free
9# Software Foundation; either version 3, or (at your option) any later
10# version.
11#
12# GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13# WARRANTY; without even the implied warranty of MERCHANTABILITY or
14# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
15# for more details.
16#
17# You should have received a copy of the GNU General Public License
18# along with GCC; see the file COPYING3.  If not see
19# <http://www.gnu.org/licenses/>.  */
20#
21#
22#
23# This script is used to calculate two basic properties of the branch prediction
24# heuristics - coverage and hitrate.  Coverage is number of executions
25# of a given branch matched by the heuristics and hitrate is probability
26# that once branch is predicted as taken it is really taken.
27#
28# These values are useful to determine the quality of given heuristics.
29# Hitrate may be directly used in predict.def.
30#
31# Usage:
32#  Step 1: Compile and profile your program.  You need to use -fprofile-generate
33#    flag to get the profiles.
34#  Step 2: Make a reference run of the intrumented application.
35#  Step 3: Compile the program with collected profile and dump IPA profiles
36#          (-fprofile-use -fdump-ipa-profile-details)
37#  Step 4: Collect all generated dump files:
38#          find . -name '*.profile' | xargs cat > dump_file
39#  Step 5: Run the script:
40#          ./analyze_brprob.py dump_file
41#          and read results.  Basically the following table is printed:
42#
43# HEURISTICS                           BRANCHES  (REL)  HITRATE                COVERAGE  (REL)
44# early return (on trees)                     3   0.2%  35.83% /  93.64%          66360   0.0%
45# guess loop iv compare                       8   0.6%  53.35% /  53.73%       11183344   0.0%
46# call                                       18   1.4%  31.95% /  69.95%       51880179   0.2%
47# loop guard                                 23   1.8%  84.13% /  84.85%    13749065956  42.2%
48# opcode values positive (on trees)          42   3.3%  15.71% /  84.81%     6771097902  20.8%
49# opcode values nonequal (on trees)         226  17.6%  72.48% /  72.84%      844753864   2.6%
50# loop exit                                 231  18.0%  86.97% /  86.98%     8952666897  27.5%
51# loop iterations                           239  18.6%  91.10% /  91.10%     3062707264   9.4%
52# DS theory                                 281  21.9%  82.08% /  83.39%     7787264075  23.9%
53# no prediction                             293  22.9%  46.92% /  70.70%     2293267840   7.0%
54# guessed loop iterations                   313  24.4%  76.41% /  76.41%    10782750177  33.1%
55# first match                               708  55.2%  82.30% /  82.31%    22489588691  69.0%
56# combined                                 1282 100.0%  79.76% /  81.75%    32570120606 100.0%
57#
58#
59#  The heuristics called "first match" is a heuristics used by GCC branch
60#  prediction pass and it predicts 55.2% branches correctly. As you can,
61#  the heuristics has very good covertage (69.05%).  On the other hand,
62#  "opcode values nonequal (on trees)" heuristics has good hirate, but poor
63#  coverage.
64
65import sys
66import os
67import re
68import argparse
69
70from math import *
71
72counter_aggregates = set(['combined', 'first match', 'DS theory',
73    'no prediction'])
74hot_threshold = 10
75
76def percentage(a, b):
77    return 100.0 * a / b
78
79def average(values):
80    return 1.0 * sum(values) / len(values)
81
82def average_cutoff(values, cut):
83    l = len(values)
84    skip = floor(l * cut / 2)
85    if skip > 0:
86        values.sort()
87        values = values[skip:-skip]
88    return average(values)
89
90def median(values):
91    values.sort()
92    return values[int(len(values) / 2)]
93
94class PredictDefFile:
95    def __init__(self, path):
96        self.path = path
97        self.predictors = {}
98
99    def parse_and_modify(self, heuristics, write_def_file):
100        lines = [x.rstrip() for x in open(self.path).readlines()]
101
102        p = None
103        modified_lines = []
104        for l in lines:
105            if l.startswith('DEF_PREDICTOR'):
106                m = re.match('.*"(.*)".*', l)
107                p = m.group(1)
108            elif l == '':
109                p = None
110
111            if p != None:
112                heuristic = [x for x in heuristics if x.name == p]
113                heuristic = heuristic[0] if len(heuristic) == 1 else None
114
115                m = re.match('.*HITRATE \(([^)]*)\).*', l)
116                if (m != None):
117                    self.predictors[p] = int(m.group(1))
118
119                    # modify the line
120                    if heuristic != None:
121                        new_line = (l[:m.start(1)]
122                            + str(round(heuristic.get_hitrate()))
123                            + l[m.end(1):])
124                        l = new_line
125                    p = None
126                elif 'PROB_VERY_LIKELY' in l:
127                    self.predictors[p] = 100
128            modified_lines.append(l)
129
130        # save the file
131        if write_def_file:
132            with open(self.path, 'w+') as f:
133                for l in modified_lines:
134                    f.write(l + '\n')
135class Heuristics:
136    def __init__(self, count, hits, fits):
137        self.count = count
138        self.hits = hits
139        self.fits = fits
140
141class Summary:
142    def __init__(self, name):
143        self.name = name
144        self.edges= []
145
146    def branches(self):
147        return len(self.edges)
148
149    def hits(self):
150        return sum([x.hits for x in self.edges])
151
152    def fits(self):
153        return sum([x.fits for x in self.edges])
154
155    def count(self):
156        return sum([x.count for x in self.edges])
157
158    def successfull_branches(self):
159        return len([x for x in self.edges if 2 * x.hits >= x.count])
160
161    def get_hitrate(self):
162        return 100.0 * self.hits() / self.count()
163
164    def get_branch_hitrate(self):
165        return 100.0 * self.successfull_branches() / self.branches()
166
167    def count_formatted(self):
168        v = self.count()
169        for unit in ['', 'k', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y']:
170            if v < 1000:
171                return "%3.2f%s" % (v, unit)
172            v /= 1000.0
173        return "%.1f%s" % (v, 'Y')
174
175    def count(self):
176        return sum([x.count for x in self.edges])
177
178    def print(self, branches_max, count_max, predict_def):
179        # filter out most hot edges (if requested)
180        self.edges = sorted(self.edges, reverse = True, key = lambda x: x.count)
181        if args.coverage_threshold != None:
182            threshold = args.coverage_threshold * self.count() / 100
183            edges = [x for x in self.edges if x.count < threshold]
184            if len(edges) != 0:
185                self.edges = edges
186
187        predicted_as = None
188        if predict_def != None and self.name in predict_def.predictors:
189            predicted_as = predict_def.predictors[self.name]
190
191        print('%-40s %8i %5.1f%% %11.2f%% %7.2f%% / %6.2f%% %14i %8s %5.1f%%' %
192            (self.name, self.branches(),
193                percentage(self.branches(), branches_max),
194                self.get_branch_hitrate(),
195                self.get_hitrate(),
196                percentage(self.fits(), self.count()),
197                self.count(), self.count_formatted(),
198                percentage(self.count(), count_max)), end = '')
199
200        if predicted_as != None:
201            print('%12i%% %5.1f%%' % (predicted_as,
202                self.get_hitrate() - predicted_as), end = '')
203        else:
204            print(' ' * 20, end = '')
205
206        # print details about the most important edges
207        if args.coverage_threshold == None:
208            edges = [x for x in self.edges[:100] if x.count * hot_threshold > self.count()]
209            if args.verbose:
210                for c in edges:
211                    r = 100.0 * c.count / self.count()
212                    print(' %.0f%%:%d' % (r, c.count), end = '')
213            elif len(edges) > 0:
214                print(' %0.0f%%:%d' % (100.0 * sum([x.count for x in edges]) / self.count(), len(edges)), end = '')
215
216        print()
217
218class Profile:
219    def __init__(self, filename):
220        self.filename = filename
221        self.heuristics = {}
222        self.niter_vector = []
223
224    def add(self, name, prediction, count, hits):
225        if not name in self.heuristics:
226            self.heuristics[name] = Summary(name)
227
228        s = self.heuristics[name]
229
230        if prediction < 50:
231            hits = count - hits
232        remaining = count - hits
233        fits = max(hits, remaining)
234
235        s.edges.append(Heuristics(count, hits, fits))
236
237    def add_loop_niter(self, niter):
238        if niter > 0:
239            self.niter_vector.append(niter)
240
241    def branches_max(self):
242        return max([v.branches() for k, v in self.heuristics.items()])
243
244    def count_max(self):
245        return max([v.count() for k, v in self.heuristics.items()])
246
247    def print_group(self, sorting, group_name, heuristics, predict_def):
248        count_max = self.count_max()
249        branches_max = self.branches_max()
250
251        sorter = lambda x: x.branches()
252        if sorting == 'branch-hitrate':
253            sorter = lambda x: x.get_branch_hitrate()
254        elif sorting == 'hitrate':
255            sorter = lambda x: x.get_hitrate()
256        elif sorting == 'coverage':
257            sorter = lambda x: x.count
258        elif sorting == 'name':
259            sorter = lambda x: x.name.lower()
260
261        print('%-40s %8s %6s %12s %18s %14s %8s %6s %12s %6s %s' %
262            ('HEURISTICS', 'BRANCHES', '(REL)',
263            'BR. HITRATE', 'HITRATE', 'COVERAGE', 'COVERAGE', '(REL)',
264            'predict.def', '(REL)', 'HOT branches (>%d%%)' % hot_threshold))
265        for h in sorted(heuristics, key = sorter):
266            h.print(branches_max, count_max, predict_def)
267
268    def dump(self, sorting):
269        heuristics = self.heuristics.values()
270        if len(heuristics) == 0:
271            print('No heuristics available')
272            return
273
274        predict_def = None
275        if args.def_file != None:
276            predict_def = PredictDefFile(args.def_file)
277            predict_def.parse_and_modify(heuristics, args.write_def_file)
278
279        special = list(filter(lambda x: x.name in counter_aggregates,
280            heuristics))
281        normal = list(filter(lambda x: x.name not in counter_aggregates,
282            heuristics))
283
284        self.print_group(sorting, 'HEURISTICS', normal, predict_def)
285        print()
286        self.print_group(sorting, 'HEURISTIC AGGREGATES', special, predict_def)
287
288        if len(self.niter_vector) > 0:
289            print ('\nLoop count: %d' % len(self.niter_vector)),
290            print('  avg. # of iter: %.2f' % average(self.niter_vector))
291            print('  median # of iter: %.2f' % median(self.niter_vector))
292            for v in [1, 5, 10, 20, 30]:
293                cut = 0.01 * v
294                print('  avg. (%d%% cutoff) # of iter: %.2f'
295                    % (v, average_cutoff(self.niter_vector, cut)))
296
297parser = argparse.ArgumentParser()
298parser.add_argument('dump_file', metavar = 'dump_file',
299    help = 'IPA profile dump file')
300parser.add_argument('-s', '--sorting', dest = 'sorting',
301    choices = ['branches', 'branch-hitrate', 'hitrate', 'coverage', 'name'],
302    default = 'branches')
303parser.add_argument('-d', '--def-file', help = 'path to predict.def')
304parser.add_argument('-w', '--write-def-file', action = 'store_true',
305    help = 'Modify predict.def file in order to set new numbers')
306parser.add_argument('-c', '--coverage-threshold', type = int,
307    help = 'Ignore edges that have percentage coverage >= coverage-threshold')
308parser.add_argument('-v', '--verbose', action = 'store_true', help = 'Print verbose informations')
309
310args = parser.parse_args()
311
312profile = Profile(args.dump_file)
313loop_niter_str = ';;  profile-based iteration count: '
314
315for l in open(args.dump_file):
316    if l.startswith(';;heuristics;'):
317        parts = l.strip().split(';')
318        assert len(parts) == 8
319        name = parts[3]
320        prediction = float(parts[6])
321        count = int(parts[4])
322        hits = int(parts[5])
323
324        profile.add(name, prediction, count, hits)
325    elif l.startswith(loop_niter_str):
326        v = int(l[len(loop_niter_str):])
327        profile.add_loop_niter(v)
328
329profile.dump(args.sorting)
330