1# DExTer : Debugging Experience Tester
2# ~~~~~~   ~         ~~         ~   ~~
3#
4# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5# See https://llvm.org/LICENSE.txt for license information.
6# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7"""Clang opt-bisect tool."""
8
9from collections import defaultdict
10import os
11import csv
12import re
13import pickle
14
15from dex.builder import run_external_build_script
16from dex.command.ParseCommand import get_command_infos
17from dex.debugger.Debuggers import run_debugger_subprocess
18from dex.debugger.DebuggerControllers.DefaultController import DefaultController
19from dex.dextIR.DextIR import DextIR
20from dex.heuristic import Heuristic
21from dex.tools import TestToolBase
22from dex.utils.Exceptions import DebuggerException, Error
23from dex.utils.Exceptions import BuildScriptException, HeuristicException
24from dex.utils.PrettyOutputBase import Stream
25from dex.utils.ReturnCode import ReturnCode
26
27
28class BisectPass(object):
29    def __init__(self, no, description, description_no_loc):
30        self.no = no
31        self.description = description
32        self.description_no_loc = description_no_loc
33
34        self.penalty = 0
35        self.differences = []
36
37
38class Tool(TestToolBase):
39    """Use the LLVM "-opt-bisect-limit=<n>" flag to get information on the
40    contribution of each LLVM pass to the overall DExTer score when using
41    clang.
42
43    Clang is run multiple times, with an increasing value of n, measuring the
44    debugging experience at each value.
45    """
46
47    _re_running_pass = re.compile(
48        r'^BISECT\: running pass \((\d+)\) (.+?)( \(.+\))?$')
49
50    def __init__(self, *args, **kwargs):
51        super(Tool, self).__init__(*args, **kwargs)
52        self._all_bisect_pass_summary = defaultdict(list)
53
54    @property
55    def name(self):
56        return 'DExTer clang opt bisect'
57
58    def _get_bisect_limits(self):
59        options = self.context.options
60
61        max_limit = 999999
62        limits = [max_limit for _ in options.source_files]
63        all_passes = [
64            l for l in self._clang_opt_bisect_build(limits)[1].splitlines()
65            if l.startswith('BISECT: running pass (')
66        ]
67
68        results = []
69        for i, pass_ in enumerate(all_passes[1:]):
70            if pass_.startswith('BISECT: running pass (1)'):
71                results.append(all_passes[i])
72        results.append(all_passes[-1])
73
74        assert len(results) == len(
75            options.source_files), (results, options.source_files)
76
77        limits = [
78            int(Tool._re_running_pass.match(r).group(1)) for r in results
79        ]
80
81        return limits
82
83    def handle_options(self, defaults):
84        options = self.context.options
85        if "clang" not in options.builder.lower():
86            raise Error("--builder %s is not supported by the clang-opt-bisect tool - only 'clang' is "
87                        "supported " % options.builder)
88        super(Tool, self).handle_options(defaults)
89
90    def _init_debugger_controller(self):
91        step_collection = DextIR(
92            executable_path=self.context.options.executable,
93            source_paths=self.context.options.source_files,
94            dexter_version=self.context.version)
95
96        step_collection.commands, new_source_files = get_command_infos(
97            self.context.options.source_files, self.context.options.source_root_dir)
98        self.context.options.source_files.extend(list(new_source_files))
99
100        debugger_controller = DefaultController(self.context, step_collection)
101        return debugger_controller
102
103    def _run_test(self, test_name):  # noqa
104        options = self.context.options
105
106        per_pass_score = []
107        current_bisect_pass_summary = defaultdict(list)
108
109        max_limits = self._get_bisect_limits()
110        overall_limit = sum(max_limits)
111        prev_score = 1.0
112        prev_steps_str = None
113
114        for current_limit in range(overall_limit + 1):
115            # Take the overall limit number and split it across buckets for
116            # each source file.
117            limit_remaining = current_limit
118            file_limits = [0] * len(max_limits)
119            for i, max_limit in enumerate(max_limits):
120                if limit_remaining < max_limit:
121                    file_limits[i] += limit_remaining
122                    break
123                else:
124                    file_limits[i] = max_limit
125                    limit_remaining -= file_limits[i]
126
127            f = [l for l in file_limits if l]
128            current_file_index = len(f) - 1 if f else 0
129
130            _, err, builderIR = self._clang_opt_bisect_build(file_limits)
131            err_lines = err.splitlines()
132            # Find the last line that specified a running pass.
133            for l in err_lines[::-1]:
134                match = Tool._re_running_pass.match(l)
135                if match:
136                    pass_info = match.groups()
137                    break
138            else:
139                pass_info = (0, None, None)
140
141            try:
142                debugger_controller =self._init_debugger_controller()
143                debugger_controller = run_debugger_subprocess(
144                    debugger_controller, self.context.working_directory.path)
145                steps = debugger_controller.step_collection
146            except DebuggerException:
147                steps =  DextIR(
148                    executable_path=self.context.options.executable,
149                    source_paths=self.context.options.source_files,
150                    dexter_version=self.context.version)
151
152            steps.builder = builderIR
153
154            try:
155                heuristic = Heuristic(self.context, steps)
156            except HeuristicException as e:
157                raise Error(e)
158
159            score_difference = heuristic.score - prev_score
160            prev_score = heuristic.score
161
162            isnan = heuristic.score != heuristic.score
163            if isnan or score_difference < 0:
164                color1 = 'r'
165                color2 = 'r'
166            elif score_difference > 0:
167                color1 = 'g'
168                color2 = 'g'
169            else:
170                color1 = 'y'
171                color2 = 'd'
172
173            summary = '<{}>running pass {}/{} on "{}"'.format(
174                color2, pass_info[0], max_limits[current_file_index],
175                test_name)
176            if len(options.source_files) > 1:
177                summary += ' [{}/{}]'.format(current_limit, overall_limit)
178
179            pass_text = ''.join(p for p in pass_info[1:] if p)
180            summary += ': {} <{}>{:+.4f}</> <{}>{}</></>\n'.format(
181                heuristic.summary_string, color1, score_difference, color2,
182                pass_text)
183
184            self.context.o.auto(summary)
185
186            heuristic_verbose_output = heuristic.verbose_output
187
188            if options.verbose:
189                self.context.o.auto(heuristic_verbose_output)
190
191            steps_str = str(steps)
192            steps_changed = steps_str != prev_steps_str
193            prev_steps_str = steps_str
194
195            # If this is the first pass, or something has changed, write a text
196            # file containing verbose information on the current status.
197            if current_limit == 0 or score_difference or steps_changed:
198                file_name = '-'.join(
199                    str(s) for s in [
200                        'status', test_name, '{{:0>{}}}'.format(
201                            len(str(overall_limit))).format(current_limit),
202                        '{:.4f}'.format(heuristic.score).replace(
203                            '.', '_'), pass_info[1]
204                    ] if s is not None)
205
206                file_name = ''.join(
207                    c for c in file_name
208                    if c.isalnum() or c in '()-_./ ').strip().replace(
209                    ' ', '_').replace('/', '_')
210
211                output_text_path = os.path.join(options.results_directory,
212                                                '{}.txt'.format(file_name))
213                with open(output_text_path, 'w') as fp:
214                    self.context.o.auto(summary + '\n', stream=Stream(fp))
215                    self.context.o.auto(str(steps) + '\n', stream=Stream(fp))
216                    self.context.o.auto(
217                        heuristic_verbose_output + '\n', stream=Stream(fp))
218
219                output_dextIR_path = os.path.join(options.results_directory,
220                                                  '{}.dextIR'.format(file_name))
221                with open(output_dextIR_path, 'wb') as fp:
222                    pickle.dump(steps, fp, protocol=pickle.HIGHEST_PROTOCOL)
223
224            per_pass_score.append((test_name, pass_text,
225                                   heuristic.score))
226
227            if pass_info[1]:
228                self._all_bisect_pass_summary[pass_info[1]].append(
229                    score_difference)
230
231                current_bisect_pass_summary[pass_info[1]].append(
232                    score_difference)
233
234        per_pass_score_path = os.path.join(
235            options.results_directory,
236            '{}-per_pass_score.csv'.format(test_name))
237
238        with open(per_pass_score_path, mode='w', newline='') as fp:
239            writer = csv.writer(fp, delimiter=',')
240            writer.writerow(['Source File', 'Pass', 'Score'])
241
242            for path, pass_, score in per_pass_score:
243                writer.writerow([path, pass_, score])
244        self.context.o.blue('wrote "{}"\n'.format(per_pass_score_path))
245
246        pass_summary_path = os.path.join(
247            options.results_directory, '{}-pass-summary.csv'.format(test_name))
248
249        self._write_pass_summary(pass_summary_path,
250                                 current_bisect_pass_summary)
251
252    def _handle_results(self) -> ReturnCode:
253        options = self.context.options
254        pass_summary_path = os.path.join(options.results_directory,
255                                         'overall-pass-summary.csv')
256
257        self._write_pass_summary(pass_summary_path,
258                                 self._all_bisect_pass_summary)
259        return ReturnCode.OK
260
261    def _clang_opt_bisect_build(self, opt_bisect_limits):
262        options = self.context.options
263        compiler_options = [
264            '{} -mllvm -opt-bisect-limit={}'.format(options.cflags,
265                                                    opt_bisect_limit)
266            for opt_bisect_limit in opt_bisect_limits
267        ]
268        linker_options = options.ldflags
269
270        try:
271            return run_external_build_script(
272                self.context,
273                source_files=options.source_files,
274                compiler_options=compiler_options,
275                linker_options=linker_options,
276                script_path=self.build_script,
277                executable_file=options.executable)
278        except BuildScriptException as e:
279            raise Error(e)
280
281    def _write_pass_summary(self, path, pass_summary):
282        # Get a list of tuples.
283        pass_summary_list = list(pass_summary.items())
284
285        for i, item in enumerate(pass_summary_list):
286            # Add elems for the sum, min, and max of the values, as well as
287            # 'interestingness' which is whether any of these values are
288            # non-zero.
289            pass_summary_list[i] += (sum(item[1]), min(item[1]), max(item[1]),
290                                     any(item[1]))
291
292            # Split the pass name into the basic name and kind.
293            pass_summary_list[i] += tuple(item[0].rsplit(' on ', 1))
294
295        # Sort the list by the following columns in order of precedence:
296        #   - Is interesting (True first)
297        #   - Sum (smallest first)
298        #   - Number of times pass ran (largest first)
299        #   - Kind (alphabetically)
300        #   - Name (alphabetically)
301        pass_summary_list.sort(
302            key=lambda tup: (not tup[5], tup[2], -len(tup[1]), tup[7], tup[6]))
303
304        with open(path, mode='w', newline='') as fp:
305            writer = csv.writer(fp, delimiter=',')
306            writer.writerow(
307                ['Pass', 'Kind', 'Sum', 'Min', 'Max', 'Interesting'])
308
309            for (_, vals, sum_, min_, max_, interesting, name,
310                 kind) in pass_summary_list:
311                writer.writerow([name, kind, sum_, min_, max_, interesting] +
312                                vals)
313
314        self.context.o.blue('wrote "{}"\n'.format(path))
315