1# DExTer : Debugging Experience Tester 2# ~~~~~~ ~ ~~ ~ ~~ 3# 4# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 5# See https://llvm.org/LICENSE.txt for license information. 6# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 7"""Clang opt-bisect tool.""" 8 9from collections import defaultdict 10import os 11import csv 12import re 13import pickle 14 15from dex.builder import run_external_build_script 16from dex.command.ParseCommand import get_command_infos 17from dex.debugger.Debuggers import run_debugger_subprocess 18from dex.debugger.DebuggerControllers.DefaultController import DefaultController 19from dex.dextIR.DextIR import DextIR 20from dex.heuristic import Heuristic 21from dex.tools import TestToolBase 22from dex.utils.Exceptions import DebuggerException, Error 23from dex.utils.Exceptions import BuildScriptException, HeuristicException 24from dex.utils.PrettyOutputBase import Stream 25from dex.utils.ReturnCode import ReturnCode 26 27 28class BisectPass(object): 29 def __init__(self, no, description, description_no_loc): 30 self.no = no 31 self.description = description 32 self.description_no_loc = description_no_loc 33 34 self.penalty = 0 35 self.differences = [] 36 37 38class Tool(TestToolBase): 39 """Use the LLVM "-opt-bisect-limit=<n>" flag to get information on the 40 contribution of each LLVM pass to the overall DExTer score when using 41 clang. 42 43 Clang is run multiple times, with an increasing value of n, measuring the 44 debugging experience at each value. 45 """ 46 47 _re_running_pass = re.compile( 48 r'^BISECT\: running pass \((\d+)\) (.+?)( \(.+\))?$') 49 50 def __init__(self, *args, **kwargs): 51 super(Tool, self).__init__(*args, **kwargs) 52 self._all_bisect_pass_summary = defaultdict(list) 53 54 @property 55 def name(self): 56 return 'DExTer clang opt bisect' 57 58 def _get_bisect_limits(self): 59 options = self.context.options 60 61 max_limit = 999999 62 limits = [max_limit for _ in options.source_files] 63 all_passes = [ 64 l for l in self._clang_opt_bisect_build(limits)[1].splitlines() 65 if l.startswith('BISECT: running pass (') 66 ] 67 68 results = [] 69 for i, pass_ in enumerate(all_passes[1:]): 70 if pass_.startswith('BISECT: running pass (1)'): 71 results.append(all_passes[i]) 72 results.append(all_passes[-1]) 73 74 assert len(results) == len( 75 options.source_files), (results, options.source_files) 76 77 limits = [ 78 int(Tool._re_running_pass.match(r).group(1)) for r in results 79 ] 80 81 return limits 82 83 def handle_options(self, defaults): 84 options = self.context.options 85 if "clang" not in options.builder.lower(): 86 raise Error("--builder %s is not supported by the clang-opt-bisect tool - only 'clang' is " 87 "supported " % options.builder) 88 super(Tool, self).handle_options(defaults) 89 90 def _init_debugger_controller(self): 91 step_collection = DextIR( 92 executable_path=self.context.options.executable, 93 source_paths=self.context.options.source_files, 94 dexter_version=self.context.version) 95 96 step_collection.commands, new_source_files = get_command_infos( 97 self.context.options.source_files, self.context.options.source_root_dir) 98 self.context.options.source_files.extend(list(new_source_files)) 99 100 debugger_controller = DefaultController(self.context, step_collection) 101 return debugger_controller 102 103 def _run_test(self, test_name): # noqa 104 options = self.context.options 105 106 per_pass_score = [] 107 current_bisect_pass_summary = defaultdict(list) 108 109 max_limits = self._get_bisect_limits() 110 overall_limit = sum(max_limits) 111 prev_score = 1.0 112 prev_steps_str = None 113 114 for current_limit in range(overall_limit + 1): 115 # Take the overall limit number and split it across buckets for 116 # each source file. 117 limit_remaining = current_limit 118 file_limits = [0] * len(max_limits) 119 for i, max_limit in enumerate(max_limits): 120 if limit_remaining < max_limit: 121 file_limits[i] += limit_remaining 122 break 123 else: 124 file_limits[i] = max_limit 125 limit_remaining -= file_limits[i] 126 127 f = [l for l in file_limits if l] 128 current_file_index = len(f) - 1 if f else 0 129 130 _, err, builderIR = self._clang_opt_bisect_build(file_limits) 131 err_lines = err.splitlines() 132 # Find the last line that specified a running pass. 133 for l in err_lines[::-1]: 134 match = Tool._re_running_pass.match(l) 135 if match: 136 pass_info = match.groups() 137 break 138 else: 139 pass_info = (0, None, None) 140 141 try: 142 debugger_controller =self._init_debugger_controller() 143 debugger_controller = run_debugger_subprocess( 144 debugger_controller, self.context.working_directory.path) 145 steps = debugger_controller.step_collection 146 except DebuggerException: 147 steps = DextIR( 148 executable_path=self.context.options.executable, 149 source_paths=self.context.options.source_files, 150 dexter_version=self.context.version) 151 152 steps.builder = builderIR 153 154 try: 155 heuristic = Heuristic(self.context, steps) 156 except HeuristicException as e: 157 raise Error(e) 158 159 score_difference = heuristic.score - prev_score 160 prev_score = heuristic.score 161 162 isnan = heuristic.score != heuristic.score 163 if isnan or score_difference < 0: 164 color1 = 'r' 165 color2 = 'r' 166 elif score_difference > 0: 167 color1 = 'g' 168 color2 = 'g' 169 else: 170 color1 = 'y' 171 color2 = 'd' 172 173 summary = '<{}>running pass {}/{} on "{}"'.format( 174 color2, pass_info[0], max_limits[current_file_index], 175 test_name) 176 if len(options.source_files) > 1: 177 summary += ' [{}/{}]'.format(current_limit, overall_limit) 178 179 pass_text = ''.join(p for p in pass_info[1:] if p) 180 summary += ': {} <{}>{:+.4f}</> <{}>{}</></>\n'.format( 181 heuristic.summary_string, color1, score_difference, color2, 182 pass_text) 183 184 self.context.o.auto(summary) 185 186 heuristic_verbose_output = heuristic.verbose_output 187 188 if options.verbose: 189 self.context.o.auto(heuristic_verbose_output) 190 191 steps_str = str(steps) 192 steps_changed = steps_str != prev_steps_str 193 prev_steps_str = steps_str 194 195 # If this is the first pass, or something has changed, write a text 196 # file containing verbose information on the current status. 197 if current_limit == 0 or score_difference or steps_changed: 198 file_name = '-'.join( 199 str(s) for s in [ 200 'status', test_name, '{{:0>{}}}'.format( 201 len(str(overall_limit))).format(current_limit), 202 '{:.4f}'.format(heuristic.score).replace( 203 '.', '_'), pass_info[1] 204 ] if s is not None) 205 206 file_name = ''.join( 207 c for c in file_name 208 if c.isalnum() or c in '()-_./ ').strip().replace( 209 ' ', '_').replace('/', '_') 210 211 output_text_path = os.path.join(options.results_directory, 212 '{}.txt'.format(file_name)) 213 with open(output_text_path, 'w') as fp: 214 self.context.o.auto(summary + '\n', stream=Stream(fp)) 215 self.context.o.auto(str(steps) + '\n', stream=Stream(fp)) 216 self.context.o.auto( 217 heuristic_verbose_output + '\n', stream=Stream(fp)) 218 219 output_dextIR_path = os.path.join(options.results_directory, 220 '{}.dextIR'.format(file_name)) 221 with open(output_dextIR_path, 'wb') as fp: 222 pickle.dump(steps, fp, protocol=pickle.HIGHEST_PROTOCOL) 223 224 per_pass_score.append((test_name, pass_text, 225 heuristic.score)) 226 227 if pass_info[1]: 228 self._all_bisect_pass_summary[pass_info[1]].append( 229 score_difference) 230 231 current_bisect_pass_summary[pass_info[1]].append( 232 score_difference) 233 234 per_pass_score_path = os.path.join( 235 options.results_directory, 236 '{}-per_pass_score.csv'.format(test_name)) 237 238 with open(per_pass_score_path, mode='w', newline='') as fp: 239 writer = csv.writer(fp, delimiter=',') 240 writer.writerow(['Source File', 'Pass', 'Score']) 241 242 for path, pass_, score in per_pass_score: 243 writer.writerow([path, pass_, score]) 244 self.context.o.blue('wrote "{}"\n'.format(per_pass_score_path)) 245 246 pass_summary_path = os.path.join( 247 options.results_directory, '{}-pass-summary.csv'.format(test_name)) 248 249 self._write_pass_summary(pass_summary_path, 250 current_bisect_pass_summary) 251 252 def _handle_results(self) -> ReturnCode: 253 options = self.context.options 254 pass_summary_path = os.path.join(options.results_directory, 255 'overall-pass-summary.csv') 256 257 self._write_pass_summary(pass_summary_path, 258 self._all_bisect_pass_summary) 259 return ReturnCode.OK 260 261 def _clang_opt_bisect_build(self, opt_bisect_limits): 262 options = self.context.options 263 compiler_options = [ 264 '{} -mllvm -opt-bisect-limit={}'.format(options.cflags, 265 opt_bisect_limit) 266 for opt_bisect_limit in opt_bisect_limits 267 ] 268 linker_options = options.ldflags 269 270 try: 271 return run_external_build_script( 272 self.context, 273 source_files=options.source_files, 274 compiler_options=compiler_options, 275 linker_options=linker_options, 276 script_path=self.build_script, 277 executable_file=options.executable) 278 except BuildScriptException as e: 279 raise Error(e) 280 281 def _write_pass_summary(self, path, pass_summary): 282 # Get a list of tuples. 283 pass_summary_list = list(pass_summary.items()) 284 285 for i, item in enumerate(pass_summary_list): 286 # Add elems for the sum, min, and max of the values, as well as 287 # 'interestingness' which is whether any of these values are 288 # non-zero. 289 pass_summary_list[i] += (sum(item[1]), min(item[1]), max(item[1]), 290 any(item[1])) 291 292 # Split the pass name into the basic name and kind. 293 pass_summary_list[i] += tuple(item[0].rsplit(' on ', 1)) 294 295 # Sort the list by the following columns in order of precedence: 296 # - Is interesting (True first) 297 # - Sum (smallest first) 298 # - Number of times pass ran (largest first) 299 # - Kind (alphabetically) 300 # - Name (alphabetically) 301 pass_summary_list.sort( 302 key=lambda tup: (not tup[5], tup[2], -len(tup[1]), tup[7], tup[6])) 303 304 with open(path, mode='w', newline='') as fp: 305 writer = csv.writer(fp, delimiter=',') 306 writer.writerow( 307 ['Pass', 'Kind', 'Sum', 'Min', 'Max', 'Interesting']) 308 309 for (_, vals, sum_, min_, max_, interesting, name, 310 kind) in pass_summary_list: 311 writer.writerow([name, kind, sum_, min_, max_, interesting] + 312 vals) 313 314 self.context.o.blue('wrote "{}"\n'.format(path)) 315