1*13fbcb42Sjoerg""" 2*13fbcb42SjoergStatic Analyzer qualification infrastructure. 3*13fbcb42Sjoerg 4*13fbcb42SjoergThis source file contains all the functionality related to benchmarking 5*13fbcb42Sjoergthe analyzer on a set projects. Right now, this includes measuring 6*13fbcb42Sjoergexecution time and peak memory usage. Benchmark runs analysis on every 7*13fbcb42Sjoergproject multiple times to get a better picture about the distribution 8*13fbcb42Sjoergof measured values. 9*13fbcb42Sjoerg 10*13fbcb42SjoergAdditionally, this file includes a comparison routine for two benchmarking 11*13fbcb42Sjoergresults that plots the result together on one chart. 12*13fbcb42Sjoerg""" 13*13fbcb42Sjoerg 14*13fbcb42Sjoergimport SATestUtils as utils 15*13fbcb42Sjoergfrom SATestBuild import ProjectTester, stdout, TestInfo 16*13fbcb42Sjoergfrom ProjectMap import ProjectInfo 17*13fbcb42Sjoerg 18*13fbcb42Sjoergimport pandas as pd 19*13fbcb42Sjoergfrom typing import List, Tuple 20*13fbcb42Sjoerg 21*13fbcb42Sjoerg 22*13fbcb42SjoergINDEX_COLUMN = "index" 23*13fbcb42Sjoerg 24*13fbcb42Sjoerg 25*13fbcb42Sjoergdef _save(data: pd.DataFrame, file_path: str): 26*13fbcb42Sjoerg data.to_csv(file_path, index_label=INDEX_COLUMN) 27*13fbcb42Sjoerg 28*13fbcb42Sjoerg 29*13fbcb42Sjoergdef _load(file_path: str) -> pd.DataFrame: 30*13fbcb42Sjoerg return pd.read_csv(file_path, index_col=INDEX_COLUMN) 31*13fbcb42Sjoerg 32*13fbcb42Sjoerg 33*13fbcb42Sjoergclass Benchmark: 34*13fbcb42Sjoerg """ 35*13fbcb42Sjoerg Becnhmark class encapsulates one functionality: it runs the analysis 36*13fbcb42Sjoerg multiple times for the given set of projects and stores results in the 37*13fbcb42Sjoerg specified file. 38*13fbcb42Sjoerg """ 39*13fbcb42Sjoerg def __init__(self, projects: List[ProjectInfo], iterations: int, 40*13fbcb42Sjoerg output_path: str): 41*13fbcb42Sjoerg self.projects = projects 42*13fbcb42Sjoerg self.iterations = iterations 43*13fbcb42Sjoerg self.out = output_path 44*13fbcb42Sjoerg 45*13fbcb42Sjoerg def run(self): 46*13fbcb42Sjoerg results = [self._benchmark_project(project) 47*13fbcb42Sjoerg for project in self.projects] 48*13fbcb42Sjoerg 49*13fbcb42Sjoerg data = pd.concat(results, ignore_index=True) 50*13fbcb42Sjoerg _save(data, self.out) 51*13fbcb42Sjoerg 52*13fbcb42Sjoerg def _benchmark_project(self, project: ProjectInfo) -> pd.DataFrame: 53*13fbcb42Sjoerg if not project.enabled: 54*13fbcb42Sjoerg stdout(f" \n\n--- Skipping disabled project {project.name}\n") 55*13fbcb42Sjoerg return 56*13fbcb42Sjoerg 57*13fbcb42Sjoerg stdout(f" \n\n--- Benchmarking project {project.name}\n") 58*13fbcb42Sjoerg 59*13fbcb42Sjoerg test_info = TestInfo(project) 60*13fbcb42Sjoerg tester = ProjectTester(test_info, silent=True) 61*13fbcb42Sjoerg project_dir = tester.get_project_dir() 62*13fbcb42Sjoerg output_dir = tester.get_output_dir() 63*13fbcb42Sjoerg 64*13fbcb42Sjoerg raw_data = [] 65*13fbcb42Sjoerg 66*13fbcb42Sjoerg for i in range(self.iterations): 67*13fbcb42Sjoerg stdout(f"Iteration #{i + 1}") 68*13fbcb42Sjoerg time, mem = tester.build(project_dir, output_dir) 69*13fbcb42Sjoerg raw_data.append({"time": time, "memory": mem, 70*13fbcb42Sjoerg "iteration": i, "project": project.name}) 71*13fbcb42Sjoerg stdout(f"time: {utils.time_to_str(time)}, " 72*13fbcb42Sjoerg f"peak memory: {utils.memory_to_str(mem)}") 73*13fbcb42Sjoerg 74*13fbcb42Sjoerg return pd.DataFrame(raw_data) 75*13fbcb42Sjoerg 76*13fbcb42Sjoerg 77*13fbcb42Sjoergdef compare(old_path: str, new_path: str, plot_file: str): 78*13fbcb42Sjoerg """ 79*13fbcb42Sjoerg Compare two benchmarking results stored as .csv files 80*13fbcb42Sjoerg and produce a plot in the specified file. 81*13fbcb42Sjoerg """ 82*13fbcb42Sjoerg old = _load(old_path) 83*13fbcb42Sjoerg new = _load(new_path) 84*13fbcb42Sjoerg 85*13fbcb42Sjoerg old_projects = set(old["project"]) 86*13fbcb42Sjoerg new_projects = set(new["project"]) 87*13fbcb42Sjoerg common_projects = old_projects & new_projects 88*13fbcb42Sjoerg 89*13fbcb42Sjoerg # Leave only rows for projects common to both dataframes. 90*13fbcb42Sjoerg old = old[old["project"].isin(common_projects)] 91*13fbcb42Sjoerg new = new[new["project"].isin(common_projects)] 92*13fbcb42Sjoerg 93*13fbcb42Sjoerg old, new = _normalize(old, new) 94*13fbcb42Sjoerg 95*13fbcb42Sjoerg # Seaborn prefers all the data to be in one dataframe. 96*13fbcb42Sjoerg old["kind"] = "old" 97*13fbcb42Sjoerg new["kind"] = "new" 98*13fbcb42Sjoerg data = pd.concat([old, new], ignore_index=True) 99*13fbcb42Sjoerg 100*13fbcb42Sjoerg # TODO: compare data in old and new dataframes using statistical tests 101*13fbcb42Sjoerg # to check if they belong to the same distribution 102*13fbcb42Sjoerg _plot(data, plot_file) 103*13fbcb42Sjoerg 104*13fbcb42Sjoerg 105*13fbcb42Sjoergdef _normalize(old: pd.DataFrame, 106*13fbcb42Sjoerg new: pd.DataFrame) -> Tuple[pd.DataFrame, pd.DataFrame]: 107*13fbcb42Sjoerg # This creates a dataframe with all numerical data averaged. 108*13fbcb42Sjoerg means = old.groupby("project").mean() 109*13fbcb42Sjoerg return _normalize_impl(old, means), _normalize_impl(new, means) 110*13fbcb42Sjoerg 111*13fbcb42Sjoerg 112*13fbcb42Sjoergdef _normalize_impl(data: pd.DataFrame, means: pd.DataFrame): 113*13fbcb42Sjoerg # Right now 'means' has one row corresponding to one project, 114*13fbcb42Sjoerg # while 'data' has N rows for each project (one for each iteration). 115*13fbcb42Sjoerg # 116*13fbcb42Sjoerg # In order for us to work easier with this data, we duplicate 117*13fbcb42Sjoerg # 'means' data to match the size of the 'data' dataframe. 118*13fbcb42Sjoerg # 119*13fbcb42Sjoerg # All the columns from 'data' will maintain their names, while 120*13fbcb42Sjoerg # new columns coming from 'means' will have "_mean" suffix. 121*13fbcb42Sjoerg joined_data = data.merge(means, on="project", suffixes=("", "_mean")) 122*13fbcb42Sjoerg _normalize_key(joined_data, "time") 123*13fbcb42Sjoerg _normalize_key(joined_data, "memory") 124*13fbcb42Sjoerg return joined_data 125*13fbcb42Sjoerg 126*13fbcb42Sjoerg 127*13fbcb42Sjoergdef _normalize_key(data: pd.DataFrame, key: str): 128*13fbcb42Sjoerg norm_key = _normalized_name(key) 129*13fbcb42Sjoerg mean_key = f"{key}_mean" 130*13fbcb42Sjoerg data[norm_key] = data[key] / data[mean_key] 131*13fbcb42Sjoerg 132*13fbcb42Sjoerg 133*13fbcb42Sjoergdef _normalized_name(name: str) -> str: 134*13fbcb42Sjoerg return f"normalized {name}" 135*13fbcb42Sjoerg 136*13fbcb42Sjoerg 137*13fbcb42Sjoergdef _plot(data: pd.DataFrame, plot_file: str): 138*13fbcb42Sjoerg import matplotlib 139*13fbcb42Sjoerg import seaborn as sns 140*13fbcb42Sjoerg from matplotlib import pyplot as plt 141*13fbcb42Sjoerg 142*13fbcb42Sjoerg sns.set_style("whitegrid") 143*13fbcb42Sjoerg # We want to have time and memory charts one above the other. 144*13fbcb42Sjoerg figure, (ax1, ax2) = plt.subplots(2, 1, figsize=(8, 6)) 145*13fbcb42Sjoerg 146*13fbcb42Sjoerg def _subplot(key: str, ax: matplotlib.axes.Axes): 147*13fbcb42Sjoerg sns.boxplot(x="project", y=_normalized_name(key), hue="kind", 148*13fbcb42Sjoerg data=data, palette=sns.color_palette("BrBG", 2), ax=ax) 149*13fbcb42Sjoerg 150*13fbcb42Sjoerg _subplot("time", ax1) 151*13fbcb42Sjoerg # No need to have xlabels on both top and bottom charts. 152*13fbcb42Sjoerg ax1.set_xlabel("") 153*13fbcb42Sjoerg 154*13fbcb42Sjoerg _subplot("memory", ax2) 155*13fbcb42Sjoerg # The legend on the top chart is enough. 156*13fbcb42Sjoerg ax2.get_legend().remove() 157*13fbcb42Sjoerg 158*13fbcb42Sjoerg figure.savefig(plot_file) 159