1#!/usr/local/bin/python3.8 2 3# Copyright Hans Dembinski 2019 4# Distributed under the Boost Software License, Version 1.0. 5# See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt 6 7""" 8This script runs the benchmarks on previous versions of this library to track changes 9in performance. 10 11Run this from a special build directory that uses the benchmark folder as root 12 13 cd my_build_dir 14 cmake ../benchmark 15 ../run_benchmarks.py 16 17This creates a database, benchmark_results. Plot it: 18 19 ../plot_benchmarks.py 20 21The script leaves the include folder in a modified state. To clean up, do: 22 23 git checkout HEAD -- ../include 24 git clean -f -- ../include 25 26""" 27import subprocess as subp 28import tempfile 29import os 30import shelve 31import json 32import argparse 33 34 35def get_commits(): 36 commits = [] 37 comments = {} 38 for line in subp.check_output(("git", "log", "--oneline")).decode("ascii").split("\n"): 39 if line: 40 ispace = line.index(" ") 41 hash = line[:ispace] 42 commits.append(hash) 43 comments[hash] = line[ispace+1:] 44 commits = commits[::-1] 45 return commits, comments 46 47 48def recursion(results, commits, comments, ia, ib): 49 ic = int((ia + ib) / 2) 50 if ic == ia: 51 return 52 run(results, comments, commits[ic], False) 53 if all([results[commits[i]] is None for i in (ia, ib, ic)]): 54 return 55 recursion(results, commits, comments, ic, ib) 56 recursion(results, commits, comments, ia, ic) 57 58 59def run(results, comments, hash, update): 60 if not update and hash in results: 61 return 62 print(hash, comments[hash]) 63 subp.call(("rm", "-rf", "../include")) 64 if subp.call(("git", "checkout", hash, "--", "../include")) != 0: 65 print("[Benchmark] Cannot checkout include folder\n") 66 return 67 print(hash, "make") 68 with tempfile.TemporaryFile() as out: 69 if subp.call(("make", "-j4", "histogram_filling"), stdout=out, stderr=out) != 0: 70 print("[Benchmark] Cannot make benchmarks\n") 71 out.seek(0) 72 print(out.read().decode("utf-8") + "\n") 73 return 74 print(hash, "run") 75 s = subp.check_output(("./histogram_filling", "--benchmark_format=json", "--benchmark_filter=normal")) 76 d = json.loads(s) 77 if update and hash in results and results[hash] is not None: 78 d2 = results[hash] 79 for i, (b, b2) in enumerate(zip(d["benchmarks"], d2["benchmarks"])): 80 d["benchmarks"][i] = b if b["cpu_time"] < b2["cpu_time"] else b2 81 results[hash] = d 82 for benchmark in d["benchmarks"]: 83 print(benchmark["name"], min(benchmark["real_time"], benchmark["cpu_time"])) 84 85 86def main(): 87 commits, comments = get_commits() 88 89 parser = argparse.ArgumentParser(description=__doc__, 90 formatter_class=argparse.RawDescriptionHelpFormatter) 91 parser.add_argument("first", type=str, default="begin", 92 help="first commit in range, special value `begin` is allowed") 93 parser.add_argument("last", type=str, default="end", 94 help="last commit in range, special value `end` is allowed") 95 parser.add_argument("-f", action="store_true", 96 help="override previous results") 97 98 args = parser.parse_args() 99 100 if args.first == "begin": 101 args.first = commits[0] 102 if args.last == "end": 103 args.last = commits[-1] 104 105 with shelve.open("benchmark_results") as results: 106 a = commits.index(args.first) 107 b = commits.index(args.last) 108 if args.f: 109 for hash in commits[a:b+1]: 110 del results[hash] 111 run(results, comments, args.first, False) 112 run(results, comments, args.last, False) 113 recursion(results, commits, comments, a, b) 114 115if __name__ == "__main__": 116 main() 117