1#!/usr/local/bin/python3.8
2
3# Copyright Hans Dembinski 2019
4# Distributed under the Boost Software License, Version 1.0.
5# See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt
6
7"""
8This script runs the benchmarks on previous versions of this library to track changes
9in performance.
10
11Run this from a special build directory that uses the benchmark folder as root
12
13    cd my_build_dir
14    cmake ../benchmark
15    ../run_benchmarks.py
16
17This creates a database, benchmark_results. Plot it:
18
19    ../plot_benchmarks.py
20
21The script leaves the include folder in a modified state. To clean up, do:
22
23    git checkout HEAD -- ../include
24    git clean -f -- ../include
25
26"""
27import subprocess as subp
28import tempfile
29import os
30import shelve
31import json
32import argparse
33
34
35def get_commits():
36    commits = []
37    comments = {}
38    for line in subp.check_output(("git", "log", "--oneline")).decode("ascii").split("\n"):
39        if line:
40            ispace = line.index(" ")
41            hash = line[:ispace]
42            commits.append(hash)
43            comments[hash] = line[ispace+1:]
44    commits = commits[::-1]
45    return commits, comments
46
47
48def recursion(results, commits, comments, ia, ib):
49    ic = int((ia + ib) / 2)
50    if ic == ia:
51        return
52    run(results, comments, commits[ic], False)
53    if all([results[commits[i]] is None for i in (ia, ib, ic)]):
54        return
55    recursion(results, commits, comments, ic, ib)
56    recursion(results, commits, comments, ia, ic)
57
58
59def run(results, comments, hash, update):
60    if not update and hash in results:
61        return
62    print(hash, comments[hash])
63    subp.call(("rm", "-rf", "../include"))
64    if subp.call(("git", "checkout", hash, "--", "../include")) != 0:
65        print("[Benchmark] Cannot checkout include folder\n")
66        return
67    print(hash, "make")
68    with tempfile.TemporaryFile() as out:
69        if subp.call(("make", "-j4", "histogram_filling"), stdout=out, stderr=out) != 0:
70            print("[Benchmark] Cannot make benchmarks\n")
71            out.seek(0)
72            print(out.read().decode("utf-8") + "\n")
73            return
74    print(hash, "run")
75    s = subp.check_output(("./histogram_filling", "--benchmark_format=json", "--benchmark_filter=normal"))
76    d = json.loads(s)
77    if update and hash in results and results[hash] is not None:
78        d2 = results[hash]
79        for i, (b, b2) in enumerate(zip(d["benchmarks"], d2["benchmarks"])):
80            d["benchmarks"][i] = b if b["cpu_time"] < b2["cpu_time"] else b2
81    results[hash] = d
82    for benchmark in d["benchmarks"]:
83        print(benchmark["name"], min(benchmark["real_time"], benchmark["cpu_time"]))
84
85
86def main():
87    commits, comments = get_commits()
88
89    parser = argparse.ArgumentParser(description=__doc__,
90                                     formatter_class=argparse.RawDescriptionHelpFormatter)
91    parser.add_argument("first", type=str, default="begin",
92                        help="first commit in range, special value `begin` is allowed")
93    parser.add_argument("last", type=str, default="end",
94                        help="last commit in range, special value `end` is allowed")
95    parser.add_argument("-f", action="store_true",
96                        help="override previous results")
97
98    args = parser.parse_args()
99
100    if args.first == "begin":
101        args.first = commits[0]
102    if args.last == "end":
103        args.last = commits[-1]
104
105    with shelve.open("benchmark_results") as results:
106        a = commits.index(args.first)
107        b = commits.index(args.last)
108        if args.f:
109            for hash in commits[a:b+1]:
110                del results[hash]
111        run(results, comments, args.first, False)
112        run(results, comments, args.last, False)
113        recursion(results, commits, comments, a, b)
114
115if __name__ == "__main__":
116    main()
117