1#!/usr/bin/python 2# 3# Copyright (c) 2009-2021, Google LLC 4# All rights reserved. 5# 6# Redistribution and use in source and binary forms, with or without 7# modification, are permitted provided that the following conditions are met: 8# * Redistributions of source code must retain the above copyright 9# notice, this list of conditions and the following disclaimer. 10# * Redistributions in binary form must reproduce the above copyright 11# notice, this list of conditions and the following disclaimer in the 12# documentation and/or other materials provided with the distribution. 13# * Neither the name of Google LLC nor the 14# names of its contributors may be used to endorse or promote products 15# derived from this software without specific prior written permission. 16# 17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 18# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 20# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY 21# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 22# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 23# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 24# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 26# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 28"""Benchmarks the current working directory against a given baseline. 29 30This script benchmarks both size and speed. Sample output: 31""" 32 33import contextlib 34import json 35import os 36import re 37import subprocess 38import sys 39import tempfile 40 41@contextlib.contextmanager 42def GitWorktree(commit): 43 tmpdir = tempfile.mkdtemp() 44 subprocess.run(['git', 'worktree', 'add', '-q', '-d', tmpdir, commit], check=True) 45 cwd = os.getcwd() 46 os.chdir(tmpdir) 47 try: 48 yield tmpdir 49 finally: 50 os.chdir(cwd) 51 subprocess.run(['git', 'worktree', 'remove', tmpdir], check=True) 52 53def Run(cmd): 54 subprocess.check_call(cmd, shell=True) 55 56def Benchmark(outbase, bench_cpu=True, runs=12, fasttable=False): 57 tmpfile = "/tmp/bench-output.json" 58 Run("rm -rf {}".format(tmpfile)) 59 #Run("CC=clang bazel test ...") 60 if fasttable: 61 extra_args = " --//:fasttable_enabled=true" 62 else: 63 extra_args = "" 64 65 if bench_cpu: 66 Run("CC=clang bazel build -c opt --copt=-march=native benchmarks:benchmark" + extra_args) 67 Run("./bazel-bin/benchmarks/benchmark --benchmark_out_format=json --benchmark_out={} --benchmark_repetitions={}".format(tmpfile, runs)) 68 with open(tmpfile) as f: 69 bench_json = json.load(f) 70 71 # Translate into the format expected by benchstat. 72 with open(outbase + ".txt", "w") as f: 73 for run in bench_json["benchmarks"]: 74 name = run["name"] 75 name = name.replace(" ", "") 76 name = re.sub(r'^BM_', 'Benchmark', name) 77 if name.endswith("_mean") or name.endswith("_median") or name.endswith("_stddev"): 78 continue 79 values = (name, run["iterations"], run["cpu_time"]) 80 print("{} {} {} ns/op".format(*values), file=f) 81 82 Run("CC=clang bazel build -c opt --copt=-g tests:conformance_upb" + extra_args) 83 Run("cp -f bazel-bin/tests/conformance_upb {}.bin".format(outbase)) 84 85 86baseline = "master" 87bench_cpu = True 88fasttable = False 89 90if len(sys.argv) > 1: 91 baseline = sys.argv[1] 92 93 # Quickly verify that the baseline exists. 94 with GitWorktree(baseline): 95 pass 96 97# Benchmark our current directory first, since it's more likely to be broken. 98Benchmark("/tmp/new", bench_cpu, fasttable=fasttable) 99 100# Benchmark the baseline. 101with GitWorktree(baseline): 102 Benchmark("/tmp/old", bench_cpu, fasttable=fasttable) 103 104print() 105print() 106 107if bench_cpu: 108 Run("~/go/bin/benchstat /tmp/old.txt /tmp/new.txt") 109 110print() 111print() 112 113Run("objcopy --strip-debug /tmp/old.bin /tmp/old.bin.stripped") 114Run("objcopy --strip-debug /tmp/new.bin /tmp/new.bin.stripped") 115Run("~/code/bloaty/bloaty /tmp/new.bin.stripped -- /tmp/old.bin.stripped --debug-file=/tmp/old.bin --debug-file=/tmp/new.bin -d compileunits,symbols") 116