1#!/usr/bin/env bash
2#
3# Generates, builds and runs parsers from grammar directory for each git reference supplied as argument.
4# Each action is performed multiple times and the times are averaged. First reference is always
5# taken as a "baseline" and others are compared to it. This should allow to compare how any given commit
6# affects PackCCs performance.
7#
8# Usage:
9#   ./benchmark.sh <git ref> ...
10#
11# Environment:
12#   CC              Compiler to use, default: "cc -O2"
13#   GEN_REPEATS     How many times to generate the parser, default: 10
14#   BUILD_REPEATS   How many times to build the parser, default: 5
15#   RUN_REPEATS     How many times to run the given parser, default: 20
16#
17# Example:
18#   CC="clang -O3" ./benchmark.sh origin/master 6015afc HEAD
19
20build() {
21    echo "Building packcc..."
22    $CC -o "$PACKCC" $ROOTDIR/src/packcc.c
23}
24
25clean() {
26    rm -rf "$BENCHDIR/tmp"
27}
28
29format() {
30    TIME="$1"
31    if [ $((TIME / 1000000000)) -gt 10 ]; then
32        echo "$((TIME / 1000000000)) s"
33    elif [ $((TIME / 1000000)) -gt 10 ]; then
34        echo "$((TIME / 1000000)) ms"
35    elif [ $((TIME / 1000)) -gt 10 ]; then
36        echo "$((TIME / 1000)) us"
37    else
38        echo "$((TIME)) ns"
39    fi
40}
41
42measure() {
43    COUNT="$1"
44    shift
45    START="$(date '+%s%N')"
46    for ((i=0; i<COUNT; i++)); do
47        "$@"
48    done
49    END="$(date '+%s%N')"
50    TIME=$(( END - START ))
51}
52
53run() {
54    "$1" < "$2" > /dev/null
55}
56
57benchmark() {
58    KEY="${GRAMMAR}_${REF//\//_}"
59    NAME="tmp/parser_$KEY"
60
61    echo "Generating $GRAMMAR parser in $REF ($GEN_REPEATS times)..."
62    measure "$GEN_REPEATS" "$PACKCC" -o "$NAME" "$GRAMMAR_FILE"
63    GEN["$KEY"]=$TIME
64    echo "  Repeated $GEN_REPEATS times in $(format $TIME)"
65
66    echo "Building $GRAMMAR parser in $REF ($BUILD_REPEATS times)..."
67    measure "$BUILD_REPEATS" $CC -I. "$NAME".c -o "$NAME"
68    BUILD["$KEY"]=$TIME
69    echo "  Built $BUILD_REPEATS times in $(format $TIME)"
70
71    echo "Running $GRAMMAR parser in $REF ($RUN_REPEATS times)..."
72    measure "$RUN_REPEATS" run "./$NAME" "$INPUT"
73    RUN["$KEY"]=$TIME
74    echo "  Repeated $RUN_REPEATS times in $(format $TIME)"
75}
76
77print_table() {
78    declare -n RESULTS="$1"
79    printf "%-12s" ""
80    for REF in "${REFS[@]}"; do
81        printf "%-16s" "$REF"
82    done
83    printf "\n"
84    for GRAMMAR in "${GRAMMARS[@]}"; do
85        printf "%-12s" "$GRAMMAR"
86        for REF in "${REFS[@]}"; do
87            KEY="${GRAMMAR}_${REF//\//_}"
88            BASE="${GRAMMAR}_${REFS[0]//\//_}"
89            TIME="$((${RESULTS["$KEY"]} / RUN_REPEATS))"
90            RELATIVE="$((100 * RESULTS["$KEY"] / RESULTS["$BASE"]))"
91            COLOR=$((RELATIVE == 100 ? 0 : ( RELATIVE > 100 ? 31 : 32)))
92            printf "\033[0;${COLOR}m%-16s\033[0m" "$(format $TIME) ($RELATIVE%)"
93        done
94        printf "\n"
95    done
96}
97
98print_results() {
99    echo
100    echo "Generation times:"
101    echo "================="
102    print_table GEN
103    echo
104    echo "Build times:"
105    echo "============"
106    print_table BUILD
107    echo
108    echo "Run times:"
109    echo "=========="
110    print_table RUN
111}
112
113main() {
114    set -e
115
116    BENCHDIR="$(cd "$(dirname "$0")" && pwd)"
117    ROOTDIR="$BENCHDIR/.."
118    declare -a GRAMMARS=()
119    declare -A BUILD=()
120    declare -A GEN=()
121    declare -A RUN=()
122
123    declare -i GEN_REPEATS="${GEN_REPEATS:-10}"
124    declare -i BUILD_REPEATS="${BUILD_REPEATS:-5}"
125    declare -i RUN_REPEATS="${RUN_REPEATS:-20}"
126    CC="${CC:-cc -O2}"
127    REFS=("$@")
128
129    if [[ $# -eq 0 || "$1" =~ -h|--help|--usage ]]; then
130        sed -n '3,/^$/s/^#//p' "$0"
131        exit 0
132    fi
133
134    START_REF="$(git name-rev --name-only HEAD)"
135    trap "echo 'Returning to $START_REF...' && git checkout $START_REF" EXIT ERR INT
136
137    cd "$BENCHDIR"
138    clean
139    mkdir "tmp"
140    cp -aL inputs grammars tmp/
141
142    for REF in "${REFS[@]}"; do
143        PACKCC="tmp/packcc_${REF//\//_}"
144        git checkout "$REF"
145        build
146        for GRAMMAR_FILE in "tmp/grammars"/*.peg ; do
147            GRAMMAR="$(basename "$GRAMMAR_FILE" .peg)"
148            [ "$REF" == "${REFS[0]}" ] && GRAMMARS+=("$GRAMMAR")
149            INPUT="$(ls "tmp/inputs/$GRAMMAR"*)"
150            benchmark
151        done
152    done
153
154    print_results
155}
156
157main "$@"
158