1#!/usr/bin/env bash 2# 3# Generates, builds and runs parsers from grammar directory for each git reference supplied as argument. 4# Each action is performed multiple times and the times are averaged. First reference is always 5# taken as a "baseline" and others are compared to it. This should allow to compare how any given commit 6# affects PackCCs performance. 7# 8# Usage: 9# ./benchmark.sh <git ref> ... 10# 11# Environment: 12# CC Compiler to use, default: "cc -O2" 13# GEN_REPEATS How many times to generate the parser, default: 10 14# BUILD_REPEATS How many times to build the parser, default: 5 15# RUN_REPEATS How many times to run the given parser, default: 20 16# 17# Example: 18# CC="clang -O3" ./benchmark.sh origin/master 6015afc HEAD 19 20build() { 21 echo "Building packcc..." 22 $CC -o "$PACKCC" $ROOTDIR/src/packcc.c 23} 24 25clean() { 26 rm -rf "$BENCHDIR/tmp" 27} 28 29format() { 30 TIME="$1" 31 if [ $((TIME / 1000000000)) -gt 10 ]; then 32 echo "$((TIME / 1000000000)) s" 33 elif [ $((TIME / 1000000)) -gt 10 ]; then 34 echo "$((TIME / 1000000)) ms" 35 elif [ $((TIME / 1000)) -gt 10 ]; then 36 echo "$((TIME / 1000)) us" 37 else 38 echo "$((TIME)) ns" 39 fi 40} 41 42measure() { 43 COUNT="$1" 44 shift 45 START="$(date '+%s%N')" 46 for ((i=0; i<COUNT; i++)); do 47 "$@" 48 done 49 END="$(date '+%s%N')" 50 TIME=$(( END - START )) 51} 52 53run() { 54 "$1" < "$2" > /dev/null 55} 56 57benchmark() { 58 KEY="${GRAMMAR}_${REF//\//_}" 59 NAME="tmp/parser_$KEY" 60 61 echo "Generating $GRAMMAR parser in $REF ($GEN_REPEATS times)..." 62 measure "$GEN_REPEATS" "$PACKCC" -o "$NAME" "$GRAMMAR_FILE" 63 GEN["$KEY"]=$TIME 64 echo " Repeated $GEN_REPEATS times in $(format $TIME)" 65 66 echo "Building $GRAMMAR parser in $REF ($BUILD_REPEATS times)..." 67 measure "$BUILD_REPEATS" $CC -I. "$NAME".c -o "$NAME" 68 BUILD["$KEY"]=$TIME 69 echo " Built $BUILD_REPEATS times in $(format $TIME)" 70 71 echo "Running $GRAMMAR parser in $REF ($RUN_REPEATS times)..." 72 measure "$RUN_REPEATS" run "./$NAME" "$INPUT" 73 RUN["$KEY"]=$TIME 74 echo " Repeated $RUN_REPEATS times in $(format $TIME)" 75} 76 77print_table() { 78 declare -n RESULTS="$1" 79 printf "%-12s" "" 80 for REF in "${REFS[@]}"; do 81 printf "%-16s" "$REF" 82 done 83 printf "\n" 84 for GRAMMAR in "${GRAMMARS[@]}"; do 85 printf "%-12s" "$GRAMMAR" 86 for REF in "${REFS[@]}"; do 87 KEY="${GRAMMAR}_${REF//\//_}" 88 BASE="${GRAMMAR}_${REFS[0]//\//_}" 89 TIME="$((${RESULTS["$KEY"]} / RUN_REPEATS))" 90 RELATIVE="$((100 * RESULTS["$KEY"] / RESULTS["$BASE"]))" 91 COLOR=$((RELATIVE == 100 ? 0 : ( RELATIVE > 100 ? 31 : 32))) 92 printf "\033[0;${COLOR}m%-16s\033[0m" "$(format $TIME) ($RELATIVE%)" 93 done 94 printf "\n" 95 done 96} 97 98print_results() { 99 echo 100 echo "Generation times:" 101 echo "=================" 102 print_table GEN 103 echo 104 echo "Build times:" 105 echo "============" 106 print_table BUILD 107 echo 108 echo "Run times:" 109 echo "==========" 110 print_table RUN 111} 112 113main() { 114 set -e 115 116 BENCHDIR="$(cd "$(dirname "$0")" && pwd)" 117 ROOTDIR="$BENCHDIR/.." 118 declare -a GRAMMARS=() 119 declare -A BUILD=() 120 declare -A GEN=() 121 declare -A RUN=() 122 123 declare -i GEN_REPEATS="${GEN_REPEATS:-10}" 124 declare -i BUILD_REPEATS="${BUILD_REPEATS:-5}" 125 declare -i RUN_REPEATS="${RUN_REPEATS:-20}" 126 CC="${CC:-cc -O2}" 127 REFS=("$@") 128 129 if [[ $# -eq 0 || "$1" =~ -h|--help|--usage ]]; then 130 sed -n '3,/^$/s/^#//p' "$0" 131 exit 0 132 fi 133 134 START_REF="$(git name-rev --name-only HEAD)" 135 trap "echo 'Returning to $START_REF...' && git checkout $START_REF" EXIT ERR INT 136 137 cd "$BENCHDIR" 138 clean 139 mkdir "tmp" 140 cp -aL inputs grammars tmp/ 141 142 for REF in "${REFS[@]}"; do 143 PACKCC="tmp/packcc_${REF//\//_}" 144 git checkout "$REF" 145 build 146 for GRAMMAR_FILE in "tmp/grammars"/*.peg ; do 147 GRAMMAR="$(basename "$GRAMMAR_FILE" .peg)" 148 [ "$REF" == "${REFS[0]}" ] && GRAMMARS+=("$GRAMMAR") 149 INPUT="$(ls "tmp/inputs/$GRAMMAR"*)" 150 benchmark 151 done 152 done 153 154 print_results 155} 156 157main "$@" 158