1#!/usr/bin/env bash 2# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. 3# The RocksDB regression test script. 4# REQUIREMENT: must be able to run make db_bench in the current directory 5# 6# This script will do the following things in order: 7# 8# 1. check out the specified rocksdb commit. 9# 2. build db_bench using the specified commit 10# 3. setup test directory $TEST_PATH. If not specified, then the test directory 11# will be "/tmp/rocksdb/regression_test" 12# 4. run set of benchmarks on the specified host 13# (can be either locally or remotely) 14# 5. generate report in the $RESULT_PATH. If RESULT_PATH is not specified, 15# RESULT_PATH will be set to $TEST_PATH/current_time 16# 17# = Examples = 18# * Run the regression test using rocksdb commit abcdef that outputs results 19# and temp files in "/my/output/dir" 20#r 21# TEST_PATH=/my/output/dir COMMIT_ID=abcdef ./tools/regression_test.sh 22# 23# * Run the regression test on a remost host under "/my/output/dir" directory 24# and stores the result locally in "/my/benchmark/results" using commit 25# abcdef and with the rocksdb options specified in /my/path/to/OPTIONS-012345 26# with 1000000000 keys in each benchmark in the regression test where each 27# key and value are 100 and 900 bytes respectively: 28# 29# REMOTE_USER_AT_HOST=yhchiang@my.remote.host \ 30# TEST_PATH=/my/output/dir \ 31# RESULT_PATH=/my/benchmark/results \ 32# COMMIT_ID=abcdef \ 33# OPTIONS_FILE=/my/path/to/OPTIONS-012345 \ 34# NUM_KEYS=1000000000 \ 35# KEY_SIZE=100 \ 36# VALUE_SIZE=900 \ 37# ./tools/regression_test.sh 38# 39# = Regression test environmental parameters = 40# DEBUG: If true, then the script will not checkout master and build db_bench 41# if db_bench already exists 42# Default: 0 43# TEST_MODE: If 1, run fillseqdeterminstic and benchmarks both 44# if 0, only run fillseqdeterministc 45# if 2, only run benchmarks 46# Default: 1 47# TEST_PATH: the root directory of the regression test. 48# Default: "/tmp/rocksdb/regression_test" 49# RESULT_PATH: the directory where the regression results will be generated. 50# Default: "$TEST_PATH/current_time" 51# REMOTE_USER_AT_HOST: If set, then test will run on the specified host under 52# TEST_PATH directory and outputs test results locally in RESULT_PATH 53# The REMOTE_USER_AT_HOST should follow the format user-id@host.name 54# DB_PATH: the path where the rocksdb database will be created during the 55# regression test. Default: $TEST_PATH/db 56# WAL_PATH: the path where the rocksdb WAL will be outputed. 57# Default: $TEST_PATH/wal 58# OPTIONS_FILE: If specified, then the regression test will use the specified 59# file to initialize the RocksDB options in its benchmarks. Note that 60# this feature only work for commits after 88acd93 or rocksdb version 61# later than 4.9. 62# DELETE_TEST_PATH: If true, then the test directory will be deleted 63# after the script ends. 64# Default: 0 65# 66# = db_bench parameters = 67# NUM_THREADS: The number of concurrent foreground threads that will issue 68# database operations in the benchmark. Default: 16. 69# NUM_KEYS: The key range that will be used in the entire regression test. 70# Default: 1G. 71# NUM_OPS: The number of operations (reads, writes, or deletes) that will 72# be issued in EACH thread. 73# Default: $NUM_KEYS / $NUM_THREADS 74# KEY_SIZE: The size of each key in bytes in db_bench. Default: 100. 75# VALUE_SIZE: The size of each value in bytes in db_bench. Default: 900. 76# CACHE_SIZE: The size of RocksDB block cache used in db_bench. Default: 1G 77# STATISTICS: If 1, then statistics is on in db_bench. Default: 0. 78# COMPRESSION_RATIO: The compression ratio of the key generated in db_bench. 79# Default: 0.5. 80# HISTOGRAM: If 1, then the histogram feature on performance feature is on. 81# STATS_PER_INTERVAL: If 1, then the statistics will be reported for every 82# STATS_INTERVAL_SECONDS seconds. Default 1. 83# STATS_INTERVAL_SECONDS: If STATS_PER_INTERVAL is set to 1, then statistics 84# will be reported for every STATS_INTERVAL_SECONDS. Default 60. 85# MAX_BACKGROUND_FLUSHES: The maxinum number of concurrent flushes in 86# db_bench. Default: 4. 87# MAX_BACKGROUND_COMPACTIONS: The maximum number of concurrent compactions 88# in db_bench. Default: 16. 89# NUM_HIGH_PRI_THREADS: The number of high-pri threads available for 90# concurrent flushes in db_bench. Default: 4. 91# NUM_LOW_PRI_THREADS: The number of low-pri threads available for 92# concurrent compactions in db_bench. Default: 16. 93# SEEK_NEXTS: Controls how many Next() will be called after seek. 94# Default: 10. 95# SEED: random seed that controls the randomness of the benchmark. 96# Default: $( date +%s ) 97 98#============================================================================== 99# CONSTANT 100#============================================================================== 101TITLE_FORMAT="%40s,%25s,%30s,%7s,%9s,%8s," 102TITLE_FORMAT+="%10s,%13s,%14s,%11s,%12s," 103TITLE_FORMAT+="%7s,%11s," 104TITLE_FORMAT+="%9s,%10s,%10s,%10s,%10s,%10s,%5s," 105TITLE_FORMAT+="%5s,%5s,%5s" # time 106TITLE_FORMAT+="\n" 107 108DATA_FORMAT="%40s,%25s,%30s,%7s,%9s,%8s," 109DATA_FORMAT+="%10s,%13.0f,%14s,%11s,%12s," 110DATA_FORMAT+="%7s,%11s," 111DATA_FORMAT+="%9.0f,%10.0f,%10.0f,%10.0f,%10.0f,%10.0f,%5.0f," 112DATA_FORMAT+="%5.0f,%5.0f,%5.0f" # time 113DATA_FORMAT+="\n" 114 115MAIN_PATTERN="$1""[[:blank:]]+:.*[[:blank:]]+([0-9\.]+)[[:blank:]]+ops/sec" 116PERC_PATTERN="Percentiles: P50: ([0-9\.]+) P75: ([0-9\.]+) " 117PERC_PATTERN+="P99: ([0-9\.]+) P99.9: ([0-9\.]+) P99.99: ([0-9\.]+)" 118#============================================================================== 119 120function main { 121 TEST_ROOT_DIR=${TEST_PATH:-"/tmp/rocksdb/regression_test"} 122 init_arguments $TEST_ROOT_DIR 123 124 build_db_bench_and_ldb 125 126 setup_test_directory 127 if [ $TEST_MODE -le 1 ]; then 128 tmp=$DB_PATH 129 DB_PATH=$ORIGIN_PATH 130 test_remote "test -d $DB_PATH" 131 if [[ $? -ne 0 ]]; then 132 echo "Building DB..." 133 # compactall alone will not print ops or threads, which will fail update_report 134 run_db_bench "fillseq,compactall" $NUM_KEYS 1 0 0 135 fi 136 DB_PATH=$tmp 137 fi 138 if [ $TEST_MODE -ge 1 ]; then 139 build_checkpoint 140 run_db_bench "readrandom" 141 run_db_bench "readwhilewriting" 142 run_db_bench "deleterandom" $((NUM_KEYS / 10 / $NUM_THREADS)) 143 run_db_bench "seekrandom" 144 run_db_bench "seekrandomwhilewriting" 145 run_db_bench "multireadrandom" 146 fi 147 148 cleanup_test_directory $TEST_ROOT_DIR 149 echo "" 150 echo "Benchmark completed! Results are available in $RESULT_PATH" 151} 152 153############################################################################ 154function init_arguments { 155 K=1024 156 M=$((1024 * K)) 157 G=$((1024 * M)) 158 159 current_time=$(date +"%F-%H:%M:%S") 160 RESULT_PATH=${RESULT_PATH:-"$1/results/$current_time"} 161 COMMIT_ID=`hg id -i` 162 SUMMARY_FILE="$RESULT_PATH/SUMMARY.csv" 163 164 DB_PATH=${3:-"$1/db"} 165 ORIGIN_PATH=${ORIGIN_PATH:-"$(dirname $(dirname $DB_PATH))/db"} 166 WAL_PATH=${4:-""} 167 if [ -z "$REMOTE_USER_AT_HOST" ]; then 168 DB_BENCH_DIR=${5:-"."} 169 else 170 DB_BENCH_DIR=${5:-"$1/db_bench"} 171 fi 172 173 DEBUG=${DEBUG:-0} 174 TEST_MODE=${TEST_MODE:-1} 175 SCP=${SCP:-"scp"} 176 SSH=${SSH:-"ssh"} 177 NUM_THREADS=${NUM_THREADS:-16} 178 NUM_KEYS=${NUM_KEYS:-$((1 * G))} # key range 179 NUM_OPS=${NUM_OPS:-$(($NUM_KEYS / $NUM_THREADS))} 180 KEY_SIZE=${KEY_SIZE:-100} 181 VALUE_SIZE=${VALUE_SIZE:-900} 182 CACHE_SIZE=${CACHE_SIZE:-$((1 * G))} 183 STATISTICS=${STATISTICS:-0} 184 COMPRESSION_RATIO=${COMPRESSION_RATIO:-0.5} 185 HISTOGRAM=${HISTOGRAM:-1} 186 NUM_MULTI_DB=${NUM_MULTI_DB:-1} 187 STATS_PER_INTERVAL=${STATS_PER_INTERVAL:-1} 188 STATS_INTERVAL_SECONDS=${STATS_INTERVAL_SECONDS:-600} 189 MAX_BACKGROUND_FLUSHES=${MAX_BACKGROUND_FLUSHES:-4} 190 MAX_BACKGROUND_COMPACTIONS=${MAX_BACKGROUND_COMPACTIONS:-16} 191 NUM_HIGH_PRI_THREADS=${NUM_HIGH_PRI_THREADS:-4} 192 NUM_LOW_PRI_THREADS=${NUM_LOW_PRI_THREADS:-16} 193 DELETE_TEST_PATH=${DELETE_TEST_PATH:-0} 194 SEEK_NEXTS=${SEEK_NEXTS:-10} 195 SEED=${SEED:-$( date +%s )} 196 MULTIREAD_BATCH_SIZE=${MULTIREAD_BATCH_SIZE:-128} 197 MULTIREAD_STRIDE=${MULTIREAD_STRIDE:-12} 198} 199 200# $1 --- benchmark name 201# $2 --- number of operations. Default: $NUM_KEYS 202# $3 --- number of threads. Default $NUM_THREADS 203# $4 --- use_existing_db. Default: 1 204# $5 --- update_report. Default: 1 205function run_db_bench { 206 # this will terminate all currently-running db_bench 207 find_db_bench_cmd="ps aux | grep db_bench | grep -v grep | grep -v aux | awk '{print \$2}'" 208 209 ops=${2:-$NUM_OPS} 210 threads=${3:-$NUM_THREADS} 211 USE_EXISTING_DB=${4:-1} 212 UPDATE_REPORT=${5:-1} 213 echo "" 214 echo "=======================================================================" 215 echo "Benchmark $1" 216 echo "=======================================================================" 217 echo "" 218 db_bench_error=0 219 options_file_arg=$(setup_options_file) 220 echo "$options_file_arg" 221 # use `which time` to avoid using bash's internal time command 222 db_bench_cmd="("'\$(which time)'" -p $DB_BENCH_DIR/db_bench \ 223 --benchmarks=$1 --db=$DB_PATH --wal_dir=$WAL_PATH \ 224 --use_existing_db=$USE_EXISTING_DB \ 225 --disable_auto_compactions \ 226 --threads=$threads \ 227 --num=$NUM_KEYS \ 228 --reads=$ops \ 229 --writes=$ops \ 230 --deletes=$ops \ 231 --key_size=$KEY_SIZE \ 232 --value_size=$VALUE_SIZE \ 233 --cache_size=$CACHE_SIZE \ 234 --statistics=$STATISTICS \ 235 $options_file_arg \ 236 --compression_ratio=$COMPRESSION_RATIO \ 237 --histogram=$HISTOGRAM \ 238 --seek_nexts=$SEEK_NEXTS \ 239 --stats_per_interval=$STATS_PER_INTERVAL \ 240 --stats_interval_seconds=$STATS_INTERVAL_SECONDS \ 241 --max_background_flushes=$MAX_BACKGROUND_FLUSHES \ 242 --num_multi_db=$NUM_MULTI_DB \ 243 --max_background_compactions=$MAX_BACKGROUND_COMPACTIONS \ 244 --num_high_pri_threads=$NUM_HIGH_PRI_THREADS \ 245 --num_low_pri_threads=$NUM_LOW_PRI_THREADS \ 246 --seed=$SEED \ 247 --multiread_batched=true \ 248 --batch_size=$MULTIREAD_BATCH_SIZE \ 249 --multiread_stride=$MULTIREAD_STRIDE) 2>&1" 250 ps_cmd="ps aux" 251 if ! [ -z "$REMOTE_USER_AT_HOST" ]; then 252 echo "Running benchmark remotely on $REMOTE_USER_AT_HOST" 253 db_bench_cmd="$SSH $REMOTE_USER_AT_HOST \"$db_bench_cmd\"" 254 ps_cmd="$SSH $REMOTE_USER_AT_HOST $ps_cmd" 255 fi 256 257 ## make sure no db_bench is running 258 # The following statement is necessary make sure "eval $ps_cmd" will success. 259 # Otherwise, if we simply check whether "$(eval $ps_cmd | grep db_bench)" is 260 # successful or not, then it will always be false since grep will return 261 # non-zero status when there's no matching output. 262 ps_output="$(eval $ps_cmd)" 263 exit_on_error $? "$ps_cmd" 264 265 # perform the actual command to check whether db_bench is running 266 grep_output="$(eval $ps_cmd | grep db_bench | grep -v grep)" 267 if [ "$grep_output" != "" ]; then 268 echo "Stopped regression_test.sh as there're still db_bench processes running:" 269 echo $grep_output 270 echo "Clean up test directory" 271 cleanup_test_directory $TEST_ROOT_DIR 272 exit 2 273 fi 274 275 ## run the db_bench 276 cmd="($db_bench_cmd || db_bench_error=1) | tee -a $RESULT_PATH/$1" 277 exit_on_error $? 278 echo $cmd 279 eval $cmd 280 exit_on_error $db_bench_error 281 if [ $UPDATE_REPORT -ne 0 ]; then 282 update_report "$1" "$RESULT_PATH/$1" $ops $threads 283 fi 284} 285 286function build_checkpoint { 287 cmd_prefix="" 288 if ! [ -z "$REMOTE_USER_AT_HOST" ]; then 289 cmd_prefix="$SSH $REMOTE_USER_AT_HOST " 290 fi 291 if [ $NUM_MULTI_DB -gt 1 ]; then 292 dirs=$($cmd_prefix find $ORIGIN_PATH -type d -links 2) 293 for dir in $dirs; do 294 db_index=$(basename $dir) 295 echo "Building checkpoints: $ORIGIN_PATH/$db_index -> $DB_PATH/$db_index ..." 296 $cmd_prefix $DB_BENCH_DIR/ldb checkpoint --checkpoint_dir=$DB_PATH/$db_index \ 297 --db=$ORIGIN_PATH/$db_index --try_load_options 2>&1 298 done 299 else 300 # checkpoint cannot build in directory already exists 301 $cmd_prefix rm -rf $DB_PATH 302 echo "Building checkpoint: $ORIGIN_PATH -> $DB_PATH ..." 303 $cmd_prefix $DB_BENCH_DIR/ldb checkpoint --checkpoint_dir=$DB_PATH \ 304 --db=$ORIGIN_PATH --try_load_options 2>&1 305 fi 306} 307 308function multiply { 309 echo "$1 * $2" | bc 310} 311 312# $1 --- name of the benchmark 313# $2 --- the filename of the output log of db_bench 314function update_report { 315 main_result=`cat $2 | grep $1` 316 exit_on_error $? 317 perc_statement=`cat $2 | grep Percentile` 318 exit_on_error $? 319 320 # Obtain micros / op 321 322 [[ $main_result =~ $MAIN_PATTERN ]] 323 ops_per_s=${BASH_REMATCH[1]} 324 325 # Obtain percentile information 326 [[ $perc_statement =~ $PERC_PATTERN ]] 327 perc[0]=${BASH_REMATCH[1]} # p50 328 perc[1]=${BASH_REMATCH[2]} # p75 329 perc[2]=${BASH_REMATCH[3]} # p99 330 perc[3]=${BASH_REMATCH[4]} # p99.9 331 perc[4]=${BASH_REMATCH[5]} # p99.99 332 333 # Parse the output of the time command 334 real_sec=`tail -3 $2 | grep real | awk '{print $2}'` 335 user_sec=`tail -3 $2 | grep user | awk '{print $2}'` 336 sys_sec=`tail -3 $2 | grep sys | awk '{print $2}'` 337 338 (printf "$DATA_FORMAT" \ 339 $COMMIT_ID $1 $REMOTE_USER_AT_HOST $NUM_MULTI_DB $NUM_KEYS $KEY_SIZE $VALUE_SIZE \ 340 $(multiply $COMPRESSION_RATIO 100) \ 341 $3 $4 $CACHE_SIZE \ 342 $MAX_BACKGROUND_FLUSHES $MAX_BACKGROUND_COMPACTIONS \ 343 $ops_per_s \ 344 $(multiply ${perc[0]} 1000) \ 345 $(multiply ${perc[1]} 1000) \ 346 $(multiply ${perc[2]} 1000) \ 347 $(multiply ${perc[3]} 1000) \ 348 $(multiply ${perc[4]} 1000) \ 349 $DEBUG \ 350 $real_sec \ 351 $user_sec \ 352 $sys_sec \ 353 >> $SUMMARY_FILE) 354 exit_on_error $? 355} 356 357function exit_on_error { 358 if [ $1 -ne 0 ]; then 359 echo "" 360 echo "ERROR: Benchmark did not complete successfully." 361 if ! [ -z "$2" ]; then 362 echo "Failure command: $2" 363 fi 364 echo "Partial results are output to $RESULT_PATH" 365 echo "ERROR" >> $SUMMARY_FILE 366 exit $1 367 fi 368} 369 370function build_db_bench_and_ldb { 371 echo "Building db_bench & ldb ..." 372 373 make clean 374 exit_on_error $? 375 376 DEBUG_LEVEL=0 make db_bench ldb -j32 377 exit_on_error $? 378} 379 380function run_remote { 381 test_remote "$1" 382 exit_on_error $? "$1" 383} 384 385function test_remote { 386 if ! [ -z "$REMOTE_USER_AT_HOST" ]; then 387 cmd="$SSH $REMOTE_USER_AT_HOST '$1'" 388 else 389 cmd="$1" 390 fi 391 eval "$cmd" 392} 393 394function run_local { 395 eval "$1" 396 exit_on_error $? 397} 398 399function setup_options_file { 400 if ! [ -z "$OPTIONS_FILE" ]; then 401 if ! [ -z "$REMOTE_USER_AT_HOST" ]; then 402 options_file="$DB_BENCH_DIR/OPTIONS_FILE" 403 run_local "$SCP $OPTIONS_FILE $REMOTE_USER_AT_HOST:$options_file" 404 else 405 options_file="$OPTIONS_FILE" 406 fi 407 echo "--options_file=$options_file" 408 fi 409 echo "" 410} 411 412function setup_test_directory { 413 echo "Deleting old regression test directories and creating new ones" 414 415 run_remote "rm -rf $DB_PATH" 416 run_remote "rm -rf $DB_BENCH_DIR" 417 run_local "rm -rf $RESULT_PATH" 418 419 if ! [ -z "$WAL_PATH" ]; then 420 run_remote "rm -rf $WAL_PATH" 421 run_remote "mkdir -p $WAL_PATH" 422 fi 423 424 run_remote "mkdir -p $DB_PATH" 425 426 run_remote "mkdir -p $DB_BENCH_DIR" 427 run_remote "ls -l $DB_BENCH_DIR" 428 429 if ! [ -z "$REMOTE_USER_AT_HOST" ]; then 430 run_local "$SCP ./db_bench $REMOTE_USER_AT_HOST:$DB_BENCH_DIR/db_bench" 431 run_local "$SCP ./ldb $REMOTE_USER_AT_HOST:$DB_BENCH_DIR/ldb" 432 fi 433 434 run_local "mkdir -p $RESULT_PATH" 435 436 (printf $TITLE_FORMAT \ 437 "commit id" "benchmark" "user@host" "num-dbs" "key-range" "key-size" \ 438 "value-size" "compress-rate" "ops-per-thread" "num-threads" "cache-size" \ 439 "flushes" "compactions" \ 440 "ops-per-s" "p50" "p75" "p99" "p99.9" "p99.99" "debug" \ 441 "real-sec" "user-sec" "sys-sec" \ 442 >> $SUMMARY_FILE) 443 exit_on_error $? 444} 445 446function cleanup_test_directory { 447 448 if [ $DELETE_TEST_PATH -ne 0 ]; then 449 echo "Clear old regression test directories and creating new ones" 450 run_remote "rm -rf $DB_PATH" 451 run_remote "rm -rf $WAL_PATH" 452 if ! [ -z "$REMOTE_USER_AT_HOST" ]; then 453 run_remote "rm -rf $DB_BENCH_DIR" 454 fi 455 run_remote "rm -rf $1" 456 else 457 echo "------------ DEBUG MODE ------------" 458 echo "DB PATH: $DB_PATH" 459 echo "WAL PATH: $WAL_PATH" 460 fi 461} 462 463############################################################################ 464 465# shellcheck disable=SC2068 466main $@ 467