1# Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2# This source code is licensed under both the GPLv2 (found in the 3# COPYING file in the root directory) and Apache 2.0 License 4# (found in the LICENSE.Apache file in the root directory). 5 6import argparse 7from advisor.db_config_optimizer import ConfigOptimizer 8from advisor.db_log_parser import NO_COL_FAMILY 9from advisor.db_options_parser import DatabaseOptions 10from advisor.rule_parser import RulesSpec 11 12 13CONFIG_OPT_NUM_ITER = 10 14 15 16def main(args): 17 # initialise the RulesSpec parser 18 rule_spec_parser = RulesSpec(args.rules_spec) 19 # initialise the benchmark runner 20 bench_runner_module = __import__( 21 args.benchrunner_module, fromlist=[args.benchrunner_class] 22 ) 23 bench_runner_class = getattr(bench_runner_module, args.benchrunner_class) 24 ods_args = {} 25 if args.ods_client and args.ods_entity: 26 ods_args['client_script'] = args.ods_client 27 ods_args['entity'] = args.ods_entity 28 if args.ods_key_prefix: 29 ods_args['key_prefix'] = args.ods_key_prefix 30 db_bench_runner = bench_runner_class(args.benchrunner_pos_args, ods_args) 31 # initialise the database configuration 32 db_options = DatabaseOptions(args.rocksdb_options, args.misc_options) 33 # set the frequency at which stats are dumped in the LOG file and the 34 # location of the LOG file. 35 db_log_dump_settings = { 36 "DBOptions.stats_dump_period_sec": { 37 NO_COL_FAMILY: args.stats_dump_period_sec 38 } 39 } 40 db_options.update_options(db_log_dump_settings) 41 # initialise the configuration optimizer 42 config_optimizer = ConfigOptimizer( 43 db_bench_runner, 44 db_options, 45 rule_spec_parser, 46 args.base_db_path 47 ) 48 # run the optimiser to improve the database configuration for given 49 # benchmarks, with the help of expert-specified rules 50 final_db_options = config_optimizer.run() 51 # generate the final rocksdb options file 52 print( 53 'Final configuration in: ' + 54 final_db_options.generate_options_config('final') 55 ) 56 print( 57 'Final miscellaneous options: ' + 58 repr(final_db_options.get_misc_options()) 59 ) 60 61 62if __name__ == '__main__': 63 ''' 64 An example run of this tool from the command-line would look like: 65 python3 -m advisor.config_optimizer_example 66 --base_db_path=/tmp/rocksdbtest-155919/dbbench 67 --rocksdb_options=temp/OPTIONS_boot.tmp --misc_options bloom_bits=2 68 --rules_spec=advisor/rules.ini --stats_dump_period_sec=20 69 --benchrunner_module=advisor.db_bench_runner 70 --benchrunner_class=DBBenchRunner --benchrunner_pos_args ./../../db_bench 71 readwhilewriting use_existing_db=true duration=90 72 ''' 73 parser = argparse.ArgumentParser(description='This script is used for\ 74 searching for a better database configuration') 75 parser.add_argument( 76 '--rocksdb_options', required=True, type=str, 77 help='path of the starting Rocksdb OPTIONS file' 78 ) 79 # these are options that are column-family agnostic and are not yet 80 # supported by the Rocksdb Options file: eg. bloom_bits=2 81 parser.add_argument( 82 '--misc_options', nargs='*', 83 help='whitespace-separated list of options that are not supported ' + 84 'by the Rocksdb OPTIONS file, given in the ' + 85 '<option_name>=<option_value> format eg. "bloom_bits=2 ' + 86 'rate_limiter_bytes_per_sec=128000000"') 87 parser.add_argument( 88 '--base_db_path', required=True, type=str, 89 help='path for the Rocksdb database' 90 ) 91 parser.add_argument( 92 '--rules_spec', required=True, type=str, 93 help='path of the file containing the expert-specified Rules' 94 ) 95 parser.add_argument( 96 '--stats_dump_period_sec', required=True, type=int, 97 help='the frequency (in seconds) at which STATISTICS are printed to ' + 98 'the Rocksdb LOG file' 99 ) 100 # ODS arguments 101 parser.add_argument( 102 '--ods_client', type=str, help='the ODS client binary' 103 ) 104 parser.add_argument( 105 '--ods_entity', type=str, 106 help='the servers for which the ODS stats need to be fetched' 107 ) 108 parser.add_argument( 109 '--ods_key_prefix', type=str, 110 help='the prefix that needs to be attached to the keys of time ' + 111 'series to be fetched from ODS' 112 ) 113 # benchrunner_module example: advisor.db_benchmark_client 114 parser.add_argument( 115 '--benchrunner_module', required=True, type=str, 116 help='the module containing the BenchmarkRunner class to be used by ' + 117 'the Optimizer, example: advisor.db_bench_runner' 118 ) 119 # benchrunner_class example: DBBenchRunner 120 parser.add_argument( 121 '--benchrunner_class', required=True, type=str, 122 help='the name of the BenchmarkRunner class to be used by the ' + 123 'Optimizer, should be present in the module provided in the ' + 124 'benchrunner_module argument, example: DBBenchRunner' 125 ) 126 parser.add_argument( 127 '--benchrunner_pos_args', nargs='*', 128 help='whitespace-separated positional arguments that are passed on ' + 129 'to the constructor of the BenchmarkRunner class provided in the ' + 130 'benchrunner_class argument, example: "use_existing_db=true ' + 131 'duration=900"' 132 ) 133 args = parser.parse_args() 134 main(args) 135