1# Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
2#  This source code is licensed under both the GPLv2 (found in the
3#  COPYING file in the root directory) and Apache 2.0 License
4#  (found in the LICENSE.Apache file in the root directory).
5
6import argparse
7from advisor.db_config_optimizer import ConfigOptimizer
8from advisor.db_log_parser import NO_COL_FAMILY
9from advisor.db_options_parser import DatabaseOptions
10from advisor.rule_parser import RulesSpec
11
12
13CONFIG_OPT_NUM_ITER = 10
14
15
16def main(args):
17    # initialise the RulesSpec parser
18    rule_spec_parser = RulesSpec(args.rules_spec)
19    # initialise the benchmark runner
20    bench_runner_module = __import__(
21        args.benchrunner_module, fromlist=[args.benchrunner_class]
22    )
23    bench_runner_class = getattr(bench_runner_module, args.benchrunner_class)
24    ods_args = {}
25    if args.ods_client and args.ods_entity:
26        ods_args['client_script'] = args.ods_client
27        ods_args['entity'] = args.ods_entity
28        if args.ods_key_prefix:
29            ods_args['key_prefix'] = args.ods_key_prefix
30    db_bench_runner = bench_runner_class(args.benchrunner_pos_args, ods_args)
31    # initialise the database configuration
32    db_options = DatabaseOptions(args.rocksdb_options, args.misc_options)
33    # set the frequency at which stats are dumped in the LOG file and the
34    # location of the LOG file.
35    db_log_dump_settings = {
36        "DBOptions.stats_dump_period_sec": {
37            NO_COL_FAMILY: args.stats_dump_period_sec
38        }
39    }
40    db_options.update_options(db_log_dump_settings)
41    # initialise the configuration optimizer
42    config_optimizer = ConfigOptimizer(
43        db_bench_runner,
44        db_options,
45        rule_spec_parser,
46        args.base_db_path
47    )
48    # run the optimiser to improve the database configuration for given
49    # benchmarks, with the help of expert-specified rules
50    final_db_options = config_optimizer.run()
51    # generate the final rocksdb options file
52    print(
53        'Final configuration in: ' +
54        final_db_options.generate_options_config('final')
55    )
56    print(
57        'Final miscellaneous options: ' +
58        repr(final_db_options.get_misc_options())
59    )
60
61
62if __name__ == '__main__':
63    '''
64    An example run of this tool from the command-line would look like:
65    python3 -m advisor.config_optimizer_example
66    --base_db_path=/tmp/rocksdbtest-155919/dbbench
67    --rocksdb_options=temp/OPTIONS_boot.tmp --misc_options bloom_bits=2
68    --rules_spec=advisor/rules.ini --stats_dump_period_sec=20
69    --benchrunner_module=advisor.db_bench_runner
70    --benchrunner_class=DBBenchRunner --benchrunner_pos_args ./../../db_bench
71    readwhilewriting use_existing_db=true duration=90
72    '''
73    parser = argparse.ArgumentParser(description='This script is used for\
74        searching for a better database configuration')
75    parser.add_argument(
76        '--rocksdb_options', required=True, type=str,
77        help='path of the starting Rocksdb OPTIONS file'
78    )
79    # these are options that are column-family agnostic and are not yet
80    # supported by the Rocksdb Options file: eg. bloom_bits=2
81    parser.add_argument(
82        '--misc_options', nargs='*',
83        help='whitespace-separated list of options that are not supported ' +
84        'by the Rocksdb OPTIONS file, given in the ' +
85        '<option_name>=<option_value> format eg. "bloom_bits=2 ' +
86        'rate_limiter_bytes_per_sec=128000000"')
87    parser.add_argument(
88        '--base_db_path', required=True, type=str,
89        help='path for the Rocksdb database'
90    )
91    parser.add_argument(
92        '--rules_spec', required=True, type=str,
93        help='path of the file containing the expert-specified Rules'
94    )
95    parser.add_argument(
96        '--stats_dump_period_sec', required=True, type=int,
97        help='the frequency (in seconds) at which STATISTICS are printed to ' +
98        'the Rocksdb LOG file'
99    )
100    # ODS arguments
101    parser.add_argument(
102        '--ods_client', type=str, help='the ODS client binary'
103    )
104    parser.add_argument(
105        '--ods_entity', type=str,
106        help='the servers for which the ODS stats need to be fetched'
107    )
108    parser.add_argument(
109        '--ods_key_prefix', type=str,
110        help='the prefix that needs to be attached to the keys of time ' +
111        'series to be fetched from ODS'
112    )
113    # benchrunner_module example: advisor.db_benchmark_client
114    parser.add_argument(
115        '--benchrunner_module', required=True, type=str,
116        help='the module containing the BenchmarkRunner class to be used by ' +
117        'the Optimizer, example: advisor.db_bench_runner'
118    )
119    # benchrunner_class example: DBBenchRunner
120    parser.add_argument(
121        '--benchrunner_class', required=True, type=str,
122        help='the name of the BenchmarkRunner class to be used by the ' +
123        'Optimizer, should be present in the module provided in the ' +
124        'benchrunner_module argument, example: DBBenchRunner'
125    )
126    parser.add_argument(
127        '--benchrunner_pos_args', nargs='*',
128        help='whitespace-separated positional arguments that are passed on ' +
129        'to the constructor of the BenchmarkRunner class provided in the ' +
130        'benchrunner_class argument, example: "use_existing_db=true ' +
131        'duration=900"'
132    )
133    args = parser.parse_args()
134    main(args)
135