1# Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2# This source code is licensed under both the GPLv2 (found in the 3# COPYING file in the root directory) and Apache 2.0 License 4# (found in the LICENSE.Apache file in the root directory). 5 6from advisor.db_log_parser import NO_COL_FAMILY 7from advisor.db_options_parser import DatabaseOptions 8from advisor.rule_parser import Suggestion 9import copy 10import random 11 12 13class ConfigOptimizer: 14 SCOPE = 'scope' 15 SUGG_VAL = 'suggested values' 16 17 @staticmethod 18 def apply_action_on_value(old_value, action, suggested_values): 19 chosen_sugg_val = None 20 if suggested_values: 21 chosen_sugg_val = random.choice(list(suggested_values)) 22 new_value = None 23 if action is Suggestion.Action.set or not old_value: 24 assert(chosen_sugg_val) 25 new_value = chosen_sugg_val 26 else: 27 # For increase/decrease actions, currently the code tries to make 28 # a 30% change in the option's value per iteration. An addend is 29 # also present (+1 or -1) to handle the cases when the option's 30 # old value was 0 or the final int() conversion suppressed the 30% 31 # change made to the option 32 old_value = float(old_value) 33 mul = 0 34 add = 0 35 if action is Suggestion.Action.increase: 36 if old_value < 0: 37 mul = 0.7 38 add = 2 39 else: 40 mul = 1.3 41 add = 2 42 elif action is Suggestion.Action.decrease: 43 if old_value < 0: 44 mul = 1.3 45 add = -2 46 else: 47 mul = 0.7 48 add = -2 49 new_value = int(old_value * mul + add) 50 return new_value 51 52 @staticmethod 53 def improve_db_config(options, rule, suggestions_dict): 54 # this method takes ONE 'rule' and applies all its suggestions on the 55 # appropriate options 56 required_options = [] 57 rule_suggestions = [] 58 for sugg_name in rule.get_suggestions(): 59 option = suggestions_dict[sugg_name].option 60 action = suggestions_dict[sugg_name].action 61 # A Suggestion in the rules spec must have the 'option' and 62 # 'action' fields defined, always call perform_checks() method 63 # after parsing the rules file using RulesSpec 64 assert(option) 65 assert(action) 66 required_options.append(option) 67 rule_suggestions.append(suggestions_dict[sugg_name]) 68 current_config = options.get_options(required_options) 69 # Create the updated configuration from the rule's suggestions 70 updated_config = {} 71 for sugg in rule_suggestions: 72 # case: when the option is not present in the current configuration 73 if sugg.option not in current_config: 74 try: 75 new_value = ConfigOptimizer.apply_action_on_value( 76 None, sugg.action, sugg.suggested_values 77 ) 78 if sugg.option not in updated_config: 79 updated_config[sugg.option] = {} 80 if DatabaseOptions.is_misc_option(sugg.option): 81 # this suggestion is on an option that is not yet 82 # supported by the Rocksdb OPTIONS file and so it is 83 # not prefixed by a section type. 84 updated_config[sugg.option][NO_COL_FAMILY] = new_value 85 else: 86 for col_fam in rule.get_trigger_column_families(): 87 updated_config[sugg.option][col_fam] = new_value 88 except AssertionError: 89 print( 90 'WARNING(ConfigOptimizer): provide suggested_values ' + 91 'for ' + sugg.option 92 ) 93 continue 94 # case: when the option is present in the current configuration 95 if NO_COL_FAMILY in current_config[sugg.option]: 96 old_value = current_config[sugg.option][NO_COL_FAMILY] 97 try: 98 new_value = ConfigOptimizer.apply_action_on_value( 99 old_value, sugg.action, sugg.suggested_values 100 ) 101 if sugg.option not in updated_config: 102 updated_config[sugg.option] = {} 103 updated_config[sugg.option][NO_COL_FAMILY] = new_value 104 except AssertionError: 105 print( 106 'WARNING(ConfigOptimizer): provide suggested_values ' + 107 'for ' + sugg.option 108 ) 109 else: 110 for col_fam in rule.get_trigger_column_families(): 111 old_value = None 112 if col_fam in current_config[sugg.option]: 113 old_value = current_config[sugg.option][col_fam] 114 try: 115 new_value = ConfigOptimizer.apply_action_on_value( 116 old_value, sugg.action, sugg.suggested_values 117 ) 118 if sugg.option not in updated_config: 119 updated_config[sugg.option] = {} 120 updated_config[sugg.option][col_fam] = new_value 121 except AssertionError: 122 print( 123 'WARNING(ConfigOptimizer): provide ' + 124 'suggested_values for ' + sugg.option 125 ) 126 return current_config, updated_config 127 128 @staticmethod 129 def pick_rule_to_apply(rules, last_rule_name, rules_tried, backtrack): 130 if not rules: 131 print('\nNo more rules triggered!') 132 return None 133 # if the last rule provided an improvement in the database performance, 134 # and it was triggered again (i.e. it is present in 'rules'), then pick 135 # the same rule for this iteration too. 136 if last_rule_name and not backtrack: 137 for rule in rules: 138 if rule.name == last_rule_name: 139 return rule 140 # there was no previous rule OR the previous rule did not improve db 141 # performance OR it was not triggered for this iteration, 142 # then pick another rule that has not been tried yet 143 for rule in rules: 144 if rule.name not in rules_tried: 145 return rule 146 print('\nAll rules have been exhausted') 147 return None 148 149 @staticmethod 150 def apply_suggestions( 151 triggered_rules, 152 current_rule_name, 153 rules_tried, 154 backtrack, 155 curr_options, 156 suggestions_dict 157 ): 158 curr_rule = ConfigOptimizer.pick_rule_to_apply( 159 triggered_rules, current_rule_name, rules_tried, backtrack 160 ) 161 if not curr_rule: 162 return tuple([None]*4) 163 # if a rule has been picked for improving db_config, update rules_tried 164 rules_tried.add(curr_rule.name) 165 # get updated config based on the picked rule 166 curr_conf, updated_conf = ConfigOptimizer.improve_db_config( 167 curr_options, curr_rule, suggestions_dict 168 ) 169 conf_diff = DatabaseOptions.get_options_diff(curr_conf, updated_conf) 170 if not conf_diff: # the current and updated configs are the same 171 curr_rule, rules_tried, curr_conf, updated_conf = ( 172 ConfigOptimizer.apply_suggestions( 173 triggered_rules, 174 None, 175 rules_tried, 176 backtrack, 177 curr_options, 178 suggestions_dict 179 ) 180 ) 181 print('returning from apply_suggestions') 182 return (curr_rule, rules_tried, curr_conf, updated_conf) 183 184 # TODO(poojam23): check if this method is required or can we directly set 185 # the config equal to the curr_config 186 @staticmethod 187 def get_backtrack_config(curr_config, updated_config): 188 diff = DatabaseOptions.get_options_diff(curr_config, updated_config) 189 bt_config = {} 190 for option in diff: 191 bt_config[option] = {} 192 for col_fam in diff[option]: 193 bt_config[option][col_fam] = diff[option][col_fam][0] 194 print(bt_config) 195 return bt_config 196 197 def __init__(self, bench_runner, db_options, rule_parser, base_db): 198 self.bench_runner = bench_runner 199 self.db_options = db_options 200 self.rule_parser = rule_parser 201 self.base_db_path = base_db 202 203 def run(self): 204 # In every iteration of this method's optimization loop we pick ONE 205 # RULE from all the triggered rules and apply all its suggestions to 206 # the appropriate options. 207 # bootstrapping the optimizer 208 print('Bootstrapping optimizer:') 209 options = copy.deepcopy(self.db_options) 210 old_data_sources, old_metric = ( 211 self.bench_runner.run_experiment(options, self.base_db_path) 212 ) 213 print('Initial metric: ' + str(old_metric)) 214 self.rule_parser.load_rules_from_spec() 215 self.rule_parser.perform_section_checks() 216 triggered_rules = self.rule_parser.get_triggered_rules( 217 old_data_sources, options.get_column_families() 218 ) 219 print('\nTriggered:') 220 self.rule_parser.print_rules(triggered_rules) 221 backtrack = False 222 rules_tried = set() 223 curr_rule, rules_tried, curr_conf, updated_conf = ( 224 ConfigOptimizer.apply_suggestions( 225 triggered_rules, 226 None, 227 rules_tried, 228 backtrack, 229 options, 230 self.rule_parser.get_suggestions_dict() 231 ) 232 ) 233 # the optimizer loop 234 while curr_rule: 235 print('\nRule picked for next iteration:') 236 print(curr_rule.name) 237 print('\ncurrent config:') 238 print(curr_conf) 239 print('updated config:') 240 print(updated_conf) 241 options.update_options(updated_conf) 242 # run bench_runner with updated config 243 new_data_sources, new_metric = ( 244 self.bench_runner.run_experiment(options, self.base_db_path) 245 ) 246 print('\nnew metric: ' + str(new_metric)) 247 backtrack = not self.bench_runner.is_metric_better( 248 new_metric, old_metric 249 ) 250 # update triggered_rules, metric, data_sources, if required 251 if backtrack: 252 # revert changes to options config 253 print('\nBacktracking to previous configuration') 254 backtrack_conf = ConfigOptimizer.get_backtrack_config( 255 curr_conf, updated_conf 256 ) 257 options.update_options(backtrack_conf) 258 else: 259 # run advisor on new data sources 260 self.rule_parser.load_rules_from_spec() # reboot the advisor 261 self.rule_parser.perform_section_checks() 262 triggered_rules = self.rule_parser.get_triggered_rules( 263 new_data_sources, options.get_column_families() 264 ) 265 print('\nTriggered:') 266 self.rule_parser.print_rules(triggered_rules) 267 old_metric = new_metric 268 old_data_sources = new_data_sources 269 rules_tried = set() 270 # pick rule to work on and set curr_rule to that 271 curr_rule, rules_tried, curr_conf, updated_conf = ( 272 ConfigOptimizer.apply_suggestions( 273 triggered_rules, 274 curr_rule.name, 275 rules_tried, 276 backtrack, 277 options, 278 self.rule_parser.get_suggestions_dict() 279 ) 280 ) 281 # return the final database options configuration 282 return options 283