1# Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
2#  This source code is licensed under both the GPLv2 (found in the
3#  COPYING file in the root directory) and Apache 2.0 License
4#  (found in the LICENSE.Apache file in the root directory).
5
6from advisor.db_log_parser import NO_COL_FAMILY
7from advisor.db_options_parser import DatabaseOptions
8from advisor.rule_parser import Suggestion
9import copy
10import random
11
12
13class ConfigOptimizer:
14    SCOPE = 'scope'
15    SUGG_VAL = 'suggested values'
16
17    @staticmethod
18    def apply_action_on_value(old_value, action, suggested_values):
19        chosen_sugg_val = None
20        if suggested_values:
21            chosen_sugg_val = random.choice(list(suggested_values))
22        new_value = None
23        if action is Suggestion.Action.set or not old_value:
24            assert(chosen_sugg_val)
25            new_value = chosen_sugg_val
26        else:
27            # For increase/decrease actions, currently the code tries to make
28            # a 30% change in the option's value per iteration. An addend is
29            # also present (+1 or -1) to handle the cases when the option's
30            # old value was 0 or the final int() conversion suppressed the 30%
31            # change made to the option
32            old_value = float(old_value)
33            mul = 0
34            add = 0
35            if action is Suggestion.Action.increase:
36                if old_value < 0:
37                    mul = 0.7
38                    add = 2
39                else:
40                    mul = 1.3
41                    add = 2
42            elif action is Suggestion.Action.decrease:
43                if old_value < 0:
44                    mul = 1.3
45                    add = -2
46                else:
47                    mul = 0.7
48                    add = -2
49            new_value = int(old_value * mul + add)
50        return new_value
51
52    @staticmethod
53    def improve_db_config(options, rule, suggestions_dict):
54        # this method takes ONE 'rule' and applies all its suggestions on the
55        # appropriate options
56        required_options = []
57        rule_suggestions = []
58        for sugg_name in rule.get_suggestions():
59            option = suggestions_dict[sugg_name].option
60            action = suggestions_dict[sugg_name].action
61            # A Suggestion in the rules spec must have the 'option' and
62            # 'action' fields defined, always call perform_checks() method
63            # after parsing the rules file using RulesSpec
64            assert(option)
65            assert(action)
66            required_options.append(option)
67            rule_suggestions.append(suggestions_dict[sugg_name])
68        current_config = options.get_options(required_options)
69        # Create the updated configuration from the rule's suggestions
70        updated_config = {}
71        for sugg in rule_suggestions:
72            # case: when the option is not present in the current configuration
73            if sugg.option not in current_config:
74                try:
75                    new_value = ConfigOptimizer.apply_action_on_value(
76                        None, sugg.action, sugg.suggested_values
77                    )
78                    if sugg.option not in updated_config:
79                        updated_config[sugg.option] = {}
80                    if DatabaseOptions.is_misc_option(sugg.option):
81                        # this suggestion is on an option that is not yet
82                        # supported by the Rocksdb OPTIONS file and so it is
83                        # not prefixed by a section type.
84                        updated_config[sugg.option][NO_COL_FAMILY] = new_value
85                    else:
86                        for col_fam in rule.get_trigger_column_families():
87                            updated_config[sugg.option][col_fam] = new_value
88                except AssertionError:
89                    print(
90                        'WARNING(ConfigOptimizer): provide suggested_values ' +
91                        'for ' + sugg.option
92                    )
93                continue
94            # case: when the option is present in the current configuration
95            if NO_COL_FAMILY in current_config[sugg.option]:
96                old_value = current_config[sugg.option][NO_COL_FAMILY]
97                try:
98                    new_value = ConfigOptimizer.apply_action_on_value(
99                        old_value, sugg.action, sugg.suggested_values
100                    )
101                    if sugg.option not in updated_config:
102                        updated_config[sugg.option] = {}
103                    updated_config[sugg.option][NO_COL_FAMILY] = new_value
104                except AssertionError:
105                    print(
106                        'WARNING(ConfigOptimizer): provide suggested_values ' +
107                        'for ' + sugg.option
108                    )
109            else:
110                for col_fam in rule.get_trigger_column_families():
111                    old_value = None
112                    if col_fam in current_config[sugg.option]:
113                        old_value = current_config[sugg.option][col_fam]
114                    try:
115                        new_value = ConfigOptimizer.apply_action_on_value(
116                            old_value, sugg.action, sugg.suggested_values
117                        )
118                        if sugg.option not in updated_config:
119                            updated_config[sugg.option] = {}
120                        updated_config[sugg.option][col_fam] = new_value
121                    except AssertionError:
122                        print(
123                            'WARNING(ConfigOptimizer): provide ' +
124                            'suggested_values for ' + sugg.option
125                        )
126        return current_config, updated_config
127
128    @staticmethod
129    def pick_rule_to_apply(rules, last_rule_name, rules_tried, backtrack):
130        if not rules:
131            print('\nNo more rules triggered!')
132            return None
133        # if the last rule provided an improvement in the database performance,
134        # and it was triggered again (i.e. it is present in 'rules'), then pick
135        # the same rule for this iteration too.
136        if last_rule_name and not backtrack:
137            for rule in rules:
138                if rule.name == last_rule_name:
139                    return rule
140        # there was no previous rule OR the previous rule did not improve db
141        # performance OR it was not triggered for this iteration,
142        # then pick another rule that has not been tried yet
143        for rule in rules:
144            if rule.name not in rules_tried:
145                return rule
146        print('\nAll rules have been exhausted')
147        return None
148
149    @staticmethod
150    def apply_suggestions(
151        triggered_rules,
152        current_rule_name,
153        rules_tried,
154        backtrack,
155        curr_options,
156        suggestions_dict
157    ):
158        curr_rule = ConfigOptimizer.pick_rule_to_apply(
159            triggered_rules, current_rule_name, rules_tried, backtrack
160        )
161        if not curr_rule:
162            return tuple([None]*4)
163        # if a rule has been picked for improving db_config, update rules_tried
164        rules_tried.add(curr_rule.name)
165        # get updated config based on the picked rule
166        curr_conf, updated_conf = ConfigOptimizer.improve_db_config(
167            curr_options, curr_rule, suggestions_dict
168        )
169        conf_diff = DatabaseOptions.get_options_diff(curr_conf, updated_conf)
170        if not conf_diff:  # the current and updated configs are the same
171            curr_rule, rules_tried, curr_conf, updated_conf = (
172                ConfigOptimizer.apply_suggestions(
173                    triggered_rules,
174                    None,
175                    rules_tried,
176                    backtrack,
177                    curr_options,
178                    suggestions_dict
179                )
180            )
181        print('returning from apply_suggestions')
182        return (curr_rule, rules_tried, curr_conf, updated_conf)
183
184    # TODO(poojam23): check if this method is required or can we directly set
185    # the config equal to the curr_config
186    @staticmethod
187    def get_backtrack_config(curr_config, updated_config):
188        diff = DatabaseOptions.get_options_diff(curr_config, updated_config)
189        bt_config = {}
190        for option in diff:
191            bt_config[option] = {}
192            for col_fam in diff[option]:
193                bt_config[option][col_fam] = diff[option][col_fam][0]
194        print(bt_config)
195        return bt_config
196
197    def __init__(self, bench_runner, db_options, rule_parser, base_db):
198        self.bench_runner = bench_runner
199        self.db_options = db_options
200        self.rule_parser = rule_parser
201        self.base_db_path = base_db
202
203    def run(self):
204        # In every iteration of this method's optimization loop we pick ONE
205        # RULE from all the triggered rules and apply all its suggestions to
206        # the appropriate options.
207        # bootstrapping the optimizer
208        print('Bootstrapping optimizer:')
209        options = copy.deepcopy(self.db_options)
210        old_data_sources, old_metric = (
211            self.bench_runner.run_experiment(options, self.base_db_path)
212        )
213        print('Initial metric: ' + str(old_metric))
214        self.rule_parser.load_rules_from_spec()
215        self.rule_parser.perform_section_checks()
216        triggered_rules = self.rule_parser.get_triggered_rules(
217            old_data_sources, options.get_column_families()
218        )
219        print('\nTriggered:')
220        self.rule_parser.print_rules(triggered_rules)
221        backtrack = False
222        rules_tried = set()
223        curr_rule, rules_tried, curr_conf, updated_conf = (
224            ConfigOptimizer.apply_suggestions(
225                triggered_rules,
226                None,
227                rules_tried,
228                backtrack,
229                options,
230                self.rule_parser.get_suggestions_dict()
231            )
232        )
233        # the optimizer loop
234        while curr_rule:
235            print('\nRule picked for next iteration:')
236            print(curr_rule.name)
237            print('\ncurrent config:')
238            print(curr_conf)
239            print('updated config:')
240            print(updated_conf)
241            options.update_options(updated_conf)
242            # run bench_runner with updated config
243            new_data_sources, new_metric = (
244                self.bench_runner.run_experiment(options, self.base_db_path)
245            )
246            print('\nnew metric: ' + str(new_metric))
247            backtrack = not self.bench_runner.is_metric_better(
248                new_metric, old_metric
249            )
250            # update triggered_rules, metric, data_sources, if required
251            if backtrack:
252                # revert changes to options config
253                print('\nBacktracking to previous configuration')
254                backtrack_conf = ConfigOptimizer.get_backtrack_config(
255                    curr_conf, updated_conf
256                )
257                options.update_options(backtrack_conf)
258            else:
259                # run advisor on new data sources
260                self.rule_parser.load_rules_from_spec()  # reboot the advisor
261                self.rule_parser.perform_section_checks()
262                triggered_rules = self.rule_parser.get_triggered_rules(
263                    new_data_sources, options.get_column_families()
264                )
265                print('\nTriggered:')
266                self.rule_parser.print_rules(triggered_rules)
267                old_metric = new_metric
268                old_data_sources = new_data_sources
269                rules_tried = set()
270            # pick rule to work on and set curr_rule to that
271            curr_rule, rules_tried, curr_conf, updated_conf = (
272                ConfigOptimizer.apply_suggestions(
273                    triggered_rules,
274                    curr_rule.name,
275                    rules_tried,
276                    backtrack,
277                    options,
278                    self.rule_parser.get_suggestions_dict()
279                )
280            )
281        # return the final database options configuration
282        return options
283