1# Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2# This source code is licensed under both the GPLv2 (found in the 3# COPYING file in the root directory) and Apache 2.0 License 4# (found in the LICENSE.Apache file in the root directory). 5 6import os 7import unittest 8from advisor.rule_parser import RulesSpec 9from advisor.db_log_parser import DatabaseLogs, DataSource 10from advisor.db_options_parser import DatabaseOptions 11 12RuleToSuggestions = { 13 "stall-too-many-memtables": [ 14 'inc-bg-flush', 15 'inc-write-buffer' 16 ], 17 "stall-too-many-L0": [ 18 'inc-max-subcompactions', 19 'inc-max-bg-compactions', 20 'inc-write-buffer-size', 21 'dec-max-bytes-for-level-base', 22 'inc-l0-slowdown-writes-trigger' 23 ], 24 "stop-too-many-L0": [ 25 'inc-max-bg-compactions', 26 'inc-write-buffer-size', 27 'inc-l0-stop-writes-trigger' 28 ], 29 "stall-too-many-compaction-bytes": [ 30 'inc-max-bg-compactions', 31 'inc-write-buffer-size', 32 'inc-hard-pending-compaction-bytes-limit', 33 'inc-soft-pending-compaction-bytes-limit' 34 ], 35 "level0-level1-ratio": [ 36 'l0-l1-ratio-health-check' 37 ] 38} 39 40 41class TestAllRulesTriggered(unittest.TestCase): 42 def setUp(self): 43 # load the Rules 44 this_path = os.path.abspath(os.path.dirname(__file__)) 45 ini_path = os.path.join(this_path, 'input_files/triggered_rules.ini') 46 self.db_rules = RulesSpec(ini_path) 47 self.db_rules.load_rules_from_spec() 48 self.db_rules.perform_section_checks() 49 # load the data sources: LOG and OPTIONS 50 log_path = os.path.join(this_path, 'input_files/LOG-0') 51 options_path = os.path.join(this_path, 'input_files/OPTIONS-000005') 52 db_options_parser = DatabaseOptions(options_path) 53 self.column_families = db_options_parser.get_column_families() 54 db_logs_parser = DatabaseLogs(log_path, self.column_families) 55 self.data_sources = { 56 DataSource.Type.DB_OPTIONS: [db_options_parser], 57 DataSource.Type.LOG: [db_logs_parser] 58 } 59 60 def test_triggered_conditions(self): 61 conditions_dict = self.db_rules.get_conditions_dict() 62 rules_dict = self.db_rules.get_rules_dict() 63 # Make sure none of the conditions is triggered beforehand 64 for cond in conditions_dict.values(): 65 self.assertFalse(cond.is_triggered(), repr(cond)) 66 for rule in rules_dict.values(): 67 self.assertFalse( 68 rule.is_triggered(conditions_dict, self.column_families), 69 repr(rule) 70 ) 71 72 # # Trigger the conditions as per the data sources. 73 # trigger_conditions(, conditions_dict) 74 75 # Get the set of rules that have been triggered 76 triggered_rules = self.db_rules.get_triggered_rules( 77 self.data_sources, self.column_families 78 ) 79 80 # Make sure each condition and rule is triggered 81 for cond in conditions_dict.values(): 82 if cond.get_data_source() is DataSource.Type.TIME_SERIES: 83 continue 84 self.assertTrue(cond.is_triggered(), repr(cond)) 85 86 for rule in rules_dict.values(): 87 self.assertIn(rule, triggered_rules) 88 # Check the suggestions made by the triggered rules 89 for sugg in rule.get_suggestions(): 90 self.assertIn(sugg, RuleToSuggestions[rule.name]) 91 92 for rule in triggered_rules: 93 self.assertIn(rule, rules_dict.values()) 94 for sugg in RuleToSuggestions[rule.name]: 95 self.assertIn(sugg, rule.get_suggestions()) 96 97 98class TestConditionsConjunctions(unittest.TestCase): 99 def setUp(self): 100 # load the Rules 101 this_path = os.path.abspath(os.path.dirname(__file__)) 102 ini_path = os.path.join(this_path, 'input_files/test_rules.ini') 103 self.db_rules = RulesSpec(ini_path) 104 self.db_rules.load_rules_from_spec() 105 self.db_rules.perform_section_checks() 106 # load the data sources: LOG and OPTIONS 107 log_path = os.path.join(this_path, 'input_files/LOG-1') 108 options_path = os.path.join(this_path, 'input_files/OPTIONS-000005') 109 db_options_parser = DatabaseOptions(options_path) 110 self.column_families = db_options_parser.get_column_families() 111 db_logs_parser = DatabaseLogs(log_path, self.column_families) 112 self.data_sources = { 113 DataSource.Type.DB_OPTIONS: [db_options_parser], 114 DataSource.Type.LOG: [db_logs_parser] 115 } 116 117 def test_condition_conjunctions(self): 118 conditions_dict = self.db_rules.get_conditions_dict() 119 rules_dict = self.db_rules.get_rules_dict() 120 # Make sure none of the conditions is triggered beforehand 121 for cond in conditions_dict.values(): 122 self.assertFalse(cond.is_triggered(), repr(cond)) 123 for rule in rules_dict.values(): 124 self.assertFalse( 125 rule.is_triggered(conditions_dict, self.column_families), 126 repr(rule) 127 ) 128 129 # Trigger the conditions as per the data sources. 130 self.db_rules.trigger_conditions(self.data_sources) 131 132 # Check for the conditions 133 conds_triggered = ['log-1-true', 'log-2-true', 'log-3-true'] 134 conds_not_triggered = ['log-4-false', 'options-1-false'] 135 for cond in conds_triggered: 136 self.assertTrue(conditions_dict[cond].is_triggered(), repr(cond)) 137 for cond in conds_not_triggered: 138 self.assertFalse(conditions_dict[cond].is_triggered(), repr(cond)) 139 140 # Check for the rules 141 rules_triggered = ['multiple-conds-true'] 142 rules_not_triggered = [ 143 'single-condition-false', 144 'multiple-conds-one-false', 145 'multiple-conds-all-false' 146 ] 147 for rule_name in rules_triggered: 148 rule = rules_dict[rule_name] 149 self.assertTrue( 150 rule.is_triggered(conditions_dict, self.column_families), 151 repr(rule) 152 ) 153 for rule_name in rules_not_triggered: 154 rule = rules_dict[rule_name] 155 self.assertFalse( 156 rule.is_triggered(conditions_dict, self.column_families), 157 repr(rule) 158 ) 159 160 161class TestSanityChecker(unittest.TestCase): 162 def setUp(self): 163 this_path = os.path.abspath(os.path.dirname(__file__)) 164 ini_path = os.path.join(this_path, 'input_files/rules_err1.ini') 165 db_rules = RulesSpec(ini_path) 166 db_rules.load_rules_from_spec() 167 self.rules_dict = db_rules.get_rules_dict() 168 self.conditions_dict = db_rules.get_conditions_dict() 169 self.suggestions_dict = db_rules.get_suggestions_dict() 170 171 def test_rule_missing_suggestions(self): 172 regex = '.*rule must have at least one suggestion.*' 173 with self.assertRaisesRegex(ValueError, regex): 174 self.rules_dict['missing-suggestions'].perform_checks() 175 176 def test_rule_missing_conditions(self): 177 regex = '.*rule must have at least one condition.*' 178 with self.assertRaisesRegex(ValueError, regex): 179 self.rules_dict['missing-conditions'].perform_checks() 180 181 def test_condition_missing_regex(self): 182 regex = '.*provide regex for log condition.*' 183 with self.assertRaisesRegex(ValueError, regex): 184 self.conditions_dict['missing-regex'].perform_checks() 185 186 def test_condition_missing_options(self): 187 regex = '.*options missing in condition.*' 188 with self.assertRaisesRegex(ValueError, regex): 189 self.conditions_dict['missing-options'].perform_checks() 190 191 def test_condition_missing_expression(self): 192 regex = '.*expression missing in condition.*' 193 with self.assertRaisesRegex(ValueError, regex): 194 self.conditions_dict['missing-expression'].perform_checks() 195 196 def test_suggestion_missing_option(self): 197 regex = '.*provide option or description.*' 198 with self.assertRaisesRegex(ValueError, regex): 199 self.suggestions_dict['missing-option'].perform_checks() 200 201 def test_suggestion_missing_description(self): 202 regex = '.*provide option or description.*' 203 with self.assertRaisesRegex(ValueError, regex): 204 self.suggestions_dict['missing-description'].perform_checks() 205 206 207class TestParsingErrors(unittest.TestCase): 208 def setUp(self): 209 self.this_path = os.path.abspath(os.path.dirname(__file__)) 210 211 def test_condition_missing_source(self): 212 ini_path = os.path.join(self.this_path, 'input_files/rules_err2.ini') 213 db_rules = RulesSpec(ini_path) 214 regex = '.*provide source for condition.*' 215 with self.assertRaisesRegex(NotImplementedError, regex): 216 db_rules.load_rules_from_spec() 217 218 def test_suggestion_missing_action(self): 219 ini_path = os.path.join(self.this_path, 'input_files/rules_err3.ini') 220 db_rules = RulesSpec(ini_path) 221 regex = '.*provide action for option.*' 222 with self.assertRaisesRegex(ValueError, regex): 223 db_rules.load_rules_from_spec() 224 225 def test_section_no_name(self): 226 ini_path = os.path.join(self.this_path, 'input_files/rules_err4.ini') 227 db_rules = RulesSpec(ini_path) 228 regex = 'Parsing error: needed section header:.*' 229 with self.assertRaisesRegex(ValueError, regex): 230 db_rules.load_rules_from_spec() 231 232 233if __name__ == '__main__': 234 unittest.main() 235