1# Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2# This source code is licensed under both the GPLv2 (found in the 3# COPYING file in the root directory) and Apache 2.0 License 4# (found in the LICENSE.Apache file in the root directory). 5 6import copy 7from advisor.db_log_parser import DataSource, NO_COL_FAMILY 8from advisor.ini_parser import IniParser 9import os 10 11 12class OptionsSpecParser(IniParser): 13 @staticmethod 14 def is_new_option(line): 15 return '=' in line 16 17 @staticmethod 18 def get_section_type(line): 19 ''' 20 Example section header: [TableOptions/BlockBasedTable "default"] 21 Here ConfigurationOptimizer returned would be 22 'TableOptions.BlockBasedTable' 23 ''' 24 section_path = line.strip()[1:-1].split()[0] 25 section_type = '.'.join(section_path.split('/')) 26 return section_type 27 28 @staticmethod 29 def get_section_name(line): 30 # example: get_section_name('[CFOptions "default"]') 31 token_list = line.strip()[1:-1].split('"') 32 # token_list = ['CFOptions', 'default', ''] 33 if len(token_list) < 3: 34 return None 35 return token_list[1] # return 'default' 36 37 @staticmethod 38 def get_section_str(section_type, section_name): 39 # Example: 40 # Case 1: get_section_str('DBOptions', NO_COL_FAMILY) 41 # Case 2: get_section_str('TableOptions.BlockBasedTable', 'default') 42 section_type = '/'.join(section_type.strip().split('.')) 43 # Case 1: section_type = 'DBOptions' 44 # Case 2: section_type = 'TableOptions/BlockBasedTable' 45 section_str = '[' + section_type 46 if section_name == NO_COL_FAMILY: 47 # Case 1: '[DBOptions]' 48 return (section_str + ']') 49 else: 50 # Case 2: '[TableOptions/BlockBasedTable "default"]' 51 return section_str + ' "' + section_name + '"]' 52 53 @staticmethod 54 def get_option_str(key, values): 55 option_str = key + '=' 56 # get_option_str('db_log_dir', None), returns 'db_log_dir=' 57 if values: 58 # example: 59 # get_option_str('max_bytes_for_level_multiplier_additional', 60 # [1,1,1,1,1,1,1]), returned string: 61 # 'max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1' 62 if isinstance(values, list): 63 for value in values: 64 option_str += (str(value) + ':') 65 option_str = option_str[:-1] 66 else: 67 # example: get_option_str('write_buffer_size', 1048576) 68 # returned string: 'write_buffer_size=1048576' 69 option_str += str(values) 70 return option_str 71 72 73class DatabaseOptions(DataSource): 74 75 @staticmethod 76 def is_misc_option(option_name): 77 # these are miscellaneous options that are not yet supported by the 78 # Rocksdb options file, hence they are not prefixed with any section 79 # name 80 return '.' not in option_name 81 82 @staticmethod 83 def get_options_diff(opt_old, opt_new): 84 # type: Dict[option, Dict[col_fam, value]] X 2 -> 85 # Dict[option, Dict[col_fam, Tuple(old_value, new_value)]] 86 # note: diff should contain a tuple of values only if they are 87 # different from each other 88 options_union = set(opt_old.keys()).union(set(opt_new.keys())) 89 diff = {} 90 for opt in options_union: 91 diff[opt] = {} 92 # if option in options_union, then it must be in one of the configs 93 if opt not in opt_old: 94 for col_fam in opt_new[opt]: 95 diff[opt][col_fam] = (None, opt_new[opt][col_fam]) 96 elif opt not in opt_new: 97 for col_fam in opt_old[opt]: 98 diff[opt][col_fam] = (opt_old[opt][col_fam], None) 99 else: 100 for col_fam in opt_old[opt]: 101 if col_fam in opt_new[opt]: 102 if opt_old[opt][col_fam] != opt_new[opt][col_fam]: 103 diff[opt][col_fam] = ( 104 opt_old[opt][col_fam], 105 opt_new[opt][col_fam] 106 ) 107 else: 108 diff[opt][col_fam] = (opt_old[opt][col_fam], None) 109 for col_fam in opt_new[opt]: 110 if col_fam in opt_old[opt]: 111 if opt_old[opt][col_fam] != opt_new[opt][col_fam]: 112 diff[opt][col_fam] = ( 113 opt_old[opt][col_fam], 114 opt_new[opt][col_fam] 115 ) 116 else: 117 diff[opt][col_fam] = (None, opt_new[opt][col_fam]) 118 if not diff[opt]: 119 diff.pop(opt) 120 return diff 121 122 def __init__(self, rocksdb_options, misc_options=None): 123 super().__init__(DataSource.Type.DB_OPTIONS) 124 # The options are stored in the following data structure: 125 # Dict[section_type, Dict[section_name, Dict[option_name, value]]] 126 self.options_dict = None 127 self.column_families = None 128 # Load the options from the given file to a dictionary. 129 self.load_from_source(rocksdb_options) 130 # Setup the miscellaneous options expected to be List[str], where each 131 # element in the List has the format "<option_name>=<option_value>" 132 # These options are the ones that are not yet supported by the Rocksdb 133 # OPTIONS file, so they are provided separately 134 self.setup_misc_options(misc_options) 135 136 def setup_misc_options(self, misc_options): 137 self.misc_options = {} 138 if misc_options: 139 for option_pair_str in misc_options: 140 option_name = option_pair_str.split('=')[0].strip() 141 option_value = option_pair_str.split('=')[1].strip() 142 self.misc_options[option_name] = option_value 143 144 def load_from_source(self, options_path): 145 self.options_dict = {} 146 with open(options_path, 'r') as db_options: 147 for line in db_options: 148 line = OptionsSpecParser.remove_trailing_comment(line) 149 if not line: 150 continue 151 if OptionsSpecParser.is_section_header(line): 152 curr_sec_type = ( 153 OptionsSpecParser.get_section_type(line) 154 ) 155 curr_sec_name = OptionsSpecParser.get_section_name(line) 156 if curr_sec_type not in self.options_dict: 157 self.options_dict[curr_sec_type] = {} 158 if not curr_sec_name: 159 curr_sec_name = NO_COL_FAMILY 160 self.options_dict[curr_sec_type][curr_sec_name] = {} 161 # example: if the line read from the Rocksdb OPTIONS file 162 # is [CFOptions "default"], then the section type is 163 # CFOptions and 'default' is the name of a column family 164 # that for this database, so it's added to the list of 165 # column families stored in this object 166 if curr_sec_type == 'CFOptions': 167 if not self.column_families: 168 self.column_families = [] 169 self.column_families.append(curr_sec_name) 170 elif OptionsSpecParser.is_new_option(line): 171 key, value = OptionsSpecParser.get_key_value_pair(line) 172 self.options_dict[curr_sec_type][curr_sec_name][key] = ( 173 value 174 ) 175 else: 176 error = 'Not able to parse line in Options file.' 177 OptionsSpecParser.exit_with_parse_error(line, error) 178 179 def get_misc_options(self): 180 # these are options that are not yet supported by the Rocksdb OPTIONS 181 # file, hence they are provided and stored separately 182 return self.misc_options 183 184 def get_column_families(self): 185 return self.column_families 186 187 def get_all_options(self): 188 # This method returns all the options that are stored in this object as 189 # a: Dict[<sec_type>.<option_name>: Dict[col_fam, option_value]] 190 all_options = [] 191 # Example: in the section header '[CFOptions "default"]' read from the 192 # OPTIONS file, sec_type='CFOptions' 193 for sec_type in self.options_dict: 194 for col_fam in self.options_dict[sec_type]: 195 for opt_name in self.options_dict[sec_type][col_fam]: 196 option = sec_type + '.' + opt_name 197 all_options.append(option) 198 all_options.extend(list(self.misc_options.keys())) 199 return self.get_options(all_options) 200 201 def get_options(self, reqd_options): 202 # type: List[str] -> Dict[str, Dict[str, Any]] 203 # List[option] -> Dict[option, Dict[col_fam, value]] 204 reqd_options_dict = {} 205 for option in reqd_options: 206 if DatabaseOptions.is_misc_option(option): 207 # the option is not prefixed by '<section_type>.' because it is 208 # not yet supported by the Rocksdb OPTIONS file; so it has to 209 # be fetched from the misc_options dictionary 210 if option not in self.misc_options: 211 continue 212 if option not in reqd_options_dict: 213 reqd_options_dict[option] = {} 214 reqd_options_dict[option][NO_COL_FAMILY] = ( 215 self.misc_options[option] 216 ) 217 else: 218 # Example: option = 'TableOptions.BlockBasedTable.block_align' 219 # then, sec_type = 'TableOptions.BlockBasedTable' 220 sec_type = '.'.join(option.split('.')[:-1]) 221 # opt_name = 'block_align' 222 opt_name = option.split('.')[-1] 223 if sec_type not in self.options_dict: 224 continue 225 for col_fam in self.options_dict[sec_type]: 226 if opt_name in self.options_dict[sec_type][col_fam]: 227 if option not in reqd_options_dict: 228 reqd_options_dict[option] = {} 229 reqd_options_dict[option][col_fam] = ( 230 self.options_dict[sec_type][col_fam][opt_name] 231 ) 232 return reqd_options_dict 233 234 def update_options(self, options): 235 # An example 'options' object looks like: 236 # {'DBOptions.max_background_jobs': {NO_COL_FAMILY: 2}, 237 # 'CFOptions.write_buffer_size': {'default': 1048576, 'cf_A': 128000}, 238 # 'bloom_bits': {NO_COL_FAMILY: 4}} 239 for option in options: 240 if DatabaseOptions.is_misc_option(option): 241 # this is a misc_option i.e. an option that is not yet 242 # supported by the Rocksdb OPTIONS file, so it is not prefixed 243 # by '<section_type>.' and must be stored in the separate 244 # misc_options dictionary 245 if NO_COL_FAMILY not in options[option]: 246 print( 247 'WARNING(DatabaseOptions.update_options): not ' + 248 'updating option ' + option + ' because it is in ' + 249 'misc_option format but its scope is not ' + 250 NO_COL_FAMILY + '. Check format of option.' 251 ) 252 continue 253 self.misc_options[option] = options[option][NO_COL_FAMILY] 254 else: 255 sec_name = '.'.join(option.split('.')[:-1]) 256 opt_name = option.split('.')[-1] 257 if sec_name not in self.options_dict: 258 self.options_dict[sec_name] = {} 259 for col_fam in options[option]: 260 # if the option is not already present in the dictionary, 261 # it will be inserted, else it will be updated to the new 262 # value 263 if col_fam not in self.options_dict[sec_name]: 264 self.options_dict[sec_name][col_fam] = {} 265 self.options_dict[sec_name][col_fam][opt_name] = ( 266 copy.deepcopy(options[option][col_fam]) 267 ) 268 269 def generate_options_config(self, nonce): 270 # this method generates a Rocksdb OPTIONS file in the INI format from 271 # the options stored in self.options_dict 272 this_path = os.path.abspath(os.path.dirname(__file__)) 273 file_name = '../temp/OPTIONS_' + str(nonce) + '.tmp' 274 file_path = os.path.join(this_path, file_name) 275 with open(file_path, 'w') as fp: 276 for section in self.options_dict: 277 for col_fam in self.options_dict[section]: 278 fp.write( 279 OptionsSpecParser.get_section_str(section, col_fam) + 280 '\n' 281 ) 282 for option in self.options_dict[section][col_fam]: 283 values = self.options_dict[section][col_fam][option] 284 fp.write( 285 OptionsSpecParser.get_option_str(option, values) + 286 '\n' 287 ) 288 fp.write('\n') 289 return file_path 290 291 def check_and_trigger_conditions(self, conditions): 292 for cond in conditions: 293 reqd_options_dict = self.get_options(cond.options) 294 # This contains the indices of options that are specific to some 295 # column family and are not database-wide options. 296 incomplete_option_ix = [] 297 options = [] 298 missing_reqd_option = False 299 for ix, option in enumerate(cond.options): 300 if option not in reqd_options_dict: 301 print( 302 'WARNING(DatabaseOptions.check_and_trigger): ' + 303 'skipping condition ' + cond.name + ' because it ' 304 'requires option ' + option + ' but this option is' + 305 ' not available' 306 ) 307 missing_reqd_option = True 308 break # required option is absent 309 if NO_COL_FAMILY in reqd_options_dict[option]: 310 options.append(reqd_options_dict[option][NO_COL_FAMILY]) 311 else: 312 options.append(None) 313 incomplete_option_ix.append(ix) 314 315 if missing_reqd_option: 316 continue 317 318 # if all the options are database-wide options 319 if not incomplete_option_ix: 320 try: 321 if eval(cond.eval_expr): 322 cond.set_trigger({NO_COL_FAMILY: options}) 323 except Exception as e: 324 print( 325 'WARNING(DatabaseOptions) check_and_trigger:' + str(e) 326 ) 327 continue 328 329 # for all the options that are not database-wide, we look for their 330 # values specific to column families 331 col_fam_options_dict = {} 332 for col_fam in self.column_families: 333 present = True 334 for ix in incomplete_option_ix: 335 option = cond.options[ix] 336 if col_fam not in reqd_options_dict[option]: 337 present = False 338 break 339 options[ix] = reqd_options_dict[option][col_fam] 340 if present: 341 try: 342 if eval(cond.eval_expr): 343 col_fam_options_dict[col_fam] = ( 344 copy.deepcopy(options) 345 ) 346 except Exception as e: 347 print( 348 'WARNING(DatabaseOptions) check_and_trigger: ' + 349 str(e) 350 ) 351 # Trigger for an OptionCondition object is of the form: 352 # Dict[col_fam_name: List[option_value]] 353 # where col_fam_name is the name of a column family for which 354 # 'eval_expr' evaluated to True and List[option_value] is the list 355 # of values of the options specified in the condition's 'options' 356 # field 357 if col_fam_options_dict: 358 cond.set_trigger(col_fam_options_dict) 359