1# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. 2from __future__ import absolute_import 3from __future__ import division 4from __future__ import print_function 5from __future__ import unicode_literals 6try: 7 from builtins import str 8except ImportError: 9 from __builtin__ import str 10from targets_builder import TARGETSBuilder 11import json 12import os 13import fnmatch 14import sys 15 16from util import ColorString 17 18# This script generates TARGETS file for Buck. 19# Buck is a build tool specifying dependencies among different build targets. 20# User can pass extra dependencies as a JSON object via command line, and this 21# script can include these dependencies in the generate TARGETS file. 22# Usage: 23# $python buckifier/buckify_rocksdb.py 24# (This generates a TARGET file without user-specified dependency for unit 25# tests.) 26# $python buckifier/buckify_rocksdb.py \ 27# '{"fake": { \ 28# "extra_deps": [":test_dep", "//fakes/module:mock1"], \ 29# "extra_compiler_flags": ["-DROCKSDB_LITE", "-Os"], \ 30# } \ 31# }' 32# (Generated TARGETS file has test_dep and mock1 as dependencies for RocksDB 33# unit tests, and will use the extra_compiler_flags to compile the unit test 34# source.) 35 36# tests to export as libraries for inclusion in other projects 37_EXPORTED_TEST_LIBS = ["env_basic_test"] 38 39# Parse src.mk files as a Dictionary of 40# VAR_NAME => list of files 41def parse_src_mk(repo_path): 42 src_mk = repo_path + "/src.mk" 43 src_files = {} 44 for line in open(src_mk): 45 line = line.strip() 46 if len(line) == 0 or line[0] == '#': 47 continue 48 if '=' in line: 49 current_src = line.split('=')[0].strip() 50 src_files[current_src] = [] 51 elif '.cc' in line: 52 src_path = line.split('.cc')[0].strip() + '.cc' 53 src_files[current_src].append(src_path) 54 return src_files 55 56 57# get all .cc / .c files 58def get_cc_files(repo_path): 59 cc_files = [] 60 for root, dirnames, filenames in os.walk(repo_path): # noqa: B007 T25377293 Grandfathered in 61 root = root[(len(repo_path) + 1):] 62 if "java" in root: 63 # Skip java 64 continue 65 for filename in fnmatch.filter(filenames, '*.cc'): 66 cc_files.append(os.path.join(root, filename)) 67 for filename in fnmatch.filter(filenames, '*.c'): 68 cc_files.append(os.path.join(root, filename)) 69 return cc_files 70 71 72# Get tests from Makefile 73def get_tests(repo_path): 74 Makefile = repo_path + "/Makefile" 75 76 # Dictionary TEST_NAME => IS_PARALLEL 77 tests = {} 78 79 found_tests = False 80 for line in open(Makefile): 81 line = line.strip() 82 if line.startswith("TESTS ="): 83 found_tests = True 84 elif found_tests: 85 if line.endswith("\\"): 86 # remove the trailing \ 87 line = line[:-1] 88 line = line.strip() 89 tests[line] = False 90 else: 91 # we consumed all the tests 92 break 93 94 found_parallel_tests = False 95 for line in open(Makefile): 96 line = line.strip() 97 if line.startswith("PARALLEL_TEST ="): 98 found_parallel_tests = True 99 elif found_parallel_tests: 100 if line.endswith("\\"): 101 # remove the trailing \ 102 line = line[:-1] 103 line = line.strip() 104 tests[line] = True 105 else: 106 # we consumed all the parallel tests 107 break 108 109 return tests 110 111 112# Parse extra dependencies passed by user from command line 113def get_dependencies(): 114 deps_map = { 115 '': { 116 'extra_deps': [], 117 'extra_compiler_flags': [] 118 } 119 } 120 if len(sys.argv) < 2: 121 return deps_map 122 123 def encode_dict(data): 124 rv = {} 125 for k, v in data.items(): 126 if isinstance(v, dict): 127 v = encode_dict(v) 128 rv[k] = v 129 return rv 130 extra_deps = json.loads(sys.argv[1], object_hook=encode_dict) 131 for target_alias, deps in extra_deps.items(): 132 deps_map[target_alias] = deps 133 return deps_map 134 135 136# Prepare TARGETS file for buck 137def generate_targets(repo_path, deps_map): 138 print(ColorString.info("Generating TARGETS")) 139 # parsed src.mk file 140 src_mk = parse_src_mk(repo_path) 141 # get all .cc files 142 cc_files = get_cc_files(repo_path) 143 # get tests from Makefile 144 tests = get_tests(repo_path) 145 146 if src_mk is None or cc_files is None or tests is None: 147 return False 148 149 TARGETS = TARGETSBuilder("%s/TARGETS" % repo_path) 150 # rocksdb_lib 151 TARGETS.add_library( 152 "rocksdb_lib", 153 src_mk["LIB_SOURCES"] + 154 src_mk["TOOL_LIB_SOURCES"]) 155 # rocksdb_test_lib 156 TARGETS.add_library( 157 "rocksdb_test_lib", 158 src_mk.get("MOCK_LIB_SOURCES", []) + 159 src_mk.get("TEST_LIB_SOURCES", []) + 160 src_mk.get("EXP_LIB_SOURCES", []) + 161 src_mk.get("ANALYZER_LIB_SOURCES", []), 162 [":rocksdb_lib"]) 163 # rocksdb_tools_lib 164 TARGETS.add_library( 165 "rocksdb_tools_lib", 166 src_mk.get("BENCH_LIB_SOURCES", []) + 167 src_mk.get("ANALYZER_LIB_SOURCES", []) + 168 ["test_util/testutil.cc"], 169 [":rocksdb_lib"]) 170 # rocksdb_stress_lib 171 TARGETS.add_library( 172 "rocksdb_stress_lib", 173 src_mk.get("ANALYZER_LIB_SOURCES", []) 174 + src_mk.get('STRESS_LIB_SOURCES', []) 175 + ["test_util/testutil.cc"], 176 [":rocksdb_lib"]) 177 178 print("Extra dependencies:\n{0}".format(str(deps_map))) 179 # test for every test we found in the Makefile 180 for target_alias, deps in deps_map.items(): 181 for test in sorted(tests): 182 match_src = [src for src in cc_files if ("/%s.c" % test) in src] 183 if len(match_src) == 0: 184 print(ColorString.warning("Cannot find .cc file for %s" % test)) 185 continue 186 elif len(match_src) > 1: 187 print(ColorString.warning("Found more than one .cc for %s" % test)) 188 print(match_src) 189 continue 190 191 assert(len(match_src) == 1) 192 is_parallel = tests[test] 193 test_target_name = \ 194 test if not target_alias else test + "_" + target_alias 195 TARGETS.register_test( 196 test_target_name, 197 match_src[0], 198 is_parallel, 199 deps['extra_deps'], 200 deps['extra_compiler_flags']) 201 202 if test in _EXPORTED_TEST_LIBS: 203 test_library = "%s_lib" % test_target_name 204 TARGETS.add_library(test_library, match_src, [":rocksdb_test_lib"]) 205 TARGETS.flush_tests() 206 207 print(ColorString.info("Generated TARGETS Summary:")) 208 print(ColorString.info("- %d libs" % TARGETS.total_lib)) 209 print(ColorString.info("- %d binarys" % TARGETS.total_bin)) 210 print(ColorString.info("- %d tests" % TARGETS.total_test)) 211 return True 212 213 214def get_rocksdb_path(): 215 # rocksdb = {script_dir}/.. 216 script_dir = os.path.dirname(sys.argv[0]) 217 script_dir = os.path.abspath(script_dir) 218 rocksdb_path = os.path.abspath( 219 os.path.join(script_dir, "../")) 220 221 return rocksdb_path 222 223def exit_with_error(msg): 224 print(ColorString.error(msg)) 225 sys.exit(1) 226 227 228def main(): 229 deps_map = get_dependencies() 230 # Generate TARGETS file for buck 231 ok = generate_targets(get_rocksdb_path(), deps_map) 232 if not ok: 233 exit_with_error("Failed to generate TARGETS files") 234 235if __name__ == "__main__": 236 main() 237