1# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
2from __future__ import absolute_import
3from __future__ import division
4from __future__ import print_function
5from __future__ import unicode_literals
6try:
7    from builtins import str
8except ImportError:
9    from __builtin__ import str
10from targets_builder import TARGETSBuilder
11import json
12import os
13import fnmatch
14import sys
15
16from util import ColorString
17
18# This script generates TARGETS file for Buck.
19# Buck is a build tool specifying dependencies among different build targets.
20# User can pass extra dependencies as a JSON object via command line, and this
21# script can include these dependencies in the generate TARGETS file.
22# Usage:
23# $python buckifier/buckify_rocksdb.py
24# (This generates a TARGET file without user-specified dependency for unit
25# tests.)
26# $python buckifier/buckify_rocksdb.py \
27#        '{"fake": { \
28#                      "extra_deps": [":test_dep", "//fakes/module:mock1"],  \
29#                      "extra_compiler_flags": ["-DROCKSDB_LITE", "-Os"], \
30#                  } \
31#         }'
32# (Generated TARGETS file has test_dep and mock1 as dependencies for RocksDB
33# unit tests, and will use the extra_compiler_flags to compile the unit test
34# source.)
35
36# tests to export as libraries for inclusion in other projects
37_EXPORTED_TEST_LIBS = ["env_basic_test"]
38
39# Parse src.mk files as a Dictionary of
40# VAR_NAME => list of files
41def parse_src_mk(repo_path):
42    src_mk = repo_path + "/src.mk"
43    src_files = {}
44    for line in open(src_mk):
45        line = line.strip()
46        if len(line) == 0 or line[0] == '#':
47            continue
48        if '=' in line:
49            current_src = line.split('=')[0].strip()
50            src_files[current_src] = []
51        elif '.cc' in line:
52            src_path = line.split('.cc')[0].strip() + '.cc'
53            src_files[current_src].append(src_path)
54    return src_files
55
56
57# get all .cc / .c files
58def get_cc_files(repo_path):
59    cc_files = []
60    for root, dirnames, filenames in os.walk(repo_path):  # noqa: B007 T25377293 Grandfathered in
61        root = root[(len(repo_path) + 1):]
62        if "java" in root:
63            # Skip java
64            continue
65        for filename in fnmatch.filter(filenames, '*.cc'):
66            cc_files.append(os.path.join(root, filename))
67        for filename in fnmatch.filter(filenames, '*.c'):
68            cc_files.append(os.path.join(root, filename))
69    return cc_files
70
71
72# Get tests from Makefile
73def get_tests(repo_path):
74    Makefile = repo_path + "/Makefile"
75
76    # Dictionary TEST_NAME => IS_PARALLEL
77    tests = {}
78
79    found_tests = False
80    for line in open(Makefile):
81        line = line.strip()
82        if line.startswith("TESTS ="):
83            found_tests = True
84        elif found_tests:
85            if line.endswith("\\"):
86                # remove the trailing \
87                line = line[:-1]
88                line = line.strip()
89                tests[line] = False
90            else:
91                # we consumed all the tests
92                break
93
94    found_parallel_tests = False
95    for line in open(Makefile):
96        line = line.strip()
97        if line.startswith("PARALLEL_TEST ="):
98            found_parallel_tests = True
99        elif found_parallel_tests:
100            if line.endswith("\\"):
101                # remove the trailing \
102                line = line[:-1]
103                line = line.strip()
104                tests[line] = True
105            else:
106                # we consumed all the parallel tests
107                break
108
109    return tests
110
111
112# Parse extra dependencies passed by user from command line
113def get_dependencies():
114    deps_map = {
115        '': {
116            'extra_deps': [],
117            'extra_compiler_flags': []
118        }
119    }
120    if len(sys.argv) < 2:
121        return deps_map
122
123    def encode_dict(data):
124        rv = {}
125        for k, v in data.items():
126            if isinstance(v, dict):
127                v = encode_dict(v)
128            rv[k] = v
129        return rv
130    extra_deps = json.loads(sys.argv[1], object_hook=encode_dict)
131    for target_alias, deps in extra_deps.items():
132        deps_map[target_alias] = deps
133    return deps_map
134
135
136# Prepare TARGETS file for buck
137def generate_targets(repo_path, deps_map):
138    print(ColorString.info("Generating TARGETS"))
139    # parsed src.mk file
140    src_mk = parse_src_mk(repo_path)
141    # get all .cc files
142    cc_files = get_cc_files(repo_path)
143    # get tests from Makefile
144    tests = get_tests(repo_path)
145
146    if src_mk is None or cc_files is None or tests is None:
147        return False
148
149    TARGETS = TARGETSBuilder("%s/TARGETS" % repo_path)
150    # rocksdb_lib
151    TARGETS.add_library(
152        "rocksdb_lib",
153        src_mk["LIB_SOURCES"] +
154        src_mk["TOOL_LIB_SOURCES"])
155    # rocksdb_test_lib
156    TARGETS.add_library(
157        "rocksdb_test_lib",
158        src_mk.get("MOCK_LIB_SOURCES", []) +
159        src_mk.get("TEST_LIB_SOURCES", []) +
160        src_mk.get("EXP_LIB_SOURCES", []) +
161        src_mk.get("ANALYZER_LIB_SOURCES", []),
162        [":rocksdb_lib"])
163    # rocksdb_tools_lib
164    TARGETS.add_library(
165        "rocksdb_tools_lib",
166        src_mk.get("BENCH_LIB_SOURCES", []) +
167        src_mk.get("ANALYZER_LIB_SOURCES", []) +
168        ["test_util/testutil.cc"],
169        [":rocksdb_lib"])
170    # rocksdb_stress_lib
171    TARGETS.add_library(
172        "rocksdb_stress_lib",
173        src_mk.get("ANALYZER_LIB_SOURCES", [])
174        + src_mk.get('STRESS_LIB_SOURCES', [])
175        + ["test_util/testutil.cc"],
176        [":rocksdb_lib"])
177
178    print("Extra dependencies:\n{0}".format(str(deps_map)))
179    # test for every test we found in the Makefile
180    for target_alias, deps in deps_map.items():
181        for test in sorted(tests):
182            match_src = [src for src in cc_files if ("/%s.c" % test) in src]
183            if len(match_src) == 0:
184                print(ColorString.warning("Cannot find .cc file for %s" % test))
185                continue
186            elif len(match_src) > 1:
187                print(ColorString.warning("Found more than one .cc for %s" % test))
188                print(match_src)
189                continue
190
191            assert(len(match_src) == 1)
192            is_parallel = tests[test]
193            test_target_name = \
194                test if not target_alias else test + "_" + target_alias
195            TARGETS.register_test(
196                test_target_name,
197                match_src[0],
198                is_parallel,
199                deps['extra_deps'],
200                deps['extra_compiler_flags'])
201
202            if test in _EXPORTED_TEST_LIBS:
203                test_library = "%s_lib" % test_target_name
204                TARGETS.add_library(test_library, match_src, [":rocksdb_test_lib"])
205    TARGETS.flush_tests()
206
207    print(ColorString.info("Generated TARGETS Summary:"))
208    print(ColorString.info("- %d libs" % TARGETS.total_lib))
209    print(ColorString.info("- %d binarys" % TARGETS.total_bin))
210    print(ColorString.info("- %d tests" % TARGETS.total_test))
211    return True
212
213
214def get_rocksdb_path():
215    # rocksdb = {script_dir}/..
216    script_dir = os.path.dirname(sys.argv[0])
217    script_dir = os.path.abspath(script_dir)
218    rocksdb_path = os.path.abspath(
219        os.path.join(script_dir, "../"))
220
221    return rocksdb_path
222
223def exit_with_error(msg):
224    print(ColorString.error(msg))
225    sys.exit(1)
226
227
228def main():
229    deps_map = get_dependencies()
230    # Generate TARGETS file for buck
231    ok = generate_targets(get_rocksdb_path(), deps_map)
232    if not ok:
233        exit_with_error("Failed to generate TARGETS files")
234
235if __name__ == "__main__":
236    main()
237