1#!/usr/bin/env python3
2
3# Copyright 2019 gRPC authors.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17import collections
18import os
19import re
20import subprocess
21import xml.etree.ElementTree as ET
22import yaml
23
24ABSEIL_PATH = "third_party/abseil-cpp"
25OUTPUT_PATH = "src/abseil-cpp/preprocessed_builds.yaml"
26CAPITAL_WORD = re.compile("[A-Z]+")
27ABSEIL_CMAKE_RULE_BEGIN = re.compile("^absl_cc_.*\(", re.MULTILINE)
28ABSEIL_CMAKE_RULE_END = re.compile("^\)", re.MULTILINE)
29
30# Rule object representing the rule of Bazel BUILD.
31Rule = collections.namedtuple(
32    "Rule", "type name package srcs hdrs textual_hdrs deps visibility testonly")
33
34
35def get_elem_value(elem, name):
36  """Returns the value of XML element with the given name."""
37  for child in elem:
38    if child.attrib.get("name") == name:
39      if child.tag == "string":
40        return child.attrib.get("value")
41      elif child.tag == "boolean":
42        return child.attrib.get("value") == "true"
43      elif child.tag == "list":
44        return [nested_child.attrib.get("value") for nested_child in child]
45      else:
46        raise "Cannot recognize tag: " + child.tag
47  return None
48
49
50def normalize_paths(paths):
51  """Returns the list of normalized path."""
52  # e.g. ["//absl/strings:dir/header.h"] -> ["absl/strings/dir/header.h"]
53  return [path.lstrip("/").replace(":", "/") for path in paths]
54
55
56def parse_bazel_rule(elem, package):
57  """Returns a rule from bazel XML rule."""
58  return Rule(
59      type=elem.attrib["class"],
60      name=get_elem_value(elem, "name"),
61      package=package,
62      srcs=normalize_paths(get_elem_value(elem, "srcs") or []),
63      hdrs=normalize_paths(get_elem_value(elem, "hdrs") or []),
64      textual_hdrs=normalize_paths(get_elem_value(elem, "textual_hdrs") or []),
65      deps=get_elem_value(elem, "deps") or [],
66      visibility=get_elem_value(elem, "visibility") or [],
67      testonly=get_elem_value(elem, "testonly") or False)
68
69
70def read_bazel_build(package):
71  """Runs bazel query on given package file and returns all cc rules."""
72  # Use a wrapper version of bazel in gRPC not to use system-wide bazel
73  # to avoid bazel conflict when running on Kokoro.
74  BAZEL_BIN = "../../tools/bazel"
75  result = subprocess.check_output(
76      [BAZEL_BIN, "query", package + ":all", "--output", "xml"])
77  root = ET.fromstring(result)
78  return [
79      parse_bazel_rule(elem, package)
80      for elem in root
81      if elem.tag == "rule" and elem.attrib["class"].startswith("cc_")
82  ]
83
84
85def collect_bazel_rules(root_path):
86  """Collects and returns all bazel rules from root path recursively."""
87  rules = []
88  for cur, _, _ in os.walk(root_path):
89    build_path = os.path.join(cur, "BUILD.bazel")
90    if os.path.exists(build_path):
91      rules.extend(read_bazel_build("//" + cur))
92  return rules
93
94
95def parse_cmake_rule(rule, package):
96  """Returns a rule from absl cmake rule.
97     Reference: https://github.com/abseil/abseil-cpp/blob/master/CMake/AbseilHelpers.cmake
98  """
99  kv = {}
100  bucket = None
101  lines = rule.splitlines()
102  for line in lines[1:-1]:
103    if CAPITAL_WORD.match(line.strip()):
104      bucket = kv.setdefault(line.strip(), [])
105    else:
106      if bucket is not None:
107        bucket.append(line.strip())
108      else:
109        raise ValueError("Illegal syntax: {}".format(rule))
110  return Rule(
111      type=lines[0].rstrip("("),
112      name="absl::" + kv["NAME"][0],
113      package=package,
114      srcs=[package + "/" + f.strip('"') for f in kv.get("SRCS", [])],
115      hdrs=[package + "/" + f.strip('"') for f in kv.get("HDRS", [])],
116      textual_hdrs=[],
117      deps=kv.get("DEPS", []),
118      visibility="PUBLIC" in kv,
119      testonly="TESTONLY" in kv,
120  )
121
122
123def read_cmake_build(build_path, package):
124  """Parses given CMakeLists.txt file and returns all cc rules."""
125  rules = []
126  with open(build_path, "r") as f:
127    src = f.read()
128    for begin_mo in ABSEIL_CMAKE_RULE_BEGIN.finditer(src):
129      end_mo = ABSEIL_CMAKE_RULE_END.search(src[begin_mo.start(0):])
130      expr = src[begin_mo.start(0):begin_mo.start(0) + end_mo.start(0) + 1]
131      rules.append(parse_cmake_rule(expr, package))
132  return rules
133
134
135def collect_cmake_rules(root_path):
136  """Collects and returns all cmake rules from root path recursively."""
137  rules = []
138  for cur, _, _ in os.walk(root_path):
139    build_path = os.path.join(cur, "CMakeLists.txt")
140    if os.path.exists(build_path):
141      rules.extend(read_cmake_build(build_path, cur))
142  return rules
143
144
145def pairing_bazel_and_cmake_rules(bazel_rules, cmake_rules):
146  """Returns a pair map between bazel rules and cmake rules based on
147     the similarity of the file list in the rule. This is because
148     cmake build and bazel build of abseil are not identical.
149  """
150  pair_map = {}
151  for rule in bazel_rules:
152    best_crule, best_similarity = None, 0
153    for crule in cmake_rules:
154      similarity = len(
155          set(rule.srcs + rule.hdrs + rule.textual_hdrs).intersection(
156              set(crule.srcs + crule.hdrs + crule.textual_hdrs)))
157      if similarity > best_similarity:
158        best_crule, best_similarity = crule, similarity
159    if best_crule:
160      pair_map[(rule.package, rule.name)] = best_crule.name
161  return pair_map
162
163
164def resolve_hdrs(files):
165  return [ABSEIL_PATH + "/" + f for f in files if f.endswith((".h", ".inc"))]
166
167
168def resolve_srcs(files):
169  return [ABSEIL_PATH + "/" + f for f in files if f.endswith(".cc")]
170
171
172def resolve_deps(targets):
173  return [(t[2:] if t.startswith("//") else t) for t in targets]
174
175
176def generate_builds(root_path):
177  """Generates builds from all BUILD files under absl directory."""
178  bazel_rules = list(
179      filter(lambda r: r.type == "cc_library" and not r.testonly,
180             collect_bazel_rules(root_path)))
181  cmake_rules = list(
182      filter(lambda r: r.type == "absl_cc_library" and not r.testonly,
183             collect_cmake_rules(root_path)))
184  pair_map = pairing_bazel_and_cmake_rules(bazel_rules, cmake_rules)
185  builds = []
186  for rule in sorted(bazel_rules, key=lambda r: r.package[2:] + ":" + r.name):
187    p = {
188        "name":
189            rule.package[2:] + ":" + rule.name,
190        "cmake_target":
191            pair_map.get((rule.package, rule.name)) or "",
192        "headers":
193            sorted(resolve_hdrs(rule.srcs + rule.hdrs + rule.textual_hdrs)),
194        "src":
195            sorted(resolve_srcs(rule.srcs + rule.hdrs + rule.textual_hdrs)),
196        "deps":
197            sorted(resolve_deps(rule.deps)),
198    }
199    builds.append(p)
200  return builds
201
202
203def main():
204  previous_dir = os.getcwd()
205  os.chdir(ABSEIL_PATH)
206  builds = generate_builds("absl")
207  os.chdir(previous_dir)
208  with open(OUTPUT_PATH, 'w') as outfile:
209    outfile.write(yaml.dump(builds, indent=2))
210
211
212if __name__ == "__main__":
213  main()
214