1#!/usr/bin/env python
2# A tool to parse ASTMatchers.h and update the documentation in
3# ../LibASTMatchersReference.html automatically. Run from the
4# directory in which this file is located to update the docs.
5
6import collections
7import re
8import urllib2
9
10MATCHERS_FILE = '../../include/clang/ASTMatchers/ASTMatchers.h'
11
12# Each matcher is documented in one row of the form:
13#   result | name | argA
14# The subsequent row contains the documentation and is hidden by default,
15# becoming visible via javascript when the user clicks the matcher name.
16TD_TEMPLATE="""
17<tr><td>%(result)s</td><td class="name" onclick="toggle('%(id)s')"><a name="%(id)sAnchor">%(name)s</a></td><td>%(args)s</td></tr>
18<tr><td colspan="4" class="doc" id="%(id)s"><pre>%(comment)s</pre></td></tr>
19"""
20
21# We categorize the matchers into these three categories in the reference:
22node_matchers = {}
23narrowing_matchers = {}
24traversal_matchers = {}
25
26# We output multiple rows per matcher if the matcher can be used on multiple
27# node types. Thus, we need a new id per row to control the documentation
28# pop-up. ids[name] keeps track of those ids.
29ids = collections.defaultdict(int)
30
31# Cache for doxygen urls we have already verified.
32doxygen_probes = {}
33
34def esc(text):
35  """Escape any html in the given text."""
36  text = re.sub(r'&', '&amp;', text)
37  text = re.sub(r'<', '&lt;', text)
38  text = re.sub(r'>', '&gt;', text)
39  def link_if_exists(m):
40    name = m.group(1)
41    url = 'http://clang.llvm.org/doxygen/classclang_1_1%s.html' % name
42    if url not in doxygen_probes:
43      try:
44        print 'Probing %s...' % url
45        urllib2.urlopen(url)
46        doxygen_probes[url] = True
47      except:
48        doxygen_probes[url] = False
49    if doxygen_probes[url]:
50      return r'Matcher&lt<a href="%s">%s</a>&gt;' % (url, name)
51    else:
52      return m.group(0)
53  text = re.sub(
54    r'Matcher&lt;([^\*&]+)&gt;', link_if_exists, text)
55  return text
56
57def extract_result_types(comment):
58  """Extracts a list of result types from the given comment.
59
60     We allow annotations in the comment of the matcher to specify what
61     nodes a matcher can match on. Those comments have the form:
62       Usable as: Any Matcher | (Matcher<T1>[, Matcher<t2>[, ...]])
63
64     Returns ['*'] in case of 'Any Matcher', or ['T1', 'T2', ...].
65     Returns the empty list if no 'Usable as' specification could be
66     parsed.
67  """
68  result_types = []
69  m = re.search(r'Usable as: Any Matcher[\s\n]*$', comment, re.S)
70  if m:
71    return ['*']
72  while True:
73    m = re.match(r'^(.*)Matcher<([^>]+)>\s*,?[\s\n]*$', comment, re.S)
74    if not m:
75      if re.search(r'Usable as:\s*$', comment):
76        return result_types
77      else:
78        return None
79    result_types += [m.group(2)]
80    comment = m.group(1)
81
82def strip_doxygen(comment):
83  """Returns the given comment without \-escaped words."""
84  # If there is only a doxygen keyword in the line, delete the whole line.
85  comment = re.sub(r'^\\[^\s]+\n', r'', comment, flags=re.M)
86  # Delete the doxygen command and the following whitespace.
87  comment = re.sub(r'\\[^\s]+\s+', r'', comment)
88  return comment
89
90def unify_arguments(args):
91  """Gets rid of anything the user doesn't care about in the argument list."""
92  args = re.sub(r'internal::', r'', args)
93  args = re.sub(r'const\s+', r'', args)
94  args = re.sub(r'&', r' ', args)
95  args = re.sub(r'(^|\s)M\d?(\s)', r'\1Matcher<*>\2', args)
96  return args
97
98def add_matcher(result_type, name, args, comment, is_dyncast=False):
99  """Adds a matcher to one of our categories."""
100  if name == 'id':
101     # FIXME: Figure out whether we want to support the 'id' matcher.
102     return
103  matcher_id = '%s%d' % (name, ids[name])
104  ids[name] += 1
105  args = unify_arguments(args)
106  matcher_html = TD_TEMPLATE % {
107    'result': esc('Matcher<%s>' % result_type),
108    'name': name,
109    'args': esc(args),
110    'comment': esc(strip_doxygen(comment)),
111    'id': matcher_id,
112  }
113  if is_dyncast:
114    node_matchers[result_type + name] = matcher_html
115  # Use a heuristic to figure out whether a matcher is a narrowing or
116  # traversal matcher. By default, matchers that take other matchers as
117  # arguments (and are not node matchers) do traversal. We specifically
118  # exclude known narrowing matchers that also take other matchers as
119  # arguments.
120  elif ('Matcher<' not in args or
121        name in ['allOf', 'anyOf', 'anything', 'unless']):
122    narrowing_matchers[result_type + name] = matcher_html
123  else:
124    traversal_matchers[result_type + name] = matcher_html
125
126def act_on_decl(declaration, comment, allowed_types):
127  """Parse the matcher out of the given declaration and comment.
128
129     If 'allowed_types' is set, it contains a list of node types the matcher
130     can match on, as extracted from the static type asserts in the matcher
131     definition.
132  """
133  if declaration.strip():
134    # Node matchers are defined by writing:
135    #   VariadicDynCastAllOfMatcher<ResultType, ArgumentType> name;
136    m = re.match(r""".*Variadic(?:DynCast)?AllOfMatcher\s*<
137                       \s*([^\s,]+)\s*(?:,
138                       \s*([^\s>]+)\s*)?>
139                       \s*([^\s;]+)\s*;\s*$""", declaration, flags=re.X)
140    if m:
141      result, inner, name = m.groups()
142      if not inner:
143        inner = result
144      add_matcher(result, name, 'Matcher<%s>...' % inner,
145                  comment, is_dyncast=True)
146      return
147
148    # Parse the various matcher definition macros.
149    m = re.match(""".*AST_TYPE_MATCHER\(
150                       \s*([^\s,]+\s*),
151                       \s*([^\s,]+\s*)
152                     \)\s*;\s*$""", declaration, flags=re.X)
153    if m:
154      inner, name = m.groups()
155      add_matcher('Type', name, 'Matcher<%s>...' % inner,
156                  comment, is_dyncast=True)
157      # FIXME: re-enable once we have implemented casting on the TypeLoc
158      # hierarchy.
159      # add_matcher('TypeLoc', '%sLoc' % name, 'Matcher<%sLoc>...' % inner,
160      #             comment, is_dyncast=True)
161      return
162
163    m = re.match(""".*AST_TYPE(LOC)?_TRAVERSE_MATCHER\(
164                       \s*([^\s,]+\s*),
165                       \s*(?:[^\s,]+\s*),
166                       \s*AST_POLYMORPHIC_SUPPORTED_TYPES_([^(]*)\(([^)]*)\)
167                     \)\s*;\s*$""", declaration, flags=re.X)
168    if m:
169      loc, name, n_results, results = m.groups()[0:4]
170      result_types = [r.strip() for r in results.split(',')]
171
172      comment_result_types = extract_result_types(comment)
173      if (comment_result_types and
174          sorted(result_types) != sorted(comment_result_types)):
175        raise Exception('Inconsistent documentation for: %s' % name)
176      for result_type in result_types:
177        add_matcher(result_type, name, 'Matcher<Type>', comment)
178        if loc:
179          add_matcher('%sLoc' % result_type, '%sLoc' % name, 'Matcher<TypeLoc>',
180                      comment)
181      return
182
183    m = re.match(r"""^\s*AST_POLYMORPHIC_MATCHER(_P)?(.?)(?:_OVERLOAD)?\(
184                          \s*([^\s,]+)\s*,
185                          \s*AST_POLYMORPHIC_SUPPORTED_TYPES_([^(]*)\(([^)]*)\)
186                       (?:,\s*([^\s,]+)\s*
187                          ,\s*([^\s,]+)\s*)?
188                       (?:,\s*([^\s,]+)\s*
189                          ,\s*([^\s,]+)\s*)?
190                       (?:,\s*\d+\s*)?
191                      \)\s*{\s*$""", declaration, flags=re.X)
192
193    if m:
194      p, n, name, n_results, results = m.groups()[0:5]
195      args = m.groups()[5:]
196      result_types = [r.strip() for r in results.split(',')]
197      if allowed_types and allowed_types != result_types:
198        raise Exception('Inconsistent documentation for: %s' % name)
199      if n not in ['', '2']:
200        raise Exception('Cannot parse "%s"' % declaration)
201      args = ', '.join('%s %s' % (args[i], args[i+1])
202                       for i in range(0, len(args), 2) if args[i])
203      for result_type in result_types:
204        add_matcher(result_type, name, args, comment)
205      return
206
207    m = re.match(r"""^\s*AST_MATCHER(_P)?(.?)(?:_OVERLOAD)?\(
208                       (?:\s*([^\s,]+)\s*,)?
209                          \s*([^\s,]+)\s*
210                       (?:,\s*([^\s,]+)\s*
211                          ,\s*([^\s,]+)\s*)?
212                       (?:,\s*([^\s,]+)\s*
213                          ,\s*([^\s,]+)\s*)?
214                       (?:,\s*\d+\s*)?
215                      \)\s*{\s*$""", declaration, flags=re.X)
216    if m:
217      p, n, result, name = m.groups()[0:4]
218      args = m.groups()[4:]
219      if not result:
220        if not allowed_types:
221          raise Exception('Did not find allowed result types for: %s' % name)
222        result_types = allowed_types
223      else:
224        result_types = [result]
225      if n not in ['', '2']:
226        raise Exception('Cannot parse "%s"' % declaration)
227      args = ', '.join('%s %s' % (args[i], args[i+1])
228                       for i in range(0, len(args), 2) if args[i])
229      for result_type in result_types:
230        add_matcher(result_type, name, args, comment)
231      return
232
233    # Parse ArgumentAdapting matchers.
234    m = re.match(
235        r"""^.*ArgumentAdaptingMatcherFunc<.*>\s*(?:LLVM_ATTRIBUTE_UNUSED\s*)
236              ([a-zA-Z]*)\s*=\s*{};$""",
237        declaration, flags=re.X)
238    if m:
239      name = m.groups()[0]
240      add_matcher('*', name, 'Matcher<*>', comment)
241      return
242
243    # Parse Variadic operator matchers.
244    m = re.match(
245        r"""^.*VariadicOperatorMatcherFunc\s*([a-zA-Z]*)\s*=\s*{.*};$""",
246        declaration, flags=re.X)
247    if m:
248      name = m.groups()[0]
249      add_matcher('*', name, 'Matcher<*>, ..., Matcher<*>', comment)
250      return
251
252
253    # Parse free standing matcher functions, like:
254    #   Matcher<ResultType> Name(Matcher<ArgumentType> InnerMatcher) {
255    m = re.match(r"""^\s*(.*)\s+
256                     ([^\s\(]+)\s*\(
257                     (.*)
258                     \)\s*{""", declaration, re.X)
259    if m:
260      result, name, args = m.groups()
261      args = ', '.join(p.strip() for p in args.split(','))
262      m = re.match(r'.*\s+internal::(Bindable)?Matcher<([^>]+)>$', result)
263      if m:
264        result_types = [m.group(2)]
265      else:
266        result_types = extract_result_types(comment)
267      if not result_types:
268        if not comment:
269          # Only overloads don't have their own doxygen comments; ignore those.
270          print 'Ignoring "%s"' % name
271        else:
272          print 'Cannot determine result type for "%s"' % name
273      else:
274        for result_type in result_types:
275          add_matcher(result_type, name, args, comment)
276    else:
277      print '*** Unparsable: "' + declaration + '" ***'
278
279def sort_table(matcher_type, matcher_map):
280  """Returns the sorted html table for the given row map."""
281  table = ''
282  for key in sorted(matcher_map.keys()):
283    table += matcher_map[key] + '\n'
284  return ('<!-- START_%(type)s_MATCHERS -->\n' +
285          '%(table)s' +
286          '<!--END_%(type)s_MATCHERS -->') % {
287    'type': matcher_type,
288    'table': table,
289  }
290
291# Parse the ast matchers.
292# We alternate between two modes:
293# body = True: We parse the definition of a matcher. We need
294#   to parse the full definition before adding a matcher, as the
295#   definition might contain static asserts that specify the result
296#   type.
297# body = False: We parse the comments and declaration of the matcher.
298comment = ''
299declaration = ''
300allowed_types = []
301body = False
302for line in open(MATCHERS_FILE).read().splitlines():
303  if body:
304    if line.strip() and line[0] == '}':
305      if declaration:
306        act_on_decl(declaration, comment, allowed_types)
307        comment = ''
308        declaration = ''
309        allowed_types = []
310      body = False
311    else:
312      m = re.search(r'is_base_of<([^,]+), NodeType>', line)
313      if m and m.group(1):
314        allowed_types += [m.group(1)]
315    continue
316  if line.strip() and line.lstrip()[0] == '/':
317    comment += re.sub(r'/+\s?', '', line) + '\n'
318  else:
319    declaration += ' ' + line
320    if ((not line.strip()) or
321        line.rstrip()[-1] == ';' or
322        (line.rstrip()[-1] == '{' and line.rstrip()[-3:] != '= {')):
323      if line.strip() and line.rstrip()[-1] == '{':
324        body = True
325      else:
326        act_on_decl(declaration, comment, allowed_types)
327        comment = ''
328        declaration = ''
329        allowed_types = []
330
331node_matcher_table = sort_table('DECL', node_matchers)
332narrowing_matcher_table = sort_table('NARROWING', narrowing_matchers)
333traversal_matcher_table = sort_table('TRAVERSAL', traversal_matchers)
334
335reference = open('../LibASTMatchersReference.html').read()
336reference = re.sub(r'<!-- START_DECL_MATCHERS.*END_DECL_MATCHERS -->',
337                   '%s', reference, flags=re.S) % node_matcher_table
338reference = re.sub(r'<!-- START_NARROWING_MATCHERS.*END_NARROWING_MATCHERS -->',
339                   '%s', reference, flags=re.S) % narrowing_matcher_table
340reference = re.sub(r'<!-- START_TRAVERSAL_MATCHERS.*END_TRAVERSAL_MATCHERS -->',
341                   '%s', reference, flags=re.S) % traversal_matcher_table
342
343with open('../LibASTMatchersReference.html', 'w') as output:
344  output.write(reference)
345
346