1#!/usr/bin/env python 2# A tool to parse ASTMatchers.h and update the documentation in 3# ../LibASTMatchersReference.html automatically. Run from the 4# directory in which this file is located to update the docs. 5 6import collections 7import re 8import urllib2 9 10MATCHERS_FILE = '../../include/clang/ASTMatchers/ASTMatchers.h' 11 12# Each matcher is documented in one row of the form: 13# result | name | argA 14# The subsequent row contains the documentation and is hidden by default, 15# becoming visible via javascript when the user clicks the matcher name. 16TD_TEMPLATE=""" 17<tr><td>%(result)s</td><td class="name" onclick="toggle('%(id)s')"><a name="%(id)sAnchor">%(name)s</a></td><td>%(args)s</td></tr> 18<tr><td colspan="4" class="doc" id="%(id)s"><pre>%(comment)s</pre></td></tr> 19""" 20 21# We categorize the matchers into these three categories in the reference: 22node_matchers = {} 23narrowing_matchers = {} 24traversal_matchers = {} 25 26# We output multiple rows per matcher if the matcher can be used on multiple 27# node types. Thus, we need a new id per row to control the documentation 28# pop-up. ids[name] keeps track of those ids. 29ids = collections.defaultdict(int) 30 31# Cache for doxygen urls we have already verified. 32doxygen_probes = {} 33 34def esc(text): 35 """Escape any html in the given text.""" 36 text = re.sub(r'&', '&', text) 37 text = re.sub(r'<', '<', text) 38 text = re.sub(r'>', '>', text) 39 def link_if_exists(m): 40 name = m.group(1) 41 url = 'http://clang.llvm.org/doxygen/classclang_1_1%s.html' % name 42 if url not in doxygen_probes: 43 try: 44 print 'Probing %s...' % url 45 urllib2.urlopen(url) 46 doxygen_probes[url] = True 47 except: 48 doxygen_probes[url] = False 49 if doxygen_probes[url]: 50 return r'Matcher<<a href="%s">%s</a>>' % (url, name) 51 else: 52 return m.group(0) 53 text = re.sub( 54 r'Matcher<([^\*&]+)>', link_if_exists, text) 55 return text 56 57def extract_result_types(comment): 58 """Extracts a list of result types from the given comment. 59 60 We allow annotations in the comment of the matcher to specify what 61 nodes a matcher can match on. Those comments have the form: 62 Usable as: Any Matcher | (Matcher<T1>[, Matcher<t2>[, ...]]) 63 64 Returns ['*'] in case of 'Any Matcher', or ['T1', 'T2', ...]. 65 Returns the empty list if no 'Usable as' specification could be 66 parsed. 67 """ 68 result_types = [] 69 m = re.search(r'Usable as: Any Matcher[\s\n]*$', comment, re.S) 70 if m: 71 return ['*'] 72 while True: 73 m = re.match(r'^(.*)Matcher<([^>]+)>\s*,?[\s\n]*$', comment, re.S) 74 if not m: 75 if re.search(r'Usable as:\s*$', comment): 76 return result_types 77 else: 78 return None 79 result_types += [m.group(2)] 80 comment = m.group(1) 81 82def strip_doxygen(comment): 83 """Returns the given comment without \-escaped words.""" 84 # If there is only a doxygen keyword in the line, delete the whole line. 85 comment = re.sub(r'^\\[^\s]+\n', r'', comment, flags=re.M) 86 # Delete the doxygen command and the following whitespace. 87 comment = re.sub(r'\\[^\s]+\s+', r'', comment) 88 return comment 89 90def unify_arguments(args): 91 """Gets rid of anything the user doesn't care about in the argument list.""" 92 args = re.sub(r'internal::', r'', args) 93 args = re.sub(r'const\s+', r'', args) 94 args = re.sub(r'&', r' ', args) 95 args = re.sub(r'(^|\s)M\d?(\s)', r'\1Matcher<*>\2', args) 96 return args 97 98def add_matcher(result_type, name, args, comment, is_dyncast=False): 99 """Adds a matcher to one of our categories.""" 100 if name == 'id': 101 # FIXME: Figure out whether we want to support the 'id' matcher. 102 return 103 matcher_id = '%s%d' % (name, ids[name]) 104 ids[name] += 1 105 args = unify_arguments(args) 106 matcher_html = TD_TEMPLATE % { 107 'result': esc('Matcher<%s>' % result_type), 108 'name': name, 109 'args': esc(args), 110 'comment': esc(strip_doxygen(comment)), 111 'id': matcher_id, 112 } 113 if is_dyncast: 114 node_matchers[result_type + name] = matcher_html 115 # Use a heuristic to figure out whether a matcher is a narrowing or 116 # traversal matcher. By default, matchers that take other matchers as 117 # arguments (and are not node matchers) do traversal. We specifically 118 # exclude known narrowing matchers that also take other matchers as 119 # arguments. 120 elif ('Matcher<' not in args or 121 name in ['allOf', 'anyOf', 'anything', 'unless']): 122 narrowing_matchers[result_type + name] = matcher_html 123 else: 124 traversal_matchers[result_type + name] = matcher_html 125 126def act_on_decl(declaration, comment, allowed_types): 127 """Parse the matcher out of the given declaration and comment. 128 129 If 'allowed_types' is set, it contains a list of node types the matcher 130 can match on, as extracted from the static type asserts in the matcher 131 definition. 132 """ 133 if declaration.strip(): 134 # Node matchers are defined by writing: 135 # VariadicDynCastAllOfMatcher<ResultType, ArgumentType> name; 136 m = re.match(r""".*Variadic(?:DynCast)?AllOfMatcher\s*< 137 \s*([^\s,]+)\s*(?:, 138 \s*([^\s>]+)\s*)?> 139 \s*([^\s;]+)\s*;\s*$""", declaration, flags=re.X) 140 if m: 141 result, inner, name = m.groups() 142 if not inner: 143 inner = result 144 add_matcher(result, name, 'Matcher<%s>...' % inner, 145 comment, is_dyncast=True) 146 return 147 148 # Parse the various matcher definition macros. 149 m = re.match(""".*AST_TYPE_MATCHER\( 150 \s*([^\s,]+\s*), 151 \s*([^\s,]+\s*) 152 \)\s*;\s*$""", declaration, flags=re.X) 153 if m: 154 inner, name = m.groups() 155 add_matcher('Type', name, 'Matcher<%s>...' % inner, 156 comment, is_dyncast=True) 157 # FIXME: re-enable once we have implemented casting on the TypeLoc 158 # hierarchy. 159 # add_matcher('TypeLoc', '%sLoc' % name, 'Matcher<%sLoc>...' % inner, 160 # comment, is_dyncast=True) 161 return 162 163 m = re.match(""".*AST_TYPE(LOC)?_TRAVERSE_MATCHER\( 164 \s*([^\s,]+\s*), 165 \s*(?:[^\s,]+\s*), 166 \s*AST_POLYMORPHIC_SUPPORTED_TYPES_([^(]*)\(([^)]*)\) 167 \)\s*;\s*$""", declaration, flags=re.X) 168 if m: 169 loc, name, n_results, results = m.groups()[0:4] 170 result_types = [r.strip() for r in results.split(',')] 171 172 comment_result_types = extract_result_types(comment) 173 if (comment_result_types and 174 sorted(result_types) != sorted(comment_result_types)): 175 raise Exception('Inconsistent documentation for: %s' % name) 176 for result_type in result_types: 177 add_matcher(result_type, name, 'Matcher<Type>', comment) 178 if loc: 179 add_matcher('%sLoc' % result_type, '%sLoc' % name, 'Matcher<TypeLoc>', 180 comment) 181 return 182 183 m = re.match(r"""^\s*AST_POLYMORPHIC_MATCHER(_P)?(.?)(?:_OVERLOAD)?\( 184 \s*([^\s,]+)\s*, 185 \s*AST_POLYMORPHIC_SUPPORTED_TYPES_([^(]*)\(([^)]*)\) 186 (?:,\s*([^\s,]+)\s* 187 ,\s*([^\s,]+)\s*)? 188 (?:,\s*([^\s,]+)\s* 189 ,\s*([^\s,]+)\s*)? 190 (?:,\s*\d+\s*)? 191 \)\s*{\s*$""", declaration, flags=re.X) 192 193 if m: 194 p, n, name, n_results, results = m.groups()[0:5] 195 args = m.groups()[5:] 196 result_types = [r.strip() for r in results.split(',')] 197 if allowed_types and allowed_types != result_types: 198 raise Exception('Inconsistent documentation for: %s' % name) 199 if n not in ['', '2']: 200 raise Exception('Cannot parse "%s"' % declaration) 201 args = ', '.join('%s %s' % (args[i], args[i+1]) 202 for i in range(0, len(args), 2) if args[i]) 203 for result_type in result_types: 204 add_matcher(result_type, name, args, comment) 205 return 206 207 m = re.match(r"""^\s*AST_MATCHER(_P)?(.?)(?:_OVERLOAD)?\( 208 (?:\s*([^\s,]+)\s*,)? 209 \s*([^\s,]+)\s* 210 (?:,\s*([^\s,]+)\s* 211 ,\s*([^\s,]+)\s*)? 212 (?:,\s*([^\s,]+)\s* 213 ,\s*([^\s,]+)\s*)? 214 (?:,\s*\d+\s*)? 215 \)\s*{\s*$""", declaration, flags=re.X) 216 if m: 217 p, n, result, name = m.groups()[0:4] 218 args = m.groups()[4:] 219 if not result: 220 if not allowed_types: 221 raise Exception('Did not find allowed result types for: %s' % name) 222 result_types = allowed_types 223 else: 224 result_types = [result] 225 if n not in ['', '2']: 226 raise Exception('Cannot parse "%s"' % declaration) 227 args = ', '.join('%s %s' % (args[i], args[i+1]) 228 for i in range(0, len(args), 2) if args[i]) 229 for result_type in result_types: 230 add_matcher(result_type, name, args, comment) 231 return 232 233 # Parse ArgumentAdapting matchers. 234 m = re.match( 235 r"""^.*ArgumentAdaptingMatcherFunc<.*>\s*(?:LLVM_ATTRIBUTE_UNUSED\s*) 236 ([a-zA-Z]*)\s*=\s*{};$""", 237 declaration, flags=re.X) 238 if m: 239 name = m.groups()[0] 240 add_matcher('*', name, 'Matcher<*>', comment) 241 return 242 243 # Parse Variadic operator matchers. 244 m = re.match( 245 r"""^.*VariadicOperatorMatcherFunc\s*([a-zA-Z]*)\s*=\s*{.*};$""", 246 declaration, flags=re.X) 247 if m: 248 name = m.groups()[0] 249 add_matcher('*', name, 'Matcher<*>, ..., Matcher<*>', comment) 250 return 251 252 253 # Parse free standing matcher functions, like: 254 # Matcher<ResultType> Name(Matcher<ArgumentType> InnerMatcher) { 255 m = re.match(r"""^\s*(.*)\s+ 256 ([^\s\(]+)\s*\( 257 (.*) 258 \)\s*{""", declaration, re.X) 259 if m: 260 result, name, args = m.groups() 261 args = ', '.join(p.strip() for p in args.split(',')) 262 m = re.match(r'.*\s+internal::(Bindable)?Matcher<([^>]+)>$', result) 263 if m: 264 result_types = [m.group(2)] 265 else: 266 result_types = extract_result_types(comment) 267 if not result_types: 268 if not comment: 269 # Only overloads don't have their own doxygen comments; ignore those. 270 print 'Ignoring "%s"' % name 271 else: 272 print 'Cannot determine result type for "%s"' % name 273 else: 274 for result_type in result_types: 275 add_matcher(result_type, name, args, comment) 276 else: 277 print '*** Unparsable: "' + declaration + '" ***' 278 279def sort_table(matcher_type, matcher_map): 280 """Returns the sorted html table for the given row map.""" 281 table = '' 282 for key in sorted(matcher_map.keys()): 283 table += matcher_map[key] + '\n' 284 return ('<!-- START_%(type)s_MATCHERS -->\n' + 285 '%(table)s' + 286 '<!--END_%(type)s_MATCHERS -->') % { 287 'type': matcher_type, 288 'table': table, 289 } 290 291# Parse the ast matchers. 292# We alternate between two modes: 293# body = True: We parse the definition of a matcher. We need 294# to parse the full definition before adding a matcher, as the 295# definition might contain static asserts that specify the result 296# type. 297# body = False: We parse the comments and declaration of the matcher. 298comment = '' 299declaration = '' 300allowed_types = [] 301body = False 302for line in open(MATCHERS_FILE).read().splitlines(): 303 if body: 304 if line.strip() and line[0] == '}': 305 if declaration: 306 act_on_decl(declaration, comment, allowed_types) 307 comment = '' 308 declaration = '' 309 allowed_types = [] 310 body = False 311 else: 312 m = re.search(r'is_base_of<([^,]+), NodeType>', line) 313 if m and m.group(1): 314 allowed_types += [m.group(1)] 315 continue 316 if line.strip() and line.lstrip()[0] == '/': 317 comment += re.sub(r'/+\s?', '', line) + '\n' 318 else: 319 declaration += ' ' + line 320 if ((not line.strip()) or 321 line.rstrip()[-1] == ';' or 322 (line.rstrip()[-1] == '{' and line.rstrip()[-3:] != '= {')): 323 if line.strip() and line.rstrip()[-1] == '{': 324 body = True 325 else: 326 act_on_decl(declaration, comment, allowed_types) 327 comment = '' 328 declaration = '' 329 allowed_types = [] 330 331node_matcher_table = sort_table('DECL', node_matchers) 332narrowing_matcher_table = sort_table('NARROWING', narrowing_matchers) 333traversal_matcher_table = sort_table('TRAVERSAL', traversal_matchers) 334 335reference = open('../LibASTMatchersReference.html').read() 336reference = re.sub(r'<!-- START_DECL_MATCHERS.*END_DECL_MATCHERS -->', 337 '%s', reference, flags=re.S) % node_matcher_table 338reference = re.sub(r'<!-- START_NARROWING_MATCHERS.*END_NARROWING_MATCHERS -->', 339 '%s', reference, flags=re.S) % narrowing_matcher_table 340reference = re.sub(r'<!-- START_TRAVERSAL_MATCHERS.*END_TRAVERSAL_MATCHERS -->', 341 '%s', reference, flags=re.S) % traversal_matcher_table 342 343with open('../LibASTMatchersReference.html', 'w') as output: 344 output.write(reference) 345 346