1#!/usr/bin/env python2 2# 3# Copyright 2016 Jeremy Kerr <jk@ozlabs.org> 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 14# implied. 15# See the License for the specific language governing permissions and 16# limitations under the License. 17 18import os.path 19import re 20import sys 21import string 22import json 23import argparse 24import subprocess 25from pyparsing import Regex, Literal, Word, Combine, OneOrMore, QuotedString, \ 26 lineno 27 28json_params = { 29 'indent': 1, 30 'sort_keys': True, 31 'separators': (',', ': '), 32} 33 34def create_parser(): 35 # Match a C-style comment starting with two *s 36 comment = Regex(r'/\*\*(?P<content>.*?)\*/', re.DOTALL) 37 38 # Match an @fwts-<tag> annotation (within the comment), plus the proceeding 39 # text 40 annotation = Regex(r'@fwts-(?P<tag>\w+)\W+(?P<text>.*?)(?=@fwts-|\Z)', 41 re.DOTALL) 42 43 # Match the following prlog() call 44 log_call = (((Literal("prerror") + Literal('(').suppress()) | 45 (Literal("prlog") + Literal('(').suppress() + 46 Word(string.letters + string.digits + '_') + 47 Literal(',').suppress())) + 48 Combine(OneOrMore(QuotedString('"')), adjacent=False) + 49 (Literal(')') | Literal(',')).suppress() 50 ) 51 52 pattern = comment + log_call 53 pattern.setWhitespaceChars(string.whitespace + '\n') 54 55 def comment_action(tok): 56 patterns = {} 57 for result in annotation.scanString(tok['content']): 58 patterns.update(result[0][0]) 59 return patterns 60 61 def annotation_action(tok): 62 return { 63 tok['tag']: cleanup_content(tok['text']) 64 } 65 66 comment.setParseAction(comment_action) 67 annotation.setParseAction(annotation_action) 68 pattern.parseWithTabs() 69 70 return pattern 71 72def find_sources(dirname): 73 sources = [] 74 75 def is_source(fname): 76 return fname.endswith('.c') 77 78 def add_fn(s, dname, fnames): 79 s.extend([ os.path.join(dname, fname) 80 for fname in fnames if is_source(fname) ]) 81 82 os.path.walk(dirname, add_fn, sources) 83 return sources 84 85def cleanup_content(content): 86 comment_prefix_re = re.compile(r'^\s*\*\s*', re.MULTILINE) 87 whitespace_re = re.compile(r'\s+') 88 89 content = comment_prefix_re.sub(' ', content) 90 content = whitespace_re.sub(' ', content) 91 return content.strip() 92 93def warn(loc, message): 94 print >>sys.stderr, 'WARNING:%s:%d: %s' % (loc[0], loc[1], message) 95 96def log_level_to_fwts(level): 97 level_map = { 98 'PR_EMERG': 'LOG_LEVEL_CRITICAL', 99 'PR_ALERT': 'LOG_LEVEL_CRITICAL', 100 'PR_CRIT': 'LOG_LEVEL_CRITICAL', 101 'PR_ERR': 'LOG_LEVEL_CRITICAL', 102 'PR_WARNING': 'LOG_LEVEL_HIGH', 103 'PR_NOTICE': 'LOG_LEVEL_MEDIUM', 104 'PR_PRINTF': 'LOG_LEVEL_MEDIUM', 105 } 106 return level_map.get(level, 'LOG_LEVEL_LOW') 107 108def message_to_pattern(loc, msg): 109 """ Convert a C printf()-style template to a pattern suitable for fwts """ 110 111 # Somewhat-simplified match for a %-template 112 template_re = re.compile( 113 '%(?P<flag>[-#0 +]*)' 114 '(?P<width>(?:[0-9]*|\*))?' 115 '(?P<precision>\.*(?:[1-9][0-9]*|\*))?' 116 '(?:hh|h|ll|l|L|j|z|t)?' 117 '(?P<conversion>[a-zA-Z%])') 118 global is_regex 119 is_regex = False 120 121 def expand_template(match): 122 global is_regex 123 c = match.group('conversion').lower() 124 if c == '%': 125 return '%' 126 is_regex = True 127 if c in ['d', 'i', 'u']: 128 return '[0-9]+' 129 elif c == 'o': 130 return '[0-7]+' 131 elif c == 'x': 132 return '[0-9a-f]+' 133 elif c == 'p': 134 return '(0x[0-9a-f]+|nil)' 135 elif c == 's': 136 return '.*' 137 else: 138 warn(loc, "Unknown template conversion '%s'" % match.group(0)) 139 return '.*' 140 141 escape_re = re.compile(r'\\(?P<char>.)', re.DOTALL) 142 def expand_escape(match): 143 global is_regex 144 c = match.group('char') 145 if c == 'n': 146 return '\n' 147 elif c in ['\\', '"']: 148 return c 149 else: 150 warn(loc, "Unhandled escape sequence '%s'" % match.group(0)) 151 is_regex = True 152 return '.' 153 154 pattern = template_re.sub(expand_template, msg) 155 pattern = escape_re.sub(expand_escape, pattern) 156 pattern = pattern.strip() 157 158 compare_mode = "string" 159 if is_regex: 160 compare_mode = "regex" 161 162 return (compare_mode, pattern) 163 164def parse_patterns(parser, fname, tag): 165 patterns = [] 166 data = open(fname).read() 167 i = 1 168 for result in parser.scanString(data): 169 (token, loc, _) = result 170 if token[1] == 'prlog': 171 (annotations, logfn, level, msg) = token 172 else: 173 (annotations, logfn, msg) = token 174 level = 'PR_ERR' 175 176 loc = (fname, lineno(loc, data)) 177 178 if logfn != 'prlog' and logfn != 'prerror': 179 warn(loc, "unknown log output function '%s'" % logfn) 180 181 compare_mode, pattern_str = message_to_pattern(loc, msg) 182 183 pattern = { 184 'log_level': log_level_to_fwts(level), 185 'compare_mode': compare_mode, 186 'pattern': pattern_str, 187 'last_tag': tag, 188 } 189 190 pattern.update(annotations) 191 192 if not 'label' in pattern: 193 warn(loc, "missing label") 194 pattern['label'] = '%s:%d' % (fname, i) 195 i += 1 196 197 if not 'advice' in pattern: 198 warn(loc, "missing advice") 199 200 allowed_data = ['compare_mode', 'log_level', 201 'pattern', 'advice', 'label', 'last_tag'] 202 extras = set(pattern.keys()) - set(allowed_data) 203 if extras: 204 warn(loc, "unknown pattern annotation: %s" % 205 ','.join([ "'%s'" % e for e in extras])) 206 for e in extras: 207 del pattern[e] 208 209 patterns.append(pattern) 210 211 return patterns 212 213if __name__ == '__main__': 214 argparser = argparse.ArgumentParser( 215 description='Generate FWTS olog definitions from the skiboot ' 216 'source tree') 217 argparser.add_argument('directories', metavar='DIR', nargs='*', 218 help='path to source files (default .)', default=['.']) 219 argparser.add_argument('--output', '-o', metavar='FILE', 220 type=argparse.FileType('w'), default=sys.stdout, 221 help='output to FILE (default to stdout)', nargs='?') 222 args = argparser.parse_args() 223 224 sources = [] 225 for directory in args.directories: 226 try: 227 git_tag = subprocess.check_output(["git","-C", directory, "describe", "--abbrev=0" ]) 228 except: 229 git_tag = "???" 230 git_tag = git_tag.replace("\n", "") 231 sources.extend([ (x, git_tag) for x in find_sources(directory)]) 232 233 parser = create_parser() 234 patterns = [] 235 for source, tag in sources: 236 patterns.extend(parse_patterns(parser, source, tag)) 237 238 data = {'olog_error_warning_patterns': patterns} 239 240 args.output.write(json.dumps(data, **json_params) + '\n') 241 242