1#!/usr/bin/env python3 2# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later 3# 4# Copyright 2016 Jeremy Kerr <jk@ozlabs.org> 5 6import os.path 7import re 8import sys 9import string 10import json 11import argparse 12import subprocess 13from pyparsing import Regex, Literal, Word, Combine, OneOrMore, QuotedString, \ 14 lineno 15 16json_params = { 17 'indent': 1, 18 'sort_keys': True, 19 'separators': (',', ': '), 20} 21 22def create_parser(): 23 # Match a C-style comment starting with two *s 24 comment = Regex(r'/\*\*(?P<content>.*?)\*/', re.DOTALL) 25 26 # Match an @fwts-<tag> annotation (within the comment), plus the proceeding 27 # text 28 annotation = Regex(r'@fwts-(?P<tag>\w+)\W+(?P<text>.*?)(?=@fwts-|\Z)', 29 re.DOTALL) 30 31 # Match the following prlog() call 32 log_call = (((Literal("prerror") + Literal('(').suppress()) | 33 (Literal("prlog") + Literal('(').suppress() + 34 Word(string.ascii_letters + string.digits + '_') + 35 Literal(',').suppress())) + 36 Combine(OneOrMore(QuotedString('"')), adjacent=False) + 37 (Literal(')') | Literal(',')).suppress() 38 ) 39 40 pattern = comment + log_call 41 pattern.setWhitespaceChars(string.whitespace + '\n') 42 43 def comment_action(tok): 44 patterns = {} 45 for result in annotation.scanString(tok['content']): 46 patterns.update(result[0][0]) 47 return patterns 48 49 def annotation_action(tok): 50 return { 51 tok['tag']: cleanup_content(tok['text']) 52 } 53 54 comment.setParseAction(comment_action) 55 annotation.setParseAction(annotation_action) 56 pattern.parseWithTabs() 57 58 return pattern 59 60def find_sources(dirname): 61 sources = [] 62 63 def is_source(fname): 64 return fname.endswith('.c') 65 66 for directory, dirnames, filenames in os.walk(dirname): 67 sources.extend([ os.path.join(directory, fname) 68 for fname in filenames if is_source(fname) ]) 69 return sources 70 71def cleanup_content(content): 72 comment_prefix_re = re.compile(r'^\s*\*\s*', re.MULTILINE) 73 whitespace_re = re.compile(r'\s+') 74 75 content = comment_prefix_re.sub(' ', content) 76 content = whitespace_re.sub(' ', content) 77 return content.strip() 78 79def warn(loc, message): 80 print >>sys.stderr, 'WARNING:%s:%d: %s' % (loc[0], loc[1], message) 81 82def log_level_to_fwts(level): 83 level_map = { 84 'PR_EMERG': 'LOG_LEVEL_CRITICAL', 85 'PR_ALERT': 'LOG_LEVEL_CRITICAL', 86 'PR_CRIT': 'LOG_LEVEL_CRITICAL', 87 'PR_ERR': 'LOG_LEVEL_CRITICAL', 88 'PR_WARNING': 'LOG_LEVEL_HIGH', 89 'PR_NOTICE': 'LOG_LEVEL_MEDIUM', 90 'PR_PRINTF': 'LOG_LEVEL_MEDIUM', 91 } 92 return level_map.get(level, 'LOG_LEVEL_LOW') 93 94def message_to_pattern(loc, msg): 95 """ Convert a C printf()-style template to a pattern suitable for fwts """ 96 97 # Somewhat-simplified match for a %-template 98 template_re = re.compile( 99 '%(?P<flag>[-#0 +]*)' 100 '(?P<width>(?:[0-9]*|\*))?' 101 '(?P<precision>\.*(?:[1-9][0-9]*|\*))?' 102 '(?:hh|h|ll|l|L|j|z|t)?' 103 '(?P<conversion>[a-zA-Z%])') 104 global is_regex 105 is_regex = False 106 107 def expand_template(match): 108 global is_regex 109 c = match.group('conversion').lower() 110 if c == '%': 111 return '%' 112 is_regex = True 113 if c in ['d', 'i', 'u']: 114 return '[0-9]+' 115 elif c == 'o': 116 return '[0-7]+' 117 elif c == 'x': 118 return '[0-9a-f]+' 119 elif c == 'p': 120 return '(0x[0-9a-f]+|nil)' 121 elif c == 's': 122 return '.*' 123 else: 124 warn(loc, "Unknown template conversion '%s'" % match.group(0)) 125 return '.*' 126 127 escape_re = re.compile(r'\\(?P<char>.)', re.DOTALL) 128 def expand_escape(match): 129 global is_regex 130 c = match.group('char') 131 if c == 'n': 132 return '\n' 133 elif c in ['\\', '"']: 134 return c 135 else: 136 warn(loc, "Unhandled escape sequence '%s'" % match.group(0)) 137 is_regex = True 138 return '.' 139 140 pattern = template_re.sub(expand_template, msg) 141 pattern = escape_re.sub(expand_escape, pattern) 142 pattern = pattern.strip() 143 144 compare_mode = "string" 145 if is_regex: 146 compare_mode = "regex" 147 148 return (compare_mode, pattern) 149 150def parse_patterns(parser, fname, tag): 151 patterns = [] 152 data = open(fname).read() 153 i = 1 154 for result in parser.scanString(data): 155 (token, loc, _) = result 156 if token[1] == 'prlog': 157 (annotations, logfn, level, msg) = token 158 else: 159 (annotations, logfn, msg) = token 160 level = 'PR_ERR' 161 162 loc = (fname, lineno(loc, data)) 163 164 if logfn != 'prlog' and logfn != 'prerror': 165 warn(loc, "unknown log output function '%s'" % logfn) 166 167 compare_mode, pattern_str = message_to_pattern(loc, msg) 168 169 pattern = { 170 'log_level': log_level_to_fwts(level), 171 'compare_mode': compare_mode, 172 'pattern': pattern_str, 173 'last_tag': tag, 174 } 175 176 pattern.update(annotations) 177 178 if not 'label' in pattern: 179 warn(loc, "missing label") 180 pattern['label'] = '%s:%d' % (fname, i) 181 i += 1 182 183 if not 'advice' in pattern: 184 warn(loc, "missing advice") 185 186 allowed_data = ['compare_mode', 'log_level', 187 'pattern', 'advice', 'label', 'last_tag'] 188 extras = set(pattern.keys()) - set(allowed_data) 189 if extras: 190 warn(loc, "unknown pattern annotation: %s" % 191 ','.join([ "'%s'" % e for e in extras])) 192 for e in extras: 193 del pattern[e] 194 195 patterns.append(pattern) 196 197 return patterns 198 199if __name__ == '__main__': 200 argparser = argparse.ArgumentParser( 201 description='Generate FWTS olog definitions from the skiboot ' 202 'source tree') 203 argparser.add_argument('directories', metavar='DIR', nargs='*', 204 help='path to source files (default .)', default=['.']) 205 argparser.add_argument('--output', '-o', metavar='FILE', 206 type=argparse.FileType('w'), default=sys.stdout, 207 help='output to FILE (default to stdout)', nargs='?') 208 args = argparser.parse_args() 209 210 sources = [] 211 for directory in args.directories: 212 try: 213 git_tag = subprocess.check_output(["git","-C", directory, "describe", "--abbrev=0" ], text=True) 214 except: 215 git_tag = "???" 216 git_tag = git_tag.replace("\n", "") 217 sources.extend([ (x, git_tag) for x in find_sources(directory)]) 218 219 parser = create_parser() 220 patterns = [] 221 for source, tag in sources: 222 patterns.extend(parse_patterns(parser, source, tag)) 223 224 data = {'olog_error_warning_patterns': patterns} 225 226 args.output.write(json.dumps(data, **json_params) + '\n') 227 228