1#!/usr/bin/env python2
2#
3# Copyright 2016 Jeremy Kerr <jk@ozlabs.org>
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# 	http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
14# implied.
15# See the License for the specific language governing permissions and
16# limitations under the License.
17
18import os.path
19import re
20import sys
21import string
22import json
23import argparse
24import subprocess
25from pyparsing import Regex, Literal, Word, Combine, OneOrMore, QuotedString, \
26         lineno
27
28json_params = {
29    'indent': 1,
30    'sort_keys': True,
31    'separators': (',', ': '),
32}
33
34def create_parser():
35    # Match a C-style comment starting with two *s
36    comment = Regex(r'/\*\*(?P<content>.*?)\*/', re.DOTALL)
37
38    # Match an @fwts-<tag> annotation (within the comment), plus the proceeding
39    # text
40    annotation = Regex(r'@fwts-(?P<tag>\w+)\W+(?P<text>.*?)(?=@fwts-|\Z)',
41                re.DOTALL)
42
43    # Match the following prlog() call
44    log_call = (((Literal("prerror") + Literal('(').suppress()) |
45                 (Literal("prlog")   + Literal('(').suppress() +
46                  Word(string.letters + string.digits + '_') +
47                  Literal(',').suppress())) +
48                Combine(OneOrMore(QuotedString('"')), adjacent=False) +
49                (Literal(')') | Literal(',')).suppress()
50               )
51
52    pattern = comment + log_call
53    pattern.setWhitespaceChars(string.whitespace + '\n')
54
55    def comment_action(tok):
56        patterns = {}
57        for result in annotation.scanString(tok['content']):
58            patterns.update(result[0][0])
59        return patterns
60
61    def annotation_action(tok):
62        return {
63            tok['tag']: cleanup_content(tok['text'])
64        }
65
66    comment.setParseAction(comment_action)
67    annotation.setParseAction(annotation_action)
68    pattern.parseWithTabs()
69
70    return pattern
71
72def find_sources(dirname):
73    sources = []
74
75    def is_source(fname):
76        return fname.endswith('.c')
77
78    def add_fn(s, dname, fnames):
79        s.extend([ os.path.join(dname, fname)
80                         for fname in fnames if is_source(fname) ])
81
82    os.path.walk(dirname, add_fn, sources)
83    return sources
84
85def cleanup_content(content):
86    comment_prefix_re = re.compile(r'^\s*\*\s*', re.MULTILINE)
87    whitespace_re = re.compile(r'\s+')
88
89    content = comment_prefix_re.sub(' ', content)
90    content = whitespace_re.sub(' ', content)
91    return content.strip()
92
93def warn(loc, message):
94    print >>sys.stderr, 'WARNING:%s:%d: %s' % (loc[0], loc[1], message)
95
96def log_level_to_fwts(level):
97    level_map = {
98        'PR_EMERG':     'LOG_LEVEL_CRITICAL',
99        'PR_ALERT':     'LOG_LEVEL_CRITICAL',
100        'PR_CRIT':      'LOG_LEVEL_CRITICAL',
101        'PR_ERR':       'LOG_LEVEL_CRITICAL',
102        'PR_WARNING':   'LOG_LEVEL_HIGH',
103        'PR_NOTICE':    'LOG_LEVEL_MEDIUM',
104        'PR_PRINTF':    'LOG_LEVEL_MEDIUM',
105    }
106    return level_map.get(level, 'LOG_LEVEL_LOW')
107
108def message_to_pattern(loc, msg):
109    """ Convert a C printf()-style template to a pattern suitable for fwts """
110
111    # Somewhat-simplified match for a %-template
112    template_re = re.compile(
113            '%(?P<flag>[-#0 +]*)'
114            '(?P<width>(?:[0-9]*|\*))?'
115            '(?P<precision>\.*(?:[1-9][0-9]*|\*))?'
116            '(?:hh|h|ll|l|L|j|z|t)?'
117            '(?P<conversion>[a-zA-Z%])')
118    global is_regex
119    is_regex = False
120
121    def expand_template(match):
122        global is_regex
123        c = match.group('conversion').lower()
124        if c == '%':
125            return '%'
126        is_regex = True
127        if c in ['d', 'i', 'u']:
128            return '[0-9]+'
129        elif c == 'o':
130            return '[0-7]+'
131        elif c == 'x':
132            return '[0-9a-f]+'
133        elif c == 'p':
134            return '(0x[0-9a-f]+|nil)'
135        elif c == 's':
136            return '.*'
137        else:
138            warn(loc, "Unknown template conversion '%s'" % match.group(0))
139            return '.*'
140
141    escape_re = re.compile(r'\\(?P<char>.)', re.DOTALL)
142    def expand_escape(match):
143        global is_regex
144        c = match.group('char')
145        if c == 'n':
146            return '\n'
147        elif c in ['\\', '"']:
148            return c
149        else:
150            warn(loc, "Unhandled escape sequence '%s'" % match.group(0))
151            is_regex = True
152            return '.'
153
154    pattern = template_re.sub(expand_template, msg)
155    pattern = escape_re.sub(expand_escape, pattern)
156    pattern = pattern.strip()
157
158    compare_mode = "string"
159    if is_regex:
160        compare_mode = "regex"
161
162    return (compare_mode, pattern)
163
164def parse_patterns(parser, fname, tag):
165    patterns = []
166    data = open(fname).read()
167    i = 1
168    for result in parser.scanString(data):
169        (token, loc, _) = result
170        if token[1] == 'prlog':
171            (annotations, logfn, level, msg) = token
172        else:
173            (annotations, logfn, msg) = token
174            level = 'PR_ERR'
175
176        loc = (fname, lineno(loc, data))
177
178        if logfn != 'prlog' and logfn != 'prerror':
179            warn(loc, "unknown log output function '%s'" % logfn)
180
181        compare_mode, pattern_str = message_to_pattern(loc, msg)
182
183        pattern = {
184            'log_level': log_level_to_fwts(level),
185            'compare_mode': compare_mode,
186            'pattern': pattern_str,
187            'last_tag': tag,
188        }
189
190        pattern.update(annotations)
191
192        if not 'label' in pattern:
193            warn(loc, "missing label")
194            pattern['label'] = '%s:%d' % (fname, i)
195            i += 1
196
197        if not 'advice' in pattern:
198            warn(loc, "missing advice")
199
200        allowed_data = ['compare_mode', 'log_level',
201                        'pattern', 'advice', 'label', 'last_tag']
202        extras = set(pattern.keys()) - set(allowed_data)
203        if extras:
204            warn(loc, "unknown pattern annotation: %s" %
205                    ','.join([ "'%s'" % e for e in extras]))
206            for e in extras:
207                del pattern[e]
208
209        patterns.append(pattern)
210
211    return patterns
212
213if __name__ == '__main__':
214    argparser = argparse.ArgumentParser(
215            description='Generate FWTS olog definitions from the skiboot '
216                        'source tree')
217    argparser.add_argument('directories', metavar='DIR', nargs='*',
218            help='path to source files (default .)', default=['.'])
219    argparser.add_argument('--output', '-o', metavar='FILE',
220            type=argparse.FileType('w'), default=sys.stdout,
221            help='output to FILE (default to stdout)', nargs='?')
222    args = argparser.parse_args()
223
224    sources = []
225    for directory in args.directories:
226        try:
227            git_tag = subprocess.check_output(["git","-C", directory, "describe", "--abbrev=0" ])
228        except:
229            git_tag = "???"
230        git_tag = git_tag.replace("\n", "")
231        sources.extend([ (x, git_tag) for x in find_sources(directory)])
232
233    parser = create_parser()
234    patterns = []
235    for source, tag in sources:
236        patterns.extend(parse_patterns(parser, source, tag))
237
238    data = {'olog_error_warning_patterns': patterns}
239
240    args.output.write(json.dumps(data, **json_params) + '\n')
241
242