#!/usr/bin/env python2 # # Copyright 2016 Jeremy Kerr # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import os.path import re import sys import string import json import argparse import subprocess from pyparsing import Regex, Literal, Word, Combine, OneOrMore, QuotedString, \ lineno json_params = { 'indent': 1, 'sort_keys': True, 'separators': (',', ': '), } def create_parser(): # Match a C-style comment starting with two *s comment = Regex(r'/\*\*(?P.*?)\*/', re.DOTALL) # Match an @fwts- annotation (within the comment), plus the proceeding # text annotation = Regex(r'@fwts-(?P\w+)\W+(?P.*?)(?=@fwts-|\Z)', re.DOTALL) # Match the following prlog() call log_call = (((Literal("prerror") + Literal('(').suppress()) | (Literal("prlog") + Literal('(').suppress() + Word(string.letters + string.digits + '_') + Literal(',').suppress())) + Combine(OneOrMore(QuotedString('"')), adjacent=False) + (Literal(')') | Literal(',')).suppress() ) pattern = comment + log_call pattern.setWhitespaceChars(string.whitespace + '\n') def comment_action(tok): patterns = {} for result in annotation.scanString(tok['content']): patterns.update(result[0][0]) return patterns def annotation_action(tok): return { tok['tag']: cleanup_content(tok['text']) } comment.setParseAction(comment_action) annotation.setParseAction(annotation_action) pattern.parseWithTabs() return pattern def find_sources(dirname): sources = [] def is_source(fname): return fname.endswith('.c') def add_fn(s, dname, fnames): s.extend([ os.path.join(dname, fname) for fname in fnames if is_source(fname) ]) os.path.walk(dirname, add_fn, sources) return sources def cleanup_content(content): comment_prefix_re = re.compile(r'^\s*\*\s*', re.MULTILINE) whitespace_re = re.compile(r'\s+') content = comment_prefix_re.sub(' ', content) content = whitespace_re.sub(' ', content) return content.strip() def warn(loc, message): print >>sys.stderr, 'WARNING:%s:%d: %s' % (loc[0], loc[1], message) def log_level_to_fwts(level): level_map = { 'PR_EMERG': 'LOG_LEVEL_CRITICAL', 'PR_ALERT': 'LOG_LEVEL_CRITICAL', 'PR_CRIT': 'LOG_LEVEL_CRITICAL', 'PR_ERR': 'LOG_LEVEL_CRITICAL', 'PR_WARNING': 'LOG_LEVEL_HIGH', 'PR_NOTICE': 'LOG_LEVEL_MEDIUM', 'PR_PRINTF': 'LOG_LEVEL_MEDIUM', } return level_map.get(level, 'LOG_LEVEL_LOW') def message_to_pattern(loc, msg): """ Convert a C printf()-style template to a pattern suitable for fwts """ # Somewhat-simplified match for a %-template template_re = re.compile( '%(?P[-#0 +]*)' '(?P(?:[0-9]*|\*))?' '(?P\.*(?:[1-9][0-9]*|\*))?' '(?:hh|h|ll|l|L|j|z|t)?' '(?P[a-zA-Z%])') global is_regex is_regex = False def expand_template(match): global is_regex c = match.group('conversion').lower() if c == '%': return '%' is_regex = True if c in ['d', 'i', 'u']: return '[0-9]+' elif c == 'o': return '[0-7]+' elif c == 'x': return '[0-9a-f]+' elif c == 'p': return '(0x[0-9a-f]+|nil)' elif c == 's': return '.*' else: warn(loc, "Unknown template conversion '%s'" % match.group(0)) return '.*' escape_re = re.compile(r'\\(?P.)', re.DOTALL) def expand_escape(match): global is_regex c = match.group('char') if c == 'n': return '\n' elif c in ['\\', '"']: return c else: warn(loc, "Unhandled escape sequence '%s'" % match.group(0)) is_regex = True return '.' pattern = template_re.sub(expand_template, msg) pattern = escape_re.sub(expand_escape, pattern) pattern = pattern.strip() compare_mode = "string" if is_regex: compare_mode = "regex" return (compare_mode, pattern) def parse_patterns(parser, fname, tag): patterns = [] data = open(fname).read() i = 1 for result in parser.scanString(data): (token, loc, _) = result if token[1] == 'prlog': (annotations, logfn, level, msg) = token else: (annotations, logfn, msg) = token level = 'PR_ERR' loc = (fname, lineno(loc, data)) if logfn != 'prlog' and logfn != 'prerror': warn(loc, "unknown log output function '%s'" % logfn) compare_mode, pattern_str = message_to_pattern(loc, msg) pattern = { 'log_level': log_level_to_fwts(level), 'compare_mode': compare_mode, 'pattern': pattern_str, 'last_tag': tag, } pattern.update(annotations) if not 'label' in pattern: warn(loc, "missing label") pattern['label'] = '%s:%d' % (fname, i) i += 1 if not 'advice' in pattern: warn(loc, "missing advice") allowed_data = ['compare_mode', 'log_level', 'pattern', 'advice', 'label', 'last_tag'] extras = set(pattern.keys()) - set(allowed_data) if extras: warn(loc, "unknown pattern annotation: %s" % ','.join([ "'%s'" % e for e in extras])) for e in extras: del pattern[e] patterns.append(pattern) return patterns if __name__ == '__main__': argparser = argparse.ArgumentParser( description='Generate FWTS olog definitions from the skiboot ' 'source tree') argparser.add_argument('directories', metavar='DIR', nargs='*', help='path to source files (default .)', default=['.']) argparser.add_argument('--output', '-o', metavar='FILE', type=argparse.FileType('w'), default=sys.stdout, help='output to FILE (default to stdout)', nargs='?') args = argparser.parse_args() sources = [] for directory in args.directories: try: git_tag = subprocess.check_output(["git","-C", directory, "describe", "--abbrev=0" ]) except: git_tag = "???" git_tag = git_tag.replace("\n", "") sources.extend([ (x, git_tag) for x in find_sources(directory)]) parser = create_parser() patterns = [] for source, tag in sources: patterns.extend(parse_patterns(parser, source, tag)) data = {'olog_error_warning_patterns': patterns} args.output.write(json.dumps(data, **json_params) + '\n')