diff options
Diffstat (limited to 'poky/scripts/oe-build-perf-report')
| -rwxr-xr-x | poky/scripts/oe-build-perf-report | 661 |
1 files changed, 661 insertions, 0 deletions
diff --git a/poky/scripts/oe-build-perf-report b/poky/scripts/oe-build-perf-report new file mode 100755 index 000000000..dc999c45c --- /dev/null +++ b/poky/scripts/oe-build-perf-report @@ -0,0 +1,661 @@ +#!/usr/bin/python3 +# +# Examine build performance test results +# +# Copyright (c) 2017, Intel Corporation. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms and conditions of the GNU General Public License, +# version 2, as published by the Free Software Foundation. +# +# This program is distributed in the hope it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +import argparse +import json +import logging +import os +import re +import sys +from collections import namedtuple, OrderedDict +from operator import attrgetter +from xml.etree import ElementTree as ET + +# Import oe libs +scripts_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(scripts_path, 'lib')) +import scriptpath +from build_perf import print_table +from build_perf.report import (metadata_xml_to_json, results_xml_to_json, + aggregate_data, aggregate_metadata, measurement_stats, + AggregateTestData) +from build_perf import html +from buildstats import BuildStats, diff_buildstats, BSVerDiff + +scriptpath.add_oe_lib_path() + +from oeqa.utils.git import GitRepo, GitError + + +# Setup logging +logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") +log = logging.getLogger('oe-build-perf-report') + + +# Container class for tester revisions +TestedRev = namedtuple('TestedRev', 'commit commit_number tags') + + +def get_test_runs(repo, tag_name, **kwargs): + """Get a sorted list of test runs, matching given pattern""" + # First, get field names from the tag name pattern + field_names = [m.group(1) for m in re.finditer(r'{(\w+)}', tag_name)] + undef_fields = [f for f in field_names if f not in kwargs.keys()] + + # Fields for formatting tag name pattern + str_fields = dict([(f, '*') for f in field_names]) + str_fields.update(kwargs) + + # Get a list of all matching tags + tag_pattern = tag_name.format(**str_fields) + tags = repo.run_cmd(['tag', '-l', tag_pattern]).splitlines() + log.debug("Found %d tags matching pattern '%s'", len(tags), tag_pattern) + + # Parse undefined fields from tag names + str_fields = dict([(f, r'(?P<{}>[\w\-.()]+)'.format(f)) for f in field_names]) + str_fields['branch'] = r'(?P<branch>[\w\-.()/]+)' + str_fields['commit'] = '(?P<commit>[0-9a-f]{7,40})' + str_fields['commit_number'] = '(?P<commit_number>[0-9]{1,7})' + str_fields['tag_number'] = '(?P<tag_number>[0-9]{1,5})' + # escape parenthesis in fields in order to not messa up the regexp + fixed_fields = dict([(k, v.replace('(', r'\(').replace(')', r'\)')) for k, v in kwargs.items()]) + str_fields.update(fixed_fields) + tag_re = re.compile(tag_name.format(**str_fields)) + + # Parse fields from tags + revs = [] + for tag in tags: + m = tag_re.match(tag) + groups = m.groupdict() + revs.append([groups[f] for f in undef_fields] + [tag]) + + # Return field names and a sorted list of revs + return undef_fields, sorted(revs) + +def list_test_revs(repo, tag_name, verbosity, **kwargs): + """Get list of all tested revisions""" + valid_kwargs = dict([(k, v) for k, v in kwargs.items() if v is not None]) + + fields, revs = get_test_runs(repo, tag_name, **valid_kwargs) + ignore_fields = ['tag_number'] + if verbosity < 2: + extra_fields = ['COMMITS', 'TEST RUNS'] + ignore_fields.extend(['commit_number', 'commit']) + else: + extra_fields = ['TEST RUNS'] + + print_fields = [i for i, f in enumerate(fields) if f not in ignore_fields] + + # Sort revs + rows = [[fields[i].upper() for i in print_fields] + extra_fields] + + prev = [''] * len(print_fields) + prev_commit = None + commit_cnt = 0 + commit_field = fields.index('commit') + for rev in revs: + # Only use fields that we want to print + cols = [rev[i] for i in print_fields] + + + if cols != prev: + commit_cnt = 1 + test_run_cnt = 1 + new_row = [''] * (len(print_fields) + len(extra_fields)) + + for i in print_fields: + if cols[i] != prev[i]: + break + new_row[i:-len(extra_fields)] = cols[i:] + rows.append(new_row) + else: + if rev[commit_field] != prev_commit: + commit_cnt += 1 + test_run_cnt += 1 + + if verbosity < 2: + new_row[-2] = commit_cnt + new_row[-1] = test_run_cnt + prev = cols + prev_commit = rev[commit_field] + + print_table(rows) + +def get_test_revs(repo, tag_name, **kwargs): + """Get list of all tested revisions""" + fields, runs = get_test_runs(repo, tag_name, **kwargs) + + revs = {} + commit_i = fields.index('commit') + commit_num_i = fields.index('commit_number') + for run in runs: + commit = run[commit_i] + commit_num = run[commit_num_i] + tag = run[-1] + if not commit in revs: + revs[commit] = TestedRev(commit, commit_num, [tag]) + else: + assert commit_num == revs[commit].commit_number, "Commit numbers do not match" + revs[commit].tags.append(tag) + + # Return in sorted table + revs = sorted(revs.values(), key=attrgetter('commit_number')) + log.debug("Found %d tested revisions:\n %s", len(revs), + "\n ".join(['{} ({})'.format(rev.commit_number, rev.commit) for rev in revs])) + return revs + +def rev_find(revs, attr, val): + """Search from a list of TestedRev""" + for i, rev in enumerate(revs): + if getattr(rev, attr) == val: + return i + raise ValueError("Unable to find '{}' value '{}'".format(attr, val)) + +def is_xml_format(repo, commit): + """Check if the commit contains xml (or json) data""" + if repo.rev_parse(commit + ':results.xml'): + log.debug("Detected report in xml format in %s", commit) + return True + else: + log.debug("No xml report in %s, assuming json formatted results", commit) + return False + +def read_results(repo, tags, xml=True): + """Read result files from repo""" + + def parse_xml_stream(data): + """Parse multiple concatenated XML objects""" + objs = [] + xml_d = "" + for line in data.splitlines(): + if xml_d and line.startswith('<?xml version='): + objs.append(ET.fromstring(xml_d)) + xml_d = line + else: + xml_d += line + objs.append(ET.fromstring(xml_d)) + return objs + + def parse_json_stream(data): + """Parse multiple concatenated JSON objects""" + objs = [] + json_d = "" + for line in data.splitlines(): + if line == '}{': + json_d += '}' + objs.append(json.loads(json_d, object_pairs_hook=OrderedDict)) + json_d = '{' + else: + json_d += line + objs.append(json.loads(json_d, object_pairs_hook=OrderedDict)) + return objs + + num_revs = len(tags) + + # Optimize by reading all data with one git command + log.debug("Loading raw result data from %d tags, %s...", num_revs, tags[0]) + if xml: + git_objs = [tag + ':metadata.xml' for tag in tags] + [tag + ':results.xml' for tag in tags] + data = parse_xml_stream(repo.run_cmd(['show'] + git_objs + ['--'])) + return ([metadata_xml_to_json(e) for e in data[0:num_revs]], + [results_xml_to_json(e) for e in data[num_revs:]]) + else: + git_objs = [tag + ':metadata.json' for tag in tags] + [tag + ':results.json' for tag in tags] + data = parse_json_stream(repo.run_cmd(['show'] + git_objs + ['--'])) + return data[0:num_revs], data[num_revs:] + + +def get_data_item(data, key): + """Nested getitem lookup""" + for k in key.split('.'): + data = data[k] + return data + + +def metadata_diff(metadata_l, metadata_r): + """Prepare a metadata diff for printing""" + keys = [('Hostname', 'hostname', 'hostname'), + ('Branch', 'branch', 'layers.meta.branch'), + ('Commit number', 'commit_num', 'layers.meta.commit_count'), + ('Commit', 'commit', 'layers.meta.commit'), + ('Number of test runs', 'testrun_count', 'testrun_count') + ] + + def _metadata_diff(key): + """Diff metadata from two test reports""" + try: + val1 = get_data_item(metadata_l, key) + except KeyError: + val1 = '(N/A)' + try: + val2 = get_data_item(metadata_r, key) + except KeyError: + val2 = '(N/A)' + return val1, val2 + + metadata = OrderedDict() + for title, key, key_json in keys: + value_l, value_r = _metadata_diff(key_json) + metadata[key] = {'title': title, + 'value_old': value_l, + 'value': value_r} + return metadata + + +def print_diff_report(metadata_l, data_l, metadata_r, data_r): + """Print differences between two data sets""" + + # First, print general metadata + print("\nTEST METADATA:\n==============") + meta_diff = metadata_diff(metadata_l, metadata_r) + rows = [] + row_fmt = ['{:{wid}} ', '{:<{wid}} ', '{:<{wid}}'] + rows = [['', 'CURRENT COMMIT', 'COMPARING WITH']] + for key, val in meta_diff.items(): + # Shorten commit hashes + if key == 'commit': + rows.append([val['title'] + ':', val['value'][:20], val['value_old'][:20]]) + else: + rows.append([val['title'] + ':', val['value'], val['value_old']]) + print_table(rows, row_fmt) + + + # Print test results + print("\nTEST RESULTS:\n=============") + + tests = list(data_l['tests'].keys()) + # Append tests that are only present in 'right' set + tests += [t for t in list(data_r['tests'].keys()) if t not in tests] + + # Prepare data to be printed + rows = [] + row_fmt = ['{:8}', '{:{wid}}', '{:{wid}}', ' {:>{wid}}', ' {:{wid}} ', '{:{wid}}', + ' {:>{wid}}', ' {:>{wid}}'] + num_cols = len(row_fmt) + for test in tests: + test_l = data_l['tests'][test] if test in data_l['tests'] else None + test_r = data_r['tests'][test] if test in data_r['tests'] else None + pref = ' ' + if test_l is None: + pref = '+' + elif test_r is None: + pref = '-' + descr = test_l['description'] if test_l else test_r['description'] + heading = "{} {}: {}".format(pref, test, descr) + + rows.append([heading]) + + # Generate the list of measurements + meas_l = test_l['measurements'] if test_l else {} + meas_r = test_r['measurements'] if test_r else {} + measurements = list(meas_l.keys()) + measurements += [m for m in list(meas_r.keys()) if m not in measurements] + + for meas in measurements: + m_pref = ' ' + if meas in meas_l: + stats_l = measurement_stats(meas_l[meas], 'l.') + else: + stats_l = measurement_stats(None, 'l.') + m_pref = '+' + if meas in meas_r: + stats_r = measurement_stats(meas_r[meas], 'r.') + else: + stats_r = measurement_stats(None, 'r.') + m_pref = '-' + stats = stats_l.copy() + stats.update(stats_r) + + absdiff = stats['val_cls'](stats['r.mean'] - stats['l.mean']) + reldiff = "{:+.1f} %".format(absdiff * 100 / stats['l.mean']) + if stats['r.mean'] > stats['l.mean']: + absdiff = '+' + str(absdiff) + else: + absdiff = str(absdiff) + rows.append(['', m_pref, stats['name'] + ' ' + stats['quantity'], + str(stats['l.mean']), '->', str(stats['r.mean']), + absdiff, reldiff]) + rows.append([''] * num_cols) + + print_table(rows, row_fmt) + + print() + + +class BSSummary(object): + def __init__(self, bs1, bs2): + self.tasks = {'count': bs2.num_tasks, + 'change': '{:+d}'.format(bs2.num_tasks - bs1.num_tasks)} + self.top_consumer = None + self.top_decrease = None + self.top_increase = None + self.ver_diff = OrderedDict() + + tasks_diff = diff_buildstats(bs1, bs2, 'cputime') + + # Get top consumers of resources + tasks_diff = sorted(tasks_diff, key=attrgetter('value2')) + self.top_consumer = tasks_diff[-5:] + + # Get biggest increase and decrease in resource usage + tasks_diff = sorted(tasks_diff, key=attrgetter('absdiff')) + self.top_decrease = tasks_diff[0:5] + self.top_increase = tasks_diff[-5:] + + # Compare recipe versions and prepare data for display + ver_diff = BSVerDiff(bs1, bs2) + if ver_diff: + if ver_diff.new: + self.ver_diff['New recipes'] = [(n, r.evr) for n, r in ver_diff.new.items()] + if ver_diff.dropped: + self.ver_diff['Dropped recipes'] = [(n, r.evr) for n, r in ver_diff.dropped.items()] + if ver_diff.echanged: + self.ver_diff['Epoch changed'] = [(n, "{} → {}".format(r.left.evr, r.right.evr)) for n, r in ver_diff.echanged.items()] + if ver_diff.vchanged: + self.ver_diff['Version changed'] = [(n, "{} → {}".format(r.left.version, r.right.version)) for n, r in ver_diff.vchanged.items()] + if ver_diff.rchanged: + self.ver_diff['Revision changed'] = [(n, "{} → {}".format(r.left.evr, r.right.evr)) for n, r in ver_diff.rchanged.items()] + + +def print_html_report(data, id_comp, buildstats): + """Print report in html format""" + # Handle metadata + metadata = metadata_diff(data[id_comp].metadata, data[-1].metadata) + + # Generate list of tests + tests = [] + for test in data[-1].results['tests'].keys(): + test_r = data[-1].results['tests'][test] + new_test = {'name': test_r['name'], + 'description': test_r['description'], + 'status': test_r['status'], + 'measurements': [], + 'err_type': test_r.get('err_type'), + } + # Limit length of err output shown + if 'message' in test_r: + lines = test_r['message'].splitlines() + if len(lines) > 20: + new_test['message'] = '...\n' + '\n'.join(lines[-20:]) + else: + new_test['message'] = test_r['message'] + + + # Generate the list of measurements + for meas in test_r['measurements'].keys(): + meas_r = test_r['measurements'][meas] + meas_type = 'time' if meas_r['type'] == 'sysres' else 'size' + new_meas = {'name': meas_r['name'], + 'legend': meas_r['legend'], + 'description': meas_r['name'] + ' ' + meas_type, + } + samples = [] + + # Run through all revisions in our data + for meta, test_data in data: + if (not test in test_data['tests'] or + not meas in test_data['tests'][test]['measurements']): + samples.append(measurement_stats(None)) + continue + test_i = test_data['tests'][test] + meas_i = test_i['measurements'][meas] + commit_num = get_data_item(meta, 'layers.meta.commit_count') + samples.append(measurement_stats(meas_i)) + samples[-1]['commit_num'] = commit_num + + absdiff = samples[-1]['val_cls'](samples[-1]['mean'] - samples[id_comp]['mean']) + new_meas['absdiff'] = absdiff + new_meas['absdiff_str'] = str(absdiff) if absdiff < 0 else '+' + str(absdiff) + new_meas['reldiff'] = "{:+.1f} %".format(absdiff * 100 / samples[id_comp]['mean']) + new_meas['samples'] = samples + new_meas['value'] = samples[-1] + new_meas['value_type'] = samples[-1]['val_cls'] + + # Compare buildstats + bs_key = test + '.' + meas + rev = metadata['commit_num']['value'] + comp_rev = metadata['commit_num']['value_old'] + if (rev in buildstats and bs_key in buildstats[rev] and + comp_rev in buildstats and bs_key in buildstats[comp_rev]): + new_meas['buildstats'] = BSSummary(buildstats[comp_rev][bs_key], + buildstats[rev][bs_key]) + + + new_test['measurements'].append(new_meas) + tests.append(new_test) + + # Chart options + chart_opts = {'haxis': {'min': get_data_item(data[0][0], 'layers.meta.commit_count'), + 'max': get_data_item(data[-1][0], 'layers.meta.commit_count')} + } + + print(html.template.render(title="Build Perf Test Report", + metadata=metadata, test_data=tests, + chart_opts=chart_opts)) + + +def get_buildstats(repo, notes_ref, revs, outdir=None): + """Get the buildstats from git notes""" + full_ref = 'refs/notes/' + notes_ref + if not repo.rev_parse(full_ref): + log.error("No buildstats found, please try running " + "'git fetch origin %s:%s' to fetch them from the remote", + full_ref, full_ref) + return + + missing = False + buildstats = {} + log.info("Parsing buildstats from 'refs/notes/%s'", notes_ref) + for rev in revs: + buildstats[rev.commit_number] = {} + log.debug('Dumping buildstats for %s (%s)', rev.commit_number, + rev.commit) + for tag in rev.tags: + log.debug(' %s', tag) + try: + bs_all = json.loads(repo.run_cmd(['notes', '--ref', notes_ref, + 'show', tag + '^0'])) + except GitError: + log.warning("Buildstats not found for %s", tag) + bs_all = {} + missing = True + + for measurement, bs in bs_all.items(): + # Write out onto disk + if outdir: + tag_base, run_id = tag.rsplit('/', 1) + tag_base = tag_base.replace('/', '_') + bs_dir = os.path.join(outdir, measurement, tag_base) + if not os.path.exists(bs_dir): + os.makedirs(bs_dir) + with open(os.path.join(bs_dir, run_id + '.json'), 'w') as f: + json.dump(bs, f, indent=2) + + # Read buildstats into a dict + _bs = BuildStats.from_json(bs) + if measurement not in buildstats[rev.commit_number]: + buildstats[rev.commit_number][measurement] = _bs + else: + buildstats[rev.commit_number][measurement].aggregate(_bs) + + if missing: + log.info("Buildstats were missing for some test runs, please " + "run 'git fetch origin %s:%s' and try again", + full_ref, full_ref) + + return buildstats + + +def auto_args(repo, args): + """Guess arguments, if not defined by the user""" + # Get the latest commit in the repo + log.debug("Guessing arguments from the latest commit") + msg = repo.run_cmd(['log', '-1', '--branches', '--remotes', '--format=%b']) + for line in msg.splitlines(): + split = line.split(':', 1) + if len(split) != 2: + continue + + key = split[0] + val = split[1].strip() + if key == 'hostname': + log.debug("Using hostname %s", val) + args.hostname = val + elif key == 'branch': + log.debug("Using branch %s", val) + args.branch = val + + +def parse_args(argv): + """Parse command line arguments""" + description = """ +Examine build performance test results from a Git repository""" + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + description=description) + + parser.add_argument('--debug', '-d', action='store_true', + help="Verbose logging") + parser.add_argument('--repo', '-r', required=True, + help="Results repository (local git clone)") + parser.add_argument('--list', '-l', action='count', + help="List available test runs") + parser.add_argument('--html', action='store_true', + help="Generate report in html format") + group = parser.add_argument_group('Tag and revision') + group.add_argument('--tag-name', '-t', + default='{hostname}/{branch}/{machine}/{commit_number}-g{commit}/{tag_number}', + help="Tag name (pattern) for finding results") + group.add_argument('--hostname', '-H') + group.add_argument('--branch', '-B', default='master') + group.add_argument('--machine', default='qemux86') + group.add_argument('--history-length', default=25, type=int, + help="Number of tested revisions to plot in html report") + group.add_argument('--commit', + help="Revision to search for") + group.add_argument('--commit-number', + help="Revision number to search for, redundant if " + "--commit is specified") + group.add_argument('--commit2', + help="Revision to compare with") + group.add_argument('--commit-number2', + help="Revision number to compare with, redundant if " + "--commit2 is specified") + parser.add_argument('--dump-buildstats', nargs='?', const='.', + help="Dump buildstats of the tests") + + return parser.parse_args(argv) + + +def main(argv=None): + """Script entry point""" + args = parse_args(argv) + if args.debug: + log.setLevel(logging.DEBUG) + + repo = GitRepo(args.repo) + + if args.list: + list_test_revs(repo, args.tag_name, args.list, hostname=args.hostname) + return 0 + + # Determine hostname which to use + if not args.hostname: + auto_args(repo, args) + + revs = get_test_revs(repo, args.tag_name, hostname=args.hostname, + branch=args.branch, machine=args.machine) + if len(revs) < 2: + log.error("%d tester revisions found, unable to generate report", + len(revs)) + return 1 + + # Pick revisions + if args.commit: + if args.commit_number: + log.warning("Ignoring --commit-number as --commit was specified") + index1 = rev_find(revs, 'commit', args.commit) + elif args.commit_number: + index1 = rev_find(revs, 'commit_number', args.commit_number) + else: + index1 = len(revs) - 1 + + if args.commit2: + if args.commit_number2: + log.warning("Ignoring --commit-number2 as --commit2 was specified") + index2 = rev_find(revs, 'commit', args.commit2) + elif args.commit_number2: + index2 = rev_find(revs, 'commit_number', args.commit_number2) + else: + if index1 > 0: + index2 = index1 - 1 + else: + log.error("Unable to determine the other commit, use " + "--commit2 or --commit-number2 to specify it") + return 1 + + index_l = min(index1, index2) + index_r = max(index1, index2) + + rev_l = revs[index_l] + rev_r = revs[index_r] + log.debug("Using 'left' revision %s (%s), %s test runs:\n %s", + rev_l.commit_number, rev_l.commit, len(rev_l.tags), + '\n '.join(rev_l.tags)) + log.debug("Using 'right' revision %s (%s), %s test runs:\n %s", + rev_r.commit_number, rev_r.commit, len(rev_r.tags), + '\n '.join(rev_r.tags)) + + # Check report format used in the repo (assume all reports in the same fmt) + xml = is_xml_format(repo, revs[index_r].tags[-1]) + + if args.html: + index_0 = max(0, min(index_l, index_r - args.history_length)) + rev_range = range(index_0, index_r + 1) + else: + # We do not need range of commits for text report (no graphs) + index_0 = index_l + rev_range = (index_l, index_r) + + # Read raw data + log.debug("Reading %d revisions, starting from %s (%s)", + len(rev_range), revs[index_0].commit_number, revs[index_0].commit) + raw_data = [read_results(repo, revs[i].tags, xml) for i in rev_range] + + data = [] + for raw_m, raw_d in raw_data: + data.append(AggregateTestData(aggregate_metadata(raw_m), + aggregate_data(raw_d))) + + # Read buildstats only when needed + buildstats = None + if args.dump_buildstats or args.html: + outdir = 'oe-build-perf-buildstats' if args.dump_buildstats else None + notes_ref = 'buildstats/{}/{}/{}'.format(args.hostname, args.branch, + args.machine) + buildstats = get_buildstats(repo, notes_ref, [rev_l, rev_r], outdir) + + # Print report + if not args.html: + print_diff_report(data[0].metadata, data[0].results, + data[1].metadata, data[1].results) + else: + # Re-map 'left' list index to the data table where index_0 maps to 0 + print_html_report(data, index_l - index_0, buildstats) + + return 0 + +if __name__ == "__main__": + sys.exit(main()) |
