diff options
Diffstat (limited to 'import-layers/yocto-poky/scripts/contrib')
11 files changed, 348 insertions, 128 deletions
diff --git a/import-layers/yocto-poky/scripts/contrib/bbvars.py b/import-layers/yocto-poky/scripts/contrib/bbvars.py index 0896d6444..d8d059477 100755 --- a/import-layers/yocto-poky/scripts/contrib/bbvars.py +++ b/import-layers/yocto-poky/scripts/contrib/bbvars.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -24,12 +24,12 @@ import os.path import re def usage(): - print 'Usage: %s -d FILENAME [-d FILENAME]* -m METADIR [-m MATADIR]*' % os.path.basename(sys.argv[0]) - print ' -d FILENAME documentation file to search' - print ' -h, --help display this help and exit' - print ' -m METADIR meta directory to search for recipes' - print ' -t FILENAME documentation config file (for doc tags)' - print ' -T Only display variables with doc tags (requires -t)' + print('Usage: %s -d FILENAME [-d FILENAME]* -m METADIR [-m MATADIR]*' % os.path.basename(sys.argv[0])) + print(' -d FILENAME documentation file to search') + print(' -h, --help display this help and exit') + print(' -m METADIR meta directory to search for recipes') + print(' -t FILENAME documentation config file (for doc tags)') + print(' -T Only display variables with doc tags (requires -t)') def recipe_bbvars(recipe): ''' Return a unique set of every bbvar encountered in the recipe ''' @@ -37,9 +37,9 @@ def recipe_bbvars(recipe): vset = set() try: r = open(recipe) - except IOError as (errno, strerror): - print 'WARNING: Failed to open recipe ', recipe - print strerror + except IOError as err: + print('WARNING: Failed to open recipe ', recipe) + print(err.args[1]) for line in r: # Strip any comments from the line @@ -59,8 +59,8 @@ def collect_bbvars(metadir): for root,dirs,files in os.walk(metadir): for name in files: if name.find(".bb") >= 0: - for key in recipe_bbvars(os.path.join(root,name)).iterkeys(): - if bbvars.has_key(key): + for key in recipe_bbvars(os.path.join(root,name)).keys(): + if key in bbvars: bbvars[key] = bbvars[key] + 1 else: bbvars[key] = 1 @@ -71,9 +71,9 @@ def bbvar_is_documented(var, docfiles): for doc in docfiles: try: f = open(doc) - except IOError as (errno, strerror): - print 'WARNING: Failed to open doc ', doc - print strerror + except IOError as err: + print('WARNING: Failed to open doc ', doc) + print(err.args[1]) for line in f: if prog.match(line): return True @@ -87,8 +87,8 @@ def bbvar_doctag(var, docconf): try: f = open(docconf) - except IOError as (errno, strerror): - return strerror + except IOError as err: + return err.args[1] for line in f: m = prog.search(line) @@ -109,8 +109,8 @@ def main(): # Collect and validate input try: opts, args = getopt.getopt(sys.argv[1:], "d:hm:t:T", ["help"]) - except getopt.GetoptError, err: - print '%s' % str(err) + except getopt.GetoptError as err: + print('%s' % str(err)) usage() sys.exit(2) @@ -122,13 +122,13 @@ def main(): if os.path.isfile(a): docfiles.append(a) else: - print 'ERROR: documentation file %s is not a regular file' % (a) + print('ERROR: documentation file %s is not a regular file' % a) sys.exit(3) elif o == '-m': if os.path.isdir(a): metadirs.append(a) else: - print 'ERROR: meta directory %s is not a directory' % (a) + print('ERROR: meta directory %s is not a directory' % a) sys.exit(4) elif o == "-t": if os.path.isfile(a): @@ -139,31 +139,31 @@ def main(): assert False, "unhandled option" if len(docfiles) == 0: - print 'ERROR: no docfile specified' + print('ERROR: no docfile specified') usage() sys.exit(5) if len(metadirs) == 0: - print 'ERROR: no metadir specified' + print('ERROR: no metadir specified') usage() sys.exit(6) if onlydoctags and docconf == "": - print 'ERROR: no docconf specified' + print('ERROR: no docconf specified') usage() sys.exit(7) # Collect all the variable names from the recipes in the metadirs for m in metadirs: - for key,cnt in collect_bbvars(m).iteritems(): - if bbvars.has_key(key): + for key,cnt in collect_bbvars(m).items(): + if key in bbvars: bbvars[key] = bbvars[key] + cnt else: bbvars[key] = cnt # Check each var for documentation varlen = 0 - for v in bbvars.iterkeys(): + for v in bbvars.keys(): if len(v) > varlen: varlen = len(v) if not bbvar_is_documented(v, docfiles): @@ -172,14 +172,14 @@ def main(): varlen = varlen + 1 # Report all undocumented variables - print 'Found %d undocumented bb variables (out of %d):' % (len(undocumented), len(bbvars)) + print('Found %d undocumented bb variables (out of %d):' % (len(undocumented), len(bbvars))) header = '%s%s%s' % (str("VARIABLE").ljust(varlen), str("COUNT").ljust(6), str("DOCTAG").ljust(7)) - print header - print str("").ljust(len(header), '=') + print(header) + print(str("").ljust(len(header), '=')) for v in undocumented: doctag = bbvar_doctag(v, docconf) if not onlydoctags or not doctag == "": - print '%s%s%s' % (v.ljust(varlen), str(bbvars[v]).ljust(6), doctag) + print('%s%s%s' % (v.ljust(varlen), str(bbvars[v]).ljust(6), doctag)) if __name__ == "__main__": diff --git a/import-layers/yocto-poky/scripts/contrib/build-perf-test-wrapper.sh b/import-layers/yocto-poky/scripts/contrib/build-perf-test-wrapper.sh new file mode 100755 index 000000000..e03ea978b --- /dev/null +++ b/import-layers/yocto-poky/scripts/contrib/build-perf-test-wrapper.sh @@ -0,0 +1,153 @@ +#!/bin/bash +# +# Build performance test script wrapper +# +# Copyright (c) 2016, Intel Corporation. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms and conditions of the GNU General Public License, +# version 2, as published by the Free Software Foundation. +# +# This program is distributed in the hope it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# +# This script is a simple wrapper around the actual build performance tester +# script. This script initializes the build environment, runs +# oe-build-perf-test and archives the results. + +script=`basename $0` +archive_dir=~/perf-results/archives + +usage () { +cat << EOF +Usage: $script [-h] [-c COMMITISH] [-C GIT_REPO] + +Optional arguments: + -h show this help and exit. + -a ARCHIVE_DIR archive results tarball here, give an empty string to + disable tarball archiving (default: $archive_dir) + -c COMMITISH test (checkout) this commit + -C GIT_REPO commit results into Git + -w WORK_DIR work dir for this script + (default: GIT_TOP_DIR/build-perf-test) +EOF +} + + +# Parse command line arguments +commitish="" +while getopts "ha:c:C:w:" opt; do + case $opt in + h) usage + exit 0 + ;; + a) archive_dir=`realpath "$OPTARG"` + ;; + c) commitish=$OPTARG + ;; + C) results_repo=`realpath "$OPTARG"` + commit_results=("--commit-results" "$results_repo") + ;; + w) base_dir=`realpath "$OPTARG"` + ;; + *) usage + exit 1 + ;; + esac +done + +# Check positional args +shift "$((OPTIND - 1))" +if [ $# -ne 0 ]; then + echo "ERROR: No positional args are accepted." + usage + exit 1 +fi + +echo "Running on `uname -n`" +if ! git_topdir=$(git rev-parse --show-toplevel); then + echo "The current working dir doesn't seem to be a git clone. Please cd there before running `basename $0`" + exit 1 +fi + +cd "$git_topdir" + +if [ -n "$commitish" ]; then + # Checkout correct revision + echo "Checking out $commitish" + git fetch &> /dev/null + git checkout HEAD^0 &> /dev/null + git branch -D $commitish &> /dev/null + if ! git checkout -f $commitish &> /dev/null; then + echo "Git checkout failed" + exit 1 + fi +fi + +# Setup build environment +if [ -z "$base_dir" ]; then + base_dir="$git_topdir/build-perf-test" +fi +echo "Using working dir $base_dir" + +timestamp=`date "+%Y%m%d%H%M%S"` +git_rev=$(git rev-parse --short HEAD) || exit 1 +build_dir="$base_dir/build-$git_rev-$timestamp" +results_dir="$base_dir/results-$git_rev-$timestamp" +globalres_log="$base_dir/globalres.log" +machine="qemux86" + +mkdir -p "$base_dir" +source ./oe-init-build-env $build_dir >/dev/null || exit 1 + +# Additional config +auto_conf="$build_dir/conf/auto.conf" +echo "MACHINE = \"$machine\"" > "$auto_conf" +echo 'BB_NUMBER_THREADS = "8"' >> "$auto_conf" +echo 'PARALLEL_MAKE = "-j 8"' >> "$auto_conf" +echo "DL_DIR = \"$base_dir/downloads\"" >> "$auto_conf" +# Disabling network sanity check slightly reduces the variance of timing results +echo 'CONNECTIVITY_CHECK_URIS = ""' >> "$auto_conf" +# Possibility to define extra settings +if [ -f "$base_dir/auto.conf.extra" ]; then + cat "$base_dir/auto.conf.extra" >> "$auto_conf" +fi + +# Run actual test script +oe-build-perf-test --out-dir "$results_dir" \ + --globalres-file "$globalres_log" \ + --lock-file "$base_dir/oe-build-perf.lock" \ + "${commit_results[@]}" \ + --commit-results-branch "{tester_host}/{git_branch}/$machine" \ + --commit-results-tag "{tester_host}/{git_branch}/$machine/{git_commit_count}-g{git_commit}/{tag_num}" + +case $? in + 1) echo "ERROR: oe-build-perf-test script failed!" + exit 1 + ;; + 2) echo "NOTE: some tests failed!" + ;; +esac + +echo -ne "\n\n-----------------\n" +echo "Global results file:" +echo -ne "\n" + +cat "$globalres_log" + +if [ -n "$archive_dir" ]; then + echo -ne "\n\n-----------------\n" + echo "Archiving results in $archive_dir" + mkdir -p "$archive_dir" + results_basename=`basename "$results_dir"` + results_dirname=`dirname "$results_dir"` + tar -czf "$archive_dir/`uname -n`-${results_basename}.tar.gz" -C "$results_dirname" "$results_basename" +fi + +rm -rf "$build_dir" +rm -rf "$results_dir" + +echo "DONE" diff --git a/import-layers/yocto-poky/scripts/contrib/ddimage b/import-layers/yocto-poky/scripts/contrib/ddimage index a503f11d0..ab929957a 100755 --- a/import-layers/yocto-poky/scripts/contrib/ddimage +++ b/import-layers/yocto-poky/scripts/contrib/ddimage @@ -100,5 +100,9 @@ if [ "$RESPONSE" != "y" ]; then fi echo "Writing image..." -dd if="$IMAGE" of="$DEVICE" bs="$BLOCKSIZE" +if which pv >/dev/null 2>&1; then + pv "$IMAGE" | dd of="$DEVICE" bs="$BLOCKSIZE" +else + dd if="$IMAGE" of="$DEVICE" bs="$BLOCKSIZE" +fi sync diff --git a/import-layers/yocto-poky/scripts/contrib/devtool-stress.py b/import-layers/yocto-poky/scripts/contrib/devtool-stress.py index 8cf92ca2f..d555c51a6 100755 --- a/import-layers/yocto-poky/scripts/contrib/devtool-stress.py +++ b/import-layers/yocto-poky/scripts/contrib/devtool-stress.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # devtool stress tester # @@ -43,15 +43,15 @@ def select_recipes(args): tinfoil = bb.tinfoil.Tinfoil() tinfoil.prepare(False) - pkg_pn = tinfoil.cooker.recipecache.pkg_pn - (latest_versions, preferred_versions) = bb.providers.findProviders(tinfoil.config_data, tinfoil.cooker.recipecache, pkg_pn) + pkg_pn = tinfoil.cooker.recipecaches[''].pkg_pn + (latest_versions, preferred_versions) = bb.providers.findProviders(tinfoil.config_data, tinfoil.cooker.recipecaches[''], pkg_pn) skip_classes = args.skip_classes.split(',') recipelist = [] for pn in sorted(pkg_pn): pref = preferred_versions[pn] - inherits = [os.path.splitext(os.path.basename(f))[0] for f in tinfoil.cooker.recipecache.inherits[pref[1]]] + inherits = [os.path.splitext(os.path.basename(f))[0] for f in tinfoil.cooker.recipecaches[''].inherits[pref[1]]] for cls in skip_classes: if cls in inherits: break @@ -121,14 +121,18 @@ def stress_extract(args): sys.stdout.write('Testing %s ' % (pn + ' ').ljust(40, '.')) sys.stdout.flush() failed = False + skipped = None srctree = os.path.join(tmpdir, pn) try: bb.process.run('devtool extract %s %s' % (pn, srctree)) - except bb.process.CmdError as exc: - failed = True - with open('stress_%s_extract.log' % pn, 'w') as f: - f.write(str(exc)) + except bb.process.ExecutionError as exc: + if exc.exitcode == 4: + skipped = 'incompatible' + else: + failed = True + with open('stress_%s_extract.log' % pn, 'w') as f: + f.write(str(exc)) if os.path.exists(srctree): shutil.rmtree(srctree) @@ -136,6 +140,8 @@ def stress_extract(args): if failed: print('failed') failures += 1 + elif skipped: + print('skipped (%s)' % skipped) else: print('ok') except KeyboardInterrupt: @@ -162,29 +168,34 @@ def stress_modify(args): sys.stdout.flush() failed = False reset = True + skipped = None srctree = os.path.join(tmpdir, pn) try: bb.process.run('devtool modify -x %s %s' % (pn, srctree)) - except bb.process.CmdError as exc: - with open('stress_%s_modify.log' % pn, 'w') as f: - f.write(str(exc)) - failed = 'modify' - reset = False - - if not failed: - try: - bb.process.run('bitbake -c install %s' % pn) - except bb.process.CmdError as exc: - with open('stress_%s_install.log' % pn, 'w') as f: + except bb.process.ExecutionError as exc: + if exc.exitcode == 4: + skipped = 'incompatible' + else: + with open('stress_%s_modify.log' % pn, 'w') as f: f.write(str(exc)) - failed = 'build' - if reset: - try: - bb.process.run('devtool reset %s' % pn) - except bb.process.CmdError as exc: - print('devtool reset failed: %s' % str(exc)) - break + failed = 'modify' + reset = False + + if not skipped: + if not failed: + try: + bb.process.run('bitbake -c install %s' % pn) + except bb.process.CmdError as exc: + with open('stress_%s_install.log' % pn, 'w') as f: + f.write(str(exc)) + failed = 'build' + if reset: + try: + bb.process.run('devtool reset %s' % pn) + except bb.process.CmdError as exc: + print('devtool reset failed: %s' % str(exc)) + break if os.path.exists(srctree): shutil.rmtree(srctree) @@ -192,6 +203,8 @@ def stress_modify(args): if failed: print('failed (%s)' % failed) failures += 1 + elif skipped: + print('skipped (%s)' % skipped) else: print('ok') except KeyboardInterrupt: @@ -210,9 +223,10 @@ def main(): parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true') parser.add_argument('-r', '--resume-from', help='Resume from specified recipe', metavar='PN') parser.add_argument('-o', '--only', help='Only test specified recipes (comma-separated without spaces, wildcards allowed)', metavar='PNLIST') - parser.add_argument('-s', '--skip', help='Skip specified recipes (comma-separated without spaces, wildcards allowed)', metavar='PNLIST') + parser.add_argument('-s', '--skip', help='Skip specified recipes (comma-separated without spaces, wildcards allowed)', metavar='PNLIST', default='gcc-source-*,kernel-devsrc,package-index,perf,meta-world-pkgdata,glibc-locale,glibc-mtrace,glibc-scripts,os-release') parser.add_argument('-c', '--skip-classes', help='Skip recipes inheriting specified classes (comma-separated) - default %(default)s', metavar='CLASSLIST', default='native,nativesdk,cross,cross-canadian,image,populate_sdk,meta,packagegroup') subparsers = parser.add_subparsers(title='subcommands', metavar='<subcommand>') + subparsers.required = True parser_modify = subparsers.add_parser('modify', help='Run "devtool modify" followed by a build with bitbake on matching recipes', diff --git a/import-layers/yocto-poky/scripts/contrib/graph-tool b/import-layers/yocto-poky/scripts/contrib/graph-tool index 6dc7d337f..1df5b8c34 100755 --- a/import-layers/yocto-poky/scripts/contrib/graph-tool +++ b/import-layers/yocto-poky/scripts/contrib/graph-tool @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Simple graph query utility # useful for getting answers from .dot files produced by bitbake -g @@ -30,8 +30,7 @@ def get_path_networkx(dotfile, fromnode, tonode): print('ERROR: Please install the networkx python module') sys.exit(1) - graph = networkx.DiGraph(networkx.read_dot(dotfile)) - + graph = networkx.DiGraph(networkx.nx_pydot.read_dot(dotfile)) def node_missing(node): import difflib close_matches = difflib.get_close_matches(node, graph.nodes(), cutoff=0.7) @@ -53,11 +52,11 @@ def find_paths(args, usage): fromnode = args[1] tonode = args[2] - paths = list(get_path_networkx(args[0], fromnode, tonode)) - if paths: - for path in paths: - print ' -> '.join(path) - else: + + path = None + for path in get_path_networkx(args[0], fromnode, tonode): + print(" -> ".join(map(str, path))) + if not path: print("ERROR: no path from %s to %s in graph" % (fromnode, tonode)) sys.exit(1) diff --git a/import-layers/yocto-poky/scripts/contrib/list-packageconfig-flags.py b/import-layers/yocto-poky/scripts/contrib/list-packageconfig-flags.py index 2f3b8b06a..389fb97f6 100755 --- a/import-layers/yocto-poky/scripts/contrib/list-packageconfig-flags.py +++ b/import-layers/yocto-poky/scripts/contrib/list-packageconfig-flags.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -37,7 +37,6 @@ if not bitbakepath: sys.stderr.write("Unable to find bitbake by searching parent directory of this script or PATH\n") sys.exit(1) -import bb.cache import bb.cooker import bb.providers import bb.tinfoil @@ -45,7 +44,7 @@ import bb.tinfoil def get_fnlist(bbhandler, pkg_pn, preferred): ''' Get all recipe file names ''' if preferred: - (latest_versions, preferred_versions) = bb.providers.findProviders(bbhandler.config_data, bbhandler.cooker.recipecache, pkg_pn) + (latest_versions, preferred_versions) = bb.providers.findProviders(bbhandler.config_data, bbhandler.cooker.recipecaches[''], pkg_pn) fn_list = [] for pn in sorted(pkg_pn): @@ -58,11 +57,11 @@ def get_fnlist(bbhandler, pkg_pn, preferred): def get_recipesdata(bbhandler, preferred): ''' Get data of all available recipes which have PACKAGECONFIG flags ''' - pkg_pn = bbhandler.cooker.recipecache.pkg_pn + pkg_pn = bbhandler.cooker.recipecaches[''].pkg_pn data_dict = {} for fn in get_fnlist(bbhandler, pkg_pn, preferred): - data = bb.cache.Cache.loadDataFull(fn, bbhandler.cooker.collection.get_file_appends(fn), bbhandler.config_data) + data = bbhandler.parse_recipe_file(fn) flags = data.getVarFlags("PACKAGECONFIG") flags.pop('doc', None) if flags: @@ -86,7 +85,7 @@ def collect_flags(pkg_dict): ''' Collect available PACKAGECONFIG flags and all affected pkgs ''' # flag_dict = {'flag': ['pkg1', 'pkg2',...]} flag_dict = {} - for pkgname, flaglist in pkg_dict.iteritems(): + for pkgname, flaglist in pkg_dict.items(): for flag in flaglist: if flag in flag_dict: flag_dict[flag].append(pkgname) @@ -104,8 +103,8 @@ def display_pkgs(pkg_dict): pkgname_len += 1 header = '%-*s%s' % (pkgname_len, str("RECIPE NAME"), str("PACKAGECONFIG FLAGS")) - print header - print str("").ljust(len(header), '=') + print(header) + print(str("").ljust(len(header), '=')) for pkgname in sorted(pkg_dict): print('%-*s%s' % (pkgname_len, pkgname, ' '.join(pkg_dict[pkgname]))) @@ -115,28 +114,28 @@ def display_flags(flag_dict): flag_len = len("PACKAGECONFIG FLAG") + 5 header = '%-*s%s' % (flag_len, str("PACKAGECONFIG FLAG"), str("RECIPE NAMES")) - print header - print str("").ljust(len(header), '=') + print(header) + print(str("").ljust(len(header), '=')) for flag in sorted(flag_dict): print('%-*s%s' % (flag_len, flag, ' '.join(sorted(flag_dict[flag])))) def display_all(data_dict): ''' Display all pkgs and PACKAGECONFIG information ''' - print str("").ljust(50, '=') + print(str("").ljust(50, '=')) for fn in data_dict: print('%s' % data_dict[fn].getVar("P", True)) - print fn + print(fn) packageconfig = data_dict[fn].getVar("PACKAGECONFIG", True) or '' if packageconfig.strip() == '': packageconfig = 'None' print('PACKAGECONFIG %s' % packageconfig) - for flag,flag_val in data_dict[fn].getVarFlags("PACKAGECONFIG").iteritems(): + for flag,flag_val in data_dict[fn].getVarFlags("PACKAGECONFIG").items(): if flag == "doc": continue print('PACKAGECONFIG[%s] %s' % (flag, flag_val)) - print '' + print('') def main(): pkg_dict = {} @@ -160,20 +159,20 @@ def main(): options, args = parser.parse_args(sys.argv) - bbhandler = bb.tinfoil.Tinfoil() - bbhandler.prepare() - print("Gathering recipe data...") - data_dict = get_recipesdata(bbhandler, options.preferred) - - if options.listtype == 'flags': - pkg_dict = collect_pkgs(data_dict) - flag_dict = collect_flags(pkg_dict) - display_flags(flag_dict) - elif options.listtype == 'recipes': - pkg_dict = collect_pkgs(data_dict) - display_pkgs(pkg_dict) - elif options.listtype == 'all': - display_all(data_dict) + with bb.tinfoil.Tinfoil() as bbhandler: + bbhandler.prepare() + print("Gathering recipe data...") + data_dict = get_recipesdata(bbhandler, options.preferred) + + if options.listtype == 'flags': + pkg_dict = collect_pkgs(data_dict) + flag_dict = collect_flags(pkg_dict) + display_flags(flag_dict) + elif options.listtype == 'recipes': + pkg_dict = collect_pkgs(data_dict) + display_pkgs(pkg_dict) + elif options.listtype == 'all': + display_all(data_dict) if __name__ == "__main__": main() diff --git a/import-layers/yocto-poky/scripts/contrib/mkefidisk.sh b/import-layers/yocto-poky/scripts/contrib/mkefidisk.sh index 333284ff5..d8db3c016 100755 --- a/import-layers/yocto-poky/scripts/contrib/mkefidisk.sh +++ b/import-layers/yocto-poky/scripts/contrib/mkefidisk.sh @@ -369,8 +369,8 @@ mkswap $SWAP >$OUT 2>&1 || die "Failed to prepare swap" # Installing to $DEVICE # debug "Mounting images and device in preparation for installation" -mount -o loop $HDDIMG $HDDIMG_MNT >$OUT 2>&1 || error "Failed to mount $HDDIMG" -mount -o loop $HDDIMG_MNT/rootfs.img $HDDIMG_ROOTFS_MNT >$OUT 2>&1 || error "Failed to mount rootfs.img" +mount -o ro,loop $HDDIMG $HDDIMG_MNT >$OUT 2>&1 || error "Failed to mount $HDDIMG" +mount -o ro,loop $HDDIMG_MNT/rootfs.img $HDDIMG_ROOTFS_MNT >$OUT 2>&1 || error "Failed to mount rootfs.img" mount $ROOTFS $ROOTFS_MNT >$OUT 2>&1 || error "Failed to mount $ROOTFS on $ROOTFS_MNT" mount $BOOTFS $BOOTFS_MNT >$OUT 2>&1 || error "Failed to mount $BOOTFS on $BOOTFS_MNT" diff --git a/import-layers/yocto-poky/scripts/contrib/python/generate-manifest-2.7.py b/import-layers/yocto-poky/scripts/contrib/python/generate-manifest-2.7.py index d93c943c6..f2ecf8d3f 100755 --- a/import-layers/yocto-poky/scripts/contrib/python/generate-manifest-2.7.py +++ b/import-layers/yocto-poky/scripts/contrib/python/generate-manifest-2.7.py @@ -97,7 +97,7 @@ class MakefileMaker: # generate package variables # - for name, data in sorted(self.packages.iteritems()): + for name, data in sorted(self.packages.items()): desc, deps, files = data # @@ -130,7 +130,7 @@ class MakefileMaker: self.out( 'SUMMARY_${PN}-modules="All Python modules"' ) line = 'RDEPENDS_${PN}-modules="' - for name, data in sorted(self.packages.iteritems()): + for name, data in sorted(self.packages.items()): if name not in ['${PN}-dev', '${PN}-distutils-staticdev']: line += "%s " % name @@ -153,7 +153,7 @@ if __name__ == "__main__": os.unlink(sys.argv[1]) except Exception: sys.exc_clear() - outfile = file( sys.argv[1], "w" ) + outfile = open( sys.argv[1], "w" ) else: outfile = sys.stdout diff --git a/import-layers/yocto-poky/scripts/contrib/python/generate-manifest-3.5.py b/import-layers/yocto-poky/scripts/contrib/python/generate-manifest-3.5.py index 367b4b8b4..2906cc66d 100755 --- a/import-layers/yocto-poky/scripts/contrib/python/generate-manifest-3.5.py +++ b/import-layers/yocto-poky/scripts/contrib/python/generate-manifest-3.5.py @@ -100,7 +100,7 @@ class MakefileMaker: # generate package variables # - for name, data in sorted(self.packages.iteritems()): + for name, data in sorted(self.packages.items()): desc, deps, files = data # @@ -133,7 +133,7 @@ class MakefileMaker: self.out( 'SUMMARY_${PN}-modules="All Python modules"' ) line = 'RDEPENDS_${PN}-modules="' - for name, data in sorted(self.packages.iteritems()): + for name, data in sorted(self.packages.items()): if name not in ['${PN}-dev', '${PN}-distutils-staticdev']: line += "%s " % name @@ -156,7 +156,7 @@ if __name__ == "__main__": os.unlink(sys.argv[1]) except Exception: sys.exc_clear() - outfile = file( sys.argv[1], "w" ) + outfile = open( sys.argv[1], "w" ) else: outfile = sys.stdout @@ -167,7 +167,7 @@ if __name__ == "__main__": # m.addPackage( "${PN}-core", "Python interpreter and core modules", "${PN}-lang ${PN}-re ${PN}-reprlib ${PN}-codecs ${PN}-io ${PN}-math", - "__future__.* _abcoll.* abc.* ast.* copy.* copyreg.* ConfigParser.* " + + "__future__.* _abcoll.* abc.* ast.* copy.* copyreg.* configparser.* " + "genericpath.* getopt.* linecache.* new.* " + "os.* posixpath.* struct.* " + "warnings.* site.* stat.* " + @@ -208,6 +208,9 @@ if __name__ == "__main__": m.addPackage( "${PN}-audio", "Python Audio Handling", "${PN}-core", "wave.* chunk.* sndhdr.* lib-dynload/ossaudiodev.*.so lib-dynload/audioop.*.so audiodev.* sunaudio.* sunau.* toaiff.*" ) + m.addPackage( "${PN}-argparse", "Python command line argument parser", "${PN}-core ${PN}-codecs ${PN}-textutils", + "argparse.*" ) + m.addPackage( "${PN}-asyncio", "Python Asynchronous I/O, event loop, coroutines and tasks", "${PN}-core", "asyncio" ) @@ -217,8 +220,8 @@ if __name__ == "__main__": m.addPackage( "${PN}-compile", "Python bytecode compilation support", "${PN}-core", "py_compile.* compileall.*" ) - m.addPackage( "${PN}-compression", "Python high-level compression support", "${PN}-core ${PN}-codecs", - "gzip.* zipfile.* tarfile.* lib-dynload/bz2.*.so" ) + m.addPackage( "${PN}-compression", "Python high-level compression support", "${PN}-core ${PN}-codecs ${PN}-importlib ${PN}-threading ${PN}-shell", + "gzip.* zipfile.* tarfile.* lib-dynload/bz2.*.so lib-dynload/zlib.*.so" ) m.addPackage( "${PN}-crypt", "Python basic cryptographic and hashing support", "${PN}-core", "hashlib.* md5.* sha.* lib-dynload/crypt.*.so lib-dynload/_hashlib.*.so lib-dynload/_sha256.*.so lib-dynload/_sha512.*.so" ) @@ -229,11 +232,11 @@ if __name__ == "__main__": m.addPackage( "${PN}-curses", "Python curses support", "${PN}-core", "curses lib-dynload/_curses.*.so lib-dynload/_curses_panel.*.so" ) # directory + low level module - m.addPackage( "${PN}-ctypes", "Python C types support", "${PN}-core", + m.addPackage( "${PN}-ctypes", "Python C types support", "${PN}-core ${PN}-subprocess", "ctypes lib-dynload/_ctypes.*.so lib-dynload/_ctypes_test.*.so" ) # directory + low level module m.addPackage( "${PN}-datetime", "Python calendar and time support", "${PN}-core ${PN}-codecs", - "_strptime.* calendar.* lib-dynload/datetime.*.so" ) + "_strptime.* calendar.* datetime.* lib-dynload/_datetime.*.so" ) m.addPackage( "${PN}-db", "Python file-based database support", "${PN}-core", "anydbm.* dumbdbm.* whichdb.* dbm lib-dynload/_dbm.*.so" ) @@ -256,13 +259,16 @@ if __name__ == "__main__": m.addPackage( "${PN}-email", "Python email support", "${PN}-core ${PN}-io ${PN}-re ${PN}-mime ${PN}-audio ${PN}-image ${PN}-netclient", "imaplib.* email" ) # package + m.addPackage( "${PN}-enum", "Python support for enumerations", "${PN}-core", + "enum.*" ) + m.addPackage( "${PN}-fcntl", "Python's fcntl interface", "${PN}-core", "lib-dynload/fcntl.*.so" ) m.addPackage( "${PN}-html", "Python HTML processing support", "${PN}-core", "formatter.* htmlentitydefs.* htmllib.* markupbase.* sgmllib.* HTMLParser.* " ) - m.addPackage( "${PN}-importlib", "Python import implementation library", "${PN}-core", + m.addPackage( "${PN}-importlib", "Python import implementation library", "${PN}-core ${PN}-lang", "importlib" ) m.addPackage( "${PN}-gdbm", "Python GNU database support", "${PN}-core", @@ -278,10 +284,10 @@ if __name__ == "__main__": m.addPackage( "${PN}-json", "Python JSON support", "${PN}-core ${PN}-math ${PN}-re", "json lib-dynload/_json.*.so" ) # package - m.addPackage( "${PN}-lang", "Python low-level language support", "${PN}-core", + m.addPackage( "${PN}-lang", "Python low-level language support", "${PN}-core ${PN}-importlib", "lib-dynload/_bisect.*.so lib-dynload/_collections.*.so lib-dynload/_heapq.*.so lib-dynload/_weakref.*.so lib-dynload/_functools.*.so " + "lib-dynload/array.*.so lib-dynload/itertools.*.so lib-dynload/operator.*.so lib-dynload/parser.*.so " + - "atexit.* bisect.* code.* codeop.* collections.* _collections_abc.* dis.* functools.* heapq.* inspect.* keyword.* opcode.* symbol.* repr.* token.* " + + "atexit.* bisect.* code.* codeop.* collections.* _collections_abc.* contextlib.* dis.* functools.* heapq.* inspect.* keyword.* opcode.* operator.* symbol.* repr.* token.* " + "tokenize.* traceback.* weakref.*" ) m.addPackage( "${PN}-logging", "Python logging support", "${PN}-core ${PN}-io ${PN}-lang ${PN}-pickle ${PN}-stringold", @@ -290,7 +296,7 @@ if __name__ == "__main__": m.addPackage( "${PN}-mailbox", "Python mailbox format support", "${PN}-core ${PN}-mime", "mailbox.*" ) - m.addPackage( "${PN}-math", "Python math support", "${PN}-core", + m.addPackage( "${PN}-math", "Python math support", "${PN}-core ${PN}-crypt", "lib-dynload/cmath.*.so lib-dynload/math.*.so lib-dynload/_random.*.so random.* sets.*" ) m.addPackage( "${PN}-mime", "Python MIME handling APIs", "${PN}-core ${PN}-io", @@ -336,11 +342,17 @@ if __name__ == "__main__": m.addPackage( "${PN}-resource", "Python resource control interface", "${PN}-core", "lib-dynload/resource.*.so" ) - m.addPackage( "${PN}-shell", "Python shell-like functionality", "${PN}-core ${PN}-re", + m.addPackage( "${PN}-selectors", "Python High-level I/O multiplexing", "${PN}-core", + "selectors.*" ) + + m.addPackage( "${PN}-shell", "Python shell-like functionality", "${PN}-core ${PN}-re ${PN}-compression", "cmd.* commands.* dircache.* fnmatch.* glob.* popen2.* shlex.* shutil.*" ) - m.addPackage( "${PN}-subprocess", "Python subprocess support", "${PN}-core ${PN}-io ${PN}-re ${PN}-fcntl ${PN}-pickle", - "subprocess.*" ) + m.addPackage( "${PN}-signal", "Python set handlers for asynchronous events support", "${PN}-core ${PN}-enum", + "signal.*" ) + + m.addPackage( "${PN}-subprocess", "Python subprocess support", "${PN}-core ${PN}-io ${PN}-re ${PN}-fcntl ${PN}-pickle ${PN}-threading ${PN}-signal ${PN}-selectors", + "subprocess.* lib-dynload/_posixsubprocess.*.so" ) m.addPackage( "${PN}-sqlite3", "Python Sqlite3 database support", "${PN}-core ${PN}-datetime ${PN}-lang ${PN}-crypt ${PN}-io ${PN}-threading", "lib-dynload/_sqlite3.*.so sqlite3/dbapi2.* sqlite3/__init__.* sqlite3/dump.*" ) @@ -361,7 +373,7 @@ if __name__ == "__main__": "test" ) # package m.addPackage( "${PN}-threading", "Python threading & synchronization support", "${PN}-core ${PN}-lang", - "_threading_local.* dummy_thread.* dummy_threading.* mutex.* threading.* Queue.*" ) + "_threading_local.* dummy_thread.* dummy_threading.* mutex.* threading.* queue.*" ) m.addPackage( "${PN}-tkinter", "Python Tcl/Tk bindings", "${PN}-core", "lib-dynload/_tkinter.*.so lib-tk tkinter" ) # package diff --git a/import-layers/yocto-poky/scripts/contrib/uncovered b/import-layers/yocto-poky/scripts/contrib/uncovered new file mode 100755 index 000000000..a8399ad17 --- /dev/null +++ b/import-layers/yocto-poky/scripts/contrib/uncovered @@ -0,0 +1,39 @@ +#!/bin/bash -eur +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Find python modules uncovered by oe-seltest +# +# Copyright (c) 2016, Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Author: Ed Bartosh <ed.bartosh@linux.intel.com> +# + +if [ ! "$#" -eq 1 -o -t 0 ] ; then + echo 'Usage: coverage report | ./scripts/contrib/uncovered <dir>' 1>&2 + exit 1 +fi + +path=$(readlink -ev $1) + +if [ ! -d "$path" ] ; then + echo "directory $1 doesn't exist" 1>&2 + exit 1 +fi + +diff -u <(grep "$path" | grep -v '0%$' | cut -f1 -d: | sort) \ + <(find $path | xargs file | grep 'Python script' | cut -f1 -d:| sort) | \ + grep "^+$path" | cut -c2- diff --git a/import-layers/yocto-poky/scripts/contrib/verify-homepage.py b/import-layers/yocto-poky/scripts/contrib/verify-homepage.py index 265ff65d3..d39dd1d97 100755 --- a/import-layers/yocto-poky/scripts/contrib/verify-homepage.py +++ b/import-layers/yocto-poky/scripts/contrib/verify-homepage.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # This script can be used to verify HOMEPAGE values for all recipes in # the current configuration. @@ -7,7 +7,7 @@ import sys import os import subprocess -import urllib2 +import urllib.request # Allow importing scripts/lib modules @@ -33,30 +33,30 @@ def wgetHomepage(pn, homepage): return 0 def verifyHomepage(bbhandler): - pkg_pn = bbhandler.cooker.recipecache.pkg_pn + pkg_pn = bbhandler.cooker.recipecaches[''].pkg_pn pnlist = sorted(pkg_pn) count = 0 checked = [] for pn in pnlist: for fn in pkg_pn[pn]: # There's no point checking multiple BBCLASSEXTENDed variants of the same recipe - realfn, _ = bb.cache.Cache.virtualfn2realfn(fn) + realfn, _, _ = bb.cache.virtualfn2realfn(fn) if realfn in checked: continue - data = bb.cache.Cache.loadDataFull(realfn, bbhandler.cooker.collection.get_file_appends(realfn), bbhandler.config_data) + data = bbhandler.parse_recipe_file(realfn) homepage = data.getVar("HOMEPAGE", True) if homepage: try: - urllib2.urlopen(homepage, timeout=5) + urllib.request.urlopen(homepage, timeout=5) except Exception: count = count + wgetHomepage(os.path.basename(realfn), homepage) checked.append(realfn) return count if __name__=='__main__': - bbhandler = bb.tinfoil.Tinfoil() - bbhandler.prepare() - logger.info("Start verifying HOMEPAGE:") - failcount = verifyHomepage(bbhandler) - logger.info("Finished verifying HOMEPAGE.") - logger.info("Summary: %s failed" % failcount) + with bb.tinfoil.Tinfoil() as bbhandler: + bbhandler.prepare() + logger.info("Start verifying HOMEPAGE:") + failcount = verifyHomepage(bbhandler) + logger.info("Finished verifying HOMEPAGE.") + logger.info("Summary: %s failed" % failcount) |
