diff options
Diffstat (limited to 'import-layers/yocto-poky/scripts/contrib')
22 files changed, 0 insertions, 3637 deletions
diff --git a/import-layers/yocto-poky/scripts/contrib/bb-perf/bb-matrix-plot.sh b/import-layers/yocto-poky/scripts/contrib/bb-perf/bb-matrix-plot.sh deleted file mode 100755 index 136a25570..000000000 --- a/import-layers/yocto-poky/scripts/contrib/bb-perf/bb-matrix-plot.sh +++ /dev/null @@ -1,137 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2011, Intel Corporation. -# All rights reserved. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -# -# DESCRIPTION -# This script operates on the .dat file generated by bb-matrix.sh. It tolerates -# the header by skipping the first line, but error messages and bad data records -# need to be removed first. It will generate three views of the plot, and leave -# an interactive view open for further analysis. -# -# AUTHORS -# Darren Hart <dvhart@linux.intel.com> -# - -# Setup the defaults -DATFILE="bb-matrix.dat" -XLABEL="BB_NUMBER_THREADS" -YLABEL="PARALLEL_MAKE" -FIELD=3 -DEF_TITLE="Elapsed Time (seconds)" -PM3D_FRAGMENT="unset surface; set pm3d at s hidden3d 100" -SIZE="640,480" - -function usage { -CMD=$(basename $0) -cat <<EOM -Usage: $CMD [-d datfile] [-f field] [-h] [-t title] [-w] - -d datfile The data file generated by bb-matrix.sh (default: $DATFILE) - -f field The field index to plot as the Z axis from the data file - (default: $FIELD, "$DEF_TITLE") - -h Display this help message - -s W,H PNG and window size in pixels (default: $SIZE) - -t title The title to display, should describe the field (-f) and units - (default: "$DEF_TITLE") - -w Render the plot as wireframe with a 2D colormap projected on the - XY plane rather than as the texture for the surface -EOM -} - -# Parse and validate arguments -while getopts "d:f:hs:t:w" OPT; do - case $OPT in - d) - DATFILE="$OPTARG" - ;; - f) - FIELD="$OPTARG" - ;; - h) - usage - exit 0 - ;; - s) - SIZE="$OPTARG" - ;; - t) - TITLE="$OPTARG" - ;; - w) - PM3D_FRAGMENT="set pm3d at b" - W="-w" - ;; - *) - usage - exit 1 - ;; - esac -done - -# Ensure the data file exists -if [ ! -f "$DATFILE" ]; then - echo "ERROR: $DATFILE does not exist" - usage - exit 1 -fi -PLOT_BASENAME=${DATFILE%.*}-f$FIELD$W - -# Set a sane title -# TODO: parse the header and define titles for each format parameter for TIME(1) -if [ -z "$TITLE" ]; then - if [ ! "$FIELD" == "3" ]; then - TITLE="Field $FIELD" - else - TITLE="$DEF_TITLE" - fi -fi - -# Determine the dgrid3d mesh dimensions size -MIN=$(tail -n +2 "$DATFILE" | cut -d ' ' -f 1 | sed 's/^0*//' | sort -n | uniq | head -n1) -MAX=$(tail -n +2 "$DATFILE" | cut -d ' ' -f 1 | sed 's/^0*//' | sort -n | uniq | tail -n1) -BB_CNT=$[${MAX} - $MIN + 1] -MIN=$(tail -n +2 "$DATFILE" | cut -d ' ' -f 2 | sed 's/^0*//' | sort -n | uniq | head -n1) -MAX=$(tail -n +2 "$DATFILE" | cut -d ' ' -f 2 | sed 's/^0*//' | sort -n | uniq | tail -n1) -PM_CNT=$[${MAX} - $MIN + 1] - - -(cat <<EOF -set title "$TITLE" -set xlabel "$XLABEL" -set ylabel "$YLABEL" -set style line 100 lt 5 lw 1.5 -$PM3D_FRAGMENT -set dgrid3d $PM_CNT,$BB_CNT splines -set ticslevel 0.2 - -set term png size $SIZE -set output "$PLOT_BASENAME.png" -splot "$DATFILE" every ::1 using 1:2:$FIELD with lines ls 100 - -set view 90,0 -set output "$PLOT_BASENAME-bb.png" -replot - -set view 90,90 -set output "$PLOT_BASENAME-pm.png" -replot - -set view 60,30 -set term wxt size $SIZE -replot -EOF -) | gnuplot --persist diff --git a/import-layers/yocto-poky/scripts/contrib/bb-perf/bb-matrix.sh b/import-layers/yocto-poky/scripts/contrib/bb-perf/bb-matrix.sh deleted file mode 100755 index 106456584..000000000 --- a/import-layers/yocto-poky/scripts/contrib/bb-perf/bb-matrix.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2011, Intel Corporation. -# All rights reserved. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -# -# DESCRIPTION -# This script runs BB_CMD (typically building core-image-sato) for all -# combincations of BB_RANGE and PM_RANGE values. It saves off all the console -# logs, the buildstats directories, and creates a bb-pm-runtime.dat file which -# can be used to postprocess the results with a plotting tool, spreadsheet, etc. -# Before running this script, it is recommended that you pre-download all the -# necessary sources by performing the BB_CMD once manually. It is also a good -# idea to disable cron to avoid runtime variations caused by things like the -# locate process. Be sure to sanitize the dat file prior to post-processing as -# it may contain error messages or bad runs that should be removed. -# -# AUTHORS -# Darren Hart <dvhart@linux.intel.com> -# - -# The following ranges are appropriate for a 4 core system with 8 logical units -# Use leading 0s to ensure all digits are the same string length, this results -# in nice log file names and columnar dat files. -BB_RANGE="04 05 06 07 08 09 10 11 12 13 14 15 16" -PM_RANGE="04 05 06 07 08 09 10 11 12 13 14 15 16" - -DATADIR="bb-matrix-$$" -BB_CMD="bitbake core-image-minimal" -RUNTIME_LOG="$DATADIR/bb-matrix.dat" - -# See TIME(1) for a description of the time format parameters -# The following all report 0: W K r s t w -TIME_STR="%e %S %U %P %c %w %R %F %M %x" - -# Prepare the DATADIR -mkdir $DATADIR -if [ $? -ne 0 ]; then - echo "Failed to create $DATADIR." - exit 1 -fi - -# Add a simple header -echo "BB PM $TIME_STR" > $RUNTIME_LOG -for BB in $BB_RANGE; do - for PM in $PM_RANGE; do - RUNDIR="$DATADIR/$BB-$PM-build" - mkdir $RUNDIR - BB_LOG=$RUNDIR/$BB-$PM-bitbake.log - date - echo "BB=$BB PM=$PM Logging to $BB_LOG" - - echo -n " Preparing the work directory... " - rm -rf pseudodone tmp sstate-cache tmp-eglibc &> /dev/null - echo "done" - - # Export the variables under test and run the bitbake command - # Strip any leading zeroes before passing to bitbake - export BB_NUMBER_THREADS=$(echo $BB | sed 's/^0*//') - export PARALLEL_MAKE="-j $(echo $PM | sed 's/^0*//')" - /usr/bin/time -f "$BB $PM $TIME_STR" -a -o $RUNTIME_LOG $BB_CMD &> $BB_LOG - - echo " $(tail -n1 $RUNTIME_LOG)" - cp -a tmp/buildstats $RUNDIR/$BB-$PM-buildstats - done -done diff --git a/import-layers/yocto-poky/scripts/contrib/bb-perf/buildstats-plot.sh b/import-layers/yocto-poky/scripts/contrib/bb-perf/buildstats-plot.sh deleted file mode 100755 index 7e8ae0410..000000000 --- a/import-layers/yocto-poky/scripts/contrib/bb-perf/buildstats-plot.sh +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2011, Intel Corporation. -# All rights reserved. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -# -# DESCRIPTION -# -# Produces script data to be consumed by gnuplot. There are two possible plots -# depending if either the -S parameter is present or not: -# -# * without -S: Produces a histogram listing top N recipes/tasks versus -# stats. The first stat defined in the -s parameter is the one taken -# into account for ranking -# * -S: Produces a histogram listing tasks versus stats. In this case, -# the value of each stat is the sum for that particular stat in all recipes found. -# Stats values are in descending order defined by the first stat defined on -s -# -# EXAMPLES -# -# 1. Top recipes' tasks taking into account utime -# -# $ buildstats-plot.sh -s utime | gnuplot -p -# -# 2. Tasks versus utime:stime -# -# $ buildstats-plot.sh -s utime:stime -S | gnuplot -p -# -# 3. Tasks versus IO write_bytes:IO read_bytes -# -# $ buildstats-plot.sh -s 'IO write_bytes:IO read_bytes' -S | gnuplot -p -# -# AUTHORS -# Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com> -# - -set -o nounset -set -o errexit - -BS_DIR="tmp/buildstats" -N=10 -STATS="utime" -SUM="" -OUTDATA_FILE="$PWD/buildstats-plot.out" - -function usage { - CMD=$(basename $0) - cat <<EOM -Usage: $CMD [-b buildstats_dir] [-t do_task] - -b buildstats The path where the folder resides - (default: "$BS_DIR") - -n N Top N recipes to display. Ignored if -S is present - (default: "$N") - -s stats The stats to be matched. If more that one stat, units - should be the same because data is plot as histogram. - (see buildstats.sh -h for all options) or any other defined - (build)stat separated by colons, i.e. stime:utime - (default: "$STATS") - -S Sum values for a particular stat for found recipes - -o Output data file. - (default: "$OUTDATA_FILE") - -h Display this help message -EOM -} - -# Parse and validate arguments -while getopts "b:n:s:o:Sh" OPT; do - case $OPT in - b) - BS_DIR="$OPTARG" - ;; - n) - N="$OPTARG" - ;; - s) - STATS="$OPTARG" - ;; - S) - SUM="y" - ;; - o) - OUTDATA_FILE="$OPTARG" - ;; - h) - usage - exit 0 - ;; - *) - usage - exit 1 - ;; - esac -done - -# Get number of stats -IFS=':'; statsarray=(${STATS}); unset IFS -nstats=${#statsarray[@]} - -# Get script folder, use to run buildstats.sh -CD=$(dirname $0) - -# Parse buildstats recipes to produce a single table -OUTBUILDSTATS="$PWD/buildstats.log" -$CD/buildstats.sh -H -s "$STATS" -H > $OUTBUILDSTATS - -# Get headers -HEADERS=$(cat $OUTBUILDSTATS | sed -n -e '1s/ /-/g' -e '1s/:/ /gp') - -echo -e "set boxwidth 0.9 relative" -echo -e "set style data histograms" -echo -e "set style fill solid 1.0 border lt -1" -echo -e "set xtics rotate by 45 right" - -# Get output data -if [ -z "$SUM" ]; then - cat $OUTBUILDSTATS | sed -e '1d' | sort -k3 -n -r | head -$N > $OUTDATA_FILE - # include task at recipe column - sed -i -e "1i\ -${HEADERS}" $OUTDATA_FILE - echo -e "set title \"Top task/recipes\"" - echo -e "plot for [COL=3:`expr 3 + ${nstats} - 1`] '${OUTDATA_FILE}' using COL:xtic(stringcolumn(1).' '.stringcolumn(2)) title columnheader(COL)" -else - - # Construct datatamash sum argument (sum 3 sum 4 ...) - declare -a sumargs - j=0 - for i in `seq $nstats`; do - sumargs[j]=sum; j=$(( $j + 1 )) - sumargs[j]=`expr 3 + $i - 1`; j=$(( $j + 1 )) - done - - # Do the processing with datamash - cat $OUTBUILDSTATS | sed -e '1d' | datamash -t ' ' -g1 ${sumargs[*]} | sort -k2 -n -r > $OUTDATA_FILE - - # Include headers into resulted file, so we can include gnuplot xtics - HEADERS=$(echo $HEADERS | sed -e 's/recipe//1') - sed -i -e "1i\ -${HEADERS}" $OUTDATA_FILE - - # Plot - echo -e "set title \"Sum stats values per task for all recipes\"" - echo -e "plot for [COL=2:`expr 2 + ${nstats} - 1`] '${OUTDATA_FILE}' using COL:xtic(1) title columnheader(COL)" -fi - diff --git a/import-layers/yocto-poky/scripts/contrib/bb-perf/buildstats.sh b/import-layers/yocto-poky/scripts/contrib/bb-perf/buildstats.sh deleted file mode 100755 index 8d7e2488f..000000000 --- a/import-layers/yocto-poky/scripts/contrib/bb-perf/buildstats.sh +++ /dev/null @@ -1,155 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2011, Intel Corporation. -# All rights reserved. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -# -# DESCRIPTION -# Given 'buildstats' data (generate by bitbake when setting -# USER_CLASSES ?= "buildstats" on local.conf), task names and a stats values -# (these are the ones preset on the buildstats files), outputs -# '<task> <recipe> <value_1> <value_2> ... <value_n>'. The units are the ones -# defined at buildstats, which in turn takes data from /proc/[pid] files -# -# Some useful pipelines -# -# 1. Tasks with largest stime (Amount of time that this process has been scheduled -# in kernel mode) values -# $ buildstats.sh -b <buildstats> -s stime | sort -k3 -n -r | head -# -# 2. Min, max, sum utime (Amount of time that this process has been scheduled -# in user mode) per task (in needs GNU datamash) -# $ buildstats.sh -b <buildstats> -s utime | datamash -t' ' -g1 min 3 max 3 sum 3 | sort -k4 -n -r -# -# AUTHORS -# Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com> -# - -# Stats, by type -TIME="utime:stime:cutime:cstime" -IO="IO wchar:IO write_bytes:IO syscr:IO read_bytes:IO rchar:IO syscw:IO cancelled_write_bytes" -RUSAGE="rusage ru_utime:rusage ru_stime:rusage ru_maxrss:rusage ru_minflt:rusage ru_majflt:\ -rusage ru_inblock:rusage ru_oublock:rusage ru_nvcsw:rusage ru_nivcsw" - -CHILD_RUSAGE="Child rusage ru_utime:Child rusage ru_stime:Child rusage ru_maxrss:Child rusage ru_minflt:\ -Child rusage ru_majflt:Child rusage ru_inblock:Child rusage ru_oublock:Child rusage ru_nvcsw:\ -Child rusage ru_nivcsw" - -BS_DIR="tmp/buildstats" -TASKS="compile:configure:fetch:install:patch:populate_lic:populate_sysroot:unpack" -STATS="$TIME" -HEADER="" # No header by default - -function usage { -CMD=$(basename $0) -cat <<EOM -Usage: $CMD [-b buildstats_dir] [-t do_task] - -b buildstats The path where the folder resides - (default: "$BS_DIR") - -t tasks The tasks to be computed - (default: "$TASKS") - -s stats The stats to be matched. Options: TIME, IO, RUSAGE, CHILD_RUSAGE - or any other defined buildstat separated by colons, i.e. stime:utime - (default: "$STATS") - Default stat sets: - TIME=$TIME - IO=$IO - RUSAGE=$RUSAGE - CHILD_RUSAGE=$CHILD_RUSAGE - -h Display this help message -EOM -} - -# Parse and validate arguments -while getopts "b:t:s:Hh" OPT; do - case $OPT in - b) - BS_DIR="$OPTARG" - ;; - t) - TASKS="$OPTARG" - ;; - s) - STATS="$OPTARG" - ;; - H) - HEADER="y" - ;; - h) - usage - exit 0 - ;; - *) - usage - exit 1 - ;; - esac -done - -# Ensure the buildstats folder exists -if [ ! -d "$BS_DIR" ]; then - echo "ERROR: $BS_DIR does not exist" - usage - exit 1 -fi - -stats="" -IFS=":" -for stat in ${STATS}; do - case $stat in - TIME) - stats="${stats}:${TIME}" - ;; - IO) - stats="${stats}:${IO}" - ;; - RUSAGE) - stats="${stats}:${RUSAGE}" - ;; - CHILD_RUSAGE) - stats="${stats}:${CHILD_RUSAGE}" - ;; - *) - stats="${STATS}" - esac -done - -# remove possible colon at the beginning -stats="$(echo "$stats" | sed -e 's/^://1')" - -# Provide a header if required by the user -[ -n "$HEADER" ] && { echo "task:recipe:$stats"; } - -for task in ${TASKS}; do - task="do_${task}" - for file in $(find ${BS_DIR} -type f -name ${task} | awk 'BEGIN{ ORS=""; OFS=":" } { print $0,"" }'); do - recipe="$(basename $(dirname $file))" - times="" - for stat in ${stats}; do - [ -z "$stat" ] && { echo "empty stats"; } - time=$(sed -n -e "s/^\($stat\): \\(.*\\)/\\2/p" $file) - # in case the stat is not present, set the value as NA - [ -z "$time" ] && { time="NA"; } - # Append it to times - if [ -z "$times" ]; then - times="${time}" - else - times="${times} ${time}" - fi - done - echo "${task} ${recipe} ${times}" - done -done diff --git a/import-layers/yocto-poky/scripts/contrib/bbvars.py b/import-layers/yocto-poky/scripts/contrib/bbvars.py deleted file mode 100755 index 286b5a940..000000000 --- a/import-layers/yocto-poky/scripts/contrib/bbvars.py +++ /dev/null @@ -1,180 +0,0 @@ -#!/usr/bin/env python3 - -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -# -# Copyright (C) Darren Hart <dvhart@linux.intel.com>, 2010 - - -import sys -import getopt -import os -import os.path -import re - -# Set up sys.path to let us import tinfoil -scripts_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -lib_path = scripts_path + '/lib' -sys.path.insert(0, lib_path) -import scriptpath -scriptpath.add_bitbake_lib_path() -import bb.tinfoil - -def usage(): - print('Usage: %s -d FILENAME [-d FILENAME]*' % os.path.basename(sys.argv[0])) - print(' -d FILENAME documentation file to search') - print(' -h, --help display this help and exit') - print(' -t FILENAME documentation config file (for doc tags)') - print(' -T Only display variables with doc tags (requires -t)') - -def bbvar_is_documented(var, documented_vars): - ''' Check if variable (var) is in the list of documented variables(documented_vars) ''' - if var in documented_vars: - return True - else: - return False - -def collect_documented_vars(docfiles): - ''' Walk the docfiles and collect the documented variables ''' - documented_vars = [] - prog = re.compile(".*($|[^A-Z_])<glossentry id=\'var-") - var_prog = re.compile('<glossentry id=\'var-(.*)\'>') - for d in docfiles: - with open(d) as f: - documented_vars += var_prog.findall(f.read()) - - return documented_vars - -def bbvar_doctag(var, docconf): - prog = re.compile('^%s\[doc\] *= *"(.*)"' % (var)) - if docconf == "": - return "?" - - try: - f = open(docconf) - except IOError as err: - return err.args[1] - - for line in f: - m = prog.search(line) - if m: - return m.group(1) - - f.close() - return "" - -def main(): - docfiles = [] - bbvars = set() - undocumented = [] - docconf = "" - onlydoctags = False - - # Collect and validate input - try: - opts, args = getopt.getopt(sys.argv[1:], "d:hm:t:T", ["help"]) - except getopt.GetoptError as err: - print('%s' % str(err)) - usage() - sys.exit(2) - - for o, a in opts: - if o in ('-h', '--help'): - usage() - sys.exit(0) - elif o == '-d': - if os.path.isfile(a): - docfiles.append(a) - else: - print('ERROR: documentation file %s is not a regular file' % a) - sys.exit(3) - elif o == "-t": - if os.path.isfile(a): - docconf = a - elif o == "-T": - onlydoctags = True - else: - assert False, "unhandled option" - - if len(docfiles) == 0: - print('ERROR: no docfile specified') - usage() - sys.exit(5) - - if onlydoctags and docconf == "": - print('ERROR: no docconf specified') - usage() - sys.exit(7) - - prog = re.compile("^[^a-z]*$") - with bb.tinfoil.Tinfoil() as tinfoil: - tinfoil.prepare(config_only=False) - parser = bb.codeparser.PythonParser('parser', None) - datastore = tinfoil.config_data - - def bbvars_update(data): - if prog.match(data): - bbvars.add(data) - if tinfoil.config_data.getVarFlag(data, 'python'): - try: - parser.parse_python(tinfoil.config_data.getVar(data)) - except bb.data_smart.ExpansionError: - pass - for var in parser.references: - if prog.match(var): - bbvars.add(var) - else: - try: - expandedVar = datastore.expandWithRefs(datastore.getVar(data, False), data) - for var in expandedVar.references: - if prog.match(var): - bbvars.add(var) - except bb.data_smart.ExpansionError: - pass - - # Use tinfoil to collect all the variable names globally - for data in datastore: - bbvars_update(data) - - # Collect variables from all recipes - for recipe in tinfoil.all_recipe_files(variants=False): - print("Checking %s" % recipe) - for data in tinfoil.parse_recipe_file(recipe): - bbvars_update(data) - - documented_vars = collect_documented_vars(docfiles) - - # Check each var for documentation - varlen = 0 - for v in bbvars: - if len(v) > varlen: - varlen = len(v) - if not bbvar_is_documented(v, documented_vars): - undocumented.append(v) - undocumented.sort() - varlen = varlen + 1 - - # Report all undocumented variables - print('Found %d undocumented bb variables (out of %d):' % (len(undocumented), len(bbvars))) - header = '%s%s' % (str("VARIABLE").ljust(varlen), str("DOCTAG").ljust(7)) - print(header) - print(str("").ljust(len(header), '=')) - for v in undocumented: - doctag = bbvar_doctag(v, docconf) - if not onlydoctags or not doctag == "": - print('%s%s' % (v.ljust(varlen), doctag)) - - -if __name__ == "__main__": - main() diff --git a/import-layers/yocto-poky/scripts/contrib/build-perf-test-wrapper.sh b/import-layers/yocto-poky/scripts/contrib/build-perf-test-wrapper.sh deleted file mode 100755 index 19bee1dd0..000000000 --- a/import-layers/yocto-poky/scripts/contrib/build-perf-test-wrapper.sh +++ /dev/null @@ -1,239 +0,0 @@ -#!/bin/bash -# -# Build performance test script wrapper -# -# Copyright (c) 2016, Intel Corporation. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms and conditions of the GNU General Public License, -# version 2, as published by the Free Software Foundation. -# -# This program is distributed in the hope it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# -# This script is a simple wrapper around the actual build performance tester -# script. This script initializes the build environment, runs -# oe-build-perf-test and archives the results. - -script=`basename $0` -script_dir=$(realpath $(dirname $0)) -archive_dir=~/perf-results/archives - -usage () { -cat << EOF -Usage: $script [-h] [-c COMMITISH] [-C GIT_REPO] - -Optional arguments: - -h show this help and exit. - -a ARCHIVE_DIR archive results tarball here, give an empty string to - disable tarball archiving (default: $archive_dir) - -c COMMITISH test (checkout) this commit, <branch>:<commit> can be - specified to test specific commit of certain branch - -C GIT_REPO commit results into Git - -E EMAIL_ADDR send email report - -P GIT_REMOTE push results to a remote Git repository - -R DEST rsync reports to a remote destination - -w WORK_DIR work dir for this script - (default: GIT_TOP_DIR/build-perf-test) - -x create xml report (instead of json) -EOF -} - -get_os_release_var () { - ( source /etc/os-release; eval echo '$'$1 ) -} - - -# Parse command line arguments -commitish="" -oe_build_perf_test_extra_opts=() -oe_git_archive_extra_opts=() -while getopts "ha:c:C:E:P:R:w:x" opt; do - case $opt in - h) usage - exit 0 - ;; - a) archive_dir=`realpath -s "$OPTARG"` - ;; - c) commitish=$OPTARG - ;; - C) results_repo=`realpath -s "$OPTARG"` - ;; - E) email_to="$OPTARG" - ;; - P) oe_git_archive_extra_opts+=("--push" "$OPTARG") - ;; - R) rsync_dst="$OPTARG" - ;; - w) base_dir=`realpath -s "$OPTARG"` - ;; - x) oe_build_perf_test_extra_opts+=("--xml") - ;; - *) usage - exit 1 - ;; - esac -done - -# Check positional args -shift "$((OPTIND - 1))" -if [ $# -ne 0 ]; then - echo "ERROR: No positional args are accepted." - usage - exit 1 -fi - -# Open a file descriptor for flock and acquire lock -LOCK_FILE="/tmp/oe-build-perf-test-wrapper.lock" -if ! exec 3> "$LOCK_FILE"; then - echo "ERROR: Unable to open lock file" - exit 1 -fi -if ! flock -n 3; then - echo "ERROR: Another instance of this script is running" - exit 1 -fi - -echo "Running on `uname -n`" -if ! git_topdir=$(git rev-parse --show-toplevel); then - echo "The current working dir doesn't seem to be a git clone. Please cd there before running `basename $0`" - exit 1 -fi - -cd "$git_topdir" - -if [ -n "$commitish" ]; then - echo "Running git fetch" - git fetch &> /dev/null - git checkout HEAD^0 &> /dev/null - - # Handle <branch>:<commit> format - if echo "$commitish" | grep -q ":"; then - commit=`echo "$commitish" | cut -d":" -f2` - branch=`echo "$commitish" | cut -d":" -f1` - else - commit="$commitish" - branch="$commitish" - fi - - echo "Checking out $commitish" - git branch -D $branch &> /dev/null - if ! git checkout -f $branch &> /dev/null; then - echo "ERROR: Git checkout failed" - exit 1 - fi - - # Check that the specified branch really contains the commit - commit_hash=`git rev-parse --revs-only $commit --` - if [ -z "$commit_hash" -o "`git merge-base $branch $commit`" != "$commit_hash" ]; then - echo "ERROR: branch $branch does not contain commit $commit" - exit 1 - fi - git reset --hard $commit > /dev/null -fi - -# Determine name of the current branch -branch=`git symbolic-ref HEAD 2> /dev/null` -# Strip refs/heads/ -branch=${branch:11} - -# Setup build environment -if [ -z "$base_dir" ]; then - base_dir="$git_topdir/build-perf-test" -fi -echo "Using working dir $base_dir" - -timestamp=`date "+%Y%m%d%H%M%S"` -git_rev=$(git rev-parse --short HEAD) || exit 1 -build_dir="$base_dir/build-$git_rev-$timestamp" -results_dir="$base_dir/results-$git_rev-$timestamp" -globalres_log="$base_dir/globalres.log" -machine="qemux86" - -mkdir -p "$base_dir" -source ./oe-init-build-env $build_dir >/dev/null || exit 1 - -# Additional config -auto_conf="$build_dir/conf/auto.conf" -echo "MACHINE = \"$machine\"" > "$auto_conf" -echo 'BB_NUMBER_THREADS = "8"' >> "$auto_conf" -echo 'PARALLEL_MAKE = "-j 8"' >> "$auto_conf" -echo "DL_DIR = \"$base_dir/downloads\"" >> "$auto_conf" -# Disabling network sanity check slightly reduces the variance of timing results -echo 'CONNECTIVITY_CHECK_URIS = ""' >> "$auto_conf" -# Possibility to define extra settings -if [ -f "$base_dir/auto.conf.extra" ]; then - cat "$base_dir/auto.conf.extra" >> "$auto_conf" -fi - -# Run actual test script -oe-build-perf-test --out-dir "$results_dir" \ - --globalres-file "$globalres_log" \ - "${oe_build_perf_test_extra_opts[@]}" \ - --lock-file "$base_dir/oe-build-perf.lock" - -case $? in - 1) echo "ERROR: oe-build-perf-test script failed!" - exit 1 - ;; - 2) echo "NOTE: some tests failed!" - ;; -esac - -# Commit results to git -if [ -n "$results_repo" ]; then - echo -e "\nArchiving results in $results_repo" - oe-git-archive \ - --git-dir "$results_repo" \ - --branch-name "{hostname}/{branch}/{machine}" \ - --tag-name "{hostname}/{branch}/{machine}/{commit_count}-g{commit}/{tag_number}" \ - --exclude "buildstats.json" \ - --notes "buildstats/{branch_name}" "$results_dir/buildstats.json" \ - "${oe_git_archive_extra_opts[@]}" \ - "$results_dir" - - # Generate test reports - sanitized_branch=`echo $branch | tr / _` - report_txt=`hostname`_${sanitized_branch}_${machine}.txt - report_html=`hostname`_${sanitized_branch}_${machine}.html - echo -e "\nGenerating test report" - oe-build-perf-report -r "$results_repo" > $report_txt - oe-build-perf-report -r "$results_repo" --html > $report_html - - # Send email report - if [ -n "$email_to" ]; then - echo "Emailing test report" - os_name=`get_os_release_var PRETTY_NAME` - "$script_dir"/oe-build-perf-report-email.py --to "$email_to" --subject "Build Perf Test Report for $os_name" --text $report_txt --html $report_html "${OE_BUILD_PERF_REPORT_EMAIL_EXTRA_ARGS[@]}" - fi - - # Upload report files, unless we're on detached head - if [ -n "$rsync_dst" -a -n "$branch" ]; then - echo "Uploading test report" - rsync $report_txt $report_html $rsync_dst - fi -fi - - -echo -ne "\n\n-----------------\n" -echo "Global results file:" -echo -ne "\n" - -cat "$globalres_log" - -if [ -n "$archive_dir" ]; then - echo -ne "\n\n-----------------\n" - echo "Archiving results in $archive_dir" - mkdir -p "$archive_dir" - results_basename=`basename "$results_dir"` - results_dirname=`dirname "$results_dir"` - tar -czf "$archive_dir/`uname -n`-${results_basename}.tar.gz" -C "$results_dirname" "$results_basename" -fi - -rm -rf "$build_dir" -rm -rf "$results_dir" - -echo "DONE" diff --git a/import-layers/yocto-poky/scripts/contrib/build-perf-test.sh b/import-layers/yocto-poky/scripts/contrib/build-perf-test.sh deleted file mode 100755 index 9a091edb0..000000000 --- a/import-layers/yocto-poky/scripts/contrib/build-perf-test.sh +++ /dev/null @@ -1,400 +0,0 @@ -#!/bin/bash -# -# This script runs a series of tests (with and without sstate) and reports build time (and tmp/ size) -# -# Build performance test script -# -# Copyright 2013 Intel Corporation -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# -# -# AUTHORS: -# Stefan Stanacar <stefanx.stanacar@intel.com> - - -ME=$(basename $0) - -# -# usage and setup -# - -usage () { -cat << EOT -Usage: $ME [-h] - $ME [-c <commit>] [-v] [-m <val>] [-j <val>] [-t <val>] [-i <image-name>] [-d <path>] -Options: - -h - Display this help and exit. - -c <commit> - git checkout <commit> before anything else - -v - Show bitbake output, don't redirect it to a log. - -m <machine> - Value for MACHINE. Default is qemux86. - -j <val> - Value for PARALLEL_MAKE. Default is 8. - -t <val> - Value for BB_NUMBER_THREADS. Default is 8. - -i <image-name> - Instead of timing against core-image-sato, use <image-name> - -d <path> - Use <path> as DL_DIR - -p <githash> - Cherry pick githash onto the commit - -Note: current working directory must be inside a poky git clone. - -EOT -} - - -if clonedir=$(git rev-parse --show-toplevel); then - cd $clonedir -else - echo "The current working dir doesn't seem to be a poky git clone. Please cd there before running $ME" - exit 1 -fi - -IMAGE="core-image-sato" -verbose=0 -dldir= -commit= -pmake= -cherrypicks= -while getopts "hvc:m:j:t:i:d:p:" opt; do - case $opt in - h) usage - exit 0 - ;; - v) verbose=1 - ;; - c) commit=$OPTARG - ;; - m) export MACHINE=$OPTARG - ;; - j) pmake=$OPTARG - ;; - t) export BB_NUMBER_THREADS=$OPTARG - ;; - i) IMAGE=$OPTARG - ;; - d) dldir=$OPTARG - ;; - p) cherrypicks="$cherrypicks $OPTARG" - ;; - *) usage - exit 1 - ;; - esac -done - - -#drop cached credentials and test for sudo access without a password -sudo -k -n ls > /dev/null 2>&1 -reqpass=$? -if [ $reqpass -ne 0 ]; then - echo "The script requires sudo access to drop caches between builds (echo 3 > /proc/sys/vm/drop_caches)" - read -s -p "Please enter your sudo password: " pass - echo -fi - -if [ -n "$commit" ]; then - echo "git checkout -f $commit" - git pull > /dev/null 2>&1 - git checkout -f $commit || exit 1 - git pull > /dev/null 2>&1 -fi - -if [ -n "$cherrypicks" ]; then - for c in $cherrypicks; do - git cherry-pick $c - done -fi - -rev=$(git rev-parse --short HEAD) || exit 1 -OUTDIR="$clonedir/build-perf-test/results-$rev-`date "+%Y%m%d%H%M%S"`" -BUILDDIR="$OUTDIR/build" -resultsfile="$OUTDIR/results.log" -cmdoutput="$OUTDIR/commands.log" -myoutput="$OUTDIR/output.log" -globalres="$clonedir/build-perf-test/globalres.log" - -mkdir -p $OUTDIR || exit 1 - -log () { - local msg="$1" - echo "`date`: $msg" | tee -a $myoutput -} - - -# -# Config stuff -# - -branch=`git branch 2>&1 | grep "^* " | tr -d "* "` -gitcommit=$(git rev-parse HEAD) || exit 1 -log "Running on $branch:$gitcommit" - -source ./oe-init-build-env $OUTDIR/build >/dev/null || exit 1 -cd $OUTDIR/build - -[ -n "$MACHINE" ] || export MACHINE="qemux86" -[ -n "$BB_NUMBER_THREADS" ] || export BB_NUMBER_THREADS="8" - -if [ -n "$pmake" ]; then - export PARALLEL_MAKE="-j $pmake" -else - export PARALLEL_MAKE="-j 8" -fi - -if [ -n "$dldir" ]; then - echo "DL_DIR = \"$dldir\"" >> conf/local.conf -else - echo "DL_DIR = \"$clonedir/build-perf-test/downloads\"" >> conf/local.conf -fi - -# Sometimes I've noticed big differences in timings for the same commit, on the same machine -# Disabling the network sanity check helps a bit (because of my crappy network connection and/or proxy) -echo "CONNECTIVITY_CHECK_URIS =\"\"" >> conf/local.conf - - -# -# Functions -# - -declare -a TIMES -time_count=0 -declare -a SIZES -size_count=0 - -time_cmd () { - log " Timing: $*" - - if [ $verbose -eq 0 ]; then - /usr/bin/time -v -o $resultsfile "$@" >> $cmdoutput - else - /usr/bin/time -v -o $resultsfile "$@" - fi - ret=$? - if [ $ret -eq 0 ]; then - t=`grep wall $resultsfile | sed 's/.*m:ss): //'` - log " TIME: $t" - TIMES[(( time_count++ ))]="$t" - else - log "ERROR: exit status was non-zero, will report time as 0." - TIMES[(( time_count++ ))]="0" - fi - - #time by default overwrites the output file and we want to keep the results - #it has an append option but I don't want to clobber the results in the same file - i=`ls $OUTDIR/results.log* |wc -l` - mv $resultsfile "${resultsfile}.${i}" - log "More stats can be found in ${resultsfile}.${i}" -} - -bbtime () { - time_cmd bitbake "$@" -} - -#we don't time bitbake here -bbnotime () { - local arg="$@" - log " Running: bitbake ${arg}" - if [ $verbose -eq 0 ]; then - bitbake ${arg} >> $cmdoutput - else - bitbake ${arg} - fi - ret=$? - if [ $ret -eq 0 ]; then - log " Finished bitbake ${arg}" - else - log "ERROR: exit status was non-zero. Exit.." - exit $ret - fi - -} - -do_rmtmp() { - log " Removing tmp" - rm -rf bitbake.lock pseudodone conf/sanity_info cache tmp -} -do_rmsstate () { - log " Removing sstate-cache" - rm -rf sstate-cache -} -do_sync () { - log " Syncing and dropping caches" - sync; sync - if [ $reqpass -eq 0 ]; then - sudo sh -c "echo 3 > /proc/sys/vm/drop_caches" - else - echo "$pass" | sudo -S sh -c "echo 3 > /proc/sys/vm/drop_caches" - echo - fi - sleep 3 -} - -write_results() { - echo -n "`uname -n`,$branch:$gitcommit,`git describe`," >> $globalres - for i in "${TIMES[@]}"; do - echo -n "$i," >> $globalres - done - for i in "${SIZES[@]}"; do - echo -n "$i," >> $globalres - done - echo >> $globalres - sed -i '$ s/,$//' $globalres -} - -#### - -# -# Test 1 -# Measure: Wall clock of "bitbake core-image-sato" and size of tmp/dir (w/o rm_work and w/ rm_work) -# Pre: Downloaded sources, no sstate -# Steps: -# Part1: -# - fetchall -# - clean build dir -# - time bitbake core-image-sato -# - collect data -# Part2: -# - bitbake virtual/kernel -c cleansstate -# - time bitbake virtual/kernel -# Part3: -# - add INHERIT to local.conf -# - clean build dir -# - build -# - report size, remove INHERIT - -test1_p1 () { - log "Running Test 1, part 1/3: Measure wall clock of bitbake $IMAGE and size of tmp/ dir" - bbnotime $IMAGE --runall=fetch - do_rmtmp - do_rmsstate - do_sync - bbtime $IMAGE - s=`du -s tmp | sed 's/tmp//' | sed 's/[ \t]*$//'` - SIZES[(( size_count++ ))]="$s" - log "SIZE of tmp dir is: $s" - log "Buildstats are saved in $OUTDIR/buildstats-test1" - mv tmp/buildstats $OUTDIR/buildstats-test1 -} - - -test1_p2 () { - log "Running Test 1, part 2/3: bitbake virtual/kernel -c cleansstate and time bitbake virtual/kernel" - bbnotime virtual/kernel -c cleansstate - do_sync - bbtime virtual/kernel -} - -test1_p3 () { - log "Running Test 1, part 3/3: Build $IMAGE w/o sstate and report size of tmp/dir with rm_work enabled" - echo "INHERIT += \"rm_work\"" >> conf/local.conf - do_rmtmp - do_rmsstate - do_sync - bbtime $IMAGE - sed -i 's/INHERIT += \"rm_work\"//' conf/local.conf - s=`du -s tmp | sed 's/tmp//' | sed 's/[ \t]*$//'` - SIZES[(( size_count++ ))]="$s" - log "SIZE of tmp dir is: $s" - log "Buildstats are saved in $OUTDIR/buildstats-test13" - mv tmp/buildstats $OUTDIR/buildstats-test13 -} - - -# -# Test 2 -# Measure: Wall clock of "bitbake core-image-sato" and size of tmp/dir -# Pre: populated sstate cache - -test2 () { - # Assuming test 1 has run - log "Running Test 2: Measure wall clock of bitbake $IMAGE -c rootfs with sstate" - do_rmtmp - do_sync - bbtime $IMAGE -c rootfs -} - - -# Test 3 -# parsing time metrics -# -# Start with -# i) "rm -rf tmp/cache; time bitbake -p" -# ii) "rm -rf tmp/cache/default-glibc/; time bitbake -p" -# iii) "time bitbake -p" - - -test3 () { - log "Running Test 3: Parsing time metrics (bitbake -p)" - log " Removing tmp/cache && cache" - rm -rf tmp/cache cache - bbtime -p - log " Removing tmp/cache/default-glibc/" - rm -rf tmp/cache/default-glibc/ - bbtime -p - bbtime -p -} - -# -# Test 4 - eSDK -# Measure: eSDK size and installation time -test4 () { - log "Running Test 4: eSDK size and installation time" - bbnotime $IMAGE -c do_populate_sdk_ext - - esdk_installer=(tmp/deploy/sdk/*-toolchain-ext-*.sh) - - if [ ${#esdk_installer[*]} -eq 1 ]; then - s=$((`stat -c %s "$esdk_installer"` / 1024)) - SIZES[(( size_count++ ))]="$s" - log "Download SIZE of eSDK is: $s kB" - - do_sync - time_cmd "$esdk_installer" -y -d "tmp/esdk-deploy" - - s=$((`du -sb "tmp/esdk-deploy" | cut -f1` / 1024)) - SIZES[(( size_count++ ))]="$s" - log "Install SIZE of eSDK is: $s kB" - else - log "ERROR: other than one sdk found (${esdk_installer[*]}), reporting size and time as 0." - SIZES[(( size_count++ ))]="0" - TIMES[(( time_count++ ))]="0" - fi - -} - - -# RUN! - -test1_p1 -test1_p2 -test1_p3 -test2 -test3 -test4 - -# if we got til here write to global results -write_results - -log "All done, cleaning up..." - -do_rmtmp -do_rmsstate diff --git a/import-layers/yocto-poky/scripts/contrib/ddimage b/import-layers/yocto-poky/scripts/contrib/ddimage deleted file mode 100755 index ab929957a..000000000 --- a/import-layers/yocto-poky/scripts/contrib/ddimage +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/sh - -# Default to avoiding the first two disks on typical Linux and Mac OS installs -# Better safe than sorry :-) -BLACKLIST_DEVICES="/dev/sda /dev/sdb /dev/disk1 /dev/disk2" - -# 1MB blocksize -BLOCKSIZE=1048576 - -usage() { - echo "Usage: $(basename $0) IMAGE DEVICE" -} - -image_details() { - IMG=$1 - echo "Image details" - echo "=============" - echo " image: $(basename $IMG)" - # stat format is different on Mac OS and Linux - if [ "$(uname)" = "Darwin" ]; then - echo " size: $(stat -L -f '%z bytes' $IMG)" - echo " modified: $(stat -L -f '%Sm' $IMG)" - else - echo " size: $(stat -L -c '%s bytes' $IMG)" - echo " modified: $(stat -L -c '%y' $IMG)" - fi - echo " type: $(file -L -b $IMG)" - echo "" -} - -device_details() { - DEV=$1 - BLOCK_SIZE=512 - - echo "Device details" - echo "==============" - - # Collect disk info using diskutil on Mac OS - if [ "$(uname)" = "Darwin" ]; then - diskutil info $DEVICE | egrep "(Device Node|Media Name|Total Size)" - return - fi - - # Default / Linux information collection - echo " device: $DEVICE" - if [ -f "/sys/class/block/$DEV/device/vendor" ]; then - echo " vendor: $(cat /sys/class/block/$DEV/device/vendor)" - else - echo " vendor: UNKOWN" - fi - if [ -f "/sys/class/block/$DEV/device/model" ]; then - echo " model: $(cat /sys/class/block/$DEV/device/model)" - else - echo " model: UNKNOWN" - fi - if [ -f "/sys/class/block/$DEV/size" ]; then - echo " size: $(($(cat /sys/class/block/$DEV/size) * $BLOCK_SIZE)) bytes" - else - echo " size: UNKNOWN" - fi - echo "" -} - -if [ $# -ne 2 ]; then - usage - exit 1 -fi - -IMAGE=$1 -DEVICE=$2 - -if [ ! -e "$IMAGE" ]; then - echo "ERROR: Image $IMAGE does not exist" - usage - exit 1 -fi - - -for i in ${BLACKLIST_DEVICES}; do - if [ "$i" = "$DEVICE" ]; then - echo "ERROR: Device $DEVICE is blacklisted" - exit 1 - fi -done - -if [ ! -w "$DEVICE" ]; then - echo "ERROR: Device $DEVICE does not exist or is not writable" - usage - exit 1 -fi - -image_details $IMAGE -device_details $(basename $DEVICE) - -printf "Write $IMAGE to $DEVICE [y/N]? " -read RESPONSE -if [ "$RESPONSE" != "y" ]; then - echo "Write aborted" - exit 0 -fi - -echo "Writing image..." -if which pv >/dev/null 2>&1; then - pv "$IMAGE" | dd of="$DEVICE" bs="$BLOCKSIZE" -else - dd if="$IMAGE" of="$DEVICE" bs="$BLOCKSIZE" -fi -sync diff --git a/import-layers/yocto-poky/scripts/contrib/devtool-stress.py b/import-layers/yocto-poky/scripts/contrib/devtool-stress.py deleted file mode 100755 index d555c51a6..000000000 --- a/import-layers/yocto-poky/scripts/contrib/devtool-stress.py +++ /dev/null @@ -1,256 +0,0 @@ -#!/usr/bin/env python3 - -# devtool stress tester -# -# Written by: Paul Eggleton <paul.eggleton@linux.intel.com> -# -# Copyright 2015 Intel Corporation -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# - -import sys -import os -import os.path -import subprocess -import re -import argparse -import logging -import tempfile -import shutil -import signal -import fnmatch - -scripts_lib_path = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'lib')) -sys.path.insert(0, scripts_lib_path) -import scriptutils -import argparse_oe -logger = scriptutils.logger_create('devtool-stress') - -def select_recipes(args): - import bb.tinfoil - tinfoil = bb.tinfoil.Tinfoil() - tinfoil.prepare(False) - - pkg_pn = tinfoil.cooker.recipecaches[''].pkg_pn - (latest_versions, preferred_versions) = bb.providers.findProviders(tinfoil.config_data, tinfoil.cooker.recipecaches[''], pkg_pn) - - skip_classes = args.skip_classes.split(',') - - recipelist = [] - for pn in sorted(pkg_pn): - pref = preferred_versions[pn] - inherits = [os.path.splitext(os.path.basename(f))[0] for f in tinfoil.cooker.recipecaches[''].inherits[pref[1]]] - for cls in skip_classes: - if cls in inherits: - break - else: - recipelist.append(pn) - - tinfoil.shutdown() - - resume_from = args.resume_from - if resume_from: - if not resume_from in recipelist: - print('%s is not a testable recipe' % resume_from) - return 1 - if args.only: - only = args.only.split(',') - for onlyitem in only: - for pn in recipelist: - if fnmatch.fnmatch(pn, onlyitem): - break - else: - print('%s does not match any testable recipe' % onlyitem) - return 1 - else: - only = None - if args.skip: - skip = args.skip.split(',') - else: - skip = [] - - recipes = [] - for pn in recipelist: - if resume_from: - if pn == resume_from: - resume_from = None - else: - continue - - if args.only: - for item in only: - if fnmatch.fnmatch(pn, item): - break - else: - continue - - skipit = False - for item in skip: - if fnmatch.fnmatch(pn, item): - skipit = True - if skipit: - continue - - recipes.append(pn) - - return recipes - - -def stress_extract(args): - import bb.process - - recipes = select_recipes(args) - - failures = 0 - tmpdir = tempfile.mkdtemp() - os.setpgrp() - try: - for pn in recipes: - sys.stdout.write('Testing %s ' % (pn + ' ').ljust(40, '.')) - sys.stdout.flush() - failed = False - skipped = None - - srctree = os.path.join(tmpdir, pn) - try: - bb.process.run('devtool extract %s %s' % (pn, srctree)) - except bb.process.ExecutionError as exc: - if exc.exitcode == 4: - skipped = 'incompatible' - else: - failed = True - with open('stress_%s_extract.log' % pn, 'w') as f: - f.write(str(exc)) - - if os.path.exists(srctree): - shutil.rmtree(srctree) - - if failed: - print('failed') - failures += 1 - elif skipped: - print('skipped (%s)' % skipped) - else: - print('ok') - except KeyboardInterrupt: - # We want any child processes killed. This is crude, but effective. - os.killpg(0, signal.SIGTERM) - - if failures: - return 1 - else: - return 0 - - -def stress_modify(args): - import bb.process - - recipes = select_recipes(args) - - failures = 0 - tmpdir = tempfile.mkdtemp() - os.setpgrp() - try: - for pn in recipes: - sys.stdout.write('Testing %s ' % (pn + ' ').ljust(40, '.')) - sys.stdout.flush() - failed = False - reset = True - skipped = None - - srctree = os.path.join(tmpdir, pn) - try: - bb.process.run('devtool modify -x %s %s' % (pn, srctree)) - except bb.process.ExecutionError as exc: - if exc.exitcode == 4: - skipped = 'incompatible' - else: - with open('stress_%s_modify.log' % pn, 'w') as f: - f.write(str(exc)) - failed = 'modify' - reset = False - - if not skipped: - if not failed: - try: - bb.process.run('bitbake -c install %s' % pn) - except bb.process.CmdError as exc: - with open('stress_%s_install.log' % pn, 'w') as f: - f.write(str(exc)) - failed = 'build' - if reset: - try: - bb.process.run('devtool reset %s' % pn) - except bb.process.CmdError as exc: - print('devtool reset failed: %s' % str(exc)) - break - - if os.path.exists(srctree): - shutil.rmtree(srctree) - - if failed: - print('failed (%s)' % failed) - failures += 1 - elif skipped: - print('skipped (%s)' % skipped) - else: - print('ok') - except KeyboardInterrupt: - # We want any child processes killed. This is crude, but effective. - os.killpg(0, signal.SIGTERM) - - if failures: - return 1 - else: - return 0 - - -def main(): - parser = argparse_oe.ArgumentParser(description="devtool stress tester", - epilog="Use %(prog)s <subcommand> --help to get help on a specific command") - parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true') - parser.add_argument('-r', '--resume-from', help='Resume from specified recipe', metavar='PN') - parser.add_argument('-o', '--only', help='Only test specified recipes (comma-separated without spaces, wildcards allowed)', metavar='PNLIST') - parser.add_argument('-s', '--skip', help='Skip specified recipes (comma-separated without spaces, wildcards allowed)', metavar='PNLIST', default='gcc-source-*,kernel-devsrc,package-index,perf,meta-world-pkgdata,glibc-locale,glibc-mtrace,glibc-scripts,os-release') - parser.add_argument('-c', '--skip-classes', help='Skip recipes inheriting specified classes (comma-separated) - default %(default)s', metavar='CLASSLIST', default='native,nativesdk,cross,cross-canadian,image,populate_sdk,meta,packagegroup') - subparsers = parser.add_subparsers(title='subcommands', metavar='<subcommand>') - subparsers.required = True - - parser_modify = subparsers.add_parser('modify', - help='Run "devtool modify" followed by a build with bitbake on matching recipes', - description='Runs "devtool modify" followed by a build with bitbake on matching recipes') - parser_modify.set_defaults(func=stress_modify) - - parser_extract = subparsers.add_parser('extract', - help='Run "devtool extract" on matching recipes', - description='Runs "devtool extract" on matching recipes') - parser_extract.set_defaults(func=stress_extract) - - args = parser.parse_args() - - if args.debug: - logger.setLevel(logging.DEBUG) - - import scriptpath - bitbakepath = scriptpath.add_bitbake_lib_path() - if not bitbakepath: - logger.error("Unable to find bitbake by searching parent directory of this script or PATH") - return 1 - logger.debug('Found bitbake path: %s' % bitbakepath) - - ret = args.func(args) - -if __name__ == "__main__": - main() diff --git a/import-layers/yocto-poky/scripts/contrib/dialog-power-control b/import-layers/yocto-poky/scripts/contrib/dialog-power-control deleted file mode 100755 index 7550ea53b..000000000 --- a/import-layers/yocto-poky/scripts/contrib/dialog-power-control +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/sh -# -# Simple script to show a manual power prompt for when you want to use -# automated hardware testing with testimage.bbclass but you don't have a -# web-enabled power strip or similar to do the power on/off/cycle. -# -# You can enable it by enabling testimage (see the Yocto Project -# Development manual "Performing Automated Runtime Testing" section) -# and setting the following in your local.conf: -# -# TEST_POWERCONTROL_CMD = "${COREBASE}/scripts/contrib/dialog-power-control" -# - -PROMPT="" -while true; do - case $1 in - on) - PROMPT="Please turn device power on";; - off) - PROMPT="Please turn device power off";; - cycle) - PROMPT="Please click Done, then turn the device power off then on";; - "") - break;; - esac - shift -done - -if [ "$PROMPT" = "" ] ; then - echo "ERROR: no power action specified on command line" - exit 2 -fi - -if [ "`which kdialog 2>/dev/null`" != "" ] ; then - DIALOGUTIL="kdialog" -elif [ "`which zenity 2>/dev/null`" != "" ] ; then - DIALOGUTIL="zenity" -else - echo "ERROR: couldn't find program to display a message, install kdialog or zenity" - exit 3 -fi - -if [ "$DIALOGUTIL" = "kdialog" ] ; then - kdialog --yesno "$PROMPT" --title "TestImage Power Control" --yes-label "Done" --no-label "Cancel test" -elif [ "$DIALOGUTIL" = "zenity" ] ; then - zenity --question --text="$PROMPT" --title="TestImage Power Control" --ok-label="Done" --cancel-label="Cancel test" -fi - -if [ "$?" != "0" ] ; then - echo "User cancelled test at power prompt" - exit 1 -fi - diff --git a/import-layers/yocto-poky/scripts/contrib/documentation-audit.sh b/import-layers/yocto-poky/scripts/contrib/documentation-audit.sh deleted file mode 100755 index 2144aac93..000000000 --- a/import-layers/yocto-poky/scripts/contrib/documentation-audit.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/bin/bash -# -# Perform an audit of which packages provide documentation and which -# are missing -doc packages. -# -# Setup requirements: be sure to be building for MACHINE=qemux86. Run -# this script after source'ing the build environment script, so you're -# running it from build/ directory. -# -# Maintainer: Scott Garman <scott.a.garman@intel.com> - -REPORT_DOC_SIMPLE="documentation_exists.txt" -REPORT_DOC_DETAIL="documentation_exists_detail.txt" -REPORT_MISSING_SIMPLE="documentation_missing.txt" -REPORT_MISSING_DETAIL="documentation_missing_detail.txt" -REPORT_BUILD_ERRORS="build_errors.txt" - -rm -rf $REPORT_DOC_SIMPLE $REPORT_DOC_DETAIL $REPORT_MISSING_SIMPLE $REPORT_MISSING_DETAIL - -BITBAKE=`which bitbake` -if [ -z "$BITBAKE" ]; then - echo "Error: bitbake command not found." - echo "Did you forget to source the build environment script?" - exit 1 -fi - -echo "REMINDER: you need to build for MACHINE=qemux86 or you won't get useful results" -echo "REMINDER: you need to set LICENSE_FLAGS_WHITELIST appropriately in local.conf or " -echo " you'll get false positives. For example, LICENSE_FLAGS_WHITELIST = \"Commercial\"" - -for pkg in `bitbake -s | awk '{ print \$1 }'`; do - if [[ "$pkg" == "Loading" || "$pkg" == "Loaded" || - "$pkg" == "Recipe" || - "$pkg" == "Parsing" || "$pkg" == "Package" || - "$pkg" == "NOTE:" || "$pkg" == "WARNING:" || - "$pkg" == "done." || "$pkg" == "===========" ]] - then - # Skip initial bitbake output - continue - fi - if [[ "$pkg" =~ -native$ || "$pkg" =~ -nativesdk$ || - "$pkg" =~ -cross-canadian ]]; then - # Skip native/nativesdk/cross-canadian recipes - continue - fi - if [[ "$pkg" =~ ^meta- || "$pkg" =~ ^packagegroup- || "$pkg" =~ -image ]]; then - # Skip meta, task and image recipes - continue - fi - if [[ "$pkg" =~ ^glibc- || "$pkg" =~ ^libiconv$ || - "$pkg" =~ -toolchain$ || "$pkg" =~ ^package-index$ || - "$pkg" =~ ^linux- || "$pkg" =~ ^adt-installer$ || - "$pkg" =~ ^eds-tools$ || "$pkg" =~ ^external-python-tarball$ || - "$pkg" =~ ^qt4-embedded$ || "$pkg" =~ ^qt-mobility ]]; then - # Skip glibc, libiconv, -toolchain, and other recipes known - # to cause build conflicts or trigger false positives. - continue - fi - - echo "Building package $pkg..." - bitbake $pkg > /dev/null - if [ $? -ne 0 ]; then - echo "There was an error building package $pkg" >> "$REPORT_MISSING_DETAIL" - echo "$pkg" >> $REPORT_BUILD_ERRORS - - # Do not skip the remaining tests, as sometimes the - # exit status is 1 due to QA errors, and we can still - # perform the -doc checks. - fi - - echo "$pkg built successfully, checking for a documentation package..." - WORKDIR=`bitbake -e $pkg | grep ^WORKDIR | awk -F '=' '{ print \$2 }' | awk -F '"' '{ print \$2 }'` - FIND_DOC_PKG=`find $WORKDIR/packages-split/*-doc -maxdepth 0 -type d` - if [ -z "$FIND_DOC_PKG" ]; then - # No -doc package was generated: - echo "No -doc package: $pkg" >> "$REPORT_MISSING_DETAIL" - echo "$pkg" >> $REPORT_MISSING_SIMPLE - continue - fi - - FIND_DOC_FILES=`find $FIND_DOC_PKG -type f` - if [ -z "$FIND_DOC_FILES" ]; then - # No files shipped with the -doc package: - echo "No files shipped with the -doc package: $pkg" >> "$REPORT_MISSING_DETAIL" - echo "$pkg" >> $REPORT_MISSING_SIMPLE - continue - fi - - echo "Documentation shipped with $pkg:" >> "$REPORT_DOC_DETAIL" - echo "$FIND_DOC_FILES" >> "$REPORT_DOC_DETAIL" - echo "" >> "$REPORT_DOC_DETAIL" - - echo "$pkg" >> "$REPORT_DOC_SIMPLE" -done diff --git a/import-layers/yocto-poky/scripts/contrib/graph-tool b/import-layers/yocto-poky/scripts/contrib/graph-tool deleted file mode 100755 index 1df5b8c34..000000000 --- a/import-layers/yocto-poky/scripts/contrib/graph-tool +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env python3 - -# Simple graph query utility -# useful for getting answers from .dot files produced by bitbake -g -# -# Written by: Paul Eggleton <paul.eggleton@linux.intel.com> -# -# Copyright 2013 Intel Corporation -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# - -import sys - -def get_path_networkx(dotfile, fromnode, tonode): - try: - import networkx - except ImportError: - print('ERROR: Please install the networkx python module') - sys.exit(1) - - graph = networkx.DiGraph(networkx.nx_pydot.read_dot(dotfile)) - def node_missing(node): - import difflib - close_matches = difflib.get_close_matches(node, graph.nodes(), cutoff=0.7) - if close_matches: - print('ERROR: no node "%s" in graph. Close matches:\n %s' % (node, '\n '.join(close_matches))) - sys.exit(1) - - if not fromnode in graph: - node_missing(fromnode) - if not tonode in graph: - node_missing(tonode) - return networkx.all_simple_paths(graph, source=fromnode, target=tonode) - - -def find_paths(args, usage): - if len(args) < 3: - usage() - sys.exit(1) - - fromnode = args[1] - tonode = args[2] - - path = None - for path in get_path_networkx(args[0], fromnode, tonode): - print(" -> ".join(map(str, path))) - if not path: - print("ERROR: no path from %s to %s in graph" % (fromnode, tonode)) - sys.exit(1) - -def main(): - import optparse - parser = optparse.OptionParser( - usage = '''%prog [options] <command> <arguments> - -Available commands: - find-paths <dotfile> <from> <to> - Find all of the paths between two nodes in a dot graph''') - - #parser.add_option("-d", "--debug", - # help = "Report all SRCREV values, not just ones where AUTOREV has been used", - # action="store_true", dest="debug", default=False) - - options, args = parser.parse_args(sys.argv) - args = args[1:] - - if len(args) < 1: - parser.print_help() - sys.exit(1) - - if args[0] == "find-paths": - find_paths(args[1:], parser.print_help) - else: - parser.print_help() - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/import-layers/yocto-poky/scripts/contrib/list-packageconfig-flags.py b/import-layers/yocto-poky/scripts/contrib/list-packageconfig-flags.py deleted file mode 100755 index 7ce718624..000000000 --- a/import-layers/yocto-poky/scripts/contrib/list-packageconfig-flags.py +++ /dev/null @@ -1,178 +0,0 @@ -#!/usr/bin/env python3 - -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation. -# -# Copyright (C) 2013 Wind River Systems, Inc. -# Copyright (C) 2014 Intel Corporation -# -# - list available recipes which have PACKAGECONFIG flags -# - list available PACKAGECONFIG flags and all affected recipes -# - list all recipes and PACKAGECONFIG information - -import sys -import optparse -import os - - -scripts_path = os.path.abspath(os.path.dirname(os.path.abspath(sys.argv[0]))) -lib_path = os.path.abspath(scripts_path + '/../lib') -sys.path = sys.path + [lib_path] - -import scriptpath - -# For importing the following modules -bitbakepath = scriptpath.add_bitbake_lib_path() -if not bitbakepath: - sys.stderr.write("Unable to find bitbake by searching parent directory of this script or PATH\n") - sys.exit(1) - -import bb.cooker -import bb.providers -import bb.tinfoil - -def get_fnlist(bbhandler, pkg_pn, preferred): - ''' Get all recipe file names ''' - if preferred: - (latest_versions, preferred_versions) = bb.providers.findProviders(bbhandler.config_data, bbhandler.cooker.recipecaches[''], pkg_pn) - - fn_list = [] - for pn in sorted(pkg_pn): - if preferred: - fn_list.append(preferred_versions[pn][1]) - else: - fn_list.extend(pkg_pn[pn]) - - return fn_list - -def get_recipesdata(bbhandler, preferred): - ''' Get data of all available recipes which have PACKAGECONFIG flags ''' - pkg_pn = bbhandler.cooker.recipecaches[''].pkg_pn - - data_dict = {} - for fn in get_fnlist(bbhandler, pkg_pn, preferred): - data = bbhandler.parse_recipe_file(fn) - flags = data.getVarFlags("PACKAGECONFIG") - flags.pop('doc', None) - if flags: - data_dict[fn] = data - - return data_dict - -def collect_pkgs(data_dict): - ''' Collect available pkgs in which have PACKAGECONFIG flags ''' - # pkg_dict = {'pkg1': ['flag1', 'flag2',...]} - pkg_dict = {} - for fn in data_dict: - pkgconfigflags = data_dict[fn].getVarFlags("PACKAGECONFIG") - pkgconfigflags.pop('doc', None) - pkgname = data_dict[fn].getVar("P") - pkg_dict[pkgname] = sorted(pkgconfigflags.keys()) - - return pkg_dict - -def collect_flags(pkg_dict): - ''' Collect available PACKAGECONFIG flags and all affected pkgs ''' - # flag_dict = {'flag': ['pkg1', 'pkg2',...]} - flag_dict = {} - for pkgname, flaglist in pkg_dict.items(): - for flag in flaglist: - if flag in flag_dict: - flag_dict[flag].append(pkgname) - else: - flag_dict[flag] = [pkgname] - - return flag_dict - -def display_pkgs(pkg_dict): - ''' Display available pkgs which have PACKAGECONFIG flags ''' - pkgname_len = len("RECIPE NAME") + 1 - for pkgname in pkg_dict: - if pkgname_len < len(pkgname): - pkgname_len = len(pkgname) - pkgname_len += 1 - - header = '%-*s%s' % (pkgname_len, str("RECIPE NAME"), str("PACKAGECONFIG FLAGS")) - print(header) - print(str("").ljust(len(header), '=')) - for pkgname in sorted(pkg_dict): - print('%-*s%s' % (pkgname_len, pkgname, ' '.join(pkg_dict[pkgname]))) - - -def display_flags(flag_dict): - ''' Display available PACKAGECONFIG flags and all affected pkgs ''' - flag_len = len("PACKAGECONFIG FLAG") + 5 - - header = '%-*s%s' % (flag_len, str("PACKAGECONFIG FLAG"), str("RECIPE NAMES")) - print(header) - print(str("").ljust(len(header), '=')) - - for flag in sorted(flag_dict): - print('%-*s%s' % (flag_len, flag, ' '.join(sorted(flag_dict[flag])))) - -def display_all(data_dict): - ''' Display all pkgs and PACKAGECONFIG information ''' - print(str("").ljust(50, '=')) - for fn in data_dict: - print('%s' % data_dict[fn].getVar("P")) - print(fn) - packageconfig = data_dict[fn].getVar("PACKAGECONFIG") or '' - if packageconfig.strip() == '': - packageconfig = 'None' - print('PACKAGECONFIG %s' % packageconfig) - - for flag,flag_val in data_dict[fn].getVarFlags("PACKAGECONFIG").items(): - if flag == "doc": - continue - print('PACKAGECONFIG[%s] %s' % (flag, flag_val)) - print('') - -def main(): - pkg_dict = {} - flag_dict = {} - - # Collect and validate input - parser = optparse.OptionParser( - description = "Lists recipes and PACKAGECONFIG flags. Without -a or -f, recipes and their available PACKAGECONFIG flags are listed.", - usage = """ - %prog [options]""") - - parser.add_option("-f", "--flags", - help = "list available PACKAGECONFIG flags and affected recipes", - action="store_const", dest="listtype", const="flags", default="recipes") - parser.add_option("-a", "--all", - help = "list all recipes and PACKAGECONFIG information", - action="store_const", dest="listtype", const="all") - parser.add_option("-p", "--preferred-only", - help = "where multiple recipe versions are available, list only the preferred version", - action="store_true", dest="preferred", default=False) - - options, args = parser.parse_args(sys.argv) - - with bb.tinfoil.Tinfoil() as bbhandler: - bbhandler.prepare() - print("Gathering recipe data...") - data_dict = get_recipesdata(bbhandler, options.preferred) - - if options.listtype == 'flags': - pkg_dict = collect_pkgs(data_dict) - flag_dict = collect_flags(pkg_dict) - display_flags(flag_dict) - elif options.listtype == 'recipes': - pkg_dict = collect_pkgs(data_dict) - display_pkgs(pkg_dict) - elif options.listtype == 'all': - display_all(data_dict) - -if __name__ == "__main__": - main() diff --git a/import-layers/yocto-poky/scripts/contrib/mkefidisk.sh b/import-layers/yocto-poky/scripts/contrib/mkefidisk.sh deleted file mode 100755 index ac4ec9c7f..000000000 --- a/import-layers/yocto-poky/scripts/contrib/mkefidisk.sh +++ /dev/null @@ -1,464 +0,0 @@ -#!/bin/sh -# -# Copyright (c) 2012, Intel Corporation. -# All rights reserved. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See -# the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# - -LANG=C - -echo -echo "WARNING: This script is deprecated and will be removed soon." -echo "Please consider using wic EFI images instead." -echo - -# Set to 1 to enable additional output -DEBUG=0 -OUT="/dev/null" - -# -# Defaults -# -# 20 Mb for the boot partition -BOOT_SIZE=20 -# 5% for swap -SWAP_RATIO=5 - -# Cleanup after die() -cleanup() { - debug "Syncing and unmounting devices" - # Unmount anything we mounted - unmount $ROOTFS_MNT || error "Failed to unmount $ROOTFS_MNT" - unmount $BOOTFS_MNT || error "Failed to unmount $BOOTFS_MNT" - unmount $HDDIMG_ROOTFS_MNT || error "Failed to unmount $HDDIMG_ROOTFS_MNT" - unmount $HDDIMG_MNT || error "Failed to unmount $HDDIMG_MNT" - - # Remove the TMPDIR - debug "Removing temporary files" - if [ -d "$TMPDIR" ]; then - rm -rf $TMPDIR || error "Failed to remove $TMPDIR" - fi -} - -trap 'die "Signal Received, Aborting..."' HUP INT TERM - -# Logging routines -WARNINGS=0 -ERRORS=0 -CLEAR="$(tput sgr0)" -INFO="$(tput bold)" -RED="$(tput setaf 1)$(tput bold)" -GREEN="$(tput setaf 2)$(tput bold)" -YELLOW="$(tput setaf 3)$(tput bold)" -info() { - echo "${INFO}$1${CLEAR}" -} -error() { - ERRORS=$((ERRORS+1)) - echo "${RED}$1${CLEAR}" -} -warn() { - WARNINGS=$((WARNINGS+1)) - echo "${YELLOW}$1${CLEAR}" -} -success() { - echo "${GREEN}$1${CLEAR}" -} -die() { - error "$1" - cleanup - exit 1 -} -debug() { - if [ $DEBUG -eq 1 ]; then - echo "$1" - fi -} - -usage() { - echo "Usage: $(basename $0) [-v] DEVICE HDDIMG TARGET_DEVICE" - echo " -v: Verbose debug" - echo " DEVICE: The device to write the image to, e.g. /dev/sdh" - echo " HDDIMG: The hddimg file to generate the efi disk from" - echo " TARGET_DEVICE: The device the target will boot from, e.g. /dev/mmcblk0" -} - -image_details() { - IMG=$1 - info "Image details" - echo " image: $(stat --printf '%N\n' $IMG)" - echo " size: $(stat -L --printf '%s bytes\n' $IMG)" - echo " modified: $(stat -L --printf '%y\n' $IMG)" - echo " type: $(file -L -b $IMG)" - echo "" -} - -device_details() { - DEV=$1 - BLOCK_SIZE=512 - - info "Device details" - echo " device: $DEVICE" - if [ -f "/sys/class/block/$DEV/device/vendor" ]; then - echo " vendor: $(cat /sys/class/block/$DEV/device/vendor)" - else - echo " vendor: UNKOWN" - fi - if [ -f "/sys/class/block/$DEV/device/model" ]; then - echo " model: $(cat /sys/class/block/$DEV/device/model)" - else - echo " model: UNKNOWN" - fi - if [ -f "/sys/class/block/$DEV/size" ]; then - echo " size: $(($(cat /sys/class/block/$DEV/size) * $BLOCK_SIZE)) bytes" - else - echo " size: UNKNOWN" - fi - echo "" -} - -unmount_device() { - grep -q $DEVICE /proc/mounts - if [ $? -eq 0 ]; then - warn "$DEVICE listed in /proc/mounts, attempting to unmount" - umount $DEVICE* 2>/dev/null - return $? - fi - return 0 -} - -unmount() { - if [ "$1" = "" ] ; then - return 0 - fi - grep -q $1 /proc/mounts - if [ $? -eq 0 ]; then - debug "Unmounting $1" - umount $1 - return $? - fi - return 0 -} - -# -# Parse and validate arguments -# -if [ $# -lt 3 ] || [ $# -gt 4 ]; then - if [ $# -eq 1 ]; then - AVAILABLE_DISK=`lsblk | grep "disk" | cut -f 1 -d " "` - X=0 - for disk in `echo $AVAILABLE_DISK`; do - mounted=`lsblk /dev/$disk | awk {'print $7'} | sed "s/MOUNTPOINT//"` - if [ -z "$mounted" ]; then - UNMOUNTED_AVAILABLES="$UNMOUNTED_AVAILABLES /dev/$disk" - info "$X - /dev/$disk" - X=`expr $X + 1` - fi - done - if [ $X -eq 0 ]; then - die "No unmounted device found." - fi - read -p "Choose unmounted device number: " DISK_NUMBER - X=0 - for line in `echo $UNMOUNTED_AVAILABLES`; do - if [ $DISK_NUMBER -eq $X ]; then - DISK_TO_BE_FLASHED=$line - break - else - X=`expr $X + 1` - fi - done - if [ -z "$DISK_TO_BE_FLASHED" ]; then - die "Option \"$DISK_NUMBER\" is invalid. Choose a valid option" - else - if [ -z `echo $DISK_TO_BE_FLASHED | grep "mmc"` ]; then - TARGET_TO_BE_BOOT="/dev/sda" - else - TARGET_TO_BE_BOOT="/dev/mmcblk0" - fi - fi - echo "" - echo "Choose a name of the device that will be boot from" - echo -n "Recommended name is: " - info "$TARGET_TO_BE_BOOT" - read -p "Is target device okay? [y/N]: " RESPONSE - if [ "$RESPONSE" != "y" ]; then - read -p "Choose target device name: " TARGET_TO_BE_BOOT - fi - echo "" - if [ -z "$TARGET_TO_BE_BOOT" ]; then - die "Error: choose a valid target name" - fi - else - usage - exit 1 - fi -fi - -if [ "$1" = "-v" ]; then - DEBUG=1 - OUT="1" - shift -fi - -if [ -z "$AVAILABLE_DISK" ]; then - DEVICE=$1 - HDDIMG=$2 - TARGET_DEVICE=$3 -else - DEVICE=$DISK_TO_BE_FLASHED - HDDIMG=$1 - TARGET_DEVICE=$TARGET_TO_BE_BOOT -fi - -LINK=$(readlink $DEVICE) -if [ $? -eq 0 ]; then - DEVICE="$LINK" -fi - -if [ ! -w "$DEVICE" ]; then - usage - if [ ! -e "${DEVICE}" ] ; then - die "Device $DEVICE cannot be found" - else - die "Device $DEVICE is not writable (need to run under sudo?)" - fi -fi - -if [ ! -e "$HDDIMG" ]; then - usage - die "HDDIMG $HDDIMG does not exist" -fi - -# -# Ensure the hddimg is not mounted -# -unmount "$HDDIMG" || die "Failed to unmount $HDDIMG" - -# -# Check if any $DEVICE partitions are mounted -# -unmount_device || die "Failed to unmount $DEVICE" - -# -# Confirm device with user -# -image_details $HDDIMG -device_details $(basename $DEVICE) -echo -n "${INFO}Prepare EFI image on $DEVICE [y/N]?${CLEAR} " -read RESPONSE -if [ "$RESPONSE" != "y" ]; then - echo "Image creation aborted" - exit 0 -fi - - -# -# Prepare the temporary working space -# -TMPDIR=$(mktemp -d mkefidisk-XXX) || die "Failed to create temporary mounting directory." -HDDIMG_MNT=$TMPDIR/hddimg -HDDIMG_ROOTFS_MNT=$TMPDIR/hddimg_rootfs -ROOTFS_MNT=$TMPDIR/rootfs -BOOTFS_MNT=$TMPDIR/bootfs -mkdir $HDDIMG_MNT || die "Failed to create $HDDIMG_MNT" -mkdir $HDDIMG_ROOTFS_MNT || die "Failed to create $HDDIMG_ROOTFS_MNT" -mkdir $ROOTFS_MNT || die "Failed to create $ROOTFS_MNT" -mkdir $BOOTFS_MNT || die "Failed to create $BOOTFS_MNT" - - -# -# Partition $DEVICE -# -DEVICE_SIZE=$(parted -s $DEVICE unit mb print | grep ^Disk | cut -d" " -f 3 | sed -e "s/MB//") -# If the device size is not reported there may not be a valid label -if [ "$DEVICE_SIZE" = "" ] ; then - parted -s $DEVICE mklabel msdos || die "Failed to create MSDOS partition table" - DEVICE_SIZE=$(parted -s $DEVICE unit mb print | grep ^Disk | cut -d" " -f 3 | sed -e "s/MB//") -fi -SWAP_SIZE=$((DEVICE_SIZE*SWAP_RATIO/100)) -ROOTFS_SIZE=$((DEVICE_SIZE-BOOT_SIZE-SWAP_SIZE)) -ROOTFS_START=$((BOOT_SIZE)) -ROOTFS_END=$((ROOTFS_START+ROOTFS_SIZE)) -SWAP_START=$((ROOTFS_END)) - -# MMC devices use a partition prefix character 'p' -PART_PREFIX="" -if [ ! "${DEVICE#/dev/mmcblk}" = "${DEVICE}" ] || [ ! "${DEVICE#/dev/loop}" = "${DEVICE}" ]; then - PART_PREFIX="p" -fi -BOOTFS=$DEVICE${PART_PREFIX}1 -ROOTFS=$DEVICE${PART_PREFIX}2 -SWAP=$DEVICE${PART_PREFIX}3 - -TARGET_PART_PREFIX="" -if [ ! "${TARGET_DEVICE#/dev/mmcblk}" = "${TARGET_DEVICE}" ]; then - TARGET_PART_PREFIX="p" -fi -TARGET_ROOTFS=$TARGET_DEVICE${TARGET_PART_PREFIX}2 -TARGET_SWAP=$TARGET_DEVICE${TARGET_PART_PREFIX}3 - -echo "" -info "Boot partition size: $BOOT_SIZE MB ($BOOTFS)" -info "ROOTFS partition size: $ROOTFS_SIZE MB ($ROOTFS)" -info "Swap partition size: $SWAP_SIZE MB ($SWAP)" -echo "" - -# Use MSDOS by default as GPT cannot be reliably distributed in disk image form -# as it requires the backup table to be on the last block of the device, which -# of course varies from device to device. - -info "Partitioning installation media ($DEVICE)" - -debug "Deleting partition table on $DEVICE" -dd if=/dev/zero of=$DEVICE bs=512 count=2 >$OUT 2>&1 || die "Failed to zero beginning of $DEVICE" - -debug "Creating new partition table (MSDOS) on $DEVICE" -parted -s $DEVICE mklabel msdos >$OUT 2>&1 || die "Failed to create MSDOS partition table" - -debug "Creating boot partition on $BOOTFS" -parted -s $DEVICE mkpart primary 0% $BOOT_SIZE >$OUT 2>&1 || die "Failed to create BOOT partition" - -debug "Enabling boot flag on $BOOTFS" -parted -s $DEVICE set 1 boot on >$OUT 2>&1 || die "Failed to enable boot flag" - -debug "Creating ROOTFS partition on $ROOTFS" -parted -s $DEVICE mkpart primary $ROOTFS_START $ROOTFS_END >$OUT 2>&1 || die "Failed to create ROOTFS partition" - -debug "Creating swap partition on $SWAP" -parted -s $DEVICE mkpart primary $SWAP_START 100% >$OUT 2>&1 || die "Failed to create SWAP partition" - -if [ $DEBUG -eq 1 ]; then - parted -s $DEVICE print -fi - - -# -# Check if any $DEVICE partitions are mounted after partitioning -# -unmount_device || die "Failed to unmount $DEVICE partitions" - - -# -# Format $DEVICE partitions -# -info "Formatting partitions" -debug "Formatting $BOOTFS as vfat" -if [ ! "${DEVICE#/dev/loop}" = "${DEVICE}" ]; then - mkfs.vfat -I $BOOTFS -n "EFI" >$OUT 2>&1 || die "Failed to format $BOOTFS" -else - mkfs.vfat $BOOTFS -n "EFI" >$OUT 2>&1 || die "Failed to format $BOOTFS" -fi - -debug "Formatting $ROOTFS as ext3" -mkfs.ext3 -F $ROOTFS -L "ROOT" >$OUT 2>&1 || die "Failed to format $ROOTFS" - -debug "Formatting swap partition ($SWAP)" -mkswap $SWAP >$OUT 2>&1 || die "Failed to prepare swap" - - -# -# Installing to $DEVICE -# -debug "Mounting images and device in preparation for installation" -mount -o ro,loop $HDDIMG $HDDIMG_MNT >$OUT 2>&1 || error "Failed to mount $HDDIMG" -mount -o ro,loop $HDDIMG_MNT/rootfs.img $HDDIMG_ROOTFS_MNT >$OUT 2>&1 || error "Failed to mount rootfs.img" -mount $ROOTFS $ROOTFS_MNT >$OUT 2>&1 || error "Failed to mount $ROOTFS on $ROOTFS_MNT" -mount $BOOTFS $BOOTFS_MNT >$OUT 2>&1 || error "Failed to mount $BOOTFS on $BOOTFS_MNT" - -info "Preparing boot partition" -EFIDIR="$BOOTFS_MNT/EFI/BOOT" -cp $HDDIMG_MNT/vmlinuz $BOOTFS_MNT >$OUT 2>&1 || error "Failed to copy vmlinuz" -# Copy the efi loader and configs (booti*.efi and grub.cfg if it exists) -cp -r $HDDIMG_MNT/EFI $BOOTFS_MNT >$OUT 2>&1 || error "Failed to copy EFI dir" -# Silently ignore a missing systemd-boot loader dir (we might just be a GRUB image) -cp -r $HDDIMG_MNT/loader $BOOTFS_MNT >$OUT 2>&1 - -# Update the boot loaders configurations for an installed image -# Remove any existing root= kernel parameters and: -# o Add a root= parameter with the target rootfs -# o Specify ro so fsck can be run during boot -# o Specify rootwait in case the target media is an asyncronous block device -# such as MMC or USB disks -# o Specify "quiet" to minimize boot time when using slow serial consoles - -# Look for a GRUB installation -GRUB_CFG="$EFIDIR/grub.cfg" -if [ -e "$GRUB_CFG" ]; then - info "Configuring GRUB" - # Delete the install entry - sed -i "/menuentry 'install'/,/^}/d" $GRUB_CFG - # Delete the initrd lines - sed -i "/initrd /d" $GRUB_CFG - # Delete any LABEL= strings - sed -i "s/ LABEL=[^ ]*/ /" $GRUB_CFG - - sed -i "s@ root=[^ ]*@ @" $GRUB_CFG - sed -i "s@vmlinuz @vmlinuz root=$TARGET_ROOTFS ro rootwait console=ttyS0 console=tty0 @" $GRUB_CFG -fi - -# Look for a systemd-boot installation -SYSTEMD_BOOT_ENTRIES="$BOOTFS_MNT/loader/entries" -SYSTEMD_BOOT_CFG="$SYSTEMD_BOOT_ENTRIES/boot.conf" -if [ -d "$SYSTEMD_BOOT_ENTRIES" ]; then - info "Configuring SystemD-boot" - # remove the install target if it exists - rm $SYSTEMD_BOOT_ENTRIES/install.conf >$OUT 2>&1 - - if [ ! -e "$SYSTEMD_BOOT_CFG" ]; then - echo "ERROR: $SYSTEMD_BOOT_CFG not found" - fi - - sed -i "/initrd /d" $SYSTEMD_BOOT_CFG - sed -i "s@ root=[^ ]*@ @" $SYSTEMD_BOOT_CFG - sed -i "s@options *LABEL=boot @options LABEL=Boot root=$TARGET_ROOTFS ro rootwait console=ttyS0 console=tty0 @" $SYSTEMD_BOOT_CFG -fi - -# Ensure we have at least one EFI bootloader configured -if [ ! -e $GRUB_CFG ] && [ ! -e $SYSTEMD_BOOT_CFG ]; then - die "No EFI bootloader configuration found" -fi - - -info "Copying ROOTFS files (this may take a while)" -cp -a $HDDIMG_ROOTFS_MNT/* $ROOTFS_MNT >$OUT 2>&1 || die "Root FS copy failed" - -echo "$TARGET_SWAP swap swap defaults 0 0" >> $ROOTFS_MNT/etc/fstab - -# We dont want udev to mount our root device while we're booting... -if [ -d $ROOTFS_MNT/etc/udev/ ] ; then - echo "$TARGET_DEVICE" >> $ROOTFS_MNT/etc/udev/mount.blacklist -fi - -# Add startup.nsh script for automated boot -printf "fs0:\%s\BOOT\%s\n" "EFI" "bootx64.efi" > $BOOTFS_MNT/startup.nsh - - -# Call cleanup to unmount devices and images and remove the TMPDIR -cleanup - -echo "" -if [ $WARNINGS -ne 0 ] && [ $ERRORS -eq 0 ]; then - echo "${YELLOW}Installation completed with warnings${CLEAR}" - echo "${YELLOW}Warnings: $WARNINGS${CLEAR}" -elif [ $ERRORS -ne 0 ]; then - echo "${RED}Installation encountered errors${CLEAR}" - echo "${RED}Errors: $ERRORS${CLEAR}" - echo "${YELLOW}Warnings: $WARNINGS${CLEAR}" -else - success "Installation completed successfully" -fi -echo "" diff --git a/import-layers/yocto-poky/scripts/contrib/oe-build-perf-report-email.py b/import-layers/yocto-poky/scripts/contrib/oe-build-perf-report-email.py deleted file mode 100755 index 913847bbe..000000000 --- a/import-layers/yocto-poky/scripts/contrib/oe-build-perf-report-email.py +++ /dev/null @@ -1,282 +0,0 @@ -#!/usr/bin/python3 -# -# Send build performance test report emails -# -# Copyright (c) 2017, Intel Corporation. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms and conditions of the GNU General Public License, -# version 2, as published by the Free Software Foundation. -# -# This program is distributed in the hope it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -import argparse -import base64 -import logging -import os -import pwd -import re -import shutil -import smtplib -import socket -import subprocess -import sys -import tempfile -from email.mime.image import MIMEImage -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText - - -# Setup logging -logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") -log = logging.getLogger('oe-build-perf-report') - - -# Find js scaper script -SCRAPE_JS = os.path.join(os.path.dirname(__file__), '..', 'lib', 'build_perf', - 'scrape-html-report.js') -if not os.path.isfile(SCRAPE_JS): - log.error("Unableto find oe-build-perf-report-scrape.js") - sys.exit(1) - - -class ReportError(Exception): - """Local errors""" - pass - - -def check_utils(): - """Check that all needed utils are installed in the system""" - missing = [] - for cmd in ('phantomjs', 'optipng'): - if not shutil.which(cmd): - missing.append(cmd) - if missing: - log.error("The following tools are missing: %s", ' '.join(missing)) - sys.exit(1) - - -def parse_args(argv): - """Parse command line arguments""" - description = """Email build perf test report""" - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description=description) - - parser.add_argument('--debug', '-d', action='store_true', - help="Verbose logging") - parser.add_argument('--quiet', '-q', action='store_true', - help="Only print errors") - parser.add_argument('--to', action='append', - help="Recipients of the email") - parser.add_argument('--cc', action='append', - help="Carbon copy recipients of the email") - parser.add_argument('--bcc', action='append', - help="Blind carbon copy recipients of the email") - parser.add_argument('--subject', default="Yocto build perf test report", - help="Email subject") - parser.add_argument('--outdir', '-o', - help="Store files in OUTDIR. Can be used to preserve " - "the email parts") - parser.add_argument('--text', - help="Plain text message") - parser.add_argument('--html', - help="HTML peport generated by oe-build-perf-report") - parser.add_argument('--phantomjs-args', action='append', - help="Extra command line arguments passed to PhantomJS") - - args = parser.parse_args(argv) - - if not args.html and not args.text: - parser.error("Please specify --html and/or --text") - - return args - - -def decode_png(infile, outfile): - """Parse/decode/optimize png data from a html element""" - with open(infile) as f: - raw_data = f.read() - - # Grab raw base64 data - b64_data = re.sub('^.*href="data:image/png;base64,', '', raw_data, 1) - b64_data = re.sub('">.+$', '', b64_data, 1) - - # Replace file with proper decoded png - with open(outfile, 'wb') as f: - f.write(base64.b64decode(b64_data)) - - subprocess.check_output(['optipng', outfile], stderr=subprocess.STDOUT) - - -def mangle_html_report(infile, outfile, pngs): - """Mangle html file into a email compatible format""" - paste = True - png_dir = os.path.dirname(outfile) - with open(infile) as f_in: - with open(outfile, 'w') as f_out: - for line in f_in.readlines(): - stripped = line.strip() - # Strip out scripts - if stripped == '<!--START-OF-SCRIPTS-->': - paste = False - elif stripped == '<!--END-OF-SCRIPTS-->': - paste = True - elif paste: - if re.match('^.+href="data:image/png;base64', stripped): - # Strip out encoded pngs (as they're huge in size) - continue - elif 'www.gstatic.com' in stripped: - # HACK: drop references to external static pages - continue - - # Replace charts with <img> elements - match = re.match('<div id="(?P<id>\w+)"', stripped) - if match and match.group('id') in pngs: - f_out.write('<img src="cid:{}"\n'.format(match.group('id'))) - else: - f_out.write(line) - - -def scrape_html_report(report, outdir, phantomjs_extra_args=None): - """Scrape html report into a format sendable by email""" - tmpdir = tempfile.mkdtemp(dir='.') - log.debug("Using tmpdir %s for phantomjs output", tmpdir) - - if not os.path.isdir(outdir): - os.mkdir(outdir) - if os.path.splitext(report)[1] not in ('.html', '.htm'): - raise ReportError("Invalid file extension for report, needs to be " - "'.html' or '.htm'") - - try: - log.info("Scraping HTML report with PhangomJS") - extra_args = phantomjs_extra_args if phantomjs_extra_args else [] - subprocess.check_output(['phantomjs', '--debug=true'] + extra_args + - [SCRAPE_JS, report, tmpdir], - stderr=subprocess.STDOUT) - - pngs = [] - images = [] - for fname in os.listdir(tmpdir): - base, ext = os.path.splitext(fname) - if ext == '.png': - log.debug("Decoding %s", fname) - decode_png(os.path.join(tmpdir, fname), - os.path.join(outdir, fname)) - pngs.append(base) - images.append(fname) - elif ext in ('.html', '.htm'): - report_file = fname - else: - log.warning("Unknown file extension: '%s'", ext) - #shutil.move(os.path.join(tmpdir, fname), outdir) - - log.debug("Mangling html report file %s", report_file) - mangle_html_report(os.path.join(tmpdir, report_file), - os.path.join(outdir, report_file), pngs) - return (os.path.join(outdir, report_file), - [os.path.join(outdir, i) for i in images]) - finally: - shutil.rmtree(tmpdir) - -def send_email(text_fn, html_fn, image_fns, subject, recipients, copy=[], - blind_copy=[]): - """Send email""" - # Generate email message - text_msg = html_msg = None - if text_fn: - with open(text_fn) as f: - text_msg = MIMEText("Yocto build performance test report.\n" + - f.read(), 'plain') - if html_fn: - html_msg = msg = MIMEMultipart('related') - with open(html_fn) as f: - html_msg.attach(MIMEText(f.read(), 'html')) - for img_fn in image_fns: - # Expect that content id is same as the filename - cid = os.path.splitext(os.path.basename(img_fn))[0] - with open(img_fn, 'rb') as f: - image_msg = MIMEImage(f.read()) - image_msg['Content-ID'] = '<{}>'.format(cid) - html_msg.attach(image_msg) - - if text_msg and html_msg: - msg = MIMEMultipart('alternative') - msg.attach(text_msg) - msg.attach(html_msg) - elif text_msg: - msg = text_msg - elif html_msg: - msg = html_msg - else: - raise ReportError("Neither plain text nor html body specified") - - pw_data = pwd.getpwuid(os.getuid()) - full_name = pw_data.pw_gecos.split(',')[0] - email = os.environ.get('EMAIL', - '{}@{}'.format(pw_data.pw_name, socket.getfqdn())) - msg['From'] = "{} <{}>".format(full_name, email) - msg['To'] = ', '.join(recipients) - if copy: - msg['Cc'] = ', '.join(copy) - if blind_copy: - msg['Bcc'] = ', '.join(blind_copy) - msg['Subject'] = subject - - # Send email - with smtplib.SMTP('localhost') as smtp: - smtp.send_message(msg) - - -def main(argv=None): - """Script entry point""" - args = parse_args(argv) - if args.quiet: - log.setLevel(logging.ERROR) - if args.debug: - log.setLevel(logging.DEBUG) - - check_utils() - - if args.outdir: - outdir = args.outdir - if not os.path.exists(outdir): - os.mkdir(outdir) - else: - outdir = tempfile.mkdtemp(dir='.') - - try: - log.debug("Storing email parts in %s", outdir) - html_report = images = None - if args.html: - html_report, images = scrape_html_report(args.html, outdir, - args.phantomjs_args) - - if args.to: - log.info("Sending email to %s", ', '.join(args.to)) - if args.cc: - log.info("Copying to %s", ', '.join(args.cc)) - if args.bcc: - log.info("Blind copying to %s", ', '.join(args.bcc)) - send_email(args.text, html_report, images, args.subject, - args.to, args.cc, args.bcc) - except subprocess.CalledProcessError as err: - log.error("%s, with output:\n%s", str(err), err.output.decode()) - return 1 - except ReportError as err: - log.error(err) - return 1 - finally: - if not args.outdir: - log.debug("Wiping %s", outdir) - shutil.rmtree(outdir) - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/import-layers/yocto-poky/scripts/contrib/patchreview.py b/import-layers/yocto-poky/scripts/contrib/patchreview.py deleted file mode 100755 index 4e3e73c7a..000000000 --- a/import-layers/yocto-poky/scripts/contrib/patchreview.py +++ /dev/null @@ -1,211 +0,0 @@ -#! /usr/bin/env python3 - -# TODO -# - option to just list all broken files -# - test suite -# - validate signed-off-by - - -class Result: - # Whether the patch has an Upstream-Status or not - missing_upstream_status = False - # If the Upstream-Status tag is malformed in some way (string for bad bit) - malformed_upstream_status = None - # If the Upstream-Status value is unknown (boolean) - unknown_upstream_status = False - # The upstream status value (Pending, etc) - upstream_status = None - # Whether the patch has a Signed-off-by or not - missing_sob = False - # Whether the Signed-off-by tag is malformed in some way - malformed_sob = False - # The Signed-off-by tag value - sob = None - # Whether a patch looks like a CVE but doesn't have a CVE tag - missing_cve = False - -def blame_patch(patch): - """ - From a patch filename, return a list of "commit summary (author name <author - email>)" strings representing the history. - """ - import subprocess - return subprocess.check_output(("git", "log", - "--follow", "--find-renames", "--diff-filter=A", - "--format=%s (%aN <%aE>)", - "--", patch)).decode("utf-8").splitlines() - -def patchreview(patches): - import re - - # General pattern: start of line, optional whitespace, tag with optional - # hyphen or spaces, maybe a colon, some whitespace, then the value, all case - # insensitive. - sob_re = re.compile(r"^[\t ]*(Signed[-_ ]off[-_ ]by:?)[\t ]*(.+)", re.IGNORECASE | re.MULTILINE) - status_re = re.compile(r"^[\t ]*(Upstream[-_ ]Status:?)[\t ]*(\w*)", re.IGNORECASE | re.MULTILINE) - status_values = ("accepted", "pending", "inappropriate", "backport", "submitted", "denied") - cve_tag_re = re.compile(r"^[\t ]*(CVE:)[\t ]*(.*)", re.IGNORECASE | re.MULTILINE) - cve_re = re.compile(r"cve-[0-9]{4}-[0-9]{4,6}", re.IGNORECASE) - - results = {} - - for patch in patches: - result = Result() - results[patch] = result - - content = open(patch, encoding='ascii', errors='ignore').read() - - # Find the Signed-off-by tag - match = sob_re.search(content) - if match: - value = match.group(1) - if value != "Signed-off-by:": - result.malformed_sob = value - result.sob = match.group(2) - else: - result.missing_sob = True - - - # Find the Upstream-Status tag - match = status_re.search(content) - if match: - value = match.group(1) - if value != "Upstream-Status:": - result.malformed_upstream_status = value - - value = match.group(2).lower() - # TODO: check case - if value not in status_values: - result.unknown_upstream_status = True - result.upstream_status = value - else: - result.missing_upstream_status = True - - # Check that patches which looks like CVEs have CVE tags - if cve_re.search(patch) or cve_re.search(content): - if not cve_tag_re.search(content): - result.missing_cve = True - # TODO: extract CVE list - - return results - - -def analyse(results, want_blame=False, verbose=True): - """ - want_blame: display blame data for each malformed patch - verbose: display per-file results instead of just summary - """ - - # want_blame requires verbose, so disable blame if we're not verbose - if want_blame and not verbose: - want_blame = False - - total_patches = 0 - missing_sob = 0 - malformed_sob = 0 - missing_status = 0 - malformed_status = 0 - missing_cve = 0 - pending_patches = 0 - - for patch in sorted(results): - r = results[patch] - total_patches += 1 - need_blame = False - - # Build statistics - if r.missing_sob: - missing_sob += 1 - if r.malformed_sob: - malformed_sob += 1 - if r.missing_upstream_status: - missing_status += 1 - if r.malformed_upstream_status or r.unknown_upstream_status: - malformed_status += 1 - if r.missing_cve: - missing_cve += 1 - if r.upstream_status == "pending": - pending_patches += 1 - - # Output warnings - if r.missing_sob: - need_blame = True - if verbose: - print("Missing Signed-off-by tag (%s)" % patch) - # TODO: disable this for now as too much fails - if False and r.malformed_sob: - need_blame = True - if verbose: - print("Malformed Signed-off-by '%s' (%s)" % (r.malformed_sob, patch)) - if r.missing_cve: - need_blame = True - if verbose: - print("Missing CVE tag (%s)" % patch) - if r.missing_upstream_status: - need_blame = True - if verbose: - print("Missing Upstream-Status tag (%s)" % patch) - if r.malformed_upstream_status: - need_blame = True - if verbose: - print("Malformed Upstream-Status '%s' (%s)" % (r.malformed_upstream_status, patch)) - if r.unknown_upstream_status: - need_blame = True - if verbose: - print("Unknown Upstream-Status value '%s' (%s)" % (r.upstream_status, patch)) - - if want_blame and need_blame: - print("\n".join(blame_patch(patch)) + "\n") - - def percent(num): - try: - return "%d (%d%%)" % (num, round(num * 100.0 / total_patches)) - except ZeroDivisionError: - return "N/A" - - if verbose: - print() - - print("""Total patches found: %d -Patches missing Signed-off-by: %s -Patches with malformed Signed-off-by: %s -Patches missing CVE: %s -Patches missing Upstream-Status: %s -Patches with malformed Upstream-Status: %s -Patches in Pending state: %s""" % (total_patches, - percent(missing_sob), - percent(malformed_sob), - percent(missing_cve), - percent(missing_status), - percent(malformed_status), - percent(pending_patches))) - - - -def histogram(results): - from toolz import recipes, dicttoolz - import math - counts = recipes.countby(lambda r: r.upstream_status, results.values()) - bars = dicttoolz.valmap(lambda v: "#" * int(math.ceil(float(v) / len(results) * 100)), counts) - for k in bars: - print("%-20s %s (%d)" % (k.capitalize() if k else "No status", bars[k], counts[k])) - - -if __name__ == "__main__": - import argparse, subprocess, os - - args = argparse.ArgumentParser(description="Patch Review Tool") - args.add_argument("-b", "--blame", action="store_true", help="show blame for malformed patches") - args.add_argument("-v", "--verbose", action="store_true", help="show per-patch results") - args.add_argument("-g", "--histogram", action="store_true", help="show patch histogram") - args.add_argument("directory", nargs="?", help="directory to scan") - args = args.parse_args() - - if args.directory: - os.chdir(args.directory) - patches = subprocess.check_output(("git", "ls-files", "*.patch", "*.diff")).decode("utf-8").split() - results = patchreview(patches) - analyse(results, want_blame=args.blame, verbose=args.verbose) - if args.histogram: - print() - histogram(results) diff --git a/import-layers/yocto-poky/scripts/contrib/patchtest.sh b/import-layers/yocto-poky/scripts/contrib/patchtest.sh deleted file mode 100755 index 7fe566666..000000000 --- a/import-layers/yocto-poky/scripts/contrib/patchtest.sh +++ /dev/null @@ -1,118 +0,0 @@ -#!/bin/bash -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -# -# patchtest: Run patchtest on commits starting at master -# -# Copyright (c) 2017, Intel Corporation. -# All rights reserved. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -# -set -o errexit - -# Default values -pokydir='' - -usage() { -CMD=$(basename $0) -cat <<EOM -Usage: $CMD [-h] [-p pokydir] - -p pokydir Defaults to current directory -EOM ->&2 - exit 1 -} - -function clone() { - local REPOREMOTE=$1 - local REPODIR=$2 - if [ ! -d $REPODIR ]; then - git clone $REPOREMOTE $REPODIR --quiet - else - ( cd $REPODIR; git pull --quiet ) - fi -} - -while getopts ":p:h" opt; do - case $opt in - p) - pokydir=$OPTARG - ;; - h) - usage - ;; - \?) - echo "Invalid option: -$OPTARG" >&2 - usage - ;; - :) - echo "Option -$OPTARG requires an argument." >&2 - usage - ;; - esac -done -shift $((OPTIND-1)) - -CDIR="$PWD" - -# default pokydir to current directory if user did not specify one -if [ -z "$pokydir" ]; then - pokydir="$CDIR" -fi - -PTENV="$PWD/patchtest" -PT="$PTENV/patchtest" -PTOE="$PTENV/patchtest-oe" - -if ! which virtualenv > /dev/null; then - echo "Install virtualenv before proceeding" - exit 1; -fi - -# activate the virtual env -virtualenv $PTENV --quiet -source $PTENV/bin/activate - -cd $PTENV - -# clone or pull -clone git://git.yoctoproject.org/patchtest $PT -clone git://git.yoctoproject.org/patchtest-oe $PTOE - -# install requirements -pip install -r $PT/requirements.txt --quiet -pip install -r $PTOE/requirements.txt --quiet - -PATH="$PT:$PT/scripts:$PATH" - -# loop through parent to HEAD and execute patchtest on each commit -for commit in $(git rev-list master..HEAD --reverse) -do - shortlog="$(git log "$commit^1..$commit" --pretty='%h: %aN: %cd: %s')" - log="$(git format-patch "$commit^1..$commit" --stdout | patchtest - -r $pokydir -s $PTOE/tests --base-commit $commit^1 --json 2>/dev/null | create-summary --fail --only-results)" - if [ -z "$log" ]; then - shortlog="$shortlog: OK" - else - shortlog="$shortlog: FAIL" - fi - echo "$shortlog" - echo "$log" | sed -n -e '/Issue/p' -e '/Suggested fix/p' - echo "" -done - -deactivate - -cd $CDIR diff --git a/import-layers/yocto-poky/scripts/contrib/serdevtry b/import-layers/yocto-poky/scripts/contrib/serdevtry deleted file mode 100755 index 74bd7b716..000000000 --- a/import-layers/yocto-poky/scripts/contrib/serdevtry +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/sh - -# Copyright (C) 2014 Intel Corporation -# -# Released under the MIT license (see COPYING.MIT) - -if [ "$1" = "" -o "$1" = "--help" ] ; then - echo "Usage: $0 <serial terminal command>" - echo - echo "Simple script to handle maintaining a terminal for serial devices that" - echo "disappear when a device is powered down or reset, such as the USB" - echo "serial console on the original BeagleBone (white version)." - echo - echo "e.g. $0 picocom -b 115200 /dev/ttyUSB0" - echo - exit -fi - -args="$@" -DEVICE="" -while [ "$1" != "" ]; do - case "$1" in - /dev/*) - DEVICE=$1 - break;; - esac - shift -done - -if [ "$DEVICE" != "" ] ; then - while true; do - if [ ! -e $DEVICE ] ; then - echo "serdevtry: waiting for $DEVICE to exist..." - while [ ! -e $DEVICE ]; do - sleep 0.1 - done - fi - if [ ! -w $DEVICE ] ; then - # Sometimes (presumably because of a race with udev) we get to - # the device before its permissions have been set up - RETRYNUM=0 - while [ ! -w $DEVICE ]; do - if [ "$RETRYNUM" = "2" ] ; then - echo "Device $DEVICE exists but is not writable!" - exit 1 - fi - RETRYNUM=$((RETRYNUM+1)) - sleep 0.1 - done - fi - $args - if [ -e $DEVICE ] ; then - break - fi - done -else - echo "Unable to determine device node from command: $args" - exit 1 -fi - diff --git a/import-layers/yocto-poky/scripts/contrib/test_build_time.sh b/import-layers/yocto-poky/scripts/contrib/test_build_time.sh deleted file mode 100755 index 9e5725ae5..000000000 --- a/import-layers/yocto-poky/scripts/contrib/test_build_time.sh +++ /dev/null @@ -1,237 +0,0 @@ -#!/bin/bash - -# Build performance regression test script -# -# Copyright 2011 Intel Corporation -# All rights reserved. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# -# -# DESCRIPTION -# This script is intended to be used in conjunction with "git bisect run" -# in order to find regressions in build time, however it can also be used -# independently. It cleans out the build output directories, runs a -# specified worker script (an example is test_build_time_worker.sh) under -# TIME(1), logs the results to TEST_LOGDIR (default /tmp) and returns a -# value telling "git bisect run" whether the build time is good (under -# the specified threshold) or bad (over it). There is also a tolerance -# option but it is not particularly useful as it only subtracts the -# tolerance from the given threshold and uses it as the actual threshold. -# -# It is also capable of taking a file listing git revision hashes to be -# test-applied to the repository in order to get past build failures that -# would otherwise cause certain revisions to have to be skipped; if a -# revision does not apply cleanly then the script assumes it does not -# need to be applied and ignores it. -# -# Please see the help output (syntax below) for some important setup -# instructions. -# -# AUTHORS -# Paul Eggleton <paul.eggleton@linux.intel.com> - - -syntax() { - echo "syntax: $0 <script> <time> <tolerance> [patchrevlist]" - echo "" - echo " script - worker script file (if in current dir, prefix with ./)" - echo " time - time threshold (in seconds, suffix m for minutes)" - echo " tolerance - tolerance (in seconds, suffix m for minutes or % for" - echo " percentage, can be 0)" - echo " patchrevlist - optional file listing revisions to apply as patches on top" - echo "" - echo "You must set TEST_BUILDDIR to point to a previously created build directory," - echo "however please note that this script will wipe out the TMPDIR defined in" - echo "TEST_BUILDDIR/conf/local.conf as part of its initial setup (as well as your" - echo "~/.ccache)" - echo "" - echo "To get rid of the sudo prompt, please add the following line to /etc/sudoers" - echo "(use 'visudo' to edit this; also it is assumed that the user you are running" - echo "as is a member of the 'wheel' group):" - echo "" - echo "%wheel ALL=(ALL) NOPASSWD: /sbin/sysctl -w vm.drop_caches=[1-3]" - echo "" - echo "Note: it is recommended that you disable crond and any other process that" - echo "may cause significant CPU or I/O usage during build performance tests." -} - -# Note - we exit with 250 here because that will tell git bisect run that -# something bad happened and stop -if [ "$1" = "" ] ; then - syntax - exit 250 -fi - -if [ "$2" = "" ] ; then - syntax - exit 250 -fi - -if [ "$3" = "" ] ; then - syntax - exit 250 -fi - -if ! [[ "$2" =~ ^[0-9][0-9m.]*$ ]] ; then - echo "'$2' is not a valid number for threshold" - exit 250 -fi - -if ! [[ "$3" =~ ^[0-9][0-9m.%]*$ ]] ; then - echo "'$3' is not a valid number for tolerance" - exit 250 -fi - -if [ "$TEST_BUILDDIR" = "" ] ; then - echo "Please set TEST_BUILDDIR to a previously created build directory" - exit 250 -fi - -if [ ! -d "$TEST_BUILDDIR" ] ; then - echo "TEST_BUILDDIR $TEST_BUILDDIR not found" - exit 250 -fi - -git diff --quiet -if [ $? != 0 ] ; then - echo "Working tree is dirty, cannot proceed" - exit 251 -fi - -if [ "$BB_ENV_EXTRAWHITE" != "" ] ; then - echo "WARNING: you are running after sourcing the build environment script, this is not recommended" -fi - -runscript=$1 -timethreshold=$2 -tolerance=$3 - -if [ "$4" != "" ] ; then - patchrevlist=`cat $4` -else - patchrevlist="" -fi - -if [[ timethreshold == *m* ]] ; then - timethreshold=`echo $timethreshold | sed s/m/*60/ | bc` -fi - -if [[ $tolerance == *m* ]] ; then - tolerance=`echo $tolerance | sed s/m/*60/ | bc` -elif [[ $tolerance == *%* ]] ; then - tolerance=`echo $tolerance | sed s/%//` - tolerance=`echo "scale = 2; (($tolerance * $timethreshold) / 100)" | bc` -fi - -tmpdir=`grep "^TMPDIR" $TEST_BUILDDIR/conf/local.conf | sed -e 's/TMPDIR[ \t]*=[ \t\?]*"//' -e 's/"//'` -if [ "x$tmpdir" = "x" ]; then - echo "Unable to determine TMPDIR from $TEST_BUILDDIR/conf/local.conf, bailing out" - exit 250 -fi -sstatedir=`grep "^SSTATE_DIR" $TEST_BUILDDIR/conf/local.conf | sed -e 's/SSTATE_DIR[ \t\?]*=[ \t]*"//' -e 's/"//'` -if [ "x$sstatedir" = "x" ]; then - echo "Unable to determine SSTATE_DIR from $TEST_BUILDDIR/conf/local.conf, bailing out" - exit 250 -fi - -if [ `expr length $tmpdir` -lt 4 ] ; then - echo "TMPDIR $tmpdir is less than 4 characters, bailing out" - exit 250 -fi - -if [ `expr length $sstatedir` -lt 4 ] ; then - echo "SSTATE_DIR $sstatedir is less than 4 characters, bailing out" - exit 250 -fi - -echo -n "About to wipe out TMPDIR $tmpdir, press Ctrl+C to break out... " -for i in 9 8 7 6 5 4 3 2 1 -do - echo -ne "\x08$i" - sleep 1 -done -echo - -pushd . > /dev/null - -rm -f pseudodone -echo "Removing TMPDIR $tmpdir..." -rm -rf $tmpdir -echo "Removing TMPDIR $tmpdir-*libc..." -rm -rf $tmpdir-*libc -echo "Removing SSTATE_DIR $sstatedir..." -rm -rf $sstatedir -echo "Removing ~/.ccache..." -rm -rf ~/.ccache - -echo "Syncing..." -sync -sync -echo "Dropping VM cache..." -#echo 3 > /proc/sys/vm/drop_caches -sudo /sbin/sysctl -w vm.drop_caches=3 > /dev/null - -if [ "$TEST_LOGDIR" = "" ] ; then - logdir="/tmp" -else - logdir="$TEST_LOGDIR" -fi -rev=`git rev-parse HEAD` -logfile="$logdir/timelog_$rev.log" -echo -n > $logfile - -gitroot=`git rev-parse --show-toplevel` -cd $gitroot -for patchrev in $patchrevlist ; do - echo "Applying $patchrev" - patchfile=`mktemp` - git show $patchrev > $patchfile - git apply --check $patchfile &> /dev/null - if [ $? != 0 ] ; then - echo " ... patch does not apply without errors, ignoring" - else - echo "Applied $patchrev" >> $logfile - git apply $patchfile &> /dev/null - fi - rm $patchfile -done - -sync -echo "Quiescing for 5s..." -sleep 5 - -echo "Running $runscript at $rev..." -timeoutfile=`mktemp` -/usr/bin/time -o $timeoutfile -f "%e\nreal\t%E\nuser\t%Us\nsys\t%Ss\nmaxm\t%Mk" $runscript 2>&1 | tee -a $logfile -exitstatus=$PIPESTATUS - -git reset --hard HEAD > /dev/null -popd > /dev/null - -timeresult=`head -n1 $timeoutfile` -cat $timeoutfile | tee -a $logfile -rm $timeoutfile - -if [ $exitstatus != 0 ] ; then - # Build failed, exit with 125 to tell git bisect run to skip this rev - echo "*** Build failed (exit code $exitstatus), skipping..." | tee -a $logfile - exit 125 -fi - -ret=`echo "scale = 2; $timeresult > $timethreshold - $tolerance" | bc` -echo "Returning $ret" | tee -a $logfile -exit $ret - diff --git a/import-layers/yocto-poky/scripts/contrib/test_build_time_worker.sh b/import-layers/yocto-poky/scripts/contrib/test_build_time_worker.sh deleted file mode 100755 index 8e20a9ea7..000000000 --- a/import-layers/yocto-poky/scripts/contrib/test_build_time_worker.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -# This is an example script to be used in conjunction with test_build_time.sh - -if [ "$TEST_BUILDDIR" = "" ] ; then - echo "TEST_BUILDDIR is not set" - exit 1 -fi - -buildsubdir=`basename $TEST_BUILDDIR` -if [ ! -d $buildsubdir ] ; then - echo "Unable to find build subdir $buildsubdir in current directory" - exit 1 -fi - -if [ -f oe-init-build-env ] ; then - . ./oe-init-build-env $buildsubdir -elif [ -f poky-init-build-env ] ; then - . ./poky-init-build-env $buildsubdir -else - echo "Unable to find build environment setup script" - exit 1 -fi - -if [ -f ../meta/recipes-sato/images/core-image-sato.bb ] ; then - target="core-image-sato" -else - target="poky-image-sato" -fi - -echo "Build started at `date "+%Y-%m-%d %H:%M:%S"`" -echo "bitbake $target" -bitbake $target -ret=$? -echo "Build finished at `date "+%Y-%m-%d %H:%M:%S"`" -exit $ret - diff --git a/import-layers/yocto-poky/scripts/contrib/uncovered b/import-layers/yocto-poky/scripts/contrib/uncovered deleted file mode 100755 index a8399ad17..000000000 --- a/import-layers/yocto-poky/scripts/contrib/uncovered +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -eur -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -# -# Find python modules uncovered by oe-seltest -# -# Copyright (c) 2016, Intel Corporation -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# Author: Ed Bartosh <ed.bartosh@linux.intel.com> -# - -if [ ! "$#" -eq 1 -o -t 0 ] ; then - echo 'Usage: coverage report | ./scripts/contrib/uncovered <dir>' 1>&2 - exit 1 -fi - -path=$(readlink -ev $1) - -if [ ! -d "$path" ] ; then - echo "directory $1 doesn't exist" 1>&2 - exit 1 -fi - -diff -u <(grep "$path" | grep -v '0%$' | cut -f1 -d: | sort) \ - <(find $path | xargs file | grep 'Python script' | cut -f1 -d:| sort) | \ - grep "^+$path" | cut -c2- diff --git a/import-layers/yocto-poky/scripts/contrib/verify-homepage.py b/import-layers/yocto-poky/scripts/contrib/verify-homepage.py deleted file mode 100755 index 76f1749cf..000000000 --- a/import-layers/yocto-poky/scripts/contrib/verify-homepage.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python3 - -# This script can be used to verify HOMEPAGE values for all recipes in -# the current configuration. -# The result is influenced by network environment, since the timeout of connect url is 5 seconds as default. - -import sys -import os -import subprocess -import urllib.request - - -# Allow importing scripts/lib modules -scripts_path = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + '/..') -lib_path = scripts_path + '/lib' -sys.path = sys.path + [lib_path] -import scriptpath -import scriptutils - -# Allow importing bitbake modules -bitbakepath = scriptpath.add_bitbake_lib_path() - -import bb.tinfoil - -logger = scriptutils.logger_create('verify_homepage') - -def wgetHomepage(pn, homepage): - result = subprocess.call('wget ' + '-q -T 5 -t 1 --spider ' + homepage, shell = True) - if result: - logger.warn("%s: failed to verify HOMEPAGE: %s " % (pn, homepage)) - return 1 - else: - return 0 - -def verifyHomepage(bbhandler): - pkg_pn = bbhandler.cooker.recipecaches[''].pkg_pn - pnlist = sorted(pkg_pn) - count = 0 - checked = [] - for pn in pnlist: - for fn in pkg_pn[pn]: - # There's no point checking multiple BBCLASSEXTENDed variants of the same recipe - realfn, _, _ = bb.cache.virtualfn2realfn(fn) - if realfn in checked: - continue - data = bbhandler.parse_recipe_file(realfn) - homepage = data.getVar("HOMEPAGE") - if homepage: - try: - urllib.request.urlopen(homepage, timeout=5) - except Exception: - count = count + wgetHomepage(os.path.basename(realfn), homepage) - checked.append(realfn) - return count - -if __name__=='__main__': - with bb.tinfoil.Tinfoil() as bbhandler: - bbhandler.prepare() - logger.info("Start verifying HOMEPAGE:") - failcount = verifyHomepage(bbhandler) - logger.info("Finished verifying HOMEPAGE.") - logger.info("Summary: %s failed" % failcount) |
