diff options
| author | Brad Bishop <bradleyb@fuzziesquirrel.com> | 2018-06-25 12:45:53 -0400 |
|---|---|---|
| committer | Brad Bishop <bradleyb@fuzziesquirrel.com> | 2018-06-27 14:38:15 -0400 |
| commit | 316dfdd917bec6a218f431211d28bf8df6b6fb0f (patch) | |
| tree | 5541073f9851f44c2bd67b4959dc776ee3c3810f /import-layers/yocto-poky/bitbake | |
| parent | 36acd3e888044dea2ac0b2946f15616f968388c9 (diff) | |
| download | blackbird-openbmc-316dfdd917bec6a218f431211d28bf8df6b6fb0f.tar.gz blackbird-openbmc-316dfdd917bec6a218f431211d28bf8df6b6fb0f.zip | |
Yocto 2.5
Move OpenBMC to Yocto 2.5(sumo)
Signed-off-by: Brad Bishop <bradleyb@fuzziesquirrel.com>
Change-Id: I5c5ad6904a16e14c1c397f0baf10c9d465594a78
Diffstat (limited to 'import-layers/yocto-poky/bitbake')
38 files changed, 1305 insertions, 1127 deletions
diff --git a/import-layers/yocto-poky/bitbake/bin/bitbake b/import-layers/yocto-poky/bitbake/bin/bitbake index 3acc23af1..342cef916 100755 --- a/import-layers/yocto-poky/bitbake/bin/bitbake +++ b/import-layers/yocto-poky/bitbake/bin/bitbake @@ -38,7 +38,7 @@ from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException if sys.getfilesystemencoding() != "utf-8": sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.") -__version__ = "1.36.0" +__version__ = "1.38.0" if __name__ == "__main__": if __version__ != bb.__version__: diff --git a/import-layers/yocto-poky/bitbake/bin/toaster b/import-layers/yocto-poky/bitbake/bin/toaster index 4036f0ad5..ed365ee82 100755 --- a/import-layers/yocto-poky/bitbake/bin/toaster +++ b/import-layers/yocto-poky/bitbake/bin/toaster @@ -68,7 +68,7 @@ webserverKillAll() if [ -f ${pidfile} ]; then pid=`cat ${pidfile}` while kill -0 $pid 2>/dev/null; do - kill -SIGTERM -$pid 2>/dev/null + kill -SIGTERM $pid 2>/dev/null sleep 1 done rm ${pidfile} @@ -91,7 +91,7 @@ webserverStartAll() echo "Starting webserver..." - $MANAGE runserver "$ADDR_PORT" \ + $MANAGE runserver --noreload "$ADDR_PORT" \ </dev/null >>${BUILDDIR}/toaster_web.log 2>&1 \ & echo $! >${BUILDDIR}/.toastermain.pid diff --git a/import-layers/yocto-poky/bitbake/contrib/dump_cache.py b/import-layers/yocto-poky/bitbake/contrib/dump_cache.py index f4d4c1b12..8963ca4b0 100755 --- a/import-layers/yocto-poky/bitbake/contrib/dump_cache.py +++ b/import-layers/yocto-poky/bitbake/contrib/dump_cache.py @@ -2,7 +2,7 @@ # ex:ts=4:sw=4:sts=4:et # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- # -# Copyright (C) 2012 Wind River Systems, Inc. +# Copyright (C) 2012, 2018 Wind River Systems, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -18,51 +18,68 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # -# This is used for dumping the bb_cache.dat, the output format is: -# recipe_path PN PV PACKAGES +# Used for dumping the bb_cache.dat # import os import sys -import warnings +import argparse # For importing bb.cache sys.path.insert(0, os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])), '../lib')) from bb.cache import CoreRecipeInfo -import pickle as pickle +import pickle -def main(argv=None): - """ - Get the mapping for the target recipe. - """ - if len(argv) != 1: - print("Error, need one argument!", file=sys.stderr) - return 2 +class DumpCache(object): + def __init__(self): + parser = argparse.ArgumentParser( + description="bb_cache.dat's dumper", + epilog="Use %(prog)s --help to get help") + parser.add_argument("-r", "--recipe", + help="specify the recipe, default: all recipes", action="store") + parser.add_argument("-m", "--members", + help = "specify the member, use comma as separator for multiple ones, default: all members", action="store", default="") + parser.add_argument("-s", "--skip", + help = "skip skipped recipes", action="store_true") + parser.add_argument("cachefile", + help = "specify bb_cache.dat", nargs = 1, action="store", default="") - cachefile = argv[0] + self.args = parser.parse_args() - with open(cachefile, "rb") as cachefile: - pickled = pickle.Unpickler(cachefile) - while cachefile: - try: - key = pickled.load() - val = pickled.load() - except Exception: - break - if isinstance(val, CoreRecipeInfo) and (not val.skipped): - pn = val.pn - # Filter out the native recipes. - if key.startswith('virtual:native:') or pn.endswith("-native"): - continue + def main(self): + with open(self.args.cachefile[0], "rb") as cachefile: + pickled = pickle.Unpickler(cachefile) + while True: + try: + key = pickled.load() + val = pickled.load() + except Exception: + break + if isinstance(val, CoreRecipeInfo): + pn = val.pn - # 1.0 is the default version for a no PV recipe. - if "pv" in val.__dict__: - pv = val.pv - else: - pv = "1.0" + if self.args.recipe and self.args.recipe != pn: + continue - print("%s %s %s %s" % (key, pn, pv, ' '.join(val.packages))) + if self.args.skip and val.skipped: + continue -if __name__ == "__main__": - sys.exit(main(sys.argv[1:])) + if self.args.members: + out = key + for member in self.args.members.split(','): + out += ": %s" % val.__dict__.get(member) + print("%s" % out) + else: + print("%s: %s" % (key, val.__dict__)) + elif not self.args.recipe: + print("%s %s" % (key, val)) +if __name__ == "__main__": + try: + dump = DumpCache() + ret = dump.main() + except Exception as esc: + ret = 1 + import traceback + traceback.print_exc() + sys.exit(ret) diff --git a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.xml b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.xml index e4cc422ea..f1caaecd2 100644 --- a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.xml +++ b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.xml @@ -781,7 +781,7 @@ The code in <filename>meta/lib/oe/sstatesig.py</filename> shows two examples of this and also illustrates how you can insert your own policy into the system if so desired. - This file defines the two basic signature generators OpenEmbedded Core + This file defines the two basic signature generators OpenEmbedded-Core uses: "OEBasic" and "OEBasicHash". By default, there is a dummy "noop" signature handler enabled in BitBake. This means that behavior is unchanged from previous versions. diff --git a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml index c721e86eb..29ae486a7 100644 --- a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml +++ b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml @@ -777,6 +777,43 @@ </para> </section> + <section id='repo-fetcher'> + <title>Repo Fetcher (<filename>repo://</filename>)</title> + + <para> + This fetcher submodule fetches code from + <filename>google-repo</filename> source control system. + The fetcher works by initiating and syncing sources of the + repository into + <link linkend='var-REPODIR'><filename>REPODIR</filename></link>, + which is usually + <link linkend='var-DL_DIR'><filename>DL_DIR</filename></link><filename>/repo</filename>. + </para> + + <para> + This fetcher supports the following parameters: + <itemizedlist> + <listitem><para> + <emphasis>"protocol":</emphasis> + Protocol to fetch the repository manifest (default: git). + </para></listitem> + <listitem><para> + <emphasis>"branch":</emphasis> + Branch or tag of repository to get (default: master). + </para></listitem> + <listitem><para> + <emphasis>"manifest":</emphasis> + Name of the manifest file (default: <filename>default.xml</filename>). + </para></listitem> + </itemizedlist> + Here are some example URLs: + <literallayout class='monospaced'> + SRC_URI = "repo://REPOROOT;protocol=git;branch=some_branch;manifest=my_manifest.xml" + SRC_URI = "repo://REPOROOT;protocol=file;branch=some_branch;manifest=my_manifest.xml" + </literallayout> + </para> + </section> + <section id='other-fetchers'> <title>Other Fetchers</title> @@ -796,9 +833,6 @@ Secure Shell (<filename>ssh://</filename>) </para></listitem> <listitem><para> - Repo (<filename>repo://</filename>) - </para></listitem> - <listitem><para> OSC (<filename>osc://</filename>) </para></listitem> <listitem><para> diff --git a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.xml b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.xml index 9253eaf9d..9076f0fcd 100644 --- a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.xml +++ b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.xml @@ -260,7 +260,7 @@ files. For this example, you need to create the file in your project directory and define some key BitBake variables. - For more information on the <filename>bitbake.conf</filename>, + For more information on the <filename>bitbake.conf</filename> file, see <ulink url='http://git.openembedded.org/bitbake/tree/conf/bitbake.conf'></ulink>. </para> @@ -273,14 +273,32 @@ some editor to create the <filename>bitbake.conf</filename> so that it contains the following: <literallayout class='monospaced'> + <link linkend='var-PN'>PN</link> = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}" + </literallayout> + <literallayout class='monospaced'> TMPDIR = "${<link linkend='var-TOPDIR'>TOPDIR</link>}/tmp" <link linkend='var-CACHE'>CACHE</link> = "${TMPDIR}/cache" - <link linkend='var-STAMP'>STAMP</link> = "${TMPDIR}/stamps" - <link linkend='var-T'>T</link> = "${TMPDIR}/work" - <link linkend='var-B'>B</link> = "${TMPDIR}" + <link linkend='var-STAMP'>STAMP</link> = "${TMPDIR}/${PN}/stamps" + <link linkend='var-T'>T</link> = "${TMPDIR}/${PN}/work" + <link linkend='var-B'>B</link> = "${TMPDIR}/${PN}" </literallayout> + <note> + Without a value for <filename>PN</filename>, the + variables <filename>STAMP</filename>, + <filename>T</filename>, and <filename>B</filename>, + prevent more than one recipe from working. You can fix + this by either setting <filename>PN</filename> to have + a value similar to what OpenEmbedded and BitBake use + in the default <filename>bitbake.conf</filename> file + (see previous example). Or, by manually updating each + recipe to set <filename>PN</filename>. You will also + need to include <filename>PN</filename> as part of the + <filename>STAMP</filename>, <filename>T</filename>, and + <filename>B</filename> variable definitions in the + <filename>local.conf</filename> file. + </note> The <filename>TMPDIR</filename> variable establishes a directory - that BitBake uses for build output and intermediate files (other + that BitBake uses for build output and intermediate files other than the cached information used by the <link linkend='setscene'>Setscene</link> process. Here, the <filename>TMPDIR</filename> directory is set to @@ -300,19 +318,19 @@ file exists, you can run the <filename>bitbake</filename> command again: <literallayout class='monospaced'> -$ bitbake -ERROR: Traceback (most recent call last): - File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 163, in wrapped - return func(fn, *args) - File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 177, in _inherit - bb.parse.BBHandler.inherit(bbclass, "configuration INHERITs", 0, data) - File "/home/scott-lenovo/bitbake/lib/bb/parse/parse_py/BBHandler.py", line 92, in inherit - include(fn, file, lineno, d, "inherit") - File "/home/scott-lenovo/bitbake/lib/bb/parse/parse_py/ConfHandler.py", line 100, in include - raise ParseError("Could not %(error_out)s file %(fn)s" % vars(), oldfn, lineno) -ParseError: ParseError in configuration INHERITs: Could not inherit file classes/base.bbclass + $ bitbake + ERROR: Traceback (most recent call last): + File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 163, in wrapped + return func(fn, *args) + File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 177, in _inherit + bb.parse.BBHandler.inherit(bbclass, "configuration INHERITs", 0, data) + File "/home/scott-lenovo/bitbake/lib/bb/parse/parse_py/BBHandler.py", line 92, in inherit + include(fn, file, lineno, d, "inherit") + File "/home/scott-lenovo/bitbake/lib/bb/parse/parse_py/ConfHandler.py", line 100, in include + raise ParseError("Could not %(error_out)s file %(fn)s" % vars(), oldfn, lineno) + ParseError: ParseError in configuration INHERITs: Could not inherit file classes/base.bbclass -ERROR: Unable to parse base: ParseError in configuration INHERITs: Could not inherit file classes/base.bbclass + ERROR: Unable to parse base: ParseError in configuration INHERITs: Could not inherit file classes/base.bbclass </literallayout> In the sample output, BitBake could not find the <filename>classes/base.bbclass</filename> file. @@ -365,10 +383,10 @@ ERROR: Unable to parse base: ParseError in configuration INHERITs: Could not inh code separate from the general metadata used by BitBake. Thus, this example creates and uses a layer called "mylayer". <note> - You can find additional information on layers at - <ulink url='http://www.yoctoproject.org/docs/2.3/bitbake-user-manual/bitbake-user-manual.html#layers'></ulink>. - </note> - </para> + You can find additional information on layers in the + "<link linkend='layers'>Layers</link>" section. + </note></para> + <para>Minimally, you need a recipe file and a layer configuration file in your layer. The configuration file needs to be in the <filename>conf</filename> diff --git a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.xml b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.xml index 08d9afdf2..4cf0ed9d1 100644 --- a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.xml +++ b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.xml @@ -488,8 +488,6 @@ target that failed and anything depending on it cannot be built, as much as possible will be built before stopping. - -a, --tryaltconfigs Continue with builds by trying to use alternative - providers where possible. -f, --force Force the specified targets/task to run (invalidating any existing stamp file). -c CMD, --cmd=CMD Specify the task to execute. The exact options @@ -504,19 +502,20 @@ Read the specified file before bitbake.conf. -R POSTFILE, --postread=POSTFILE Read the specified file after bitbake.conf. - -v, --verbose Enable tracing of shell tasks (with 'set -x'). - Also print bb.note(...) messages to stdout (in - addition to writing them to ${T}/log.do_<task>). - -D, --debug Increase the debug level. You can specify this - more than once. -D sets the debug level to 1, - where only bb.debug(1, ...) messages are printed - to stdout; -DD sets the debug level to 2, where - both bb.debug(1, ...) and bb.debug(2, ...) - messages are printed; etc. Without -D, no debug - messages are printed. Note that -D only affects - output to stdout. All debug messages are written - to ${T}/log.do_taskname, regardless of the debug - level. + -v, --verbose Enable tracing of shell tasks (with 'set -x'). Also + print bb.note(...) messages to stdout (in addition to + writing them to ${T}/log.do_<task>). + -D, --debug Increase the debug level. You can specify this more + than once. -D sets the debug level to 1, where only + bb.debug(1, ...) messages are printed to stdout; -DD + sets the debug level to 2, where both bb.debug(1, ...) + and bb.debug(2, ...) messages are printed; etc. + Without -D, no debug messages are printed. Note that + -D only affects output to stdout. All debug messages + are written to ${T}/log.do_taskname, regardless of the + debug level. + -q, --quiet Output less log message data to the terminal. You can + specify this more than once. -n, --dry-run Don't execute, just go through the motions. -S SIGNATURE_HANDLER, --dump-signatures=SIGNATURE_HANDLER Dump out the signature construction information, with @@ -539,30 +538,38 @@ -l DEBUG_DOMAINS, --log-domains=DEBUG_DOMAINS Show debug logging for the specified logging domains -P, --profile Profile the command and save reports. - -u UI, --ui=UI The user interface to use (taskexp, knotty or - ncurses - default knotty). - -t SERVERTYPE, --servertype=SERVERTYPE - Choose which server type to use (process or xmlrpc - - default process). + -u UI, --ui=UI The user interface to use (knotty, ncurses or taskexp + - default knotty). --token=XMLRPCTOKEN Specify the connection token to be used when connecting to a remote server. --revisions-changed Set the exit code depending on whether upstream floating revisions have changed or not. --server-only Run bitbake without a UI, only starting a server (cooker) process. - -B BIND, --bind=BIND The name/address for the bitbake server to bind to. + -B BIND, --bind=BIND The name/address for the bitbake xmlrpc server to bind + to. + -T SERVER_TIMEOUT, --idle-timeout=SERVER_TIMEOUT + Set timeout to unload bitbake server due to + inactivity, set to -1 means no unload, default: + Environment variable BB_SERVER_TIMEOUT. --no-setscene Do not run any setscene tasks. sstate will be ignored and everything needed, built. --setscene-only Only run setscene tasks, don't run any real tasks. --remote-server=REMOTE_SERVER Connect to the specified server. - -m, --kill-server Terminate the remote server. + -m, --kill-server Terminate any running bitbake server. --observe-only Connect to a server as an observing-only client. --status-only Check the status of the remote bitbake server. -w WRITEEVENTLOG, --write-log=WRITEEVENTLOG Writes the event log of the build to a bitbake event json file. Use '' (empty string) to assign the name automatically. + --runall=RUNALL Run the specified task for any recipe in the taskgraph + of the specified target (even if it wouldn't otherwise + have run). + --runonly=RUNONLY Run only the specified task within the taskgraph of + the specified targets (and any task dependencies those + tasks may have). </literallayout> </para> </section> diff --git a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml index 0cfa53d02..b4fc64e75 100644 --- a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml +++ b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml @@ -2132,6 +2132,8 @@ <listitem><para> <filename>bb.event.BuildStarted()</filename>: Fired when a new build starts. + BitBake fires multiple "BuildStarted" events (one per configuration) + when multiple configuration (multiconfig) is enabled. </para></listitem> <listitem><para> <filename>bb.build.TaskStarted()</filename>: @@ -2650,47 +2652,70 @@ </para> <para> - This list is a place holder of content existed from previous work - on the manual. - Some or all of it probably needs integrated into the subsections - that make up this section. - For now, I have just provided a short glossary-like description - for each variable. - Ultimately, this list goes away. + These checksums are stored in + <link linkend='var-STAMP'><filename>STAMP</filename></link>. + You can examine the checksums using the following BitBake command: + <literallayout class='monospaced'> + $ bitbake-dumpsigs + </literallayout> + This command returns the signature data in a readable format + that allows you to examine the inputs used when the + OpenEmbedded build system generates signatures. + For example, using <filename>bitbake-dumpsigs</filename> + allows you to examine the <filename>do_compile</filename> + task's “sigdata” for a C application (e.g. + <filename>bash</filename>). + Running the command also reveals that the “CC” variable is part of + the inputs that are hashed. + Any changes to this variable would invalidate the stamp and + cause the <filename>do_compile</filename> task to run. + </para> + + <para> + The following list describes related variables: <itemizedlist> - <listitem><para><filename>STAMP</filename>: - The base path to create stamp files.</para></listitem> - <listitem><para><filename>STAMPCLEAN</filename> - Again, the base path to create stamp files but can use wildcards - for matching a range of files for clean operations. - </para></listitem> - <listitem><para><filename>BB_STAMP_WHITELIST</filename> - Lists stamp files that are looked at when the stamp policy - is "whitelist". - </para></listitem> - <listitem><para><filename>BB_STAMP_POLICY</filename> - Defines the mode for comparing timestamps of stamp files. - </para></listitem> - <listitem><para><filename>BB_HASHCHECK_FUNCTION</filename> + <listitem><para> + <link linkend='var-BB_HASHCHECK_FUNCTION'><filename>BB_HASHCHECK_FUNCTION</filename></link>: Specifies the name of the function to call during the "setscene" part of the task's execution in order to validate the list of task hashes. </para></listitem> - <listitem><para><filename>BB_SETSCENE_VERIFY_FUNCTION2</filename> + <listitem><para> + <link linkend='var-BB_SETSCENE_DEPVALID'><filename>BB_SETSCENE_DEPVALID</filename></link>: + Specifies a function BitBake calls that determines + whether BitBake requires a setscene dependency to + be met. + </para></listitem> + <listitem><para> + <link linkend='var-BB_SETSCENE_VERIFY_FUNCTION2'><filename>BB_SETSCENE_VERIFY_FUNCTION2</filename></link>: Specifies a function to call that verifies the list of planned task execution before the main task execution happens. </para></listitem> - <listitem><para><filename>BB_SETSCENE_DEPVALID</filename> - Specifies a function BitBake calls that determines - whether BitBake requires a setscene dependency to - be met. + <listitem><para> + <link linkend='var-BB_STAMP_POLICY'><filename>BB_STAMP_POLICY</filename></link>: + Defines the mode for comparing timestamps of stamp files. + </para></listitem> + <listitem><para> + <link linkend='var-BB_STAMP_WHITELIST'><filename>BB_STAMP_WHITELIST</filename></link>: + Lists stamp files that are looked at when the stamp policy + is "whitelist". </para></listitem> - <listitem><para><filename>BB_TASKHASH</filename> + <listitem><para> + <link linkend='var-BB_TASKHASH'><filename>BB_TASKHASH</filename></link>: Within an executing task, this variable holds the hash of the task as returned by the currently enabled signature generator. </para></listitem> + <listitem><para> + <link linkend='var-STAMP'><filename>STAMP</filename></link>: + The base path to create stamp files. + </para></listitem> + <listitem><para> + <link linkend='var-STAMPCLEAN'><filename>STAMPCLEAN</filename></link>: + Again, the base path to create stamp files but can use wildcards + for matching a range of files for clean operations. + </para></listitem> </itemizedlist> </para> </section> diff --git a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml index d89e123e0..0313359d9 100644 --- a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml +++ b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml @@ -78,7 +78,7 @@ </para> <para> - In OpenEmbedded Core, <filename>ASSUME_PROVIDED</filename> + In OpenEmbedded-Core, <filename>ASSUME_PROVIDED</filename> mostly specifies native tools that should not be built. An example is <filename>git-native</filename>, which when specified allows for the Git binary from the host to @@ -964,7 +964,7 @@ Allows you to extend a recipe so that it builds variants of the software. Some examples of these variants for recipes from the - OpenEmbedded Core metadata are "natives" such as + OpenEmbedded-Core metadata are "natives" such as <filename>quilt-native</filename>, which is a copy of Quilt built to run on the build system; "crosses" such as <filename>gcc-cross</filename>, which is a compiler @@ -980,7 +980,7 @@ amount of code, it usually is as simple as adding the variable to your recipe. Here are two examples. - The "native" variants are from the OpenEmbedded Core + The "native" variants are from the OpenEmbedded-Core metadata: <literallayout class='monospaced'> BBCLASSEXTEND =+ "native nativesdk" @@ -2089,6 +2089,16 @@ </glossdef> </glossentry> + <glossentry id='var-REPODIR'><glossterm>REPODIR</glossterm> + <glossdef> + <para> + The directory in which a local copy of a + <filename>google-repo</filename> directory is stored + when it is synced. + </para> + </glossdef> + </glossentry> + <glossentry id='var-RPROVIDES'><glossterm>RPROVIDES</glossterm> <glossdef> <para> diff --git a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual.xml b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual.xml index 4aef4e72f..d793265c9 100644 --- a/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual.xml +++ b/import-layers/yocto-poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual.xml @@ -56,7 +56,7 @@ --> <copyright> - <year>2004-2016</year> + <year>2004-2018</year> <holder>Richard Purdie</holder> <holder>Chris Larson</holder> <holder>and Phil Blundell</holder> diff --git a/import-layers/yocto-poky/bitbake/lib/bb/__init__.py b/import-layers/yocto-poky/bitbake/lib/bb/__init__.py index 526883154..d24adb8ea 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/__init__.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/__init__.py @@ -21,7 +21,7 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -__version__ = "1.36.0" +__version__ = "1.38.0" import sys if sys.version_info < (3, 4, 0): diff --git a/import-layers/yocto-poky/bitbake/lib/bb/build.py b/import-layers/yocto-poky/bitbake/lib/bb/build.py index 0d0100a06..4631abdde 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/build.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/build.py @@ -872,6 +872,12 @@ def preceedtask(task, with_recrdeptasks, d): that this may lead to the task itself being listed. """ preceed = set() + + # Ignore tasks which don't exist + tasks = d.getVar('__BBTASKS', False) + if task not in tasks: + return preceed + preceed.update(d.getVarFlag(task, 'deps') or []) if with_recrdeptasks: recrdeptask = d.getVarFlag(task, 'recrdeptask') diff --git a/import-layers/yocto-poky/bitbake/lib/bb/cache.py b/import-layers/yocto-poky/bitbake/lib/bb/cache.py index 86ce0e786..168a77ac0 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/cache.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/cache.py @@ -395,7 +395,7 @@ class Cache(NoCache): self.has_cache = True self.cachefile = getCacheFile(self.cachedir, "bb_cache.dat", self.data_hash) - logger.debug(1, "Using cache in '%s'", self.cachedir) + logger.debug(1, "Cache dir: %s", self.cachedir) bb.utils.mkdirhier(self.cachedir) cache_ok = True @@ -408,6 +408,8 @@ class Cache(NoCache): self.load_cachefile() elif os.path.isfile(self.cachefile): logger.info("Out of date cache found, rebuilding...") + else: + logger.debug(1, "Cache file %s not found, building..." % self.cachefile) def load_cachefile(self): cachesize = 0 @@ -424,6 +426,7 @@ class Cache(NoCache): for cache_class in self.caches_array: cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) + logger.debug(1, 'Loading cache file: %s' % cachefile) with open(cachefile, "rb") as cachefile: pickled = pickle.Unpickler(cachefile) # Check cache version information diff --git a/import-layers/yocto-poky/bitbake/lib/bb/cooker.py b/import-layers/yocto-poky/bitbake/lib/bb/cooker.py index c7fdd7290..1fda40dd4 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/cooker.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/cooker.py @@ -516,6 +516,8 @@ class BBCooker: fn = runlist[0][3] else: envdata = self.data + data.expandKeys(envdata) + parse.ast.runAnonFuncs(envdata) if fn: try: @@ -536,7 +538,6 @@ class BBCooker: logger.plain(env.getvalue()) # emit the metadata which isnt valid shell - data.expandKeys(envdata) for e in sorted(envdata.keys()): if envdata.getVarFlag(e, 'func', False) and envdata.getVarFlag(e, 'python', False): logger.plain("\npython %s () {\n%s}\n", e, envdata.getVar(e, False)) @@ -856,12 +857,12 @@ class BBCooker: with open('task-depends.dot', 'w') as f: f.write("digraph depends {\n") - for task in depgraph["tdepends"]: + for task in sorted(depgraph["tdepends"]): (pn, taskname) = task.rsplit(".", 1) fn = depgraph["pn"][pn]["filename"] version = depgraph["pn"][pn]["version"] f.write('"%s.%s" [label="%s %s\\n%s\\n%s"]\n' % (pn, taskname, pn, taskname, version, fn)) - for dep in depgraph["tdepends"][task]: + for dep in sorted(depgraph["tdepends"][task]): f.write('"%s" -> "%s"\n' % (task, dep)) f.write("}\n") logger.info("Task dependencies saved to 'task-depends.dot'") @@ -869,23 +870,23 @@ class BBCooker: with open('recipe-depends.dot', 'w') as f: f.write("digraph depends {\n") pndeps = {} - for task in depgraph["tdepends"]: + for task in sorted(depgraph["tdepends"]): (pn, taskname) = task.rsplit(".", 1) if pn not in pndeps: pndeps[pn] = set() - for dep in depgraph["tdepends"][task]: + for dep in sorted(depgraph["tdepends"][task]): (deppn, deptaskname) = dep.rsplit(".", 1) pndeps[pn].add(deppn) - for pn in pndeps: + for pn in sorted(pndeps): fn = depgraph["pn"][pn]["filename"] version = depgraph["pn"][pn]["version"] f.write('"%s" [label="%s\\n%s\\n%s"]\n' % (pn, pn, version, fn)) - for dep in pndeps[pn]: + for dep in sorted(pndeps[pn]): if dep == pn: continue f.write('"%s" -> "%s"\n' % (pn, dep)) f.write("}\n") - logger.info("Flatened recipe dependencies saved to 'recipe-depends.dot'") + logger.info("Flattened recipe dependencies saved to 'recipe-depends.dot'") def show_appends_with_no_recipes(self): # Determine which bbappends haven't been applied @@ -1170,6 +1171,7 @@ class BBCooker: elif regex == "": parselog.debug(1, "BBFILE_PATTERN_%s is empty" % c) errors = False + continue else: try: cre = re.compile(regex) @@ -1603,8 +1605,6 @@ class BBCooker: if self.parser: self.parser.shutdown(clean=not force, force=force) - self.notifier.stop() - self.confignotifier.stop() def finishcommand(self): self.state = state.initial @@ -1807,21 +1807,25 @@ class CookerCollectFiles(object): realfn, cls, mc = bb.cache.virtualfn2realfn(p) priorities[p] = self.calc_bbfile_priority(realfn, matched) - # Don't show the warning if the BBFILE_PATTERN did match .bbappend files unmatched = set() for _, _, regex, pri in self.bbfile_config_priorities: if not regex in matched: unmatched.add(regex) - def findmatch(regex): + # Don't show the warning if the BBFILE_PATTERN did match .bbappend files + def find_bbappend_match(regex): for b in self.bbappends: (bbfile, append) = b if regex.match(append): + # If the bbappend is matched by already "matched set", return False + for matched_regex in matched: + if matched_regex.match(append): + return False return True return False for unmatch in unmatched.copy(): - if findmatch(unmatch): + if find_bbappend_match(unmatch): unmatched.remove(unmatch) for collection, pattern, regex, _ in self.bbfile_config_priorities: diff --git a/import-layers/yocto-poky/bitbake/lib/bb/cookerdata.py b/import-layers/yocto-poky/bitbake/lib/bb/cookerdata.py index fab47c75f..5df66e617 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/cookerdata.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/cookerdata.py @@ -143,7 +143,8 @@ class CookerConfiguration(object): self.writeeventlog = False self.server_only = False self.limited_deps = False - self.runall = None + self.runall = [] + self.runonly = [] self.env = {} @@ -395,6 +396,8 @@ class CookerDataBuilder(object): if compat and not (compat & layerseries): bb.fatal("Layer %s is not compatible with the core layer which only supports these series: %s (layer is compatible with %s)" % (c, " ".join(layerseries), " ".join(compat))) + elif not compat and not data.getVar("BB_WORKERCONTEXT"): + bb.warn("Layer %s should set LAYERSERIES_COMPAT_%s in its conf/layer.conf file to list the core layer names it is compatible with." % (c, c)) if not data.getVar("BBPATH"): msg = "The BBPATH variable is not set" diff --git a/import-layers/yocto-poky/bitbake/lib/bb/event.py b/import-layers/yocto-poky/bitbake/lib/bb/event.py index 52072b580..5d0049626 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/event.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/event.py @@ -449,12 +449,6 @@ class BuildBase(Event): def setName(self, name): self._name = name - def getCfg(self): - return self.data - - def setCfg(self, cfg): - self.data = cfg - def getFailures(self): """ Return the number of failed packages @@ -463,9 +457,6 @@ class BuildBase(Event): pkgs = property(getPkgs, setPkgs, None, "pkgs property") name = property(getName, setName, None, "name property") - cfg = property(getCfg, setCfg, None, "cfg property") - - class BuildInit(BuildBase): """buildFile or buildTargets was invoked""" diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/__init__.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/__init__.py index f70f1b515..72d6092de 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/__init__.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/__init__.py @@ -643,26 +643,25 @@ def verify_donestamp(ud, d, origud=None): if not ud.needdonestamp or (origud and not origud.needdonestamp): return True - if not os.path.exists(ud.donestamp): + if not os.path.exists(ud.localpath): + # local path does not exist + if os.path.exists(ud.donestamp): + # done stamp exists, but the downloaded file does not; the done stamp + # must be incorrect, re-trigger the download + bb.utils.remove(ud.donestamp) return False if (not ud.method.supports_checksum(ud) or (origud and not origud.method.supports_checksum(origud))): - # done stamp exists, checksums not supported; assume the local file is - # current - return True - - if not os.path.exists(ud.localpath): - # done stamp exists, but the downloaded file does not; the done stamp - # must be incorrect, re-trigger the download - bb.utils.remove(ud.donestamp) - return False + # if done stamp exists and checksums not supported; assume the local + # file is current + return os.path.exists(ud.donestamp) precomputed_checksums = {} # Only re-use the precomputed checksums if the donestamp is newer than the # file. Do not rely on the mtime of directories, though. If ud.localpath is # a directory, there will probably not be any checksums anyway. - if (os.path.isdir(ud.localpath) or + if os.path.exists(ud.donestamp) and (os.path.isdir(ud.localpath) or os.path.getmtime(ud.localpath) < os.path.getmtime(ud.donestamp)): try: with open(ud.donestamp, "rb") as cachefile: @@ -853,6 +852,9 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None): if val: cmd = 'export ' + var + '=\"%s\"; %s' % (val, cmd) + # Disable pseudo as it may affect ssh, potentially causing it to hang. + cmd = 'export PSEUDO_DISABLED=1; ' + cmd + logger.debug(1, "Running %s", cmd) success = False @@ -1424,7 +1426,7 @@ class FetchMethod(object): cmd = 'gzip -dc %s > %s' % (file, efile) elif file.endswith('.bz2'): cmd = 'bzip2 -dc %s > %s' % (file, efile) - elif file.endswith('.tar.xz'): + elif file.endswith('.txz') or file.endswith('.tar.xz'): cmd = 'xz -dc %s | tar x --no-same-owner -f -' % file elif file.endswith('.xz'): cmd = 'xz -dc %s > %s' % (file, efile) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/git.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/git.py index 5ef8cd69e..3de83bed1 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/git.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/git.py @@ -125,6 +125,9 @@ class GitProgressHandler(bb.progress.LineFilterProgressHandler): class Git(FetchMethod): + bitbake_dir = os.path.abspath(os.path.join(os.path.dirname(os.path.join(os.path.abspath(__file__))), '..', '..', '..')) + make_shallow_path = os.path.join(bitbake_dir, 'bin', 'git-make-shallow') + """Class to fetch a module or modules from git repositories""" def init(self, d): pass @@ -363,6 +366,7 @@ class Git(FetchMethod): progresshandler = GitProgressHandler(d) runfetchcmd(fetch_cmd, d, log=progresshandler, workdir=ud.clonedir) runfetchcmd("%s prune-packed" % ud.basecmd, d, workdir=ud.clonedir) + runfetchcmd("%s pack-refs --all" % ud.basecmd, d, workdir=ud.clonedir) runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d, workdir=ud.clonedir) try: os.unlink(ud.fullmirror) @@ -445,7 +449,7 @@ class Git(FetchMethod): shallow_branches.append(r) # Make the repository shallow - shallow_cmd = ['git', 'make-shallow', '-s'] + shallow_cmd = [self.make_shallow_path, '-s'] for b in shallow_branches: shallow_cmd.append('-r') shallow_cmd.append(b) @@ -591,7 +595,8 @@ class Git(FetchMethod): tagregex = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or "(?P<pver>([0-9][\.|_]?)+)") try: output = self._lsremote(ud, d, "refs/tags/*") - except bb.fetch2.FetchError or bb.fetch2.NetworkAccess: + except (bb.fetch2.FetchError, bb.fetch2.NetworkAccess) as e: + bb.note("Could not list remote: %s" % str(e)) return pupver verstring = "" diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/npm.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/npm.py index b5f148ca0..730c346a9 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/npm.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/npm.py @@ -195,9 +195,11 @@ class Npm(FetchMethod): outputurl = pdata['dist']['tarball'] data[pkg] = {} data[pkg]['tgz'] = os.path.basename(outputurl) - if not outputurl in fetchedlist: - self._runwget(ud, d, "%s --directory-prefix=%s %s" % (self.basecmd, ud.prefixdir, outputurl), False) - fetchedlist.append(outputurl) + if outputurl in fetchedlist: + return + + self._runwget(ud, d, "%s --directory-prefix=%s %s" % (self.basecmd, ud.prefixdir, outputurl), False) + fetchedlist.append(outputurl) dependencies = pdata.get('dependencies', {}) optionalDependencies = pdata.get('optionalDependencies', {}) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/wget.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/wget.py index 7c49c2b12..8f505b6de 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/wget.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/wget.py @@ -250,6 +250,7 @@ class Wget(FetchMethod): return "" def close(self): pass + closed = False resp = addinfourl(fp_dummy(), r.msg, req.get_full_url()) resp.code = r.status @@ -332,7 +333,8 @@ class Wget(FetchMethod): except (TypeError, ImportError, IOError, netrc.NetrcParseError): pass - opener.open(r) + with opener.open(r) as response: + pass except urllib.error.URLError as e: if try_again: logger.debug(2, "checkstatus: trying again") diff --git a/import-layers/yocto-poky/bitbake/lib/bb/main.py b/import-layers/yocto-poky/bitbake/lib/bb/main.py index 7711b290d..f4474e410 100755 --- a/import-layers/yocto-poky/bitbake/lib/bb/main.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/main.py @@ -292,8 +292,12 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters): help="Writes the event log of the build to a bitbake event json file. " "Use '' (empty string) to assign the name automatically.") - parser.add_option("", "--runall", action="store", dest="runall", - help="Run the specified task for all build targets and their dependencies.") + parser.add_option("", "--runall", action="append", dest="runall", + help="Run the specified task for any recipe in the taskgraph of the specified target (even if it wouldn't otherwise have run).") + + parser.add_option("", "--runonly", action="append", dest="runonly", + help="Run only the specified task within the taskgraph of the specified targets (and any task dependencies those tasks may have).") + options, targets = parser.parse_args(argv) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/parse/__init__.py b/import-layers/yocto-poky/bitbake/lib/bb/parse/__init__.py index 2fc4002db..5397d57a5 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/parse/__init__.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/parse/__init__.py @@ -134,8 +134,9 @@ def resolve_file(fn, d): if not newfn: raise IOError(errno.ENOENT, "file %s not found in %s" % (fn, bbpath)) fn = newfn + else: + mark_dependency(d, fn) - mark_dependency(d, fn) if not os.path.isfile(fn): raise IOError(errno.ENOENT, "file %s not found" % fn) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/parse/ast.py b/import-layers/yocto-poky/bitbake/lib/bb/parse/ast.py index dba4540f5..6690dc51c 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/parse/ast.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/parse/ast.py @@ -335,6 +335,12 @@ def handleInherit(statements, filename, lineno, m): classes = m.group(1) statements.append(InheritNode(filename, lineno, classes)) +def runAnonFuncs(d): + code = [] + for funcname in d.getVar("__BBANONFUNCS", False) or []: + code.append("%s(d)" % funcname) + bb.utils.better_exec("\n".join(code), {"d": d}) + def finalize(fn, d, variant = None): saved_handlers = bb.event.get_handlers().copy() @@ -349,10 +355,7 @@ def finalize(fn, d, variant = None): bb.event.fire(bb.event.RecipePreFinalise(fn), d) bb.data.expandKeys(d) - code = [] - for funcname in d.getVar("__BBANONFUNCS", False) or []: - code.append("%s(d)" % funcname) - bb.utils.better_exec("\n".join(code), {"d": d}) + runAnonFuncs(d) tasklist = d.getVar('__BBTASKS', False) or [] bb.event.fire(bb.event.RecipeTaskPreProcess(fn, list(tasklist)), d) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/parse/parse_py/BBHandler.py b/import-layers/yocto-poky/bitbake/lib/bb/parse/parse_py/BBHandler.py index f89ad2427..e5039e3bd 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/parse/parse_py/BBHandler.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/parse/parse_py/BBHandler.py @@ -131,9 +131,6 @@ def handle(fn, d, include): abs_fn = resolve_file(fn, d) - if include: - bb.parse.mark_dependency(d, abs_fn) - # actual loading statements = get_statements(fn, abs_fn, base_name) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/parse/parse_py/ConfHandler.py b/import-layers/yocto-poky/bitbake/lib/bb/parse/parse_py/ConfHandler.py index 97aa13043..9d3ebe16f 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/parse/parse_py/ConfHandler.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/parse/parse_py/ConfHandler.py @@ -134,9 +134,6 @@ def handle(fn, data, include): abs_fn = resolve_file(fn, data) f = open(abs_fn, 'r') - if include: - bb.parse.mark_dependency(data, abs_fn) - statements = ast.StatementGroup() lineno = 0 while True: diff --git a/import-layers/yocto-poky/bitbake/lib/bb/runqueue.py b/import-layers/yocto-poky/bitbake/lib/bb/runqueue.py index ae12c2504..f2e52cf75 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/runqueue.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/runqueue.py @@ -181,7 +181,7 @@ class RunQueueScheduler(object): if self.rq.stats.active < self.rq.number_tasks: return self.next_buildable_task() - def newbuilable(self, task): + def newbuildable(self, task): self.buildable.append(task) def describe_task(self, taskid): @@ -581,12 +581,6 @@ class RunQueueData: if t in taskData[mc].taskentries: depends.add(t) - def add_resolved_dependencies(mc, fn, tasknames, depends): - for taskname in tasknames: - tid = build_tid(mc, fn, taskname) - if tid in self.runtaskentries: - depends.add(tid) - for mc in taskData: for tid in taskData[mc].taskentries: @@ -673,57 +667,106 @@ class RunQueueData: recursiveitasks[tid].append(newdep) self.runtaskentries[tid].depends = depends + # Remove all self references + self.runtaskentries[tid].depends.discard(tid) #self.dump_data() + self.init_progress_reporter.next_stage() + # Resolve recursive 'recrdeptask' dependencies (Part B) # # e.g. do_sometask[recrdeptask] = "do_someothertask" # (makes sure sometask runs after someothertask of all DEPENDS, RDEPENDS and intertask dependencies, recursively) # We need to do this separately since we need all of runtaskentries[*].depends to be complete before this is processed - self.init_progress_reporter.next_stage(len(recursivetasks)) - extradeps = {} - for taskcounter, tid in enumerate(recursivetasks): - extradeps[tid] = set(self.runtaskentries[tid].depends) - - tasknames = recursivetasks[tid] - seendeps = set() - - def generate_recdeps(t): - newdeps = set() - (mc, fn, taskname, _) = split_tid_mcfn(t) - add_resolved_dependencies(mc, fn, tasknames, newdeps) - extradeps[tid].update(newdeps) - seendeps.add(t) - newdeps.add(t) - for i in newdeps: - if i not in self.runtaskentries: - # Not all recipes might have the recrdeptask task as a task - continue - task = self.runtaskentries[i].task - for n in self.runtaskentries[i].depends: - if n not in seendeps: - generate_recdeps(n) - generate_recdeps(tid) - if tid in recursiveitasks: - for dep in recursiveitasks[tid]: - generate_recdeps(dep) - self.init_progress_reporter.update(taskcounter) + # Generating/interating recursive lists of dependencies is painful and potentially slow + # Precompute recursive task dependencies here by: + # a) create a temp list of reverse dependencies (revdeps) + # b) walk up the ends of the chains (when a given task no longer has dependencies i.e. len(deps) == 0) + # c) combine the total list of dependencies in cumulativedeps + # d) optimise by pre-truncating 'task' off the items in cumulativedeps (keeps items in sets lower) - # Remove circular references so that do_a[recrdeptask] = "do_a do_b" can work - for tid in recursivetasks: - extradeps[tid].difference_update(recursivetasksselfref) + revdeps = {} + deps = {} + cumulativedeps = {} for tid in self.runtaskentries: - task = self.runtaskentries[tid].task - # Add in extra dependencies - if tid in extradeps: - self.runtaskentries[tid].depends = extradeps[tid] - # Remove all self references - if tid in self.runtaskentries[tid].depends: - logger.debug(2, "Task %s contains self reference!", tid) - self.runtaskentries[tid].depends.remove(tid) + deps[tid] = set(self.runtaskentries[tid].depends) + revdeps[tid] = set() + cumulativedeps[tid] = set() + # Generate a temp list of reverse dependencies + for tid in self.runtaskentries: + for dep in self.runtaskentries[tid].depends: + revdeps[dep].add(tid) + # Find the dependency chain endpoints + endpoints = set() + for tid in self.runtaskentries: + if len(deps[tid]) == 0: + endpoints.add(tid) + # Iterate the chains collating dependencies + while endpoints: + next = set() + for tid in endpoints: + for dep in revdeps[tid]: + cumulativedeps[dep].add(fn_from_tid(tid)) + cumulativedeps[dep].update(cumulativedeps[tid]) + if tid in deps[dep]: + deps[dep].remove(tid) + if len(deps[dep]) == 0: + next.add(dep) + endpoints = next + #for tid in deps: + # if len(deps[tid]) != 0: + # bb.warn("Sanity test failure, dependencies left for %s (%s)" % (tid, deps[tid])) + + # Loop here since recrdeptasks can depend upon other recrdeptasks and we have to + # resolve these recursively until we aren't adding any further extra dependencies + extradeps = True + while extradeps: + extradeps = 0 + for tid in recursivetasks: + tasknames = recursivetasks[tid] + + totaldeps = set(self.runtaskentries[tid].depends) + if tid in recursiveitasks: + totaldeps.update(recursiveitasks[tid]) + for dep in recursiveitasks[tid]: + if dep not in self.runtaskentries: + continue + totaldeps.update(self.runtaskentries[dep].depends) + + deps = set() + for dep in totaldeps: + if dep in cumulativedeps: + deps.update(cumulativedeps[dep]) + + for t in deps: + for taskname in tasknames: + newtid = t + ":" + taskname + if newtid == tid: + continue + if newtid in self.runtaskentries and newtid not in self.runtaskentries[tid].depends: + extradeps += 1 + self.runtaskentries[tid].depends.add(newtid) + + # Handle recursive tasks which depend upon other recursive tasks + deps = set() + for dep in self.runtaskentries[tid].depends.intersection(recursivetasks): + deps.update(self.runtaskentries[dep].depends.difference(self.runtaskentries[tid].depends)) + for newtid in deps: + for taskname in tasknames: + if not newtid.endswith(":" + taskname): + continue + if newtid in self.runtaskentries: + extradeps += 1 + self.runtaskentries[tid].depends.add(newtid) + + bb.debug(1, "Added %s recursive dependencies in this loop" % extradeps) + + # Remove recrdeptask circular references so that do_a[recrdeptask] = "do_a do_b" can work + for tid in recursivetasksselfref: + self.runtaskentries[tid].depends.difference_update(recursivetasksselfref) self.init_progress_reporter.next_stage() @@ -798,30 +841,57 @@ class RunQueueData: # # Once all active tasks are marked, prune the ones we don't need. - delcount = 0 + delcount = {} for tid in list(self.runtaskentries.keys()): if tid not in runq_build: + delcount[tid] = self.runtaskentries[tid] del self.runtaskentries[tid] - delcount += 1 - self.init_progress_reporter.next_stage() + # Handle --runall + if self.cooker.configuration.runall: + # re-run the mark_active and then drop unused tasks from new list + runq_build = {} - if self.cooker.configuration.runall is not None: - runall = "do_%s" % self.cooker.configuration.runall - runall_tids = { k: v for k, v in self.runtaskentries.items() if taskname_from_tid(k) == runall } + for task in self.cooker.configuration.runall: + runall_tids = set() + for tid in list(self.runtaskentries): + wanttid = fn_from_tid(tid) + ":do_%s" % task + if wanttid in delcount: + self.runtaskentries[wanttid] = delcount[wanttid] + if wanttid in self.runtaskentries: + runall_tids.add(wanttid) + for tid in list(runall_tids): + mark_active(tid,1) + + for tid in list(self.runtaskentries.keys()): + if tid not in runq_build: + delcount[tid] = self.runtaskentries[tid] + del self.runtaskentries[tid] + + if len(self.runtaskentries) == 0: + bb.msg.fatal("RunQueue", "Could not find any tasks with the tasknames %s to run within the recipes of the taskgraphs of the targets %s" % (str(self.cooker.configuration.runall), str(self.targets))) + + self.init_progress_reporter.next_stage() + + # Handle runonly + if self.cooker.configuration.runonly: # re-run the mark_active and then drop unused tasks from new list runq_build = {} - for tid in list(runall_tids): - mark_active(tid,1) + + for task in self.cooker.configuration.runonly: + runonly_tids = { k: v for k, v in self.runtaskentries.items() if taskname_from_tid(k) == "do_%s" % task } + + for tid in list(runonly_tids): + mark_active(tid,1) for tid in list(self.runtaskentries.keys()): if tid not in runq_build: + delcount[tid] = self.runtaskentries[tid] del self.runtaskentries[tid] - delcount += 1 if len(self.runtaskentries) == 0: - bb.msg.fatal("RunQueue", "No remaining tasks to run for build target %s with runall %s" % (target, runall)) + bb.msg.fatal("RunQueue", "Could not find any tasks with the tasknames %s to run within the taskgraphs of the targets %s" % (str(self.cooker.configuration.runonly), str(self.targets))) # # Step D - Sanity checks and computation @@ -834,7 +904,7 @@ class RunQueueData: else: bb.msg.fatal("RunQueue", "No active tasks and not in --continue mode?! Please report this bug.") - logger.verbose("Pruned %s inactive tasks, %s left", delcount, len(self.runtaskentries)) + logger.verbose("Pruned %s inactive tasks, %s left", len(delcount), len(self.runtaskentries)) logger.verbose("Assign Weightings") @@ -1781,7 +1851,7 @@ class RunQueueExecuteTasks(RunQueueExecute): def setbuildable(self, task): self.runq_buildable.add(task) - self.sched.newbuilable(task) + self.sched.newbuildable(task) def task_completeoutright(self, task): """ diff --git a/import-layers/yocto-poky/bitbake/lib/bb/server/process.py b/import-layers/yocto-poky/bitbake/lib/bb/server/process.py index 3d31355fd..828159ed7 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/server/process.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/server/process.py @@ -223,6 +223,8 @@ class ProcessServer(multiprocessing.Process): try: self.cooker.shutdown(True) + self.cooker.notifier.stop() + self.cooker.confignotifier.stop() except: pass diff --git a/import-layers/yocto-poky/bitbake/lib/bb/shell.py b/import-layers/yocto-poky/bitbake/lib/bb/shell.py deleted file mode 100644 index 1dd8d54bd..000000000 --- a/import-layers/yocto-poky/bitbake/lib/bb/shell.py +++ /dev/null @@ -1,820 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -########################################################################## -# -# Copyright (C) 2005-2006 Michael 'Mickey' Lauer <mickey@Vanille.de> -# Copyright (C) 2005-2006 Vanille Media -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -########################################################################## -# -# Thanks to: -# * Holger Freyther <zecke@handhelds.org> -# * Justin Patrin <papercrane@reversefold.com> -# -########################################################################## - -""" -BitBake Shell - -IDEAS: - * list defined tasks per package - * list classes - * toggle force - * command to reparse just one (or more) bbfile(s) - * automatic check if reparsing is necessary (inotify?) - * frontend for bb file manipulation - * more shell-like features: - - output control, i.e. pipe output into grep, sort, etc. - - job control, i.e. bring running commands into background and foreground - * start parsing in background right after startup - * ncurses interface - -PROBLEMS: - * force doesn't always work - * readline completion for commands with more than one parameters - -""" - -########################################################################## -# Import and setup global variables -########################################################################## - -from __future__ import print_function -from functools import reduce -try: - set -except NameError: - from sets import Set as set -import sys, os, readline, socket, httplib, urllib, commands, popen2, shlex, Queue, fnmatch -from bb import data, parse, build, cache, taskdata, runqueue, providers as Providers - -__version__ = "0.5.3.1" -__credits__ = """BitBake Shell Version %s (C) 2005 Michael 'Mickey' Lauer <mickey@Vanille.de> -Type 'help' for more information, press CTRL-D to exit.""" % __version__ - -cmds = {} -leave_mainloop = False -last_exception = None -cooker = None -parsed = False -debug = os.environ.get( "BBSHELL_DEBUG", "" ) - -########################################################################## -# Class BitBakeShellCommands -########################################################################## - -class BitBakeShellCommands: - """This class contains the valid commands for the shell""" - - def __init__( self, shell ): - """Register all the commands""" - self._shell = shell - for attr in BitBakeShellCommands.__dict__: - if not attr.startswith( "_" ): - if attr.endswith( "_" ): - command = attr[:-1].lower() - else: - command = attr[:].lower() - method = getattr( BitBakeShellCommands, attr ) - debugOut( "registering command '%s'" % command ) - # scan number of arguments - usage = getattr( method, "usage", "" ) - if usage != "<...>": - numArgs = len( usage.split() ) - else: - numArgs = -1 - shell.registerCommand( command, method, numArgs, "%s %s" % ( command, usage ), method.__doc__ ) - - def _checkParsed( self ): - if not parsed: - print("SHELL: This command needs to parse bbfiles...") - self.parse( None ) - - def _findProvider( self, item ): - self._checkParsed() - # Need to use taskData for this information - preferred = data.getVar( "PREFERRED_PROVIDER_%s" % item, cooker.configuration.data, 1 ) - if not preferred: preferred = item - try: - lv, lf, pv, pf = Providers.findBestProvider(preferred, cooker.configuration.data, cooker.status) - except KeyError: - if item in cooker.status.providers: - pf = cooker.status.providers[item][0] - else: - pf = None - return pf - - def alias( self, params ): - """Register a new name for a command""" - new, old = params - if not old in cmds: - print("ERROR: Command '%s' not known" % old) - else: - cmds[new] = cmds[old] - print("OK") - alias.usage = "<alias> <command>" - - def buffer( self, params ): - """Dump specified output buffer""" - index = params[0] - print(self._shell.myout.buffer( int( index ) )) - buffer.usage = "<index>" - - def buffers( self, params ): - """Show the available output buffers""" - commands = self._shell.myout.bufferedCommands() - if not commands: - print("SHELL: No buffered commands available yet. Start doing something.") - else: - print("="*35, "Available Output Buffers", "="*27) - for index, cmd in enumerate( commands ): - print("| %s %s" % ( str( index ).ljust( 3 ), cmd )) - print("="*88) - - def build( self, params, cmd = "build" ): - """Build a providee""" - global last_exception - globexpr = params[0] - self._checkParsed() - names = globfilter( cooker.status.pkg_pn, globexpr ) - if len( names ) == 0: names = [ globexpr ] - print("SHELL: Building %s" % ' '.join( names )) - - td = taskdata.TaskData(cooker.configuration.abort) - localdata = data.createCopy(cooker.configuration.data) - data.update_data(localdata) - data.expandKeys(localdata) - - try: - tasks = [] - for name in names: - td.add_provider(localdata, cooker.status, name) - providers = td.get_provider(name) - - if len(providers) == 0: - raise Providers.NoProvider - - tasks.append([name, "do_%s" % cmd]) - - td.add_unresolved(localdata, cooker.status) - - rq = runqueue.RunQueue(cooker, localdata, cooker.status, td, tasks) - rq.prepare_runqueue() - rq.execute_runqueue() - - except Providers.NoProvider: - print("ERROR: No Provider") - last_exception = Providers.NoProvider - - except runqueue.TaskFailure as fnids: - last_exception = runqueue.TaskFailure - - except build.FuncFailed as e: - print("ERROR: Couldn't build '%s'" % names) - last_exception = e - - - build.usage = "<providee>" - - def clean( self, params ): - """Clean a providee""" - self.build( params, "clean" ) - clean.usage = "<providee>" - - def compile( self, params ): - """Execute 'compile' on a providee""" - self.build( params, "compile" ) - compile.usage = "<providee>" - - def configure( self, params ): - """Execute 'configure' on a providee""" - self.build( params, "configure" ) - configure.usage = "<providee>" - - def install( self, params ): - """Execute 'install' on a providee""" - self.build( params, "install" ) - install.usage = "<providee>" - - def edit( self, params ): - """Call $EDITOR on a providee""" - name = params[0] - bbfile = self._findProvider( name ) - if bbfile is not None: - os.system( "%s %s" % ( os.environ.get( "EDITOR", "vi" ), bbfile ) ) - else: - print("ERROR: Nothing provides '%s'" % name) - edit.usage = "<providee>" - - def environment( self, params ): - """Dump out the outer BitBake environment""" - cooker.showEnvironment() - - def exit_( self, params ): - """Leave the BitBake Shell""" - debugOut( "setting leave_mainloop to true" ) - global leave_mainloop - leave_mainloop = True - - def fetch( self, params ): - """Fetch a providee""" - self.build( params, "fetch" ) - fetch.usage = "<providee>" - - def fileBuild( self, params, cmd = "build" ): - """Parse and build a .bb file""" - global last_exception - name = params[0] - bf = completeFilePath( name ) - print("SHELL: Calling '%s' on '%s'" % ( cmd, bf )) - - try: - cooker.buildFile(bf, cmd) - except parse.ParseError: - print("ERROR: Unable to open or parse '%s'" % bf) - except build.FuncFailed as e: - print("ERROR: Couldn't build '%s'" % name) - last_exception = e - - fileBuild.usage = "<bbfile>" - - def fileClean( self, params ): - """Clean a .bb file""" - self.fileBuild( params, "clean" ) - fileClean.usage = "<bbfile>" - - def fileEdit( self, params ): - """Call $EDITOR on a .bb file""" - name = params[0] - os.system( "%s %s" % ( os.environ.get( "EDITOR", "vi" ), completeFilePath( name ) ) ) - fileEdit.usage = "<bbfile>" - - def fileRebuild( self, params ): - """Rebuild (clean & build) a .bb file""" - self.fileBuild( params, "rebuild" ) - fileRebuild.usage = "<bbfile>" - - def fileReparse( self, params ): - """(re)Parse a bb file""" - bbfile = params[0] - print("SHELL: Parsing '%s'" % bbfile) - parse.update_mtime( bbfile ) - cooker.parser.reparse(bbfile) - if False: #fromCache: - print("SHELL: File has not been updated, not reparsing") - else: - print("SHELL: Parsed") - fileReparse.usage = "<bbfile>" - - def abort( self, params ): - """Toggle abort task execution flag (see bitbake -k)""" - cooker.configuration.abort = not cooker.configuration.abort - print("SHELL: Abort Flag is now '%s'" % repr( cooker.configuration.abort )) - - def force( self, params ): - """Toggle force task execution flag (see bitbake -f)""" - cooker.configuration.force = not cooker.configuration.force - print("SHELL: Force Flag is now '%s'" % repr( cooker.configuration.force )) - - def help( self, params ): - """Show a comprehensive list of commands and their purpose""" - print("="*30, "Available Commands", "="*30) - for cmd in sorted(cmds): - function, numparams, usage, helptext = cmds[cmd] - print("| %s | %s" % (usage.ljust(30), helptext)) - print("="*78) - - def lastError( self, params ): - """Show the reason or log that was produced by the last BitBake event exception""" - if last_exception is None: - print("SHELL: No Errors yet (Phew)...") - else: - reason, event = last_exception.args - print("SHELL: Reason for the last error: '%s'" % reason) - if ':' in reason: - msg, filename = reason.split( ':' ) - filename = filename.strip() - print("SHELL: Dumping log file for last error:") - try: - print(open( filename ).read()) - except IOError: - print("ERROR: Couldn't open '%s'" % filename) - - def match( self, params ): - """Dump all files or providers matching a glob expression""" - what, globexpr = params - if what == "files": - self._checkParsed() - for key in globfilter( cooker.status.pkg_fn, globexpr ): print(key) - elif what == "providers": - self._checkParsed() - for key in globfilter( cooker.status.pkg_pn, globexpr ): print(key) - else: - print("Usage: match %s" % self.print_.usage) - match.usage = "<files|providers> <glob>" - - def new( self, params ): - """Create a new .bb file and open the editor""" - dirname, filename = params - packages = '/'.join( data.getVar( "BBFILES", cooker.configuration.data, 1 ).split('/')[:-2] ) - fulldirname = "%s/%s" % ( packages, dirname ) - - if not os.path.exists( fulldirname ): - print("SHELL: Creating '%s'" % fulldirname) - os.mkdir( fulldirname ) - if os.path.exists( fulldirname ) and os.path.isdir( fulldirname ): - if os.path.exists( "%s/%s" % ( fulldirname, filename ) ): - print("SHELL: ERROR: %s/%s already exists" % ( fulldirname, filename )) - return False - print("SHELL: Creating '%s/%s'" % ( fulldirname, filename )) - newpackage = open( "%s/%s" % ( fulldirname, filename ), "w" ) - print("""DESCRIPTION = "" -SECTION = "" -AUTHOR = "" -HOMEPAGE = "" -MAINTAINER = "" -LICENSE = "GPL" -PR = "r0" - -SRC_URI = "" - -#inherit base - -#do_configure() { -# -#} - -#do_compile() { -# -#} - -#do_stage() { -# -#} - -#do_install() { -# -#} -""", file=newpackage) - newpackage.close() - os.system( "%s %s/%s" % ( os.environ.get( "EDITOR" ), fulldirname, filename ) ) - new.usage = "<directory> <filename>" - - def package( self, params ): - """Execute 'package' on a providee""" - self.build( params, "package" ) - package.usage = "<providee>" - - def pasteBin( self, params ): - """Send a command + output buffer to the pastebin at http://rafb.net/paste""" - index = params[0] - contents = self._shell.myout.buffer( int( index ) ) - sendToPastebin( "output of " + params[0], contents ) - pasteBin.usage = "<index>" - - def pasteLog( self, params ): - """Send the last event exception error log (if there is one) to http://rafb.net/paste""" - if last_exception is None: - print("SHELL: No Errors yet (Phew)...") - else: - reason, event = last_exception.args - print("SHELL: Reason for the last error: '%s'" % reason) - if ':' in reason: - msg, filename = reason.split( ':' ) - filename = filename.strip() - print("SHELL: Pasting log file to pastebin...") - - file = open( filename ).read() - sendToPastebin( "contents of " + filename, file ) - - def patch( self, params ): - """Execute 'patch' command on a providee""" - self.build( params, "patch" ) - patch.usage = "<providee>" - - def parse( self, params ): - """(Re-)parse .bb files and calculate the dependency graph""" - cooker.status = cache.CacheData(cooker.caches_array) - ignore = data.getVar("ASSUME_PROVIDED", cooker.configuration.data, 1) or "" - cooker.status.ignored_dependencies = set( ignore.split() ) - cooker.handleCollections( data.getVar("BBFILE_COLLECTIONS", cooker.configuration.data, 1) ) - - (filelist, masked) = cooker.collect_bbfiles() - cooker.parse_bbfiles(filelist, masked, cooker.myProgressCallback) - cooker.buildDepgraph() - global parsed - parsed = True - print() - - def reparse( self, params ): - """(re)Parse a providee's bb file""" - bbfile = self._findProvider( params[0] ) - if bbfile is not None: - print("SHELL: Found bbfile '%s' for '%s'" % ( bbfile, params[0] )) - self.fileReparse( [ bbfile ] ) - else: - print("ERROR: Nothing provides '%s'" % params[0]) - reparse.usage = "<providee>" - - def getvar( self, params ): - """Dump the contents of an outer BitBake environment variable""" - var = params[0] - value = data.getVar( var, cooker.configuration.data, 1 ) - print(value) - getvar.usage = "<variable>" - - def peek( self, params ): - """Dump contents of variable defined in providee's metadata""" - name, var = params - bbfile = self._findProvider( name ) - if bbfile is not None: - the_data = cache.Cache.loadDataFull(bbfile, cooker.configuration.data) - value = the_data.getVar( var, 1 ) - print(value) - else: - print("ERROR: Nothing provides '%s'" % name) - peek.usage = "<providee> <variable>" - - def poke( self, params ): - """Set contents of variable defined in providee's metadata""" - name, var, value = params - bbfile = self._findProvider( name ) - if bbfile is not None: - print("ERROR: Sorry, this functionality is currently broken") - #d = cooker.pkgdata[bbfile] - #data.setVar( var, value, d ) - - # mark the change semi persistant - #cooker.pkgdata.setDirty(bbfile, d) - #print "OK" - else: - print("ERROR: Nothing provides '%s'" % name) - poke.usage = "<providee> <variable> <value>" - - def print_( self, params ): - """Dump all files or providers""" - what = params[0] - if what == "files": - self._checkParsed() - for key in cooker.status.pkg_fn: print(key) - elif what == "providers": - self._checkParsed() - for key in cooker.status.providers: print(key) - else: - print("Usage: print %s" % self.print_.usage) - print_.usage = "<files|providers>" - - def python( self, params ): - """Enter the expert mode - an interactive BitBake Python Interpreter""" - sys.ps1 = "EXPERT BB>>> " - sys.ps2 = "EXPERT BB... " - import code - interpreter = code.InteractiveConsole( dict( globals() ) ) - interpreter.interact( "SHELL: Expert Mode - BitBake Python %s\nType 'help' for more information, press CTRL-D to switch back to BBSHELL." % sys.version ) - - def showdata( self, params ): - """Execute 'showdata' on a providee""" - cooker.showEnvironment(None, params) - showdata.usage = "<providee>" - - def setVar( self, params ): - """Set an outer BitBake environment variable""" - var, value = params - data.setVar( var, value, cooker.configuration.data ) - print("OK") - setVar.usage = "<variable> <value>" - - def rebuild( self, params ): - """Clean and rebuild a .bb file or a providee""" - self.build( params, "clean" ) - self.build( params, "build" ) - rebuild.usage = "<providee>" - - def shell( self, params ): - """Execute a shell command and dump the output""" - if params != "": - print(commands.getoutput( " ".join( params ) )) - shell.usage = "<...>" - - def stage( self, params ): - """Execute 'stage' on a providee""" - self.build( params, "populate_staging" ) - stage.usage = "<providee>" - - def status( self, params ): - """<just for testing>""" - print("-" * 78) - print("building list = '%s'" % cooker.building_list) - print("build path = '%s'" % cooker.build_path) - print("consider_msgs_cache = '%s'" % cooker.consider_msgs_cache) - print("build stats = '%s'" % cooker.stats) - if last_exception is not None: print("last_exception = '%s'" % repr( last_exception.args )) - print("memory output contents = '%s'" % self._shell.myout._buffer) - - def test( self, params ): - """<just for testing>""" - print("testCommand called with '%s'" % params) - - def unpack( self, params ): - """Execute 'unpack' on a providee""" - self.build( params, "unpack" ) - unpack.usage = "<providee>" - - def which( self, params ): - """Computes the providers for a given providee""" - # Need to use taskData for this information - item = params[0] - - self._checkParsed() - - preferred = data.getVar( "PREFERRED_PROVIDER_%s" % item, cooker.configuration.data, 1 ) - if not preferred: preferred = item - - try: - lv, lf, pv, pf = Providers.findBestProvider(preferred, cooker.configuration.data, cooker.status) - except KeyError: - lv, lf, pv, pf = (None,)*4 - - try: - providers = cooker.status.providers[item] - except KeyError: - print("SHELL: ERROR: Nothing provides", preferred) - else: - for provider in providers: - if provider == pf: provider = " (***) %s" % provider - else: provider = " %s" % provider - print(provider) - which.usage = "<providee>" - -########################################################################## -# Common helper functions -########################################################################## - -def completeFilePath( bbfile ): - """Get the complete bbfile path""" - if not cooker.status: return bbfile - if not cooker.status.pkg_fn: return bbfile - for key in cooker.status.pkg_fn: - if key.endswith( bbfile ): - return key - return bbfile - -def sendToPastebin( desc, content ): - """Send content to http://oe.pastebin.com""" - mydata = {} - mydata["lang"] = "Plain Text" - mydata["desc"] = desc - mydata["cvt_tabs"] = "No" - mydata["nick"] = "%s@%s" % ( os.environ.get( "USER", "unknown" ), socket.gethostname() or "unknown" ) - mydata["text"] = content - params = urllib.urlencode( mydata ) - headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"} - - host = "rafb.net" - conn = httplib.HTTPConnection( "%s:80" % host ) - conn.request("POST", "/paste/paste.php", params, headers ) - - response = conn.getresponse() - conn.close() - - if response.status == 302: - location = response.getheader( "location" ) or "unknown" - print("SHELL: Pasted to http://%s%s" % ( host, location )) - else: - print("ERROR: %s %s" % ( response.status, response.reason )) - -def completer( text, state ): - """Return a possible readline completion""" - debugOut( "completer called with text='%s', state='%d'" % ( text, state ) ) - - if state == 0: - line = readline.get_line_buffer() - if " " in line: - line = line.split() - # we are in second (or more) argument - if line[0] in cmds and hasattr( cmds[line[0]][0], "usage" ): # known command and usage - u = getattr( cmds[line[0]][0], "usage" ).split()[0] - if u == "<variable>": - allmatches = cooker.configuration.data.keys() - elif u == "<bbfile>": - if cooker.status.pkg_fn is None: allmatches = [ "(No Matches Available. Parsed yet?)" ] - else: allmatches = [ x.split("/")[-1] for x in cooker.status.pkg_fn ] - elif u == "<providee>": - if cooker.status.pkg_fn is None: allmatches = [ "(No Matches Available. Parsed yet?)" ] - else: allmatches = cooker.status.providers.iterkeys() - else: allmatches = [ "(No tab completion available for this command)" ] - else: allmatches = [ "(No tab completion available for this command)" ] - else: - # we are in first argument - allmatches = cmds.iterkeys() - - completer.matches = [ x for x in allmatches if x[:len(text)] == text ] - #print "completer.matches = '%s'" % completer.matches - if len( completer.matches ) > state: - return completer.matches[state] - else: - return None - -def debugOut( text ): - if debug: - sys.stderr.write( "( %s )\n" % text ) - -def columnize( alist, width = 80 ): - """ - A word-wrap function that preserves existing line breaks - and most spaces in the text. Expects that existing line - breaks are posix newlines (\n). - """ - return reduce(lambda line, word, width=width: '%s%s%s' % - (line, - ' \n'[(len(line[line.rfind('\n')+1:]) - + len(word.split('\n', 1)[0] - ) >= width)], - word), - alist - ) - -def globfilter( names, pattern ): - return fnmatch.filter( names, pattern ) - -########################################################################## -# Class MemoryOutput -########################################################################## - -class MemoryOutput: - """File-like output class buffering the output of the last 10 commands""" - def __init__( self, delegate ): - self.delegate = delegate - self._buffer = [] - self.text = [] - self._command = None - - def startCommand( self, command ): - self._command = command - self.text = [] - def endCommand( self ): - if self._command is not None: - if len( self._buffer ) == 10: del self._buffer[0] - self._buffer.append( ( self._command, self.text ) ) - def removeLast( self ): - if self._buffer: - del self._buffer[ len( self._buffer ) - 1 ] - self.text = [] - self._command = None - def lastBuffer( self ): - if self._buffer: - return self._buffer[ len( self._buffer ) -1 ][1] - def bufferedCommands( self ): - return [ cmd for cmd, output in self._buffer ] - def buffer( self, i ): - if i < len( self._buffer ): - return "BB>> %s\n%s" % ( self._buffer[i][0], "".join( self._buffer[i][1] ) ) - else: return "ERROR: Invalid buffer number. Buffer needs to be in (0, %d)" % ( len( self._buffer ) - 1 ) - def write( self, text ): - if self._command is not None and text != "BB>> ": self.text.append( text ) - if self.delegate is not None: self.delegate.write( text ) - def flush( self ): - return self.delegate.flush() - def fileno( self ): - return self.delegate.fileno() - def isatty( self ): - return self.delegate.isatty() - -########################################################################## -# Class BitBakeShell -########################################################################## - -class BitBakeShell: - - def __init__( self ): - """Register commands and set up readline""" - self.commandQ = Queue.Queue() - self.commands = BitBakeShellCommands( self ) - self.myout = MemoryOutput( sys.stdout ) - self.historyfilename = os.path.expanduser( "~/.bbsh_history" ) - self.startupfilename = os.path.expanduser( "~/.bbsh_startup" ) - - readline.set_completer( completer ) - readline.set_completer_delims( " " ) - readline.parse_and_bind("tab: complete") - - try: - readline.read_history_file( self.historyfilename ) - except IOError: - pass # It doesn't exist yet. - - print(__credits__) - - def cleanup( self ): - """Write readline history and clean up resources""" - debugOut( "writing command history" ) - try: - readline.write_history_file( self.historyfilename ) - except: - print("SHELL: Unable to save command history") - - def registerCommand( self, command, function, numparams = 0, usage = "", helptext = "" ): - """Register a command""" - if usage == "": usage = command - if helptext == "": helptext = function.__doc__ or "<not yet documented>" - cmds[command] = ( function, numparams, usage, helptext ) - - def processCommand( self, command, params ): - """Process a command. Check number of params and print a usage string, if appropriate""" - debugOut( "processing command '%s'..." % command ) - try: - function, numparams, usage, helptext = cmds[command] - except KeyError: - print("SHELL: ERROR: '%s' command is not a valid command." % command) - self.myout.removeLast() - else: - if (numparams != -1) and (not len( params ) == numparams): - print("Usage: '%s'" % usage) - return - - result = function( self.commands, params ) - debugOut( "result was '%s'" % result ) - - def processStartupFile( self ): - """Read and execute all commands found in $HOME/.bbsh_startup""" - if os.path.exists( self.startupfilename ): - startupfile = open( self.startupfilename, "r" ) - for cmdline in startupfile: - debugOut( "processing startup line '%s'" % cmdline ) - if not cmdline: - continue - if "|" in cmdline: - print("ERROR: '|' in startup file is not allowed. Ignoring line") - continue - self.commandQ.put( cmdline.strip() ) - - def main( self ): - """The main command loop""" - while not leave_mainloop: - try: - if self.commandQ.empty(): - sys.stdout = self.myout.delegate - cmdline = raw_input( "BB>> " ) - sys.stdout = self.myout - else: - cmdline = self.commandQ.get() - if cmdline: - allCommands = cmdline.split( ';' ) - for command in allCommands: - pipecmd = None - # - # special case for expert mode - if command == 'python': - sys.stdout = self.myout.delegate - self.processCommand( command, "" ) - sys.stdout = self.myout - else: - self.myout.startCommand( command ) - if '|' in command: # disable output - command, pipecmd = command.split( '|' ) - delegate = self.myout.delegate - self.myout.delegate = None - tokens = shlex.split( command, True ) - self.processCommand( tokens[0], tokens[1:] or "" ) - self.myout.endCommand() - if pipecmd is not None: # restore output - self.myout.delegate = delegate - - pipe = popen2.Popen4( pipecmd ) - pipe.tochild.write( "\n".join( self.myout.lastBuffer() ) ) - pipe.tochild.close() - sys.stdout.write( pipe.fromchild.read() ) - # - except EOFError: - print() - return - except KeyboardInterrupt: - print() - -########################################################################## -# Start function - called from the BitBake command line utility -########################################################################## - -def start( aCooker ): - global cooker - cooker = aCooker - bbshell = BitBakeShell() - bbshell.processStartupFile() - bbshell.main() - bbshell.cleanup() - -if __name__ == "__main__": - print("SHELL: Sorry, this program should only be called by BitBake.") diff --git a/import-layers/yocto-poky/bitbake/lib/bb/tests/event.py b/import-layers/yocto-poky/bitbake/lib/bb/tests/event.py index c7eb1fe44..d3a5f6269 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/tests/event.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/tests/event.py @@ -30,28 +30,45 @@ import time import pickle from unittest.mock import Mock from unittest.mock import call +from bb.msg import BBLogFormatter -class EventQueueStub(): - """ Class used as specification for UI event handler queue stub objects """ +class EventQueueStubBase(object): + """ Base class for EventQueueStub classes """ def __init__(self): + self.event_calls = [] return - def send(self, event): + def _store_event_data_string(self, event): + if isinstance(event, logging.LogRecord): + formatter = BBLogFormatter("%(levelname)s: %(message)s") + self.event_calls.append(formatter.format(event)) + else: + self.event_calls.append(bb.event.getName(event)) return -class PickleEventQueueStub(): +class EventQueueStub(EventQueueStubBase): + """ Class used as specification for UI event handler queue stub objects """ + def __init__(self): + super(EventQueueStub, self).__init__() + + def send(self, event): + super(EventQueueStub, self)._store_event_data_string(event) + + +class PickleEventQueueStub(EventQueueStubBase): """ Class used as specification for UI event handler queue stub objects with sendpickle method """ def __init__(self): - return + super(PickleEventQueueStub, self).__init__() def sendpickle(self, pickled_event): - return + event = pickle.loads(pickled_event) + super(PickleEventQueueStub, self)._store_event_data_string(event) -class UIClientStub(): +class UIClientStub(object): """ Class used as specification for UI event handler stub objects """ def __init__(self): self.event = None @@ -59,7 +76,7 @@ class UIClientStub(): class EventHandlingTest(unittest.TestCase): """ Event handling test class """ - _threadlock_test_calls = [] + def setUp(self): self._test_process = Mock() @@ -179,6 +196,33 @@ class EventHandlingTest(unittest.TestCase): self.assertEqual(self._test_process.event_handler2.call_args_list, expected_event_handler2) + def test_class_handler_filters(self): + """ Test filters for class handlers """ + mask = ["bb.event.OperationStarted"] + result = bb.event.register("event_handler1", + self._test_process.event_handler1, + mask) + self.assertEqual(result, bb.event.Registered) + result = bb.event.register("event_handler2", + self._test_process.event_handler2, + "*") + self.assertEqual(result, bb.event.Registered) + bb.event.set_eventfilter( + lambda name, handler, event, d : + name == 'event_handler2' and + bb.event.getName(event) == "OperationStarted") + event1 = bb.event.OperationStarted() + event2 = bb.event.OperationCompleted(total=123) + bb.event.fire_class_handlers(event1, None) + bb.event.fire_class_handlers(event2, None) + bb.event.fire_class_handlers(event2, None) + expected_event_handler1 = [] + expected_event_handler2 = [call(event1)] + self.assertEqual(self._test_process.event_handler1.call_args_list, + expected_event_handler1) + self.assertEqual(self._test_process.event_handler2.call_args_list, + expected_event_handler2) + def test_change_handler_event_mapping(self): """ Test changing the event mapping for class handlers """ event1 = bb.event.OperationStarted() @@ -196,8 +240,8 @@ class EventHandlingTest(unittest.TestCase): expected) # unregister handler and register it only for OperationStarted - result = bb.event.remove("event_handler1", - self._test_process.event_handler1) + bb.event.remove("event_handler1", + self._test_process.event_handler1) mask = ["bb.event.OperationStarted"] result = bb.event.register("event_handler1", self._test_process.event_handler1, @@ -210,8 +254,8 @@ class EventHandlingTest(unittest.TestCase): expected) # unregister handler and register it only for OperationCompleted - result = bb.event.remove("event_handler1", - self._test_process.event_handler1) + bb.event.remove("event_handler1", + self._test_process.event_handler1) mask = ["bb.event.OperationCompleted"] result = bb.event.register("event_handler1", self._test_process.event_handler1, @@ -259,6 +303,61 @@ class EventHandlingTest(unittest.TestCase): self.assertEqual(self._test_ui2.event.sendpickle.call_args_list, expected) + def test_ui_handler_mask_filter(self): + """ Test filters for UI handlers """ + mask = ["bb.event.OperationStarted"] + debug_domains = {} + self._test_ui1.event = Mock(spec_set=EventQueueStub) + result = bb.event.register_UIHhandler(self._test_ui1, mainui=True) + bb.event.set_UIHmask(result, logging.INFO, debug_domains, mask) + self._test_ui2.event = Mock(spec_set=PickleEventQueueStub) + result = bb.event.register_UIHhandler(self._test_ui2, mainui=True) + bb.event.set_UIHmask(result, logging.INFO, debug_domains, mask) + + event1 = bb.event.OperationStarted() + event2 = bb.event.OperationCompleted(total=1) + + bb.event.fire_ui_handlers(event1, None) + bb.event.fire_ui_handlers(event2, None) + expected = [call(event1)] + self.assertEqual(self._test_ui1.event.send.call_args_list, + expected) + expected = [call(pickle.dumps(event1))] + self.assertEqual(self._test_ui2.event.sendpickle.call_args_list, + expected) + + def test_ui_handler_log_filter(self): + """ Test log filters for UI handlers """ + mask = ["*"] + debug_domains = {'BitBake.Foo': logging.WARNING} + + self._test_ui1.event = EventQueueStub() + result = bb.event.register_UIHhandler(self._test_ui1, mainui=True) + bb.event.set_UIHmask(result, logging.ERROR, debug_domains, mask) + self._test_ui2.event = PickleEventQueueStub() + result = bb.event.register_UIHhandler(self._test_ui2, mainui=True) + bb.event.set_UIHmask(result, logging.ERROR, debug_domains, mask) + + event1 = bb.event.OperationStarted() + bb.event.fire_ui_handlers(event1, None) # All events match + + event_log_handler = bb.event.LogHandler() + logger = logging.getLogger("BitBake") + logger.addHandler(event_log_handler) + logger1 = logging.getLogger("BitBake.Foo") + logger1.warning("Test warning LogRecord1") # Matches debug_domains level + logger1.info("Test info LogRecord") # Filtered out + logger2 = logging.getLogger("BitBake.Bar") + logger2.error("Test error LogRecord") # Matches filter base level + logger2.warning("Test warning LogRecord2") # Filtered out + logger.removeHandler(event_log_handler) + + expected = ['OperationStarted', + 'WARNING: Test warning LogRecord1', + 'ERROR: Test error LogRecord'] + self.assertEqual(self._test_ui1.event.event_calls, expected) + self.assertEqual(self._test_ui2.event.event_calls, expected) + def test_fire(self): """ Test fire method used to trigger class and ui event handlers """ mask = ["bb.event.ConfigParsed"] @@ -289,18 +388,28 @@ class EventHandlingTest(unittest.TestCase): self.assertEqual(self._test_ui1.event.send.call_args_list, expected) + def test_worker_fire(self): + """ Test the triggering of bb.event.worker_fire callback """ + bb.event.worker_fire = Mock() + event = bb.event.Event() + bb.event.fire(event, None) + expected = [call(event, None)] + self.assertEqual(bb.event.worker_fire.call_args_list, expected) + def test_print_ui_queue(self): """ Test print_ui_queue method """ event1 = bb.event.OperationStarted() event2 = bb.event.OperationCompleted(total=123) bb.event.fire(event1, None) bb.event.fire(event2, None) + event_log_handler = bb.event.LogHandler() logger = logging.getLogger("BitBake") - logger.addHandler(bb.event.LogHandler()) + logger.addHandler(event_log_handler) logger.info("Test info LogRecord") logger.warning("Test warning LogRecord") with self.assertLogs("BitBake", level="INFO") as cm: bb.event.print_ui_queue() + logger.removeHandler(event_log_handler) self.assertEqual(cm.output, ["INFO:BitBake:Test info LogRecord", "WARNING:BitBake:Test warning LogRecord"]) @@ -364,6 +473,7 @@ class EventHandlingTest(unittest.TestCase): self.assertEqual(self._threadlock_test_calls, ["w1_ui1", "w1_ui2", "w2_ui1", "w2_ui2"]) + def test_disable_threadlock(self): """ Test disable_threadlock method """ self._set_threadlock_test_mockups() @@ -375,3 +485,502 @@ class EventHandlingTest(unittest.TestCase): # processed before finishing handling the first worker event. self.assertEqual(self._threadlock_test_calls, ["w1_ui1", "w2_ui1", "w1_ui2", "w2_ui2"]) + + +class EventClassesTest(unittest.TestCase): + """ Event classes test class """ + + _worker_pid = 54321 + + def setUp(self): + bb.event.worker_pid = EventClassesTest._worker_pid + + def test_Event(self): + """ Test the Event base class """ + event = bb.event.Event() + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_HeartbeatEvent(self): + """ Test the HeartbeatEvent class """ + time = 10 + event = bb.event.HeartbeatEvent(time) + self.assertEqual(event.time, time) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_OperationStarted(self): + """ Test OperationStarted event class """ + msg = "Foo Bar" + event = bb.event.OperationStarted(msg) + self.assertEqual(event.msg, msg) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_OperationCompleted(self): + """ Test OperationCompleted event class """ + msg = "Foo Bar" + total = 123 + event = bb.event.OperationCompleted(total, msg) + self.assertEqual(event.msg, msg) + self.assertEqual(event.total, total) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_OperationProgress(self): + """ Test OperationProgress event class """ + msg = "Foo Bar" + total = 123 + current = 111 + event = bb.event.OperationProgress(current, total, msg) + self.assertEqual(event.msg, msg + ": %s/%s" % (current, total)) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_ConfigParsed(self): + """ Test the ConfigParsed class """ + event = bb.event.ConfigParsed() + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_MultiConfigParsed(self): + """ Test MultiConfigParsed event class """ + mcdata = {"foobar": "Foo Bar"} + event = bb.event.MultiConfigParsed(mcdata) + self.assertEqual(event.mcdata, mcdata) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_RecipeEvent(self): + """ Test RecipeEvent event base class """ + callback = lambda a: 2 * a + event = bb.event.RecipeEvent(callback) + self.assertEqual(event.fn(1), callback(1)) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_RecipePreFinalise(self): + """ Test RecipePreFinalise event class """ + callback = lambda a: 2 * a + event = bb.event.RecipePreFinalise(callback) + self.assertEqual(event.fn(1), callback(1)) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_RecipeTaskPreProcess(self): + """ Test RecipeTaskPreProcess event class """ + callback = lambda a: 2 * a + tasklist = [("foobar", callback)] + event = bb.event.RecipeTaskPreProcess(callback, tasklist) + self.assertEqual(event.fn(1), callback(1)) + self.assertEqual(event.tasklist, tasklist) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_RecipeParsed(self): + """ Test RecipeParsed event base class """ + callback = lambda a: 2 * a + event = bb.event.RecipeParsed(callback) + self.assertEqual(event.fn(1), callback(1)) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_StampUpdate(self): + targets = ["foo", "bar"] + stampfns = [lambda:"foobar"] + event = bb.event.StampUpdate(targets, stampfns) + self.assertEqual(event.targets, targets) + self.assertEqual(event.stampPrefix, stampfns) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_BuildBase(self): + """ Test base class for bitbake build events """ + name = "foo" + pkgs = ["bar"] + failures = 123 + event = bb.event.BuildBase(name, pkgs, failures) + self.assertEqual(event.name, name) + self.assertEqual(event.pkgs, pkgs) + self.assertEqual(event.getFailures(), failures) + name = event.name = "bar" + pkgs = event.pkgs = ["foo"] + self.assertEqual(event.name, name) + self.assertEqual(event.pkgs, pkgs) + self.assertEqual(event.getFailures(), failures) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_BuildInit(self): + """ Test class for bitbake build invocation events """ + event = bb.event.BuildInit() + self.assertEqual(event.name, None) + self.assertEqual(event.pkgs, []) + self.assertEqual(event.getFailures(), 0) + name = event.name = "bar" + pkgs = event.pkgs = ["foo"] + self.assertEqual(event.name, name) + self.assertEqual(event.pkgs, pkgs) + self.assertEqual(event.getFailures(), 0) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_BuildStarted(self): + """ Test class for build started events """ + name = "foo" + pkgs = ["bar"] + failures = 123 + event = bb.event.BuildStarted(name, pkgs, failures) + self.assertEqual(event.name, name) + self.assertEqual(event.pkgs, pkgs) + self.assertEqual(event.getFailures(), failures) + self.assertEqual(event.msg, "Building Started") + name = event.name = "bar" + pkgs = event.pkgs = ["foo"] + msg = event.msg = "foobar" + self.assertEqual(event.name, name) + self.assertEqual(event.pkgs, pkgs) + self.assertEqual(event.getFailures(), failures) + self.assertEqual(event.msg, msg) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_BuildCompleted(self): + """ Test class for build completed events """ + total = 1000 + name = "foo" + pkgs = ["bar"] + failures = 123 + interrupted = 1 + event = bb.event.BuildCompleted(total, name, pkgs, failures, + interrupted) + self.assertEqual(event.name, name) + self.assertEqual(event.pkgs, pkgs) + self.assertEqual(event.getFailures(), failures) + self.assertEqual(event.msg, "Building Failed") + event2 = bb.event.BuildCompleted(total, name, pkgs) + self.assertEqual(event2.name, name) + self.assertEqual(event2.pkgs, pkgs) + self.assertEqual(event2.getFailures(), 0) + self.assertEqual(event2.msg, "Building Succeeded") + self.assertEqual(event2.pid, EventClassesTest._worker_pid) + + def test_DiskFull(self): + """ Test DiskFull event class """ + dev = "/dev/foo" + type = "ext4" + freespace = "104M" + mountpoint = "/" + event = bb.event.DiskFull(dev, type, freespace, mountpoint) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_MonitorDiskEvent(self): + """ Test MonitorDiskEvent class """ + available_bytes = 10000000 + free_bytes = 90000000 + total_bytes = 1000000000 + du = bb.event.DiskUsageSample(available_bytes, free_bytes, + total_bytes) + event = bb.event.MonitorDiskEvent(du) + self.assertEqual(event.disk_usage.available_bytes, available_bytes) + self.assertEqual(event.disk_usage.free_bytes, free_bytes) + self.assertEqual(event.disk_usage.total_bytes, total_bytes) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_NoProvider(self): + """ Test NoProvider event class """ + item = "foobar" + event1 = bb.event.NoProvider(item) + self.assertEqual(event1.getItem(), item) + self.assertEqual(event1.isRuntime(), False) + self.assertEqual(str(event1), "Nothing PROVIDES 'foobar'") + runtime = True + dependees = ["foo", "bar"] + reasons = None + close_matches = ["foibar", "footbar"] + event2 = bb.event.NoProvider(item, runtime, dependees, reasons, + close_matches) + self.assertEqual(event2.isRuntime(), True) + expected = ("Nothing RPROVIDES 'foobar' (but foo, bar RDEPENDS" + " on or otherwise requires it). Close matches:\n" + " foibar\n" + " footbar") + self.assertEqual(str(event2), expected) + reasons = ["Item does not exist on database"] + close_matches = ["foibar", "footbar"] + event3 = bb.event.NoProvider(item, runtime, dependees, reasons, + close_matches) + expected = ("Nothing RPROVIDES 'foobar' (but foo, bar RDEPENDS" + " on or otherwise requires it)\n" + "Item does not exist on database") + self.assertEqual(str(event3), expected) + self.assertEqual(event3.pid, EventClassesTest._worker_pid) + + def test_MultipleProviders(self): + """ Test MultipleProviders event class """ + item = "foobar" + candidates = ["foobarv1", "foobars"] + event1 = bb.event.MultipleProviders(item, candidates) + self.assertEqual(event1.isRuntime(), False) + self.assertEqual(event1.getItem(), item) + self.assertEqual(event1.getCandidates(), candidates) + expected = ("Multiple providers are available for foobar (foobarv1," + " foobars)\n" + "Consider defining a PREFERRED_PROVIDER entry to match " + "foobar") + self.assertEqual(str(event1), expected) + runtime = True + event2 = bb.event.MultipleProviders(item, candidates, runtime) + self.assertEqual(event2.isRuntime(), runtime) + expected = ("Multiple providers are available for runtime foobar " + "(foobarv1, foobars)\n" + "Consider defining a PREFERRED_RPROVIDER entry to match " + "foobar") + self.assertEqual(str(event2), expected) + self.assertEqual(event2.pid, EventClassesTest._worker_pid) + + def test_ParseStarted(self): + """ Test ParseStarted event class """ + total = 123 + event = bb.event.ParseStarted(total) + self.assertEqual(event.msg, "Recipe parsing Started") + self.assertEqual(event.total, total) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_ParseCompleted(self): + """ Test ParseCompleted event class """ + cached = 10 + parsed = 13 + skipped = 7 + virtuals = 2 + masked = 1 + errors = 0 + total = 23 + event = bb.event.ParseCompleted(cached, parsed, skipped, masked, + virtuals, errors, total) + self.assertEqual(event.msg, "Recipe parsing Completed") + expected = [cached, parsed, skipped, virtuals, masked, errors, + cached + parsed, total] + actual = [event.cached, event.parsed, event.skipped, event.virtuals, + event.masked, event.errors, event.sofar, event.total] + self.assertEqual(str(actual), str(expected)) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_ParseProgress(self): + """ Test ParseProgress event class """ + current = 10 + total = 100 + event = bb.event.ParseProgress(current, total) + self.assertEqual(event.msg, + "Recipe parsing" + ": %s/%s" % (current, total)) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_CacheLoadStarted(self): + """ Test CacheLoadStarted event class """ + total = 123 + event = bb.event.CacheLoadStarted(total) + self.assertEqual(event.msg, "Loading cache Started") + self.assertEqual(event.total, total) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_CacheLoadProgress(self): + """ Test CacheLoadProgress event class """ + current = 10 + total = 100 + event = bb.event.CacheLoadProgress(current, total) + self.assertEqual(event.msg, + "Loading cache" + ": %s/%s" % (current, total)) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_CacheLoadCompleted(self): + """ Test CacheLoadCompleted event class """ + total = 23 + num_entries = 12 + event = bb.event.CacheLoadCompleted(total, num_entries) + self.assertEqual(event.msg, "Loading cache Completed") + expected = [total, num_entries] + actual = [event.total, event.num_entries] + self.assertEqual(str(actual), str(expected)) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_TreeDataPreparationStarted(self): + """ Test TreeDataPreparationStarted event class """ + event = bb.event.TreeDataPreparationStarted() + self.assertEqual(event.msg, "Preparing tree data Started") + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_TreeDataPreparationProgress(self): + """ Test TreeDataPreparationProgress event class """ + current = 10 + total = 100 + event = bb.event.TreeDataPreparationProgress(current, total) + self.assertEqual(event.msg, + "Preparing tree data" + ": %s/%s" % (current, total)) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_TreeDataPreparationCompleted(self): + """ Test TreeDataPreparationCompleted event class """ + total = 23 + event = bb.event.TreeDataPreparationCompleted(total) + self.assertEqual(event.msg, "Preparing tree data Completed") + self.assertEqual(event.total, total) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_DepTreeGenerated(self): + """ Test DepTreeGenerated event class """ + depgraph = Mock() + event = bb.event.DepTreeGenerated(depgraph) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_TargetsTreeGenerated(self): + """ Test TargetsTreeGenerated event class """ + model = Mock() + event = bb.event.TargetsTreeGenerated(model) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_ReachableStamps(self): + """ Test ReachableStamps event class """ + stamps = [Mock(), Mock()] + event = bb.event.ReachableStamps(stamps) + self.assertEqual(event.stamps, stamps) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_FilesMatchingFound(self): + """ Test FilesMatchingFound event class """ + pattern = "foo.*bar" + matches = ["foobar"] + event = bb.event.FilesMatchingFound(pattern, matches) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_ConfigFilesFound(self): + """ Test ConfigFilesFound event class """ + variable = "FOO_BAR" + values = ["foo", "bar"] + event = bb.event.ConfigFilesFound(variable, values) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_ConfigFilePathFound(self): + """ Test ConfigFilePathFound event class """ + path = "/foo/bar" + event = bb.event.ConfigFilePathFound(path) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_message_classes(self): + """ Test message event classes """ + msg = "foobar foo bar" + event = bb.event.MsgBase(msg) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + event = bb.event.MsgDebug(msg) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + event = bb.event.MsgNote(msg) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + event = bb.event.MsgWarn(msg) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + event = bb.event.MsgError(msg) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + event = bb.event.MsgFatal(msg) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + event = bb.event.MsgPlain(msg) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_LogExecTTY(self): + """ Test LogExecTTY event class """ + msg = "foo bar" + prog = "foo.sh" + sleep_delay = 10 + retries = 3 + event = bb.event.LogExecTTY(msg, prog, sleep_delay, retries) + self.assertEqual(event.msg, msg) + self.assertEqual(event.prog, prog) + self.assertEqual(event.sleep_delay, sleep_delay) + self.assertEqual(event.retries, retries) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def _throw_zero_division_exception(self): + a = 1 / 0 + return + + def _worker_handler(self, event, d): + self._returned_event = event + return + + def test_LogHandler(self): + """ Test LogHandler class """ + logger = logging.getLogger("TestEventClasses") + logger.propagate = False + handler = bb.event.LogHandler(logging.INFO) + logger.addHandler(handler) + bb.event.worker_fire = self._worker_handler + try: + self._throw_zero_division_exception() + except ZeroDivisionError as ex: + logger.exception(ex) + event = self._returned_event + try: + pe = pickle.dumps(event) + newevent = pickle.loads(pe) + except: + self.fail('Logged event is not serializable') + self.assertEqual(event.taskpid, EventClassesTest._worker_pid) + + def test_MetadataEvent(self): + """ Test MetadataEvent class """ + eventtype = "footype" + eventdata = {"foo": "bar"} + event = bb.event.MetadataEvent(eventtype, eventdata) + self.assertEqual(event.type, eventtype) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_ProcessStarted(self): + """ Test ProcessStarted class """ + processname = "foo" + total = 9783128974 + event = bb.event.ProcessStarted(processname, total) + self.assertEqual(event.processname, processname) + self.assertEqual(event.total, total) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_ProcessProgress(self): + """ Test ProcessProgress class """ + processname = "foo" + progress = 243224 + event = bb.event.ProcessProgress(processname, progress) + self.assertEqual(event.processname, processname) + self.assertEqual(event.progress, progress) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_ProcessFinished(self): + """ Test ProcessFinished class """ + processname = "foo" + total = 1242342344 + event = bb.event.ProcessFinished(processname) + self.assertEqual(event.processname, processname) + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_SanityCheck(self): + """ Test SanityCheck class """ + event1 = bb.event.SanityCheck() + self.assertEqual(event1.generateevents, True) + self.assertEqual(event1.pid, EventClassesTest._worker_pid) + generateevents = False + event2 = bb.event.SanityCheck(generateevents) + self.assertEqual(event2.generateevents, generateevents) + self.assertEqual(event2.pid, EventClassesTest._worker_pid) + + def test_SanityCheckPassed(self): + """ Test SanityCheckPassed class """ + event = bb.event.SanityCheckPassed() + self.assertEqual(event.pid, EventClassesTest._worker_pid) + + def test_SanityCheckFailed(self): + """ Test SanityCheckFailed class """ + msg = "The sanity test failed." + event1 = bb.event.SanityCheckFailed(msg) + self.assertEqual(event1.pid, EventClassesTest._worker_pid) + network_error = True + event2 = bb.event.SanityCheckFailed(msg, network_error) + self.assertEqual(event2.pid, EventClassesTest._worker_pid) + + def test_network_event_classes(self): + """ Test network event classes """ + event1 = bb.event.NetworkTest() + generateevents = False + self.assertEqual(event1.pid, EventClassesTest._worker_pid) + event2 = bb.event.NetworkTest(generateevents) + self.assertEqual(event2.pid, EventClassesTest._worker_pid) + event3 = bb.event.NetworkTestPassed() + self.assertEqual(event3.pid, EventClassesTest._worker_pid) + event4 = bb.event.NetworkTestFailed() + self.assertEqual(event4.pid, EventClassesTest._worker_pid) + + def test_FindSigInfoResult(self): + """ Test FindSigInfoResult event class """ + result = [Mock()] + event = bb.event.FindSigInfoResult(result) + self.assertEqual(event.result, result) + self.assertEqual(event.pid, EventClassesTest._worker_pid) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/tests/fetch.py b/import-layers/yocto-poky/bitbake/lib/bb/tests/fetch.py index 7d7c5d7ff..74859f9d3 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/tests/fetch.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/tests/fetch.py @@ -20,6 +20,7 @@ # import unittest +import hashlib import tempfile import subprocess import collections @@ -522,6 +523,109 @@ class FetcherLocalTest(FetcherTest): with self.assertRaises(bb.fetch2.UnpackError): self.fetchUnpack(['file://a;subdir=/bin/sh']) +class FetcherNoNetworkTest(FetcherTest): + def setUp(self): + super().setUp() + # all test cases are based on not having network + self.d.setVar("BB_NO_NETWORK", "1") + + def test_missing(self): + string = "this is a test file\n".encode("utf-8") + self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest()) + self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest()) + + self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) + self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) + fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d) + with self.assertRaises(bb.fetch2.NetworkAccess): + fetcher.download() + + def test_valid_missing_donestamp(self): + # create the file in the download directory with correct hash + string = "this is a test file\n".encode("utf-8") + with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb") as f: + f.write(string) + + self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest()) + self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest()) + + self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) + self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) + fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d) + fetcher.download() + self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) + + def test_invalid_missing_donestamp(self): + # create an invalid file in the download directory with incorrect hash + string = "this is a test file\n".encode("utf-8") + with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"): + pass + + self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest()) + self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest()) + + self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) + self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) + fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d) + with self.assertRaises(bb.fetch2.NetworkAccess): + fetcher.download() + # the existing file should not exist or should have be moved to "bad-checksum" + self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) + + def test_nochecksums_missing(self): + self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) + self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) + # ssh fetch does not support checksums + fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) + # attempts to download with missing donestamp + with self.assertRaises(bb.fetch2.NetworkAccess): + fetcher.download() + + def test_nochecksums_missing_donestamp(self): + # create a file in the download directory + with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"): + pass + + self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) + self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) + # ssh fetch does not support checksums + fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) + # attempts to download with missing donestamp + with self.assertRaises(bb.fetch2.NetworkAccess): + fetcher.download() + + def test_nochecksums_has_donestamp(self): + # create a file in the download directory with the donestamp + with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"): + pass + with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"): + pass + + self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) + self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) + # ssh fetch does not support checksums + fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) + # should not fetch + fetcher.download() + # both files should still exist + self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) + self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) + + def test_nochecksums_missing_has_donestamp(self): + # create a file in the download directory with the donestamp + with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"): + pass + + self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) + self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) + # ssh fetch does not support checksums + fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) + with self.assertRaises(bb.fetch2.NetworkAccess): + fetcher.download() + # both files should still exist + self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) + self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) + class FetcherNetworkTest(FetcherTest): @skipIfNoNetwork() def test_fetch(self): @@ -809,7 +913,7 @@ class FetchLatestVersionTest(FetcherTest): ud = bb.fetch2.FetchData(k[1], self.d) pupver= ud.method.latest_versionstring(ud, self.d) verstring = pupver[0] - self.assertTrue(verstring, msg="Could not find upstream version") + self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0]) r = bb.utils.vercmp_string(v, verstring) self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring)) @@ -822,7 +926,7 @@ class FetchLatestVersionTest(FetcherTest): ud = bb.fetch2.FetchData(k[1], self.d) pupver = ud.method.latest_versionstring(ud, self.d) verstring = pupver[0] - self.assertTrue(verstring, msg="Could not find upstream version") + self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0]) r = bb.utils.vercmp_string(v, verstring) self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring)) @@ -874,9 +978,6 @@ class FetchCheckStatusTest(FetcherTest): class GitMakeShallowTest(FetcherTest): - bitbake_dir = os.path.join(os.path.dirname(os.path.join(__file__)), '..', '..', '..') - make_shallow_path = os.path.join(bitbake_dir, 'bin', 'git-make-shallow') - def setUp(self): FetcherTest.setUp(self) self.gitdir = os.path.join(self.tempdir, 'gitshallow') @@ -905,7 +1006,7 @@ class GitMakeShallowTest(FetcherTest): def make_shallow(self, args=None): if args is None: args = ['HEAD'] - return bb.process.run([self.make_shallow_path] + args, cwd=self.gitdir) + return bb.process.run([bb.fetch2.git.Git.make_shallow_path] + args, cwd=self.gitdir) def add_empty_file(self, path, msg=None): if msg is None: diff --git a/import-layers/yocto-poky/bitbake/lib/bb/tinfoil.py b/import-layers/yocto-poky/bitbake/lib/bb/tinfoil.py index fa95f6329..368264f39 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/tinfoil.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/tinfoil.py @@ -604,13 +604,16 @@ class Tinfoil: recipecache = self.cooker.recipecaches[mc] prov = self.find_best_provider(pn) fn = prov[3] - actual_pn = recipecache.pkg_fn[fn] - recipe = TinfoilRecipeInfo(recipecache, - self.config_data, - pn=actual_pn, - fn=fn, - fns=recipecache.pkg_pn[actual_pn]) - return recipe + if fn: + actual_pn = recipecache.pkg_fn[fn] + recipe = TinfoilRecipeInfo(recipecache, + self.config_data, + pn=actual_pn, + fn=fn, + fns=recipecache.pkg_pn[actual_pn]) + return recipe + else: + return None def parse_recipe(self, pn): """ diff --git a/import-layers/yocto-poky/bitbake/lib/bb/utils.py b/import-layers/yocto-poky/bitbake/lib/bb/utils.py index c540b49cf..378e699e0 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/utils.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/utils.py @@ -187,7 +187,7 @@ def explode_deps(s): #r[-1] += ' ' + ' '.join(j) return r -def explode_dep_versions2(s): +def explode_dep_versions2(s, *, sort=True): """ Take an RDEPENDS style string of format: "DEPEND1 (optional version) DEPEND2 (optional version) ..." @@ -250,7 +250,8 @@ def explode_dep_versions2(s): if not (i in r and r[i]): r[lastdep] = [] - r = collections.OrderedDict(sorted(r.items(), key=lambda x: x[0])) + if sort: + r = collections.OrderedDict(sorted(r.items(), key=lambda x: x[0])) return r def explode_dep_versions(s): @@ -806,8 +807,8 @@ def movefile(src, dest, newmtime = None, sstat = None): return None # failure try: if didcopy: - os.lchown(dest, sstat[stat.ST_UID], sstat[stat.ST_GID]) - os.chmod(dest, stat.S_IMODE(sstat[stat.ST_MODE])) # Sticky is reset on chown + os.lchown(destpath, sstat[stat.ST_UID], sstat[stat.ST_GID]) + os.chmod(destpath, stat.S_IMODE(sstat[stat.ST_MODE])) # Sticky is reset on chown os.unlink(src) except Exception as e: print("movefile: Failed to chown/chmod/unlink", dest, e) diff --git a/import-layers/yocto-poky/bitbake/lib/bblayers/action.py b/import-layers/yocto-poky/bitbake/lib/bblayers/action.py index b1326e5f5..aa575d1c0 100644 --- a/import-layers/yocto-poky/bitbake/lib/bblayers/action.py +++ b/import-layers/yocto-poky/bitbake/lib/bblayers/action.py @@ -18,16 +18,18 @@ def plugin_init(plugins): class ActionPlugin(LayerPlugin): def do_add_layer(self, args): - """Add a layer to bblayers.conf.""" - layerdir = os.path.abspath(args.layerdir) - if not os.path.exists(layerdir): - sys.stderr.write("Specified layer directory doesn't exist\n") - return 1 + """Add one or more layers to bblayers.conf.""" + layerdirs = [os.path.abspath(ldir) for ldir in args.layerdir] - layer_conf = os.path.join(layerdir, 'conf', 'layer.conf') - if not os.path.exists(layer_conf): - sys.stderr.write("Specified layer directory doesn't contain a conf/layer.conf file\n") - return 1 + for layerdir in layerdirs: + if not os.path.exists(layerdir): + sys.stderr.write("Specified layer directory %s doesn't exist\n" % layerdir) + return 1 + + layer_conf = os.path.join(layerdir, 'conf', 'layer.conf') + if not os.path.exists(layer_conf): + sys.stderr.write("Specified layer directory %s doesn't contain a conf/layer.conf file\n" % layerdir) + return 1 bblayers_conf = os.path.join('conf', 'bblayers.conf') if not os.path.exists(bblayers_conf): @@ -40,7 +42,7 @@ class ActionPlugin(LayerPlugin): shutil.copy2(bblayers_conf, backup) try: - notadded, _ = bb.utils.edit_bblayers_conf(bblayers_conf, layerdir, None) + notadded, _ = bb.utils.edit_bblayers_conf(bblayers_conf, layerdirs, None) if not (args.force or notadded): try: self.tinfoil.parseRecipes() @@ -56,19 +58,22 @@ class ActionPlugin(LayerPlugin): shutil.rmtree(tempdir) def do_remove_layer(self, args): - """Remove a layer from bblayers.conf.""" + """Remove one or more layers from bblayers.conf.""" bblayers_conf = os.path.join('conf', 'bblayers.conf') if not os.path.exists(bblayers_conf): sys.stderr.write("Unable to find bblayers.conf\n") return 1 - if args.layerdir.startswith('*'): - layerdir = args.layerdir - elif not '/' in args.layerdir: - layerdir = '*/%s' % args.layerdir - else: - layerdir = os.path.abspath(args.layerdir) - (_, notremoved) = bb.utils.edit_bblayers_conf(bblayers_conf, None, layerdir) + layerdirs = [] + for item in args.layerdir: + if item.startswith('*'): + layerdir = item + elif not '/' in item: + layerdir = '*/%s' % item + else: + layerdir = os.path.abspath(item) + layerdirs.append(layerdir) + (_, notremoved) = bb.utils.edit_bblayers_conf(bblayers_conf, None, layerdirs) if notremoved: for item in notremoved: sys.stderr.write("No layers matching %s found in BBLAYERS\n" % item) @@ -240,10 +245,10 @@ build results (as the layer priority order has effectively changed). def register_commands(self, sp): parser_add_layer = self.add_command(sp, 'add-layer', self.do_add_layer, parserecipes=False) - parser_add_layer.add_argument('layerdir', help='Layer directory to add') + parser_add_layer.add_argument('layerdir', nargs='+', help='Layer directory/directories to add') parser_remove_layer = self.add_command(sp, 'remove-layer', self.do_remove_layer, parserecipes=False) - parser_remove_layer.add_argument('layerdir', help='Layer directory to remove (wildcards allowed, enclose in quotes to avoid shell expansion)') + parser_remove_layer.add_argument('layerdir', nargs='+', help='Layer directory/directories to remove (wildcards allowed, enclose in quotes to avoid shell expansion)') parser_remove_layer.set_defaults(func=self.do_remove_layer) parser_flatten = self.add_command(sp, 'flatten', self.do_flatten) diff --git a/import-layers/yocto-poky/bitbake/lib/bblayers/query.py b/import-layers/yocto-poky/bitbake/lib/bblayers/query.py index bef3af31a..9294dfa88 100644 --- a/import-layers/yocto-poky/bitbake/lib/bblayers/query.py +++ b/import-layers/yocto-poky/bitbake/lib/bblayers/query.py @@ -161,7 +161,12 @@ skipped recipes will also be listed, with a " (skipped)" suffix. items_listed = False for p in sorted(pkg_pn): if pnspec: - if not fnmatch.fnmatch(p, pnspec): + found=False + for pnm in pnspec: + if fnmatch.fnmatch(p, pnm): + found=True + break + if not found: continue if len(allproviders[p]) > 1 or not show_multi_provider_only: @@ -251,8 +256,14 @@ Lists recipes with the bbappends that apply to them as subitems. pnlist.sort() appends = False for pn in pnlist: - if args.pnspec and pn != args.pnspec: - continue + if args.pnspec: + found=False + for pnm in args.pnspec: + if fnmatch.fnmatch(pn, pnm): + found=True + break + if not found: + continue if self.show_appends_for_pn(pn): appends = True @@ -479,11 +490,11 @@ NOTE: .bbappend files can impact the dependencies. parser_show_recipes = self.add_command(sp, 'show-recipes', self.do_show_recipes) parser_show_recipes.add_argument('-f', '--filenames', help='instead of the default formatting, list filenames of higher priority recipes with the ones they overlay indented underneath', action='store_true') parser_show_recipes.add_argument('-m', '--multiple', help='only list where multiple recipes (in the same layer or different layers) exist for the same recipe name', action='store_true') - parser_show_recipes.add_argument('-i', '--inherits', help='only list recipes that inherit the named class', metavar='CLASS', default='') - parser_show_recipes.add_argument('pnspec', nargs='?', help='optional recipe name specification (wildcards allowed, enclose in quotes to avoid shell expansion)') + parser_show_recipes.add_argument('-i', '--inherits', help='only list recipes that inherit the named class(es) - separate multiple classes using , (without spaces)', metavar='CLASS', default='') + parser_show_recipes.add_argument('pnspec', nargs='*', help='optional recipe name specification (wildcards allowed, enclose in quotes to avoid shell expansion)') parser_show_appends = self.add_command(sp, 'show-appends', self.do_show_appends) - parser_show_appends.add_argument('pnspec', nargs='?', help='optional recipe name specification (wildcards allowed, enclose in quotes to avoid shell expansion)') + parser_show_appends.add_argument('pnspec', nargs='*', help='optional recipe name specification (wildcards allowed, enclose in quotes to avoid shell expansion)') parser_show_cross_depends = self.add_command(sp, 'show-cross-depends', self.do_show_cross_depends) parser_show_cross_depends.add_argument('-f', '--filenames', help='show full file path', action='store_true') diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py index 4c175625d..16c7c8044 100644 --- a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py @@ -192,7 +192,7 @@ class LocalhostBEController(BuildEnvironmentController): if commit != "HEAD": logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname)) ref = commit if re.match('^[a-fA-F0-9]+$', commit) else 'origin/%s' % commit - self._shellcmd('git fetch --all && git reset --hard "%s"' % ref, localdirname,env=git_env) + self._shellcmd('git fetch && git reset --hard "%s"' % ref, localdirname,env=git_env) # take the localdirname as poky dir if we can find the oe-init-build-env if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")): diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py index 582114ac9..823c6f154 100644 --- a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py @@ -107,7 +107,10 @@ class Command(BaseCommand): action="ignore", message="^.*No fixture named.*$") print("Importing custom settings if present") - call_command("loaddata", "custom") + try: + call_command("loaddata", "custom") + except: + print("NOTE: optional fixture 'custom' not found") # we run lsupdates after config update print("\nFetching information from the layer index, " diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/orm/fixtures/oe-core.xml b/import-layers/yocto-poky/bitbake/lib/toaster/orm/fixtures/oe-core.xml index 00720c3da..d7ea78dc2 100644 --- a/import-layers/yocto-poky/bitbake/lib/toaster/orm/fixtures/oe-core.xml +++ b/import-layers/yocto-poky/bitbake/lib/toaster/orm/fixtures/oe-core.xml @@ -8,9 +8,9 @@ <!-- Bitbake versions which correspond to the metadata release --> <object model="orm.bitbakeversion" pk="1"> - <field type="CharField" name="name">rocko</field> + <field type="CharField" name="name">sumo</field> <field type="CharField" name="giturl">git://git.openembedded.org/bitbake</field> - <field type="CharField" name="branch">1.36</field> + <field type="CharField" name="branch">1.38</field> </object> <object model="orm.bitbakeversion" pk="2"> <field type="CharField" name="name">HEAD</field> @@ -22,14 +22,19 @@ <field type="CharField" name="giturl">git://git.openembedded.org/bitbake</field> <field type="CharField" name="branch">master</field> </object> + <object model="orm.bitbakeversion" pk="4"> + <field type="CharField" name="name">rocko</field> + <field type="CharField" name="giturl">git://git.openembedded.org/bitbake</field> + <field type="CharField" name="branch">1.36</field> + </object> <!-- Releases available --> <object model="orm.release" pk="1"> <field type="CharField" name="name">rocko</field> - <field type="CharField" name="description">Openembedded Rocko</field> + <field type="CharField" name="description">Openembedded Sumo</field> <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">1</field> - <field type="CharField" name="branch_name">rocko</field> - <field type="TextField" name="helptext">Toaster will run your builds using the tip of the <a href=\"http://cgit.openembedded.org/openembedded-core/log/?h=rocko\">OpenEmbedded Rocko</a> branch.</field> + <field type="CharField" name="branch_name">sumo</field> + <field type="TextField" name="helptext">Toaster will run your builds using the tip of the <a href=\"http://cgit.openembedded.org/openembedded-core/log/?h=sumo\">OpenEmbedded Sumo</a> branch.</field> </object> <object model="orm.release" pk="2"> <field type="CharField" name="name">local</field> @@ -45,6 +50,13 @@ <field type="CharField" name="branch_name">master</field> <field type="TextField" name="helptext">Toaster will run your builds using the tip of the <a href=\"http://cgit.openembedded.org/openembedded-core/log/\">OpenEmbedded master</a> branch.</field> </object> + <object model="orm.release" pk="4"> + <field type="CharField" name="name">rocko</field> + <field type="CharField" name="description">Openembedded Rocko</field> + <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">1</field> + <field type="CharField" name="branch_name">rocko</field> + <field type="TextField" name="helptext">Toaster will run your builds using the tip of the <a href=\"http://cgit.openembedded.org/openembedded-core/log/?h=rocko\">OpenEmbedded Rocko</a> branch.</field> + </object> <!-- Default layers for each release --> <object model="orm.releasedefaultlayer" pk="1"> @@ -59,6 +71,10 @@ <field rel="ManyToOneRel" to="orm.release" name="release">3</field> <field type="CharField" name="layer_name">openembedded-core</field> </object> + <object model="orm.releasedefaultlayer" pk="4"> + <field rel="ManyToOneRel" to="orm.release" name="release">4</field> + <field type="CharField" name="layer_name">openembedded-core</field> + </object> <!-- Layer for the Local release --> diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/orm/fixtures/poky.xml b/import-layers/yocto-poky/bitbake/lib/toaster/orm/fixtures/poky.xml index 2f39d7776..6c966da4a 100644 --- a/import-layers/yocto-poky/bitbake/lib/toaster/orm/fixtures/poky.xml +++ b/import-layers/yocto-poky/bitbake/lib/toaster/orm/fixtures/poky.xml @@ -8,9 +8,9 @@ <!-- Bitbake versions which correspond to the metadata release --> <object model="orm.bitbakeversion" pk="1"> - <field type="CharField" name="name">rocko</field> + <field type="CharField" name="name">sumo</field> <field type="CharField" name="giturl">git://git.yoctoproject.org/poky</field> - <field type="CharField" name="branch">rocko</field> + <field type="CharField" name="branch">sumo</field> <field type="CharField" name="dirpath">bitbake</field> </object> <object model="orm.bitbakeversion" pk="2"> @@ -25,15 +25,21 @@ <field type="CharField" name="branch">master</field> <field type="CharField" name="dirpath">bitbake</field> </object> + <object model="orm.bitbakeversion" pk="4"> + <field type="CharField" name="name">rocko</field> + <field type="CharField" name="giturl">git://git.yoctoproject.org/poky</field> + <field type="CharField" name="branch">rocko</field> + <field type="CharField" name="dirpath">bitbake</field> + </object> <!-- Releases available --> <object model="orm.release" pk="1"> - <field type="CharField" name="name">rocko</field> - <field type="CharField" name="description">Yocto Project 2.4 "Rocko"</field> + <field type="CharField" name="name">sumo</field> + <field type="CharField" name="description">Yocto Project 2.5 "Sumo"</field> <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">1</field> - <field type="CharField" name="branch_name">rocko</field> - <field type="TextField" name="helptext">Toaster will run your builds using the tip of the <a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=rocko">Yocto Project Rocko branch</a>.</field> + <field type="CharField" name="branch_name">sumo</field> + <field type="TextField" name="helptext">Toaster will run your builds using the tip of the <a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=sumo">Yocto Project Sumo branch</a>.</field> </object> <object model="orm.release" pk="2"> <field type="CharField" name="name">local</field> @@ -49,6 +55,13 @@ <field type="CharField" name="branch_name">master</field> <field type="TextField" name="helptext">Toaster will run your builds using the tip of the <a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/">Yocto Project Master branch</a>.</field> </object> + <object model="orm.release" pk="4"> + <field type="CharField" name="name">rocko</field> + <field type="CharField" name="description">Yocto Project 2.4 "Rocko"</field> + <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">1</field> + <field type="CharField" name="branch_name">rocko</field> + <field type="TextField" name="helptext">Toaster will run your builds using the tip of the <a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=rocko">Yocto Project Rocko branch</a>.</field> + </object> <!-- Default project layers for each release --> <object model="orm.releasedefaultlayer" pk="1"> @@ -87,6 +100,18 @@ <field rel="ManyToOneRel" to="orm.release" name="release">3</field> <field type="CharField" name="layer_name">meta-yocto-bsp</field> </object> + <object model="orm.releasedefaultlayer" pk="10"> + <field rel="ManyToOneRel" to="orm.release" name="release">4</field> + <field type="CharField" name="layer_name">openembedded-core</field> + </object> + <object model="orm.releasedefaultlayer" pk="11"> + <field rel="ManyToOneRel" to="orm.release" name="release">4</field> + <field type="CharField" name="layer_name">meta-poky</field> + </object> + <object model="orm.releasedefaultlayer" pk="12"> + <field rel="ManyToOneRel" to="orm.release" name="release">4</field> + <field type="CharField" name="layer_name">meta-yocto-bsp</field> + </object> <!-- Default layers provided by poky openembedded-core @@ -105,7 +130,7 @@ <field rel="ManyToOneRel" to="orm.layer" name="layer">1</field> <field type="IntegerField" name="layer_source">0</field> <field rel="ManyToOneRel" to="orm.release" name="release">1</field> - <field type="CharField" name="branch">rocko</field> + <field type="CharField" name="branch">sumo</field> <field type="CharField" name="dirpath">meta</field> </object> <object model="orm.layer_version" pk="2"> @@ -123,6 +148,13 @@ <field type="CharField" name="branch">master</field> <field type="CharField" name="dirpath">meta</field> </object> + <object model="orm.layer_version" pk="4"> + <field rel="ManyToOneRel" to="orm.layer" name="layer">1</field> + <field type="IntegerField" name="layer_source">0</field> + <field rel="ManyToOneRel" to="orm.release" name="release">4</field> + <field type="CharField" name="branch">rocko</field> + <field type="CharField" name="dirpath">meta</field> + </object> <object model="orm.layer" pk="2"> <field type="CharField" name="name">meta-poky</field> @@ -132,14 +164,14 @@ <field type="CharField" name="vcs_web_tree_base_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field> <field type="CharField" name="vcs_web_file_base_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field> </object> - <object model="orm.layer_version" pk="4"> + <object model="orm.layer_version" pk="5"> <field rel="ManyToOneRel" to="orm.layer" name="layer">2</field> <field type="IntegerField" name="layer_source">0</field> <field rel="ManyToOneRel" to="orm.release" name="release">1</field> - <field type="CharField" name="branch">rocko</field> + <field type="CharField" name="branch">sumo</field> <field type="CharField" name="dirpath">meta-poky</field> </object> - <object model="orm.layer_version" pk="5"> + <object model="orm.layer_version" pk="6"> <field rel="ManyToOneRel" to="orm.layer" name="layer">2</field> <field type="IntegerField" name="layer_source">0</field> <field rel="ManyToOneRel" to="orm.release" name="release">2</field> @@ -147,13 +179,20 @@ <field type="CharField" name="commit">HEAD</field> <field type="CharField" name="dirpath">meta-poky</field> </object> - <object model="orm.layer_version" pk="6"> + <object model="orm.layer_version" pk="7"> <field rel="ManyToOneRel" to="orm.layer" name="layer">2</field> <field type="IntegerField" name="layer_source">0</field> <field rel="ManyToOneRel" to="orm.release" name="release">3</field> <field type="CharField" name="branch">master</field> <field type="CharField" name="dirpath">meta-poky</field> </object> + <object model="orm.layer_version" pk="8"> + <field rel="ManyToOneRel" to="orm.layer" name="layer">2</field> + <field type="IntegerField" name="layer_source">0</field> + <field rel="ManyToOneRel" to="orm.release" name="release">4</field> + <field type="CharField" name="branch">rocko</field> + <field type="CharField" name="dirpath">meta-poky</field> + </object> <object model="orm.layer" pk="3"> <field type="CharField" name="name">meta-yocto-bsp</field> @@ -163,14 +202,14 @@ <field type="CharField" name="vcs_web_tree_base_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field> <field type="CharField" name="vcs_web_file_base_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field> </object> - <object model="orm.layer_version" pk="7"> + <object model="orm.layer_version" pk="9"> <field rel="ManyToOneRel" to="orm.layer" name="layer">3</field> <field type="IntegerField" name="layer_source">0</field> <field rel="ManyToOneRel" to="orm.release" name="release">1</field> - <field type="CharField" name="branch">rocko</field> + <field type="CharField" name="branch">sumo</field> <field type="CharField" name="dirpath">meta-yocto-bsp</field> </object> - <object model="orm.layer_version" pk="8"> + <object model="orm.layer_version" pk="10"> <field rel="ManyToOneRel" to="orm.layer" name="layer">3</field> <field type="IntegerField" name="layer_source">0</field> <field rel="ManyToOneRel" to="orm.release" name="release">2</field> @@ -178,11 +217,18 @@ <field type="CharField" name="commit">HEAD</field> <field type="CharField" name="dirpath">meta-yocto-bsp</field> </object> - <object model="orm.layer_version" pk="9"> + <object model="orm.layer_version" pk="11"> <field rel="ManyToOneRel" to="orm.layer" name="layer">3</field> <field type="IntegerField" name="layer_source">0</field> <field rel="ManyToOneRel" to="orm.release" name="release">3</field> <field type="CharField" name="branch">master</field> <field type="CharField" name="dirpath">meta-yocto-bsp</field> </object> + <object model="orm.layer_version" pk="12"> + <field rel="ManyToOneRel" to="orm.layer" name="layer">3</field> + <field type="IntegerField" name="layer_source">0</field> + <field rel="ManyToOneRel" to="orm.release" name="release">4</field> + <field type="CharField" name="branch">rocko</field> + <field type="CharField" name="dirpath">meta-yocto-bsp</field> + </object> </django-objects> |
