summaryrefslogtreecommitdiffstats
path: root/yocto-poky/bitbake/lib/bb
diff options
context:
space:
mode:
Diffstat (limited to 'yocto-poky/bitbake/lib/bb')
-rw-r--r--yocto-poky/bitbake/lib/bb/cache.py15
-rw-r--r--yocto-poky/bitbake/lib/bb/data_smart.py38
-rw-r--r--yocto-poky/bitbake/lib/bb/fetch2/__init__.py16
-rw-r--r--yocto-poky/bitbake/lib/bb/fetch2/git.py11
-rw-r--r--yocto-poky/bitbake/lib/bb/fetch2/hg.py10
-rw-r--r--yocto-poky/bitbake/lib/bb/tests/fetch.py10
-rw-r--r--yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py8
-rw-r--r--yocto-poky/bitbake/lib/bb/ui/toasterui.py2
8 files changed, 83 insertions, 27 deletions
diff --git a/yocto-poky/bitbake/lib/bb/cache.py b/yocto-poky/bitbake/lib/bb/cache.py
index ef4d660e8..ab09b08b5 100644
--- a/yocto-poky/bitbake/lib/bb/cache.py
+++ b/yocto-poky/bitbake/lib/bb/cache.py
@@ -528,7 +528,20 @@ class Cache(object):
if hasattr(info_array[0], 'file_checksums'):
for _, fl in info_array[0].file_checksums.items():
- for f in fl.split():
+ fl = fl.strip()
+ while fl:
+ # A .split() would be simpler but means spaces or colons in filenames would break
+ a = fl.find(":True")
+ b = fl.find(":False")
+ if ((a < 0) and b) or ((b > 0) and (b < a)):
+ f = fl[:b+6]
+ fl = fl[b+7:]
+ elif ((b < 0) and a) or ((a > 0) and (a < b)):
+ f = fl[:a+5]
+ fl = fl[a+6:]
+ else:
+ break
+ fl = fl.strip()
if "*" in f:
continue
f, exist = f.split(":")
diff --git a/yocto-poky/bitbake/lib/bb/data_smart.py b/yocto-poky/bitbake/lib/bb/data_smart.py
index 79b4ed932..70558c15a 100644
--- a/yocto-poky/bitbake/lib/bb/data_smart.py
+++ b/yocto-poky/bitbake/lib/bb/data_smart.py
@@ -413,9 +413,11 @@ class DataSmart(MutableMapping):
self.overrides = None
def need_overrides(self):
- if self.overrides is None:
- if self.inoverride:
- return
+ if self.overrides is not None:
+ return
+ if self.inoverride:
+ return
+ for count in range(5):
self.inoverride = True
# Can end up here recursively so setup dummy values
self.overrides = []
@@ -424,6 +426,13 @@ class DataSmart(MutableMapping):
self.overridesset = set(self.overrides)
self.inoverride = False
self.expand_cache = {}
+ newoverrides = (self.getVar("OVERRIDES", True) or "").split(":") or []
+ if newoverrides == self.overrides:
+ break
+ self.overrides = newoverrides
+ self.overridesset = set(self.overrides)
+ else:
+ bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work.")
def initVar(self, var):
self.expand_cache = {}
@@ -484,10 +493,8 @@ class DataSmart(MutableMapping):
if '_' in var:
self._setvar_update_overrides(base, **loginfo)
-
if base in self.overridevars:
- self.overridevars.update(self.expandWithRefs(value, var).references)
- self.internal_finalize(True)
+ self._setvar_update_overridevars(var, value)
return
if not var in self.dict:
@@ -520,8 +527,21 @@ class DataSmart(MutableMapping):
self.varhistory.record(**loginfo)
if var in self.overridevars:
- self.overridevars.update(self.expandWithRefs(value, var).references)
- self.internal_finalize(True)
+ self._setvar_update_overridevars(var, value)
+
+ def _setvar_update_overridevars(self, var, value):
+ vardata = self.expandWithRefs(value, var)
+ new = vardata.references
+ new.update(vardata.contains.keys())
+ while not new.issubset(self.overridevars):
+ nextnew = set()
+ self.overridevars.update(new)
+ for i in new:
+ vardata = self.expandWithRefs(self.getVar(i, True), i)
+ nextnew.update(vardata.references)
+ nextnew.update(vardata.contains.keys())
+ new = nextnew
+ self.internal_finalize(True)
def _setvar_update_overrides(self, var, **loginfo):
# aka pay the cookie monster
@@ -628,6 +648,8 @@ class DataSmart(MutableMapping):
if flag == "_defaultval" and '_' in var:
self._setvar_update_overrides(var, **loginfo)
+ if flag == "_defaultval" and var in self.overridevars:
+ self._setvar_update_overridevars(var, value)
if flag == "unexport" or flag == "export":
if not "__exportlist" in self.dict:
diff --git a/yocto-poky/bitbake/lib/bb/fetch2/__init__.py b/yocto-poky/bitbake/lib/bb/fetch2/__init__.py
index 3d53b63b3..288a1c8fd 100644
--- a/yocto-poky/bitbake/lib/bb/fetch2/__init__.py
+++ b/yocto-poky/bitbake/lib/bb/fetch2/__init__.py
@@ -464,7 +464,7 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d):
for k in replacements:
uri_replace_decoded[loc] = uri_replace_decoded[loc].replace(k, replacements[k])
#bb.note("%s %s %s" % (regexp, uri_replace_decoded[loc], uri_decoded[loc]))
- result_decoded[loc] = re.sub(regexp, uri_replace_decoded[loc], uri_decoded[loc])
+ result_decoded[loc] = re.sub(regexp, uri_replace_decoded[loc], uri_decoded[loc], 1)
if loc == 2:
# Handle path manipulations
basename = None
@@ -862,7 +862,7 @@ def build_mirroruris(origud, mirrors, ld):
replacements["BASENAME"] = origud.path.split("/")[-1]
replacements["MIRRORNAME"] = origud.host.replace(':','.') + origud.path.replace('/', '.').replace('*', '.')
- def adduri(ud, uris, uds):
+ def adduri(ud, uris, uds, mirrors):
for line in mirrors:
try:
(find, replace) = line
@@ -876,6 +876,12 @@ def build_mirroruris(origud, mirrors, ld):
logger.debug(1, "Mirror %s not in the list of trusted networks, skipping" % (newuri))
continue
+ # Create a local copy of the mirrors minus the current line
+ # this will prevent us from recursively processing the same line
+ # as well as indirect recursion A -> B -> C -> A
+ localmirrors = list(mirrors)
+ localmirrors.remove(line)
+
try:
newud = FetchData(newuri, ld)
newud.setup_localpath(ld)
@@ -885,16 +891,16 @@ def build_mirroruris(origud, mirrors, ld):
try:
# setup_localpath of file:// urls may fail, we should still see
# if mirrors of the url exist
- adduri(newud, uris, uds)
+ adduri(newud, uris, uds, localmirrors)
except UnboundLocalError:
pass
continue
uris.append(newuri)
uds.append(newud)
- adduri(newud, uris, uds)
+ adduri(newud, uris, uds, localmirrors)
- adduri(origud, uris, uds)
+ adduri(origud, uris, uds, mirrors)
return uris, uds
diff --git a/yocto-poky/bitbake/lib/bb/fetch2/git.py b/yocto-poky/bitbake/lib/bb/fetch2/git.py
index 40658ff94..9bd87ad25 100644
--- a/yocto-poky/bitbake/lib/bb/fetch2/git.py
+++ b/yocto-poky/bitbake/lib/bb/fetch2/git.py
@@ -66,6 +66,7 @@ Supported SRC_URI options are:
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+import errno
import os
import re
import bb
@@ -181,8 +182,6 @@ class Git(FetchMethod):
def download(self, ud, d):
"""Fetch url"""
- ud.repochanged = not os.path.exists(ud.fullmirror)
-
# If the checkout doesn't exist and the mirror tarball does, extract it
if not os.path.exists(ud.clonedir) and os.path.exists(ud.fullmirror):
bb.utils.mkdirhier(ud.clonedir)
@@ -220,7 +219,11 @@ class Git(FetchMethod):
runfetchcmd(fetch_cmd, d)
runfetchcmd("%s prune-packed" % ud.basecmd, d)
runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d)
- ud.repochanged = True
+ try:
+ os.unlink(ud.fullmirror)
+ except OSError as exc:
+ if exc.errno != errno.ENOENT:
+ raise
os.chdir(ud.clonedir)
for name in ud.names:
if not self._contains_ref(ud, d, name):
@@ -228,7 +231,7 @@ class Git(FetchMethod):
def build_mirror_data(self, ud, d):
# Generate a mirror tarball if needed
- if ud.write_tarballs and (ud.repochanged or not os.path.exists(ud.fullmirror)):
+ if ud.write_tarballs and not os.path.exists(ud.fullmirror):
# it's possible that this symlink points to read-only filesystem with PREMIRROR
if os.path.islink(ud.fullmirror):
os.unlink(ud.fullmirror)
diff --git a/yocto-poky/bitbake/lib/bb/fetch2/hg.py b/yocto-poky/bitbake/lib/bb/fetch2/hg.py
index d978630ba..bbb4ed95d 100644
--- a/yocto-poky/bitbake/lib/bb/fetch2/hg.py
+++ b/yocto-poky/bitbake/lib/bb/fetch2/hg.py
@@ -163,8 +163,6 @@ class Hg(FetchMethod):
def download(self, ud, d):
"""Fetch url"""
- ud.repochanged = not os.path.exists(ud.fullmirror)
-
logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")
# If the checkout doesn't exist and the mirror tarball does, extract it
@@ -189,7 +187,11 @@ class Hg(FetchMethod):
logger.debug(1, "Running %s", pullcmd)
bb.fetch2.check_network_access(d, pullcmd, ud.url)
runfetchcmd(pullcmd, d)
- ud.repochanged = True
+ try:
+ os.unlink(ud.fullmirror)
+ except OSError as exc:
+ if exc.errno != errno.ENOENT:
+ raise
# No source found, clone it.
if not os.path.exists(ud.moddir):
@@ -238,7 +240,7 @@ class Hg(FetchMethod):
def build_mirror_data(self, ud, d):
# Generate a mirror tarball if needed
- if ud.write_tarballs == "1" and (ud.repochanged or not os.path.exists(ud.fullmirror)):
+ if ud.write_tarballs == "1" and not os.path.exists(ud.fullmirror):
# it's possible that this symlink points to read-only filesystem with PREMIRROR
if os.path.islink(ud.fullmirror):
os.unlink(ud.fullmirror)
diff --git a/yocto-poky/bitbake/lib/bb/tests/fetch.py b/yocto-poky/bitbake/lib/bb/tests/fetch.py
index 1e61f3a11..94173c14a 100644
--- a/yocto-poky/bitbake/lib/bb/tests/fetch.py
+++ b/yocto-poky/bitbake/lib/bb/tests/fetch.py
@@ -405,6 +405,16 @@ class MirrorUriTest(FetcherTest):
'http://otherdownloads.yoctoproject.org/downloads/bitbake-1.0.tar.gz',
'http://downloads2.yoctoproject.org/downloads/bitbake-1.0.tar.gz'])
+ recmirrorvar = "https://.*/[^/]* http://AAAA/A/A/A/ \n" \
+ "https://.*/[^/]* https://BBBB/B/B/B/ \n"
+
+ def test_recursive(self):
+ fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
+ mirrors = bb.fetch2.mirror_from_string(self.recmirrorvar)
+ uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
+ self.assertEqual(uris, ['http://AAAA/A/A/A/bitbake/bitbake-1.0.tar.gz',
+ 'https://BBBB/B/B/B/bitbake/bitbake-1.0.tar.gz',
+ 'http://AAAA/A/A/A/B/B/bitbake/bitbake-1.0.tar.gz'])
class FetcherLocalTest(FetcherTest):
def setUp(self):
diff --git a/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py b/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py
index 2d1ed5111..6e313fee8 100644
--- a/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py
+++ b/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py
@@ -704,7 +704,7 @@ class BuildInfoHelper(object):
## methods to convert event/external info into objects that the ORM layer uses
- def _get_build_information(self):
+ def _get_build_information(self, consolelogfile):
build_info = {}
# Generate an identifier for each new build
@@ -713,7 +713,7 @@ class BuildInfoHelper(object):
build_info['distro_version'] = self.server.runCommand(["getVariable", "DISTRO_VERSION"])[0]
build_info['started_on'] = timezone.now()
build_info['completed_on'] = timezone.now()
- build_info['cooker_log_path'] = self.server.runCommand(["getVariable", "BB_CONSOLELOG"])[0]
+ build_info['cooker_log_path'] = consolelogfile
build_info['build_name'] = self.server.runCommand(["getVariable", "BUILDNAME"])[0]
build_info['bitbake_version'] = self.server.runCommand(["getVariable", "BB_VERSION"])[0]
@@ -847,9 +847,9 @@ class BuildInfoHelper(object):
logger.warn("buildinfohelper: cannot identify layer exception:%s ", nee)
- def store_started_build(self, event):
+ def store_started_build(self, event, consolelogfile):
assert '_pkgs' in vars(event)
- build_information = self._get_build_information()
+ build_information = self._get_build_information(consolelogfile)
build_obj = self.orm_wrapper.create_build_object(build_information, self.brbe, self.project)
diff --git a/yocto-poky/bitbake/lib/bb/ui/toasterui.py b/yocto-poky/bitbake/lib/bb/ui/toasterui.py
index 9c7e87dd1..e0c278bb3 100644
--- a/yocto-poky/bitbake/lib/bb/ui/toasterui.py
+++ b/yocto-poky/bitbake/lib/bb/ui/toasterui.py
@@ -126,7 +126,7 @@ def main(server, eventHandler, params ):
# the code will look into the protected variables of the event; no easy way around this
if isinstance(event, bb.event.BuildStarted):
- buildinfohelper.store_started_build(event)
+ buildinfohelper.store_started_build(event, consolelogfile)
if isinstance(event, (bb.build.TaskStarted, bb.build.TaskSucceeded, bb.build.TaskFailedSilent)):
buildinfohelper.update_and_store_task(event)
OpenPOWER on IntegriCloud