diff options
Diffstat (limited to 'yocto-poky/bitbake/lib/bb')
-rw-r--r-- | yocto-poky/bitbake/lib/bb/cache.py | 15 | ||||
-rw-r--r-- | yocto-poky/bitbake/lib/bb/data_smart.py | 38 | ||||
-rw-r--r-- | yocto-poky/bitbake/lib/bb/fetch2/__init__.py | 16 | ||||
-rw-r--r-- | yocto-poky/bitbake/lib/bb/fetch2/git.py | 11 | ||||
-rw-r--r-- | yocto-poky/bitbake/lib/bb/fetch2/hg.py | 10 | ||||
-rw-r--r-- | yocto-poky/bitbake/lib/bb/tests/fetch.py | 10 | ||||
-rw-r--r-- | yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py | 8 | ||||
-rw-r--r-- | yocto-poky/bitbake/lib/bb/ui/toasterui.py | 2 |
8 files changed, 83 insertions, 27 deletions
diff --git a/yocto-poky/bitbake/lib/bb/cache.py b/yocto-poky/bitbake/lib/bb/cache.py index ef4d660e8..ab09b08b5 100644 --- a/yocto-poky/bitbake/lib/bb/cache.py +++ b/yocto-poky/bitbake/lib/bb/cache.py @@ -528,7 +528,20 @@ class Cache(object): if hasattr(info_array[0], 'file_checksums'): for _, fl in info_array[0].file_checksums.items(): - for f in fl.split(): + fl = fl.strip() + while fl: + # A .split() would be simpler but means spaces or colons in filenames would break + a = fl.find(":True") + b = fl.find(":False") + if ((a < 0) and b) or ((b > 0) and (b < a)): + f = fl[:b+6] + fl = fl[b+7:] + elif ((b < 0) and a) or ((a > 0) and (a < b)): + f = fl[:a+5] + fl = fl[a+6:] + else: + break + fl = fl.strip() if "*" in f: continue f, exist = f.split(":") diff --git a/yocto-poky/bitbake/lib/bb/data_smart.py b/yocto-poky/bitbake/lib/bb/data_smart.py index 79b4ed932..70558c15a 100644 --- a/yocto-poky/bitbake/lib/bb/data_smart.py +++ b/yocto-poky/bitbake/lib/bb/data_smart.py @@ -413,9 +413,11 @@ class DataSmart(MutableMapping): self.overrides = None def need_overrides(self): - if self.overrides is None: - if self.inoverride: - return + if self.overrides is not None: + return + if self.inoverride: + return + for count in range(5): self.inoverride = True # Can end up here recursively so setup dummy values self.overrides = [] @@ -424,6 +426,13 @@ class DataSmart(MutableMapping): self.overridesset = set(self.overrides) self.inoverride = False self.expand_cache = {} + newoverrides = (self.getVar("OVERRIDES", True) or "").split(":") or [] + if newoverrides == self.overrides: + break + self.overrides = newoverrides + self.overridesset = set(self.overrides) + else: + bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work.") def initVar(self, var): self.expand_cache = {} @@ -484,10 +493,8 @@ class DataSmart(MutableMapping): if '_' in var: self._setvar_update_overrides(base, **loginfo) - if base in self.overridevars: - self.overridevars.update(self.expandWithRefs(value, var).references) - self.internal_finalize(True) + self._setvar_update_overridevars(var, value) return if not var in self.dict: @@ -520,8 +527,21 @@ class DataSmart(MutableMapping): self.varhistory.record(**loginfo) if var in self.overridevars: - self.overridevars.update(self.expandWithRefs(value, var).references) - self.internal_finalize(True) + self._setvar_update_overridevars(var, value) + + def _setvar_update_overridevars(self, var, value): + vardata = self.expandWithRefs(value, var) + new = vardata.references + new.update(vardata.contains.keys()) + while not new.issubset(self.overridevars): + nextnew = set() + self.overridevars.update(new) + for i in new: + vardata = self.expandWithRefs(self.getVar(i, True), i) + nextnew.update(vardata.references) + nextnew.update(vardata.contains.keys()) + new = nextnew + self.internal_finalize(True) def _setvar_update_overrides(self, var, **loginfo): # aka pay the cookie monster @@ -628,6 +648,8 @@ class DataSmart(MutableMapping): if flag == "_defaultval" and '_' in var: self._setvar_update_overrides(var, **loginfo) + if flag == "_defaultval" and var in self.overridevars: + self._setvar_update_overridevars(var, value) if flag == "unexport" or flag == "export": if not "__exportlist" in self.dict: diff --git a/yocto-poky/bitbake/lib/bb/fetch2/__init__.py b/yocto-poky/bitbake/lib/bb/fetch2/__init__.py index 3d53b63b3..288a1c8fd 100644 --- a/yocto-poky/bitbake/lib/bb/fetch2/__init__.py +++ b/yocto-poky/bitbake/lib/bb/fetch2/__init__.py @@ -464,7 +464,7 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d): for k in replacements: uri_replace_decoded[loc] = uri_replace_decoded[loc].replace(k, replacements[k]) #bb.note("%s %s %s" % (regexp, uri_replace_decoded[loc], uri_decoded[loc])) - result_decoded[loc] = re.sub(regexp, uri_replace_decoded[loc], uri_decoded[loc]) + result_decoded[loc] = re.sub(regexp, uri_replace_decoded[loc], uri_decoded[loc], 1) if loc == 2: # Handle path manipulations basename = None @@ -862,7 +862,7 @@ def build_mirroruris(origud, mirrors, ld): replacements["BASENAME"] = origud.path.split("/")[-1] replacements["MIRRORNAME"] = origud.host.replace(':','.') + origud.path.replace('/', '.').replace('*', '.') - def adduri(ud, uris, uds): + def adduri(ud, uris, uds, mirrors): for line in mirrors: try: (find, replace) = line @@ -876,6 +876,12 @@ def build_mirroruris(origud, mirrors, ld): logger.debug(1, "Mirror %s not in the list of trusted networks, skipping" % (newuri)) continue + # Create a local copy of the mirrors minus the current line + # this will prevent us from recursively processing the same line + # as well as indirect recursion A -> B -> C -> A + localmirrors = list(mirrors) + localmirrors.remove(line) + try: newud = FetchData(newuri, ld) newud.setup_localpath(ld) @@ -885,16 +891,16 @@ def build_mirroruris(origud, mirrors, ld): try: # setup_localpath of file:// urls may fail, we should still see # if mirrors of the url exist - adduri(newud, uris, uds) + adduri(newud, uris, uds, localmirrors) except UnboundLocalError: pass continue uris.append(newuri) uds.append(newud) - adduri(newud, uris, uds) + adduri(newud, uris, uds, localmirrors) - adduri(origud, uris, uds) + adduri(origud, uris, uds, mirrors) return uris, uds diff --git a/yocto-poky/bitbake/lib/bb/fetch2/git.py b/yocto-poky/bitbake/lib/bb/fetch2/git.py index 40658ff94..9bd87ad25 100644 --- a/yocto-poky/bitbake/lib/bb/fetch2/git.py +++ b/yocto-poky/bitbake/lib/bb/fetch2/git.py @@ -66,6 +66,7 @@ Supported SRC_URI options are: # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import errno import os import re import bb @@ -181,8 +182,6 @@ class Git(FetchMethod): def download(self, ud, d): """Fetch url""" - ud.repochanged = not os.path.exists(ud.fullmirror) - # If the checkout doesn't exist and the mirror tarball does, extract it if not os.path.exists(ud.clonedir) and os.path.exists(ud.fullmirror): bb.utils.mkdirhier(ud.clonedir) @@ -220,7 +219,11 @@ class Git(FetchMethod): runfetchcmd(fetch_cmd, d) runfetchcmd("%s prune-packed" % ud.basecmd, d) runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d) - ud.repochanged = True + try: + os.unlink(ud.fullmirror) + except OSError as exc: + if exc.errno != errno.ENOENT: + raise os.chdir(ud.clonedir) for name in ud.names: if not self._contains_ref(ud, d, name): @@ -228,7 +231,7 @@ class Git(FetchMethod): def build_mirror_data(self, ud, d): # Generate a mirror tarball if needed - if ud.write_tarballs and (ud.repochanged or not os.path.exists(ud.fullmirror)): + if ud.write_tarballs and not os.path.exists(ud.fullmirror): # it's possible that this symlink points to read-only filesystem with PREMIRROR if os.path.islink(ud.fullmirror): os.unlink(ud.fullmirror) diff --git a/yocto-poky/bitbake/lib/bb/fetch2/hg.py b/yocto-poky/bitbake/lib/bb/fetch2/hg.py index d978630ba..bbb4ed95d 100644 --- a/yocto-poky/bitbake/lib/bb/fetch2/hg.py +++ b/yocto-poky/bitbake/lib/bb/fetch2/hg.py @@ -163,8 +163,6 @@ class Hg(FetchMethod): def download(self, ud, d): """Fetch url""" - ud.repochanged = not os.path.exists(ud.fullmirror) - logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'") # If the checkout doesn't exist and the mirror tarball does, extract it @@ -189,7 +187,11 @@ class Hg(FetchMethod): logger.debug(1, "Running %s", pullcmd) bb.fetch2.check_network_access(d, pullcmd, ud.url) runfetchcmd(pullcmd, d) - ud.repochanged = True + try: + os.unlink(ud.fullmirror) + except OSError as exc: + if exc.errno != errno.ENOENT: + raise # No source found, clone it. if not os.path.exists(ud.moddir): @@ -238,7 +240,7 @@ class Hg(FetchMethod): def build_mirror_data(self, ud, d): # Generate a mirror tarball if needed - if ud.write_tarballs == "1" and (ud.repochanged or not os.path.exists(ud.fullmirror)): + if ud.write_tarballs == "1" and not os.path.exists(ud.fullmirror): # it's possible that this symlink points to read-only filesystem with PREMIRROR if os.path.islink(ud.fullmirror): os.unlink(ud.fullmirror) diff --git a/yocto-poky/bitbake/lib/bb/tests/fetch.py b/yocto-poky/bitbake/lib/bb/tests/fetch.py index 1e61f3a11..94173c14a 100644 --- a/yocto-poky/bitbake/lib/bb/tests/fetch.py +++ b/yocto-poky/bitbake/lib/bb/tests/fetch.py @@ -405,6 +405,16 @@ class MirrorUriTest(FetcherTest): 'http://otherdownloads.yoctoproject.org/downloads/bitbake-1.0.tar.gz', 'http://downloads2.yoctoproject.org/downloads/bitbake-1.0.tar.gz']) + recmirrorvar = "https://.*/[^/]* http://AAAA/A/A/A/ \n" \ + "https://.*/[^/]* https://BBBB/B/B/B/ \n" + + def test_recursive(self): + fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) + mirrors = bb.fetch2.mirror_from_string(self.recmirrorvar) + uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) + self.assertEqual(uris, ['http://AAAA/A/A/A/bitbake/bitbake-1.0.tar.gz', + 'https://BBBB/B/B/B/bitbake/bitbake-1.0.tar.gz', + 'http://AAAA/A/A/A/B/B/bitbake/bitbake-1.0.tar.gz']) class FetcherLocalTest(FetcherTest): def setUp(self): diff --git a/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py b/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py index 2d1ed5111..6e313fee8 100644 --- a/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py +++ b/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py @@ -704,7 +704,7 @@ class BuildInfoHelper(object): ## methods to convert event/external info into objects that the ORM layer uses - def _get_build_information(self): + def _get_build_information(self, consolelogfile): build_info = {} # Generate an identifier for each new build @@ -713,7 +713,7 @@ class BuildInfoHelper(object): build_info['distro_version'] = self.server.runCommand(["getVariable", "DISTRO_VERSION"])[0] build_info['started_on'] = timezone.now() build_info['completed_on'] = timezone.now() - build_info['cooker_log_path'] = self.server.runCommand(["getVariable", "BB_CONSOLELOG"])[0] + build_info['cooker_log_path'] = consolelogfile build_info['build_name'] = self.server.runCommand(["getVariable", "BUILDNAME"])[0] build_info['bitbake_version'] = self.server.runCommand(["getVariable", "BB_VERSION"])[0] @@ -847,9 +847,9 @@ class BuildInfoHelper(object): logger.warn("buildinfohelper: cannot identify layer exception:%s ", nee) - def store_started_build(self, event): + def store_started_build(self, event, consolelogfile): assert '_pkgs' in vars(event) - build_information = self._get_build_information() + build_information = self._get_build_information(consolelogfile) build_obj = self.orm_wrapper.create_build_object(build_information, self.brbe, self.project) diff --git a/yocto-poky/bitbake/lib/bb/ui/toasterui.py b/yocto-poky/bitbake/lib/bb/ui/toasterui.py index 9c7e87dd1..e0c278bb3 100644 --- a/yocto-poky/bitbake/lib/bb/ui/toasterui.py +++ b/yocto-poky/bitbake/lib/bb/ui/toasterui.py @@ -126,7 +126,7 @@ def main(server, eventHandler, params ): # the code will look into the protected variables of the event; no easy way around this if isinstance(event, bb.event.BuildStarted): - buildinfohelper.store_started_build(event) + buildinfohelper.store_started_build(event, consolelogfile) if isinstance(event, (bb.build.TaskStarted, bb.build.TaskSucceeded, bb.build.TaskFailedSilent)): buildinfohelper.update_and_store_task(event) |