From 37a0e4ddff58c0120cc5cfef104b60d0e180638c Mon Sep 17 00:00:00 2001 From: Brad Bishop Date: Mon, 4 Dec 2017 01:01:44 -0500 Subject: Squashed 'import-layers/yocto-poky/' changes from dc8508f6099..67491b0c104 Yocto 2.2.2 (Morty) Change-Id: Id9a452e28940d9f166957de243d9cb1d8818704e git-subtree-dir: import-layers/yocto-poky git-subtree-split: 67491b0c104101bb9f366d697edd23c895be4302 Signed-off-by: Brad Bishop --- import-layers/yocto-poky/bitbake/lib/bb/build.py | 3 +- .../yocto-poky/bitbake/lib/bb/codeparser.py | 29 ++++++---- import-layers/yocto-poky/bitbake/lib/bb/cooker.py | 57 +++++++++++++++++-- .../yocto-poky/bitbake/lib/bb/cookerdata.py | 5 +- import-layers/yocto-poky/bitbake/lib/bb/data.py | 8 ++- .../yocto-poky/bitbake/lib/bb/data_smart.py | 15 ++--- .../yocto-poky/bitbake/lib/bb/fetch2/wget.py | 5 +- .../yocto-poky/bitbake/lib/bb/runqueue.py | 64 ++++++++++++---------- import-layers/yocto-poky/bitbake/lib/bb/siggen.py | 39 ++++++++----- import-layers/yocto-poky/bitbake/lib/bb/tinfoil.py | 3 + .../bitbake/lib/bb/ui/buildinfohelper.py | 27 ++++++++- .../yocto-poky/bitbake/lib/bb/ui/uihelper.py | 5 +- import-layers/yocto-poky/bitbake/lib/bb/utils.py | 2 +- 13 files changed, 181 insertions(+), 81 deletions(-) (limited to 'import-layers/yocto-poky/bitbake/lib/bb') diff --git a/import-layers/yocto-poky/bitbake/lib/bb/build.py b/import-layers/yocto-poky/bitbake/lib/bb/build.py index c4c8aeb64..b59a49bc1 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/build.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/build.py @@ -92,6 +92,7 @@ class TaskBase(event.Event): def __init__(self, t, logfile, d): self._task = t self._package = d.getVar("PF", True) + self._mc = d.getVar("BB_CURRENT_MC", True) self.taskfile = d.getVar("FILE", True) self.taskname = self._task self.logfile = logfile @@ -723,7 +724,7 @@ def make_stamp(task, d, file_name = None): for mask in cleanmask: for name in glob.glob(mask): # Preserve sigdata files in the stamps directory - if "sigdata" in name: + if "sigdata" in name or "sigbasedata" in name: continue # Preserve taint files in the stamps directory if name.endswith('.taint'): diff --git a/import-layers/yocto-poky/bitbake/lib/bb/codeparser.py b/import-layers/yocto-poky/bitbake/lib/bb/codeparser.py index 25938d658..5d2d44065 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/codeparser.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/codeparser.py @@ -342,8 +342,7 @@ class ShellParser(): except pyshlex.NeedMore: raise sherrors.ShellSyntaxError("Unexpected EOF") - for token in tokens: - self.process_tokens(token) + self.process_tokens(tokens) def process_tokens(self, tokens): """Process a supplied portion of the syntax tree as returned by @@ -389,18 +388,24 @@ class ShellParser(): "case_clause": case_clause, } - for token in tokens: - name, value = token - try: - more_tokens, words = token_handlers[name](value) - except KeyError: - raise NotImplementedError("Unsupported token type " + name) + def process_token_list(tokens): + for token in tokens: + if isinstance(token, list): + process_token_list(token) + continue + name, value = token + try: + more_tokens, words = token_handlers[name](value) + except KeyError: + raise NotImplementedError("Unsupported token type " + name) + + if more_tokens: + self.process_tokens(more_tokens) - if more_tokens: - self.process_tokens(more_tokens) + if words: + self.process_words(words) - if words: - self.process_words(words) + process_token_list(tokens) def process_words(self, words): """Process a set of 'words' in pyshyacc parlance, which includes diff --git a/import-layers/yocto-poky/bitbake/lib/bb/cooker.py b/import-layers/yocto-poky/bitbake/lib/bb/cooker.py index 42831e277..07897be27 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/cooker.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/cooker.py @@ -252,6 +252,10 @@ class BBCooker: signal.signal(signal.SIGHUP, self.sigterm_exception) def config_notifications(self, event): + if event.maskname == "IN_Q_OVERFLOW": + bb.warn("inotify event queue overflowed, invalidating caches.") + self.baseconfig_valid = False + return if not event.pathname in self.configwatcher.bbwatchedfiles: return if not event.pathname in self.inotify_modified_files: @@ -259,6 +263,10 @@ class BBCooker: self.baseconfig_valid = False def notifications(self, event): + if event.maskname == "IN_Q_OVERFLOW": + bb.warn("inotify event queue overflowed, invalidating caches.") + self.parsecache_valid = False + return if not event.pathname in self.inotify_modified_files: self.inotify_modified_files.append(event.pathname) self.parsecache_valid = False @@ -657,8 +665,40 @@ class BBCooker: # A task of None means use the default task if task is None: task = self.configuration.cmd + if not task.startswith("do_"): + task = "do_%s" % task + + targetlist = self.checkPackages(pkgs_to_build, task) + fulltargetlist = [] + defaulttask_implicit = '' + defaulttask_explicit = False + wildcard = False + + # Wild card expansion: + # Replace string such as "multiconfig:*:bash" + # into "multiconfig:A:bash multiconfig:B:bash bash" + for k in targetlist: + if k.startswith("multiconfig:"): + if wildcard: + bb.fatal('multiconfig conflict') + if k.split(":")[1] == "*": + wildcard = True + for mc in self.multiconfigs: + if mc: + fulltargetlist.append(k.replace('*', mc)) + # implicit default task + else: + defaulttask_implicit = k.split(":")[2] + else: + fulltargetlist.append(k) + else: + defaulttask_explicit = True + fulltargetlist.append(k) - fulltargetlist = self.checkPackages(pkgs_to_build, task) + if not defaulttask_explicit and defaulttask_implicit != '': + fulltargetlist.append(defaulttask_implicit) + + bb.debug(1,"Target list: %s" % (str(fulltargetlist))) taskdata = {} localdata = {} @@ -715,6 +755,9 @@ class BBCooker: Create a dependency graph of pkgs_to_build including reverse dependency information. """ + if not task.startswith("do_"): + task = "do_%s" % task + runlist, taskdata = self.prepareTreeData(pkgs_to_build, task) rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist) rq.rqdata.prepare() @@ -818,6 +861,9 @@ class BBCooker: """ Create a dependency tree of pkgs_to_build, returning the data. """ + if not task.startswith("do_"): + task = "do_%s" % task + _, taskdata = self.prepareTreeData(pkgs_to_build, task) seen_fns = [] @@ -1318,6 +1364,8 @@ class BBCooker: # If we are told to do the None task then query the default task if (task == None): task = self.configuration.cmd + if not task.startswith("do_"): + task = "do_%s" % task fn, cls, mc = bb.cache.virtualfn2realfn(buildfile) fn = self.matchFile(fn) @@ -1354,8 +1402,6 @@ class BBCooker: # Invalidate task for target if force mode active if self.configuration.force: logger.verbose("Invalidate task %s, %s", task, fn) - if not task.startswith("do_"): - task = "do_%s" % task bb.parse.siggen.invalidate_task(task, self.recipecaches[mc], fn) # Setup taskdata structure @@ -1367,8 +1413,6 @@ class BBCooker: bb.event.fire(bb.event.BuildStarted(buildname, [item]), self.expanded_data) # Execute the runqueue - if not task.startswith("do_"): - task = "do_%s" % task runlist = [[mc, item, task, fn]] rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist) @@ -1579,7 +1623,8 @@ class BBCooker: if self.state != state.parsing and not self.parsecache_valid: self.parseConfiguration () if CookerFeatures.SEND_SANITYEVENTS in self.featureset: - bb.event.fire(bb.event.SanityCheck(False), self.data) + for mc in self.multiconfigs: + bb.event.fire(bb.event.SanityCheck(False), self.databuilder.mcdata[mc]) for mc in self.multiconfigs: ignore = self.databuilder.mcdata[mc].getVar("ASSUME_PROVIDED", True) or "" diff --git a/import-layers/yocto-poky/bitbake/lib/bb/cookerdata.py b/import-layers/yocto-poky/bitbake/lib/bb/cookerdata.py index b07c26643..98f56ac7b 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/cookerdata.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/cookerdata.py @@ -288,7 +288,7 @@ class CookerDataBuilder(object): multiconfig = (self.data.getVar("BBMULTICONFIG", True) or "").split() for config in multiconfig: - mcdata = self.parseConfigurationFiles(['conf/multiconfig/%s.conf' % config] + self.prefiles, self.postfiles) + mcdata = self.parseConfigurationFiles(self.prefiles, self.postfiles, config) bb.event.fire(bb.event.ConfigParsed(), mcdata) self.mcdata[config] = mcdata @@ -304,8 +304,9 @@ class CookerDataBuilder(object): def _findLayerConf(self, data): return findConfigFile("bblayers.conf", data) - def parseConfigurationFiles(self, prefiles, postfiles): + def parseConfigurationFiles(self, prefiles, postfiles, mc = "default"): data = bb.data.createCopy(self.basedata) + data.setVar("BB_CURRENT_MC", mc) # Parse files for loading *before* bitbake.conf and any includes for f in prefiles: diff --git a/import-layers/yocto-poky/bitbake/lib/bb/data.py b/import-layers/yocto-poky/bitbake/lib/bb/data.py index c1f27cd0c..c56965c60 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/data.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/data.py @@ -258,11 +258,13 @@ def exported_keys(d): not d.getVarFlag(key, 'unexport', False)) def exported_vars(d): - for key in exported_keys(d): + k = list(exported_keys(d)) + for key in k: try: value = d.getVar(key, True) - except Exception: - pass + except Exception as err: + bb.warn("%s: Unable to export ${%s}: %s" % (d.getVar("FILE", True), key, err)) + continue if value is not None: yield key, str(value) diff --git a/import-layers/yocto-poky/bitbake/lib/bb/data_smart.py b/import-layers/yocto-poky/bitbake/lib/bb/data_smart.py index f100446dc..805a9a71f 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/data_smart.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/data_smart.py @@ -748,13 +748,14 @@ class DataSmart(MutableMapping): if match: removes.extend(self.expand(r).split()) - filtered = filter(lambda v: v not in removes, - value.split()) - value = " ".join(filtered) - if expand and var in self.expand_cache: - # We need to ensure the expand cache has the correct value - # flag == "_content" here - self.expand_cache[var].value = value + if removes: + filtered = filter(lambda v: v not in removes, + value.split()) + value = " ".join(filtered) + if expand and var in self.expand_cache: + # We need to ensure the expand cache has the correct value + # flag == "_content" here + self.expand_cache[var].value = value return value def delVarFlag(self, var, flag, **loginfo): diff --git a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/wget.py b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/wget.py index ecb946aa8..23d48acb0 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/fetch2/wget.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/fetch2/wget.py @@ -108,9 +108,8 @@ class Wget(FetchMethod): bb.utils.mkdirhier(os.path.dirname(dldir + os.sep + ud.localfile)) fetchcmd += " -O " + dldir + os.sep + ud.localfile - if ud.user: - up = ud.user.split(":") - fetchcmd += " --user=%s --password=%s --auth-no-challenge" % (up[0],up[1]) + if ud.user and ud.pswd: + fetchcmd += " --user=%s --password=%s --auth-no-challenge" % (ud.user, ud.pswd) uri = ud.url.split(";")[0] if os.path.exists(ud.localpath): diff --git a/import-layers/yocto-poky/bitbake/lib/bb/runqueue.py b/import-layers/yocto-poky/bitbake/lib/bb/runqueue.py index 84b268580..9384c72ba 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/runqueue.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/runqueue.py @@ -289,8 +289,8 @@ class RunQueueData: return tid + task_name_suffix def get_short_user_idstring(self, task, task_name_suffix = ""): - (mc, fn, taskname, _) = split_tid_mcfn(task) - pn = self.dataCaches[mc].pkg_fn[fn] + (mc, fn, taskname, taskfn) = split_tid_mcfn(task) + pn = self.dataCaches[mc].pkg_fn[taskfn] taskname = taskname_from_tid(task) + task_name_suffix return "%s:%s" % (pn, taskname) @@ -884,14 +884,14 @@ class RunQueueData: if not self.cooker.configuration.nosetscene: for tid in self.runtaskentries: (mc, fn, taskname, _) = split_tid_mcfn(tid) - setscenetid = fn + ":" + taskname + "_setscene" + setscenetid = tid + "_setscene" if setscenetid not in taskData[mc].taskentries: continue self.runq_setscene_tids.append(tid) def invalidate_task(tid, error_nostamp): - (mc, fn, taskname, _) = split_tid_mcfn(tid) - taskdep = self.dataCaches[mc].task_deps[fn] + (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) + taskdep = self.dataCaches[mc].task_deps[taskfn] if fn + ":" + taskname not in taskData[mc].taskentries: logger.warning("Task %s does not exist, invalidating this task will have no effect" % taskname) if 'nostamp' in taskdep and taskname in taskdep['nostamp']: @@ -997,8 +997,9 @@ class RunQueue: magic = "decafbadbad" if fakeroot: magic = magic + "beef" - fakerootcmd = self.cfgData.getVar("FAKEROOTCMD", True) - fakerootenv = (self.cfgData.getVar("FAKEROOTBASEENV", True) or "").split() + mcdata = self.cooker.databuilder.mcdata[mc] + fakerootcmd = mcdata.getVar("FAKEROOTCMD", True) + fakerootenv = (mcdata.getVar("FAKEROOTBASEENV", True) or "").split() env = os.environ.copy() for key, value in (var.split('=') for var in fakerootenv): env[key] = value @@ -1059,10 +1060,9 @@ class RunQueue: for mc in self.rqdata.dataCaches: self.worker[mc] = self._start_worker(mc) - def start_fakeworker(self, rqexec): - if not self.fakeworker: - for mc in self.rqdata.dataCaches: - self.fakeworker[mc] = self._start_worker(mc, True, rqexec) + def start_fakeworker(self, rqexec, mc): + if not mc in self.fakeworker: + self.fakeworker[mc] = self._start_worker(mc, True, rqexec) def teardown_workers(self): self.teardown = True @@ -1322,7 +1322,7 @@ class RunQueue: continue sq_fn.append(fn) - sq_hashfn.append(self.rqdata.dataCaches[mc].hashfn[fn]) + sq_hashfn.append(self.rqdata.dataCaches[mc].hashfn[taskfn]) sq_hash.append(self.rqdata.runtaskentries[tid].hash) sq_taskname.append(taskname) sq_task.append(tid) @@ -1402,8 +1402,8 @@ class RunQueue: for tid in invalidtasks: - (mc, fn, taskname, _) = split_tid_mcfn(tid) - pn = self.rqdata.dataCaches[mc].pkg_fn[fn] + (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) + pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn] h = self.rqdata.runtaskentries[tid].hash matches = bb.siggen.find_siginfo(pn, taskname, [], self.cfgData) match = None @@ -1506,8 +1506,8 @@ class RunQueueExecute: taskdata = {} taskdeps.add(task) for dep in taskdeps: - (mc, fn, taskname, _) = split_tid_mcfn(dep) - pn = self.rqdata.dataCaches[mc].pkg_fn[fn] + (mc, fn, taskname, taskfn) = split_tid_mcfn(dep) + pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn] taskdata[dep] = [pn, taskname, fn] call = self.rq.depvalidate + "(task, taskdata, notneeded, d)" locs = { "task" : task, "taskdata" : taskdata, "notneeded" : self.scenequeue_notneeded, "d" : self.cooker.expanded_data } @@ -1707,7 +1707,7 @@ class RunQueueExecuteTasks(RunQueueExecute): # Check tasks that are going to run against the whitelist def check_norun_task(tid, showerror=False): - (mc, fn, taskname, _) = split_tid_mcfn(tid) + (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) # Ignore covered tasks if tid in self.rq.scenequeue_covered: return False @@ -1715,11 +1715,11 @@ class RunQueueExecuteTasks(RunQueueExecute): if self.rq.check_stamp_task(tid, taskname, cache=self.stampcache): return False # Ignore noexec tasks - taskdep = self.rqdata.dataCaches[mc].task_deps[fn] + taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn] if 'noexec' in taskdep and taskname in taskdep['noexec']: return False - pn = self.rqdata.dataCaches[mc].pkg_fn[fn] + pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn] if not check_setscene_enforce_whitelist(pn, taskname, self.rqdata.setscenewhitelist): if showerror: if tid in self.rqdata.runq_setscene_tids: @@ -1787,9 +1787,9 @@ class RunQueueExecuteTasks(RunQueueExecute): taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn] if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run: - if not self.rq.fakeworker: + if not mc in self.rq.fakeworker: try: - self.rq.start_fakeworker(self) + self.rq.start_fakeworker(self, mc) except OSError as exc: logger.critical("Failed to spawn fakeroot worker to run %s: %s" % (task, str(exc))) self.rq.state = runQueueFailed @@ -1868,6 +1868,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute): sq_revdeps_new = {} sq_revdeps_squash = {} self.sq_harddeps = {} + self.stamps = {} # We need to construct a dependency graph for the setscene functions. Intermediate # dependencies between the setscene tasks only complicate the code. This code @@ -1978,9 +1979,10 @@ class RunQueueExecuteScenequeue(RunQueueExecute): # e.g. do_sometask_setscene[depends] = "targetname:do_someothertask_setscene" # Note that anything explicitly depended upon will have its reverse dependencies removed to avoid circular dependencies for tid in self.rqdata.runq_setscene_tids: - (mc, fn, taskname, _) = split_tid_mcfn(tid) - realtid = fn + ":" + taskname + "_setscene" + (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) + realtid = tid + "_setscene" idepends = self.rqdata.taskData[mc].taskentries[realtid].idepends + self.stamps[tid] = bb.build.stampfile(taskname + "_setscene", self.rqdata.dataCaches[mc], taskfn, noextra=True) for (depname, idependtask) in idepends: if depname not in self.rqdata.taskData[mc].build_targets: @@ -2044,7 +2046,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute): for tid in self.sq_revdeps: (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) - taskdep = self.rqdata.dataCaches[mc].task_deps[fn] + taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn] if 'noexec' in taskdep and taskname in taskdep['noexec']: noexec.append(tid) @@ -2065,7 +2067,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute): continue sq_fn.append(fn) - sq_hashfn.append(self.rqdata.dataCaches[mc].hashfn[fn]) + sq_hashfn.append(self.rqdata.dataCaches[mc].hashfn[taskfn]) sq_hash.append(self.rqdata.runtaskentries[tid].hash) sq_taskname.append(taskname) sq_task.append(tid) @@ -2113,8 +2115,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute): def check_taskfail(self, task): if self.rqdata.setscenewhitelist: realtask = task.split('_setscene')[0] - (mc, fn, taskname, _) = split_tid_mcfn(realtask) - pn = self.rqdata.dataCaches[mc].pkg_fn[fn] + (mc, fn, taskname, taskfn) = split_tid_mcfn(realtask) + pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn] if not check_setscene_enforce_whitelist(pn, taskname, self.rqdata.setscenewhitelist): logger.error('Task %s.%s failed' % (pn, taskname + "_setscene")) self.rq.state = runQueueCleanUp @@ -2157,7 +2159,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute): if self.stats.active < self.number_tasks: # Find the next setscene to run for nexttask in self.rqdata.runq_setscene_tids: - if nexttask in self.runq_buildable and nexttask not in self.runq_running: + if nexttask in self.runq_buildable and nexttask not in self.runq_running and self.stamps[nexttask] not in self.build_stamps.values(): if nexttask in self.unskippable: logger.debug(2, "Setscene task %s is unskippable" % nexttask) if nexttask not in self.unskippable and len(self.sq_revdeps[nexttask]) > 0 and self.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sq_revdeps[nexttask], True): @@ -2199,14 +2201,16 @@ class RunQueueExecuteScenequeue(RunQueueExecute): taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn] if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run: - if not self.rq.fakeworker: - self.rq.start_fakeworker(self) + if not mc in self.rq.fakeworker: + self.rq.start_fakeworker(self, mc) self.rq.fakeworker[mc].process.stdin.write(b"" + pickle.dumps((taskfn, task, taskname, True, self.cooker.collection.get_file_appends(taskfn), None)) + b"") self.rq.fakeworker[mc].process.stdin.flush() else: self.rq.worker[mc].process.stdin.write(b"" + pickle.dumps((taskfn, task, taskname, True, self.cooker.collection.get_file_appends(taskfn), None)) + b"") self.rq.worker[mc].process.stdin.flush() + self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True) + self.build_stamps2.append(self.build_stamps[task]) self.runq_running.add(task) self.stats.taskActive() if self.stats.active < self.number_tasks: diff --git a/import-layers/yocto-poky/bitbake/lib/bb/siggen.py b/import-layers/yocto-poky/bitbake/lib/bb/siggen.py index 3a7dac4cb..542bbb9d1 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/siggen.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/siggen.py @@ -30,6 +30,7 @@ class SignatureGenerator(object): name = "noop" def __init__(self, data): + self.basehash = {} self.taskhash = {} self.runtaskdeps = {} self.file_checksum_values = {} @@ -61,11 +62,10 @@ class SignatureGenerator(object): return def get_taskdata(self): - return (self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints) + return (self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints, self.basehash) def set_taskdata(self, data): - self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints = data - + self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints, self.basehash = data class SignatureGeneratorBasic(SignatureGenerator): """ @@ -133,7 +133,11 @@ class SignatureGeneratorBasic(SignatureGenerator): var = lookupcache[dep] if var is not None: data = data + str(var) - self.basehash[fn + "." + task] = hashlib.md5(data.encode("utf-8")).hexdigest() + datahash = hashlib.md5(data.encode("utf-8")).hexdigest() + k = fn + "." + task + if k in self.basehash and self.basehash[k] != datahash: + bb.error("When reparsing %s, the basehash value changed from %s to %s. The metadata is not deterministic and this needs to be fixed." % (k, self.basehash[k], datahash)) + self.basehash[k] = datahash taskdeps[task] = alldeps self.taskdeps[fn] = taskdeps @@ -182,6 +186,7 @@ class SignatureGeneratorBasic(SignatureGenerator): def get_taskhash(self, fn, task, deps, dataCache): k = fn + "." + task data = dataCache.basetaskhash[k] + self.basehash[k] = data self.runtaskdeps[k] = [] self.file_checksum_values[k] = [] recipename = dataCache.pkg_fn[fn] @@ -278,6 +283,15 @@ class SignatureGeneratorBasic(SignatureGenerator): if 'nostamp:' in self.taints[k]: data['taint'] = self.taints[k] + computed_basehash = calc_basehash(data) + if computed_basehash != self.basehash[k]: + bb.error("Basehash mismatch %s versus %s for %s" % (computed_basehash, self.basehash[k], k)) + if runtime and k in self.taskhash: + computed_taskhash = calc_taskhash(data) + if computed_taskhash != self.taskhash[k]: + bb.error("Taskhash mismatch %s versus %s for %s" % (computed_taskhash, self.taskhash[k], k)) + sigfile = sigfile.replace(self.taskhash[k], computed_taskhash) + fd, tmpfile = tempfile.mkstemp(dir=os.path.dirname(sigfile), prefix="sigtask.") try: with os.fdopen(fd, "wb") as stream: @@ -292,15 +306,6 @@ class SignatureGeneratorBasic(SignatureGenerator): pass raise err - computed_basehash = calc_basehash(data) - if computed_basehash != self.basehash[k]: - bb.error("Basehash mismatch %s versus %s for %s" % (computed_basehash, self.basehash[k], k)) - if runtime and k in self.taskhash: - computed_taskhash = calc_taskhash(data) - if computed_taskhash != self.taskhash[k]: - bb.error("Taskhash mismatch %s versus %s for %s" % (computed_taskhash, self.taskhash[k], k)) - - def dump_sigs(self, dataCaches, options): for fn in self.taskdeps: for task in self.taskdeps[fn]: @@ -346,9 +351,14 @@ def dump_this_task(outfile, d): bb.parse.siggen.dump_sigtask(fn, task, outfile, "customfile:" + referencestamp) def clean_basepath(a): + mc = None + if a.startswith("multiconfig:"): + _, mc, a = a.split(":", 2) b = a.rsplit("/", 2)[1] + a.rsplit("/", 2)[2] if a.startswith("virtual:"): b = b + ":" + a.rsplit(":", 1)[0] + if mc: + b = b + ":multiconfig:" + mc return b def clean_basepaths(a): @@ -554,7 +564,8 @@ def calc_taskhash(sigdata): data = data + sigdata['runtaskhashes'][dep] for c in sigdata['file_checksum_values']: - data = data + c[1] + if c[1]: + data = data + c[1] if 'taint' in sigdata: if 'nostamp:' in sigdata['taint']: diff --git a/import-layers/yocto-poky/bitbake/lib/bb/tinfoil.py b/import-layers/yocto-poky/bitbake/lib/bb/tinfoil.py index 8899e861c..9fa5b5b3d 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/tinfoil.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/tinfoil.py @@ -51,10 +51,13 @@ class Tinfoil: features = [] if tracking: features.append(CookerFeatures.BASEDATASTORE_TRACKING) + cleanedvars = bb.utils.clean_environment() self.cooker = BBCooker(self.config, features) self.config_data = self.cooker.data bb.providers.logger.setLevel(logging.ERROR) self.cooker_data = None + for k in cleanedvars: + os.environ[k] = cleanedvars[k] def register_idle_function(self, function, data): pass diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py b/import-layers/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py index 5b69660a3..3ddcb2ac6 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py @@ -982,6 +982,31 @@ class BuildInfoHelper(object): pass return task_information + def _get_layer_version_for_dependency(self, pathRE): + """ Returns the layer in the toaster db that has a full regex match to the pathRE. + pathRE - the layer path passed as a regex in the event. It is created in + cooker.py as a collection for the layer priorities. + """ + self._ensure_build() + + def _sort_longest_path(layer_version): + assert isinstance(layer_version, Layer_Version) + return len(layer_version.local_path) + + # we don't care if we match the trailing slashes + p = re.compile(re.sub("/[^/]*?$","",pathRE)) + # Heuristics: we always match recipe to the deepest layer path in the discovered layers + for lvo in sorted(self.orm_wrapper.layer_version_objects, reverse=True, key=_sort_longest_path): + if p.fullmatch(lvo.local_path): + return lvo + if lvo.layer.local_source_dir: + if p.fullmatch(lvo.layer.local_source_dir): + return lvo + #if we get here, we didn't read layers correctly; dump whatever information we have on the error log + logger.warning("Could not match layer dependency for path %s : %s", path, self.orm_wrapper.layer_version_objects) + + + def _get_layer_version_for_path(self, path): self._ensure_build() @@ -1372,7 +1397,7 @@ class BuildInfoHelper(object): if 'layer-priorities' in event._depgraph.keys(): for lv in event._depgraph['layer-priorities']: (_, path, _, priority) = lv - layer_version_obj = self._get_layer_version_for_path(path[1:]) # paths start with a ^ + layer_version_obj = self._get_layer_version_for_dependency(path) assert layer_version_obj is not None layer_version_obj.priority = priority layer_version_obj.save() diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/uihelper.py b/import-layers/yocto-poky/bitbake/lib/bb/ui/uihelper.py index fda7cc2c7..113fcedea 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/ui/uihelper.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/ui/uihelper.py @@ -32,7 +32,10 @@ class BBUIHelper: def eventHandler(self, event): if isinstance(event, bb.build.TaskStarted): - self.running_tasks[event.pid] = { 'title' : "%s %s" % (event._package, event._task), 'starttime' : time.time() } + if event._mc != "default": + self.running_tasks[event.pid] = { 'title' : "mc:%s:%s %s" % (event._mc, event._package, event._task), 'starttime' : time.time() } + else: + self.running_tasks[event.pid] = { 'title' : "%s %s" % (event._package, event._task), 'starttime' : time.time() } self.running_pids.append(event.pid) self.needUpdate = True elif isinstance(event, bb.build.TaskSucceeded): diff --git a/import-layers/yocto-poky/bitbake/lib/bb/utils.py b/import-layers/yocto-poky/bitbake/lib/bb/utils.py index 729848a1c..16fc9db25 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/utils.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/utils.py @@ -378,7 +378,7 @@ def _print_exception(t, value, tb, realfile, text, context): # If the exception is from spwaning a task, let's be helpful and display # the output (which hopefully includes stderr). - if isinstance(value, subprocess.CalledProcessError): + if isinstance(value, subprocess.CalledProcessError) and value.output: error.append("Subprocess output:") error.append(value.output.decode("utf-8", errors="ignore")) finally: -- cgit v1.2.1