diff options
Diffstat (limited to 'import-layers/yocto-poky/meta/lib/oe')
27 files changed, 1081 insertions, 559 deletions
diff --git a/import-layers/yocto-poky/meta/lib/oe/buildhistory_analysis.py b/import-layers/yocto-poky/meta/lib/oe/buildhistory_analysis.py index 5395c768a..b6c0265c1 100644 --- a/import-layers/yocto-poky/meta/lib/oe/buildhistory_analysis.py +++ b/import-layers/yocto-poky/meta/lib/oe/buildhistory_analysis.py @@ -62,7 +62,7 @@ class ChangeRecord: def pkglist_combine(depver): pkglist = [] - for k,v in depver.iteritems(): + for k,v in depver.items(): if v: pkglist.append("%s (%s)" % (k,v)) else: @@ -190,7 +190,7 @@ class FileChange: def blob_to_dict(blob): - alines = blob.data_stream.read().splitlines() + alines = [line for line in blob.data_stream.read().decode('utf-8').splitlines()] adict = {} for line in alines: splitv = [i.strip() for i in line.split('=',1)] @@ -220,7 +220,7 @@ def compare_file_lists(alines, blines): adict = file_list_to_dict(alines) bdict = file_list_to_dict(blines) filechanges = [] - for path, splitv in adict.iteritems(): + for path, splitv in adict.items(): newsplitv = bdict.pop(path, None) if newsplitv: # Check type @@ -359,6 +359,24 @@ def compare_dict_blobs(path, ablob, bblob, report_all, report_ver): if ' '.join(alist) == ' '.join(blist): continue + if key == 'PKGR' and not report_all: + vers = [] + # strip leading 'r' and dots + for ver in (astr.split()[0], bstr.split()[0]): + if ver.startswith('r'): + ver = ver[1:] + vers.append(ver.replace('.', '')) + maxlen = max(len(vers[0]), len(vers[1])) + try: + # pad with '0' and convert to int + vers = [int(ver.ljust(maxlen, '0')) for ver in vers] + except ValueError: + pass + else: + # skip decrements and increments + if abs(vers[0] - vers[1]) == 1: + continue + chg = ChangeRecord(path, key, astr, bstr, monitored) changes.append(chg) return changes @@ -378,34 +396,34 @@ def process_changes(repopath, revision1, revision2='HEAD', report_all=False, rep if filename == 'latest': changes.extend(compare_dict_blobs(path, d.a_blob, d.b_blob, report_all, report_ver)) elif filename.startswith('latest.'): - chg = ChangeRecord(path, filename, d.a_blob.data_stream.read(), d.b_blob.data_stream.read(), True) + chg = ChangeRecord(path, filename, d.a_blob.data_stream.read().decode('utf-8'), d.b_blob.data_stream.read().decode('utf-8'), True) changes.append(chg) elif path.startswith('images/'): filename = os.path.basename(d.a_blob.path) if filename in img_monitor_files: if filename == 'files-in-image.txt': - alines = d.a_blob.data_stream.read().splitlines() - blines = d.b_blob.data_stream.read().splitlines() + alines = d.a_blob.data_stream.read().decode('utf-8').splitlines() + blines = d.b_blob.data_stream.read().decode('utf-8').splitlines() filechanges = compare_file_lists(alines,blines) if filechanges: chg = ChangeRecord(path, filename, None, None, True) chg.filechanges = filechanges changes.append(chg) elif filename == 'installed-package-names.txt': - alines = d.a_blob.data_stream.read().splitlines() - blines = d.b_blob.data_stream.read().splitlines() + alines = d.a_blob.data_stream.read().decode('utf-8').splitlines() + blines = d.b_blob.data_stream.read().decode('utf-8').splitlines() filechanges = compare_lists(alines,blines) if filechanges: chg = ChangeRecord(path, filename, None, None, True) chg.filechanges = filechanges changes.append(chg) else: - chg = ChangeRecord(path, filename, d.a_blob.data_stream.read(), d.b_blob.data_stream.read(), True) + chg = ChangeRecord(path, filename, d.a_blob.data_stream.read().decode('utf-8'), d.b_blob.data_stream.read().decode('utf-8'), True) changes.append(chg) elif filename == 'image-info.txt': changes.extend(compare_dict_blobs(path, d.a_blob, d.b_blob, report_all, report_ver)) elif '/image-files/' in path: - chg = ChangeRecord(path, filename, d.a_blob.data_stream.read(), d.b_blob.data_stream.read(), True) + chg = ChangeRecord(path, filename, d.a_blob.data_stream.read().decode('utf-8'), d.b_blob.data_stream.read().decode('utf-8'), True) changes.append(chg) # Look for added preinst/postinst/prerm/postrm @@ -419,7 +437,7 @@ def process_changes(repopath, revision1, revision2='HEAD', report_all=False, rep if filename == 'latest': addedpkgs.append(path) elif filename.startswith('latest.'): - chg = ChangeRecord(path, filename[7:], '', d.b_blob.data_stream.read(), True) + chg = ChangeRecord(path, filename[7:], '', d.b_blob.data_stream.read().decode('utf-8'), True) addedchanges.append(chg) for chg in addedchanges: found = False @@ -436,7 +454,7 @@ def process_changes(repopath, revision1, revision2='HEAD', report_all=False, rep if path.startswith('packages/'): filename = os.path.basename(d.a_blob.path) if filename != 'latest' and filename.startswith('latest.'): - chg = ChangeRecord(path, filename[7:], d.a_blob.data_stream.read(), '', True) + chg = ChangeRecord(path, filename[7:], d.a_blob.data_stream.read().decode('utf-8'), '', True) changes.append(chg) # Link related changes diff --git a/import-layers/yocto-poky/meta/lib/oe/classextend.py b/import-layers/yocto-poky/meta/lib/oe/classextend.py index 5107ecde2..4c8a00070 100644 --- a/import-layers/yocto-poky/meta/lib/oe/classextend.py +++ b/import-layers/yocto-poky/meta/lib/oe/classextend.py @@ -1,3 +1,5 @@ +import collections + class ClassExtender(object): def __init__(self, extname, d): self.extname = extname @@ -77,7 +79,7 @@ class ClassExtender(object): self.d.setVar("EXTENDPKGV", orig) return deps = bb.utils.explode_dep_versions2(deps) - newdeps = {} + newdeps = collections.OrderedDict() for dep in deps: newdeps[self.map_depends(dep)] = deps[dep] diff --git a/import-layers/yocto-poky/meta/lib/oe/classutils.py b/import-layers/yocto-poky/meta/lib/oe/classutils.py index 58188fdd6..e7856c86f 100644 --- a/import-layers/yocto-poky/meta/lib/oe/classutils.py +++ b/import-layers/yocto-poky/meta/lib/oe/classutils.py @@ -1,4 +1,11 @@ -class ClassRegistry(type): + +class ClassRegistryMeta(type): + """Give each ClassRegistry their own registry""" + def __init__(cls, name, bases, attrs): + cls.registry = {} + type.__init__(cls, name, bases, attrs) + +class ClassRegistry(type, metaclass=ClassRegistryMeta): """Maintain a registry of classes, indexed by name. Note that this implementation requires that the names be unique, as it uses @@ -12,12 +19,6 @@ Subclasses of ClassRegistry may define an 'implemented' property to exert control over whether the class will be added to the registry (e.g. to keep abstract base classes out of the registry).""" priority = 0 - class __metaclass__(type): - """Give each ClassRegistry their own registry""" - def __init__(cls, name, bases, attrs): - cls.registry = {} - type.__init__(cls, name, bases, attrs) - def __init__(cls, name, bases, attrs): super(ClassRegistry, cls).__init__(name, bases, attrs) try: @@ -34,7 +35,7 @@ abstract base classes out of the registry).""" @classmethod def prioritized(tcls): - return sorted(tcls.registry.values(), + return sorted(list(tcls.registry.values()), key=lambda v: v.priority, reverse=True) def unregister(cls): diff --git a/import-layers/yocto-poky/meta/lib/oe/copy_buildsystem.py b/import-layers/yocto-poky/meta/lib/oe/copy_buildsystem.py index 7b9a0ee06..afaff6859 100644 --- a/import-layers/yocto-poky/meta/lib/oe/copy_buildsystem.py +++ b/import-layers/yocto-poky/meta/lib/oe/copy_buildsystem.py @@ -17,7 +17,7 @@ class BuildSystem(object): def __init__(self, context, d): self.d = d self.context = context - self.layerdirs = d.getVar('BBLAYERS', True).split() + self.layerdirs = [os.path.abspath(pth) for pth in d.getVar('BBLAYERS', True).split()] self.layers_exclude = (d.getVar('SDK_LAYERS_EXCLUDE', True) or "").split() def copy_bitbake_and_layers(self, destdir, workspace_name=None): @@ -26,7 +26,7 @@ class BuildSystem(object): bb.utils.mkdirhier(destdir) layers = list(self.layerdirs) - corebase = self.d.getVar('COREBASE', True) + corebase = os.path.abspath(self.d.getVar('COREBASE', True)) layers.append(corebase) # Exclude layers @@ -124,7 +124,7 @@ class BuildSystem(object): def generate_locked_sigs(sigfile, d): bb.utils.mkdirhier(os.path.dirname(sigfile)) depd = d.getVar('BB_TASKDEPDATA', False) - tasks = ['%s.%s' % (v[2], v[1]) for v in depd.itervalues()] + tasks = ['%s.%s' % (v[2], v[1]) for v in depd.values()] bb.parse.siggen.dump_lockedsigs(sigfile, tasks) def prune_lockedsigs(excluded_tasks, excluded_targets, lockedsigs, pruned_output): @@ -145,7 +145,7 @@ def prune_lockedsigs(excluded_tasks, excluded_targets, lockedsigs, pruned_output invalue = True f.write(line) -def merge_lockedsigs(copy_tasks, lockedsigs_main, lockedsigs_extra, merged_output, copy_output): +def merge_lockedsigs(copy_tasks, lockedsigs_main, lockedsigs_extra, merged_output, copy_output=None): merged = {} arch_order = [] with open(lockedsigs_main, 'r') as f: @@ -195,23 +195,46 @@ def merge_lockedsigs(copy_tasks, lockedsigs_main, lockedsigs_extra, merged_outpu fulltypes.append(typename) f.write('SIGGEN_LOCKEDSIGS_TYPES = "%s"\n' % ' '.join(fulltypes)) - write_sigs_file(copy_output, tocopy.keys(), tocopy) + if copy_output: + write_sigs_file(copy_output, list(tocopy.keys()), tocopy) if merged_output: write_sigs_file(merged_output, arch_order, merged) -def create_locked_sstate_cache(lockedsigs, input_sstate_cache, output_sstate_cache, d, fixedlsbstring=""): +def create_locked_sstate_cache(lockedsigs, input_sstate_cache, output_sstate_cache, d, fixedlsbstring="", filterfile=None): + import shutil bb.note('Generating sstate-cache...') nativelsbstring = d.getVar('NATIVELSBSTRING', True) - bb.process.run("gen-lockedsig-cache %s %s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache, nativelsbstring)) - if fixedlsbstring: + bb.process.run("gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache, nativelsbstring, filterfile or '')) + if fixedlsbstring and nativelsbstring != fixedlsbstring: nativedir = output_sstate_cache + '/' + nativelsbstring if os.path.isdir(nativedir): destdir = os.path.join(output_sstate_cache, fixedlsbstring) - bb.utils.mkdirhier(destdir) + for root, _, files in os.walk(nativedir): + for fn in files: + src = os.path.join(root, fn) + dest = os.path.join(destdir, os.path.relpath(src, nativedir)) + if os.path.exists(dest): + # Already exists, and it'll be the same file, so just delete it + os.unlink(src) + else: + bb.utils.mkdirhier(os.path.dirname(dest)) + shutil.move(src, dest) + +def check_sstate_task_list(d, targets, filteroutfile, cmdprefix='', cwd=None, logfile=None): + import subprocess - dirlist = os.listdir(nativedir) - for i in dirlist: - src = os.path.join(nativedir, i) - dest = os.path.join(destdir, i) - os.rename(src, dest) + bb.note('Generating sstate task list...') + + if not cwd: + cwd = os.getcwd() + if logfile: + logparam = '-l %s' % logfile + else: + logparam = '' + cmd = "%sBB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam) + env = dict(d.getVar('BB_ORIGENV', False)) + env.pop('BUILDDIR', '') + pathitems = env['PATH'].split(':') + env['PATH'] = ':'.join([item for item in pathitems if not item.endswith('/bitbake/bin')]) + bb.process.run(cmd, stderr=subprocess.STDOUT, env=env, cwd=cwd, executable='/bin/bash') diff --git a/import-layers/yocto-poky/meta/lib/oe/data.py b/import-layers/yocto-poky/meta/lib/oe/data.py index e49572177..ee48950a8 100644 --- a/import-layers/yocto-poky/meta/lib/oe/data.py +++ b/import-layers/yocto-poky/meta/lib/oe/data.py @@ -7,11 +7,11 @@ def typed_value(key, d): flags = d.getVarFlags(key) if flags is not None: flags = dict((flag, d.expand(value)) - for flag, value in flags.iteritems()) + for flag, value in list(flags.items())) else: flags = {} try: return oe.maketype.create(d.getVar(key, True) or '', var_type, **flags) - except (TypeError, ValueError), exc: + except (TypeError, ValueError) as exc: bb.msg.fatal("Data", "%s: %s" % (key, str(exc))) diff --git a/import-layers/yocto-poky/meta/lib/oe/distro_check.py b/import-layers/yocto-poky/meta/lib/oe/distro_check.py index 8655a6fc1..87c52fae9 100644 --- a/import-layers/yocto-poky/meta/lib/oe/distro_check.py +++ b/import-layers/yocto-poky/meta/lib/oe/distro_check.py @@ -1,53 +1,35 @@ from contextlib import contextmanager -@contextmanager + +from bb.utils import export_proxies + def create_socket(url, d): import urllib - socket = urllib.urlopen(url, proxies=get_proxies(d)) + + socket = None try: - yield socket - finally: - socket.close() + export_proxies(d) + socket = urllib.request.urlopen(url) + except: + bb.warn("distro_check: create_socket url %s can't access" % url) -def get_proxies(d): - proxies = {} - for key in ['http', 'https', 'ftp', 'ftps', 'no', 'all']: - proxy = d.getVar(key + '_proxy', True) - if proxy: - proxies[key] = proxy - return proxies + return socket def get_links_from_url(url, d): "Return all the href links found on the web location" - import sgmllib - - class LinksParser(sgmllib.SGMLParser): - def parse(self, s): - "Parse the given string 's'." - self.feed(s) - self.close() - - def __init__(self, verbose=0): - "Initialise an object passing 'verbose' to the superclass." - sgmllib.SGMLParser.__init__(self, verbose) - self.hyperlinks = [] - - def start_a(self, attributes): - "Process a hyperlink and its 'attributes'." - for name, value in attributes: - if name == "href": - self.hyperlinks.append(value.strip('/')) - - def get_hyperlinks(self): - "Return the list of hyperlinks." - return self.hyperlinks + from bs4 import BeautifulSoup, SoupStrainer + + hyperlinks = [] - with create_socket(url,d) as sock: + webpage = '' + sock = create_socket(url,d) + if sock: webpage = sock.read() - linksparser = LinksParser() - linksparser.parse(webpage) - return linksparser.get_hyperlinks() + soup = BeautifulSoup(webpage, "html.parser", parse_only=SoupStrainer("a")) + for line in soup.find_all('a', href=True): + hyperlinks.append(line['href'].strip('/')) + return hyperlinks def find_latest_numeric_release(url, d): "Find the latest listed numeric release on the given url" @@ -104,8 +86,8 @@ def get_source_package_list_from_url(url, section, d): bb.note("Reading %s: %s" % (url, section)) links = get_links_from_url(url, d) - srpms = filter(is_src_rpm, links) - names_list = map(package_name_from_srpm, srpms) + srpms = list(filter(is_src_rpm, links)) + names_list = list(map(package_name_from_srpm, srpms)) new_pkgs = [] for pkgs in names_list: @@ -162,14 +144,18 @@ def find_latest_debian_release(url, d): def get_debian_style_source_package_list(url, section, d): "Return the list of package-names stored in the debian style Sources.gz file" - with create_socket(url,d) as sock: - webpage = sock.read() - import tempfile - tmpfile = tempfile.NamedTemporaryFile(mode='wb', prefix='oecore.', suffix='.tmp', delete=False) - tmpfilename=tmpfile.name - tmpfile.write(sock.read()) - tmpfile.close() + import tempfile import gzip + + webpage = '' + sock = create_socket(url,d) + if sock: + webpage = sock.read() + + tmpfile = tempfile.NamedTemporaryFile(mode='wb', prefix='oecore.', suffix='.tmp', delete=False) + tmpfilename=tmpfile.name + tmpfile.write(sock.read()) + tmpfile.close() bb.note("Reading %s: %s" % (url, section)) f = gzip.open(tmpfilename) @@ -266,9 +252,9 @@ def update_distro_data(distro_check_dir, datetime, d): import fcntl try: if not os.path.exists(datetime_file): - open(datetime_file, 'w+b').close() # touch the file so that the next open won't fail + open(datetime_file, 'w+').close() # touch the file so that the next open won't fail - f = open(datetime_file, "r+b") + f = open(datetime_file, "r+") fcntl.lockf(f, fcntl.LOCK_EX) saved_datetime = f.read() if saved_datetime[0:8] != datetime[0:8]: @@ -357,8 +343,8 @@ def compare_in_distro_packages_list(distro_check_dir, d): if tmp != None: - list = tmp.split(' ') - for item in list: + list = tmp.split(' ') + for item in list: matching_distros.append(item) bb.note("Matching: %s" % matching_distros) return matching_distros diff --git a/import-layers/yocto-poky/meta/lib/oe/gpg_sign.py b/import-layers/yocto-poky/meta/lib/oe/gpg_sign.py index b83ee8672..38eb0cb13 100644 --- a/import-layers/yocto-poky/meta/lib/oe/gpg_sign.py +++ b/import-layers/yocto-poky/meta/lib/oe/gpg_sign.py @@ -71,11 +71,11 @@ class LocalSigner(object): passphrase = fobj.readline(); job = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE) - (_, stderr) = job.communicate(passphrase) + (_, stderr) = job.communicate(passphrase.encode("utf-8")) if job.returncode: raise bb.build.FuncFailed("GPG exited with code %d: %s" % - (job.returncode, stderr)) + (job.returncode, stderr.decode("utf-8"))) except IOError as e: bb.error("IO error (%s): %s" % (e.errno, e.strerror)) @@ -90,7 +90,7 @@ class LocalSigner(object): """Return the gpg version""" import subprocess try: - return subprocess.check_output((self.gpg_bin, "--version")).split()[2] + return subprocess.check_output((self.gpg_bin, "--version")).split()[2].decode("utf-8") except subprocess.CalledProcessError as e: raise bb.build.FuncFailed("Could not get gpg version: %s" % e) @@ -113,4 +113,3 @@ def get_signer(d, backend): return LocalSigner(d) else: bb.fatal("Unsupported signing backend '%s'" % backend) - diff --git a/import-layers/yocto-poky/meta/lib/oe/license.py b/import-layers/yocto-poky/meta/lib/oe/license.py index f0f661c3b..8d2fd1709 100644 --- a/import-layers/yocto-poky/meta/lib/oe/license.py +++ b/import-layers/yocto-poky/meta/lib/oe/license.py @@ -47,7 +47,7 @@ class LicenseVisitor(ast.NodeVisitor): """Get elements based on OpenEmbedded license strings""" def get_elements(self, licensestr): new_elements = [] - elements = filter(lambda x: x.strip(), license_operator.split(licensestr)) + elements = list([x for x in license_operator.split(licensestr) if x.strip()]) for pos, element in enumerate(elements): if license_pattern.match(element): if pos > 0 and license_pattern.match(elements[pos-1]): @@ -118,8 +118,8 @@ def is_included(licensestr, whitelist=None, blacklist=None): def choose_licenses(alpha, beta): """Select the option in an OR which is the 'best' (has the most included licenses).""" - alpha_weight = len(filter(include_license, alpha)) - beta_weight = len(filter(include_license, beta)) + alpha_weight = len(list(filter(include_license, alpha))) + beta_weight = len(list(filter(include_license, beta))) if alpha_weight > beta_weight: return alpha else: @@ -132,8 +132,8 @@ def is_included(licensestr, whitelist=None, blacklist=None): blacklist = [] licenses = flattened_licenses(licensestr, choose_licenses) - excluded = filter(lambda lic: exclude_license(lic), licenses) - included = filter(lambda lic: include_license(lic), licenses) + excluded = [lic for lic in licenses if exclude_license(lic)] + included = [lic for lic in licenses if include_license(lic)] if excluded: return False, excluded else: @@ -215,3 +215,21 @@ def manifest_licenses(licensestr, dont_want_licenses, canonical_license, d): manifest.licensestr = manifest.licensestr.replace('[', '(').replace(']', ')') return (manifest.licensestr, manifest.licenses) + +class ListVisitor(LicenseVisitor): + """Record all different licenses found in the license string""" + def __init__(self): + self.licenses = set() + + def visit_Str(self, node): + self.licenses.add(node.s) + +def list_licenses(licensestr): + """Simply get a list of all licenses mentioned in a license string. + Binary operators are not applied or taken into account in any way""" + visitor = ListVisitor() + try: + visitor.visit_string(licensestr) + except SyntaxError as exc: + raise LicenseSyntaxError(licensestr, exc) + return visitor.licenses diff --git a/import-layers/yocto-poky/meta/lib/oe/maketype.py b/import-layers/yocto-poky/meta/lib/oe/maketype.py index 139f33369..f88981dd9 100644 --- a/import-layers/yocto-poky/meta/lib/oe/maketype.py +++ b/import-layers/yocto-poky/meta/lib/oe/maketype.py @@ -6,7 +6,8 @@ the arguments of the type's factory for details. """ import inspect -import types +import oe.types as types +import collections available_types = {} @@ -53,7 +54,9 @@ def get_callable_args(obj): if type(obj) is type: obj = obj.__init__ - args, varargs, keywords, defaults = inspect.getargspec(obj) + sig = inspect.signature(obj) + args = list(sig.parameters.keys()) + defaults = list(s for s in sig.parameters.keys() if sig.parameters[s].default != inspect.Parameter.empty) flaglist = [] if args: if len(args) > 1 and args[0] == 'self': @@ -93,7 +96,7 @@ for name in dir(types): continue obj = getattr(types, name) - if not callable(obj): + if not isinstance(obj, collections.Callable): continue register(name, obj) diff --git a/import-layers/yocto-poky/meta/lib/oe/manifest.py b/import-layers/yocto-poky/meta/lib/oe/manifest.py index 42832f15d..95f8eb2df 100644 --- a/import-layers/yocto-poky/meta/lib/oe/manifest.py +++ b/import-layers/yocto-poky/meta/lib/oe/manifest.py @@ -4,11 +4,10 @@ import re import bb -class Manifest(object): +class Manifest(object, metaclass=ABCMeta): """ This is an abstract class. Do not instantiate this directly. """ - __metaclass__ = ABCMeta PKG_TYPE_MUST_INSTALL = "mip" PKG_TYPE_MULTILIB = "mlp" @@ -219,7 +218,7 @@ class RpmManifest(Manifest): if var in self.vars_to_split: split_pkgs = self._split_multilib(self.d.getVar(var, True)) if split_pkgs is not None: - pkgs = dict(pkgs.items() + split_pkgs.items()) + pkgs = dict(list(pkgs.items()) + list(split_pkgs.items())) else: pkg_list = self.d.getVar(var, True) if pkg_list is not None: @@ -269,7 +268,7 @@ class OpkgManifest(Manifest): if var in self.vars_to_split: split_pkgs = self._split_multilib(self.d.getVar(var, True)) if split_pkgs is not None: - pkgs = dict(pkgs.items() + split_pkgs.items()) + pkgs = dict(list(pkgs.items()) + list(split_pkgs.items())) else: pkg_list = self.d.getVar(var, True) if pkg_list is not None: diff --git a/import-layers/yocto-poky/meta/lib/oe/package.py b/import-layers/yocto-poky/meta/lib/oe/package.py index 288768954..02642f29f 100644 --- a/import-layers/yocto-poky/meta/lib/oe/package.py +++ b/import-layers/yocto-poky/meta/lib/oe/package.py @@ -8,7 +8,7 @@ def runstrip(arg): # 8 - shared library # 16 - kernel module - import commands, stat, subprocess + import stat, subprocess (file, elftype, strip) = arg @@ -64,8 +64,8 @@ def filedeprunner(arg): def process_deps(pipe, pkg, pkgdest, provides, requires): for line in pipe: - f = line.split(" ", 1)[0].strip() - line = line.split(" ", 1)[1].strip() + f = line.decode("utf-8").split(" ", 1)[0].strip() + line = line.decode("utf-8").split(" ", 1)[1].strip() if line.startswith("Requires:"): i = requires @@ -114,7 +114,12 @@ def read_shlib_providers(d): m = list_re.match(file) if m: dep_pkg = m.group(1) - fd = open(os.path.join(dir, file)) + try: + fd = open(os.path.join(dir, file)) + except IOError: + # During a build unrelated shlib files may be deleted, so + # handle files disappearing between the listdirs and open. + continue lines = fd.readlines() fd.close() for l in lines: @@ -149,7 +154,7 @@ def npm_split_package_dirs(pkgdir): if os.path.exists(pkgfile): with open(pkgfile, 'r') as f: data = json.loads(f.read()) - packages[pkgname] = (relpth, data) + packages[pkgname] = (relpth, data) # We want the main package for a module sorted *after* its subpackages # (so that it doesn't otherwise steal the files for the subpackage), so # this is a cheap way to do that whilst still having an otherwise diff --git a/import-layers/yocto-poky/meta/lib/oe/package_manager.py b/import-layers/yocto-poky/meta/lib/oe/package_manager.py index b4b359a8c..3cee9730a 100644 --- a/import-layers/yocto-poky/meta/lib/oe/package_manager.py +++ b/import-layers/yocto-poky/meta/lib/oe/package_manager.py @@ -5,9 +5,11 @@ import subprocess import shutil import multiprocessing import re +import collections import bb import tempfile import oe.utils +import oe.path import string from oe.gpg_sign import get_signer @@ -17,20 +19,79 @@ def create_index(arg): try: bb.note("Executing '%s' ..." % index_cmd) - result = subprocess.check_output(index_cmd, stderr=subprocess.STDOUT, shell=True) + result = subprocess.check_output(index_cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8") except subprocess.CalledProcessError as e: return("Index creation command '%s' failed with return code %d:\n%s" % - (e.cmd, e.returncode, e.output)) + (e.cmd, e.returncode, e.output.decode("utf-8"))) if result: bb.note(result) return None - -class Indexer(object): - __metaclass__ = ABCMeta - +""" +This method parse the output from the package managerand return +a dictionary with the information of the packages. This is used +when the packages are in deb or ipk format. +""" +def opkg_query(cmd_output): + verregex = re.compile(' \([=<>]* [^ )]*\)') + output = dict() + pkg = "" + arch = "" + ver = "" + filename = "" + dep = [] + pkgarch = "" + for line in cmd_output.splitlines(): + line = line.rstrip() + if ':' in line: + if line.startswith("Package: "): + pkg = line.split(": ")[1] + elif line.startswith("Architecture: "): + arch = line.split(": ")[1] + elif line.startswith("Version: "): + ver = line.split(": ")[1] + elif line.startswith("File: ") or line.startswith("Filename:"): + filename = line.split(": ")[1] + if "/" in filename: + filename = os.path.basename(filename) + elif line.startswith("Depends: "): + depends = verregex.sub('', line.split(": ")[1]) + for depend in depends.split(", "): + dep.append(depend) + elif line.startswith("Recommends: "): + recommends = verregex.sub('', line.split(": ")[1]) + for recommend in recommends.split(", "): + dep.append("%s [REC]" % recommend) + elif line.startswith("PackageArch: "): + pkgarch = line.split(": ")[1] + + # When there is a blank line save the package information + elif not line: + # IPK doesn't include the filename + if not filename: + filename = "%s_%s_%s.ipk" % (pkg, ver, arch) + if pkg: + output[pkg] = {"arch":arch, "ver":ver, + "filename":filename, "deps": dep, "pkgarch":pkgarch } + pkg = "" + arch = "" + ver = "" + filename = "" + dep = [] + pkgarch = "" + + if pkg: + if not filename: + filename = "%s_%s_%s.ipk" % (pkg, ver, arch) + output[pkg] = {"arch":arch, "ver":ver, + "filename":filename, "deps": dep } + + return output + + +class Indexer(object, metaclass=ABCMeta): def __init__(self, d, deploy_dir): self.d = d self.deploy_dir = deploy_dir @@ -42,13 +103,8 @@ class Indexer(object): class RpmIndexer(Indexer): def get_ml_prefix_and_os_list(self, arch_var=None, os_var=None): - package_archs = { - 'default': [], - } - - target_os = { - 'default': "", - } + package_archs = collections.OrderedDict() + target_os = collections.OrderedDict() if arch_var is not None and os_var is not None: package_archs['default'] = self.d.getVar(arch_var, True).split() @@ -79,7 +135,7 @@ class RpmIndexer(Indexer): target_os[eext[1]] = localdata.getVar("TARGET_OS", True).strip() - ml_prefix_list = dict() + ml_prefix_list = collections.OrderedDict() for mlib in package_archs: if mlib == 'default': ml_prefix_list[mlib] = package_archs[mlib] @@ -281,9 +337,7 @@ class DpkgIndexer(Indexer): -class PkgsList(object): - __metaclass__ = ABCMeta - +class PkgsList(object, metaclass=ABCMeta): def __init__(self, d, rootfs_dir): self.d = d self.rootfs_dir = rootfs_dir @@ -293,57 +347,6 @@ class PkgsList(object): pass - """ - This method parse the output from the package manager - and return a dictionary with the information of the - installed packages. This is used whne the packages are - in deb or ipk format - """ - def opkg_query(self, cmd_output): - verregex = re.compile(' \([=<>]* [^ )]*\)') - output = dict() - filename = "" - dep = [] - pkg = "" - for line in cmd_output.splitlines(): - line = line.rstrip() - if ':' in line: - if line.startswith("Package: "): - pkg = line.split(": ")[1] - elif line.startswith("Architecture: "): - arch = line.split(": ")[1] - elif line.startswith("Version: "): - ver = line.split(": ")[1] - elif line.startswith("File: "): - filename = line.split(": ")[1] - elif line.startswith("Depends: "): - depends = verregex.sub('', line.split(": ")[1]) - for depend in depends.split(", "): - dep.append(depend) - elif line.startswith("Recommends: "): - recommends = verregex.sub('', line.split(": ")[1]) - for recommend in recommends.split(", "): - dep.append("%s [REC]" % recommend) - else: - # IPK doesn't include the filename - if not filename: - filename = "%s_%s_%s.ipk" % (pkg, ver, arch) - if pkg: - output[pkg] = {"arch":arch, "ver":ver, - "filename":filename, "deps": dep } - pkg = "" - filename = "" - dep = [] - - if pkg: - if not filename: - filename = "%s_%s_%s.ipk" % (pkg, ver, arch) - output[pkg] = {"arch":arch, "ver":ver, - "filename":filename, "deps": dep } - - return output - - class RpmPkgsList(PkgsList): def __init__(self, d, rootfs_dir, arch_var=None, os_var=None): super(RpmPkgsList, self).__init__(d, rootfs_dir) @@ -357,10 +360,10 @@ class RpmPkgsList(PkgsList): # Determine rpm version cmd = "%s --version" % self.rpm_cmd try: - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8") except subprocess.CalledProcessError as e: bb.fatal("Getting rpm version failed. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) ''' Translate the RPM/Smart format names to the OE multilib format names @@ -401,10 +404,10 @@ class RpmPkgsList(PkgsList): "-t", self.image_rpmlib] try: - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).strip() + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).strip().decode("utf-8") except subprocess.CalledProcessError as e: bb.fatal("Cannot get the package dependencies. Command '%s' " - "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output)) + "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8"))) return output @@ -415,10 +418,10 @@ class RpmPkgsList(PkgsList): try: # bb.note(cmd) - tmp_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).strip() + tmp_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).strip().decode("utf-8") except subprocess.CalledProcessError as e: bb.fatal("Cannot get the installed packages list. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) output = dict() deps = dict() @@ -475,11 +478,13 @@ class OpkgPkgsList(PkgsList): # output streams separately and check for empty stderr. p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) cmd_output, cmd_stderr = p.communicate() + cmd_output = cmd_output.decode("utf-8") + cmd_stderr = cmd_stderr.decode("utf-8") if p.returncode or cmd_stderr: bb.fatal("Cannot get the installed packages list. Command '%s' " "returned %d and stderr:\n%s" % (cmd, p.returncode, cmd_stderr)) - return self.opkg_query(cmd_output) + return opkg_query(cmd_output) class DpkgPkgsList(PkgsList): @@ -492,19 +497,18 @@ class DpkgPkgsList(PkgsList): cmd.append("-f=Package: ${Package}\nArchitecture: ${PackageArch}\nVersion: ${Version}\nFile: ${Package}_${Version}_${Architecture}.deb\nDepends: ${Depends}\nRecommends: ${Recommends}\n\n") try: - cmd_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).strip() + cmd_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).strip().decode("utf-8") except subprocess.CalledProcessError as e: bb.fatal("Cannot get the installed packages list. Command '%s' " - "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output)) + "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8"))) - return self.opkg_query(cmd_output) + return opkg_query(cmd_output) -class PackageManager(object): +class PackageManager(object, metaclass=ABCMeta): """ This is an abstract class. Do not instantiate this directly. """ - __metaclass__ = ABCMeta def __init__(self, d): self.d = d @@ -594,15 +598,15 @@ class PackageManager(object): globs] exclude = self.d.getVar('PACKAGE_EXCLUDE_COMPLEMENTARY', True) if exclude: - cmd.extend(['-x', exclude]) + cmd.extend(['--exclude=' + '|'.join(exclude.split())]) try: bb.note("Installing complementary packages ...") bb.note('Running %s' % cmd) - complementary_pkgs = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + complementary_pkgs = subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode("utf-8") except subprocess.CalledProcessError as e: bb.fatal("Could not compute complementary packages list. Command " "'%s' returned %d:\n%s" % - (' '.join(cmd), e.returncode, e.output)) + (' '.join(cmd), e.returncode, e.output.decode("utf-8"))) self.install(complementary_pkgs.split(), attempt_only=True) os.remove(installed_pkgs_file) @@ -631,8 +635,8 @@ class PackageManager(object): def construct_uris(self, uris, base_paths): def _append(arr1, arr2, sep='/'): res = [] - narr1 = map(lambda a: string.rstrip(a, sep), arr1) - narr2 = map(lambda a: string.lstrip(string.rstrip(a, sep), sep), arr2) + narr1 = [a.rstrip(sep) for a in arr1] + narr2 = [a.rstrip(sep).lstrip(sep) for a in arr2] for a1 in narr1: if arr2: for a2 in narr2: @@ -682,7 +686,8 @@ class RpmPM(PackageManager): if not os.path.exists(self.d.expand('${T}/saved')): bb.utils.mkdirhier(self.d.expand('${T}/saved')) - self.indexer = RpmIndexer(self.d, self.deploy_dir) + packageindex_dir = os.path.join(self.d.getVar('WORKDIR', True), 'rpms') + self.indexer = RpmIndexer(self.d, packageindex_dir) self.pkgs_list = RpmPkgsList(self.d, self.target_rootfs, arch_var, os_var) self.ml_prefix_list, self.ml_os_list = self.indexer.get_ml_prefix_and_os_list(arch_var, os_var) @@ -697,18 +702,19 @@ class RpmPM(PackageManager): arch_list = self.feed_archs.split() else: # List must be prefered to least preferred order - default_platform_extra = set() - platform_extra = set() + default_platform_extra = list() + platform_extra = list() bbextendvariant = self.d.getVar('BBEXTENDVARIANT', True) or "" for mlib in self.ml_os_list: for arch in self.ml_prefix_list[mlib]: plt = arch.replace('-', '_') + '-.*-' + self.ml_os_list[mlib] if mlib == bbextendvariant: - default_platform_extra.add(plt) + if plt not in default_platform_extra: + default_platform_extra.append(plt) else: - platform_extra.add(plt) - - platform_extra = platform_extra.union(default_platform_extra) + if plt not in platform_extra: + platform_extra.append(plt) + platform_extra = default_platform_extra + platform_extra for canonical_arch in platform_extra: arch = canonical_arch.split('-')[0] @@ -724,7 +730,7 @@ class RpmPM(PackageManager): for uri in feed_uris: if arch_list: for arch in arch_list: - bb.note('Note: adding Smart channel url%d%s (%s)' % + bb.note('Adding Smart channel url%d%s (%s)' % (uri_iterator, arch, channel_priority)) self._invoke_smart('channel --add url%d-%s type=rpm-md baseurl=%s/%s -y' % (uri_iterator, arch, uri, arch)) @@ -732,7 +738,7 @@ class RpmPM(PackageManager): (uri_iterator, arch, channel_priority)) channel_priority -= 5 else: - bb.note('Note: adding Smart channel url%d (%s)' % + bb.note('Adding Smart channel url%d (%s)' % (uri_iterator, channel_priority)) self._invoke_smart('channel --add url%d type=rpm-md baseurl=%s -y' % (uri_iterator, uri)) @@ -774,12 +780,12 @@ class RpmPM(PackageManager): try: complementary_pkgs = subprocess.check_output(cmd, stderr=subprocess.STDOUT, - shell=True) + shell=True).decode("utf-8") # bb.note(complementary_pkgs) return complementary_pkgs except subprocess.CalledProcessError as e: bb.fatal("Could not invoke smart. Command " - "'%s' returned %d:\n%s" % (cmd, e.returncode, e.output)) + "'%s' returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) def _search_pkg_name_in_feeds(self, pkg, feed_archs): for arch in feed_archs: @@ -798,7 +804,7 @@ class RpmPM(PackageManager): (self.smart_cmd, self.smart_opt, pkg) cmd += " | sed -ne 's/ *Provides://p'" bb.note('cmd: %s' % cmd) - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8") # Found a provider if output: bb.note('Found providers for %s: %s' % (pkg, output)) @@ -833,11 +839,13 @@ class RpmPM(PackageManager): new_pkg = self._search_pkg_name_in_feeds(subst, feed_archs) if not new_pkg: # Failed to translate, package not found! - err_msg = '%s not found in the %s feeds (%s).\n' % \ - (pkg, mlib, " ".join(feed_archs)) + err_msg = '%s not found in the %s feeds (%s) in %s.' % \ + (pkg, mlib, " ".join(feed_archs), self.d.getVar('DEPLOY_DIR_RPM', True)) if not attempt_only: - err_msg += " ".join(self.fullpkglist) - bb.fatal(err_msg) + bb.error(err_msg) + bb.fatal("This is often caused by an empty package declared " \ + "in a recipe's PACKAGES variable. (Empty packages are " \ + "not constructed unless ALLOW_EMPTY_<pkg> = '1' is used.)") bb.warn(err_msg) else: new_pkgs.append(new_pkg) @@ -850,11 +858,13 @@ class RpmPM(PackageManager): default_archs = self.ml_prefix_list['default'] new_pkg = self._search_pkg_name_in_feeds(pkg, default_archs) if not new_pkg: - err_msg = '%s not found in the base feeds (%s).\n' % \ - (pkg, ' '.join(default_archs)) + err_msg = '%s not found in the feeds (%s) in %s.' % \ + (pkg, " ".join(default_archs), self.d.getVar('DEPLOY_DIR_RPM', True)) if not attempt_only: - err_msg += " ".join(self.fullpkglist) - bb.fatal(err_msg) + bb.error(err_msg) + bb.fatal("This is often caused by an empty package declared " \ + "in a recipe's PACKAGES variable. (Empty packages are " \ + "not constructed unless ALLOW_EMPTY_<pkg> = '1' is used.)") bb.warn(err_msg) else: new_pkgs.append(new_pkg) @@ -946,7 +956,7 @@ class RpmPM(PackageManager): subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError as e: bb.fatal("Create rpm database failed. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) # Import GPG key to RPM database of the target system if self.d.getVar('RPM_SIGN_PACKAGES', True) == '1': pubkey_path = self.d.getVar('RPM_GPG_PUBKEY', True) @@ -1006,9 +1016,19 @@ class RpmPM(PackageManager): ch_already_added = [] for canonical_arch in platform_extra: arch = canonical_arch.split('-')[0] - arch_channel = os.path.join(self.deploy_dir, arch) - if os.path.exists(arch_channel) and not arch in ch_already_added: - bb.note('Note: adding Smart channel %s (%s)' % + arch_channel = os.path.join(self.d.getVar('WORKDIR', True), 'rpms', arch) + oe.path.remove(arch_channel) + deploy_arch_dir = os.path.join(self.deploy_dir, arch) + if not os.path.exists(deploy_arch_dir): + continue + + lockfilename = self.d.getVar('DEPLOY_DIR_RPM', True) + "/rpm.lock" + lf = bb.utils.lockfile(lockfilename, False) + oe.path.copyhardlinktree(deploy_arch_dir, arch_channel) + bb.utils.unlockfile(lf) + + if not arch in ch_already_added: + bb.note('Adding Smart channel %s (%s)' % (arch, channel_priority)) self._invoke_smart('channel --add %s type=rpm-md baseurl=%s -y' % (arch, arch_channel)) @@ -1080,8 +1100,8 @@ class RpmPM(PackageManager): native_root) open(self.scriptlet_wrapper, 'w+').write(scriptlet_content) - bb.note("Note: configuring RPM cross-install scriptlet_wrapper") - os.chmod(self.scriptlet_wrapper, 0755) + bb.note("configuring RPM cross-install scriptlet_wrapper") + os.chmod(self.scriptlet_wrapper, 0o755) cmd = 'config --set rpm-extra-macros._cross_scriptlet_wrapper=%s' % \ self.scriptlet_wrapper self._invoke_smart(cmd) @@ -1099,7 +1119,7 @@ class RpmPM(PackageManager): sub_rdep = sub_data.get("RDEPENDS_" + pkg) if not sub_rdep: continue - done = bb.utils.explode_dep_versions2(sub_rdep).keys() + done = list(bb.utils.explode_dep_versions2(sub_rdep).keys()) next = done # Find all the rdepends on dependency chain while next: @@ -1182,6 +1202,9 @@ class RpmPM(PackageManager): new_depends[new_depend] = deps[depend] pkgs = bb.utils.join_deps(new_depends, commasep=True).split(', ') pkgs = self._pkg_translate_oe_to_smart(pkgs, attempt_only) + if not pkgs: + bb.note("There are no packages to install") + return if not attempt_only: bb.note('to be installed: %s' % ' '.join(pkgs)) cmd = "%s %s install -y %s" % \ @@ -1193,11 +1216,11 @@ class RpmPM(PackageManager): cmd = "%s %s install --attempt -y %s" % \ (self.smart_cmd, self.smart_opt, ' '.join(pkgs)) try: - output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) + output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT).decode("utf-8") bb.note(output) except subprocess.CalledProcessError as e: bb.fatal("Unable to install packages. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) ''' Remove pkgs with smart, the pkg name is smart/rpm format @@ -1223,11 +1246,11 @@ class RpmPM(PackageManager): try: bb.note(cmd) - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8") bb.note(output) except subprocess.CalledProcessError as e: bb.note("Unable to remove packages. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) def upgrade(self): bb.note('smart upgrade') @@ -1300,7 +1323,7 @@ class RpmPM(PackageManager): install_pkgs.append(pkg) except subprocess.CalledProcessError as e: bb.note("Unable to dump install packages. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) # Recovery rpmsys channel self._invoke_smart('channel --enable rpmsys') return install_pkgs @@ -1342,7 +1365,7 @@ class RpmPM(PackageManager): available_pkgs.append(pkg.strip()) except subprocess.CalledProcessError as e: bb.note("Unable to list all available packages. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) self.fullpkglist = available_pkgs @@ -1369,12 +1392,12 @@ class RpmPM(PackageManager): try: bb.note(cmd) - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).strip() + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).strip().decode("utf-8") bb.note(output) - os.chmod(saved_dir, 0755) + os.chmod(saved_dir, 0o755) except subprocess.CalledProcessError as e: bb.fatal("Invoke save_rpmpostinst failed. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) '''Write common configuration for target usage''' def rpm_setup_smart_target_config(self): @@ -1398,8 +1421,157 @@ class RpmPM(PackageManager): for f in rpm_db_locks: bb.utils.remove(f, True) + """ + Returns a dictionary with the package info. + """ + def package_info(self, pkg): + cmd = "%s %s info --urls %s" % (self.smart_cmd, self.smart_opt, pkg) + try: + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8") + except subprocess.CalledProcessError as e: + bb.fatal("Unable to list available packages. Command '%s' " + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) + + # Set default values to avoid UnboundLocalError + arch = "" + ver = "" + filename = "" + + #Parse output + for line in output.splitlines(): + line = line.rstrip() + if line.startswith("Name:"): + pkg = line.split(": ")[1] + elif line.startswith("Version:"): + tmp_str = line.split(": ")[1] + ver, arch = tmp_str.split("@") + break + + # Get filename + index = re.search("^URLs", output, re.MULTILINE) + tmp_str = output[index.end():] + for line in tmp_str.splitlines(): + if "/" in line: + line = line.lstrip() + filename = line.split(" ")[0] + break + + # To have the same data type than other package_info methods + filepath = os.path.join(self.deploy_dir, arch, filename) + pkg_dict = {} + pkg_dict[pkg] = {"arch":arch, "ver":ver, "filename":filename, + "filepath": filepath} + + return pkg_dict + + """ + Returns the path to a tmpdir where resides the contents of a package. + + Deleting the tmpdir is responsability of the caller. + + """ + def extract(self, pkg): + pkg_info = self.package_info(pkg) + if not pkg_info: + bb.fatal("Unable to get information for package '%s' while " + "trying to extract the package." % pkg) + + pkg_path = pkg_info[pkg]["filepath"] + + cpio_cmd = bb.utils.which(os.getenv("PATH"), "cpio") + rpm2cpio_cmd = bb.utils.which(os.getenv("PATH"), "rpm2cpio") + + if not os.path.isfile(pkg_path): + bb.fatal("Unable to extract package for '%s'." + "File %s doesn't exists" % (pkg, pkg_path)) + + tmp_dir = tempfile.mkdtemp() + current_dir = os.getcwd() + os.chdir(tmp_dir) + + try: + cmd = "%s %s | %s -idmv" % (rpm2cpio_cmd, pkg_path, cpio_cmd) + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) + except subprocess.CalledProcessError as e: + bb.utils.remove(tmp_dir, recurse=True) + bb.fatal("Unable to extract %s package. Command '%s' " + "returned %d:\n%s" % (pkg_path, cmd, e.returncode, e.output.decode("utf-8"))) + except OSError as e: + bb.utils.remove(tmp_dir, recurse=True) + bb.fatal("Unable to extract %s package. Command '%s' " + "returned %d:\n%s at %s" % (pkg_path, cmd, e.errno, e.strerror, e.filename)) + + bb.note("Extracted %s to %s" % (pkg_path, tmp_dir)) + os.chdir(current_dir) + + return tmp_dir + + +class OpkgDpkgPM(PackageManager): + """ + This is an abstract class. Do not instantiate this directly. + """ + def __init__(self, d): + super(OpkgDpkgPM, self).__init__(d) + + """ + Returns a dictionary with the package info. + + This method extracts the common parts for Opkg and Dpkg + """ + def package_info(self, pkg, cmd): + + try: + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8") + except subprocess.CalledProcessError as e: + bb.fatal("Unable to list available packages. Command '%s' " + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) + return opkg_query(output) + + """ + Returns the path to a tmpdir where resides the contents of a package. + + Deleting the tmpdir is responsability of the caller. + + This method extracts the common parts for Opkg and Dpkg + """ + def extract(self, pkg, pkg_info): + + ar_cmd = bb.utils.which(os.getenv("PATH"), "ar") + tar_cmd = bb.utils.which(os.getenv("PATH"), "tar") + pkg_path = pkg_info[pkg]["filepath"] + + if not os.path.isfile(pkg_path): + bb.fatal("Unable to extract package for '%s'." + "File %s doesn't exists" % (pkg, pkg_path)) + + tmp_dir = tempfile.mkdtemp() + current_dir = os.getcwd() + os.chdir(tmp_dir) + + try: + cmd = "%s x %s" % (ar_cmd, pkg_path) + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) + cmd = "%s xf data.tar.*" % tar_cmd + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) + except subprocess.CalledProcessError as e: + bb.utils.remove(tmp_dir, recurse=True) + bb.fatal("Unable to extract %s package. Command '%s' " + "returned %d:\n%s" % (pkg_path, cmd, e.returncode, e.output.decode("utf-8"))) + except OSError as e: + bb.utils.remove(tmp_dir, recurse=True) + bb.fatal("Unable to extract %s package. Command '%s' " + "returned %d:\n%s at %s" % (pkg_path, cmd, e.errno, e.strerror, e.filename)) + + bb.note("Extracted %s to %s" % (pkg_path, tmp_dir)) + bb.utils.remove(os.path.join(tmp_dir, "debian-binary")) + bb.utils.remove(os.path.join(tmp_dir, "control.tar.gz")) + os.chdir(current_dir) -class OpkgPM(PackageManager): + return tmp_dir + + +class OpkgPM(OpkgDpkgPM): def __init__(self, d, target_rootfs, config_file, archs, task_name='target'): super(OpkgPM, self).__init__(d) @@ -1411,7 +1583,7 @@ class OpkgPM(PackageManager): self.deploy_dir = self.d.getVar("DEPLOY_DIR_IPK", True) self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock") self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg") - self.opkg_args = "--volatile-cache -f %s -o %s " % (self.config_file, target_rootfs) + self.opkg_args = "--volatile-cache -f %s -t %s -o %s " % (self.config_file, self.d.expand('${T}/ipktemp/') ,target_rootfs) self.opkg_args += self.d.getVar("OPKG_ARGS", True) opkg_lib_dir = self.d.getVar('OPKGLIBDIR', True) @@ -1552,12 +1724,12 @@ class OpkgPM(PackageManager): for arch in archs: if (self.feed_archs is None) and (not os.path.exists(os.path.join(self.deploy_dir, arch))): continue - bb.note('Note: adding opkg feed url-%s-%d (%s)' % + bb.note('Adding opkg feed url-%s-%d (%s)' % (arch, uri_iterator, uri)) config_file.write("src/gz uri-%s-%d %s/%s\n" % (arch, uri_iterator, uri, arch)) else: - bb.note('Note: adding opkg feed url-%d (%s)' % + bb.note('Adding opkg feed url-%d (%s)' % (uri_iterator, uri)) config_file.write("src/gz uri-%d %s\n" % (uri_iterator, uri)) @@ -1574,7 +1746,7 @@ class OpkgPM(PackageManager): except subprocess.CalledProcessError as e: self.deploy_dir_unlock() bb.fatal("Unable to update the package index files. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) self.deploy_dir_unlock() @@ -1595,12 +1767,12 @@ class OpkgPM(PackageManager): try: bb.note("Installing the following packages: %s" % ' '.join(pkgs)) bb.note(cmd) - output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) + output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT).decode("utf-8") bb.note(output) except subprocess.CalledProcessError as e: (bb.fatal, bb.note)[attempt_only]("Unable to install packages. " "Command '%s' returned %d:\n%s" % - (cmd, e.returncode, e.output)) + (cmd, e.returncode, e.output.decode("utf-8"))) def remove(self, pkgs, with_dependencies=True): if with_dependencies: @@ -1612,11 +1784,11 @@ class OpkgPM(PackageManager): try: bb.note(cmd) - output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) + output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT).decode("utf-8") bb.note(output) except subprocess.CalledProcessError as e: bb.fatal("Unable to remove packages. Command '%s' " - "returned %d:\n%s" % (e.cmd, e.returncode, e.output)) + "returned %d:\n%s" % (e.cmd, e.returncode, e.output.decode("utf-8"))) def write_index(self): self.deploy_dir_lock() @@ -1659,10 +1831,10 @@ class OpkgPM(PackageManager): pkg_info = cmd + pkg try: - output = subprocess.check_output(pkg_info.split(), stderr=subprocess.STDOUT).strip() + output = subprocess.check_output(pkg_info.split(), stderr=subprocess.STDOUT).strip().decode("utf-8") except subprocess.CalledProcessError as e: bb.fatal("Cannot get package info. Command '%s' " - "returned %d:\n%s" % (pkg_info, e.returncode, e.output)) + "returned %d:\n%s" % (pkg_info, e.returncode, e.output.decode("utf-8"))) if output == "": bb.note("Ignored bad recommendation: '%s' is " @@ -1699,7 +1871,7 @@ class OpkgPM(PackageManager): subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError as e: bb.fatal("Unable to update. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) # Dummy installation cmd = "%s %s --noaction install %s " % (self.opkg_cmd, @@ -1709,7 +1881,7 @@ class OpkgPM(PackageManager): output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError as e: bb.fatal("Unable to dummy install packages. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) bb.utils.remove(temp_rootfs, True) @@ -1734,8 +1906,37 @@ class OpkgPM(PackageManager): self.opkg_dir, symlinks=True) + """ + Returns a dictionary with the package info. + """ + def package_info(self, pkg): + cmd = "%s %s info %s" % (self.opkg_cmd, self.opkg_args, pkg) + pkg_info = super(OpkgPM, self).package_info(pkg, cmd) + + pkg_arch = pkg_info[pkg]["arch"] + pkg_filename = pkg_info[pkg]["filename"] + pkg_info[pkg]["filepath"] = \ + os.path.join(self.deploy_dir, pkg_arch, pkg_filename) + + return pkg_info + + """ + Returns the path to a tmpdir where resides the contents of a package. + + Deleting the tmpdir is responsability of the caller. + """ + def extract(self, pkg): + pkg_info = self.package_info(pkg) + if not pkg_info: + bb.fatal("Unable to get information for package '%s' while " + "trying to extract the package." % pkg) + + tmp_dir = super(OpkgPM, self).extract(pkg, pkg_info) + bb.utils.remove(os.path.join(tmp_dir, "data.tar.gz")) -class DpkgPM(PackageManager): + return tmp_dir + +class DpkgPM(OpkgDpkgPM): def __init__(self, d, target_rootfs, archs, base_archs, apt_conf_dir=None): super(DpkgPM, self).__init__(d) self.target_rootfs = target_rootfs @@ -1746,6 +1947,7 @@ class DpkgPM(PackageManager): self.apt_conf_dir = apt_conf_dir self.apt_conf_file = os.path.join(self.apt_conf_dir, "apt.conf") self.apt_get_cmd = bb.utils.which(os.getenv('PATH'), "apt-get") + self.apt_cache_cmd = bb.utils.which(os.getenv('PATH'), "apt-cache") self.apt_args = d.getVar("APT_ARGS", True) @@ -1823,7 +2025,7 @@ class DpkgPM(PackageManager): subprocess.check_output(p_full, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: bb.note("%s for package %s failed with %d:\n%s" % - (suffix[1], pkg_name, e.returncode, e.output)) + (suffix[1], pkg_name, e.returncode, e.output.decode("utf-8"))) failed_pkgs.append(pkg_name) break @@ -1841,7 +2043,7 @@ class DpkgPM(PackageManager): subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: bb.fatal("Unable to update the package index files. Command '%s' " - "returned %d:\n%s" % (e.cmd, e.returncode, e.output)) + "returned %d:\n%s" % (e.cmd, e.returncode, e.output.decode("utf-8"))) self.deploy_dir_unlock() @@ -1860,7 +2062,7 @@ class DpkgPM(PackageManager): except subprocess.CalledProcessError as e: (bb.fatal, bb.note)[attempt_only]("Unable to install packages. " "Command '%s' returned %d:\n%s" % - (cmd, e.returncode, e.output)) + (cmd, e.returncode, e.output.decode("utf-8"))) # rename *.dpkg-new files/dirs for root, dirs, files in os.walk(self.target_rootfs): @@ -1891,7 +2093,7 @@ class DpkgPM(PackageManager): subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: bb.fatal("Unable to remove packages. Command '%s' " - "returned %d:\n%s" % (e.cmd, e.returncode, e.output)) + "returned %d:\n%s" % (e.cmd, e.returncode, e.output.decode("utf-8"))) def write_index(self): self.deploy_dir_lock() @@ -1925,11 +2127,11 @@ class DpkgPM(PackageManager): for uri in feed_uris: if arch_list: for arch in arch_list: - bb.note('Note: adding dpkg channel at (%s)' % uri) + bb.note('Adding dpkg channel at (%s)' % uri) sources_file.write("deb %s/%s ./\n" % (uri, arch)) else: - bb.note('Note: adding dpkg channel at (%s)' % uri) + bb.note('Adding dpkg channel at (%s)' % uri) sources_file.write("deb %s ./\n" % uri) def _create_configs(self, archs, base_archs): @@ -2024,11 +2226,40 @@ class DpkgPM(PackageManager): subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: bb.fatal("Cannot fix broken dependencies. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) def list_installed(self): return DpkgPkgsList(self.d, self.target_rootfs).list_pkgs() + """ + Returns a dictionary with the package info. + """ + def package_info(self, pkg): + cmd = "%s show %s" % (self.apt_cache_cmd, pkg) + pkg_info = super(DpkgPM, self).package_info(pkg, cmd) + + pkg_arch = pkg_info[pkg]["pkgarch"] + pkg_filename = pkg_info[pkg]["filename"] + pkg_info[pkg]["filepath"] = \ + os.path.join(self.deploy_dir, pkg_arch, pkg_filename) + + return pkg_info + + """ + Returns the path to a tmpdir where resides the contents of a package. + + Deleting the tmpdir is responsability of the caller. + """ + def extract(self, pkg): + pkg_info = self.package_info(pkg) + if not pkg_info: + bb.fatal("Unable to get information for package '%s' while " + "trying to extract the package." % pkg) + + tmp_dir = super(DpkgPM, self).extract(pkg, pkg_info) + bb.utils.remove(os.path.join(tmp_dir, "data.tar.xz")) + + return tmp_dir def generate_index_files(d): classes = d.getVar('PACKAGE_CLASSES', True).replace("package_", "").split() diff --git a/import-layers/yocto-poky/meta/lib/oe/packagedata.py b/import-layers/yocto-poky/meta/lib/oe/packagedata.py index bc0fd06bc..21d4de914 100644 --- a/import-layers/yocto-poky/meta/lib/oe/packagedata.py +++ b/import-layers/yocto-poky/meta/lib/oe/packagedata.py @@ -8,7 +8,7 @@ def read_pkgdatafile(fn): pkgdata = {} def decode(str): - c = codecs.getdecoder("string_escape") + c = codecs.getdecoder("unicode_escape") return c(str)[0] if os.access(fn, os.R_OK): @@ -66,7 +66,7 @@ def _pkgmap(d): bb.warn("No files in %s?" % pkgdatadir) files = [] - for pn in filter(lambda f: not os.path.isdir(os.path.join(pkgdatadir, f)), files): + for pn in [f for f in files if not os.path.isdir(os.path.join(pkgdatadir, f))]: try: pkgdata = read_pkgdatafile(os.path.join(pkgdatadir, pn)) except OSError: diff --git a/import-layers/yocto-poky/meta/lib/oe/packagegroup.py b/import-layers/yocto-poky/meta/lib/oe/packagegroup.py index a6fee5f95..97819279b 100644 --- a/import-layers/yocto-poky/meta/lib/oe/packagegroup.py +++ b/import-layers/yocto-poky/meta/lib/oe/packagegroup.py @@ -16,11 +16,11 @@ def packages(features, d): yield pkg def required_packages(features, d): - req = filter(lambda feature: not is_optional(feature, d), features) + req = [feature for feature in features if not is_optional(feature, d)] return packages(req, d) def optional_packages(features, d): - opt = filter(lambda feature: is_optional(feature, d), features) + opt = [feature for feature in features if is_optional(feature, d)] return packages(opt, d) def active_packages(features, d): diff --git a/import-layers/yocto-poky/meta/lib/oe/patch.py b/import-layers/yocto-poky/meta/lib/oe/patch.py index 9d3617290..0332f100f 100644 --- a/import-layers/yocto-poky/meta/lib/oe/patch.py +++ b/import-layers/yocto-poky/meta/lib/oe/patch.py @@ -117,43 +117,50 @@ class PatchSet(object): return None return os.sep.join(filesplit[striplevel:]) - copiedmode = False - filelist = [] - with open(patchfile) as f: - for line in f: - if line.startswith('--- '): - patchpth = patchedpath(line) - if not patchpth: - break - if copiedmode: - addedfile = patchpth - else: - removedfile = patchpth - elif line.startswith('+++ '): - addedfile = patchedpath(line) - if not addedfile: - break - elif line.startswith('*** '): - copiedmode = True - removedfile = patchedpath(line) - if not removedfile: - break - else: - removedfile = None - addedfile = None - - if addedfile and removedfile: - if removedfile == '/dev/null': - mode = 'A' - elif addedfile == '/dev/null': - mode = 'D' - else: - mode = 'M' - if srcdir: - fullpath = os.path.abspath(os.path.join(srcdir, addedfile)) - else: - fullpath = addedfile - filelist.append((fullpath, mode)) + for encoding in ['utf-8', 'latin-1']: + try: + copiedmode = False + filelist = [] + with open(patchfile) as f: + for line in f: + if line.startswith('--- '): + patchpth = patchedpath(line) + if not patchpth: + break + if copiedmode: + addedfile = patchpth + else: + removedfile = patchpth + elif line.startswith('+++ '): + addedfile = patchedpath(line) + if not addedfile: + break + elif line.startswith('*** '): + copiedmode = True + removedfile = patchedpath(line) + if not removedfile: + break + else: + removedfile = None + addedfile = None + + if addedfile and removedfile: + if removedfile == '/dev/null': + mode = 'A' + elif addedfile == '/dev/null': + mode = 'D' + else: + mode = 'M' + if srcdir: + fullpath = os.path.abspath(os.path.join(srcdir, addedfile)) + else: + fullpath = addedfile + filelist.append((fullpath, mode)) + except UnicodeDecodeError: + continue + break + else: + raise PatchError('Unable to decode %s' % patchfile) return filelist @@ -274,30 +281,43 @@ class GitApplyTree(PatchTree): def __init__(self, dir, d): PatchTree.__init__(self, dir, d) + self.commituser = d.getVar('PATCH_GIT_USER_NAME', True) + self.commitemail = d.getVar('PATCH_GIT_USER_EMAIL', True) @staticmethod def extractPatchHeader(patchfile): """ Extract just the header lines from the top of a patch file """ - lines = [] - with open(patchfile, 'r') as f: - for line in f.readlines(): - if line.startswith('Index: ') or line.startswith('diff -') or line.startswith('---'): - break - lines.append(line) + for encoding in ['utf-8', 'latin-1']: + lines = [] + try: + with open(patchfile, 'r', encoding=encoding) as f: + for line in f: + if line.startswith('Index: ') or line.startswith('diff -') or line.startswith('---'): + break + lines.append(line) + except UnicodeDecodeError: + continue + break + else: + raise PatchError('Unable to find a character encoding to decode %s' % patchfile) return lines @staticmethod def decodeAuthor(line): from email.header import decode_header authorval = line.split(':', 1)[1].strip().replace('"', '') - return decode_header(authorval)[0][0] + result = decode_header(authorval)[0][0] + if hasattr(result, 'decode'): + result = result.decode('utf-8') + return result @staticmethod def interpretPatchHeader(headerlines): import re author_re = re.compile('[\S ]+ <\S+@\S+\.\S+>') + from_commit_re = re.compile('^From [a-z0-9]{40} .*') outlines = [] author = None date = None @@ -327,11 +347,39 @@ class GitApplyTree(PatchTree): # git is fussy about author formatting i.e. it must be Name <email@domain> if author_re.match(authorval): author = authorval + elif from_commit_re.match(line): + # We don't want the From <commit> line - if it's present it will break rebasing + continue outlines.append(line) + + if not subject: + firstline = None + for line in headerlines: + line = line.strip() + if firstline: + if line: + # Second line is not blank, the first line probably isn't usable + firstline = None + break + elif line: + firstline = line + if firstline and not firstline.startswith(('#', 'Index:', 'Upstream-Status:')) and len(firstline) < 100: + subject = firstline + return outlines, author, date, subject @staticmethod - def prepareCommit(patchfile): + def gitCommandUserOptions(cmd, commituser=None, commitemail=None, d=None): + if d: + commituser = d.getVar('PATCH_GIT_USER_NAME', True) + commitemail = d.getVar('PATCH_GIT_USER_EMAIL', True) + if commituser: + cmd += ['-c', 'user.name="%s"' % commituser] + if commitemail: + cmd += ['-c', 'user.email="%s"' % commitemail] + + @staticmethod + def prepareCommit(patchfile, commituser=None, commitemail=None): """ Prepare a git commit command line based on the header from a patch file (typically this is useful for patches that cannot be applied with "git am" due to formatting) @@ -340,21 +388,24 @@ class GitApplyTree(PatchTree): # Process patch header and extract useful information lines = GitApplyTree.extractPatchHeader(patchfile) outlines, author, date, subject = GitApplyTree.interpretPatchHeader(lines) - if not author or not subject: + if not author or not subject or not date: try: - shellcmd = ["git", "log", "--format=email", "--diff-filter=A", "--", patchfile] + shellcmd = ["git", "log", "--format=email", "--follow", "--diff-filter=A", "--", patchfile] out = runcmd(["sh", "-c", " ".join(shellcmd)], os.path.dirname(patchfile)) except CmdError: out = None if out: _, newauthor, newdate, newsubject = GitApplyTree.interpretPatchHeader(out.splitlines()) - if not author or not date: - # These really need to go together + if not author: + # If we're setting the author then the date should be set as well author = newauthor date = newdate + elif not date: + # If we don't do this we'll get the current date, at least this will be closer + date = newdate if not subject: subject = newsubject - if subject: + if subject and outlines and not outlines[0].strip() == subject: outlines.insert(0, '%s\n\n' % subject.strip()) # Write out commit message to a file @@ -363,7 +414,9 @@ class GitApplyTree(PatchTree): for line in outlines: tf.write(line) # Prepare git command - cmd = ["git", "commit", "-F", tmpfile] + cmd = ["git"] + GitApplyTree.gitCommandUserOptions(cmd, commituser, commitemail) + cmd += ["commit", "-F", tmpfile] # git doesn't like plain email addresses as authors if author and '<' in author: cmd.append('--author="%s"' % author) @@ -384,16 +437,24 @@ class GitApplyTree(PatchTree): out = runcmd(["sh", "-c", " ".join(shellcmd)], tree) if out: for srcfile in out.split(): - patchlines = [] - outfile = None - with open(srcfile, 'r') as f: - for line in f: - if line.startswith(GitApplyTree.patch_line_prefix): - outfile = line.split()[-1].strip() - continue - if line.startswith(GitApplyTree.ignore_commit_prefix): - continue - patchlines.append(line) + for encoding in ['utf-8', 'latin-1']: + patchlines = [] + outfile = None + try: + with open(srcfile, 'r', encoding=encoding) as f: + for line in f: + if line.startswith(GitApplyTree.patch_line_prefix): + outfile = line.split()[-1].strip() + continue + if line.startswith(GitApplyTree.ignore_commit_prefix): + continue + patchlines.append(line) + except UnicodeDecodeError: + continue + break + else: + raise PatchError('Unable to find a character encoding to decode %s' % srcfile) + if not outfile: outfile = os.path.basename(srcfile) with open(os.path.join(outdir, outfile), 'w') as of: @@ -434,12 +495,14 @@ class GitApplyTree(PatchTree): # change other places which read it back f.write('echo >> $1\n') f.write('echo "%s: $PATCHFILE" >> $1\n' % GitApplyTree.patch_line_prefix) - os.chmod(commithook, 0755) + os.chmod(commithook, 0o755) shutil.copy2(commithook, applyhook) try: patchfilevar = 'PATCHFILE="%s"' % os.path.basename(patch['file']) try: - shellcmd = [patchfilevar, "git", "--work-tree=%s" % reporoot, "am", "-3", "--keep-cr", "-p%s" % patch['strippath']] + shellcmd = [patchfilevar, "git", "--work-tree=%s" % reporoot] + self.gitCommandUserOptions(shellcmd, self.commituser, self.commitemail) + shellcmd += ["am", "-3", "--keep-cr", "-p%s" % patch['strippath']] return _applypatchhelper(shellcmd, patch, force, reverse, run) except CmdError: # Need to abort the git am, or we'll still be within it at the end @@ -469,7 +532,7 @@ class GitApplyTree(PatchTree): shellcmd = ["git", "reset", "HEAD", self.patchdir] output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) # Commit the result - (tmpfile, shellcmd) = self.prepareCommit(patch['file']) + (tmpfile, shellcmd) = self.prepareCommit(patch['file'], self.commituser, self.commitemail) try: shellcmd.insert(0, patchfilevar) output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) @@ -672,7 +735,7 @@ class UserResolver(Resolver): f.write("echo 'Run \"quilt refresh\" when patch is corrected, press CTRL+D to exit.'\n") f.write("echo ''\n") f.write(" ".join(patchcmd) + "\n") - os.chmod(rcfile, 0775) + os.chmod(rcfile, 0o775) self.terminal("bash --rcfile " + rcfile, 'Patch Rejects: Please fix patch rejects manually', self.patchset.d) diff --git a/import-layers/yocto-poky/meta/lib/oe/path.py b/import-layers/yocto-poky/meta/lib/oe/path.py index 413ebfb39..06a5af265 100644 --- a/import-layers/yocto-poky/meta/lib/oe/path.py +++ b/import-layers/yocto-poky/meta/lib/oe/path.py @@ -65,22 +65,30 @@ def copytree(src, dst): # This way we also preserve hardlinks between files in the tree. bb.utils.mkdirhier(dst) - cmd = 'tar -cf - -C %s -p . | tar -xf - -C %s' % (src, dst) - check_output(cmd, shell=True, stderr=subprocess.STDOUT) + cmd = "tar --xattrs --xattrs-include='*' -cf - -C %s -p . | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dst) + subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) def copyhardlinktree(src, dst): """ Make the hard link when possible, otherwise copy. """ bb.utils.mkdirhier(dst) if os.path.isdir(src) and not len(os.listdir(src)): - return + return if (os.stat(src).st_dev == os.stat(dst).st_dev): # Need to copy directories only with tar first since cp will error if two # writers try and create a directory at the same time - cmd = 'cd %s; find . -type d -print | tar -cf - -C %s -p --files-from - --no-recursion | tar -xf - -C %s' % (src, src, dst) - check_output(cmd, shell=True, stderr=subprocess.STDOUT) - cmd = 'cd %s; find . -print0 | cpio --null -pdlu %s' % (src, dst) - check_output(cmd, shell=True, stderr=subprocess.STDOUT) + cmd = "cd %s; find . -type d -print | tar --xattrs --xattrs-include='*' -cf - -C %s -p --no-recursion --files-from - | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, src, dst) + subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) + source = '' + if os.path.isdir(src): + import glob + if len(glob.glob('%s/.??*' % src)) > 0: + source = '%s/.??* ' % src + source = source + '%s/*' % src + else: + source = src + cmd = 'cp -afl --preserve=xattr %s %s' % (source, dst) + subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) else: copytree(src, dst) @@ -105,47 +113,6 @@ def symlink(source, destination, force=False): if e.errno != errno.EEXIST or os.readlink(destination) != source: raise -class CalledProcessError(Exception): - def __init__(self, retcode, cmd, output = None): - self.retcode = retcode - self.cmd = cmd - self.output = output - def __str__(self): - return "Command '%s' returned non-zero exit status %d with output %s" % (self.cmd, self.retcode, self.output) - -# Not needed when we move to python 2.7 -def check_output(*popenargs, **kwargs): - r"""Run command with arguments and return its output as a byte string. - - If the exit code was non-zero it raises a CalledProcessError. The - CalledProcessError object will have the return code in the returncode - attribute and output in the output attribute. - - The arguments are the same as for the Popen constructor. Example: - - >>> check_output(["ls", "-l", "/dev/null"]) - 'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' - - The stdout argument is not allowed as it is used internally. - To capture standard error in the result, use stderr=STDOUT. - - >>> check_output(["/bin/sh", "-c", - ... "ls -l non_existent_file ; exit 0"], - ... stderr=STDOUT) - 'ls: non_existent_file: No such file or directory\n' - """ - if 'stdout' in kwargs: - raise ValueError('stdout argument not allowed, it will be overridden.') - process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) - output, unused_err = process.communicate() - retcode = process.poll() - if retcode: - cmd = kwargs.get("args") - if cmd is None: - cmd = popenargs[0] - raise CalledProcessError(retcode, cmd, output=output) - return output - def find(dir, **walkoptions): """ Given a directory, recurses into that directory, returning all files as absolute paths. """ diff --git a/import-layers/yocto-poky/meta/lib/oe/prservice.py b/import-layers/yocto-poky/meta/lib/oe/prservice.py index b0cbcb1fb..0054f954c 100644 --- a/import-layers/yocto-poky/meta/lib/oe/prservice.py +++ b/import-layers/yocto-poky/meta/lib/oe/prservice.py @@ -1,7 +1,7 @@ def prserv_make_conn(d, check = False): import prserv.serv - host_params = filter(None, (d.getVar("PRSERV_HOST", True) or '').split(':')) + host_params = list([_f for _f in (d.getVar("PRSERV_HOST", True) or '').split(':') if _f]) try: conn = None conn = prserv.serv.PRServerConnection(host_params[0], int(host_params[1])) @@ -9,7 +9,7 @@ def prserv_make_conn(d, check = False): if not conn.ping(): raise Exception('service not available') d.setVar("__PRSERV_CONN",conn) - except Exception, exc: + except Exception as exc: bb.fatal("Connecting to PR service %s:%s failed: %s" % (host_params[0], host_params[1], str(exc))) return conn @@ -114,7 +114,7 @@ def prserv_export_tofile(d, metainfo, datainfo, lockdown, nomax=False): bb.utils.unlockfile(lf) def prserv_check_avail(d): - host_params = filter(None, (d.getVar("PRSERV_HOST", True) or '').split(':')) + host_params = list([_f for _f in (d.getVar("PRSERV_HOST", True) or '').split(':') if _f]) try: if len(host_params) != 2: raise TypeError diff --git a/import-layers/yocto-poky/meta/lib/oe/qa.py b/import-layers/yocto-poky/meta/lib/oe/qa.py index 3cfeee737..fbe719d8e 100644 --- a/import-layers/yocto-poky/meta/lib/oe/qa.py +++ b/import-layers/yocto-poky/meta/lib/oe/qa.py @@ -43,48 +43,48 @@ class ELFFile: if not os.path.isfile(self.name): raise NotELFFileError("%s is not a normal file" % self.name) - self.file = file(self.name, "r") - # Read 4k which should cover most of the headers we're after - self.data = self.file.read(4096) + with open(self.name, "rb") as f: + # Read 4k which should cover most of the headers we're after + self.data = f.read(4096) if len(self.data) < ELFFile.EI_NIDENT + 4: raise NotELFFileError("%s is not an ELF" % self.name) - self.my_assert(self.data[0], chr(0x7f) ) - self.my_assert(self.data[1], 'E') - self.my_assert(self.data[2], 'L') - self.my_assert(self.data[3], 'F') + self.my_assert(self.data[0], 0x7f) + self.my_assert(self.data[1], ord('E')) + self.my_assert(self.data[2], ord('L')) + self.my_assert(self.data[3], ord('F')) if self.bits == 0: - if self.data[ELFFile.EI_CLASS] == chr(ELFFile.ELFCLASS32): + if self.data[ELFFile.EI_CLASS] == ELFFile.ELFCLASS32: self.bits = 32 - elif self.data[ELFFile.EI_CLASS] == chr(ELFFile.ELFCLASS64): + elif self.data[ELFFile.EI_CLASS] == ELFFile.ELFCLASS64: self.bits = 64 else: # Not 32-bit or 64.. lets assert raise NotELFFileError("ELF but not 32 or 64 bit.") elif self.bits == 32: - self.my_assert(self.data[ELFFile.EI_CLASS], chr(ELFFile.ELFCLASS32)) + self.my_assert(self.data[ELFFile.EI_CLASS], ELFFile.ELFCLASS32) elif self.bits == 64: - self.my_assert(self.data[ELFFile.EI_CLASS], chr(ELFFile.ELFCLASS64)) + self.my_assert(self.data[ELFFile.EI_CLASS], ELFFile.ELFCLASS64) else: raise NotELFFileError("Must specify unknown, 32 or 64 bit size.") - self.my_assert(self.data[ELFFile.EI_VERSION], chr(ELFFile.EV_CURRENT) ) + self.my_assert(self.data[ELFFile.EI_VERSION], ELFFile.EV_CURRENT) self.sex = self.data[ELFFile.EI_DATA] - if self.sex == chr(ELFFile.ELFDATANONE): + if self.sex == ELFFile.ELFDATANONE: raise NotELFFileError("self.sex == ELFDATANONE") - elif self.sex == chr(ELFFile.ELFDATA2LSB): + elif self.sex == ELFFile.ELFDATA2LSB: self.sex = "<" - elif self.sex == chr(ELFFile.ELFDATA2MSB): + elif self.sex == ELFFile.ELFDATA2MSB: self.sex = ">" else: raise NotELFFileError("Unknown self.sex") def osAbi(self): - return ord(self.data[ELFFile.EI_OSABI]) + return self.data[ELFFile.EI_OSABI] def abiVersion(self): - return ord(self.data[ELFFile.EI_ABIVERSION]) + return self.data[ELFFile.EI_ABIVERSION] def abiSize(self): return self.bits @@ -144,8 +144,28 @@ class ELFFile: bb.note("%s %s %s failed: %s" % (objdump, cmd, self.name, e)) return "" +def elf_machine_to_string(machine): + """ + Return the name of a given ELF e_machine field or the hex value as a string + if it isn't recognised. + """ + try: + return { + 0x02: "SPARC", + 0x03: "x86", + 0x08: "MIPS", + 0x14: "PowerPC", + 0x28: "ARM", + 0x2A: "SuperH", + 0x32: "IA-64", + 0x3E: "x86-64", + 0xB7: "AArch64" + }[machine] + except: + return "Unknown (%s)" % repr(machine) + if __name__ == "__main__": import sys elf = ELFFile(sys.argv[1]) elf.open() - print elf.isDynamic() + print(elf.isDynamic()) diff --git a/import-layers/yocto-poky/meta/lib/oe/recipeutils.py b/import-layers/yocto-poky/meta/lib/oe/recipeutils.py index 6c7adb5bd..58e4028ae 100644 --- a/import-layers/yocto-poky/meta/lib/oe/recipeutils.py +++ b/import-layers/yocto-poky/meta/lib/oe/recipeutils.py @@ -2,7 +2,7 @@ # # Some code borrowed from the OE layer index # -# Copyright (C) 2013-2015 Intel Corporation +# Copyright (C) 2013-2016 Intel Corporation # import sys @@ -11,31 +11,32 @@ import os.path import tempfile import textwrap import difflib -import utils +from . import utils import shutil import re import fnmatch +import glob from collections import OrderedDict, defaultdict # Help us to find places to insert values -recipe_progression = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRCREV', 'SRC_URI', 'S', 'do_fetch()', 'do_unpack()', 'do_patch()', 'EXTRA_OECONF', 'do_configure()', 'EXTRA_OEMAKE', 'do_compile()', 'do_install()', 'do_populate_sysroot()', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'do_package()', 'do_deploy()'] +recipe_progression = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LICENSE_FLAGS', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRCREV', 'SRCPV', 'SRC_URI', 'S', 'do_fetch()', 'do_unpack()', 'do_patch()', 'EXTRA_OECONF', 'EXTRA_OECMAKE', 'EXTRA_OESCONS', 'do_configure()', 'EXTRA_OEMAKE', 'do_compile()', 'do_install()', 'do_populate_sysroot()', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'populate_packages()', 'do_package()', 'do_deploy()'] # Variables that sometimes are a bit long but shouldn't be wrapped nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER', 'SRC_URI[md5sum]', 'SRC_URI[sha256sum]'] list_vars = ['SRC_URI', 'LIC_FILES_CHKSUM'] meta_vars = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION'] -def pn_to_recipe(cooker, pn): +def pn_to_recipe(cooker, pn, mc=''): """Convert a recipe name (PN) to the path to the recipe file""" import bb.providers - if pn in cooker.recipecache.pkg_pn: - best = bb.providers.findBestProvider(pn, cooker.data, cooker.recipecache, cooker.recipecache.pkg_pn) + if pn in cooker.recipecaches[mc].pkg_pn: + best = bb.providers.findBestProvider(pn, cooker.data, cooker.recipecaches[mc], cooker.recipecaches[mc].pkg_pn) return best[3] - elif pn in cooker.recipecache.providers: - filenames = cooker.recipecache.providers[pn] - eligible, foundUnique = bb.providers.filterProviders(filenames, pn, cooker.expanded_data, cooker.recipecache) + elif pn in cooker.recipecaches[mc].providers: + filenames = cooker.recipecaches[mc].providers[pn] + eligible, foundUnique = bb.providers.filterProviders(filenames, pn, cooker.expanded_data, cooker.recipecaches[mc]) filename = eligible[0] return filename else: @@ -49,13 +50,14 @@ def get_unavailable_reasons(cooker, pn): return taskdata.get_reasons(pn) -def parse_recipe(fn, appendfiles, d): +def parse_recipe(cooker, fn, appendfiles): """ Parse an individual recipe file, optionally with a list of bbappend files. """ import bb.cache - envdata = bb.cache.Cache.loadDataFull(fn, appendfiles, d) + parser = bb.cache.NoCache(cooker.databuilder) + envdata = parser.loadDataFull(fn, appendfiles) return envdata @@ -78,7 +80,7 @@ def parse_recipe_simple(cooker, pn, d, appends=True): appendfiles = cooker.collection.get_file_appends(recipefile) else: appendfiles = None - return parse_recipe(recipefile, appendfiles, d) + return parse_recipe(cooker, recipefile, appendfiles) def get_var_files(fn, varlist, d): @@ -158,15 +160,19 @@ def split_var_value(value, assignment=True): return outlist -def patch_recipe_file(fn, values, patch=False, relpath=''): - """Update or insert variable values into a recipe file (assuming you - have already identified the exact file you want to update.) +def patch_recipe_lines(fromlines, values, trailing_newline=True): + """Update or insert variable values into lines from a recipe. Note that some manual inspection/intervention may be required since this cannot handle all situations. """ import bb.utils + if trailing_newline: + newline = '\n' + else: + newline = '' + recipe_progression_res = [] recipe_progression_restrs = [] for item in recipe_progression: @@ -190,14 +196,14 @@ def patch_recipe_file(fn, values, patch=False, relpath=''): remainingnames = {} for k in values.keys(): remainingnames[k] = get_recipe_pos(k) - remainingnames = OrderedDict(sorted(remainingnames.iteritems(), key=lambda x: x[1])) + remainingnames = OrderedDict(sorted(remainingnames.items(), key=lambda x: x[1])) modifying = False def outputvalue(name, lines, rewindcomments=False): if values[name] is None: return - rawtext = '%s = "%s"\n' % (name, values[name]) + rawtext = '%s = "%s"%s' % (name, values[name], newline) addlines = [] if name in nowrap_vars: addlines.append(rawtext) @@ -205,19 +211,19 @@ def patch_recipe_file(fn, values, patch=False, relpath=''): splitvalue = split_var_value(values[name], assignment=False) if len(splitvalue) > 1: linesplit = ' \\\n' + (' ' * (len(name) + 4)) - addlines.append('%s = "%s%s"\n' % (name, linesplit.join(splitvalue), linesplit)) + addlines.append('%s = "%s%s"%s' % (name, linesplit.join(splitvalue), linesplit, newline)) else: addlines.append(rawtext) else: wrapped = textwrap.wrap(rawtext) for wrapline in wrapped[:-1]: - addlines.append('%s \\\n' % wrapline) - addlines.append('%s\n' % wrapped[-1]) + addlines.append('%s \\%s' % (wrapline, newline)) + addlines.append('%s%s' % (wrapped[-1], newline)) if rewindcomments: # Ensure we insert the lines before any leading comments # (that we'd want to ensure remain leading the next value) for i, ln in reversed(list(enumerate(lines))): - if ln[0] != '#': + if not ln.startswith('#'): lines[i+1:i+1] = addlines break else: @@ -230,7 +236,7 @@ def patch_recipe_file(fn, values, patch=False, relpath=''): if modifying: # Insert anything that should come before this variable pos = get_recipe_pos(varname) - for k in remainingnames.keys()[:]: + for k in list(remainingnames): if remainingnames[k] > -1 and pos >= remainingnames[k] and not k in existingnames: outputvalue(k, newlines, rewindcomments=True) del remainingnames[k] @@ -247,19 +253,33 @@ def patch_recipe_file(fn, values, patch=False, relpath=''): # First run - establish which values we want to set are already in the file varlist = [re.escape(item) for item in values.keys()] - with open(fn, 'r') as f: - changed, fromlines = bb.utils.edit_metadata(f, varlist, patch_recipe_varfunc) + bb.utils.edit_metadata(fromlines, varlist, patch_recipe_varfunc) # Second run - actually set everything modifying = True varlist.extend(recipe_progression_restrs) changed, tolines = bb.utils.edit_metadata(fromlines, varlist, patch_recipe_varfunc, match_overrides=True) if remainingnames: - if tolines[-1].strip() != '': + if tolines and tolines[-1].strip() != '': tolines.append('\n') for k in remainingnames.keys(): outputvalue(k, tolines) + return changed, tolines + + +def patch_recipe_file(fn, values, patch=False, relpath=''): + """Update or insert variable values into a recipe file (assuming you + have already identified the exact file you want to update.) + Note that some manual inspection/intervention may be required + since this cannot handle all situations. + """ + + with open(fn, 'r') as f: + fromlines = f.readlines() + + _, tolines = patch_recipe_lines(fromlines, values) + if patch: relfn = os.path.relpath(fn, relpath) diff = difflib.unified_diff(fromlines, tolines, 'a/%s' % relfn, 'b/%s' % relfn) @@ -318,7 +338,7 @@ def patch_recipe(d, fn, varvalues, patch=False, relpath=''): varfiles = get_var_files(fn, varlist, d) locs = localise_file_vars(fn, varfiles, varlist) patches = [] - for f,v in locs.iteritems(): + for f,v in locs.items(): vals = {k: varvalues[k] for k in v} patchdata = patch_recipe_file(f, vals, patch, relpath) if patch: @@ -347,6 +367,7 @@ def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True): # Copy local files to target directory and gather any remote files bb_dir = os.path.dirname(d.getVar('FILE', True)) + os.sep remotes = [] + copied = [] includes = [path for path in d.getVar('BBINCLUDED', True).split() if path.startswith(bb_dir) and os.path.exists(path)] for path in fetch.localpaths() + includes: @@ -358,13 +379,14 @@ def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True): if not os.path.exists(subdir): os.makedirs(subdir) shutil.copy2(path, os.path.join(tgt_dir, relpath)) + copied.append(relpath) else: remotes.append(path) # Simply copy whole meta dir, if requested if whole_dir: shutil.copytree(bb_dir, tgt_dir) - return remotes + return copied, remotes def get_recipe_local_files(d, patches=False): @@ -378,8 +400,16 @@ def get_recipe_local_files(d, patches=False): bb.utils.exec_flat_python_func('patch_path', uri, fetch, '')): continue # Skip files that are referenced by absolute path - if not os.path.isabs(fetch.ud[uri].basepath): - ret[fetch.ud[uri].basepath] = fetch.localpath(uri) + fname = fetch.ud[uri].basepath + if os.path.isabs(fname): + continue + # Handle subdir= + subdir = fetch.ud[uri].parm.get('subdir', '') + if subdir: + if os.path.isabs(subdir): + continue + fname = os.path.join(subdir, fname) + ret[fname] = fetch.localpath(uri) return ret @@ -419,8 +449,8 @@ def get_recipe_patched_files(d): def validate_pn(pn): """Perform validation on a recipe name (PN) for a new recipe.""" reserved_names = ['forcevariable', 'append', 'prepend', 'remove'] - if not re.match('[0-9a-z-.]+', pn): - return 'Recipe name "%s" is invalid: only characters 0-9, a-z, - and . are allowed' % pn + if not re.match('^[0-9a-z-.+]+$', pn): + return 'Recipe name "%s" is invalid: only characters 0-9, a-z, -, + and . are allowed' % pn elif pn in reserved_names: return 'Recipe name "%s" is invalid: is a reserved keyword' % pn elif pn.startswith('pn-'): @@ -430,6 +460,60 @@ def validate_pn(pn): return '' +def get_bbfile_path(d, destdir, extrapathhint=None): + """ + Determine the correct path for a recipe within a layer + Parameters: + d: Recipe-specific datastore + destdir: destination directory. Can be the path to the base of the layer or a + partial path somewhere within the layer. + extrapathhint: a path relative to the base of the layer to try + """ + import bb.cookerdata + + destdir = os.path.abspath(destdir) + destlayerdir = find_layerdir(destdir) + + # Parse the specified layer's layer.conf file directly, in case the layer isn't in bblayers.conf + confdata = d.createCopy() + confdata.setVar('BBFILES', '') + confdata.setVar('LAYERDIR', destlayerdir) + destlayerconf = os.path.join(destlayerdir, "conf", "layer.conf") + confdata = bb.cookerdata.parse_config_file(destlayerconf, confdata) + pn = d.getVar('PN', True) + + bbfilespecs = (confdata.getVar('BBFILES', True) or '').split() + if destdir == destlayerdir: + for bbfilespec in bbfilespecs: + if not bbfilespec.endswith('.bbappend'): + for match in glob.glob(bbfilespec): + splitext = os.path.splitext(os.path.basename(match)) + if splitext[1] == '.bb': + mpn = splitext[0].split('_')[0] + if mpn == pn: + return os.path.dirname(match) + + # Try to make up a path that matches BBFILES + # this is a little crude, but better than nothing + bpn = d.getVar('BPN', True) + recipefn = os.path.basename(d.getVar('FILE', True)) + pathoptions = [destdir] + if extrapathhint: + pathoptions.append(os.path.join(destdir, extrapathhint)) + if destdir == destlayerdir: + pathoptions.append(os.path.join(destdir, 'recipes-%s' % bpn, bpn)) + pathoptions.append(os.path.join(destdir, 'recipes', bpn)) + pathoptions.append(os.path.join(destdir, bpn)) + elif not destdir.endswith(('/' + pn, '/' + bpn)): + pathoptions.append(os.path.join(destdir, bpn)) + closepath = '' + for pathoption in pathoptions: + bbfilepath = os.path.join(pathoption, 'test.bb') + for bbfilespec in bbfilespecs: + if fnmatch.fnmatchcase(bbfilepath, bbfilespec): + return pathoption + return None + def get_bbappend_path(d, destlayerdir, wildcardver=False): """Determine how a bbappend for a recipe should be named and located within another layer""" @@ -536,7 +620,7 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, bbappendlines = [] if extralines: if isinstance(extralines, dict): - for name, value in extralines.iteritems(): + for name, value in extralines.items(): bbappendlines.append((name, '=', value)) else: # Do our best to split it @@ -550,14 +634,14 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, raise Exception('Invalid extralines value passed') def popline(varname): - for i in xrange(0, len(bbappendlines)): + for i in range(0, len(bbappendlines)): if bbappendlines[i][0] == varname: line = bbappendlines.pop(i) return line return None def appendline(varname, op, value): - for i in xrange(0, len(bbappendlines)): + for i in range(0, len(bbappendlines)): item = bbappendlines[i] if item[0] == varname: bbappendlines[i] = (item[0], item[1], item[2] + ' ' + value) @@ -576,7 +660,7 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, copyfiles = {} if srcfiles: instfunclines = [] - for newfile, origsrcfile in srcfiles.iteritems(): + for newfile, origsrcfile in srcfiles.items(): srcfile = origsrcfile srcurientry = None if not srcfile: @@ -644,7 +728,7 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, if removevar in removevalues: remove = removevalues[removevar] - if isinstance(remove, basestring): + if isinstance(remove, str): if remove in splitval: splitval.remove(remove) changed = True @@ -674,7 +758,7 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, varnames = [item[0] for item in bbappendlines] if removevalues: - varnames.extend(removevalues.keys()) + varnames.extend(list(removevalues.keys())) with open(appendpath, 'r') as f: (updated, newlines) = bb.utils.edit_metadata(f, varnames, appendfile_varfunc) @@ -699,7 +783,7 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, if copyfiles: if machine: destsubdir = os.path.join(destsubdir, machine) - for newfile, srcfile in copyfiles.iteritems(): + for newfile, srcfile in copyfiles.items(): filedest = os.path.join(appenddir, destsubdir, os.path.basename(srcfile)) if os.path.abspath(newfile) != os.path.abspath(filedest): bb.note('Copying %s to %s' % (newfile, filedest)) @@ -710,14 +794,16 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, def find_layerdir(fn): - """ Figure out relative path to base of layer for a file (e.g. a recipe)""" - pth = os.path.dirname(fn) + """ Figure out the path to the base of the layer containing a file (e.g. a recipe)""" + pth = fn layerdir = '' while pth: if os.path.exists(os.path.join(pth, 'conf', 'layer.conf')): layerdir = pth break pth = os.path.dirname(pth) + if pth == '/': + return None return layerdir @@ -725,12 +811,12 @@ def replace_dir_vars(path, d): """Replace common directory paths with appropriate variable references (e.g. /etc becomes ${sysconfdir})""" dirvars = {} # Sort by length so we get the variables we're interested in first - for var in sorted(d.keys(), key=len): + for var in sorted(list(d.keys()), key=len): if var.endswith('dir') and var.lower() == var: value = d.getVar(var, True) if value.startswith('/') and not '\n' in value and value not in dirvars: dirvars[value] = var - for dirpath in sorted(dirvars.keys(), reverse=True): + for dirpath in sorted(list(dirvars.keys()), reverse=True): path = path.replace(dirpath, '${%s}' % dirvars[dirpath]) return path diff --git a/import-layers/yocto-poky/meta/lib/oe/rootfs.py b/import-layers/yocto-poky/meta/lib/oe/rootfs.py index a95e1b739..a348b975c 100644 --- a/import-layers/yocto-poky/meta/lib/oe/rootfs.py +++ b/import-layers/yocto-poky/meta/lib/oe/rootfs.py @@ -10,17 +10,17 @@ import subprocess import re -class Rootfs(object): +class Rootfs(object, metaclass=ABCMeta): """ This is an abstract class. Do not instantiate this directly. """ - __metaclass__ = ABCMeta - def __init__(self, d): + def __init__(self, d, progress_reporter=None): self.d = d self.pm = None self.image_rootfs = self.d.getVar('IMAGE_ROOTFS', True) - self.deploy_dir_image = self.d.getVar('DEPLOY_DIR_IMAGE', True) + self.deploydir = self.d.getVar('IMGDEPLOYDIR', True) + self.progress_reporter = progress_reporter self.install_order = Manifest.INSTALL_ORDER @@ -40,51 +40,46 @@ class Rootfs(object): def _log_check(self): pass - def _log_check_warn(self): - r = re.compile('^(warn|Warn|NOTE: warn|NOTE: Warn|WARNING:)') + def _log_check_common(self, type, match): + # Ignore any lines containing log_check to avoid recursion, and ignore + # lines beginning with a + since sh -x may emit code which isn't + # actually executed, but may contain error messages + excludes = [ 'log_check', r'^\+' ] + if hasattr(self, 'log_check_expected_regexes'): + excludes.extend(self.log_check_expected_regexes) + excludes = [re.compile(x) for x in excludes] + r = re.compile(match) log_path = self.d.expand("${T}/log.do_rootfs") + messages = [] with open(log_path, 'r') as log: for line in log: - if 'log_check' in line or 'NOTE:' in line: - continue - - m = r.search(line) + for ee in excludes: + m = ee.search(line) + if m: + break if m: - bb.warn('[log_check] %s: found a warning message in the logfile (keyword \'%s\'):\n[log_check] %s' - % (self.d.getVar('PN', True), m.group(), line)) - - def _log_check_error(self): - r = re.compile(self.log_check_regex) - log_path = self.d.expand("${T}/log.do_rootfs") - with open(log_path, 'r') as log: - found_error = 0 - message = "\n" - for line in log: - if 'log_check' in line: continue - if hasattr(self, 'log_check_expected_errors_regexes'): - m = None - for ee in self.log_check_expected_errors_regexes: - m = re.search(ee, line) - if m: - break - if m: - continue - m = r.search(line) if m: - found_error = 1 - bb.warn('[log_check] In line: [%s]' % line) - bb.warn('[log_check] %s: found an error message in the logfile (keyword \'%s\'):\n[log_check] %s' - % (self.d.getVar('PN', True), m.group(), line)) + messages.append('[log_check] %s' % line) + if messages: + if len(messages) == 1: + msg = '1 %s message' % type + else: + msg = '%d %s messages' % (len(messages), type) + msg = '[log_check] %s: found %s in the logfile:\n%s' % \ + (self.d.getVar('PN', True), msg, ''.join(messages)) + if type == 'error': + bb.fatal(msg) + else: + bb.warn(msg) - if found_error >= 1 and found_error <= 5: - message += line + '\n' - found_error += 1 + def _log_check_warn(self): + self._log_check_common('warning', '^(warn|Warn|WARNING:)') - if found_error == 6: - bb.fatal(message) + def _log_check_error(self): + self._log_check_common('error', self.log_check_regex) def _insert_feed_uris(self): if bb.utils.contains("IMAGE_FEATURES", "package-management", @@ -121,8 +116,10 @@ class Rootfs(object): bb.note(" Copying back package database...") for dir in dirs: + if not os.path.isdir(self.image_rootfs + '-orig' + dir): + continue bb.utils.mkdirhier(self.image_rootfs + os.path.dirname(dir)) - shutil.copytree(self.image_rootfs + '-orig' + dir, self.image_rootfs + dir) + shutil.copytree(self.image_rootfs + '-orig' + dir, self.image_rootfs + dir, symlinks=True) cpath = oe.cachedpath.CachedPath() # Copy files located in /usr/lib/debug or /usr/src/debug @@ -185,16 +182,19 @@ class Rootfs(object): bb.utils.mkdirhier(self.image_rootfs) - bb.utils.mkdirhier(self.deploy_dir_image) + bb.utils.mkdirhier(self.deploydir) shutil.copytree(postinst_intercepts_dir, intercepts_dir) shutil.copy(self.d.expand("${COREBASE}/meta/files/deploydir_readme.txt"), - self.deploy_dir_image + + self.deploydir + "/README_-_DO_NOT_DELETE_FILES_IN_THIS_DIRECTORY.txt") execute_pre_post_process(self.d, pre_process_cmds) + if self.progress_reporter: + self.progress_reporter.next_stage() + # call the package manager dependent create method self._create() @@ -209,6 +209,9 @@ class Rootfs(object): execute_pre_post_process(self.d, post_process_cmds) + if self.progress_reporter: + self.progress_reporter.next_stage() + if bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", True, False, self.d): delayed_postinsts = self._get_delayed_postinsts() @@ -222,6 +225,9 @@ class Rootfs(object): self._uninstall_unneeded() + if self.progress_reporter: + self.progress_reporter.next_stage() + self._insert_feed_uris() self._run_ldconfig() @@ -232,6 +238,10 @@ class Rootfs(object): self._cleanup() self._log_check() + if self.progress_reporter: + self.progress_reporter.next_stage() + + def _uninstall_unneeded(self): # Remove unneeded init script symlinks delayed_postinsts = self._get_delayed_postinsts() @@ -243,7 +253,9 @@ class Rootfs(object): image_rorfs = bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", True, False, self.d) - if image_rorfs: + image_rorfs_force = self.d.getVar('FORCE_RO_REMOVE', True) + + if image_rorfs or image_rorfs_force == "1": # Remove components that we don't need if it's a read-only rootfs unneeded_pkgs = self.d.getVar("ROOTFS_RO_UNNEEDED", True).split() pkgs_installed = image_list_installed_packages(self.d) @@ -363,8 +375,8 @@ class Rootfs(object): class RpmRootfs(Rootfs): - def __init__(self, d, manifest_dir): - super(RpmRootfs, self).__init__(d) + def __init__(self, d, manifest_dir, progress_reporter=None): + super(RpmRootfs, self).__init__(d, progress_reporter) self.log_check_regex = '(unpacking of archive failed|Cannot find package'\ '|exit 1|ERROR: |Error: |Error |ERROR '\ '|Failed |Failed: |Failed$|Failed\(\d+\):)' @@ -422,11 +434,17 @@ class RpmRootfs(Rootfs): execute_pre_post_process(self.d, rpm_pre_process_cmds) + if self.progress_reporter: + self.progress_reporter.next_stage() + self.pm.dump_all_available_pkgs() if self.inc_rpm_image_gen == "1": self._create_incremental(pkgs_to_install) + if self.progress_reporter: + self.progress_reporter.next_stage() + self.pm.update() pkgs = [] @@ -437,12 +455,24 @@ class RpmRootfs(Rootfs): else: pkgs += pkgs_to_install[pkg_type] + if self.progress_reporter: + self.progress_reporter.next_stage() + self.pm.install(pkgs) + if self.progress_reporter: + self.progress_reporter.next_stage() + self.pm.install(pkgs_attempt, True) + if self.progress_reporter: + self.progress_reporter.next_stage() + self.pm.install_complementary() + if self.progress_reporter: + self.progress_reporter.next_stage() + self._setup_dbg_rootfs(['/etc/rpm', '/var/lib/rpm', '/var/lib/smart']) execute_pre_post_process(self.d, rpm_post_process_cmds) @@ -454,6 +484,10 @@ class RpmRootfs(Rootfs): self.pm.rpm_setup_smart_target_config() + if self.progress_reporter: + self.progress_reporter.next_stage() + + @staticmethod def _depends_list(): return ['DEPLOY_DIR_RPM', 'INC_RPM_IMAGE_GEN', 'RPM_PREPROCESS_COMMANDS', @@ -474,32 +508,6 @@ class RpmRootfs(Rootfs): # already saved in /etc/rpm-postinsts pass - def _log_check_error(self): - r = re.compile('(unpacking of archive failed|Cannot find package|exit 1|ERR|Fail)') - log_path = self.d.expand("${T}/log.do_rootfs") - with open(log_path, 'r') as log: - found_error = 0 - message = "\n" - for line in log.read().split('\n'): - if 'log_check' in line: - continue - # sh -x may emit code which isn't actually executed - if line.startswith('+'): - continue - - m = r.search(line) - if m: - found_error = 1 - bb.warn('log_check: There were error messages in the logfile') - bb.warn('log_check: Matched keyword: [%s]\n\n' % m.group()) - - if found_error >= 1 and found_error <= 5: - message += line + '\n' - found_error += 1 - - if found_error == 6: - bb.fatal(message) - def _log_check(self): self._log_check_warn() self._log_check_error() @@ -524,8 +532,8 @@ class RpmRootfs(Rootfs): bb.utils.remove(self.pm.install_dir_path, True) class DpkgOpkgRootfs(Rootfs): - def __init__(self, d): - super(DpkgOpkgRootfs, self).__init__(d) + def __init__(self, d, progress_reporter=None): + super(DpkgOpkgRootfs, self).__init__(d, progress_reporter) def _get_pkgs_postinsts(self, status_file): def _get_pkg_depends_list(pkg_depends): @@ -565,7 +573,7 @@ class DpkgOpkgRootfs(Rootfs): pkg_depends = m_depends.group(1) # remove package dependencies not in postinsts - pkg_names = pkgs.keys() + pkg_names = list(pkgs.keys()) for pkg_name in pkg_names: deps = pkgs[pkg_name][:] @@ -598,7 +606,7 @@ class DpkgOpkgRootfs(Rootfs): pkgs = self._get_pkgs_postinsts(status_file) if pkgs: root = "__packagegroup_postinst__" - pkgs[root] = pkgs.keys() + pkgs[root] = list(pkgs.keys()) _dep_resolve(pkgs, root, pkg_list, []) pkg_list.remove(root) @@ -619,10 +627,10 @@ class DpkgOpkgRootfs(Rootfs): num += 1 class DpkgRootfs(DpkgOpkgRootfs): - def __init__(self, d, manifest_dir): - super(DpkgRootfs, self).__init__(d) + def __init__(self, d, manifest_dir, progress_reporter=None): + super(DpkgRootfs, self).__init__(d, progress_reporter) self.log_check_regex = '^E:' - self.log_check_expected_errors_regexes = \ + self.log_check_expected_regexes = \ [ "^E: Unmet dependencies." ] @@ -648,15 +656,31 @@ class DpkgRootfs(DpkgOpkgRootfs): execute_pre_post_process(self.d, deb_pre_process_cmds) + if self.progress_reporter: + self.progress_reporter.next_stage() + # Don't support incremental, so skip that + self.progress_reporter.next_stage() + self.pm.update() + if self.progress_reporter: + self.progress_reporter.next_stage() + for pkg_type in self.install_order: if pkg_type in pkgs_to_install: self.pm.install(pkgs_to_install[pkg_type], [False, True][pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY]) + if self.progress_reporter: + # Don't support attemptonly, so skip that + self.progress_reporter.next_stage() + self.progress_reporter.next_stage() + self.pm.install_complementary() + if self.progress_reporter: + self.progress_reporter.next_stage() + self._setup_dbg_rootfs(['/var/lib/dpkg']) self.pm.fix_broken_dependencies() @@ -667,6 +691,9 @@ class DpkgRootfs(DpkgOpkgRootfs): execute_pre_post_process(self.d, deb_post_process_cmds) + if self.progress_reporter: + self.progress_reporter.next_stage() + @staticmethod def _depends_list(): return ['DEPLOY_DIR_DEB', 'DEB_SDK_ARCH', 'APTCONF_TARGET', 'APT_ARGS', 'DPKG_ARCH', 'DEB_PREPROCESS_COMMANDS', 'DEB_POSTPROCESS_COMMANDS'] @@ -692,8 +719,8 @@ class DpkgRootfs(DpkgOpkgRootfs): class OpkgRootfs(DpkgOpkgRootfs): - def __init__(self, d, manifest_dir): - super(OpkgRootfs, self).__init__(d) + def __init__(self, d, manifest_dir, progress_reporter=None): + super(OpkgRootfs, self).__init__(d, progress_reporter) self.log_check_regex = '(exit 1|Collected errors)' self.manifest = OpkgManifest(d, manifest_dir) @@ -887,13 +914,24 @@ class OpkgRootfs(DpkgOpkgRootfs): execute_pre_post_process(self.d, opkg_pre_process_cmds) + if self.progress_reporter: + self.progress_reporter.next_stage() + # Steps are a bit different in order, skip next + self.progress_reporter.next_stage() + self.pm.update() self.pm.handle_bad_recommendations() + if self.progress_reporter: + self.progress_reporter.next_stage() + if self.inc_opkg_image_gen == "1": self._remove_extra_packages(pkgs_to_install) + if self.progress_reporter: + self.progress_reporter.next_stage() + for pkg_type in self.install_order: if pkg_type in pkgs_to_install: # For multilib, we perform a sanity test before final install @@ -905,15 +943,24 @@ class OpkgRootfs(DpkgOpkgRootfs): self.pm.install(pkgs_to_install[pkg_type], [False, True][pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY]) + if self.progress_reporter: + self.progress_reporter.next_stage() + self.pm.install_complementary() - self._setup_dbg_rootfs(['/var/lib/opkg']) + if self.progress_reporter: + self.progress_reporter.next_stage() + + self._setup_dbg_rootfs(['/etc', '/var/lib/opkg', '/usr/lib/ssl']) execute_pre_post_process(self.d, opkg_post_process_cmds) if self.inc_opkg_image_gen == "1": self.pm.backup_packaging_data() + if self.progress_reporter: + self.progress_reporter.next_stage() + @staticmethod def _depends_list(): return ['IPKGCONF_SDK', 'IPK_FEED_URIS', 'DEPLOY_DIR_IPK', 'IPKGCONF_TARGET', 'INC_IPK_IMAGE_GEN', 'OPKG_ARGS', 'OPKGLIBDIR', 'OPKG_PREPROCESS_COMMANDS', 'OPKG_POSTPROCESS_COMMANDS', 'OPKGLIBDIR'] @@ -949,16 +996,16 @@ def variable_depends(d, manifest_dir=None): cls = get_class_for_type(img_type) return cls._depends_list() -def create_rootfs(d, manifest_dir=None): +def create_rootfs(d, manifest_dir=None, progress_reporter=None): env_bkp = os.environ.copy() img_type = d.getVar('IMAGE_PKGTYPE', True) if img_type == "rpm": - RpmRootfs(d, manifest_dir).create() + RpmRootfs(d, manifest_dir, progress_reporter).create() elif img_type == "ipk": - OpkgRootfs(d, manifest_dir).create() + OpkgRootfs(d, manifest_dir, progress_reporter).create() elif img_type == "deb": - DpkgRootfs(d, manifest_dir).create() + DpkgRootfs(d, manifest_dir, progress_reporter).create() os.environ.clear() os.environ.update(env_bkp) diff --git a/import-layers/yocto-poky/meta/lib/oe/sdk.py b/import-layers/yocto-poky/meta/lib/oe/sdk.py index f15fbdb36..c74525f92 100644 --- a/import-layers/yocto-poky/meta/lib/oe/sdk.py +++ b/import-layers/yocto-poky/meta/lib/oe/sdk.py @@ -8,9 +8,7 @@ import glob import traceback -class Sdk(object): - __metaclass__ = ABCMeta - +class Sdk(object, metaclass=ABCMeta): def __init__(self, d, manifest_dir): self.d = d self.sdk_output = self.d.getVar('SDK_OUTPUT', True) @@ -155,14 +153,16 @@ class RpmSdk(Sdk): execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_TARGET_COMMAND", True)) - self.target_pm.remove_packaging_data() + if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d): + self.target_pm.remove_packaging_data() bb.note("Installing NATIVESDK packages") self._populate_sysroot(self.host_pm, self.host_manifest) execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_HOST_COMMAND", True)) - self.host_pm.remove_packaging_data() + if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d): + self.host_pm.remove_packaging_data() # Move host RPM library data native_rpm_state_dir = os.path.join(self.sdk_output, @@ -232,14 +232,16 @@ class OpkgSdk(Sdk): execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_TARGET_COMMAND", True)) - self.target_pm.remove_packaging_data() + if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d): + self.target_pm.remove_packaging_data() bb.note("Installing NATIVESDK packages") self._populate_sysroot(self.host_pm, self.host_manifest) execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_HOST_COMMAND", True)) - self.host_pm.remove_packaging_data() + if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d): + self.host_pm.remove_packaging_data() target_sysconfdir = os.path.join(self.sdk_target_sysroot, self.sysconfdir) host_sysconfdir = os.path.join(self.sdk_host_sysroot, self.sysconfdir) @@ -247,12 +249,12 @@ class OpkgSdk(Sdk): self.mkdirhier(target_sysconfdir) shutil.copy(self.target_conf, target_sysconfdir) os.chmod(os.path.join(target_sysconfdir, - os.path.basename(self.target_conf)), 0644) + os.path.basename(self.target_conf)), 0o644) self.mkdirhier(host_sysconfdir) shutil.copy(self.host_conf, host_sysconfdir) os.chmod(os.path.join(host_sysconfdir, - os.path.basename(self.host_conf)), 0644) + os.path.basename(self.host_conf)), 0o644) native_opkg_state_dir = os.path.join(self.sdk_output, self.sdk_native_path, self.d.getVar('localstatedir_nativesdk', True).strip('/'), @@ -314,6 +316,9 @@ class DpkgSdk(Sdk): self._copy_apt_dir_to(os.path.join(self.sdk_target_sysroot, "etc", "apt")) + if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d): + self.target_pm.remove_packaging_data() + bb.note("Installing NATIVESDK packages") self._populate_sysroot(self.host_pm, self.host_manifest) @@ -322,6 +327,9 @@ class DpkgSdk(Sdk): self._copy_apt_dir_to(os.path.join(self.sdk_output, self.sdk_native_path, "etc", "apt")) + if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d): + self.host_pm.remove_packaging_data() + native_dpkg_state_dir = os.path.join(self.sdk_output, self.sdk_native_path, "var", "lib", "dpkg") self.mkdirhier(native_dpkg_state_dir) diff --git a/import-layers/yocto-poky/meta/lib/oe/sstatesig.py b/import-layers/yocto-poky/meta/lib/oe/sstatesig.py index 01dce660c..8224e3a12 100644 --- a/import-layers/yocto-poky/meta/lib/oe/sstatesig.py +++ b/import-layers/yocto-poky/meta/lib/oe/sstatesig.py @@ -130,7 +130,9 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): super(bb.siggen.SignatureGeneratorBasicHash, self).set_taskdata(coredata) def dump_sigs(self, dataCache, options): - self.dump_lockedsigs() + sigfile = os.getcwd() + "/locked-sigs.inc" + bb.plain("Writing locked sigs to %s" % sigfile) + self.dump_lockedsigs(sigfile) return super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigs(dataCache, options) def get_taskhash(self, fn, task, deps, dataCache): @@ -181,11 +183,7 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): return super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigtask(fn, task, stampbase, runtime) - def dump_lockedsigs(self, sigfile=None, taskfilter=None): - if not sigfile: - sigfile = os.getcwd() + "/locked-sigs.inc" - - bb.plain("Writing locked sigs to %s" % sigfile) + def dump_lockedsigs(self, sigfile, taskfilter=None): types = {} for k in self.runtaskdeps: if taskfilter: @@ -210,7 +208,7 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): continue f.write(" " + self.lockedpnmap[fn] + ":" + task + ":" + self.taskhash[k] + " \\\n") f.write(' "\n') - f.write('SIGGEN_LOCKEDSIGS_TYPES_%s = "%s"' % (self.machine, " ".join(types.keys()))) + f.write('SIGGEN_LOCKEDSIGS_TYPES_%s = "%s"' % (self.machine, " ".join(list(types.keys())))) def checkhashes(self, missed, ret, sq_fn, sq_task, sq_hash, sq_hashfn, d): warn_msgs = [] @@ -220,7 +218,7 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): for task in range(len(sq_fn)): if task not in ret: for pn in self.lockedsigs: - if sq_hash[task] in self.lockedsigs[pn].itervalues(): + if sq_hash[task] in iter(self.lockedsigs[pn].values()): if sq_task[task] == 'do_shared_workdir': continue sstate_missing_msgs.append("Locked sig is set for %s:%s (%s) yet not in sstate cache?" diff --git a/import-layers/yocto-poky/meta/lib/oe/terminal.py b/import-layers/yocto-poky/meta/lib/oe/terminal.py index 634daa903..3901ad3f2 100644 --- a/import-layers/yocto-poky/meta/lib/oe/terminal.py +++ b/import-layers/yocto-poky/meta/lib/oe/terminal.py @@ -25,9 +25,7 @@ class Registry(oe.classutils.ClassRegistry): return bool(cls.command) -class Terminal(Popen): - __metaclass__ = Registry - +class Terminal(Popen, metaclass=Registry): def __init__(self, sh_cmd, title=None, env=None, d=None): fmt_sh_cmd = self.format_command(sh_cmd, title) try: @@ -41,7 +39,7 @@ class Terminal(Popen): def format_command(self, sh_cmd, title): fmt = {'title': title or 'Terminal', 'command': sh_cmd} - if isinstance(self.command, basestring): + if isinstance(self.command, str): return shlex.split(self.command.format(**fmt)) else: return [element.format(**fmt) for element in self.command] @@ -53,7 +51,7 @@ class XTerminal(Terminal): raise UnsupportedTerminal(self.name) class Gnome(XTerminal): - command = 'gnome-terminal -t "{title}" --disable-factory -x {command}' + command = 'gnome-terminal -t "{title}" -x {command}' priority = 2 def __init__(self, sh_cmd, title=None, env=None, d=None): @@ -63,12 +61,28 @@ class Gnome(XTerminal): # Once fixed on the gnome-terminal project, this should be removed. if os.getenv('LC_ALL'): os.putenv('LC_ALL','') - # Check version - vernum = check_terminal_version("gnome-terminal") - if vernum and LooseVersion(vernum) >= '3.10': - logger.debug(1, 'Gnome-Terminal 3.10 or later does not support --disable-factory') - self.command = 'gnome-terminal -t "{title}" -x {command}' - XTerminal.__init__(self, sh_cmd, title, env, d) + # We need to know when the command completes but gnome-terminal gives us no way + # to do this. We therefore write the pid to a file using a "phonehome" wrapper + # script, then monitor the pid until it exits. Thanks gnome! + import tempfile + pidfile = tempfile.NamedTemporaryFile(delete = False).name + try: + sh_cmd = "oe-gnome-terminal-phonehome " + pidfile + " " + sh_cmd + XTerminal.__init__(self, sh_cmd, title, env, d) + while os.stat(pidfile).st_size <= 0: + continue + with open(pidfile, "r") as f: + pid = int(f.readline()) + finally: + os.unlink(pidfile) + + import time + while True: + try: + os.kill(pid, 0) + time.sleep(0.1) + except OSError: + return class Mate(XTerminal): command = 'mate-terminal -t "{title}" -x {command}' @@ -248,7 +262,7 @@ def check_terminal_version(terminalName): newenv["LANG"] = "C" p = sub.Popen(['sh', '-c', cmdversion], stdout=sub.PIPE, stderr=sub.PIPE, env=newenv) out, err = p.communicate() - ver_info = out.rstrip().split('\n') + ver_info = out.decode().rstrip().split('\n') except OSError as exc: import errno if exc.errno == errno.ENOENT: diff --git a/import-layers/yocto-poky/meta/lib/oe/tests/test_elf.py b/import-layers/yocto-poky/meta/lib/oe/tests/test_elf.py new file mode 100644 index 000000000..1f59037ed --- /dev/null +++ b/import-layers/yocto-poky/meta/lib/oe/tests/test_elf.py @@ -0,0 +1,21 @@ +import unittest +import oe.qa + +class TestElf(unittest.TestCase): + def test_machine_name(self): + """ + Test elf_machine_to_string() + """ + self.assertEqual(oe.qa.elf_machine_to_string(0x02), "SPARC") + self.assertEqual(oe.qa.elf_machine_to_string(0x03), "x86") + self.assertEqual(oe.qa.elf_machine_to_string(0x08), "MIPS") + self.assertEqual(oe.qa.elf_machine_to_string(0x14), "PowerPC") + self.assertEqual(oe.qa.elf_machine_to_string(0x28), "ARM") + self.assertEqual(oe.qa.elf_machine_to_string(0x2A), "SuperH") + self.assertEqual(oe.qa.elf_machine_to_string(0x32), "IA-64") + self.assertEqual(oe.qa.elf_machine_to_string(0x3E), "x86-64") + self.assertEqual(oe.qa.elf_machine_to_string(0xB7), "AArch64") + + self.assertEqual(oe.qa.elf_machine_to_string(0x00), "Unknown (0)") + self.assertEqual(oe.qa.elf_machine_to_string(0xDEADBEEF), "Unknown (3735928559)") + self.assertEqual(oe.qa.elf_machine_to_string("foobar"), "Unknown ('foobar')") diff --git a/import-layers/yocto-poky/meta/lib/oe/tests/test_path.py b/import-layers/yocto-poky/meta/lib/oe/tests/test_path.py index 3d41ce157..44d068143 100644 --- a/import-layers/yocto-poky/meta/lib/oe/tests/test_path.py +++ b/import-layers/yocto-poky/meta/lib/oe/tests/test_path.py @@ -55,7 +55,7 @@ class TestRealPath(unittest.TestCase): for d in self.DIRS: os.mkdir(os.path.join(self.root, d)) for f in self.FILES: - file(os.path.join(self.root, f), "w") + open(os.path.join(self.root, f), "w") for l in self.LINKS: os.symlink(l[1], os.path.join(self.root, l[0])) @@ -85,5 +85,5 @@ class TestRealPath(unittest.TestCase): def test_loop(self): for e in self.EXCEPTIONS: - self.assertRaisesRegexp(OSError, r'\[Errno %u\]' % e[1], + self.assertRaisesRegex(OSError, r'\[Errno %u\]' % e[1], self.__realpath, e[0], False, False) diff --git a/import-layers/yocto-poky/meta/lib/oe/types.py b/import-layers/yocto-poky/meta/lib/oe/types.py index 7f47c17d0..4ae58acfa 100644 --- a/import-layers/yocto-poky/meta/lib/oe/types.py +++ b/import-layers/yocto-poky/meta/lib/oe/types.py @@ -33,7 +33,7 @@ def choice(value, choices): Acts as a multiple choice for the user. To use this, set the variable type flag to 'choice', and set the 'choices' flag to a space separated list of valid values.""" - if not isinstance(value, basestring): + if not isinstance(value, str): raise TypeError("choice accepts a string, not '%s'" % type(value)) value = value.lower() @@ -106,7 +106,7 @@ def boolean(value): Valid values for false: 'no', 'n', 'false', 'f', '0' """ - if not isinstance(value, basestring): + if not isinstance(value, str): raise TypeError("boolean accepts a string, not '%s'" % type(value)) value = value.lower() diff --git a/import-layers/yocto-poky/meta/lib/oe/utils.py b/import-layers/yocto-poky/meta/lib/oe/utils.py index 30d30629f..d6545b197 100644 --- a/import-layers/yocto-poky/meta/lib/oe/utils.py +++ b/import-layers/yocto-poky/meta/lib/oe/utils.py @@ -23,13 +23,13 @@ def ifelse(condition, iftrue = True, iffalse = False): return iffalse def conditional(variable, checkvalue, truevalue, falsevalue, d): - if d.getVar(variable,1) == checkvalue: + if d.getVar(variable, True) == checkvalue: return truevalue else: return falsevalue def less_or_equal(variable, checkvalue, truevalue, falsevalue, d): - if float(d.getVar(variable,1)) <= float(checkvalue): + if float(d.getVar(variable, True)) <= float(checkvalue): return truevalue else: return falsevalue @@ -46,7 +46,7 @@ def both_contain(variable1, variable2, checkvalue, d): val2 = d.getVar(variable2, True) val1 = set(val1.split()) val2 = set(val2.split()) - if isinstance(checkvalue, basestring): + if isinstance(checkvalue, str): checkvalue = set(checkvalue.split()) else: checkvalue = set(checkvalue) @@ -85,11 +85,11 @@ def prune_suffix(var, suffixes, d): def str_filter(f, str, d): from re import match - return " ".join(filter(lambda x: match(f, x, 0), str.split())) + return " ".join([x for x in str.split() if match(f, x, 0)]) def str_filter_out(f, str, d): from re import match - return " ".join(filter(lambda x: not match(f, x, 0), str.split())) + return " ".join([x for x in str.split() if not match(f, x, 0)]) def param_bool(cfg, field, dflt = None): """Lookup <field> in <cfg> map and convert it to a boolean; take @@ -134,7 +134,7 @@ def packages_filter_out_system(d): PN-dbg PN-doc PN-locale-eb-gb removed. """ pn = d.getVar('PN', True) - blacklist = map(lambda suffix: pn + suffix, ('', '-dbg', '-dev', '-doc', '-locale', '-staticdev')) + blacklist = [pn + suffix for suffix in ('', '-dbg', '-dev', '-doc', '-locale', '-staticdev')] localepkg = pn + "-locale-" pkgs = [] @@ -235,7 +235,7 @@ def format_pkg_list(pkg_dict, ret_format=None): # so implement a version here # -from Queue import Queue +from queue import Queue from threading import Thread class ThreadedWorker(Thread): @@ -249,7 +249,7 @@ class ThreadedWorker(Thread): self.worker_end = worker_end def run(self): - from Queue import Empty + from queue import Empty if self.worker_init is not None: self.worker_init(self) @@ -264,8 +264,8 @@ class ThreadedWorker(Thread): try: func(self, *args, **kargs) - except Exception, e: - print e + except Exception as e: + print(e) finally: self.tasks.task_done() @@ -304,3 +304,16 @@ def write_ld_so_conf(d): with open(ldsoconf, "w") as f: f.write(d.getVar("base_libdir", True) + '\n') f.write(d.getVar("libdir", True) + '\n') + +class ImageQAFailed(bb.build.FuncFailed): + def __init__(self, description, name=None, logfile=None): + self.description = description + self.name = name + self.logfile=logfile + + def __str__(self): + msg = 'Function failed: %s' % self.name + if self.description: + msg = msg + ' (%s)' % self.description + + return msg |
