diff options
Diffstat (limited to 'import-layers/yocto-poky/meta/lib')
84 files changed, 3582 insertions, 1115 deletions
diff --git a/import-layers/yocto-poky/meta/lib/oe/buildhistory_analysis.py b/import-layers/yocto-poky/meta/lib/oe/buildhistory_analysis.py index 5395c768a..b6c0265c1 100644 --- a/import-layers/yocto-poky/meta/lib/oe/buildhistory_analysis.py +++ b/import-layers/yocto-poky/meta/lib/oe/buildhistory_analysis.py @@ -62,7 +62,7 @@ class ChangeRecord: def pkglist_combine(depver): pkglist = [] - for k,v in depver.iteritems(): + for k,v in depver.items(): if v: pkglist.append("%s (%s)" % (k,v)) else: @@ -190,7 +190,7 @@ class FileChange: def blob_to_dict(blob): - alines = blob.data_stream.read().splitlines() + alines = [line for line in blob.data_stream.read().decode('utf-8').splitlines()] adict = {} for line in alines: splitv = [i.strip() for i in line.split('=',1)] @@ -220,7 +220,7 @@ def compare_file_lists(alines, blines): adict = file_list_to_dict(alines) bdict = file_list_to_dict(blines) filechanges = [] - for path, splitv in adict.iteritems(): + for path, splitv in adict.items(): newsplitv = bdict.pop(path, None) if newsplitv: # Check type @@ -359,6 +359,24 @@ def compare_dict_blobs(path, ablob, bblob, report_all, report_ver): if ' '.join(alist) == ' '.join(blist): continue + if key == 'PKGR' and not report_all: + vers = [] + # strip leading 'r' and dots + for ver in (astr.split()[0], bstr.split()[0]): + if ver.startswith('r'): + ver = ver[1:] + vers.append(ver.replace('.', '')) + maxlen = max(len(vers[0]), len(vers[1])) + try: + # pad with '0' and convert to int + vers = [int(ver.ljust(maxlen, '0')) for ver in vers] + except ValueError: + pass + else: + # skip decrements and increments + if abs(vers[0] - vers[1]) == 1: + continue + chg = ChangeRecord(path, key, astr, bstr, monitored) changes.append(chg) return changes @@ -378,34 +396,34 @@ def process_changes(repopath, revision1, revision2='HEAD', report_all=False, rep if filename == 'latest': changes.extend(compare_dict_blobs(path, d.a_blob, d.b_blob, report_all, report_ver)) elif filename.startswith('latest.'): - chg = ChangeRecord(path, filename, d.a_blob.data_stream.read(), d.b_blob.data_stream.read(), True) + chg = ChangeRecord(path, filename, d.a_blob.data_stream.read().decode('utf-8'), d.b_blob.data_stream.read().decode('utf-8'), True) changes.append(chg) elif path.startswith('images/'): filename = os.path.basename(d.a_blob.path) if filename in img_monitor_files: if filename == 'files-in-image.txt': - alines = d.a_blob.data_stream.read().splitlines() - blines = d.b_blob.data_stream.read().splitlines() + alines = d.a_blob.data_stream.read().decode('utf-8').splitlines() + blines = d.b_blob.data_stream.read().decode('utf-8').splitlines() filechanges = compare_file_lists(alines,blines) if filechanges: chg = ChangeRecord(path, filename, None, None, True) chg.filechanges = filechanges changes.append(chg) elif filename == 'installed-package-names.txt': - alines = d.a_blob.data_stream.read().splitlines() - blines = d.b_blob.data_stream.read().splitlines() + alines = d.a_blob.data_stream.read().decode('utf-8').splitlines() + blines = d.b_blob.data_stream.read().decode('utf-8').splitlines() filechanges = compare_lists(alines,blines) if filechanges: chg = ChangeRecord(path, filename, None, None, True) chg.filechanges = filechanges changes.append(chg) else: - chg = ChangeRecord(path, filename, d.a_blob.data_stream.read(), d.b_blob.data_stream.read(), True) + chg = ChangeRecord(path, filename, d.a_blob.data_stream.read().decode('utf-8'), d.b_blob.data_stream.read().decode('utf-8'), True) changes.append(chg) elif filename == 'image-info.txt': changes.extend(compare_dict_blobs(path, d.a_blob, d.b_blob, report_all, report_ver)) elif '/image-files/' in path: - chg = ChangeRecord(path, filename, d.a_blob.data_stream.read(), d.b_blob.data_stream.read(), True) + chg = ChangeRecord(path, filename, d.a_blob.data_stream.read().decode('utf-8'), d.b_blob.data_stream.read().decode('utf-8'), True) changes.append(chg) # Look for added preinst/postinst/prerm/postrm @@ -419,7 +437,7 @@ def process_changes(repopath, revision1, revision2='HEAD', report_all=False, rep if filename == 'latest': addedpkgs.append(path) elif filename.startswith('latest.'): - chg = ChangeRecord(path, filename[7:], '', d.b_blob.data_stream.read(), True) + chg = ChangeRecord(path, filename[7:], '', d.b_blob.data_stream.read().decode('utf-8'), True) addedchanges.append(chg) for chg in addedchanges: found = False @@ -436,7 +454,7 @@ def process_changes(repopath, revision1, revision2='HEAD', report_all=False, rep if path.startswith('packages/'): filename = os.path.basename(d.a_blob.path) if filename != 'latest' and filename.startswith('latest.'): - chg = ChangeRecord(path, filename[7:], d.a_blob.data_stream.read(), '', True) + chg = ChangeRecord(path, filename[7:], d.a_blob.data_stream.read().decode('utf-8'), '', True) changes.append(chg) # Link related changes diff --git a/import-layers/yocto-poky/meta/lib/oe/classextend.py b/import-layers/yocto-poky/meta/lib/oe/classextend.py index 5107ecde2..4c8a00070 100644 --- a/import-layers/yocto-poky/meta/lib/oe/classextend.py +++ b/import-layers/yocto-poky/meta/lib/oe/classextend.py @@ -1,3 +1,5 @@ +import collections + class ClassExtender(object): def __init__(self, extname, d): self.extname = extname @@ -77,7 +79,7 @@ class ClassExtender(object): self.d.setVar("EXTENDPKGV", orig) return deps = bb.utils.explode_dep_versions2(deps) - newdeps = {} + newdeps = collections.OrderedDict() for dep in deps: newdeps[self.map_depends(dep)] = deps[dep] diff --git a/import-layers/yocto-poky/meta/lib/oe/classutils.py b/import-layers/yocto-poky/meta/lib/oe/classutils.py index 58188fdd6..e7856c86f 100644 --- a/import-layers/yocto-poky/meta/lib/oe/classutils.py +++ b/import-layers/yocto-poky/meta/lib/oe/classutils.py @@ -1,4 +1,11 @@ -class ClassRegistry(type): + +class ClassRegistryMeta(type): + """Give each ClassRegistry their own registry""" + def __init__(cls, name, bases, attrs): + cls.registry = {} + type.__init__(cls, name, bases, attrs) + +class ClassRegistry(type, metaclass=ClassRegistryMeta): """Maintain a registry of classes, indexed by name. Note that this implementation requires that the names be unique, as it uses @@ -12,12 +19,6 @@ Subclasses of ClassRegistry may define an 'implemented' property to exert control over whether the class will be added to the registry (e.g. to keep abstract base classes out of the registry).""" priority = 0 - class __metaclass__(type): - """Give each ClassRegistry their own registry""" - def __init__(cls, name, bases, attrs): - cls.registry = {} - type.__init__(cls, name, bases, attrs) - def __init__(cls, name, bases, attrs): super(ClassRegistry, cls).__init__(name, bases, attrs) try: @@ -34,7 +35,7 @@ abstract base classes out of the registry).""" @classmethod def prioritized(tcls): - return sorted(tcls.registry.values(), + return sorted(list(tcls.registry.values()), key=lambda v: v.priority, reverse=True) def unregister(cls): diff --git a/import-layers/yocto-poky/meta/lib/oe/copy_buildsystem.py b/import-layers/yocto-poky/meta/lib/oe/copy_buildsystem.py index 7b9a0ee06..afaff6859 100644 --- a/import-layers/yocto-poky/meta/lib/oe/copy_buildsystem.py +++ b/import-layers/yocto-poky/meta/lib/oe/copy_buildsystem.py @@ -17,7 +17,7 @@ class BuildSystem(object): def __init__(self, context, d): self.d = d self.context = context - self.layerdirs = d.getVar('BBLAYERS', True).split() + self.layerdirs = [os.path.abspath(pth) for pth in d.getVar('BBLAYERS', True).split()] self.layers_exclude = (d.getVar('SDK_LAYERS_EXCLUDE', True) or "").split() def copy_bitbake_and_layers(self, destdir, workspace_name=None): @@ -26,7 +26,7 @@ class BuildSystem(object): bb.utils.mkdirhier(destdir) layers = list(self.layerdirs) - corebase = self.d.getVar('COREBASE', True) + corebase = os.path.abspath(self.d.getVar('COREBASE', True)) layers.append(corebase) # Exclude layers @@ -124,7 +124,7 @@ class BuildSystem(object): def generate_locked_sigs(sigfile, d): bb.utils.mkdirhier(os.path.dirname(sigfile)) depd = d.getVar('BB_TASKDEPDATA', False) - tasks = ['%s.%s' % (v[2], v[1]) for v in depd.itervalues()] + tasks = ['%s.%s' % (v[2], v[1]) for v in depd.values()] bb.parse.siggen.dump_lockedsigs(sigfile, tasks) def prune_lockedsigs(excluded_tasks, excluded_targets, lockedsigs, pruned_output): @@ -145,7 +145,7 @@ def prune_lockedsigs(excluded_tasks, excluded_targets, lockedsigs, pruned_output invalue = True f.write(line) -def merge_lockedsigs(copy_tasks, lockedsigs_main, lockedsigs_extra, merged_output, copy_output): +def merge_lockedsigs(copy_tasks, lockedsigs_main, lockedsigs_extra, merged_output, copy_output=None): merged = {} arch_order = [] with open(lockedsigs_main, 'r') as f: @@ -195,23 +195,46 @@ def merge_lockedsigs(copy_tasks, lockedsigs_main, lockedsigs_extra, merged_outpu fulltypes.append(typename) f.write('SIGGEN_LOCKEDSIGS_TYPES = "%s"\n' % ' '.join(fulltypes)) - write_sigs_file(copy_output, tocopy.keys(), tocopy) + if copy_output: + write_sigs_file(copy_output, list(tocopy.keys()), tocopy) if merged_output: write_sigs_file(merged_output, arch_order, merged) -def create_locked_sstate_cache(lockedsigs, input_sstate_cache, output_sstate_cache, d, fixedlsbstring=""): +def create_locked_sstate_cache(lockedsigs, input_sstate_cache, output_sstate_cache, d, fixedlsbstring="", filterfile=None): + import shutil bb.note('Generating sstate-cache...') nativelsbstring = d.getVar('NATIVELSBSTRING', True) - bb.process.run("gen-lockedsig-cache %s %s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache, nativelsbstring)) - if fixedlsbstring: + bb.process.run("gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache, nativelsbstring, filterfile or '')) + if fixedlsbstring and nativelsbstring != fixedlsbstring: nativedir = output_sstate_cache + '/' + nativelsbstring if os.path.isdir(nativedir): destdir = os.path.join(output_sstate_cache, fixedlsbstring) - bb.utils.mkdirhier(destdir) + for root, _, files in os.walk(nativedir): + for fn in files: + src = os.path.join(root, fn) + dest = os.path.join(destdir, os.path.relpath(src, nativedir)) + if os.path.exists(dest): + # Already exists, and it'll be the same file, so just delete it + os.unlink(src) + else: + bb.utils.mkdirhier(os.path.dirname(dest)) + shutil.move(src, dest) + +def check_sstate_task_list(d, targets, filteroutfile, cmdprefix='', cwd=None, logfile=None): + import subprocess - dirlist = os.listdir(nativedir) - for i in dirlist: - src = os.path.join(nativedir, i) - dest = os.path.join(destdir, i) - os.rename(src, dest) + bb.note('Generating sstate task list...') + + if not cwd: + cwd = os.getcwd() + if logfile: + logparam = '-l %s' % logfile + else: + logparam = '' + cmd = "%sBB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam) + env = dict(d.getVar('BB_ORIGENV', False)) + env.pop('BUILDDIR', '') + pathitems = env['PATH'].split(':') + env['PATH'] = ':'.join([item for item in pathitems if not item.endswith('/bitbake/bin')]) + bb.process.run(cmd, stderr=subprocess.STDOUT, env=env, cwd=cwd, executable='/bin/bash') diff --git a/import-layers/yocto-poky/meta/lib/oe/data.py b/import-layers/yocto-poky/meta/lib/oe/data.py index e49572177..ee48950a8 100644 --- a/import-layers/yocto-poky/meta/lib/oe/data.py +++ b/import-layers/yocto-poky/meta/lib/oe/data.py @@ -7,11 +7,11 @@ def typed_value(key, d): flags = d.getVarFlags(key) if flags is not None: flags = dict((flag, d.expand(value)) - for flag, value in flags.iteritems()) + for flag, value in list(flags.items())) else: flags = {} try: return oe.maketype.create(d.getVar(key, True) or '', var_type, **flags) - except (TypeError, ValueError), exc: + except (TypeError, ValueError) as exc: bb.msg.fatal("Data", "%s: %s" % (key, str(exc))) diff --git a/import-layers/yocto-poky/meta/lib/oe/distro_check.py b/import-layers/yocto-poky/meta/lib/oe/distro_check.py index 8655a6fc1..87c52fae9 100644 --- a/import-layers/yocto-poky/meta/lib/oe/distro_check.py +++ b/import-layers/yocto-poky/meta/lib/oe/distro_check.py @@ -1,53 +1,35 @@ from contextlib import contextmanager -@contextmanager + +from bb.utils import export_proxies + def create_socket(url, d): import urllib - socket = urllib.urlopen(url, proxies=get_proxies(d)) + + socket = None try: - yield socket - finally: - socket.close() + export_proxies(d) + socket = urllib.request.urlopen(url) + except: + bb.warn("distro_check: create_socket url %s can't access" % url) -def get_proxies(d): - proxies = {} - for key in ['http', 'https', 'ftp', 'ftps', 'no', 'all']: - proxy = d.getVar(key + '_proxy', True) - if proxy: - proxies[key] = proxy - return proxies + return socket def get_links_from_url(url, d): "Return all the href links found on the web location" - import sgmllib - - class LinksParser(sgmllib.SGMLParser): - def parse(self, s): - "Parse the given string 's'." - self.feed(s) - self.close() - - def __init__(self, verbose=0): - "Initialise an object passing 'verbose' to the superclass." - sgmllib.SGMLParser.__init__(self, verbose) - self.hyperlinks = [] - - def start_a(self, attributes): - "Process a hyperlink and its 'attributes'." - for name, value in attributes: - if name == "href": - self.hyperlinks.append(value.strip('/')) - - def get_hyperlinks(self): - "Return the list of hyperlinks." - return self.hyperlinks + from bs4 import BeautifulSoup, SoupStrainer + + hyperlinks = [] - with create_socket(url,d) as sock: + webpage = '' + sock = create_socket(url,d) + if sock: webpage = sock.read() - linksparser = LinksParser() - linksparser.parse(webpage) - return linksparser.get_hyperlinks() + soup = BeautifulSoup(webpage, "html.parser", parse_only=SoupStrainer("a")) + for line in soup.find_all('a', href=True): + hyperlinks.append(line['href'].strip('/')) + return hyperlinks def find_latest_numeric_release(url, d): "Find the latest listed numeric release on the given url" @@ -104,8 +86,8 @@ def get_source_package_list_from_url(url, section, d): bb.note("Reading %s: %s" % (url, section)) links = get_links_from_url(url, d) - srpms = filter(is_src_rpm, links) - names_list = map(package_name_from_srpm, srpms) + srpms = list(filter(is_src_rpm, links)) + names_list = list(map(package_name_from_srpm, srpms)) new_pkgs = [] for pkgs in names_list: @@ -162,14 +144,18 @@ def find_latest_debian_release(url, d): def get_debian_style_source_package_list(url, section, d): "Return the list of package-names stored in the debian style Sources.gz file" - with create_socket(url,d) as sock: - webpage = sock.read() - import tempfile - tmpfile = tempfile.NamedTemporaryFile(mode='wb', prefix='oecore.', suffix='.tmp', delete=False) - tmpfilename=tmpfile.name - tmpfile.write(sock.read()) - tmpfile.close() + import tempfile import gzip + + webpage = '' + sock = create_socket(url,d) + if sock: + webpage = sock.read() + + tmpfile = tempfile.NamedTemporaryFile(mode='wb', prefix='oecore.', suffix='.tmp', delete=False) + tmpfilename=tmpfile.name + tmpfile.write(sock.read()) + tmpfile.close() bb.note("Reading %s: %s" % (url, section)) f = gzip.open(tmpfilename) @@ -266,9 +252,9 @@ def update_distro_data(distro_check_dir, datetime, d): import fcntl try: if not os.path.exists(datetime_file): - open(datetime_file, 'w+b').close() # touch the file so that the next open won't fail + open(datetime_file, 'w+').close() # touch the file so that the next open won't fail - f = open(datetime_file, "r+b") + f = open(datetime_file, "r+") fcntl.lockf(f, fcntl.LOCK_EX) saved_datetime = f.read() if saved_datetime[0:8] != datetime[0:8]: @@ -357,8 +343,8 @@ def compare_in_distro_packages_list(distro_check_dir, d): if tmp != None: - list = tmp.split(' ') - for item in list: + list = tmp.split(' ') + for item in list: matching_distros.append(item) bb.note("Matching: %s" % matching_distros) return matching_distros diff --git a/import-layers/yocto-poky/meta/lib/oe/gpg_sign.py b/import-layers/yocto-poky/meta/lib/oe/gpg_sign.py index b83ee8672..38eb0cb13 100644 --- a/import-layers/yocto-poky/meta/lib/oe/gpg_sign.py +++ b/import-layers/yocto-poky/meta/lib/oe/gpg_sign.py @@ -71,11 +71,11 @@ class LocalSigner(object): passphrase = fobj.readline(); job = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE) - (_, stderr) = job.communicate(passphrase) + (_, stderr) = job.communicate(passphrase.encode("utf-8")) if job.returncode: raise bb.build.FuncFailed("GPG exited with code %d: %s" % - (job.returncode, stderr)) + (job.returncode, stderr.decode("utf-8"))) except IOError as e: bb.error("IO error (%s): %s" % (e.errno, e.strerror)) @@ -90,7 +90,7 @@ class LocalSigner(object): """Return the gpg version""" import subprocess try: - return subprocess.check_output((self.gpg_bin, "--version")).split()[2] + return subprocess.check_output((self.gpg_bin, "--version")).split()[2].decode("utf-8") except subprocess.CalledProcessError as e: raise bb.build.FuncFailed("Could not get gpg version: %s" % e) @@ -113,4 +113,3 @@ def get_signer(d, backend): return LocalSigner(d) else: bb.fatal("Unsupported signing backend '%s'" % backend) - diff --git a/import-layers/yocto-poky/meta/lib/oe/license.py b/import-layers/yocto-poky/meta/lib/oe/license.py index f0f661c3b..8d2fd1709 100644 --- a/import-layers/yocto-poky/meta/lib/oe/license.py +++ b/import-layers/yocto-poky/meta/lib/oe/license.py @@ -47,7 +47,7 @@ class LicenseVisitor(ast.NodeVisitor): """Get elements based on OpenEmbedded license strings""" def get_elements(self, licensestr): new_elements = [] - elements = filter(lambda x: x.strip(), license_operator.split(licensestr)) + elements = list([x for x in license_operator.split(licensestr) if x.strip()]) for pos, element in enumerate(elements): if license_pattern.match(element): if pos > 0 and license_pattern.match(elements[pos-1]): @@ -118,8 +118,8 @@ def is_included(licensestr, whitelist=None, blacklist=None): def choose_licenses(alpha, beta): """Select the option in an OR which is the 'best' (has the most included licenses).""" - alpha_weight = len(filter(include_license, alpha)) - beta_weight = len(filter(include_license, beta)) + alpha_weight = len(list(filter(include_license, alpha))) + beta_weight = len(list(filter(include_license, beta))) if alpha_weight > beta_weight: return alpha else: @@ -132,8 +132,8 @@ def is_included(licensestr, whitelist=None, blacklist=None): blacklist = [] licenses = flattened_licenses(licensestr, choose_licenses) - excluded = filter(lambda lic: exclude_license(lic), licenses) - included = filter(lambda lic: include_license(lic), licenses) + excluded = [lic for lic in licenses if exclude_license(lic)] + included = [lic for lic in licenses if include_license(lic)] if excluded: return False, excluded else: @@ -215,3 +215,21 @@ def manifest_licenses(licensestr, dont_want_licenses, canonical_license, d): manifest.licensestr = manifest.licensestr.replace('[', '(').replace(']', ')') return (manifest.licensestr, manifest.licenses) + +class ListVisitor(LicenseVisitor): + """Record all different licenses found in the license string""" + def __init__(self): + self.licenses = set() + + def visit_Str(self, node): + self.licenses.add(node.s) + +def list_licenses(licensestr): + """Simply get a list of all licenses mentioned in a license string. + Binary operators are not applied or taken into account in any way""" + visitor = ListVisitor() + try: + visitor.visit_string(licensestr) + except SyntaxError as exc: + raise LicenseSyntaxError(licensestr, exc) + return visitor.licenses diff --git a/import-layers/yocto-poky/meta/lib/oe/maketype.py b/import-layers/yocto-poky/meta/lib/oe/maketype.py index 139f33369..f88981dd9 100644 --- a/import-layers/yocto-poky/meta/lib/oe/maketype.py +++ b/import-layers/yocto-poky/meta/lib/oe/maketype.py @@ -6,7 +6,8 @@ the arguments of the type's factory for details. """ import inspect -import types +import oe.types as types +import collections available_types = {} @@ -53,7 +54,9 @@ def get_callable_args(obj): if type(obj) is type: obj = obj.__init__ - args, varargs, keywords, defaults = inspect.getargspec(obj) + sig = inspect.signature(obj) + args = list(sig.parameters.keys()) + defaults = list(s for s in sig.parameters.keys() if sig.parameters[s].default != inspect.Parameter.empty) flaglist = [] if args: if len(args) > 1 and args[0] == 'self': @@ -93,7 +96,7 @@ for name in dir(types): continue obj = getattr(types, name) - if not callable(obj): + if not isinstance(obj, collections.Callable): continue register(name, obj) diff --git a/import-layers/yocto-poky/meta/lib/oe/manifest.py b/import-layers/yocto-poky/meta/lib/oe/manifest.py index 42832f15d..95f8eb2df 100644 --- a/import-layers/yocto-poky/meta/lib/oe/manifest.py +++ b/import-layers/yocto-poky/meta/lib/oe/manifest.py @@ -4,11 +4,10 @@ import re import bb -class Manifest(object): +class Manifest(object, metaclass=ABCMeta): """ This is an abstract class. Do not instantiate this directly. """ - __metaclass__ = ABCMeta PKG_TYPE_MUST_INSTALL = "mip" PKG_TYPE_MULTILIB = "mlp" @@ -219,7 +218,7 @@ class RpmManifest(Manifest): if var in self.vars_to_split: split_pkgs = self._split_multilib(self.d.getVar(var, True)) if split_pkgs is not None: - pkgs = dict(pkgs.items() + split_pkgs.items()) + pkgs = dict(list(pkgs.items()) + list(split_pkgs.items())) else: pkg_list = self.d.getVar(var, True) if pkg_list is not None: @@ -269,7 +268,7 @@ class OpkgManifest(Manifest): if var in self.vars_to_split: split_pkgs = self._split_multilib(self.d.getVar(var, True)) if split_pkgs is not None: - pkgs = dict(pkgs.items() + split_pkgs.items()) + pkgs = dict(list(pkgs.items()) + list(split_pkgs.items())) else: pkg_list = self.d.getVar(var, True) if pkg_list is not None: diff --git a/import-layers/yocto-poky/meta/lib/oe/package.py b/import-layers/yocto-poky/meta/lib/oe/package.py index 288768954..02642f29f 100644 --- a/import-layers/yocto-poky/meta/lib/oe/package.py +++ b/import-layers/yocto-poky/meta/lib/oe/package.py @@ -8,7 +8,7 @@ def runstrip(arg): # 8 - shared library # 16 - kernel module - import commands, stat, subprocess + import stat, subprocess (file, elftype, strip) = arg @@ -64,8 +64,8 @@ def filedeprunner(arg): def process_deps(pipe, pkg, pkgdest, provides, requires): for line in pipe: - f = line.split(" ", 1)[0].strip() - line = line.split(" ", 1)[1].strip() + f = line.decode("utf-8").split(" ", 1)[0].strip() + line = line.decode("utf-8").split(" ", 1)[1].strip() if line.startswith("Requires:"): i = requires @@ -114,7 +114,12 @@ def read_shlib_providers(d): m = list_re.match(file) if m: dep_pkg = m.group(1) - fd = open(os.path.join(dir, file)) + try: + fd = open(os.path.join(dir, file)) + except IOError: + # During a build unrelated shlib files may be deleted, so + # handle files disappearing between the listdirs and open. + continue lines = fd.readlines() fd.close() for l in lines: @@ -149,7 +154,7 @@ def npm_split_package_dirs(pkgdir): if os.path.exists(pkgfile): with open(pkgfile, 'r') as f: data = json.loads(f.read()) - packages[pkgname] = (relpth, data) + packages[pkgname] = (relpth, data) # We want the main package for a module sorted *after* its subpackages # (so that it doesn't otherwise steal the files for the subpackage), so # this is a cheap way to do that whilst still having an otherwise diff --git a/import-layers/yocto-poky/meta/lib/oe/package_manager.py b/import-layers/yocto-poky/meta/lib/oe/package_manager.py index b4b359a8c..3cee9730a 100644 --- a/import-layers/yocto-poky/meta/lib/oe/package_manager.py +++ b/import-layers/yocto-poky/meta/lib/oe/package_manager.py @@ -5,9 +5,11 @@ import subprocess import shutil import multiprocessing import re +import collections import bb import tempfile import oe.utils +import oe.path import string from oe.gpg_sign import get_signer @@ -17,20 +19,79 @@ def create_index(arg): try: bb.note("Executing '%s' ..." % index_cmd) - result = subprocess.check_output(index_cmd, stderr=subprocess.STDOUT, shell=True) + result = subprocess.check_output(index_cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8") except subprocess.CalledProcessError as e: return("Index creation command '%s' failed with return code %d:\n%s" % - (e.cmd, e.returncode, e.output)) + (e.cmd, e.returncode, e.output.decode("utf-8"))) if result: bb.note(result) return None - -class Indexer(object): - __metaclass__ = ABCMeta - +""" +This method parse the output from the package managerand return +a dictionary with the information of the packages. This is used +when the packages are in deb or ipk format. +""" +def opkg_query(cmd_output): + verregex = re.compile(' \([=<>]* [^ )]*\)') + output = dict() + pkg = "" + arch = "" + ver = "" + filename = "" + dep = [] + pkgarch = "" + for line in cmd_output.splitlines(): + line = line.rstrip() + if ':' in line: + if line.startswith("Package: "): + pkg = line.split(": ")[1] + elif line.startswith("Architecture: "): + arch = line.split(": ")[1] + elif line.startswith("Version: "): + ver = line.split(": ")[1] + elif line.startswith("File: ") or line.startswith("Filename:"): + filename = line.split(": ")[1] + if "/" in filename: + filename = os.path.basename(filename) + elif line.startswith("Depends: "): + depends = verregex.sub('', line.split(": ")[1]) + for depend in depends.split(", "): + dep.append(depend) + elif line.startswith("Recommends: "): + recommends = verregex.sub('', line.split(": ")[1]) + for recommend in recommends.split(", "): + dep.append("%s [REC]" % recommend) + elif line.startswith("PackageArch: "): + pkgarch = line.split(": ")[1] + + # When there is a blank line save the package information + elif not line: + # IPK doesn't include the filename + if not filename: + filename = "%s_%s_%s.ipk" % (pkg, ver, arch) + if pkg: + output[pkg] = {"arch":arch, "ver":ver, + "filename":filename, "deps": dep, "pkgarch":pkgarch } + pkg = "" + arch = "" + ver = "" + filename = "" + dep = [] + pkgarch = "" + + if pkg: + if not filename: + filename = "%s_%s_%s.ipk" % (pkg, ver, arch) + output[pkg] = {"arch":arch, "ver":ver, + "filename":filename, "deps": dep } + + return output + + +class Indexer(object, metaclass=ABCMeta): def __init__(self, d, deploy_dir): self.d = d self.deploy_dir = deploy_dir @@ -42,13 +103,8 @@ class Indexer(object): class RpmIndexer(Indexer): def get_ml_prefix_and_os_list(self, arch_var=None, os_var=None): - package_archs = { - 'default': [], - } - - target_os = { - 'default': "", - } + package_archs = collections.OrderedDict() + target_os = collections.OrderedDict() if arch_var is not None and os_var is not None: package_archs['default'] = self.d.getVar(arch_var, True).split() @@ -79,7 +135,7 @@ class RpmIndexer(Indexer): target_os[eext[1]] = localdata.getVar("TARGET_OS", True).strip() - ml_prefix_list = dict() + ml_prefix_list = collections.OrderedDict() for mlib in package_archs: if mlib == 'default': ml_prefix_list[mlib] = package_archs[mlib] @@ -281,9 +337,7 @@ class DpkgIndexer(Indexer): -class PkgsList(object): - __metaclass__ = ABCMeta - +class PkgsList(object, metaclass=ABCMeta): def __init__(self, d, rootfs_dir): self.d = d self.rootfs_dir = rootfs_dir @@ -293,57 +347,6 @@ class PkgsList(object): pass - """ - This method parse the output from the package manager - and return a dictionary with the information of the - installed packages. This is used whne the packages are - in deb or ipk format - """ - def opkg_query(self, cmd_output): - verregex = re.compile(' \([=<>]* [^ )]*\)') - output = dict() - filename = "" - dep = [] - pkg = "" - for line in cmd_output.splitlines(): - line = line.rstrip() - if ':' in line: - if line.startswith("Package: "): - pkg = line.split(": ")[1] - elif line.startswith("Architecture: "): - arch = line.split(": ")[1] - elif line.startswith("Version: "): - ver = line.split(": ")[1] - elif line.startswith("File: "): - filename = line.split(": ")[1] - elif line.startswith("Depends: "): - depends = verregex.sub('', line.split(": ")[1]) - for depend in depends.split(", "): - dep.append(depend) - elif line.startswith("Recommends: "): - recommends = verregex.sub('', line.split(": ")[1]) - for recommend in recommends.split(", "): - dep.append("%s [REC]" % recommend) - else: - # IPK doesn't include the filename - if not filename: - filename = "%s_%s_%s.ipk" % (pkg, ver, arch) - if pkg: - output[pkg] = {"arch":arch, "ver":ver, - "filename":filename, "deps": dep } - pkg = "" - filename = "" - dep = [] - - if pkg: - if not filename: - filename = "%s_%s_%s.ipk" % (pkg, ver, arch) - output[pkg] = {"arch":arch, "ver":ver, - "filename":filename, "deps": dep } - - return output - - class RpmPkgsList(PkgsList): def __init__(self, d, rootfs_dir, arch_var=None, os_var=None): super(RpmPkgsList, self).__init__(d, rootfs_dir) @@ -357,10 +360,10 @@ class RpmPkgsList(PkgsList): # Determine rpm version cmd = "%s --version" % self.rpm_cmd try: - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8") except subprocess.CalledProcessError as e: bb.fatal("Getting rpm version failed. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) ''' Translate the RPM/Smart format names to the OE multilib format names @@ -401,10 +404,10 @@ class RpmPkgsList(PkgsList): "-t", self.image_rpmlib] try: - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).strip() + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).strip().decode("utf-8") except subprocess.CalledProcessError as e: bb.fatal("Cannot get the package dependencies. Command '%s' " - "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output)) + "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8"))) return output @@ -415,10 +418,10 @@ class RpmPkgsList(PkgsList): try: # bb.note(cmd) - tmp_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).strip() + tmp_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).strip().decode("utf-8") except subprocess.CalledProcessError as e: bb.fatal("Cannot get the installed packages list. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) output = dict() deps = dict() @@ -475,11 +478,13 @@ class OpkgPkgsList(PkgsList): # output streams separately and check for empty stderr. p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) cmd_output, cmd_stderr = p.communicate() + cmd_output = cmd_output.decode("utf-8") + cmd_stderr = cmd_stderr.decode("utf-8") if p.returncode or cmd_stderr: bb.fatal("Cannot get the installed packages list. Command '%s' " "returned %d and stderr:\n%s" % (cmd, p.returncode, cmd_stderr)) - return self.opkg_query(cmd_output) + return opkg_query(cmd_output) class DpkgPkgsList(PkgsList): @@ -492,19 +497,18 @@ class DpkgPkgsList(PkgsList): cmd.append("-f=Package: ${Package}\nArchitecture: ${PackageArch}\nVersion: ${Version}\nFile: ${Package}_${Version}_${Architecture}.deb\nDepends: ${Depends}\nRecommends: ${Recommends}\n\n") try: - cmd_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).strip() + cmd_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).strip().decode("utf-8") except subprocess.CalledProcessError as e: bb.fatal("Cannot get the installed packages list. Command '%s' " - "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output)) + "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8"))) - return self.opkg_query(cmd_output) + return opkg_query(cmd_output) -class PackageManager(object): +class PackageManager(object, metaclass=ABCMeta): """ This is an abstract class. Do not instantiate this directly. """ - __metaclass__ = ABCMeta def __init__(self, d): self.d = d @@ -594,15 +598,15 @@ class PackageManager(object): globs] exclude = self.d.getVar('PACKAGE_EXCLUDE_COMPLEMENTARY', True) if exclude: - cmd.extend(['-x', exclude]) + cmd.extend(['--exclude=' + '|'.join(exclude.split())]) try: bb.note("Installing complementary packages ...") bb.note('Running %s' % cmd) - complementary_pkgs = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + complementary_pkgs = subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode("utf-8") except subprocess.CalledProcessError as e: bb.fatal("Could not compute complementary packages list. Command " "'%s' returned %d:\n%s" % - (' '.join(cmd), e.returncode, e.output)) + (' '.join(cmd), e.returncode, e.output.decode("utf-8"))) self.install(complementary_pkgs.split(), attempt_only=True) os.remove(installed_pkgs_file) @@ -631,8 +635,8 @@ class PackageManager(object): def construct_uris(self, uris, base_paths): def _append(arr1, arr2, sep='/'): res = [] - narr1 = map(lambda a: string.rstrip(a, sep), arr1) - narr2 = map(lambda a: string.lstrip(string.rstrip(a, sep), sep), arr2) + narr1 = [a.rstrip(sep) for a in arr1] + narr2 = [a.rstrip(sep).lstrip(sep) for a in arr2] for a1 in narr1: if arr2: for a2 in narr2: @@ -682,7 +686,8 @@ class RpmPM(PackageManager): if not os.path.exists(self.d.expand('${T}/saved')): bb.utils.mkdirhier(self.d.expand('${T}/saved')) - self.indexer = RpmIndexer(self.d, self.deploy_dir) + packageindex_dir = os.path.join(self.d.getVar('WORKDIR', True), 'rpms') + self.indexer = RpmIndexer(self.d, packageindex_dir) self.pkgs_list = RpmPkgsList(self.d, self.target_rootfs, arch_var, os_var) self.ml_prefix_list, self.ml_os_list = self.indexer.get_ml_prefix_and_os_list(arch_var, os_var) @@ -697,18 +702,19 @@ class RpmPM(PackageManager): arch_list = self.feed_archs.split() else: # List must be prefered to least preferred order - default_platform_extra = set() - platform_extra = set() + default_platform_extra = list() + platform_extra = list() bbextendvariant = self.d.getVar('BBEXTENDVARIANT', True) or "" for mlib in self.ml_os_list: for arch in self.ml_prefix_list[mlib]: plt = arch.replace('-', '_') + '-.*-' + self.ml_os_list[mlib] if mlib == bbextendvariant: - default_platform_extra.add(plt) + if plt not in default_platform_extra: + default_platform_extra.append(plt) else: - platform_extra.add(plt) - - platform_extra = platform_extra.union(default_platform_extra) + if plt not in platform_extra: + platform_extra.append(plt) + platform_extra = default_platform_extra + platform_extra for canonical_arch in platform_extra: arch = canonical_arch.split('-')[0] @@ -724,7 +730,7 @@ class RpmPM(PackageManager): for uri in feed_uris: if arch_list: for arch in arch_list: - bb.note('Note: adding Smart channel url%d%s (%s)' % + bb.note('Adding Smart channel url%d%s (%s)' % (uri_iterator, arch, channel_priority)) self._invoke_smart('channel --add url%d-%s type=rpm-md baseurl=%s/%s -y' % (uri_iterator, arch, uri, arch)) @@ -732,7 +738,7 @@ class RpmPM(PackageManager): (uri_iterator, arch, channel_priority)) channel_priority -= 5 else: - bb.note('Note: adding Smart channel url%d (%s)' % + bb.note('Adding Smart channel url%d (%s)' % (uri_iterator, channel_priority)) self._invoke_smart('channel --add url%d type=rpm-md baseurl=%s -y' % (uri_iterator, uri)) @@ -774,12 +780,12 @@ class RpmPM(PackageManager): try: complementary_pkgs = subprocess.check_output(cmd, stderr=subprocess.STDOUT, - shell=True) + shell=True).decode("utf-8") # bb.note(complementary_pkgs) return complementary_pkgs except subprocess.CalledProcessError as e: bb.fatal("Could not invoke smart. Command " - "'%s' returned %d:\n%s" % (cmd, e.returncode, e.output)) + "'%s' returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) def _search_pkg_name_in_feeds(self, pkg, feed_archs): for arch in feed_archs: @@ -798,7 +804,7 @@ class RpmPM(PackageManager): (self.smart_cmd, self.smart_opt, pkg) cmd += " | sed -ne 's/ *Provides://p'" bb.note('cmd: %s' % cmd) - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8") # Found a provider if output: bb.note('Found providers for %s: %s' % (pkg, output)) @@ -833,11 +839,13 @@ class RpmPM(PackageManager): new_pkg = self._search_pkg_name_in_feeds(subst, feed_archs) if not new_pkg: # Failed to translate, package not found! - err_msg = '%s not found in the %s feeds (%s).\n' % \ - (pkg, mlib, " ".join(feed_archs)) + err_msg = '%s not found in the %s feeds (%s) in %s.' % \ + (pkg, mlib, " ".join(feed_archs), self.d.getVar('DEPLOY_DIR_RPM', True)) if not attempt_only: - err_msg += " ".join(self.fullpkglist) - bb.fatal(err_msg) + bb.error(err_msg) + bb.fatal("This is often caused by an empty package declared " \ + "in a recipe's PACKAGES variable. (Empty packages are " \ + "not constructed unless ALLOW_EMPTY_<pkg> = '1' is used.)") bb.warn(err_msg) else: new_pkgs.append(new_pkg) @@ -850,11 +858,13 @@ class RpmPM(PackageManager): default_archs = self.ml_prefix_list['default'] new_pkg = self._search_pkg_name_in_feeds(pkg, default_archs) if not new_pkg: - err_msg = '%s not found in the base feeds (%s).\n' % \ - (pkg, ' '.join(default_archs)) + err_msg = '%s not found in the feeds (%s) in %s.' % \ + (pkg, " ".join(default_archs), self.d.getVar('DEPLOY_DIR_RPM', True)) if not attempt_only: - err_msg += " ".join(self.fullpkglist) - bb.fatal(err_msg) + bb.error(err_msg) + bb.fatal("This is often caused by an empty package declared " \ + "in a recipe's PACKAGES variable. (Empty packages are " \ + "not constructed unless ALLOW_EMPTY_<pkg> = '1' is used.)") bb.warn(err_msg) else: new_pkgs.append(new_pkg) @@ -946,7 +956,7 @@ class RpmPM(PackageManager): subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError as e: bb.fatal("Create rpm database failed. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) # Import GPG key to RPM database of the target system if self.d.getVar('RPM_SIGN_PACKAGES', True) == '1': pubkey_path = self.d.getVar('RPM_GPG_PUBKEY', True) @@ -1006,9 +1016,19 @@ class RpmPM(PackageManager): ch_already_added = [] for canonical_arch in platform_extra: arch = canonical_arch.split('-')[0] - arch_channel = os.path.join(self.deploy_dir, arch) - if os.path.exists(arch_channel) and not arch in ch_already_added: - bb.note('Note: adding Smart channel %s (%s)' % + arch_channel = os.path.join(self.d.getVar('WORKDIR', True), 'rpms', arch) + oe.path.remove(arch_channel) + deploy_arch_dir = os.path.join(self.deploy_dir, arch) + if not os.path.exists(deploy_arch_dir): + continue + + lockfilename = self.d.getVar('DEPLOY_DIR_RPM', True) + "/rpm.lock" + lf = bb.utils.lockfile(lockfilename, False) + oe.path.copyhardlinktree(deploy_arch_dir, arch_channel) + bb.utils.unlockfile(lf) + + if not arch in ch_already_added: + bb.note('Adding Smart channel %s (%s)' % (arch, channel_priority)) self._invoke_smart('channel --add %s type=rpm-md baseurl=%s -y' % (arch, arch_channel)) @@ -1080,8 +1100,8 @@ class RpmPM(PackageManager): native_root) open(self.scriptlet_wrapper, 'w+').write(scriptlet_content) - bb.note("Note: configuring RPM cross-install scriptlet_wrapper") - os.chmod(self.scriptlet_wrapper, 0755) + bb.note("configuring RPM cross-install scriptlet_wrapper") + os.chmod(self.scriptlet_wrapper, 0o755) cmd = 'config --set rpm-extra-macros._cross_scriptlet_wrapper=%s' % \ self.scriptlet_wrapper self._invoke_smart(cmd) @@ -1099,7 +1119,7 @@ class RpmPM(PackageManager): sub_rdep = sub_data.get("RDEPENDS_" + pkg) if not sub_rdep: continue - done = bb.utils.explode_dep_versions2(sub_rdep).keys() + done = list(bb.utils.explode_dep_versions2(sub_rdep).keys()) next = done # Find all the rdepends on dependency chain while next: @@ -1182,6 +1202,9 @@ class RpmPM(PackageManager): new_depends[new_depend] = deps[depend] pkgs = bb.utils.join_deps(new_depends, commasep=True).split(', ') pkgs = self._pkg_translate_oe_to_smart(pkgs, attempt_only) + if not pkgs: + bb.note("There are no packages to install") + return if not attempt_only: bb.note('to be installed: %s' % ' '.join(pkgs)) cmd = "%s %s install -y %s" % \ @@ -1193,11 +1216,11 @@ class RpmPM(PackageManager): cmd = "%s %s install --attempt -y %s" % \ (self.smart_cmd, self.smart_opt, ' '.join(pkgs)) try: - output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) + output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT).decode("utf-8") bb.note(output) except subprocess.CalledProcessError as e: bb.fatal("Unable to install packages. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) ''' Remove pkgs with smart, the pkg name is smart/rpm format @@ -1223,11 +1246,11 @@ class RpmPM(PackageManager): try: bb.note(cmd) - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8") bb.note(output) except subprocess.CalledProcessError as e: bb.note("Unable to remove packages. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) def upgrade(self): bb.note('smart upgrade') @@ -1300,7 +1323,7 @@ class RpmPM(PackageManager): install_pkgs.append(pkg) except subprocess.CalledProcessError as e: bb.note("Unable to dump install packages. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) # Recovery rpmsys channel self._invoke_smart('channel --enable rpmsys') return install_pkgs @@ -1342,7 +1365,7 @@ class RpmPM(PackageManager): available_pkgs.append(pkg.strip()) except subprocess.CalledProcessError as e: bb.note("Unable to list all available packages. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) self.fullpkglist = available_pkgs @@ -1369,12 +1392,12 @@ class RpmPM(PackageManager): try: bb.note(cmd) - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).strip() + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).strip().decode("utf-8") bb.note(output) - os.chmod(saved_dir, 0755) + os.chmod(saved_dir, 0o755) except subprocess.CalledProcessError as e: bb.fatal("Invoke save_rpmpostinst failed. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) '''Write common configuration for target usage''' def rpm_setup_smart_target_config(self): @@ -1398,8 +1421,157 @@ class RpmPM(PackageManager): for f in rpm_db_locks: bb.utils.remove(f, True) + """ + Returns a dictionary with the package info. + """ + def package_info(self, pkg): + cmd = "%s %s info --urls %s" % (self.smart_cmd, self.smart_opt, pkg) + try: + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8") + except subprocess.CalledProcessError as e: + bb.fatal("Unable to list available packages. Command '%s' " + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) + + # Set default values to avoid UnboundLocalError + arch = "" + ver = "" + filename = "" + + #Parse output + for line in output.splitlines(): + line = line.rstrip() + if line.startswith("Name:"): + pkg = line.split(": ")[1] + elif line.startswith("Version:"): + tmp_str = line.split(": ")[1] + ver, arch = tmp_str.split("@") + break + + # Get filename + index = re.search("^URLs", output, re.MULTILINE) + tmp_str = output[index.end():] + for line in tmp_str.splitlines(): + if "/" in line: + line = line.lstrip() + filename = line.split(" ")[0] + break + + # To have the same data type than other package_info methods + filepath = os.path.join(self.deploy_dir, arch, filename) + pkg_dict = {} + pkg_dict[pkg] = {"arch":arch, "ver":ver, "filename":filename, + "filepath": filepath} + + return pkg_dict + + """ + Returns the path to a tmpdir where resides the contents of a package. + + Deleting the tmpdir is responsability of the caller. + + """ + def extract(self, pkg): + pkg_info = self.package_info(pkg) + if not pkg_info: + bb.fatal("Unable to get information for package '%s' while " + "trying to extract the package." % pkg) + + pkg_path = pkg_info[pkg]["filepath"] + + cpio_cmd = bb.utils.which(os.getenv("PATH"), "cpio") + rpm2cpio_cmd = bb.utils.which(os.getenv("PATH"), "rpm2cpio") + + if not os.path.isfile(pkg_path): + bb.fatal("Unable to extract package for '%s'." + "File %s doesn't exists" % (pkg, pkg_path)) + + tmp_dir = tempfile.mkdtemp() + current_dir = os.getcwd() + os.chdir(tmp_dir) + + try: + cmd = "%s %s | %s -idmv" % (rpm2cpio_cmd, pkg_path, cpio_cmd) + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) + except subprocess.CalledProcessError as e: + bb.utils.remove(tmp_dir, recurse=True) + bb.fatal("Unable to extract %s package. Command '%s' " + "returned %d:\n%s" % (pkg_path, cmd, e.returncode, e.output.decode("utf-8"))) + except OSError as e: + bb.utils.remove(tmp_dir, recurse=True) + bb.fatal("Unable to extract %s package. Command '%s' " + "returned %d:\n%s at %s" % (pkg_path, cmd, e.errno, e.strerror, e.filename)) + + bb.note("Extracted %s to %s" % (pkg_path, tmp_dir)) + os.chdir(current_dir) + + return tmp_dir + + +class OpkgDpkgPM(PackageManager): + """ + This is an abstract class. Do not instantiate this directly. + """ + def __init__(self, d): + super(OpkgDpkgPM, self).__init__(d) + + """ + Returns a dictionary with the package info. + + This method extracts the common parts for Opkg and Dpkg + """ + def package_info(self, pkg, cmd): + + try: + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8") + except subprocess.CalledProcessError as e: + bb.fatal("Unable to list available packages. Command '%s' " + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) + return opkg_query(output) + + """ + Returns the path to a tmpdir where resides the contents of a package. + + Deleting the tmpdir is responsability of the caller. + + This method extracts the common parts for Opkg and Dpkg + """ + def extract(self, pkg, pkg_info): + + ar_cmd = bb.utils.which(os.getenv("PATH"), "ar") + tar_cmd = bb.utils.which(os.getenv("PATH"), "tar") + pkg_path = pkg_info[pkg]["filepath"] + + if not os.path.isfile(pkg_path): + bb.fatal("Unable to extract package for '%s'." + "File %s doesn't exists" % (pkg, pkg_path)) + + tmp_dir = tempfile.mkdtemp() + current_dir = os.getcwd() + os.chdir(tmp_dir) + + try: + cmd = "%s x %s" % (ar_cmd, pkg_path) + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) + cmd = "%s xf data.tar.*" % tar_cmd + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) + except subprocess.CalledProcessError as e: + bb.utils.remove(tmp_dir, recurse=True) + bb.fatal("Unable to extract %s package. Command '%s' " + "returned %d:\n%s" % (pkg_path, cmd, e.returncode, e.output.decode("utf-8"))) + except OSError as e: + bb.utils.remove(tmp_dir, recurse=True) + bb.fatal("Unable to extract %s package. Command '%s' " + "returned %d:\n%s at %s" % (pkg_path, cmd, e.errno, e.strerror, e.filename)) + + bb.note("Extracted %s to %s" % (pkg_path, tmp_dir)) + bb.utils.remove(os.path.join(tmp_dir, "debian-binary")) + bb.utils.remove(os.path.join(tmp_dir, "control.tar.gz")) + os.chdir(current_dir) -class OpkgPM(PackageManager): + return tmp_dir + + +class OpkgPM(OpkgDpkgPM): def __init__(self, d, target_rootfs, config_file, archs, task_name='target'): super(OpkgPM, self).__init__(d) @@ -1411,7 +1583,7 @@ class OpkgPM(PackageManager): self.deploy_dir = self.d.getVar("DEPLOY_DIR_IPK", True) self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock") self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg") - self.opkg_args = "--volatile-cache -f %s -o %s " % (self.config_file, target_rootfs) + self.opkg_args = "--volatile-cache -f %s -t %s -o %s " % (self.config_file, self.d.expand('${T}/ipktemp/') ,target_rootfs) self.opkg_args += self.d.getVar("OPKG_ARGS", True) opkg_lib_dir = self.d.getVar('OPKGLIBDIR', True) @@ -1552,12 +1724,12 @@ class OpkgPM(PackageManager): for arch in archs: if (self.feed_archs is None) and (not os.path.exists(os.path.join(self.deploy_dir, arch))): continue - bb.note('Note: adding opkg feed url-%s-%d (%s)' % + bb.note('Adding opkg feed url-%s-%d (%s)' % (arch, uri_iterator, uri)) config_file.write("src/gz uri-%s-%d %s/%s\n" % (arch, uri_iterator, uri, arch)) else: - bb.note('Note: adding opkg feed url-%d (%s)' % + bb.note('Adding opkg feed url-%d (%s)' % (uri_iterator, uri)) config_file.write("src/gz uri-%d %s\n" % (uri_iterator, uri)) @@ -1574,7 +1746,7 @@ class OpkgPM(PackageManager): except subprocess.CalledProcessError as e: self.deploy_dir_unlock() bb.fatal("Unable to update the package index files. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) self.deploy_dir_unlock() @@ -1595,12 +1767,12 @@ class OpkgPM(PackageManager): try: bb.note("Installing the following packages: %s" % ' '.join(pkgs)) bb.note(cmd) - output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) + output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT).decode("utf-8") bb.note(output) except subprocess.CalledProcessError as e: (bb.fatal, bb.note)[attempt_only]("Unable to install packages. " "Command '%s' returned %d:\n%s" % - (cmd, e.returncode, e.output)) + (cmd, e.returncode, e.output.decode("utf-8"))) def remove(self, pkgs, with_dependencies=True): if with_dependencies: @@ -1612,11 +1784,11 @@ class OpkgPM(PackageManager): try: bb.note(cmd) - output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) + output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT).decode("utf-8") bb.note(output) except subprocess.CalledProcessError as e: bb.fatal("Unable to remove packages. Command '%s' " - "returned %d:\n%s" % (e.cmd, e.returncode, e.output)) + "returned %d:\n%s" % (e.cmd, e.returncode, e.output.decode("utf-8"))) def write_index(self): self.deploy_dir_lock() @@ -1659,10 +1831,10 @@ class OpkgPM(PackageManager): pkg_info = cmd + pkg try: - output = subprocess.check_output(pkg_info.split(), stderr=subprocess.STDOUT).strip() + output = subprocess.check_output(pkg_info.split(), stderr=subprocess.STDOUT).strip().decode("utf-8") except subprocess.CalledProcessError as e: bb.fatal("Cannot get package info. Command '%s' " - "returned %d:\n%s" % (pkg_info, e.returncode, e.output)) + "returned %d:\n%s" % (pkg_info, e.returncode, e.output.decode("utf-8"))) if output == "": bb.note("Ignored bad recommendation: '%s' is " @@ -1699,7 +1871,7 @@ class OpkgPM(PackageManager): subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError as e: bb.fatal("Unable to update. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) # Dummy installation cmd = "%s %s --noaction install %s " % (self.opkg_cmd, @@ -1709,7 +1881,7 @@ class OpkgPM(PackageManager): output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError as e: bb.fatal("Unable to dummy install packages. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) bb.utils.remove(temp_rootfs, True) @@ -1734,8 +1906,37 @@ class OpkgPM(PackageManager): self.opkg_dir, symlinks=True) + """ + Returns a dictionary with the package info. + """ + def package_info(self, pkg): + cmd = "%s %s info %s" % (self.opkg_cmd, self.opkg_args, pkg) + pkg_info = super(OpkgPM, self).package_info(pkg, cmd) + + pkg_arch = pkg_info[pkg]["arch"] + pkg_filename = pkg_info[pkg]["filename"] + pkg_info[pkg]["filepath"] = \ + os.path.join(self.deploy_dir, pkg_arch, pkg_filename) + + return pkg_info + + """ + Returns the path to a tmpdir where resides the contents of a package. + + Deleting the tmpdir is responsability of the caller. + """ + def extract(self, pkg): + pkg_info = self.package_info(pkg) + if not pkg_info: + bb.fatal("Unable to get information for package '%s' while " + "trying to extract the package." % pkg) + + tmp_dir = super(OpkgPM, self).extract(pkg, pkg_info) + bb.utils.remove(os.path.join(tmp_dir, "data.tar.gz")) -class DpkgPM(PackageManager): + return tmp_dir + +class DpkgPM(OpkgDpkgPM): def __init__(self, d, target_rootfs, archs, base_archs, apt_conf_dir=None): super(DpkgPM, self).__init__(d) self.target_rootfs = target_rootfs @@ -1746,6 +1947,7 @@ class DpkgPM(PackageManager): self.apt_conf_dir = apt_conf_dir self.apt_conf_file = os.path.join(self.apt_conf_dir, "apt.conf") self.apt_get_cmd = bb.utils.which(os.getenv('PATH'), "apt-get") + self.apt_cache_cmd = bb.utils.which(os.getenv('PATH'), "apt-cache") self.apt_args = d.getVar("APT_ARGS", True) @@ -1823,7 +2025,7 @@ class DpkgPM(PackageManager): subprocess.check_output(p_full, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: bb.note("%s for package %s failed with %d:\n%s" % - (suffix[1], pkg_name, e.returncode, e.output)) + (suffix[1], pkg_name, e.returncode, e.output.decode("utf-8"))) failed_pkgs.append(pkg_name) break @@ -1841,7 +2043,7 @@ class DpkgPM(PackageManager): subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: bb.fatal("Unable to update the package index files. Command '%s' " - "returned %d:\n%s" % (e.cmd, e.returncode, e.output)) + "returned %d:\n%s" % (e.cmd, e.returncode, e.output.decode("utf-8"))) self.deploy_dir_unlock() @@ -1860,7 +2062,7 @@ class DpkgPM(PackageManager): except subprocess.CalledProcessError as e: (bb.fatal, bb.note)[attempt_only]("Unable to install packages. " "Command '%s' returned %d:\n%s" % - (cmd, e.returncode, e.output)) + (cmd, e.returncode, e.output.decode("utf-8"))) # rename *.dpkg-new files/dirs for root, dirs, files in os.walk(self.target_rootfs): @@ -1891,7 +2093,7 @@ class DpkgPM(PackageManager): subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: bb.fatal("Unable to remove packages. Command '%s' " - "returned %d:\n%s" % (e.cmd, e.returncode, e.output)) + "returned %d:\n%s" % (e.cmd, e.returncode, e.output.decode("utf-8"))) def write_index(self): self.deploy_dir_lock() @@ -1925,11 +2127,11 @@ class DpkgPM(PackageManager): for uri in feed_uris: if arch_list: for arch in arch_list: - bb.note('Note: adding dpkg channel at (%s)' % uri) + bb.note('Adding dpkg channel at (%s)' % uri) sources_file.write("deb %s/%s ./\n" % (uri, arch)) else: - bb.note('Note: adding dpkg channel at (%s)' % uri) + bb.note('Adding dpkg channel at (%s)' % uri) sources_file.write("deb %s ./\n" % uri) def _create_configs(self, archs, base_archs): @@ -2024,11 +2226,40 @@ class DpkgPM(PackageManager): subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: bb.fatal("Cannot fix broken dependencies. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) + "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) def list_installed(self): return DpkgPkgsList(self.d, self.target_rootfs).list_pkgs() + """ + Returns a dictionary with the package info. + """ + def package_info(self, pkg): + cmd = "%s show %s" % (self.apt_cache_cmd, pkg) + pkg_info = super(DpkgPM, self).package_info(pkg, cmd) + + pkg_arch = pkg_info[pkg]["pkgarch"] + pkg_filename = pkg_info[pkg]["filename"] + pkg_info[pkg]["filepath"] = \ + os.path.join(self.deploy_dir, pkg_arch, pkg_filename) + + return pkg_info + + """ + Returns the path to a tmpdir where resides the contents of a package. + + Deleting the tmpdir is responsability of the caller. + """ + def extract(self, pkg): + pkg_info = self.package_info(pkg) + if not pkg_info: + bb.fatal("Unable to get information for package '%s' while " + "trying to extract the package." % pkg) + + tmp_dir = super(DpkgPM, self).extract(pkg, pkg_info) + bb.utils.remove(os.path.join(tmp_dir, "data.tar.xz")) + + return tmp_dir def generate_index_files(d): classes = d.getVar('PACKAGE_CLASSES', True).replace("package_", "").split() diff --git a/import-layers/yocto-poky/meta/lib/oe/packagedata.py b/import-layers/yocto-poky/meta/lib/oe/packagedata.py index bc0fd06bc..21d4de914 100644 --- a/import-layers/yocto-poky/meta/lib/oe/packagedata.py +++ b/import-layers/yocto-poky/meta/lib/oe/packagedata.py @@ -8,7 +8,7 @@ def read_pkgdatafile(fn): pkgdata = {} def decode(str): - c = codecs.getdecoder("string_escape") + c = codecs.getdecoder("unicode_escape") return c(str)[0] if os.access(fn, os.R_OK): @@ -66,7 +66,7 @@ def _pkgmap(d): bb.warn("No files in %s?" % pkgdatadir) files = [] - for pn in filter(lambda f: not os.path.isdir(os.path.join(pkgdatadir, f)), files): + for pn in [f for f in files if not os.path.isdir(os.path.join(pkgdatadir, f))]: try: pkgdata = read_pkgdatafile(os.path.join(pkgdatadir, pn)) except OSError: diff --git a/import-layers/yocto-poky/meta/lib/oe/packagegroup.py b/import-layers/yocto-poky/meta/lib/oe/packagegroup.py index a6fee5f95..97819279b 100644 --- a/import-layers/yocto-poky/meta/lib/oe/packagegroup.py +++ b/import-layers/yocto-poky/meta/lib/oe/packagegroup.py @@ -16,11 +16,11 @@ def packages(features, d): yield pkg def required_packages(features, d): - req = filter(lambda feature: not is_optional(feature, d), features) + req = [feature for feature in features if not is_optional(feature, d)] return packages(req, d) def optional_packages(features, d): - opt = filter(lambda feature: is_optional(feature, d), features) + opt = [feature for feature in features if is_optional(feature, d)] return packages(opt, d) def active_packages(features, d): diff --git a/import-layers/yocto-poky/meta/lib/oe/patch.py b/import-layers/yocto-poky/meta/lib/oe/patch.py index 9d3617290..0332f100f 100644 --- a/import-layers/yocto-poky/meta/lib/oe/patch.py +++ b/import-layers/yocto-poky/meta/lib/oe/patch.py @@ -117,43 +117,50 @@ class PatchSet(object): return None return os.sep.join(filesplit[striplevel:]) - copiedmode = False - filelist = [] - with open(patchfile) as f: - for line in f: - if line.startswith('--- '): - patchpth = patchedpath(line) - if not patchpth: - break - if copiedmode: - addedfile = patchpth - else: - removedfile = patchpth - elif line.startswith('+++ '): - addedfile = patchedpath(line) - if not addedfile: - break - elif line.startswith('*** '): - copiedmode = True - removedfile = patchedpath(line) - if not removedfile: - break - else: - removedfile = None - addedfile = None - - if addedfile and removedfile: - if removedfile == '/dev/null': - mode = 'A' - elif addedfile == '/dev/null': - mode = 'D' - else: - mode = 'M' - if srcdir: - fullpath = os.path.abspath(os.path.join(srcdir, addedfile)) - else: - fullpath = addedfile - filelist.append((fullpath, mode)) + for encoding in ['utf-8', 'latin-1']: + try: + copiedmode = False + filelist = [] + with open(patchfile) as f: + for line in f: + if line.startswith('--- '): + patchpth = patchedpath(line) + if not patchpth: + break + if copiedmode: + addedfile = patchpth + else: + removedfile = patchpth + elif line.startswith('+++ '): + addedfile = patchedpath(line) + if not addedfile: + break + elif line.startswith('*** '): + copiedmode = True + removedfile = patchedpath(line) + if not removedfile: + break + else: + removedfile = None + addedfile = None + + if addedfile and removedfile: + if removedfile == '/dev/null': + mode = 'A' + elif addedfile == '/dev/null': + mode = 'D' + else: + mode = 'M' + if srcdir: + fullpath = os.path.abspath(os.path.join(srcdir, addedfile)) + else: + fullpath = addedfile + filelist.append((fullpath, mode)) + except UnicodeDecodeError: + continue + break + else: + raise PatchError('Unable to decode %s' % patchfile) return filelist @@ -274,30 +281,43 @@ class GitApplyTree(PatchTree): def __init__(self, dir, d): PatchTree.__init__(self, dir, d) + self.commituser = d.getVar('PATCH_GIT_USER_NAME', True) + self.commitemail = d.getVar('PATCH_GIT_USER_EMAIL', True) @staticmethod def extractPatchHeader(patchfile): """ Extract just the header lines from the top of a patch file """ - lines = [] - with open(patchfile, 'r') as f: - for line in f.readlines(): - if line.startswith('Index: ') or line.startswith('diff -') or line.startswith('---'): - break - lines.append(line) + for encoding in ['utf-8', 'latin-1']: + lines = [] + try: + with open(patchfile, 'r', encoding=encoding) as f: + for line in f: + if line.startswith('Index: ') or line.startswith('diff -') or line.startswith('---'): + break + lines.append(line) + except UnicodeDecodeError: + continue + break + else: + raise PatchError('Unable to find a character encoding to decode %s' % patchfile) return lines @staticmethod def decodeAuthor(line): from email.header import decode_header authorval = line.split(':', 1)[1].strip().replace('"', '') - return decode_header(authorval)[0][0] + result = decode_header(authorval)[0][0] + if hasattr(result, 'decode'): + result = result.decode('utf-8') + return result @staticmethod def interpretPatchHeader(headerlines): import re author_re = re.compile('[\S ]+ <\S+@\S+\.\S+>') + from_commit_re = re.compile('^From [a-z0-9]{40} .*') outlines = [] author = None date = None @@ -327,11 +347,39 @@ class GitApplyTree(PatchTree): # git is fussy about author formatting i.e. it must be Name <email@domain> if author_re.match(authorval): author = authorval + elif from_commit_re.match(line): + # We don't want the From <commit> line - if it's present it will break rebasing + continue outlines.append(line) + + if not subject: + firstline = None + for line in headerlines: + line = line.strip() + if firstline: + if line: + # Second line is not blank, the first line probably isn't usable + firstline = None + break + elif line: + firstline = line + if firstline and not firstline.startswith(('#', 'Index:', 'Upstream-Status:')) and len(firstline) < 100: + subject = firstline + return outlines, author, date, subject @staticmethod - def prepareCommit(patchfile): + def gitCommandUserOptions(cmd, commituser=None, commitemail=None, d=None): + if d: + commituser = d.getVar('PATCH_GIT_USER_NAME', True) + commitemail = d.getVar('PATCH_GIT_USER_EMAIL', True) + if commituser: + cmd += ['-c', 'user.name="%s"' % commituser] + if commitemail: + cmd += ['-c', 'user.email="%s"' % commitemail] + + @staticmethod + def prepareCommit(patchfile, commituser=None, commitemail=None): """ Prepare a git commit command line based on the header from a patch file (typically this is useful for patches that cannot be applied with "git am" due to formatting) @@ -340,21 +388,24 @@ class GitApplyTree(PatchTree): # Process patch header and extract useful information lines = GitApplyTree.extractPatchHeader(patchfile) outlines, author, date, subject = GitApplyTree.interpretPatchHeader(lines) - if not author or not subject: + if not author or not subject or not date: try: - shellcmd = ["git", "log", "--format=email", "--diff-filter=A", "--", patchfile] + shellcmd = ["git", "log", "--format=email", "--follow", "--diff-filter=A", "--", patchfile] out = runcmd(["sh", "-c", " ".join(shellcmd)], os.path.dirname(patchfile)) except CmdError: out = None if out: _, newauthor, newdate, newsubject = GitApplyTree.interpretPatchHeader(out.splitlines()) - if not author or not date: - # These really need to go together + if not author: + # If we're setting the author then the date should be set as well author = newauthor date = newdate + elif not date: + # If we don't do this we'll get the current date, at least this will be closer + date = newdate if not subject: subject = newsubject - if subject: + if subject and outlines and not outlines[0].strip() == subject: outlines.insert(0, '%s\n\n' % subject.strip()) # Write out commit message to a file @@ -363,7 +414,9 @@ class GitApplyTree(PatchTree): for line in outlines: tf.write(line) # Prepare git command - cmd = ["git", "commit", "-F", tmpfile] + cmd = ["git"] + GitApplyTree.gitCommandUserOptions(cmd, commituser, commitemail) + cmd += ["commit", "-F", tmpfile] # git doesn't like plain email addresses as authors if author and '<' in author: cmd.append('--author="%s"' % author) @@ -384,16 +437,24 @@ class GitApplyTree(PatchTree): out = runcmd(["sh", "-c", " ".join(shellcmd)], tree) if out: for srcfile in out.split(): - patchlines = [] - outfile = None - with open(srcfile, 'r') as f: - for line in f: - if line.startswith(GitApplyTree.patch_line_prefix): - outfile = line.split()[-1].strip() - continue - if line.startswith(GitApplyTree.ignore_commit_prefix): - continue - patchlines.append(line) + for encoding in ['utf-8', 'latin-1']: + patchlines = [] + outfile = None + try: + with open(srcfile, 'r', encoding=encoding) as f: + for line in f: + if line.startswith(GitApplyTree.patch_line_prefix): + outfile = line.split()[-1].strip() + continue + if line.startswith(GitApplyTree.ignore_commit_prefix): + continue + patchlines.append(line) + except UnicodeDecodeError: + continue + break + else: + raise PatchError('Unable to find a character encoding to decode %s' % srcfile) + if not outfile: outfile = os.path.basename(srcfile) with open(os.path.join(outdir, outfile), 'w') as of: @@ -434,12 +495,14 @@ class GitApplyTree(PatchTree): # change other places which read it back f.write('echo >> $1\n') f.write('echo "%s: $PATCHFILE" >> $1\n' % GitApplyTree.patch_line_prefix) - os.chmod(commithook, 0755) + os.chmod(commithook, 0o755) shutil.copy2(commithook, applyhook) try: patchfilevar = 'PATCHFILE="%s"' % os.path.basename(patch['file']) try: - shellcmd = [patchfilevar, "git", "--work-tree=%s" % reporoot, "am", "-3", "--keep-cr", "-p%s" % patch['strippath']] + shellcmd = [patchfilevar, "git", "--work-tree=%s" % reporoot] + self.gitCommandUserOptions(shellcmd, self.commituser, self.commitemail) + shellcmd += ["am", "-3", "--keep-cr", "-p%s" % patch['strippath']] return _applypatchhelper(shellcmd, patch, force, reverse, run) except CmdError: # Need to abort the git am, or we'll still be within it at the end @@ -469,7 +532,7 @@ class GitApplyTree(PatchTree): shellcmd = ["git", "reset", "HEAD", self.patchdir] output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) # Commit the result - (tmpfile, shellcmd) = self.prepareCommit(patch['file']) + (tmpfile, shellcmd) = self.prepareCommit(patch['file'], self.commituser, self.commitemail) try: shellcmd.insert(0, patchfilevar) output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) @@ -672,7 +735,7 @@ class UserResolver(Resolver): f.write("echo 'Run \"quilt refresh\" when patch is corrected, press CTRL+D to exit.'\n") f.write("echo ''\n") f.write(" ".join(patchcmd) + "\n") - os.chmod(rcfile, 0775) + os.chmod(rcfile, 0o775) self.terminal("bash --rcfile " + rcfile, 'Patch Rejects: Please fix patch rejects manually', self.patchset.d) diff --git a/import-layers/yocto-poky/meta/lib/oe/path.py b/import-layers/yocto-poky/meta/lib/oe/path.py index 413ebfb39..06a5af265 100644 --- a/import-layers/yocto-poky/meta/lib/oe/path.py +++ b/import-layers/yocto-poky/meta/lib/oe/path.py @@ -65,22 +65,30 @@ def copytree(src, dst): # This way we also preserve hardlinks between files in the tree. bb.utils.mkdirhier(dst) - cmd = 'tar -cf - -C %s -p . | tar -xf - -C %s' % (src, dst) - check_output(cmd, shell=True, stderr=subprocess.STDOUT) + cmd = "tar --xattrs --xattrs-include='*' -cf - -C %s -p . | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dst) + subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) def copyhardlinktree(src, dst): """ Make the hard link when possible, otherwise copy. """ bb.utils.mkdirhier(dst) if os.path.isdir(src) and not len(os.listdir(src)): - return + return if (os.stat(src).st_dev == os.stat(dst).st_dev): # Need to copy directories only with tar first since cp will error if two # writers try and create a directory at the same time - cmd = 'cd %s; find . -type d -print | tar -cf - -C %s -p --files-from - --no-recursion | tar -xf - -C %s' % (src, src, dst) - check_output(cmd, shell=True, stderr=subprocess.STDOUT) - cmd = 'cd %s; find . -print0 | cpio --null -pdlu %s' % (src, dst) - check_output(cmd, shell=True, stderr=subprocess.STDOUT) + cmd = "cd %s; find . -type d -print | tar --xattrs --xattrs-include='*' -cf - -C %s -p --no-recursion --files-from - | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, src, dst) + subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) + source = '' + if os.path.isdir(src): + import glob + if len(glob.glob('%s/.??*' % src)) > 0: + source = '%s/.??* ' % src + source = source + '%s/*' % src + else: + source = src + cmd = 'cp -afl --preserve=xattr %s %s' % (source, dst) + subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) else: copytree(src, dst) @@ -105,47 +113,6 @@ def symlink(source, destination, force=False): if e.errno != errno.EEXIST or os.readlink(destination) != source: raise -class CalledProcessError(Exception): - def __init__(self, retcode, cmd, output = None): - self.retcode = retcode - self.cmd = cmd - self.output = output - def __str__(self): - return "Command '%s' returned non-zero exit status %d with output %s" % (self.cmd, self.retcode, self.output) - -# Not needed when we move to python 2.7 -def check_output(*popenargs, **kwargs): - r"""Run command with arguments and return its output as a byte string. - - If the exit code was non-zero it raises a CalledProcessError. The - CalledProcessError object will have the return code in the returncode - attribute and output in the output attribute. - - The arguments are the same as for the Popen constructor. Example: - - >>> check_output(["ls", "-l", "/dev/null"]) - 'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' - - The stdout argument is not allowed as it is used internally. - To capture standard error in the result, use stderr=STDOUT. - - >>> check_output(["/bin/sh", "-c", - ... "ls -l non_existent_file ; exit 0"], - ... stderr=STDOUT) - 'ls: non_existent_file: No such file or directory\n' - """ - if 'stdout' in kwargs: - raise ValueError('stdout argument not allowed, it will be overridden.') - process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) - output, unused_err = process.communicate() - retcode = process.poll() - if retcode: - cmd = kwargs.get("args") - if cmd is None: - cmd = popenargs[0] - raise CalledProcessError(retcode, cmd, output=output) - return output - def find(dir, **walkoptions): """ Given a directory, recurses into that directory, returning all files as absolute paths. """ diff --git a/import-layers/yocto-poky/meta/lib/oe/prservice.py b/import-layers/yocto-poky/meta/lib/oe/prservice.py index b0cbcb1fb..0054f954c 100644 --- a/import-layers/yocto-poky/meta/lib/oe/prservice.py +++ b/import-layers/yocto-poky/meta/lib/oe/prservice.py @@ -1,7 +1,7 @@ def prserv_make_conn(d, check = False): import prserv.serv - host_params = filter(None, (d.getVar("PRSERV_HOST", True) or '').split(':')) + host_params = list([_f for _f in (d.getVar("PRSERV_HOST", True) or '').split(':') if _f]) try: conn = None conn = prserv.serv.PRServerConnection(host_params[0], int(host_params[1])) @@ -9,7 +9,7 @@ def prserv_make_conn(d, check = False): if not conn.ping(): raise Exception('service not available') d.setVar("__PRSERV_CONN",conn) - except Exception, exc: + except Exception as exc: bb.fatal("Connecting to PR service %s:%s failed: %s" % (host_params[0], host_params[1], str(exc))) return conn @@ -114,7 +114,7 @@ def prserv_export_tofile(d, metainfo, datainfo, lockdown, nomax=False): bb.utils.unlockfile(lf) def prserv_check_avail(d): - host_params = filter(None, (d.getVar("PRSERV_HOST", True) or '').split(':')) + host_params = list([_f for _f in (d.getVar("PRSERV_HOST", True) or '').split(':') if _f]) try: if len(host_params) != 2: raise TypeError diff --git a/import-layers/yocto-poky/meta/lib/oe/qa.py b/import-layers/yocto-poky/meta/lib/oe/qa.py index 3cfeee737..fbe719d8e 100644 --- a/import-layers/yocto-poky/meta/lib/oe/qa.py +++ b/import-layers/yocto-poky/meta/lib/oe/qa.py @@ -43,48 +43,48 @@ class ELFFile: if not os.path.isfile(self.name): raise NotELFFileError("%s is not a normal file" % self.name) - self.file = file(self.name, "r") - # Read 4k which should cover most of the headers we're after - self.data = self.file.read(4096) + with open(self.name, "rb") as f: + # Read 4k which should cover most of the headers we're after + self.data = f.read(4096) if len(self.data) < ELFFile.EI_NIDENT + 4: raise NotELFFileError("%s is not an ELF" % self.name) - self.my_assert(self.data[0], chr(0x7f) ) - self.my_assert(self.data[1], 'E') - self.my_assert(self.data[2], 'L') - self.my_assert(self.data[3], 'F') + self.my_assert(self.data[0], 0x7f) + self.my_assert(self.data[1], ord('E')) + self.my_assert(self.data[2], ord('L')) + self.my_assert(self.data[3], ord('F')) if self.bits == 0: - if self.data[ELFFile.EI_CLASS] == chr(ELFFile.ELFCLASS32): + if self.data[ELFFile.EI_CLASS] == ELFFile.ELFCLASS32: self.bits = 32 - elif self.data[ELFFile.EI_CLASS] == chr(ELFFile.ELFCLASS64): + elif self.data[ELFFile.EI_CLASS] == ELFFile.ELFCLASS64: self.bits = 64 else: # Not 32-bit or 64.. lets assert raise NotELFFileError("ELF but not 32 or 64 bit.") elif self.bits == 32: - self.my_assert(self.data[ELFFile.EI_CLASS], chr(ELFFile.ELFCLASS32)) + self.my_assert(self.data[ELFFile.EI_CLASS], ELFFile.ELFCLASS32) elif self.bits == 64: - self.my_assert(self.data[ELFFile.EI_CLASS], chr(ELFFile.ELFCLASS64)) + self.my_assert(self.data[ELFFile.EI_CLASS], ELFFile.ELFCLASS64) else: raise NotELFFileError("Must specify unknown, 32 or 64 bit size.") - self.my_assert(self.data[ELFFile.EI_VERSION], chr(ELFFile.EV_CURRENT) ) + self.my_assert(self.data[ELFFile.EI_VERSION], ELFFile.EV_CURRENT) self.sex = self.data[ELFFile.EI_DATA] - if self.sex == chr(ELFFile.ELFDATANONE): + if self.sex == ELFFile.ELFDATANONE: raise NotELFFileError("self.sex == ELFDATANONE") - elif self.sex == chr(ELFFile.ELFDATA2LSB): + elif self.sex == ELFFile.ELFDATA2LSB: self.sex = "<" - elif self.sex == chr(ELFFile.ELFDATA2MSB): + elif self.sex == ELFFile.ELFDATA2MSB: self.sex = ">" else: raise NotELFFileError("Unknown self.sex") def osAbi(self): - return ord(self.data[ELFFile.EI_OSABI]) + return self.data[ELFFile.EI_OSABI] def abiVersion(self): - return ord(self.data[ELFFile.EI_ABIVERSION]) + return self.data[ELFFile.EI_ABIVERSION] def abiSize(self): return self.bits @@ -144,8 +144,28 @@ class ELFFile: bb.note("%s %s %s failed: %s" % (objdump, cmd, self.name, e)) return "" +def elf_machine_to_string(machine): + """ + Return the name of a given ELF e_machine field or the hex value as a string + if it isn't recognised. + """ + try: + return { + 0x02: "SPARC", + 0x03: "x86", + 0x08: "MIPS", + 0x14: "PowerPC", + 0x28: "ARM", + 0x2A: "SuperH", + 0x32: "IA-64", + 0x3E: "x86-64", + 0xB7: "AArch64" + }[machine] + except: + return "Unknown (%s)" % repr(machine) + if __name__ == "__main__": import sys elf = ELFFile(sys.argv[1]) elf.open() - print elf.isDynamic() + print(elf.isDynamic()) diff --git a/import-layers/yocto-poky/meta/lib/oe/recipeutils.py b/import-layers/yocto-poky/meta/lib/oe/recipeutils.py index 6c7adb5bd..58e4028ae 100644 --- a/import-layers/yocto-poky/meta/lib/oe/recipeutils.py +++ b/import-layers/yocto-poky/meta/lib/oe/recipeutils.py @@ -2,7 +2,7 @@ # # Some code borrowed from the OE layer index # -# Copyright (C) 2013-2015 Intel Corporation +# Copyright (C) 2013-2016 Intel Corporation # import sys @@ -11,31 +11,32 @@ import os.path import tempfile import textwrap import difflib -import utils +from . import utils import shutil import re import fnmatch +import glob from collections import OrderedDict, defaultdict # Help us to find places to insert values -recipe_progression = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRCREV', 'SRC_URI', 'S', 'do_fetch()', 'do_unpack()', 'do_patch()', 'EXTRA_OECONF', 'do_configure()', 'EXTRA_OEMAKE', 'do_compile()', 'do_install()', 'do_populate_sysroot()', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'do_package()', 'do_deploy()'] +recipe_progression = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LICENSE_FLAGS', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRCREV', 'SRCPV', 'SRC_URI', 'S', 'do_fetch()', 'do_unpack()', 'do_patch()', 'EXTRA_OECONF', 'EXTRA_OECMAKE', 'EXTRA_OESCONS', 'do_configure()', 'EXTRA_OEMAKE', 'do_compile()', 'do_install()', 'do_populate_sysroot()', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'populate_packages()', 'do_package()', 'do_deploy()'] # Variables that sometimes are a bit long but shouldn't be wrapped nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER', 'SRC_URI[md5sum]', 'SRC_URI[sha256sum]'] list_vars = ['SRC_URI', 'LIC_FILES_CHKSUM'] meta_vars = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION'] -def pn_to_recipe(cooker, pn): +def pn_to_recipe(cooker, pn, mc=''): """Convert a recipe name (PN) to the path to the recipe file""" import bb.providers - if pn in cooker.recipecache.pkg_pn: - best = bb.providers.findBestProvider(pn, cooker.data, cooker.recipecache, cooker.recipecache.pkg_pn) + if pn in cooker.recipecaches[mc].pkg_pn: + best = bb.providers.findBestProvider(pn, cooker.data, cooker.recipecaches[mc], cooker.recipecaches[mc].pkg_pn) return best[3] - elif pn in cooker.recipecache.providers: - filenames = cooker.recipecache.providers[pn] - eligible, foundUnique = bb.providers.filterProviders(filenames, pn, cooker.expanded_data, cooker.recipecache) + elif pn in cooker.recipecaches[mc].providers: + filenames = cooker.recipecaches[mc].providers[pn] + eligible, foundUnique = bb.providers.filterProviders(filenames, pn, cooker.expanded_data, cooker.recipecaches[mc]) filename = eligible[0] return filename else: @@ -49,13 +50,14 @@ def get_unavailable_reasons(cooker, pn): return taskdata.get_reasons(pn) -def parse_recipe(fn, appendfiles, d): +def parse_recipe(cooker, fn, appendfiles): """ Parse an individual recipe file, optionally with a list of bbappend files. """ import bb.cache - envdata = bb.cache.Cache.loadDataFull(fn, appendfiles, d) + parser = bb.cache.NoCache(cooker.databuilder) + envdata = parser.loadDataFull(fn, appendfiles) return envdata @@ -78,7 +80,7 @@ def parse_recipe_simple(cooker, pn, d, appends=True): appendfiles = cooker.collection.get_file_appends(recipefile) else: appendfiles = None - return parse_recipe(recipefile, appendfiles, d) + return parse_recipe(cooker, recipefile, appendfiles) def get_var_files(fn, varlist, d): @@ -158,15 +160,19 @@ def split_var_value(value, assignment=True): return outlist -def patch_recipe_file(fn, values, patch=False, relpath=''): - """Update or insert variable values into a recipe file (assuming you - have already identified the exact file you want to update.) +def patch_recipe_lines(fromlines, values, trailing_newline=True): + """Update or insert variable values into lines from a recipe. Note that some manual inspection/intervention may be required since this cannot handle all situations. """ import bb.utils + if trailing_newline: + newline = '\n' + else: + newline = '' + recipe_progression_res = [] recipe_progression_restrs = [] for item in recipe_progression: @@ -190,14 +196,14 @@ def patch_recipe_file(fn, values, patch=False, relpath=''): remainingnames = {} for k in values.keys(): remainingnames[k] = get_recipe_pos(k) - remainingnames = OrderedDict(sorted(remainingnames.iteritems(), key=lambda x: x[1])) + remainingnames = OrderedDict(sorted(remainingnames.items(), key=lambda x: x[1])) modifying = False def outputvalue(name, lines, rewindcomments=False): if values[name] is None: return - rawtext = '%s = "%s"\n' % (name, values[name]) + rawtext = '%s = "%s"%s' % (name, values[name], newline) addlines = [] if name in nowrap_vars: addlines.append(rawtext) @@ -205,19 +211,19 @@ def patch_recipe_file(fn, values, patch=False, relpath=''): splitvalue = split_var_value(values[name], assignment=False) if len(splitvalue) > 1: linesplit = ' \\\n' + (' ' * (len(name) + 4)) - addlines.append('%s = "%s%s"\n' % (name, linesplit.join(splitvalue), linesplit)) + addlines.append('%s = "%s%s"%s' % (name, linesplit.join(splitvalue), linesplit, newline)) else: addlines.append(rawtext) else: wrapped = textwrap.wrap(rawtext) for wrapline in wrapped[:-1]: - addlines.append('%s \\\n' % wrapline) - addlines.append('%s\n' % wrapped[-1]) + addlines.append('%s \\%s' % (wrapline, newline)) + addlines.append('%s%s' % (wrapped[-1], newline)) if rewindcomments: # Ensure we insert the lines before any leading comments # (that we'd want to ensure remain leading the next value) for i, ln in reversed(list(enumerate(lines))): - if ln[0] != '#': + if not ln.startswith('#'): lines[i+1:i+1] = addlines break else: @@ -230,7 +236,7 @@ def patch_recipe_file(fn, values, patch=False, relpath=''): if modifying: # Insert anything that should come before this variable pos = get_recipe_pos(varname) - for k in remainingnames.keys()[:]: + for k in list(remainingnames): if remainingnames[k] > -1 and pos >= remainingnames[k] and not k in existingnames: outputvalue(k, newlines, rewindcomments=True) del remainingnames[k] @@ -247,19 +253,33 @@ def patch_recipe_file(fn, values, patch=False, relpath=''): # First run - establish which values we want to set are already in the file varlist = [re.escape(item) for item in values.keys()] - with open(fn, 'r') as f: - changed, fromlines = bb.utils.edit_metadata(f, varlist, patch_recipe_varfunc) + bb.utils.edit_metadata(fromlines, varlist, patch_recipe_varfunc) # Second run - actually set everything modifying = True varlist.extend(recipe_progression_restrs) changed, tolines = bb.utils.edit_metadata(fromlines, varlist, patch_recipe_varfunc, match_overrides=True) if remainingnames: - if tolines[-1].strip() != '': + if tolines and tolines[-1].strip() != '': tolines.append('\n') for k in remainingnames.keys(): outputvalue(k, tolines) + return changed, tolines + + +def patch_recipe_file(fn, values, patch=False, relpath=''): + """Update or insert variable values into a recipe file (assuming you + have already identified the exact file you want to update.) + Note that some manual inspection/intervention may be required + since this cannot handle all situations. + """ + + with open(fn, 'r') as f: + fromlines = f.readlines() + + _, tolines = patch_recipe_lines(fromlines, values) + if patch: relfn = os.path.relpath(fn, relpath) diff = difflib.unified_diff(fromlines, tolines, 'a/%s' % relfn, 'b/%s' % relfn) @@ -318,7 +338,7 @@ def patch_recipe(d, fn, varvalues, patch=False, relpath=''): varfiles = get_var_files(fn, varlist, d) locs = localise_file_vars(fn, varfiles, varlist) patches = [] - for f,v in locs.iteritems(): + for f,v in locs.items(): vals = {k: varvalues[k] for k in v} patchdata = patch_recipe_file(f, vals, patch, relpath) if patch: @@ -347,6 +367,7 @@ def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True): # Copy local files to target directory and gather any remote files bb_dir = os.path.dirname(d.getVar('FILE', True)) + os.sep remotes = [] + copied = [] includes = [path for path in d.getVar('BBINCLUDED', True).split() if path.startswith(bb_dir) and os.path.exists(path)] for path in fetch.localpaths() + includes: @@ -358,13 +379,14 @@ def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True): if not os.path.exists(subdir): os.makedirs(subdir) shutil.copy2(path, os.path.join(tgt_dir, relpath)) + copied.append(relpath) else: remotes.append(path) # Simply copy whole meta dir, if requested if whole_dir: shutil.copytree(bb_dir, tgt_dir) - return remotes + return copied, remotes def get_recipe_local_files(d, patches=False): @@ -378,8 +400,16 @@ def get_recipe_local_files(d, patches=False): bb.utils.exec_flat_python_func('patch_path', uri, fetch, '')): continue # Skip files that are referenced by absolute path - if not os.path.isabs(fetch.ud[uri].basepath): - ret[fetch.ud[uri].basepath] = fetch.localpath(uri) + fname = fetch.ud[uri].basepath + if os.path.isabs(fname): + continue + # Handle subdir= + subdir = fetch.ud[uri].parm.get('subdir', '') + if subdir: + if os.path.isabs(subdir): + continue + fname = os.path.join(subdir, fname) + ret[fname] = fetch.localpath(uri) return ret @@ -419,8 +449,8 @@ def get_recipe_patched_files(d): def validate_pn(pn): """Perform validation on a recipe name (PN) for a new recipe.""" reserved_names = ['forcevariable', 'append', 'prepend', 'remove'] - if not re.match('[0-9a-z-.]+', pn): - return 'Recipe name "%s" is invalid: only characters 0-9, a-z, - and . are allowed' % pn + if not re.match('^[0-9a-z-.+]+$', pn): + return 'Recipe name "%s" is invalid: only characters 0-9, a-z, -, + and . are allowed' % pn elif pn in reserved_names: return 'Recipe name "%s" is invalid: is a reserved keyword' % pn elif pn.startswith('pn-'): @@ -430,6 +460,60 @@ def validate_pn(pn): return '' +def get_bbfile_path(d, destdir, extrapathhint=None): + """ + Determine the correct path for a recipe within a layer + Parameters: + d: Recipe-specific datastore + destdir: destination directory. Can be the path to the base of the layer or a + partial path somewhere within the layer. + extrapathhint: a path relative to the base of the layer to try + """ + import bb.cookerdata + + destdir = os.path.abspath(destdir) + destlayerdir = find_layerdir(destdir) + + # Parse the specified layer's layer.conf file directly, in case the layer isn't in bblayers.conf + confdata = d.createCopy() + confdata.setVar('BBFILES', '') + confdata.setVar('LAYERDIR', destlayerdir) + destlayerconf = os.path.join(destlayerdir, "conf", "layer.conf") + confdata = bb.cookerdata.parse_config_file(destlayerconf, confdata) + pn = d.getVar('PN', True) + + bbfilespecs = (confdata.getVar('BBFILES', True) or '').split() + if destdir == destlayerdir: + for bbfilespec in bbfilespecs: + if not bbfilespec.endswith('.bbappend'): + for match in glob.glob(bbfilespec): + splitext = os.path.splitext(os.path.basename(match)) + if splitext[1] == '.bb': + mpn = splitext[0].split('_')[0] + if mpn == pn: + return os.path.dirname(match) + + # Try to make up a path that matches BBFILES + # this is a little crude, but better than nothing + bpn = d.getVar('BPN', True) + recipefn = os.path.basename(d.getVar('FILE', True)) + pathoptions = [destdir] + if extrapathhint: + pathoptions.append(os.path.join(destdir, extrapathhint)) + if destdir == destlayerdir: + pathoptions.append(os.path.join(destdir, 'recipes-%s' % bpn, bpn)) + pathoptions.append(os.path.join(destdir, 'recipes', bpn)) + pathoptions.append(os.path.join(destdir, bpn)) + elif not destdir.endswith(('/' + pn, '/' + bpn)): + pathoptions.append(os.path.join(destdir, bpn)) + closepath = '' + for pathoption in pathoptions: + bbfilepath = os.path.join(pathoption, 'test.bb') + for bbfilespec in bbfilespecs: + if fnmatch.fnmatchcase(bbfilepath, bbfilespec): + return pathoption + return None + def get_bbappend_path(d, destlayerdir, wildcardver=False): """Determine how a bbappend for a recipe should be named and located within another layer""" @@ -536,7 +620,7 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, bbappendlines = [] if extralines: if isinstance(extralines, dict): - for name, value in extralines.iteritems(): + for name, value in extralines.items(): bbappendlines.append((name, '=', value)) else: # Do our best to split it @@ -550,14 +634,14 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, raise Exception('Invalid extralines value passed') def popline(varname): - for i in xrange(0, len(bbappendlines)): + for i in range(0, len(bbappendlines)): if bbappendlines[i][0] == varname: line = bbappendlines.pop(i) return line return None def appendline(varname, op, value): - for i in xrange(0, len(bbappendlines)): + for i in range(0, len(bbappendlines)): item = bbappendlines[i] if item[0] == varname: bbappendlines[i] = (item[0], item[1], item[2] + ' ' + value) @@ -576,7 +660,7 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, copyfiles = {} if srcfiles: instfunclines = [] - for newfile, origsrcfile in srcfiles.iteritems(): + for newfile, origsrcfile in srcfiles.items(): srcfile = origsrcfile srcurientry = None if not srcfile: @@ -644,7 +728,7 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, if removevar in removevalues: remove = removevalues[removevar] - if isinstance(remove, basestring): + if isinstance(remove, str): if remove in splitval: splitval.remove(remove) changed = True @@ -674,7 +758,7 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, varnames = [item[0] for item in bbappendlines] if removevalues: - varnames.extend(removevalues.keys()) + varnames.extend(list(removevalues.keys())) with open(appendpath, 'r') as f: (updated, newlines) = bb.utils.edit_metadata(f, varnames, appendfile_varfunc) @@ -699,7 +783,7 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, if copyfiles: if machine: destsubdir = os.path.join(destsubdir, machine) - for newfile, srcfile in copyfiles.iteritems(): + for newfile, srcfile in copyfiles.items(): filedest = os.path.join(appenddir, destsubdir, os.path.basename(srcfile)) if os.path.abspath(newfile) != os.path.abspath(filedest): bb.note('Copying %s to %s' % (newfile, filedest)) @@ -710,14 +794,16 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, def find_layerdir(fn): - """ Figure out relative path to base of layer for a file (e.g. a recipe)""" - pth = os.path.dirname(fn) + """ Figure out the path to the base of the layer containing a file (e.g. a recipe)""" + pth = fn layerdir = '' while pth: if os.path.exists(os.path.join(pth, 'conf', 'layer.conf')): layerdir = pth break pth = os.path.dirname(pth) + if pth == '/': + return None return layerdir @@ -725,12 +811,12 @@ def replace_dir_vars(path, d): """Replace common directory paths with appropriate variable references (e.g. /etc becomes ${sysconfdir})""" dirvars = {} # Sort by length so we get the variables we're interested in first - for var in sorted(d.keys(), key=len): + for var in sorted(list(d.keys()), key=len): if var.endswith('dir') and var.lower() == var: value = d.getVar(var, True) if value.startswith('/') and not '\n' in value and value not in dirvars: dirvars[value] = var - for dirpath in sorted(dirvars.keys(), reverse=True): + for dirpath in sorted(list(dirvars.keys()), reverse=True): path = path.replace(dirpath, '${%s}' % dirvars[dirpath]) return path diff --git a/import-layers/yocto-poky/meta/lib/oe/rootfs.py b/import-layers/yocto-poky/meta/lib/oe/rootfs.py index a95e1b739..a348b975c 100644 --- a/import-layers/yocto-poky/meta/lib/oe/rootfs.py +++ b/import-layers/yocto-poky/meta/lib/oe/rootfs.py @@ -10,17 +10,17 @@ import subprocess import re -class Rootfs(object): +class Rootfs(object, metaclass=ABCMeta): """ This is an abstract class. Do not instantiate this directly. """ - __metaclass__ = ABCMeta - def __init__(self, d): + def __init__(self, d, progress_reporter=None): self.d = d self.pm = None self.image_rootfs = self.d.getVar('IMAGE_ROOTFS', True) - self.deploy_dir_image = self.d.getVar('DEPLOY_DIR_IMAGE', True) + self.deploydir = self.d.getVar('IMGDEPLOYDIR', True) + self.progress_reporter = progress_reporter self.install_order = Manifest.INSTALL_ORDER @@ -40,51 +40,46 @@ class Rootfs(object): def _log_check(self): pass - def _log_check_warn(self): - r = re.compile('^(warn|Warn|NOTE: warn|NOTE: Warn|WARNING:)') + def _log_check_common(self, type, match): + # Ignore any lines containing log_check to avoid recursion, and ignore + # lines beginning with a + since sh -x may emit code which isn't + # actually executed, but may contain error messages + excludes = [ 'log_check', r'^\+' ] + if hasattr(self, 'log_check_expected_regexes'): + excludes.extend(self.log_check_expected_regexes) + excludes = [re.compile(x) for x in excludes] + r = re.compile(match) log_path = self.d.expand("${T}/log.do_rootfs") + messages = [] with open(log_path, 'r') as log: for line in log: - if 'log_check' in line or 'NOTE:' in line: - continue - - m = r.search(line) + for ee in excludes: + m = ee.search(line) + if m: + break if m: - bb.warn('[log_check] %s: found a warning message in the logfile (keyword \'%s\'):\n[log_check] %s' - % (self.d.getVar('PN', True), m.group(), line)) - - def _log_check_error(self): - r = re.compile(self.log_check_regex) - log_path = self.d.expand("${T}/log.do_rootfs") - with open(log_path, 'r') as log: - found_error = 0 - message = "\n" - for line in log: - if 'log_check' in line: continue - if hasattr(self, 'log_check_expected_errors_regexes'): - m = None - for ee in self.log_check_expected_errors_regexes: - m = re.search(ee, line) - if m: - break - if m: - continue - m = r.search(line) if m: - found_error = 1 - bb.warn('[log_check] In line: [%s]' % line) - bb.warn('[log_check] %s: found an error message in the logfile (keyword \'%s\'):\n[log_check] %s' - % (self.d.getVar('PN', True), m.group(), line)) + messages.append('[log_check] %s' % line) + if messages: + if len(messages) == 1: + msg = '1 %s message' % type + else: + msg = '%d %s messages' % (len(messages), type) + msg = '[log_check] %s: found %s in the logfile:\n%s' % \ + (self.d.getVar('PN', True), msg, ''.join(messages)) + if type == 'error': + bb.fatal(msg) + else: + bb.warn(msg) - if found_error >= 1 and found_error <= 5: - message += line + '\n' - found_error += 1 + def _log_check_warn(self): + self._log_check_common('warning', '^(warn|Warn|WARNING:)') - if found_error == 6: - bb.fatal(message) + def _log_check_error(self): + self._log_check_common('error', self.log_check_regex) def _insert_feed_uris(self): if bb.utils.contains("IMAGE_FEATURES", "package-management", @@ -121,8 +116,10 @@ class Rootfs(object): bb.note(" Copying back package database...") for dir in dirs: + if not os.path.isdir(self.image_rootfs + '-orig' + dir): + continue bb.utils.mkdirhier(self.image_rootfs + os.path.dirname(dir)) - shutil.copytree(self.image_rootfs + '-orig' + dir, self.image_rootfs + dir) + shutil.copytree(self.image_rootfs + '-orig' + dir, self.image_rootfs + dir, symlinks=True) cpath = oe.cachedpath.CachedPath() # Copy files located in /usr/lib/debug or /usr/src/debug @@ -185,16 +182,19 @@ class Rootfs(object): bb.utils.mkdirhier(self.image_rootfs) - bb.utils.mkdirhier(self.deploy_dir_image) + bb.utils.mkdirhier(self.deploydir) shutil.copytree(postinst_intercepts_dir, intercepts_dir) shutil.copy(self.d.expand("${COREBASE}/meta/files/deploydir_readme.txt"), - self.deploy_dir_image + + self.deploydir + "/README_-_DO_NOT_DELETE_FILES_IN_THIS_DIRECTORY.txt") execute_pre_post_process(self.d, pre_process_cmds) + if self.progress_reporter: + self.progress_reporter.next_stage() + # call the package manager dependent create method self._create() @@ -209,6 +209,9 @@ class Rootfs(object): execute_pre_post_process(self.d, post_process_cmds) + if self.progress_reporter: + self.progress_reporter.next_stage() + if bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", True, False, self.d): delayed_postinsts = self._get_delayed_postinsts() @@ -222,6 +225,9 @@ class Rootfs(object): self._uninstall_unneeded() + if self.progress_reporter: + self.progress_reporter.next_stage() + self._insert_feed_uris() self._run_ldconfig() @@ -232,6 +238,10 @@ class Rootfs(object): self._cleanup() self._log_check() + if self.progress_reporter: + self.progress_reporter.next_stage() + + def _uninstall_unneeded(self): # Remove unneeded init script symlinks delayed_postinsts = self._get_delayed_postinsts() @@ -243,7 +253,9 @@ class Rootfs(object): image_rorfs = bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", True, False, self.d) - if image_rorfs: + image_rorfs_force = self.d.getVar('FORCE_RO_REMOVE', True) + + if image_rorfs or image_rorfs_force == "1": # Remove components that we don't need if it's a read-only rootfs unneeded_pkgs = self.d.getVar("ROOTFS_RO_UNNEEDED", True).split() pkgs_installed = image_list_installed_packages(self.d) @@ -363,8 +375,8 @@ class Rootfs(object): class RpmRootfs(Rootfs): - def __init__(self, d, manifest_dir): - super(RpmRootfs, self).__init__(d) + def __init__(self, d, manifest_dir, progress_reporter=None): + super(RpmRootfs, self).__init__(d, progress_reporter) self.log_check_regex = '(unpacking of archive failed|Cannot find package'\ '|exit 1|ERROR: |Error: |Error |ERROR '\ '|Failed |Failed: |Failed$|Failed\(\d+\):)' @@ -422,11 +434,17 @@ class RpmRootfs(Rootfs): execute_pre_post_process(self.d, rpm_pre_process_cmds) + if self.progress_reporter: + self.progress_reporter.next_stage() + self.pm.dump_all_available_pkgs() if self.inc_rpm_image_gen == "1": self._create_incremental(pkgs_to_install) + if self.progress_reporter: + self.progress_reporter.next_stage() + self.pm.update() pkgs = [] @@ -437,12 +455,24 @@ class RpmRootfs(Rootfs): else: pkgs += pkgs_to_install[pkg_type] + if self.progress_reporter: + self.progress_reporter.next_stage() + self.pm.install(pkgs) + if self.progress_reporter: + self.progress_reporter.next_stage() + self.pm.install(pkgs_attempt, True) + if self.progress_reporter: + self.progress_reporter.next_stage() + self.pm.install_complementary() + if self.progress_reporter: + self.progress_reporter.next_stage() + self._setup_dbg_rootfs(['/etc/rpm', '/var/lib/rpm', '/var/lib/smart']) execute_pre_post_process(self.d, rpm_post_process_cmds) @@ -454,6 +484,10 @@ class RpmRootfs(Rootfs): self.pm.rpm_setup_smart_target_config() + if self.progress_reporter: + self.progress_reporter.next_stage() + + @staticmethod def _depends_list(): return ['DEPLOY_DIR_RPM', 'INC_RPM_IMAGE_GEN', 'RPM_PREPROCESS_COMMANDS', @@ -474,32 +508,6 @@ class RpmRootfs(Rootfs): # already saved in /etc/rpm-postinsts pass - def _log_check_error(self): - r = re.compile('(unpacking of archive failed|Cannot find package|exit 1|ERR|Fail)') - log_path = self.d.expand("${T}/log.do_rootfs") - with open(log_path, 'r') as log: - found_error = 0 - message = "\n" - for line in log.read().split('\n'): - if 'log_check' in line: - continue - # sh -x may emit code which isn't actually executed - if line.startswith('+'): - continue - - m = r.search(line) - if m: - found_error = 1 - bb.warn('log_check: There were error messages in the logfile') - bb.warn('log_check: Matched keyword: [%s]\n\n' % m.group()) - - if found_error >= 1 and found_error <= 5: - message += line + '\n' - found_error += 1 - - if found_error == 6: - bb.fatal(message) - def _log_check(self): self._log_check_warn() self._log_check_error() @@ -524,8 +532,8 @@ class RpmRootfs(Rootfs): bb.utils.remove(self.pm.install_dir_path, True) class DpkgOpkgRootfs(Rootfs): - def __init__(self, d): - super(DpkgOpkgRootfs, self).__init__(d) + def __init__(self, d, progress_reporter=None): + super(DpkgOpkgRootfs, self).__init__(d, progress_reporter) def _get_pkgs_postinsts(self, status_file): def _get_pkg_depends_list(pkg_depends): @@ -565,7 +573,7 @@ class DpkgOpkgRootfs(Rootfs): pkg_depends = m_depends.group(1) # remove package dependencies not in postinsts - pkg_names = pkgs.keys() + pkg_names = list(pkgs.keys()) for pkg_name in pkg_names: deps = pkgs[pkg_name][:] @@ -598,7 +606,7 @@ class DpkgOpkgRootfs(Rootfs): pkgs = self._get_pkgs_postinsts(status_file) if pkgs: root = "__packagegroup_postinst__" - pkgs[root] = pkgs.keys() + pkgs[root] = list(pkgs.keys()) _dep_resolve(pkgs, root, pkg_list, []) pkg_list.remove(root) @@ -619,10 +627,10 @@ class DpkgOpkgRootfs(Rootfs): num += 1 class DpkgRootfs(DpkgOpkgRootfs): - def __init__(self, d, manifest_dir): - super(DpkgRootfs, self).__init__(d) + def __init__(self, d, manifest_dir, progress_reporter=None): + super(DpkgRootfs, self).__init__(d, progress_reporter) self.log_check_regex = '^E:' - self.log_check_expected_errors_regexes = \ + self.log_check_expected_regexes = \ [ "^E: Unmet dependencies." ] @@ -648,15 +656,31 @@ class DpkgRootfs(DpkgOpkgRootfs): execute_pre_post_process(self.d, deb_pre_process_cmds) + if self.progress_reporter: + self.progress_reporter.next_stage() + # Don't support incremental, so skip that + self.progress_reporter.next_stage() + self.pm.update() + if self.progress_reporter: + self.progress_reporter.next_stage() + for pkg_type in self.install_order: if pkg_type in pkgs_to_install: self.pm.install(pkgs_to_install[pkg_type], [False, True][pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY]) + if self.progress_reporter: + # Don't support attemptonly, so skip that + self.progress_reporter.next_stage() + self.progress_reporter.next_stage() + self.pm.install_complementary() + if self.progress_reporter: + self.progress_reporter.next_stage() + self._setup_dbg_rootfs(['/var/lib/dpkg']) self.pm.fix_broken_dependencies() @@ -667,6 +691,9 @@ class DpkgRootfs(DpkgOpkgRootfs): execute_pre_post_process(self.d, deb_post_process_cmds) + if self.progress_reporter: + self.progress_reporter.next_stage() + @staticmethod def _depends_list(): return ['DEPLOY_DIR_DEB', 'DEB_SDK_ARCH', 'APTCONF_TARGET', 'APT_ARGS', 'DPKG_ARCH', 'DEB_PREPROCESS_COMMANDS', 'DEB_POSTPROCESS_COMMANDS'] @@ -692,8 +719,8 @@ class DpkgRootfs(DpkgOpkgRootfs): class OpkgRootfs(DpkgOpkgRootfs): - def __init__(self, d, manifest_dir): - super(OpkgRootfs, self).__init__(d) + def __init__(self, d, manifest_dir, progress_reporter=None): + super(OpkgRootfs, self).__init__(d, progress_reporter) self.log_check_regex = '(exit 1|Collected errors)' self.manifest = OpkgManifest(d, manifest_dir) @@ -887,13 +914,24 @@ class OpkgRootfs(DpkgOpkgRootfs): execute_pre_post_process(self.d, opkg_pre_process_cmds) + if self.progress_reporter: + self.progress_reporter.next_stage() + # Steps are a bit different in order, skip next + self.progress_reporter.next_stage() + self.pm.update() self.pm.handle_bad_recommendations() + if self.progress_reporter: + self.progress_reporter.next_stage() + if self.inc_opkg_image_gen == "1": self._remove_extra_packages(pkgs_to_install) + if self.progress_reporter: + self.progress_reporter.next_stage() + for pkg_type in self.install_order: if pkg_type in pkgs_to_install: # For multilib, we perform a sanity test before final install @@ -905,15 +943,24 @@ class OpkgRootfs(DpkgOpkgRootfs): self.pm.install(pkgs_to_install[pkg_type], [False, True][pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY]) + if self.progress_reporter: + self.progress_reporter.next_stage() + self.pm.install_complementary() - self._setup_dbg_rootfs(['/var/lib/opkg']) + if self.progress_reporter: + self.progress_reporter.next_stage() + + self._setup_dbg_rootfs(['/etc', '/var/lib/opkg', '/usr/lib/ssl']) execute_pre_post_process(self.d, opkg_post_process_cmds) if self.inc_opkg_image_gen == "1": self.pm.backup_packaging_data() + if self.progress_reporter: + self.progress_reporter.next_stage() + @staticmethod def _depends_list(): return ['IPKGCONF_SDK', 'IPK_FEED_URIS', 'DEPLOY_DIR_IPK', 'IPKGCONF_TARGET', 'INC_IPK_IMAGE_GEN', 'OPKG_ARGS', 'OPKGLIBDIR', 'OPKG_PREPROCESS_COMMANDS', 'OPKG_POSTPROCESS_COMMANDS', 'OPKGLIBDIR'] @@ -949,16 +996,16 @@ def variable_depends(d, manifest_dir=None): cls = get_class_for_type(img_type) return cls._depends_list() -def create_rootfs(d, manifest_dir=None): +def create_rootfs(d, manifest_dir=None, progress_reporter=None): env_bkp = os.environ.copy() img_type = d.getVar('IMAGE_PKGTYPE', True) if img_type == "rpm": - RpmRootfs(d, manifest_dir).create() + RpmRootfs(d, manifest_dir, progress_reporter).create() elif img_type == "ipk": - OpkgRootfs(d, manifest_dir).create() + OpkgRootfs(d, manifest_dir, progress_reporter).create() elif img_type == "deb": - DpkgRootfs(d, manifest_dir).create() + DpkgRootfs(d, manifest_dir, progress_reporter).create() os.environ.clear() os.environ.update(env_bkp) diff --git a/import-layers/yocto-poky/meta/lib/oe/sdk.py b/import-layers/yocto-poky/meta/lib/oe/sdk.py index f15fbdb36..c74525f92 100644 --- a/import-layers/yocto-poky/meta/lib/oe/sdk.py +++ b/import-layers/yocto-poky/meta/lib/oe/sdk.py @@ -8,9 +8,7 @@ import glob import traceback -class Sdk(object): - __metaclass__ = ABCMeta - +class Sdk(object, metaclass=ABCMeta): def __init__(self, d, manifest_dir): self.d = d self.sdk_output = self.d.getVar('SDK_OUTPUT', True) @@ -155,14 +153,16 @@ class RpmSdk(Sdk): execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_TARGET_COMMAND", True)) - self.target_pm.remove_packaging_data() + if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d): + self.target_pm.remove_packaging_data() bb.note("Installing NATIVESDK packages") self._populate_sysroot(self.host_pm, self.host_manifest) execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_HOST_COMMAND", True)) - self.host_pm.remove_packaging_data() + if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d): + self.host_pm.remove_packaging_data() # Move host RPM library data native_rpm_state_dir = os.path.join(self.sdk_output, @@ -232,14 +232,16 @@ class OpkgSdk(Sdk): execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_TARGET_COMMAND", True)) - self.target_pm.remove_packaging_data() + if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d): + self.target_pm.remove_packaging_data() bb.note("Installing NATIVESDK packages") self._populate_sysroot(self.host_pm, self.host_manifest) execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_HOST_COMMAND", True)) - self.host_pm.remove_packaging_data() + if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d): + self.host_pm.remove_packaging_data() target_sysconfdir = os.path.join(self.sdk_target_sysroot, self.sysconfdir) host_sysconfdir = os.path.join(self.sdk_host_sysroot, self.sysconfdir) @@ -247,12 +249,12 @@ class OpkgSdk(Sdk): self.mkdirhier(target_sysconfdir) shutil.copy(self.target_conf, target_sysconfdir) os.chmod(os.path.join(target_sysconfdir, - os.path.basename(self.target_conf)), 0644) + os.path.basename(self.target_conf)), 0o644) self.mkdirhier(host_sysconfdir) shutil.copy(self.host_conf, host_sysconfdir) os.chmod(os.path.join(host_sysconfdir, - os.path.basename(self.host_conf)), 0644) + os.path.basename(self.host_conf)), 0o644) native_opkg_state_dir = os.path.join(self.sdk_output, self.sdk_native_path, self.d.getVar('localstatedir_nativesdk', True).strip('/'), @@ -314,6 +316,9 @@ class DpkgSdk(Sdk): self._copy_apt_dir_to(os.path.join(self.sdk_target_sysroot, "etc", "apt")) + if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d): + self.target_pm.remove_packaging_data() + bb.note("Installing NATIVESDK packages") self._populate_sysroot(self.host_pm, self.host_manifest) @@ -322,6 +327,9 @@ class DpkgSdk(Sdk): self._copy_apt_dir_to(os.path.join(self.sdk_output, self.sdk_native_path, "etc", "apt")) + if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d): + self.host_pm.remove_packaging_data() + native_dpkg_state_dir = os.path.join(self.sdk_output, self.sdk_native_path, "var", "lib", "dpkg") self.mkdirhier(native_dpkg_state_dir) diff --git a/import-layers/yocto-poky/meta/lib/oe/sstatesig.py b/import-layers/yocto-poky/meta/lib/oe/sstatesig.py index 01dce660c..8224e3a12 100644 --- a/import-layers/yocto-poky/meta/lib/oe/sstatesig.py +++ b/import-layers/yocto-poky/meta/lib/oe/sstatesig.py @@ -130,7 +130,9 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): super(bb.siggen.SignatureGeneratorBasicHash, self).set_taskdata(coredata) def dump_sigs(self, dataCache, options): - self.dump_lockedsigs() + sigfile = os.getcwd() + "/locked-sigs.inc" + bb.plain("Writing locked sigs to %s" % sigfile) + self.dump_lockedsigs(sigfile) return super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigs(dataCache, options) def get_taskhash(self, fn, task, deps, dataCache): @@ -181,11 +183,7 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): return super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigtask(fn, task, stampbase, runtime) - def dump_lockedsigs(self, sigfile=None, taskfilter=None): - if not sigfile: - sigfile = os.getcwd() + "/locked-sigs.inc" - - bb.plain("Writing locked sigs to %s" % sigfile) + def dump_lockedsigs(self, sigfile, taskfilter=None): types = {} for k in self.runtaskdeps: if taskfilter: @@ -210,7 +208,7 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): continue f.write(" " + self.lockedpnmap[fn] + ":" + task + ":" + self.taskhash[k] + " \\\n") f.write(' "\n') - f.write('SIGGEN_LOCKEDSIGS_TYPES_%s = "%s"' % (self.machine, " ".join(types.keys()))) + f.write('SIGGEN_LOCKEDSIGS_TYPES_%s = "%s"' % (self.machine, " ".join(list(types.keys())))) def checkhashes(self, missed, ret, sq_fn, sq_task, sq_hash, sq_hashfn, d): warn_msgs = [] @@ -220,7 +218,7 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): for task in range(len(sq_fn)): if task not in ret: for pn in self.lockedsigs: - if sq_hash[task] in self.lockedsigs[pn].itervalues(): + if sq_hash[task] in iter(self.lockedsigs[pn].values()): if sq_task[task] == 'do_shared_workdir': continue sstate_missing_msgs.append("Locked sig is set for %s:%s (%s) yet not in sstate cache?" diff --git a/import-layers/yocto-poky/meta/lib/oe/terminal.py b/import-layers/yocto-poky/meta/lib/oe/terminal.py index 634daa903..3901ad3f2 100644 --- a/import-layers/yocto-poky/meta/lib/oe/terminal.py +++ b/import-layers/yocto-poky/meta/lib/oe/terminal.py @@ -25,9 +25,7 @@ class Registry(oe.classutils.ClassRegistry): return bool(cls.command) -class Terminal(Popen): - __metaclass__ = Registry - +class Terminal(Popen, metaclass=Registry): def __init__(self, sh_cmd, title=None, env=None, d=None): fmt_sh_cmd = self.format_command(sh_cmd, title) try: @@ -41,7 +39,7 @@ class Terminal(Popen): def format_command(self, sh_cmd, title): fmt = {'title': title or 'Terminal', 'command': sh_cmd} - if isinstance(self.command, basestring): + if isinstance(self.command, str): return shlex.split(self.command.format(**fmt)) else: return [element.format(**fmt) for element in self.command] @@ -53,7 +51,7 @@ class XTerminal(Terminal): raise UnsupportedTerminal(self.name) class Gnome(XTerminal): - command = 'gnome-terminal -t "{title}" --disable-factory -x {command}' + command = 'gnome-terminal -t "{title}" -x {command}' priority = 2 def __init__(self, sh_cmd, title=None, env=None, d=None): @@ -63,12 +61,28 @@ class Gnome(XTerminal): # Once fixed on the gnome-terminal project, this should be removed. if os.getenv('LC_ALL'): os.putenv('LC_ALL','') - # Check version - vernum = check_terminal_version("gnome-terminal") - if vernum and LooseVersion(vernum) >= '3.10': - logger.debug(1, 'Gnome-Terminal 3.10 or later does not support --disable-factory') - self.command = 'gnome-terminal -t "{title}" -x {command}' - XTerminal.__init__(self, sh_cmd, title, env, d) + # We need to know when the command completes but gnome-terminal gives us no way + # to do this. We therefore write the pid to a file using a "phonehome" wrapper + # script, then monitor the pid until it exits. Thanks gnome! + import tempfile + pidfile = tempfile.NamedTemporaryFile(delete = False).name + try: + sh_cmd = "oe-gnome-terminal-phonehome " + pidfile + " " + sh_cmd + XTerminal.__init__(self, sh_cmd, title, env, d) + while os.stat(pidfile).st_size <= 0: + continue + with open(pidfile, "r") as f: + pid = int(f.readline()) + finally: + os.unlink(pidfile) + + import time + while True: + try: + os.kill(pid, 0) + time.sleep(0.1) + except OSError: + return class Mate(XTerminal): command = 'mate-terminal -t "{title}" -x {command}' @@ -248,7 +262,7 @@ def check_terminal_version(terminalName): newenv["LANG"] = "C" p = sub.Popen(['sh', '-c', cmdversion], stdout=sub.PIPE, stderr=sub.PIPE, env=newenv) out, err = p.communicate() - ver_info = out.rstrip().split('\n') + ver_info = out.decode().rstrip().split('\n') except OSError as exc: import errno if exc.errno == errno.ENOENT: diff --git a/import-layers/yocto-poky/meta/lib/oe/tests/test_elf.py b/import-layers/yocto-poky/meta/lib/oe/tests/test_elf.py new file mode 100644 index 000000000..1f59037ed --- /dev/null +++ b/import-layers/yocto-poky/meta/lib/oe/tests/test_elf.py @@ -0,0 +1,21 @@ +import unittest +import oe.qa + +class TestElf(unittest.TestCase): + def test_machine_name(self): + """ + Test elf_machine_to_string() + """ + self.assertEqual(oe.qa.elf_machine_to_string(0x02), "SPARC") + self.assertEqual(oe.qa.elf_machine_to_string(0x03), "x86") + self.assertEqual(oe.qa.elf_machine_to_string(0x08), "MIPS") + self.assertEqual(oe.qa.elf_machine_to_string(0x14), "PowerPC") + self.assertEqual(oe.qa.elf_machine_to_string(0x28), "ARM") + self.assertEqual(oe.qa.elf_machine_to_string(0x2A), "SuperH") + self.assertEqual(oe.qa.elf_machine_to_string(0x32), "IA-64") + self.assertEqual(oe.qa.elf_machine_to_string(0x3E), "x86-64") + self.assertEqual(oe.qa.elf_machine_to_string(0xB7), "AArch64") + + self.assertEqual(oe.qa.elf_machine_to_string(0x00), "Unknown (0)") + self.assertEqual(oe.qa.elf_machine_to_string(0xDEADBEEF), "Unknown (3735928559)") + self.assertEqual(oe.qa.elf_machine_to_string("foobar"), "Unknown ('foobar')") diff --git a/import-layers/yocto-poky/meta/lib/oe/tests/test_path.py b/import-layers/yocto-poky/meta/lib/oe/tests/test_path.py index 3d41ce157..44d068143 100644 --- a/import-layers/yocto-poky/meta/lib/oe/tests/test_path.py +++ b/import-layers/yocto-poky/meta/lib/oe/tests/test_path.py @@ -55,7 +55,7 @@ class TestRealPath(unittest.TestCase): for d in self.DIRS: os.mkdir(os.path.join(self.root, d)) for f in self.FILES: - file(os.path.join(self.root, f), "w") + open(os.path.join(self.root, f), "w") for l in self.LINKS: os.symlink(l[1], os.path.join(self.root, l[0])) @@ -85,5 +85,5 @@ class TestRealPath(unittest.TestCase): def test_loop(self): for e in self.EXCEPTIONS: - self.assertRaisesRegexp(OSError, r'\[Errno %u\]' % e[1], + self.assertRaisesRegex(OSError, r'\[Errno %u\]' % e[1], self.__realpath, e[0], False, False) diff --git a/import-layers/yocto-poky/meta/lib/oe/types.py b/import-layers/yocto-poky/meta/lib/oe/types.py index 7f47c17d0..4ae58acfa 100644 --- a/import-layers/yocto-poky/meta/lib/oe/types.py +++ b/import-layers/yocto-poky/meta/lib/oe/types.py @@ -33,7 +33,7 @@ def choice(value, choices): Acts as a multiple choice for the user. To use this, set the variable type flag to 'choice', and set the 'choices' flag to a space separated list of valid values.""" - if not isinstance(value, basestring): + if not isinstance(value, str): raise TypeError("choice accepts a string, not '%s'" % type(value)) value = value.lower() @@ -106,7 +106,7 @@ def boolean(value): Valid values for false: 'no', 'n', 'false', 'f', '0' """ - if not isinstance(value, basestring): + if not isinstance(value, str): raise TypeError("boolean accepts a string, not '%s'" % type(value)) value = value.lower() diff --git a/import-layers/yocto-poky/meta/lib/oe/utils.py b/import-layers/yocto-poky/meta/lib/oe/utils.py index 30d30629f..d6545b197 100644 --- a/import-layers/yocto-poky/meta/lib/oe/utils.py +++ b/import-layers/yocto-poky/meta/lib/oe/utils.py @@ -23,13 +23,13 @@ def ifelse(condition, iftrue = True, iffalse = False): return iffalse def conditional(variable, checkvalue, truevalue, falsevalue, d): - if d.getVar(variable,1) == checkvalue: + if d.getVar(variable, True) == checkvalue: return truevalue else: return falsevalue def less_or_equal(variable, checkvalue, truevalue, falsevalue, d): - if float(d.getVar(variable,1)) <= float(checkvalue): + if float(d.getVar(variable, True)) <= float(checkvalue): return truevalue else: return falsevalue @@ -46,7 +46,7 @@ def both_contain(variable1, variable2, checkvalue, d): val2 = d.getVar(variable2, True) val1 = set(val1.split()) val2 = set(val2.split()) - if isinstance(checkvalue, basestring): + if isinstance(checkvalue, str): checkvalue = set(checkvalue.split()) else: checkvalue = set(checkvalue) @@ -85,11 +85,11 @@ def prune_suffix(var, suffixes, d): def str_filter(f, str, d): from re import match - return " ".join(filter(lambda x: match(f, x, 0), str.split())) + return " ".join([x for x in str.split() if match(f, x, 0)]) def str_filter_out(f, str, d): from re import match - return " ".join(filter(lambda x: not match(f, x, 0), str.split())) + return " ".join([x for x in str.split() if not match(f, x, 0)]) def param_bool(cfg, field, dflt = None): """Lookup <field> in <cfg> map and convert it to a boolean; take @@ -134,7 +134,7 @@ def packages_filter_out_system(d): PN-dbg PN-doc PN-locale-eb-gb removed. """ pn = d.getVar('PN', True) - blacklist = map(lambda suffix: pn + suffix, ('', '-dbg', '-dev', '-doc', '-locale', '-staticdev')) + blacklist = [pn + suffix for suffix in ('', '-dbg', '-dev', '-doc', '-locale', '-staticdev')] localepkg = pn + "-locale-" pkgs = [] @@ -235,7 +235,7 @@ def format_pkg_list(pkg_dict, ret_format=None): # so implement a version here # -from Queue import Queue +from queue import Queue from threading import Thread class ThreadedWorker(Thread): @@ -249,7 +249,7 @@ class ThreadedWorker(Thread): self.worker_end = worker_end def run(self): - from Queue import Empty + from queue import Empty if self.worker_init is not None: self.worker_init(self) @@ -264,8 +264,8 @@ class ThreadedWorker(Thread): try: func(self, *args, **kargs) - except Exception, e: - print e + except Exception as e: + print(e) finally: self.tasks.task_done() @@ -304,3 +304,16 @@ def write_ld_so_conf(d): with open(ldsoconf, "w") as f: f.write(d.getVar("base_libdir", True) + '\n') f.write(d.getVar("libdir", True) + '\n') + +class ImageQAFailed(bb.build.FuncFailed): + def __init__(self, description, name=None, logfile=None): + self.description = description + self.name = name + self.logfile=logfile + + def __str__(self): + msg = 'Function failed: %s' % self.name + if self.description: + msg = msg + ' (%s)' % self.description + + return msg diff --git a/import-layers/yocto-poky/meta/lib/oeqa/__init__.py b/import-layers/yocto-poky/meta/lib/oeqa/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/import-layers/yocto-poky/meta/lib/oeqa/__init__.py +++ /dev/null diff --git a/import-layers/yocto-poky/meta/lib/oeqa/buildperf/__init__.py b/import-layers/yocto-poky/meta/lib/oeqa/buildperf/__init__.py new file mode 100644 index 000000000..605f429ec --- /dev/null +++ b/import-layers/yocto-poky/meta/lib/oeqa/buildperf/__init__.py @@ -0,0 +1,19 @@ +# Copyright (c) 2016, Intel Corporation. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms and conditions of the GNU General Public License, +# version 2, as published by the Free Software Foundation. +# +# This program is distributed in the hope it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +"""Build performance tests""" +from .base import (BuildPerfTestCase, + BuildPerfTestLoader, + BuildPerfTestResult, + BuildPerfTestRunner, + KernelDropCaches, + runCmd2) +from .test_basic import * diff --git a/import-layers/yocto-poky/meta/lib/oeqa/buildperf/base.py b/import-layers/yocto-poky/meta/lib/oeqa/buildperf/base.py new file mode 100644 index 000000000..59dd02521 --- /dev/null +++ b/import-layers/yocto-poky/meta/lib/oeqa/buildperf/base.py @@ -0,0 +1,551 @@ +# Copyright (c) 2016, Intel Corporation. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms and conditions of the GNU General Public License, +# version 2, as published by the Free Software Foundation. +# +# This program is distributed in the hope it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +"""Build performance test base classes and functionality""" +import glob +import json +import logging +import os +import re +import resource +import shutil +import socket +import time +import traceback +import unittest +from datetime import datetime, timedelta +from functools import partial +from multiprocessing import Process +from multiprocessing import SimpleQueue + +import oe.path +from oeqa.utils.commands import CommandError, runCmd, get_bb_vars +from oeqa.utils.git import GitError, GitRepo + +# Get logger for this module +log = logging.getLogger('build-perf') + +# Our own version of runCmd which does not raise AssertErrors which would cause +# errors to interpreted as failures +runCmd2 = partial(runCmd, assert_error=False) + + +class KernelDropCaches(object): + """Container of the functions for dropping kernel caches""" + sudo_passwd = None + + @classmethod + def check(cls): + """Check permssions for dropping kernel caches""" + from getpass import getpass + from locale import getdefaultlocale + cmd = ['sudo', '-k', '-n', 'tee', '/proc/sys/vm/drop_caches'] + ret = runCmd2(cmd, ignore_status=True, data=b'0') + if ret.output.startswith('sudo:'): + pass_str = getpass( + "\nThe script requires sudo access to drop caches between " + "builds (echo 3 > /proc/sys/vm/drop_caches).\n" + "Please enter your sudo password: ") + cls.sudo_passwd = bytes(pass_str, getdefaultlocale()[1]) + + @classmethod + def drop(cls): + """Drop kernel caches""" + cmd = ['sudo', '-k'] + if cls.sudo_passwd: + cmd.append('-S') + input_data = cls.sudo_passwd + b'\n' + else: + cmd.append('-n') + input_data = b'' + cmd += ['tee', '/proc/sys/vm/drop_caches'] + input_data += b'3' + runCmd2(cmd, data=input_data) + + +def str_to_fn(string): + """Convert string to a sanitized filename""" + return re.sub(r'(\W+)', '-', string, flags=re.LOCALE) + + +class ResultsJsonEncoder(json.JSONEncoder): + """Extended encoder for build perf test results""" + unix_epoch = datetime.utcfromtimestamp(0) + + def default(self, obj): + """Encoder for our types""" + if isinstance(obj, datetime): + # NOTE: we assume that all timestamps are in UTC time + return (obj - self.unix_epoch).total_seconds() + if isinstance(obj, timedelta): + return obj.total_seconds() + return json.JSONEncoder.default(self, obj) + + +class BuildPerfTestResult(unittest.TextTestResult): + """Runner class for executing the individual tests""" + # List of test cases to run + test_run_queue = [] + + def __init__(self, out_dir, *args, **kwargs): + super(BuildPerfTestResult, self).__init__(*args, **kwargs) + + self.out_dir = out_dir + # Get Git parameters + try: + self.repo = GitRepo('.') + except GitError: + self.repo = None + self.git_commit, self.git_commit_count, self.git_branch = \ + self.get_git_revision() + self.hostname = socket.gethostname() + self.product = os.getenv('OE_BUILDPERFTEST_PRODUCT', 'oe-core') + self.start_time = self.elapsed_time = None + self.successes = [] + log.info("Using Git branch:commit %s:%s (%s)", self.git_branch, + self.git_commit, self.git_commit_count) + + def get_git_revision(self): + """Get git branch and commit under testing""" + commit = os.getenv('OE_BUILDPERFTEST_GIT_COMMIT') + commit_cnt = os.getenv('OE_BUILDPERFTEST_GIT_COMMIT_COUNT') + branch = os.getenv('OE_BUILDPERFTEST_GIT_BRANCH') + if not self.repo and (not commit or not commit_cnt or not branch): + log.info("The current working directory doesn't seem to be a Git " + "repository clone. You can specify branch and commit " + "displayed in test results with OE_BUILDPERFTEST_GIT_BRANCH, " + "OE_BUILDPERFTEST_GIT_COMMIT and " + "OE_BUILDPERFTEST_GIT_COMMIT_COUNT environment variables") + else: + if not commit: + commit = self.repo.rev_parse('HEAD^0') + commit_cnt = self.repo.run_cmd(['rev-list', '--count', 'HEAD^0']) + if not branch: + branch = self.repo.get_current_branch() + if not branch: + log.debug('Currently on detached HEAD') + return str(commit), str(commit_cnt), str(branch) + + def addSuccess(self, test): + """Record results from successful tests""" + super(BuildPerfTestResult, self).addSuccess(test) + self.successes.append((test, None)) + + def startTest(self, test): + """Pre-test hook""" + test.base_dir = self.out_dir + os.mkdir(test.out_dir) + log.info("Executing test %s: %s", test.name, test.shortDescription()) + self.stream.write(datetime.now().strftime("[%Y-%m-%d %H:%M:%S] ")) + super(BuildPerfTestResult, self).startTest(test) + + def startTestRun(self): + """Pre-run hook""" + self.start_time = datetime.utcnow() + + def stopTestRun(self): + """Pre-run hook""" + self.elapsed_time = datetime.utcnow() - self.start_time + self.write_results_json() + + def all_results(self): + result_map = {'SUCCESS': self.successes, + 'FAIL': self.failures, + 'ERROR': self.errors, + 'EXP_FAIL': self.expectedFailures, + 'UNEXP_SUCCESS': self.unexpectedSuccesses, + 'SKIPPED': self.skipped} + for status, tests in result_map.items(): + for test in tests: + yield (status, test) + + + def update_globalres_file(self, filename): + """Write results to globalres csv file""" + # Map test names to time and size columns in globalres + # The tuples represent index and length of times and sizes + # respectively + gr_map = {'test1': ((0, 1), (8, 1)), + 'test12': ((1, 1), (None, None)), + 'test13': ((2, 1), (9, 1)), + 'test2': ((3, 1), (None, None)), + 'test3': ((4, 3), (None, None)), + 'test4': ((7, 1), (10, 2))} + + if self.repo: + git_tag_rev = self.repo.run_cmd(['describe', self.git_commit]) + else: + git_tag_rev = self.git_commit + + values = ['0'] * 12 + for status, (test, msg) in self.all_results(): + if status in ['ERROR', 'SKIPPED']: + continue + (t_ind, t_len), (s_ind, s_len) = gr_map[test.name] + if t_ind is not None: + values[t_ind:t_ind + t_len] = test.times + if s_ind is not None: + values[s_ind:s_ind + s_len] = test.sizes + + log.debug("Writing globalres log to %s", filename) + with open(filename, 'a') as fobj: + fobj.write('{},{}:{},{},'.format(self.hostname, + self.git_branch, + self.git_commit, + git_tag_rev)) + fobj.write(','.join(values) + '\n') + + def write_results_json(self): + """Write test results into a json-formatted file""" + results = {'tester_host': self.hostname, + 'git_branch': self.git_branch, + 'git_commit': self.git_commit, + 'git_commit_count': self.git_commit_count, + 'product': self.product, + 'start_time': self.start_time, + 'elapsed_time': self.elapsed_time} + + tests = {} + for status, (test, reason) in self.all_results(): + tests[test.name] = {'name': test.name, + 'description': test.shortDescription(), + 'status': status, + 'start_time': test.start_time, + 'elapsed_time': test.elapsed_time, + 'cmd_log_file': os.path.relpath(test.cmd_log_file, + self.out_dir), + 'measurements': test.measurements} + results['tests'] = tests + + with open(os.path.join(self.out_dir, 'results.json'), 'w') as fobj: + json.dump(results, fobj, indent=4, sort_keys=True, + cls=ResultsJsonEncoder) + + + def git_commit_results(self, repo_path, branch=None, tag=None): + """Commit results into a Git repository""" + repo = GitRepo(repo_path, is_topdir=True) + if not branch: + branch = self.git_branch + else: + # Replace keywords + branch = branch.format(git_branch=self.git_branch, + tester_host=self.hostname) + + log.info("Committing test results into %s %s", repo_path, branch) + tmp_index = os.path.join(repo_path, '.git', 'index.oe-build-perf') + try: + # Create new commit object from the new results + env_update = {'GIT_INDEX_FILE': tmp_index, + 'GIT_WORK_TREE': self.out_dir} + repo.run_cmd('add .', env_update) + tree = repo.run_cmd('write-tree', env_update) + parent = repo.rev_parse(branch) + msg = "Results of {}:{}\n".format(self.git_branch, self.git_commit) + git_cmd = ['commit-tree', tree, '-m', msg] + if parent: + git_cmd += ['-p', parent] + commit = repo.run_cmd(git_cmd, env_update) + + # Update branch head + git_cmd = ['update-ref', 'refs/heads/' + branch, commit] + if parent: + git_cmd.append(parent) + repo.run_cmd(git_cmd) + + # Update current HEAD, if we're on branch 'branch' + if repo.get_current_branch() == branch: + log.info("Updating %s HEAD to latest commit", repo_path) + repo.run_cmd('reset --hard') + + # Create (annotated) tag + if tag: + # Find tags matching the pattern + tag_keywords = dict(git_branch=self.git_branch, + git_commit=self.git_commit, + git_commit_count=self.git_commit_count, + tester_host=self.hostname, + tag_num='[0-9]{1,5}') + tag_re = re.compile(tag.format(**tag_keywords) + '$') + tag_keywords['tag_num'] = 0 + for existing_tag in repo.run_cmd('tag').splitlines(): + if tag_re.match(existing_tag): + tag_keywords['tag_num'] += 1 + + tag = tag.format(**tag_keywords) + msg = "Test run #{} of {}:{}\n".format(tag_keywords['tag_num'], + self.git_branch, + self.git_commit) + repo.run_cmd(['tag', '-a', '-m', msg, tag, commit]) + + finally: + if os.path.exists(tmp_index): + os.unlink(tmp_index) + + +class BuildPerfTestCase(unittest.TestCase): + """Base class for build performance tests""" + SYSRES = 'sysres' + DISKUSAGE = 'diskusage' + build_target = None + + def __init__(self, *args, **kwargs): + super(BuildPerfTestCase, self).__init__(*args, **kwargs) + self.name = self._testMethodName + self.base_dir = None + self.start_time = None + self.elapsed_time = None + self.measurements = [] + self.bb_vars = get_bb_vars() + # TODO: remove 'times' and 'sizes' arrays when globalres support is + # removed + self.times = [] + self.sizes = [] + + @property + def out_dir(self): + return os.path.join(self.base_dir, self.name) + + @property + def cmd_log_file(self): + return os.path.join(self.out_dir, 'commands.log') + + def setUp(self): + """Set-up fixture for each test""" + if self.build_target: + self.log_cmd_output(['bitbake', self.build_target, + '-c', 'fetchall']) + + def run(self, *args, **kwargs): + """Run test""" + self.start_time = datetime.now() + super(BuildPerfTestCase, self).run(*args, **kwargs) + self.elapsed_time = datetime.now() - self.start_time + + def log_cmd_output(self, cmd): + """Run a command and log it's output""" + cmd_str = cmd if isinstance(cmd, str) else ' '.join(cmd) + log.info("Logging command: %s", cmd_str) + try: + with open(self.cmd_log_file, 'a') as fobj: + runCmd2(cmd, stdout=fobj) + except CommandError as err: + log.error("Command failed: %s", err.retcode) + raise + + def measure_cmd_resources(self, cmd, name, legend, save_bs=False): + """Measure system resource usage of a command""" + def _worker(data_q, cmd, **kwargs): + """Worker process for measuring resources""" + try: + start_time = datetime.now() + ret = runCmd2(cmd, **kwargs) + etime = datetime.now() - start_time + rusage_struct = resource.getrusage(resource.RUSAGE_CHILDREN) + iostat = {} + with open('/proc/{}/io'.format(os.getpid())) as fobj: + for line in fobj.readlines(): + key, val = line.split(':') + iostat[key] = int(val) + rusage = {} + # Skip unused fields, (i.e. 'ru_ixrss', 'ru_idrss', 'ru_isrss', + # 'ru_nswap', 'ru_msgsnd', 'ru_msgrcv' and 'ru_nsignals') + for key in ['ru_utime', 'ru_stime', 'ru_maxrss', 'ru_minflt', + 'ru_majflt', 'ru_inblock', 'ru_oublock', + 'ru_nvcsw', 'ru_nivcsw']: + rusage[key] = getattr(rusage_struct, key) + data_q.put({'ret': ret, + 'start_time': start_time, + 'elapsed_time': etime, + 'rusage': rusage, + 'iostat': iostat}) + except Exception as err: + data_q.put(err) + + cmd_str = cmd if isinstance(cmd, str) else ' '.join(cmd) + log.info("Timing command: %s", cmd_str) + data_q = SimpleQueue() + try: + with open(self.cmd_log_file, 'a') as fobj: + proc = Process(target=_worker, args=(data_q, cmd,), + kwargs={'stdout': fobj}) + proc.start() + data = data_q.get() + proc.join() + if isinstance(data, Exception): + raise data + except CommandError: + log.error("Command '%s' failed, see %s for more details", cmd_str, + self.cmd_log_file) + raise + etime = data['elapsed_time'] + + measurement = {'type': self.SYSRES, + 'name': name, + 'legend': legend} + measurement['values'] = {'start_time': data['start_time'], + 'elapsed_time': etime, + 'rusage': data['rusage'], + 'iostat': data['iostat']} + if save_bs: + bs_file = self.save_buildstats(legend) + measurement['values']['buildstats_file'] = \ + os.path.relpath(bs_file, self.base_dir) + + self.measurements.append(measurement) + + # Append to 'times' array for globalres log + e_sec = etime.total_seconds() + self.times.append('{:d}:{:02d}:{:05.2f}'.format(int(e_sec / 3600), + int((e_sec % 3600) / 60), + e_sec % 60)) + + def measure_disk_usage(self, path, name, legend, apparent_size=False): + """Estimate disk usage of a file or directory""" + cmd = ['du', '-s', '--block-size', '1024'] + if apparent_size: + cmd.append('--apparent-size') + cmd.append(path) + + ret = runCmd2(cmd) + size = int(ret.output.split()[0]) + log.debug("Size of %s path is %s", path, size) + measurement = {'type': self.DISKUSAGE, + 'name': name, + 'legend': legend} + measurement['values'] = {'size': size} + self.measurements.append(measurement) + # Append to 'sizes' array for globalres log + self.sizes.append(str(size)) + + def save_buildstats(self, label=None): + """Save buildstats""" + def split_nevr(nevr): + """Split name and version information from recipe "nevr" string""" + n_e_v, revision = nevr.rsplit('-', 1) + match = re.match(r'^(?P<name>\S+)-((?P<epoch>[0-9]{1,5})_)?(?P<version>[0-9]\S*)$', + n_e_v) + if not match: + # If we're not able to parse a version starting with a number, just + # take the part after last dash + match = re.match(r'^(?P<name>\S+)-((?P<epoch>[0-9]{1,5})_)?(?P<version>[^-]+)$', + n_e_v) + name = match.group('name') + version = match.group('version') + epoch = match.group('epoch') + return name, epoch, version, revision + + def bs_to_json(filename): + """Convert (task) buildstats file into json format""" + bs_json = {'iostat': {}, + 'rusage': {}, + 'child_rusage': {}} + with open(filename) as fobj: + for line in fobj.readlines(): + key, val = line.split(':', 1) + val = val.strip() + if key == 'Started': + start_time = datetime.utcfromtimestamp(float(val)) + bs_json['start_time'] = start_time + elif key == 'Ended': + end_time = datetime.utcfromtimestamp(float(val)) + elif key.startswith('IO '): + split = key.split() + bs_json['iostat'][split[1]] = int(val) + elif key.find('rusage') >= 0: + split = key.split() + ru_key = split[-1] + if ru_key in ('ru_stime', 'ru_utime'): + val = float(val) + else: + val = int(val) + ru_type = 'rusage' if split[0] == 'rusage' else \ + 'child_rusage' + bs_json[ru_type][ru_key] = val + elif key == 'Status': + bs_json['status'] = val + bs_json['elapsed_time'] = end_time - start_time + return bs_json + + log.info('Saving buildstats in JSON format') + bs_dirs = sorted(os.listdir(self.bb_vars['BUILDSTATS_BASE'])) + if len(bs_dirs) > 1: + log.warning("Multiple buildstats found for test %s, only " + "archiving the last one", self.name) + bs_dir = os.path.join(self.bb_vars['BUILDSTATS_BASE'], bs_dirs[-1]) + + buildstats = [] + for fname in os.listdir(bs_dir): + recipe_dir = os.path.join(bs_dir, fname) + if not os.path.isdir(recipe_dir): + continue + name, epoch, version, revision = split_nevr(fname) + recipe_bs = {'name': name, + 'epoch': epoch, + 'version': version, + 'revision': revision, + 'tasks': {}} + for task in os.listdir(recipe_dir): + recipe_bs['tasks'][task] = bs_to_json(os.path.join(recipe_dir, + task)) + buildstats.append(recipe_bs) + + # Write buildstats into json file + postfix = '.' + str_to_fn(label) if label else '' + postfix += '.json' + outfile = os.path.join(self.out_dir, 'buildstats' + postfix) + with open(outfile, 'w') as fobj: + json.dump(buildstats, fobj, indent=4, sort_keys=True, + cls=ResultsJsonEncoder) + return outfile + + def rm_tmp(self): + """Cleanup temporary/intermediate files and directories""" + log.debug("Removing temporary and cache files") + for name in ['bitbake.lock', 'conf/sanity_info', + self.bb_vars['TMPDIR']]: + oe.path.remove(name, recurse=True) + + def rm_sstate(self): + """Remove sstate directory""" + log.debug("Removing sstate-cache") + oe.path.remove(self.bb_vars['SSTATE_DIR'], recurse=True) + + def rm_cache(self): + """Drop bitbake caches""" + oe.path.remove(self.bb_vars['PERSISTENT_DIR'], recurse=True) + + @staticmethod + def sync(): + """Sync and drop kernel caches""" + log.debug("Syncing and dropping kernel caches""") + KernelDropCaches.drop() + os.sync() + # Wait a bit for all the dirty blocks to be written onto disk + time.sleep(3) + + +class BuildPerfTestLoader(unittest.TestLoader): + """Test loader for build performance tests""" + sortTestMethodsUsing = None + + +class BuildPerfTestRunner(unittest.TextTestRunner): + """Test loader for build performance tests""" + sortTestMethodsUsing = None + + def __init__(self, out_dir, *args, **kwargs): + super(BuildPerfTestRunner, self).__init__(*args, **kwargs) + self.out_dir = out_dir + + def _makeResult(self): + return BuildPerfTestResult(self.out_dir, self.stream, self.descriptions, + self.verbosity) diff --git a/import-layers/yocto-poky/meta/lib/oeqa/buildperf/test_basic.py b/import-layers/yocto-poky/meta/lib/oeqa/buildperf/test_basic.py new file mode 100644 index 000000000..7a48c1e77 --- /dev/null +++ b/import-layers/yocto-poky/meta/lib/oeqa/buildperf/test_basic.py @@ -0,0 +1,127 @@ +# Copyright (c) 2016, Intel Corporation. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms and conditions of the GNU General Public License, +# version 2, as published by the Free Software Foundation. +# +# This program is distributed in the hope it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +"""Basic set of build performance tests""" +import os +import shutil + +import oe.path +from oeqa.buildperf import BuildPerfTestCase +from oeqa.utils.commands import get_bb_vars + + +class Test1P1(BuildPerfTestCase): + build_target = 'core-image-sato' + + def test1(self): + """Measure wall clock of bitbake core-image-sato and size of tmp dir""" + self.rm_tmp() + self.rm_sstate() + self.rm_cache() + self.sync() + self.measure_cmd_resources(['bitbake', self.build_target], 'build', + 'bitbake ' + self.build_target, save_bs=True) + self.measure_disk_usage(self.bb_vars['TMPDIR'], 'tmpdir', 'tmpdir') + + +class Test1P2(BuildPerfTestCase): + build_target = 'virtual/kernel' + + def test12(self): + """Measure bitbake virtual/kernel""" + # Build and cleans state in order to get all dependencies pre-built + self.log_cmd_output(['bitbake', self.build_target]) + self.log_cmd_output(['bitbake', self.build_target, '-c', 'cleansstate']) + + self.sync() + self.measure_cmd_resources(['bitbake', self.build_target], 'build', + 'bitbake ' + self.build_target) + + +class Test1P3(BuildPerfTestCase): + build_target = 'core-image-sato' + + def test13(self): + """Build core-image-sato with rm_work enabled""" + postfile = os.path.join(self.out_dir, 'postfile.conf') + with open(postfile, 'w') as fobj: + fobj.write('INHERIT += "rm_work"\n') + try: + self.rm_tmp() + self.rm_sstate() + self.rm_cache() + self.sync() + cmd = ['bitbake', '-R', postfile, self.build_target] + self.measure_cmd_resources(cmd, 'build', + 'bitbake' + self.build_target, + save_bs=True) + self.measure_disk_usage(self.bb_vars['TMPDIR'], 'tmpdir', 'tmpdir') + finally: + os.unlink(postfile) + + +class Test2(BuildPerfTestCase): + build_target = 'core-image-sato' + + def test2(self): + """Measure bitbake core-image-sato -c rootfs with sstate""" + # Build once in order to populate sstate cache + self.log_cmd_output(['bitbake', self.build_target]) + + self.rm_tmp() + self.rm_cache() + self.sync() + cmd = ['bitbake', self.build_target, '-c', 'rootfs'] + self.measure_cmd_resources(cmd, 'do_rootfs', 'bitbake do_rootfs') + + +class Test3(BuildPerfTestCase): + + def test3(self): + """Parsing time metrics (bitbake -p)""" + # Drop all caches and parse + self.rm_cache() + oe.path.remove(os.path.join(self.bb_vars['TMPDIR'], 'cache'), True) + self.measure_cmd_resources(['bitbake', '-p'], 'parse_1', + 'bitbake -p (no caches)') + # Drop tmp/cache + oe.path.remove(os.path.join(self.bb_vars['TMPDIR'], 'cache'), True) + self.measure_cmd_resources(['bitbake', '-p'], 'parse_2', + 'bitbake -p (no tmp/cache)') + # Parse with fully cached data + self.measure_cmd_resources(['bitbake', '-p'], 'parse_3', + 'bitbake -p (cached)') + + +class Test4(BuildPerfTestCase): + build_target = 'core-image-sato' + + def test4(self): + """eSDK metrics""" + self.log_cmd_output("bitbake {} -c do_populate_sdk_ext".format( + self.build_target)) + self.bb_vars = get_bb_vars(None, self.build_target) + tmp_dir = self.bb_vars['TMPDIR'] + installer = os.path.join( + self.bb_vars['SDK_DEPLOY'], + self.bb_vars['TOOLCHAINEXT_OUTPUTNAME'] + '.sh') + # Measure installer size + self.measure_disk_usage(installer, 'installer_bin', 'eSDK installer', + apparent_size=True) + # Measure deployment time and deployed size + deploy_dir = os.path.join(tmp_dir, 'esdk-deploy') + if os.path.exists(deploy_dir): + shutil.rmtree(deploy_dir) + self.sync() + self.measure_cmd_resources([installer, '-y', '-d', deploy_dir], + 'deploy', 'eSDK deploy') + self.measure_disk_usage(deploy_dir, 'deploy_dir', 'deploy dir', + apparent_size=True) diff --git a/import-layers/yocto-poky/meta/lib/oeqa/controllers/masterimage.py b/import-layers/yocto-poky/meta/lib/oeqa/controllers/masterimage.py index 522f9ebd7..9ce3bf803 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/controllers/masterimage.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/controllers/masterimage.py @@ -24,9 +24,7 @@ from oeqa.utils import CommandError from abc import ABCMeta, abstractmethod -class MasterImageHardwareTarget(oeqa.targetcontrol.BaseTarget): - - __metaclass__ = ABCMeta +class MasterImageHardwareTarget(oeqa.targetcontrol.BaseTarget, metaclass=ABCMeta): supported_image_fstypes = ['tar.gz', 'tar.bz2'] @@ -199,3 +197,43 @@ class GummibootTarget(MasterImageHardwareTarget): self.power_cycle(self.master) # there are better ways than a timeout but this should work for now time.sleep(120) + + +class SystemdbootTarget(MasterImageHardwareTarget): + + def __init__(self, d): + super(SystemdbootTarget, self).__init__(d) + # this the value we need to set in the LoaderEntryOneShot EFI variable + # so the system boots the 'test' bootloader label and not the default + # The first four bytes are EFI bits, and the rest is an utf-16le string + # (EFI vars values need to be utf-16) + # $ echo -en "test\0" | iconv -f ascii -t utf-16le | hexdump -C + # 00000000 74 00 65 00 73 00 74 00 00 00 |t.e.s.t...| + self.efivarvalue = r'\x07\x00\x00\x00\x74\x00\x65\x00\x73\x00\x74\x00\x00\x00' + self.deploy_cmds = [ + 'mount -L boot /boot', + 'mkdir -p /mnt/testrootfs', + 'mount -L testrootfs /mnt/testrootfs', + 'modprobe efivarfs', + 'mount -t efivarfs efivarfs /sys/firmware/efi/efivars', + 'cp ~/test-kernel /boot', + 'rm -rf /mnt/testrootfs/*', + 'tar xvf ~/test-rootfs.%s -C /mnt/testrootfs' % self.image_fstype, + 'printf "%s" > /sys/firmware/efi/efivars/LoaderEntryOneShot-4a67b082-0a4c-41cf-b6c7-440b29bb8c4f' % self.efivarvalue + ] + + def _deploy(self): + # make sure these aren't mounted + self.master.run("umount /boot; umount /mnt/testrootfs; umount /sys/firmware/efi/efivars;") + # from now on, every deploy cmd should return 0 + # else an exception will be thrown by sshcontrol + self.master.ignore_status = False + self.master.copy_to(self.rootfs, "~/test-rootfs." + self.image_fstype) + self.master.copy_to(self.kernel, "~/test-kernel") + for cmd in self.deploy_cmds: + self.master.run(cmd) + + def _start(self, params=None): + self.power_cycle(self.master) + # there are better ways than a timeout but this should work for now + time.sleep(120) diff --git a/import-layers/yocto-poky/meta/lib/oeqa/oetest.py b/import-layers/yocto-poky/meta/lib/oeqa/oetest.py index 3ed5bb8c2..95d3bf72f 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/oetest.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/oetest.py @@ -12,6 +12,8 @@ import unittest import inspect import subprocess import signal +import shutil +import functools try: import bb except ImportError: @@ -54,18 +56,29 @@ def filterByTagExp(testsuite, tagexp): @LogResults class oeTest(unittest.TestCase): + pscmd = "ps" longMessage = True @classmethod def hasPackage(self, pkg): - for item in oeTest.tc.pkgmanifest.split('\n'): - if re.match(pkg, item): + """ + True if the full package name exists in the manifest, False otherwise. + """ + return pkg in oeTest.tc.pkgmanifest + + @classmethod + def hasPackageMatch(self, match): + """ + True if match exists in the manifest as a regular expression substring, + False otherwise. + """ + for s in oeTest.tc.pkgmanifest: + if re.match(match, s): return True return False @classmethod def hasFeature(self,feature): - if feature in oeTest.tc.imagefeatures or \ feature in oeTest.tc.distrofeatures: return True @@ -78,6 +91,9 @@ class oeRuntimeTest(oeTest): super(oeRuntimeTest, self).__init__(methodName) def setUp(self): + # Install packages in the DUT + self.tc.install_uninstall_packages(self.id()) + # Check if test needs to run if self.tc.sigterm: self.fail("Got SIGTERM") @@ -91,6 +107,9 @@ class oeRuntimeTest(oeTest): pass def tearDown(self): + # Unistall packages in the DUT + self.tc.install_uninstall_packages(self.id(), False) + res = getResults() # If a test fails or there is an exception dump # for QemuTarget only @@ -117,6 +136,15 @@ class oeRuntimeTest(oeTest): if status != 0: return status +class OETestCalledProcessError(subprocess.CalledProcessError): + def __str__(self): + if hasattr(self, "stderr"): + return "Command '%s' returned non-zero exit status %d with output %s and stderr %s" % (self.cmd, self.returncode, self.output, self.stderr) + else: + return "Command '%s' returned non-zero exit status %d with output %s" % (self.cmd, self.returncode, self.output) + +subprocess.CalledProcessError = OETestCalledProcessError + class oeSDKTest(oeTest): def __init__(self, methodName='runTest'): self.sdktestdir = oeSDKTest.tc.sdktestdir @@ -124,13 +152,12 @@ class oeSDKTest(oeTest): @classmethod def hasHostPackage(self, pkg): - if re.search(pkg, oeTest.tc.hostpkgmanifest): return True return False def _run(self, cmd): - return subprocess.check_output(". %s > /dev/null; %s;" % (self.tc.sdkenv, cmd), shell=True) + return subprocess.check_output(". %s > /dev/null; %s;" % (self.tc.sdkenv, cmd), shell=True, stderr=subprocess.STDOUT).decode("utf-8") class oeSDKExtTest(oeSDKTest): def _run(self, cmd): @@ -142,7 +169,7 @@ class oeSDKExtTest(oeSDKTest): env['PATH'] = avoid_paths_in_environ(paths_to_avoid) return subprocess.check_output(". %s > /dev/null;"\ - " %s;" % (self.tc.sdkenv, cmd), shell=True, env=env) + " %s;" % (self.tc.sdkenv, cmd), stderr=subprocess.STDOUT, shell=True, env=env).decode("utf-8") def getmodule(pos=2): # stack returns a list of tuples containg frame information @@ -185,15 +212,22 @@ def custom_verbose(msg, *args, **kwargs): _buffer_logger = "" class TestContext(object): - def __init__(self, d): + def __init__(self, d, exported=False): self.d = d self.testsuites = self._get_test_suites() - self.testslist = self._get_tests_list(d.getVar("BBPATH", True).split(':')) + + if exported: + path = [os.path.dirname(os.path.abspath(__file__))] + extrapath = "" + else: + path = d.getVar("BBPATH", True).split(':') + extrapath = "lib/oeqa" + + self.testslist = self._get_tests_list(path, extrapath) self.testsrequired = self._get_test_suites_required() - self.filesdir = os.path.join(os.path.dirname(os.path.abspath( - oeqa.runtime.__file__)), "files") + self.filesdir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "runtime/files") self.imagefeatures = d.getVar("IMAGE_FEATURES", True).split() self.distrofeatures = d.getVar("DISTRO_FEATURES", True).split() @@ -212,7 +246,7 @@ class TestContext(object): return " ".join(tcs) # return test list by type also filter if TEST_SUITES is specified - def _get_tests_list(self, bbpath): + def _get_tests_list(self, bbpath, extrapath): testslist = [] type = self._get_test_namespace() @@ -226,11 +260,11 @@ class TestContext(object): continue found = False for p in bbpath: - if os.path.exists(os.path.join(p, 'lib', 'oeqa', type, testname + '.py')): + if os.path.exists(os.path.join(p, extrapath, type, testname + ".py")): testslist.append("oeqa." + type + "." + testname) found = True break - elif os.path.exists(os.path.join(p, 'lib', 'oeqa', type, testname.split(".")[0] + '.py')): + elif os.path.exists(os.path.join(p, extrapath, type, testname.split(".")[0] + ".py")): testslist.append("oeqa." + type + "." + testname) found = True break @@ -239,8 +273,6 @@ class TestContext(object): if "auto" in self.testsuites: def add_auto_list(path): - if not os.path.exists(os.path.join(path, '__init__.py')): - bb.fatal('Tests directory %s exists but is missing __init__.py' % path) files = sorted([f for f in os.listdir(path) if f.endswith('.py') and not f.startswith('_')]) for f in files: module = 'oeqa.' + type + '.' + f[:-3] @@ -255,6 +287,51 @@ class TestContext(object): return testslist + def getTestModules(self): + """ + Returns all the test modules in the testlist. + """ + + import pkgutil + + modules = [] + for test in self.testslist: + if re.search("\w+\.\w+\.test_\S+", test): + test = '.'.join(t.split('.')[:3]) + module = pkgutil.get_loader(test) + modules.append(module) + + return modules + + def getModulefromID(self, test_id): + """ + Returns the test module based on a test id. + """ + + module_name = ".".join(test_id.split(".")[:3]) + modules = self.getTestModules() + for module in modules: + if module.name == module_name: + return module + + return None + + def getTests(self, test): + '''Return all individual tests executed when running the suite.''' + # Unfortunately unittest does not have an API for this, so we have + # to rely on implementation details. This only needs to work + # for TestSuite containing TestCase. + method = getattr(test, '_testMethodName', None) + if method: + # leaf case: a TestCase + yield test + else: + # Look into TestSuite. + tests = getattr(test, '_tests', []) + for t1 in tests: + for t2 in self.getTests(t1): + yield t2 + def loadTests(self): setattr(oeTest, "tc", self) @@ -263,36 +340,20 @@ class TestContext(object): suites = [testloader.loadTestsFromName(name) for name in self.testslist] suites = filterByTagExp(suites, getattr(self, "tagexp", None)) - def getTests(test): - '''Return all individual tests executed when running the suite.''' - # Unfortunately unittest does not have an API for this, so we have - # to rely on implementation details. This only needs to work - # for TestSuite containing TestCase. - method = getattr(test, '_testMethodName', None) - if method: - # leaf case: a TestCase - yield test - else: - # Look into TestSuite. - tests = getattr(test, '_tests', []) - for t1 in tests: - for t2 in getTests(t1): - yield t2 - # Determine dependencies between suites by looking for @skipUnlessPassed # method annotations. Suite A depends on suite B if any method in A # depends on a method on B. for suite in suites: suite.dependencies = [] suite.depth = 0 - for test in getTests(suite): + for test in self.getTests(suite): methodname = getattr(test, '_testMethodName', None) if methodname: method = getattr(test, methodname) depends_on = getattr(method, '_depends_on', None) if depends_on: for dep_suite in suites: - if depends_on in [getattr(t, '_testMethodName', None) for t in getTests(dep_suite)]: + if depends_on in [getattr(t, '_testMethodName', None) for t in self.getTests(dep_suite)]: if dep_suite not in suite.dependencies and \ dep_suite is not suite: suite.dependencies.append(dep_suite) @@ -314,7 +375,14 @@ class TestContext(object): for index, suite in enumerate(suites): set_suite_depth(suite) suite.index = index - suites.sort(cmp=lambda a,b: cmp((a.depth, a.index), (b.depth, b.index))) + + def cmp(a, b): + return (a > b) - (a < b) + + def cmpfunc(a, b): + return cmp((a.depth, a.index), (b.depth, b.index)) + + suites.sort(key=functools.cmp_to_key(cmpfunc)) self.suite = testloader.suiteClass(suites) @@ -331,35 +399,24 @@ class TestContext(object): return runner.run(self.suite) -class ImageTestContext(TestContext): - def __init__(self, d, target, host_dumper): - super(ImageTestContext, self).__init__(d) - - self.tagexp = d.getVar("TEST_SUITES_TAGS", True) +class RuntimeTestContext(TestContext): + def __init__(self, d, target, exported=False): + super(RuntimeTestContext, self).__init__(d, exported) self.target = target - self.host_dumper = host_dumper + self.pkgmanifest = {} manifest = os.path.join(d.getVar("DEPLOY_DIR_IMAGE", True), d.getVar("IMAGE_LINK_NAME", True) + ".manifest") nomanifest = d.getVar("IMAGE_NO_MANIFEST", True) if nomanifest is None or nomanifest != "1": try: with open(manifest) as f: - self.pkgmanifest = f.read() + for line in f: + (pkg, arch, version) = line.strip().split() + self.pkgmanifest[pkg] = (version, arch) except IOError as e: bb.fatal("No package manifest file found. Did you build the image?\n%s" % e) - else: - self.pkgmanifest = "" - - self.sigterm = False - self.origsigtermhandler = signal.getsignal(signal.SIGTERM) - signal.signal(signal.SIGTERM, self._sigterm_exception) - - def _sigterm_exception(self, signum, stackframe): - bb.warn("TestImage received SIGTERM, shutting down...") - self.sigterm = True - self.target.stop() def _get_test_namespace(self): return "runtime" @@ -382,8 +439,223 @@ class ImageTestContext(TestContext): return [t for t in self.d.getVar("TEST_SUITES", True).split() if t != "auto"] def loadTests(self): - super(ImageTestContext, self).loadTests() - setattr(oeRuntimeTest, "pscmd", "ps -ef" if oeTest.hasPackage("procps") else "ps") + super(RuntimeTestContext, self).loadTests() + if oeTest.hasPackage("procps"): + oeRuntimeTest.pscmd = "ps -ef" + + def extract_packages(self): + """ + Find packages that will be needed during runtime. + """ + + modules = self.getTestModules() + bbpaths = self.d.getVar("BBPATH", True).split(":") + + shutil.rmtree(self.d.getVar("TEST_EXTRACTED_DIR", True)) + shutil.rmtree(self.d.getVar("TEST_PACKAGED_DIR", True)) + for module in modules: + json_file = self._getJsonFile(module) + if json_file: + needed_packages = self._getNeededPackages(json_file) + self._perform_package_extraction(needed_packages) + + def _perform_package_extraction(self, needed_packages): + """ + Extract packages that will be needed during runtime. + """ + + import oe.path + + extracted_path = self.d.getVar("TEST_EXTRACTED_DIR", True) + packaged_path = self.d.getVar("TEST_PACKAGED_DIR", True) + + for key,value in needed_packages.items(): + packages = () + if isinstance(value, dict): + packages = (value, ) + elif isinstance(value, list): + packages = value + else: + bb.fatal("Failed to process needed packages for %s; " + "Value must be a dict or list" % key) + + for package in packages: + pkg = package["pkg"] + rm = package.get("rm", False) + extract = package.get("extract", True) + if extract: + dst_dir = os.path.join(extracted_path, pkg) + else: + dst_dir = os.path.join(packaged_path) + + # Extract package and copy it to TEST_EXTRACTED_DIR + pkg_dir = self._extract_in_tmpdir(pkg) + if extract: + + # Same package used for more than one test, + # don't need to extract again. + if os.path.exists(dst_dir): + continue + oe.path.copytree(pkg_dir, dst_dir) + shutil.rmtree(pkg_dir) + + # Copy package to TEST_PACKAGED_DIR + else: + self._copy_package(pkg) + + def _getJsonFile(self, module): + """ + Returns the path of the JSON file for a module, empty if doesn't exitst. + """ + + module_file = module.path + json_file = "%s.json" % module_file.rsplit(".", 1)[0] + if os.path.isfile(module_file) and os.path.isfile(json_file): + return json_file + else: + return "" + + def _getNeededPackages(self, json_file, test=None): + """ + Returns a dict with needed packages based on a JSON file. + + + If a test is specified it will return the dict just for that test. + """ + + import json + + needed_packages = {} + + with open(json_file) as f: + test_packages = json.load(f) + for key,value in test_packages.items(): + needed_packages[key] = value + + if test: + if test in needed_packages: + needed_packages = needed_packages[test] + else: + needed_packages = {} + + return needed_packages + + def _extract_in_tmpdir(self, pkg): + """" + Returns path to a temp directory where the package was + extracted without dependencies. + """ + + from oeqa.utils.package_manager import get_package_manager + + pkg_path = os.path.join(self.d.getVar("TEST_INSTALL_TMP_DIR", True), pkg) + pm = get_package_manager(self.d, pkg_path) + extract_dir = pm.extract(pkg) + shutil.rmtree(pkg_path) + + return extract_dir + + def _copy_package(self, pkg): + """ + Copy the RPM, DEB or IPK package to dst_dir + """ + + from oeqa.utils.package_manager import get_package_manager + + pkg_path = os.path.join(self.d.getVar("TEST_INSTALL_TMP_DIR", True), pkg) + dst_dir = self.d.getVar("TEST_PACKAGED_DIR", True) + pm = get_package_manager(self.d, pkg_path) + pkg_info = pm.package_info(pkg) + file_path = pkg_info[pkg]["filepath"] + shutil.copy2(file_path, dst_dir) + shutil.rmtree(pkg_path) + + def install_uninstall_packages(self, test_id, pkg_dir, install): + """ + Check if the test requires a package and Install/Unistall it in the DUT + """ + + test = test_id.split(".")[4] + module = self.getModulefromID(test_id) + json = self._getJsonFile(module) + if json: + needed_packages = self._getNeededPackages(json, test) + if needed_packages: + self._install_uninstall_packages(needed_packages, pkg_dir, install) + + def _install_uninstall_packages(self, needed_packages, pkg_dir, install=True): + """ + Install/Unistall packages in the DUT without using a package manager + """ + + if isinstance(needed_packages, dict): + packages = [needed_packages] + elif isinstance(needed_packages, list): + packages = needed_packages + + for package in packages: + pkg = package["pkg"] + rm = package.get("rm", False) + extract = package.get("extract", True) + src_dir = os.path.join(pkg_dir, pkg) + + # Install package + if install and extract: + self.target.connection.copy_dir_to(src_dir, "/") + + # Unistall package + elif not install and rm: + self.target.connection.delete_dir_structure(src_dir, "/") + +class ImageTestContext(RuntimeTestContext): + def __init__(self, d, target, host_dumper): + super(ImageTestContext, self).__init__(d, target) + + self.tagexp = d.getVar("TEST_SUITES_TAGS", True) + + self.host_dumper = host_dumper + + self.sigterm = False + self.origsigtermhandler = signal.getsignal(signal.SIGTERM) + signal.signal(signal.SIGTERM, self._sigterm_exception) + + def _sigterm_exception(self, signum, stackframe): + bb.warn("TestImage received SIGTERM, shutting down...") + self.sigterm = True + self.target.stop() + + def install_uninstall_packages(self, test_id, install=True): + """ + Check if the test requires a package and Install/Unistall it in the DUT + """ + + pkg_dir = self.d.getVar("TEST_EXTRACTED_DIR", True) + super(ImageTestContext, self).install_uninstall_packages(test_id, pkg_dir, install) + +class ExportTestContext(RuntimeTestContext): + def __init__(self, d, target, exported=False, parsedArgs={}): + """ + This class is used when exporting tests and when are executed outside OE environment. + + parsedArgs can contain the following: + - tag: Filter test by tag. + """ + super(ExportTestContext, self).__init__(d, target, exported) + + tag = parsedArgs.get("tag", None) + self.tagexp = tag if tag != None else d.getVar("TEST_SUITES_TAGS", True) + + self.sigterm = None + + def install_uninstall_packages(self, test_id, install=True): + """ + Check if the test requires a package and Install/Unistall it in the DUT + """ + + export_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + extracted_dir = self.d.getVar("TEST_EXPORT_EXTRACTED_DIR", True) + pkg_dir = os.path.join(export_dir, extracted_dir) + super(ExportTestContext, self).install_uninstall_packages(test_id, pkg_dir, install) class SDKTestContext(TestContext): def __init__(self, d, sdktestdir, sdkenv, tcname, *args): @@ -396,8 +668,11 @@ class SDKTestContext(TestContext): if not hasattr(self, 'target_manifest'): self.target_manifest = d.getVar("SDK_TARGET_MANIFEST", True) try: + self.pkgmanifest = {} with open(self.target_manifest) as f: - self.pkgmanifest = f.read() + for line in f: + (pkg, arch, version) = line.strip().split() + self.pkgmanifest[pkg] = (version, arch) except IOError as e: bb.fatal("No package manifest file found. Did you build the sdk image?\n%s" % e) diff --git a/import-layers/yocto-poky/meta/lib/oeqa/runexported.py b/import-layers/yocto-poky/meta/lib/oeqa/runexported.py index cc89e13c0..7e245c412 100755 --- a/import-layers/yocto-poky/meta/lib/oeqa/runexported.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/runexported.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (C) 2013 Intel Corporation @@ -30,9 +30,9 @@ except ImportError: sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "oeqa"))) -from oeqa.oetest import TestContext +from oeqa.oetest import ExportTestContext +from oeqa.utils.commands import runCmd, updateEnv from oeqa.utils.sshcontrol import SSHControl -from oeqa.utils.dump import get_host_dumper # this isn't pretty but we need a fake target object # for running the tests externally as we don't care @@ -69,10 +69,6 @@ class MyDataDict(dict): def getVar(self, key, unused = None): return self.get(key, "") -class ExportTestContext(TestContext): - def __init__(self, d): - self.d = d - def main(): parser = argparse.ArgumentParser() @@ -85,6 +81,7 @@ def main(): specified in the json if that directory actually exists or it will error out.") parser.add_argument("-l", "--log-dir", dest="log_dir", help="This sets the path for TEST_LOG_DIR. If not specified \ the current dir is used. This is used for usually creating a ssh log file and a scp test file.") + parser.add_argument("-a", "--tag", dest="tag", help="Only run test with specified tag.") parser.add_argument("json", help="The json file exported by the build system", default="testdata.json", nargs='?') args = parser.parse_args() @@ -111,30 +108,41 @@ def main(): if not os.path.isdir(d["DEPLOY_DIR"]): print("WARNING: The path to DEPLOY_DIR does not exist: %s" % d["DEPLOY_DIR"]) + parsedArgs = {} + parsedArgs["tag"] = args.tag + + extract_sdk(d) target = FakeTarget(d) for key in loaded["target"].keys(): setattr(target, key, loaded["target"][key]) - host_dumper = get_host_dumper(d) - host_dumper.parent_dir = loaded["host_dumper"]["parent_dir"] - host_dumper.cmds = loaded["host_dumper"]["cmds"] - target.exportStart() - tc = ExportTestContext(d) - - setattr(tc, "d", d) - setattr(tc, "target", target) - setattr(tc, "host_dumper", host_dumper) - for key in loaded.keys(): - if key != "d" and key != "target" and key != "host_dumper": - setattr(tc, key, loaded[key]) - + tc = ExportTestContext(d, target, True, parsedArgs) tc.loadTests() tc.runTests() return 0 +def extract_sdk(d): + """ + Extract SDK if needed + """ + + export_dir = os.path.dirname(os.path.realpath(__file__)) + tools_dir = d.getVar("TEST_EXPORT_SDK_DIR", True) + tarball_name = "%s.sh" % d.getVar("TEST_EXPORT_SDK_NAME", True) + tarball_path = os.path.join(export_dir, tools_dir, tarball_name) + extract_path = os.path.join(export_dir, "sysroot") + if os.path.isfile(tarball_path): + print ("Found SDK tarball %s. Extracting..." % tarball_path) + result = runCmd("%s -y -d %s" % (tarball_path, extract_path)) + for f in os.listdir(extract_path): + if f.startswith("environment-setup"): + print("Setting up SDK environment...") + env_file = os.path.join(extract_path, f) + updateEnv(env_file) + if __name__ == "__main__": try: ret = main() diff --git a/import-layers/yocto-poky/meta/lib/oeqa/runtime/__init__.py b/import-layers/yocto-poky/meta/lib/oeqa/runtime/__init__.py deleted file mode 100644 index 4cf3fa76b..000000000 --- a/import-layers/yocto-poky/meta/lib/oeqa/runtime/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# Enable other layers to have tests in the same named directory -from pkgutil import extend_path -__path__ = extend_path(__path__, __name__) diff --git a/import-layers/yocto-poky/meta/lib/oeqa/runtime/_ptest.py b/import-layers/yocto-poky/meta/lib/oeqa/runtime/_ptest.py index 0621028b8..71324d3da 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/runtime/_ptest.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/runtime/_ptest.py @@ -11,7 +11,7 @@ import subprocess def setUpModule(): if not oeRuntimeTest.hasFeature("package-management"): skipModule("Image doesn't have package management feature") - if not oeRuntimeTest.hasPackage("smart"): + if not oeRuntimeTest.hasPackage("smartpm"): skipModule("Image doesn't have smart installed") if "package_rpm" != oeRuntimeTest.tc.d.getVar("PACKAGE_CLASSES", True).split()[0]: skipModule("Rpm is not the primary package manager") @@ -105,7 +105,7 @@ class PtestRunnerTest(oeRuntimeTest): def test_ptestrunner(self): self.add_smart_channel() (runnerstatus, result) = self.target.run('which ptest-runner', 0) - cond = oeRuntimeTest.hasPackage("ptest-runner") and oeRuntimeTest.hasFeature("ptest") and oeRuntimeTest.hasPackage("-ptest") and (runnerstatus != 0) + cond = oeRuntimeTest.hasPackage("ptest-runner") and oeRuntimeTest.hasFeature("ptest") and oeRuntimeTest.hasPackageMatch("-ptest") and (runnerstatus != 0) if cond: self.install_packages(self.install_complementary("*-ptest")) self.install_packages(['ptest-runner']) diff --git a/import-layers/yocto-poky/meta/lib/oeqa/runtime/buildgalculator.py b/import-layers/yocto-poky/meta/lib/oeqa/runtime/buildgalculator.py new file mode 100644 index 000000000..28ba29e5c --- /dev/null +++ b/import-layers/yocto-poky/meta/lib/oeqa/runtime/buildgalculator.py @@ -0,0 +1,23 @@ +from oeqa.oetest import oeRuntimeTest, skipModule +from oeqa.utils.decorators import * +from oeqa.utils.targetbuild import TargetBuildProject + +def setUpModule(): + if not oeRuntimeTest.hasFeature("tools-sdk"): + skipModule("Image doesn't have tools-sdk in IMAGE_FEATURES") + +class GalculatorTest(oeRuntimeTest): + @skipUnlessPassed("test_ssh") + def test_galculator(self): + try: + project = TargetBuildProject(oeRuntimeTest.tc.target, oeRuntimeTest.tc.d, + "http://galculator.mnim.org/downloads/galculator-2.1.4.tar.bz2") + project.download_archive() + + self.assertEqual(project.run_configure(), 0, + msg="Running configure failed") + + self.assertEqual(project.run_make(), 0, + msg="Running make failed") + finally: + project.clean() diff --git a/import-layers/yocto-poky/meta/lib/oeqa/runtime/buildiptables.py b/import-layers/yocto-poky/meta/lib/oeqa/runtime/buildiptables.py index 09e252df8..bc75d0a0c 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/runtime/buildiptables.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/runtime/buildiptables.py @@ -11,7 +11,7 @@ class BuildIptablesTest(oeRuntimeTest): @classmethod def setUpClass(self): self.project = TargetBuildProject(oeRuntimeTest.tc.target, oeRuntimeTest.tc.d, - "http://netfilter.org/projects/iptables/files/iptables-1.4.13.tar.bz2") + "http://downloads.yoctoproject.org/mirror/sources/iptables-1.4.13.tar.bz2") self.project.download_archive() @testcase(206) diff --git a/import-layers/yocto-poky/meta/lib/oeqa/runtime/buildsudoku.py b/import-layers/yocto-poky/meta/lib/oeqa/runtime/buildsudoku.py deleted file mode 100644 index 802b06001..000000000 --- a/import-layers/yocto-poky/meta/lib/oeqa/runtime/buildsudoku.py +++ /dev/null @@ -1,28 +0,0 @@ -from oeqa.oetest import oeRuntimeTest, skipModule -from oeqa.utils.decorators import * -from oeqa.utils.targetbuild import TargetBuildProject - -def setUpModule(): - if not oeRuntimeTest.hasFeature("tools-sdk"): - skipModule("Image doesn't have tools-sdk in IMAGE_FEATURES") - -class SudokuTest(oeRuntimeTest): - - @classmethod - def setUpClass(self): - self.project = TargetBuildProject(oeRuntimeTest.tc.target, oeRuntimeTest.tc.d, - "http://downloads.sourceforge.net/project/sudoku-savant/sudoku-savant/sudoku-savant-1.3/sudoku-savant-1.3.tar.bz2") - self.project.download_archive() - - @testcase(207) - @skipUnlessPassed("test_ssh") - def test_sudoku(self): - self.assertEqual(self.project.run_configure(), 0, - msg="Running configure failed") - - self.assertEqual(self.project.run_make(), 0, - msg="Running make failed") - - @classmethod - def tearDownClass(self): - self.project.clean() diff --git a/import-layers/yocto-poky/meta/lib/oeqa/runtime/connman.py b/import-layers/yocto-poky/meta/lib/oeqa/runtime/connman.py index bd9dba3bd..003fefe2c 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/runtime/connman.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/runtime/connman.py @@ -27,5 +27,5 @@ class ConnmanTest(oeRuntimeTest): def test_connmand_running(self): (status, output) = self.target.run(oeRuntimeTest.pscmd + ' | grep [c]onnmand') if status != 0: - print self.service_status("connman") + print(self.service_status("connman")) self.fail("No connmand process running") diff --git a/import-layers/yocto-poky/meta/lib/oeqa/runtime/files/test.py b/import-layers/yocto-poky/meta/lib/oeqa/runtime/files/test.py index f3a2273c5..f389225d7 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/runtime/files/test.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/runtime/files/test.py @@ -3,4 +3,4 @@ import os os.system('touch /tmp/testfile.python') a = 9.01e+21 - 9.01e+21 + 0.01 -print "the value of a is %s" % a +print("the value of a is %s" % a) diff --git a/import-layers/yocto-poky/meta/lib/oeqa/runtime/parselogs.py b/import-layers/yocto-poky/meta/lib/oeqa/runtime/parselogs.py index dec9ebe87..8efe2d1de 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/runtime/parselogs.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/runtime/parselogs.py @@ -55,18 +55,25 @@ x86_common = [ 'Could not enable PowerButton event', 'probe of LNXPWRBN:00 failed with error -22', 'pmd_set_huge: Cannot satisfy', + 'failed to setup card detect gpio', + 'amd_nb: Cannot enumerate AMD northbridges', + 'failed to retrieve link info, disabling eDP', ] + common_errors qemux86_common = [ 'wrong ELF class', "fail to add MMCONFIG information, can't access extended PCI configuration space under this bridge.", "can't claim BAR ", + 'amd_nb: Cannot enumerate AMD northbridges', + 'uvesafb: 5000 ms task timeout, infinitely waiting', + 'tsc: HPET/PMTIMER calibration failed', ] + common_errors ignore_errors = { 'default' : common_errors, 'qemux86' : [ 'Failed to access perfctr msr (MSR', + 'pci 0000:00:00.0: [Firmware Bug]: reg 0x..: invalid BAR (can\'t size)', ] + qemux86_common, 'qemux86-64' : qemux86_common, 'qemumips' : [ @@ -81,16 +88,28 @@ ignore_errors = { 'host side 80-wire cable detection failed, limiting max speed', 'mode "640x480" test failed', 'Failed to load module "glx"', + 'can\'t handle BAR above 4GB', + 'Cannot reserve Legacy IO', ] + common_errors, 'qemuarm' : [ 'mmci-pl18x: probe of fpga:05 failed with error -22', 'mmci-pl18x: probe of fpga:0b failed with error -22', - 'Failed to load module "glx"' + 'Failed to load module "glx"', + 'OF: amba_device_add() failed (-19) for /amba/smc@10100000', + 'OF: amba_device_add() failed (-19) for /amba/mpmc@10110000', + 'OF: amba_device_add() failed (-19) for /amba/sctl@101e0000', + 'OF: amba_device_add() failed (-19) for /amba/watchdog@101e1000', + 'OF: amba_device_add() failed (-19) for /amba/sci@101f0000', + 'OF: amba_device_add() failed (-19) for /amba/ssp@101f4000', + 'OF: amba_device_add() failed (-19) for /amba/fpga/sci@a000', + 'Failed to initialize \'/amba/timer@101e3000\': -22', + 'jitterentropy: Initialization failed with host not compliant with requirements: 2', ] + common_errors, 'qemuarm64' : [ 'Fatal server error:', '(EE) Server terminated with error (1). Closing log file.', 'dmi: Firmware registration failed.', + 'irq: type mismatch, failed to map hwirq-27 for /intc', ] + common_errors, 'emenlow' : [ '[Firmware Bug]: ACPI: No _BQC method, cannot determine initial brightness', @@ -110,11 +129,19 @@ ignore_errors = { '(EE) Failed to load module psbdrv', '(EE) open /dev/fb0: No such file or directory', '(EE) AIGLX: reverting to software rendering', + 'dmi: Firmware registration failed.', + 'ioremap error for 0x78', ] + x86_common, 'intel-corei7-64' : x86_common, 'crownbay' : x86_common, 'genericx86' : x86_common, - 'genericx86-64' : x86_common, + 'genericx86-64' : [ + 'Direct firmware load for i915', + 'Failed to load firmware i915', + 'Failed to fetch GuC', + 'Failed to initialize GuC', + 'The driver is built-in, so to load the firmware you need to', + ] + x86_common, 'edgerouter' : [ 'Fatal server error:', ] + common_errors, @@ -153,6 +180,9 @@ class ParseLogsTest(oeRuntimeTest): def getMachine(self): return oeRuntimeTest.tc.d.getVar("MACHINE", True) + def getWorkdir(self): + return oeRuntimeTest.tc.d.getVar("WORKDIR", True) + #get some information on the CPU of the machine to display at the beginning of the output. This info might be useful in some cases. def getHardwareInfo(self): hwi = "" @@ -190,16 +220,19 @@ class ParseLogsTest(oeRuntimeTest): #copy the log files to be parsed locally def transfer_logs(self, log_list): - target_logs = 'target_logs' + workdir = self.getWorkdir() + self.target_logs = workdir + '/' + 'target_logs' + target_logs = self.target_logs if not os.path.exists(target_logs): os.makedirs(target_logs) + bb.utils.remove(self.target_logs + "/*") for f in log_list: self.target.copy_from(f, target_logs) #get the local list of logs def get_local_log_list(self, log_locations): self.transfer_logs(self.getLogList(log_locations)) - logs = [ os.path.join('target_logs',f) for f in os.listdir('target_logs') if os.path.isfile(os.path.join('target_logs',f)) ] + logs = [ os.path.join(self.target_logs, f) for f in os.listdir(self.target_logs) if os.path.isfile(os.path.join(self.target_logs, f)) ] return logs #build the grep command to be used with filters and exclusions @@ -238,7 +271,7 @@ class ParseLogsTest(oeRuntimeTest): result = None thegrep = self.build_grepcmd(errors, ignore_errors, log) try: - result = subprocess.check_output(thegrep, shell=True) + result = subprocess.check_output(thegrep, shell=True).decode("utf-8") except: pass if (result is not None): @@ -246,7 +279,7 @@ class ParseLogsTest(oeRuntimeTest): rez = result.splitlines() for xrez in rez: try: - grep_output = subprocess.check_output(['grep', '-F', xrez, '-B', str(lines_before), '-A', str(lines_after), log]) + grep_output = subprocess.check_output(['grep', '-F', xrez, '-B', str(lines_before), '-A', str(lines_after), log]).decode("utf-8") except: pass results[log.replace('target_logs/','')][xrez]=grep_output @@ -262,7 +295,7 @@ class ParseLogsTest(oeRuntimeTest): self.write_dmesg() log_list = self.get_local_log_list(self.log_locations) result = self.parse_logs(self.errors, self.ignore_errors, log_list) - print self.getHardwareInfo() + print(self.getHardwareInfo()) errcount = 0 for log in result: self.msg += "Log: "+log+"\n" diff --git a/import-layers/yocto-poky/meta/lib/oeqa/runtime/ping.py b/import-layers/yocto-poky/meta/lib/oeqa/runtime/ping.py index 80c460161..0f2744792 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/runtime/ping.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/runtime/ping.py @@ -14,7 +14,7 @@ class PingTest(oeRuntimeTest): endtime = time.time() + 60 while count < 5 and time.time() < endtime: proc = subprocess.Popen("ping -c 1 %s" % self.target.ip, shell=True, stdout=subprocess.PIPE) - output += proc.communicate()[0] + output += proc.communicate()[0].decode("utf-8") if proc.poll() == 0: count += 1 else: diff --git a/import-layers/yocto-poky/meta/lib/oeqa/runtime/python.py b/import-layers/yocto-poky/meta/lib/oeqa/runtime/python.py index 26edb7a9b..29a231c7c 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/runtime/python.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/runtime/python.py @@ -4,7 +4,7 @@ from oeqa.oetest import oeRuntimeTest, skipModule from oeqa.utils.decorators import * def setUpModule(): - if not oeRuntimeTest.hasPackage("python"): + if not oeRuntimeTest.hasPackage("python-core"): skipModule("No python package in the image") diff --git a/import-layers/yocto-poky/meta/lib/oeqa/runtime/rpm.py b/import-layers/yocto-poky/meta/lib/oeqa/runtime/rpm.py index 624c515aa..7f514ca00 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/runtime/rpm.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/runtime/rpm.py @@ -51,12 +51,32 @@ class RpmInstallRemoveTest(oeRuntimeTest): @testcase(1096) @skipUnlessPassed('test_ssh') def test_rpm_query_nonroot(self): - (status, output) = self.target.run('useradd test1') - self.assertTrue(status == 0, msg="Failed to create new user: " + output) - (status, output) = self.target.run('sudo -u test1 id') - self.assertTrue('(test1)' in output, msg="Failed to execute as new user") - (status, output) = self.target.run('sudo -u test1 rpm -qa') - self.assertEqual(status, 0, msg="status: %s. Cannot run rpm -qa: %s" % (status, output)) + + def set_up_test_user(u): + (status, output) = self.target.run("id -u %s" % u) + if status == 0: + pass + else: + (status, output) = self.target.run("useradd %s" % u) + self.assertTrue(status == 0, msg="Failed to create new user: " + output) + + def exec_as_test_user(u): + (status, output) = self.target.run("su -c id %s" % u) + self.assertTrue("({0})".format(u) in output, msg="Failed to execute as new user") + (status, output) = self.target.run("su -c \"rpm -qa\" %s " % u) + self.assertEqual(status, 0, msg="status: %s. Cannot run rpm -qa: %s" % (status, output)) + + def unset_up_test_user(u): + (status, output) = self.target.run("userdel -r %s" % u) + self.assertTrue(status == 0, msg="Failed to erase user: %s" % output) + + tuser = 'test1' + + try: + set_up_test_user(tuser) + exec_as_test_user(tuser) + finally: + unset_up_test_user(tuser) @testcase(195) @skipUnlessPassed('test_rpm_install') @@ -98,4 +118,3 @@ class RpmInstallRemoveTest(oeRuntimeTest): @classmethod def tearDownClass(self): oeRuntimeTest.tc.target.run('rm -f /tmp/rpm-doc.rpm') - diff --git a/import-layers/yocto-poky/meta/lib/oeqa/runtime/smart.py b/import-layers/yocto-poky/meta/lib/oeqa/runtime/smart.py index 126d61463..6cdb10d63 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/runtime/smart.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/runtime/smart.py @@ -1,5 +1,7 @@ import unittest import re +import oe +import subprocess from oeqa.oetest import oeRuntimeTest, skipModule from oeqa.utils.decorators import * from oeqa.utils.httpserver import HTTPService @@ -7,7 +9,7 @@ from oeqa.utils.httpserver import HTTPService def setUpModule(): if not oeRuntimeTest.hasFeature("package-management"): skipModule("Image doesn't have package management feature") - if not oeRuntimeTest.hasPackage("smart"): + if not oeRuntimeTest.hasPackage("smartpm"): skipModule("Image doesn't have smart installed") if "package_rpm" != oeRuntimeTest.tc.d.getVar("PACKAGE_CLASSES", True).split()[0]: skipModule("Rpm is not the primary package manager") @@ -53,9 +55,50 @@ class SmartBasicTest(SmartTest): class SmartRepoTest(SmartTest): @classmethod + def create_index(self, arg): + index_cmd = arg + try: + bb.note("Executing '%s' ..." % index_cmd) + result = subprocess.check_output(index_cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8") + except subprocess.CalledProcessError as e: + return("Index creation command '%s' failed with return code %d:\n%s" % + (e.cmd, e.returncode, e.output.decode("utf-8"))) + if result: + bb.note(result) + return None + + @classmethod def setUpClass(self): self.repolist = [] - self.repo_server = HTTPService(oeRuntimeTest.tc.d.getVar('DEPLOY_DIR', True), oeRuntimeTest.tc.target.server_ip) + + # Index RPMs + rpm_createrepo = bb.utils.which(os.getenv('PATH'), "createrepo") + index_cmds = [] + rpm_dirs_found = False + archs = (oeRuntimeTest.tc.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS', True) or "").replace('-', '_').split() + for arch in archs: + rpm_dir = os.path.join(oeRuntimeTest.tc.d.getVar('DEPLOY_DIR_RPM', True), arch) + idx_path = os.path.join(oeRuntimeTest.tc.d.getVar('WORKDIR', True), 'rpm', arch) + db_path = os.path.join(oeRuntimeTest.tc.d.getVar('WORKDIR', True), 'rpmdb', arch) + if not os.path.isdir(rpm_dir): + continue + if os.path.exists(db_path): + bb.utils.remove(dbpath, True) + lockfilename = oeRuntimeTest.tc.d.getVar('DEPLOY_DIR_RPM', True) + "/rpm.lock" + lf = bb.utils.lockfile(lockfilename, False) + oe.path.copyhardlinktree(rpm_dir, idx_path) + # Full indexes overload a 256MB image so reduce the number of rpms + # in the feed. Filter to p* since we use the psplash packages and + # this leaves some allarch and machine arch packages too. + bb.utils.remove(idx_path + "*/[a-oq-z]*.rpm") + bb.utils.unlockfile(lf) + index_cmds.append("%s --dbpath %s --update -q %s" % (rpm_createrepo, db_path, idx_path)) + rpm_dirs_found = True + # Create repodata¬ + result = oe.utils.multiprocess_exec(index_cmds, self.create_index) + if result: + bb.fatal('%s' % ('\n'.join(result))) + self.repo_server = HTTPService(oeRuntimeTest.tc.d.getVar('WORKDIR', True), oeRuntimeTest.tc.target.server_ip) self.repo_server.start() @classmethod diff --git a/import-layers/yocto-poky/meta/lib/oeqa/runtime/syslog.py b/import-layers/yocto-poky/meta/lib/oeqa/runtime/syslog.py index 2601dd9ea..8f550329e 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/runtime/syslog.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/runtime/syslog.py @@ -9,7 +9,6 @@ def setUpModule(): class SyslogTest(oeRuntimeTest): @testcase(201) - @skipUnlessPassed("test_syslog_help") def test_syslog_running(self): (status,output) = self.target.run(oeRuntimeTest.pscmd + ' | grep -i [s]yslogd') self.assertEqual(status, 0, msg="no syslogd process, ps output: %s" % self.target.run(oeRuntimeTest.pscmd)[1]) @@ -19,8 +18,16 @@ class SyslogTestConfig(oeRuntimeTest): @testcase(1149) @skipUnlessPassed("test_syslog_running") def test_syslog_logger(self): - (status,output) = self.target.run('logger foobar && test -e /var/log/messages && grep foobar /var/log/messages || logread | grep foobar') - self.assertEqual(status, 0, msg="Test log string not found in /var/log/messages. Output: %s " % output) + (status, output) = self.target.run('logger foobar') + self.assertEqual(status, 0, msg="Can't log into syslog. Output: %s " % output) + + (status, output) = self.target.run('grep foobar /var/log/messages') + if status != 0: + if oeRuntimeTest.tc.d.getVar("VIRTUAL-RUNTIME_init_manager", "") == "systemd": + (status, output) = self.target.run('journalctl -o cat | grep foobar') + else: + (status, output) = self.target.run('logread | grep foobar') + self.assertEqual(status, 0, msg="Test log string not found in /var/log/messages or logread. Output: %s " % output) @testcase(1150) @skipUnlessPassed("test_syslog_running") diff --git a/import-layers/yocto-poky/meta/lib/oeqa/runtime/systemd.py b/import-layers/yocto-poky/meta/lib/oeqa/runtime/systemd.py index 2b2f10d71..8de799cd6 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/runtime/systemd.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/runtime/systemd.py @@ -57,7 +57,7 @@ class SystemdBasicTests(SystemdTest): self.systemctl('--version') @testcase(551) - @skipUnlessPassed('test_system_basic') + @skipUnlessPassed('test_systemd_basic') def test_systemd_list(self): self.systemctl('list-unit-files') @@ -153,7 +153,7 @@ class SystemdJournalTests(SystemdTest): if check_match: break # put the startup time in the test log if check_match: - print "%s" % check_match + print("%s" % check_match) else: self.skipTest("Error at obtaining the boot time from journalctl") boot_time_sec = 0 @@ -174,5 +174,5 @@ class SystemdJournalTests(SystemdTest): self.skipTest("Error when parsing time from boot string") #Assert the target boot time against systemd's unit start timeout if boot_time_sec > systemd_TimeoutStartSec: - print "Target boot time %s exceeds systemd's TimeoutStartSec %s"\ - %(boot_time_sec, systemd_TimeoutStartSec) + print("Target boot time %s exceeds systemd's TimeoutStartSec %s"\ + %(boot_time_sec, systemd_TimeoutStartSec)) diff --git a/import-layers/yocto-poky/meta/lib/oeqa/sdk/buildgalculator.py b/import-layers/yocto-poky/meta/lib/oeqa/sdk/buildgalculator.py new file mode 100644 index 000000000..dc2fa9ce1 --- /dev/null +++ b/import-layers/yocto-poky/meta/lib/oeqa/sdk/buildgalculator.py @@ -0,0 +1,27 @@ +from oeqa.oetest import oeSDKTest, skipModule +from oeqa.utils.decorators import * +from oeqa.utils.targetbuild import SDKBuildProject + +def setUpModule(): + if not (oeSDKTest.hasPackage("gtk+3") or oeSDKTest.hasPackage("libgtk-3.0")): + skipModule("Image doesn't have gtk+3 in manifest") + +class GalculatorTest(oeSDKTest): + def test_galculator(self): + try: + project = SDKBuildProject(oeSDKTest.tc.sdktestdir + "/galculator/", + oeSDKTest.tc.sdkenv, oeSDKTest.tc.d, + "http://galculator.mnim.org/downloads/galculator-2.1.4.tar.bz2") + + project.download_archive() + + # regenerate configure to get support for --with-libtool-sysroot + legacy_preconf=("autoreconf -i -f -I ${OECORE_TARGET_SYSROOT}/usr/share/aclocal -I m4;") + + self.assertEqual(project.run_configure(extra_cmds=legacy_preconf), + 0, msg="Running configure failed") + + self.assertEqual(project.run_make(), 0, + msg="Running make failed") + finally: + project.clean() diff --git a/import-layers/yocto-poky/meta/lib/oeqa/sdk/buildiptables.py b/import-layers/yocto-poky/meta/lib/oeqa/sdk/buildiptables.py index 062e5316e..f0cb8a428 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/sdk/buildiptables.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/sdk/buildiptables.py @@ -8,7 +8,7 @@ class BuildIptablesTest(oeSDKTest): @classmethod def setUpClass(self): self.project = SDKBuildProject(oeSDKTest.tc.sdktestdir + "/iptables/", oeSDKTest.tc.sdkenv, oeSDKTest.tc.d, - "http://netfilter.org/projects/iptables/files/iptables-1.4.13.tar.bz2") + "http://downloads.yoctoproject.org/mirror/sources/iptables-1.4.13.tar.bz2") self.project.download_archive() def test_iptables(self): diff --git a/import-layers/yocto-poky/meta/lib/oeqa/sdk/buildsudoku.py b/import-layers/yocto-poky/meta/lib/oeqa/sdk/buildsudoku.py deleted file mode 100644 index dea77c659..000000000 --- a/import-layers/yocto-poky/meta/lib/oeqa/sdk/buildsudoku.py +++ /dev/null @@ -1,26 +0,0 @@ -from oeqa.oetest import oeSDKTest, skipModule -from oeqa.utils.decorators import * -from oeqa.utils.targetbuild import SDKBuildProject - -def setUpModule(): - if not oeSDKTest.hasPackage("gtk\+"): - skipModule("Image doesn't have gtk+ in manifest") - -class SudokuTest(oeSDKTest): - - @classmethod - def setUpClass(self): - self.project = SDKBuildProject(oeSDKTest.tc.sdktestdir + "/sudoku/", oeSDKTest.tc.sdkenv, oeSDKTest.tc.d, - "http://downloads.sourceforge.net/project/sudoku-savant/sudoku-savant/sudoku-savant-1.3/sudoku-savant-1.3.tar.bz2") - self.project.download_archive() - - def test_sudoku(self): - self.assertEqual(self.project.run_configure(), 0, - msg="Running configure failed") - - self.assertEqual(self.project.run_make(), 0, - msg="Running make failed") - - @classmethod - def tearDownClass(self): - self.project.clean() diff --git a/import-layers/yocto-poky/meta/lib/oeqa/sdkext/devtool.py b/import-layers/yocto-poky/meta/lib/oeqa/sdkext/devtool.py index c5bb3102a..65f41f687 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/sdkext/devtool.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/sdkext/devtool.py @@ -1,32 +1,108 @@ import shutil - +import subprocess +import urllib.request from oeqa.oetest import oeSDKExtTest from oeqa.utils.decorators import * class DevtoolTest(oeSDKExtTest): - @classmethod def setUpClass(self): self.myapp_src = os.path.join(self.tc.sdkextfilesdir, "myapp") self.myapp_dst = os.path.join(self.tc.sdktestdir, "myapp") shutil.copytree(self.myapp_src, self.myapp_dst) + self.myapp_cmake_src = os.path.join(self.tc.sdkextfilesdir, "myapp_cmake") + self.myapp_cmake_dst = os.path.join(self.tc.sdktestdir, "myapp_cmake") + shutil.copytree(self.myapp_cmake_src, self.myapp_cmake_dst) + + def _test_devtool_build(self, directory): + self._run('devtool add myapp %s' % directory) + try: + self._run('devtool build myapp') + except Exception as e: + print(e.output) + self._run('devtool reset myapp') + raise e + self._run('devtool reset myapp') + + def _test_devtool_build_package(self, directory): + self._run('devtool add myapp %s' % directory) + try: + self._run('devtool package myapp') + except Exception as e: + print(e.output) + self._run('devtool reset myapp') + raise e + self._run('devtool reset myapp') + def test_devtool_location(self): output = self._run('which devtool') self.assertEqual(output.startswith(self.tc.sdktestdir), True, \ msg="Seems that devtool isn't the eSDK one: %s" % output) - + @skipUnlessPassed('test_devtool_location') def test_devtool_add_reset(self): self._run('devtool add myapp %s' % self.myapp_dst) self._run('devtool reset myapp') + + @testcase(1473) + @skipUnlessPassed('test_devtool_location') + def test_devtool_build_make(self): + self._test_devtool_build(self.myapp_dst) + + @testcase(1474) + @skipUnlessPassed('test_devtool_location') + def test_devtool_build_esdk_package(self): + self._test_devtool_build_package(self.myapp_dst) + @testcase(1479) @skipUnlessPassed('test_devtool_location') - def test_devtool_build(self): - self._run('devtool add myapp %s' % self.myapp_dst) - self._run('devtool build myapp') - self._run('devtool reset myapp') + def test_devtool_build_cmake(self): + self._test_devtool_build(self.myapp_cmake_dst) + + @testcase(1482) + @skipUnlessPassed('test_devtool_location') + def test_extend_autotools_recipe_creation(self): + req = 'https://github.com/rdfa/librdfa' + recipe = "bbexample" + self._run('devtool add %s %s' % (recipe, req) ) + try: + self._run('devtool build %s' % recipe) + except Exception as e: + print(e.output) + self._run('devtool reset %s' % recipe) + raise e + self._run('devtool reset %s' % recipe) + + @testcase(1484) + @skipUnlessPassed('test_devtool_location') + def test_devtool_kernelmodule(self): + docfile = 'https://github.com/umlaeute/v4l2loopback.git' + recipe = 'v4l2loopback-driver' + self._run('devtool add %s %s' % (recipe, docfile) ) + try: + self._run('devtool build %s' % recipe) + except Exception as e: + print(e.output) + self._run('devtool reset %s' % recipe) + raise e + self._run('devtool reset %s' % recipe) + + @testcase(1478) + @skipUnlessPassed('test_devtool_location') + def test_recipes_for_nodejs(self): + package_nodejs = "npm://registry.npmjs.org;name=winston;version=2.2.0" + self._run('devtool add %s ' % package_nodejs) + try: + self._run('devtool build %s ' % package_nodejs) + except Exception as e: + print(e.output) + self._run('devtool reset %s' % package_nodejs) + raise e + self._run('devtool reset %s '% package_nodejs) + @classmethod def tearDownClass(self): shutil.rmtree(self.myapp_dst) + shutil.rmtree(self.myapp_cmake_dst) diff --git a/import-layers/yocto-poky/meta/lib/oeqa/sdkext/files/myapp_cmake/CMakeLists.txt b/import-layers/yocto-poky/meta/lib/oeqa/sdkext/files/myapp_cmake/CMakeLists.txt new file mode 100644 index 000000000..19d773dd6 --- /dev/null +++ b/import-layers/yocto-poky/meta/lib/oeqa/sdkext/files/myapp_cmake/CMakeLists.txt @@ -0,0 +1,11 @@ +cmake_minimum_required (VERSION 2.6) +project (myapp) +# The version number. +set (myapp_VERSION_MAJOR 1) +set (myapp_VERSION_MINOR 0) + +# add the executable +add_executable (myapp myapp.c) + +install(TARGETS myapp + RUNTIME DESTINATION bin) diff --git a/import-layers/yocto-poky/meta/lib/oeqa/sdkext/files/myapp_cmake/myapp.c b/import-layers/yocto-poky/meta/lib/oeqa/sdkext/files/myapp_cmake/myapp.c new file mode 100644 index 000000000..f0b63f03f --- /dev/null +++ b/import-layers/yocto-poky/meta/lib/oeqa/sdkext/files/myapp_cmake/myapp.c @@ -0,0 +1,9 @@ +#include <stdio.h> + +int +main(int argc, char *argv[]) +{ + printf("Hello world\n"); + + return 0; +} diff --git a/import-layers/yocto-poky/meta/lib/oeqa/sdkext/sdk_update.py b/import-layers/yocto-poky/meta/lib/oeqa/sdkext/sdk_update.py index 7a2a6fe7c..2ade839c0 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/sdkext/sdk_update.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/sdkext/sdk_update.py @@ -30,9 +30,6 @@ class SdkUpdateTest(oeSDKExtTest): def test_sdk_update_http(self): output = self._run("devtool sdk-update \"%s\"" % self.http_url) - def test_sdk_update_local(self): - output = self._run("devtool sdk-update \"%s\"" % self.publish_dir) - @classmethod def tearDownClass(self): self.http_service.stop() diff --git a/import-layers/yocto-poky/meta/lib/oeqa/selftest/_toaster.py b/import-layers/yocto-poky/meta/lib/oeqa/selftest/_toaster.py index c424659fd..15ea9df9e 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/selftest/_toaster.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/selftest/_toaster.py @@ -2,7 +2,7 @@ import unittest import os import sys import shlex, subprocess -import urllib, commands, time, getpass, re, json, shlex +import urllib.request, urllib.parse, urllib.error, subprocess, time, getpass, re, json, shlex import oeqa.utils.ftools as ftools from oeqa.selftest.base import oeSelfTest @@ -290,7 +290,7 @@ class Toaster_DB_Tests(ToasterSetup): layers = Layer.objects.values('id', 'layer_index_url') cnt_err = [] for layer in layers: - resp = urllib.urlopen(layer['layer_index_url']) + resp = urllib.request.urlopen(layer['layer_index_url']) if (resp.getcode() != 200): cnt_err.append(layer['id']) self.assertEqual(len(cnt_err), 0, msg = 'Errors for layer id: %s' % cnt_err) diff --git a/import-layers/yocto-poky/meta/lib/oeqa/selftest/base.py b/import-layers/yocto-poky/meta/lib/oeqa/selftest/base.py index e10455edc..26c93f905 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/selftest/base.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/selftest/base.py @@ -28,17 +28,47 @@ class oeSelfTest(unittest.TestCase): def __init__(self, methodName="runTest"): self.builddir = os.environ.get("BUILDDIR") self.localconf_path = os.path.join(self.builddir, "conf/local.conf") + self.localconf_backup = os.path.join(self.builddir, "conf/local.bk") self.testinc_path = os.path.join(self.builddir, "conf/selftest.inc") self.local_bblayers_path = os.path.join(self.builddir, "conf/bblayers.conf") + self.local_bblayers_backup = os.path.join(self.builddir, + "conf/bblayers.bk") self.testinc_bblayers_path = os.path.join(self.builddir, "conf/bblayers.inc") self.machineinc_path = os.path.join(self.builddir, "conf/machine.inc") self.testlayer_path = oeSelfTest.testlayer_path self._extra_tear_down_commands = [] - self._track_for_cleanup = [self.testinc_path, self.testinc_bblayers_path, self.machineinc_path] + self._track_for_cleanup = [ + self.testinc_path, self.testinc_bblayers_path, + self.machineinc_path, self.localconf_backup, + self.local_bblayers_backup] super(oeSelfTest, self).__init__(methodName) def setUp(self): os.chdir(self.builddir) + # Check if local.conf or bblayers.conf files backup exists + # from a previous failed test and restore them + if os.path.isfile(self.localconf_backup) or os.path.isfile( + self.local_bblayers_backup): + self.log.debug("Found a local.conf and/or bblayers.conf backup \ +from a previously aborted test. Restoring these files now, but tests should \ +be re-executed from a clean environment to ensure accurate results.") + try: + shutil.copyfile(self.localconf_backup, self.localconf_path) + except OSError as e: + if e.errno != errno.ENOENT: + raise + try: + shutil.copyfile(self.local_bblayers_backup, + self.local_bblayers_path) + except OSError as e: + if e.errno != errno.ENOENT: + raise + else: + # backup local.conf and bblayers.conf + shutil.copyfile(self.localconf_path, self.localconf_backup) + shutil.copyfile(self.local_bblayers_path, + self.local_bblayers_backup) + self.log.debug("Creating local.conf and bblayers.conf backups.") # we don't know what the previous test left around in config or inc files # if it failed so we need a fresh start try: @@ -67,7 +97,7 @@ class oeSelfTest(unittest.TestCase): machine = custommachine machine_conf = 'MACHINE ??= "%s"\n' % machine self.set_machine_config(machine_conf) - print 'MACHINE: %s' % machine + print('MACHINE: %s' % machine) # tests might need their own setup # but if they overwrite this one they have to call diff --git a/import-layers/yocto-poky/meta/lib/oeqa/selftest/bbtests.py b/import-layers/yocto-poky/meta/lib/oeqa/selftest/bbtests.py index 26728a4b4..baae1e0e5 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/selftest/bbtests.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/selftest/bbtests.py @@ -29,7 +29,7 @@ class BitbakeTests(oeSelfTest): def test_event_handler(self): self.write_config("INHERIT += \"test_events\"") result = bitbake('m4-native') - find_build_started = re.search("NOTE: Test for bb\.event\.BuildStarted(\n.*)*NOTE: Preparing RunQueue", result.output) + find_build_started = re.search("NOTE: Test for bb\.event\.BuildStarted(\n.*)*NOTE: Executing RunQueue Tasks", result.output) find_build_completed = re.search("Tasks Summary:.*(\n.*)*NOTE: Test for bb\.event\.BuildCompleted", result.output) self.assertTrue(find_build_started, msg = "Match failed in:\n%s" % result.output) self.assertTrue(find_build_completed, msg = "Match failed in:\n%s" % result.output) @@ -64,12 +64,15 @@ class BitbakeTests(oeSelfTest): @testcase(108) def test_invalid_patch(self): + # This patch already exists in SRC_URI so adding it again will cause the + # patch to fail. self.write_recipeinc('man', 'SRC_URI += "file://man-1.5h1-make.patch"') + self.write_config("INHERIT_remove = \"report-error\"") result = bitbake('man -c patch', ignore_status=True) self.delete_recipeinc('man') bitbake('-cclean man') line = self.getline(result, "Function failed: patch_do_patch") - self.assertTrue(line and line.startswith("ERROR:"), msg = "Though no man-1.5h1-make.patch file exists, bitbake didn't output any err. message. bitbake output: %s" % result.output) + self.assertTrue(line and line.startswith("ERROR:"), msg = "Repeated patch application didn't fail. bitbake output: %s" % result.output) @testcase(1354) def test_force_task_1(self): @@ -131,6 +134,7 @@ class BitbakeTests(oeSelfTest): self.write_recipeinc('man', data) self.write_config("""DL_DIR = \"${TOPDIR}/download-selftest\" SSTATE_DIR = \"${TOPDIR}/download-selftest\" +INHERIT_remove = \"report-error\" """) self.track_for_cleanup(os.path.join(self.builddir, "download-selftest")) @@ -141,7 +145,7 @@ SSTATE_DIR = \"${TOPDIR}/download-selftest\" self.assertEqual(result.status, 1, msg="Command succeded when it should have failed. bitbake output: %s" % result.output) self.assertTrue('Fetcher failure: Unable to find file file://invalid anywhere. The paths that were searched were:' in result.output, msg = "\"invalid\" file \ doesn't exist, yet no error message encountered. bitbake output: %s" % result.output) - line = self.getline(result, 'Function failed: Fetcher failure for URL: \'file://invalid\'. Unable to fetch URL from any source.') + line = self.getline(result, 'Fetcher failure for URL: \'file://invalid\'. Unable to fetch URL from any source.') self.assertTrue(line and line.startswith("ERROR:"), msg = "\"invalid\" file \ doesn't exist, yet fetcher didn't report any error. bitbake output: %s" % result.output) @@ -212,6 +216,7 @@ SSTATE_DIR = \"${TOPDIR}/download-selftest\" def test_continue(self): self.write_config("""DL_DIR = \"${TOPDIR}/download-selftest\" SSTATE_DIR = \"${TOPDIR}/download-selftest\" +INHERIT_remove = \"report-error\" """) self.track_for_cleanup(os.path.join(self.builddir, "download-selftest")) self.write_recipeinc('man',"\ndo_fail_task () {\nexit 1 \n}\n\naddtask do_fail_task before do_fetch\n" ) diff --git a/import-layers/yocto-poky/meta/lib/oeqa/selftest/buildoptions.py b/import-layers/yocto-poky/meta/lib/oeqa/selftest/buildoptions.py index 35d5dfd29..9487898b0 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/selftest/buildoptions.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/selftest/buildoptions.py @@ -30,22 +30,6 @@ class ImageOptionsTests(oeSelfTest): incremental_removed = re.search("NOTE: load old install solution for incremental install\nNOTE: creating new install solution for incremental install(\n.*)*NOTE: incremental removed:.*openssh-sshd-.*", log_data_removed) self.assertTrue(incremental_removed, msg = "Match failed in:\n%s" % log_data_removed) - @testcase(925) - def test_rm_old_image(self): - bitbake("core-image-minimal") - deploydir = get_bb_var("DEPLOY_DIR_IMAGE", target="core-image-minimal") - imagename = get_bb_var("IMAGE_LINK_NAME", target="core-image-minimal") - deploydir_files = os.listdir(deploydir) - track_original_files = [] - for image_file in deploydir_files: - if imagename in image_file and os.path.islink(os.path.join(deploydir, image_file)): - track_original_files.append(os.path.realpath(os.path.join(deploydir, image_file))) - self.write_config("RM_OLD_IMAGE = \"1\"") - bitbake("-C rootfs core-image-minimal") - deploydir_files = os.listdir(deploydir) - remaining_not_expected = [path for path in track_original_files if os.path.basename(path) in deploydir_files] - self.assertFalse(remaining_not_expected, msg="\nThe following image files were not removed: %s" % ', '.join(map(str, remaining_not_expected))) - @testcase(286) def test_ccache_tool(self): bitbake("ccache-native") @@ -89,8 +73,9 @@ class SanityOptionsTest(oeSelfTest): def test_options_warnqa_errorqa_switch(self): bitbake("xcursor-transparent-theme -ccleansstate") + self.write_config("INHERIT_remove = \"report-error\"") if "packages-list" not in get_bb_var("ERROR_QA"): - self.write_config("ERROR_QA_append = \" packages-list\"") + self.append_config("ERROR_QA_append = \" packages-list\"") self.write_recipeinc('xcursor-transparent-theme', 'PACKAGES += \"${PN}-dbg\"') res = bitbake("xcursor-transparent-theme", ignore_status=True) @@ -199,78 +184,6 @@ class BuildhistoryTests(BuildhistoryBase): self.run_buildhistory_operation(target, target_config="PR = \"r1\"", change_bh_location=True) self.run_buildhistory_operation(target, target_config="PR = \"r0\"", change_bh_location=False, expect_error=True, error_regex=error) - @testcase(1386) - def test_buildhistory_does_not_change_signatures(self): - """ - Summary: Ensure that buildhistory does not change signatures - Expected: Only 'do_rootfs' task should be rerun - Product: oe-core - Author: Daniel Istrate <daniel.alexandrux.istrate@intel.com> - AutomatedBy: Daniel Istrate <daniel.alexandrux.istrate@intel.com> - """ - - tmpdir1_name = 'tmpsig1' - tmpdir2_name = 'tmpsig2' - builddir = os.environ.get('BUILDDIR') - tmpdir1 = os.path.join(builddir, tmpdir1_name) - tmpdir2 = os.path.join(builddir, tmpdir2_name) - - self.track_for_cleanup(tmpdir1) - self.track_for_cleanup(tmpdir2) - - features = 'TMPDIR = "%s"\n' % tmpdir1 - self.write_config(features) - bitbake('core-image-minimal -S none -c rootfs') - - features = 'TMPDIR = "%s"\n' % tmpdir2 - features += 'INHERIT += "buildhistory"\n' - self.write_config(features) - bitbake('core-image-minimal -S none -c rootfs') - - def get_files(d): - f = [] - for root, dirs, files in os.walk(d): - for name in files: - f.append(os.path.join(root, name)) - return f - - files1 = get_files(tmpdir1 + '/stamps') - files2 = get_files(tmpdir2 + '/stamps') - files2 = [x.replace(tmpdir2_name, tmpdir1_name) for x in files2] - - f1 = set(files1) - f2 = set(files2) - sigdiff = f1 - f2 - - self.assertEqual(len(sigdiff), 1, 'Expected 1 signature differences. Out: %s' % list(sigdiff)) - - unexpected_diff = [] - - # No new signatures should appear apart from do_rootfs - found_do_rootfs_flag = False - - for sig in sigdiff: - if 'do_rootfs' in sig: - found_do_rootfs_flag = True - else: - unexpected_diff.append(sig) - - self.assertTrue(found_do_rootfs_flag, 'Task do_rootfs did not rerun.') - self.assertFalse(unexpected_diff, 'Found unexpected signature differences. Out: %s' % unexpected_diff) - - -class BuildImagesTest(oeSelfTest): - @testcase(563) - def test_directfb(self): - """ - This method is used to test the build of directfb image for arm arch. - In essence we build a coreimagedirectfb and test the exitcode of bitbake that in case of success is 0. - """ - self.add_command_to_tearDown('cleanup-workdir') - self.write_config("DISTRO_FEATURES_remove = \"x11\"\nDISTRO_FEATURES_append = \" directfb\"\nMACHINE ??= \"qemuarm\"") - res = bitbake("core-image-directfb", ignore_status=True) - self.assertEqual(res.status, 0, "\ncoreimagedirectfb failed to build. Please check logs for further details.\nbitbake output %s" % res.output) - class ArchiverTest(oeSelfTest): @testcase(926) def test_arch_work_dir_and_export_source(self): diff --git a/import-layers/yocto-poky/meta/lib/oeqa/selftest/devtool.py b/import-layers/yocto-poky/meta/lib/oeqa/selftest/devtool.py index 132a73d0e..e992dcf77 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/selftest/devtool.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/selftest/devtool.py @@ -5,10 +5,11 @@ import re import shutil import tempfile import glob +import fnmatch import oeqa.utils.ftools as ftools from oeqa.selftest.base import oeSelfTest -from oeqa.utils.commands import runCmd, bitbake, get_bb_var, create_temp_layer, runqemu +from oeqa.utils.commands import runCmd, bitbake, get_bb_var, create_temp_layer, runqemu, get_test_layer from oeqa.utils.decorators import testcase class DevtoolBase(oeSelfTest): @@ -50,7 +51,7 @@ class DevtoolBase(oeSelfTest): missingvars = {} - for var, value in checkvars.iteritems(): + for var, value in checkvars.items(): if value is not None: missingvars[var] = value self.assertEqual(missingvars, {}, 'Some expected variables not found in recipe: %s' % checkvars) @@ -207,12 +208,14 @@ class DevtoolTests(DevtoolBase): tempdir = tempfile.mkdtemp(prefix='devtoolqa') self.track_for_cleanup(tempdir) pn = 'dbus-wait' + srcrev = '6cc6077a36fe2648a5f993fe7c16c9632f946517' # We choose an https:// git URL here to check rewriting the URL works url = 'https://git.yoctoproject.org/git/dbus-wait' # Force fetching to "noname" subdir so we verify we're picking up the name from autoconf # instead of the directory name result = runCmd('git clone %s noname' % url, cwd=tempdir) srcdir = os.path.join(tempdir, 'noname') + result = runCmd('git reset --hard %s' % srcrev, cwd=srcdir) self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure.ac')), 'Unable to find configure script in source directory') # Test devtool add self.track_for_cleanup(self.workspacedir) @@ -235,7 +238,7 @@ class DevtoolTests(DevtoolBase): checkvars['S'] = '${WORKDIR}/git' checkvars['PV'] = '0.1+git${SRCPV}' checkvars['SRC_URI'] = 'git://git.yoctoproject.org/git/dbus-wait;protocol=https' - checkvars['SRCREV'] = '${AUTOREV}' + checkvars['SRCREV'] = srcrev checkvars['DEPENDS'] = set(['dbus']) self._test_recipe_contents(recipefile, checkvars, []) @@ -345,7 +348,7 @@ class DevtoolTests(DevtoolBase): self.track_for_cleanup(self.workspacedir) self.add_command_to_tearDown('bitbake -c cleansstate %s' % testrecipe) self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') - result = runCmd('devtool add %s %s -f %s' % (testrecipe, srcdir, url)) + result = runCmd('devtool add %s %s -a -f %s' % (testrecipe, srcdir, url)) self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created: %s' % result.output) self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure.ac')), 'Unable to find configure.ac in source directory') # Test devtool status @@ -357,7 +360,7 @@ class DevtoolTests(DevtoolBase): self.assertIn('_git.bb', recipefile, 'Recipe file incorrectly named') checkvars = {} checkvars['S'] = '${WORKDIR}/git' - checkvars['PV'] = '1.11+git${SRCPV}' + checkvars['PV'] = '1.12+git${SRCPV}' checkvars['SRC_URI'] = url checkvars['SRCREV'] = '${AUTOREV}' self._test_recipe_contents(recipefile, checkvars, []) @@ -465,12 +468,11 @@ class DevtoolTests(DevtoolBase): testrecipes = 'perf kernel-devsrc package-index core-image-minimal meta-toolchain packagegroup-core-sdk meta-ide-support'.split() # Find actual name of gcc-source since it now includes the version - crude, but good enough for this purpose result = runCmd('bitbake-layers show-recipes gcc-source*') - reading = False for line in result.output.splitlines(): - if line.startswith('=='): - reading = True - elif reading and not line.startswith(' '): - testrecipes.append(line.split(':')[0]) + # just match those lines that contain a real target + m = re.match('(?P<recipe>^[a-zA-Z0-9.-]+)(?P<colon>:$)', line) + if m: + testrecipes.append(m.group('recipe')) for testrecipe in testrecipes: # Check it's a valid recipe bitbake('%s -e' % testrecipe) @@ -816,28 +818,28 @@ class DevtoolTests(DevtoolBase): # Check bbappend contents result = runCmd('git rev-parse HEAD', cwd=tempsrcdir) - expectedlines = ['SRCREV = "%s"\n' % result.output, - '\n', - 'SRC_URI = "%s"\n' % git_uri, - '\n'] + expectedlines = set(['SRCREV = "%s"\n' % result.output, + '\n', + 'SRC_URI = "%s"\n' % git_uri, + '\n']) with open(bbappendfile, 'r') as f: - self.assertEqual(expectedlines, f.readlines()) + self.assertEqual(expectedlines, set(f.readlines())) # Check we can run it again and bbappend isn't modified result = runCmd('devtool update-recipe -m srcrev %s -a %s' % (testrecipe, templayerdir)) with open(bbappendfile, 'r') as f: - self.assertEqual(expectedlines, f.readlines()) + self.assertEqual(expectedlines, set(f.readlines())) # Drop new commit and check SRCREV changes result = runCmd('git reset HEAD^', cwd=tempsrcdir) result = runCmd('devtool update-recipe -m srcrev %s -a %s' % (testrecipe, templayerdir)) self.assertFalse(os.path.exists(os.path.join(appenddir, testrecipe)), 'Patch directory should not be created') result = runCmd('git rev-parse HEAD', cwd=tempsrcdir) - expectedlines = ['SRCREV = "%s"\n' % result.output, - '\n', - 'SRC_URI = "%s"\n' % git_uri, - '\n'] + expectedlines = set(['SRCREV = "%s"\n' % result.output, + '\n', + 'SRC_URI = "%s"\n' % git_uri, + '\n']) with open(bbappendfile, 'r') as f: - self.assertEqual(expectedlines, f.readlines()) + self.assertEqual(expectedlines, set(f.readlines())) # Put commit back and check we can run it if layer isn't in bblayers.conf os.remove(bbappendfile) result = runCmd('git commit -a -m "Change the Makefile"', cwd=tempsrcdir) @@ -846,12 +848,12 @@ class DevtoolTests(DevtoolBase): self.assertIn('WARNING: Specified layer is not currently enabled in bblayers.conf', result.output) self.assertFalse(os.path.exists(os.path.join(appenddir, testrecipe)), 'Patch directory should not be created') result = runCmd('git rev-parse HEAD', cwd=tempsrcdir) - expectedlines = ['SRCREV = "%s"\n' % result.output, - '\n', - 'SRC_URI = "%s"\n' % git_uri, - '\n'] + expectedlines = set(['SRCREV = "%s"\n' % result.output, + '\n', + 'SRC_URI = "%s"\n' % git_uri, + '\n']) with open(bbappendfile, 'r') as f: - self.assertEqual(expectedlines, f.readlines()) + self.assertEqual(expectedlines, set(f.readlines())) # Deleting isn't expected to work under these circumstances @testcase(1370) @@ -1188,3 +1190,159 @@ class DevtoolTests(DevtoolBase): s = "Microsoft Made No Profit From Anyone's Zunes Yo" result = runCmd("devtool --quiet selftest-reverse \"%s\"" % s) self.assertEqual(result.output, s[::-1]) + + def _setup_test_devtool_finish_upgrade(self): + # Check preconditions + self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + # Use a "real" recipe from meta-selftest + recipe = 'devtool-upgrade-test1' + oldversion = '1.5.3' + newversion = '1.6.0' + oldrecipefile = get_bb_var('FILE', recipe) + recipedir = os.path.dirname(oldrecipefile) + result = runCmd('git status --porcelain .', cwd=recipedir) + if result.output.strip(): + self.fail('Recipe directory for %s contains uncommitted changes' % recipe) + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + # Check that recipe is not already under devtool control + result = runCmd('devtool status') + self.assertNotIn(recipe, result.output) + # Do the upgrade + result = runCmd('devtool upgrade %s %s -V %s' % (recipe, tempdir, newversion)) + # Check devtool status and make sure recipe is present + result = runCmd('devtool status') + self.assertIn(recipe, result.output) + self.assertIn(tempdir, result.output) + # Make a change to the source + result = runCmd('sed -i \'/^#include "pv.h"/a \\/* Here is a new comment *\\/\' src/pv/number.c', cwd=tempdir) + result = runCmd('git status --porcelain', cwd=tempdir) + self.assertIn('M src/pv/number.c', result.output) + result = runCmd('git commit src/pv/number.c -m "Add a comment to the code"', cwd=tempdir) + # Check if patch is there + recipedir = os.path.dirname(oldrecipefile) + olddir = os.path.join(recipedir, recipe + '-' + oldversion) + patchfn = '0001-Add-a-note-line-to-the-quick-reference.patch' + self.assertTrue(os.path.exists(os.path.join(olddir, patchfn)), 'Original patch file does not exist') + return recipe, oldrecipefile, recipedir, olddir, newversion, patchfn + + def test_devtool_finish_upgrade_origlayer(self): + recipe, oldrecipefile, recipedir, olddir, newversion, patchfn = self._setup_test_devtool_finish_upgrade() + # Ensure the recipe is where we think it should be (so that cleanup doesn't trash things) + self.assertIn('/meta-selftest/', recipedir) + # Try finish to the original layer + self.add_command_to_tearDown('rm -rf %s ; cd %s ; git checkout %s' % (recipedir, os.path.dirname(recipedir), recipedir)) + result = runCmd('devtool finish %s meta-selftest' % recipe) + result = runCmd('devtool status') + self.assertNotIn(recipe, result.output, 'Recipe should have been reset by finish but wasn\'t') + self.assertFalse(os.path.exists(os.path.join(self.workspacedir, 'recipes', recipe)), 'Recipe directory should not exist after finish') + self.assertFalse(os.path.exists(oldrecipefile), 'Old recipe file should have been deleted but wasn\'t') + self.assertFalse(os.path.exists(os.path.join(olddir, patchfn)), 'Old patch file should have been deleted but wasn\'t') + newrecipefile = os.path.join(recipedir, '%s_%s.bb' % (recipe, newversion)) + newdir = os.path.join(recipedir, recipe + '-' + newversion) + self.assertTrue(os.path.exists(newrecipefile), 'New recipe file should have been copied into existing layer but wasn\'t') + self.assertTrue(os.path.exists(os.path.join(newdir, patchfn)), 'Patch file should have been copied into new directory but wasn\'t') + self.assertTrue(os.path.exists(os.path.join(newdir, '0002-Add-a-comment-to-the-code.patch')), 'New patch file should have been created but wasn\'t') + + def test_devtool_finish_upgrade_otherlayer(self): + recipe, oldrecipefile, recipedir, olddir, newversion, patchfn = self._setup_test_devtool_finish_upgrade() + # Ensure the recipe is where we think it should be (so that cleanup doesn't trash things) + self.assertIn('/meta-selftest/', recipedir) + # Try finish to a different layer - should create a bbappend + # This cleanup isn't strictly necessary but do it anyway just in case it goes wrong and writes to here + self.add_command_to_tearDown('rm -rf %s ; cd %s ; git checkout %s' % (recipedir, os.path.dirname(recipedir), recipedir)) + oe_core_dir = os.path.join(get_bb_var('COREBASE'), 'meta') + newrecipedir = os.path.join(oe_core_dir, 'recipes-test', 'devtool') + newrecipefile = os.path.join(newrecipedir, '%s_%s.bb' % (recipe, newversion)) + self.track_for_cleanup(newrecipedir) + result = runCmd('devtool finish %s oe-core' % recipe) + result = runCmd('devtool status') + self.assertNotIn(recipe, result.output, 'Recipe should have been reset by finish but wasn\'t') + self.assertFalse(os.path.exists(os.path.join(self.workspacedir, 'recipes', recipe)), 'Recipe directory should not exist after finish') + self.assertTrue(os.path.exists(oldrecipefile), 'Old recipe file should not have been deleted') + self.assertTrue(os.path.exists(os.path.join(olddir, patchfn)), 'Old patch file should not have been deleted') + newdir = os.path.join(newrecipedir, recipe + '-' + newversion) + self.assertTrue(os.path.exists(newrecipefile), 'New recipe file should have been copied into existing layer but wasn\'t') + self.assertTrue(os.path.exists(os.path.join(newdir, patchfn)), 'Patch file should have been copied into new directory but wasn\'t') + self.assertTrue(os.path.exists(os.path.join(newdir, '0002-Add-a-comment-to-the-code.patch')), 'New patch file should have been created but wasn\'t') + + def _setup_test_devtool_finish_modify(self): + # Check preconditions + self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') + # Try modifying a recipe + self.track_for_cleanup(self.workspacedir) + recipe = 'mdadm' + oldrecipefile = get_bb_var('FILE', recipe) + recipedir = os.path.dirname(oldrecipefile) + result = runCmd('git status --porcelain .', cwd=recipedir) + if result.output.strip(): + self.fail('Recipe directory for %s contains uncommitted changes' % recipe) + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool modify %s %s' % (recipe, tempdir)) + self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile')), 'Extracted source could not be found') + # Test devtool status + result = runCmd('devtool status') + self.assertIn(recipe, result.output) + self.assertIn(tempdir, result.output) + # Make a change to the source + result = runCmd('sed -i \'/^#include "mdadm.h"/a \\/* Here is a new comment *\\/\' maps.c', cwd=tempdir) + result = runCmd('git status --porcelain', cwd=tempdir) + self.assertIn('M maps.c', result.output) + result = runCmd('git commit maps.c -m "Add a comment to the code"', cwd=tempdir) + for entry in os.listdir(recipedir): + filesdir = os.path.join(recipedir, entry) + if os.path.isdir(filesdir): + break + else: + self.fail('Unable to find recipe files directory for %s' % recipe) + return recipe, oldrecipefile, recipedir, filesdir + + def test_devtool_finish_modify_origlayer(self): + recipe, oldrecipefile, recipedir, filesdir = self._setup_test_devtool_finish_modify() + # Ensure the recipe is where we think it should be (so that cleanup doesn't trash things) + self.assertIn('/meta/', recipedir) + # Try finish to the original layer + self.add_command_to_tearDown('rm -rf %s ; cd %s ; git checkout %s' % (recipedir, os.path.dirname(recipedir), recipedir)) + result = runCmd('devtool finish %s meta' % recipe) + result = runCmd('devtool status') + self.assertNotIn(recipe, result.output, 'Recipe should have been reset by finish but wasn\'t') + self.assertFalse(os.path.exists(os.path.join(self.workspacedir, 'recipes', recipe)), 'Recipe directory should not exist after finish') + expected_status = [(' M', '.*/%s$' % os.path.basename(oldrecipefile)), + ('??', '.*/.*-Add-a-comment-to-the-code.patch$')] + self._check_repo_status(recipedir, expected_status) + + def test_devtool_finish_modify_otherlayer(self): + recipe, oldrecipefile, recipedir, filesdir = self._setup_test_devtool_finish_modify() + # Ensure the recipe is where we think it should be (so that cleanup doesn't trash things) + self.assertIn('/meta/', recipedir) + relpth = os.path.relpath(recipedir, os.path.join(get_bb_var('COREBASE'), 'meta')) + appenddir = os.path.join(get_test_layer(), relpth) + self.track_for_cleanup(appenddir) + # Try finish to the original layer + self.add_command_to_tearDown('rm -rf %s ; cd %s ; git checkout %s' % (recipedir, os.path.dirname(recipedir), recipedir)) + result = runCmd('devtool finish %s meta-selftest' % recipe) + result = runCmd('devtool status') + self.assertNotIn(recipe, result.output, 'Recipe should have been reset by finish but wasn\'t') + self.assertFalse(os.path.exists(os.path.join(self.workspacedir, 'recipes', recipe)), 'Recipe directory should not exist after finish') + result = runCmd('git status --porcelain .', cwd=recipedir) + if result.output.strip(): + self.fail('Recipe directory for %s contains the following unexpected changes after finish:\n%s' % (recipe, result.output.strip())) + recipefn = os.path.splitext(os.path.basename(oldrecipefile))[0] + recipefn = recipefn.split('_')[0] + '_%' + appendfile = os.path.join(appenddir, recipefn + '.bbappend') + self.assertTrue(os.path.exists(appendfile), 'bbappend %s should have been created but wasn\'t' % appendfile) + newdir = os.path.join(appenddir, recipe) + files = os.listdir(newdir) + foundpatch = None + for fn in files: + if fnmatch.fnmatch(fn, '*-Add-a-comment-to-the-code.patch'): + foundpatch = fn + if not foundpatch: + self.fail('No patch file created next to bbappend') + files.remove(foundpatch) + if files: + self.fail('Unexpected file(s) copied next to bbappend: %s' % ', '.join(files)) diff --git a/import-layers/yocto-poky/meta/lib/oeqa/selftest/eSDK.py b/import-layers/yocto-poky/meta/lib/oeqa/selftest/eSDK.py new file mode 100644 index 000000000..9d5c68094 --- /dev/null +++ b/import-layers/yocto-poky/meta/lib/oeqa/selftest/eSDK.py @@ -0,0 +1,103 @@ +import unittest +import tempfile +import shutil +import os +import glob +import logging +import subprocess +import oeqa.utils.ftools as ftools +from oeqa.utils.decorators import testcase +from oeqa.selftest.base import oeSelfTest +from oeqa.utils.commands import runCmd, bitbake, get_bb_var +from oeqa.utils.httpserver import HTTPService + +class oeSDKExtSelfTest(oeSelfTest): + """ + # Bugzilla Test Plan: 6033 + # This code is planned to be part of the automation for eSDK containig + # Install libraries and headers, image generation binary feeds. + """ + + @staticmethod + def get_esdk_environment(env_eSDK, tmpdir_eSDKQA): + # XXX: at this time use the first env need to investigate + # what environment load oe-selftest, i586, x86_64 + pattern = os.path.join(tmpdir_eSDKQA, 'environment-setup-*') + return glob.glob(pattern)[0] + + @staticmethod + def run_esdk_cmd(env_eSDK, tmpdir_eSDKQA, cmd, postconfig=None, **options): + if postconfig: + esdk_conf_file = os.path.join(tmpdir_eSDKQA, 'conf', 'local.conf') + with open(esdk_conf_file, 'a+') as f: + f.write(postconfig) + if not options: + options = {} + if not 'shell' in options: + options['shell'] = True + + runCmd("cd %s; . %s; %s" % (tmpdir_eSDKQA, env_eSDK, cmd), **options) + + @staticmethod + def generate_eSDK(image): + pn_task = '%s -c populate_sdk_ext' % image + bitbake(pn_task) + + @staticmethod + def get_eSDK_toolchain(image): + pn_task = '%s -c populate_sdk_ext' % image + + sdk_deploy = get_bb_var('SDK_DEPLOY', pn_task) + toolchain_name = get_bb_var('TOOLCHAINEXT_OUTPUTNAME', pn_task) + return os.path.join(sdk_deploy, toolchain_name + '.sh') + + + @classmethod + def setUpClass(cls): + # Start to serve sstate dir + sstate_dir = os.path.join(os.environ['BUILDDIR'], 'sstate-cache') + cls.http_service = HTTPService(sstate_dir) + cls.http_service.start() + + http_url = "127.0.0.1:%d" % cls.http_service.port + + image = 'core-image-minimal' + + cls.tmpdir_eSDKQA = tempfile.mkdtemp(prefix='eSDKQA') + oeSDKExtSelfTest.generate_eSDK(image) + + # Install eSDK + ext_sdk_path = oeSDKExtSelfTest.get_eSDK_toolchain(image) + runCmd("%s -y -d \"%s\"" % (ext_sdk_path, cls.tmpdir_eSDKQA)) + + cls.env_eSDK = oeSDKExtSelfTest.get_esdk_environment('', cls.tmpdir_eSDKQA) + + # Configure eSDK to use sstate mirror from poky + sstate_config=""" +SDK_LOCAL_CONF_WHITELIST = "SSTATE_MIRRORS" +SSTATE_MIRRORS = "file://.* http://%s/PATH" + """ % http_url + with open(os.path.join(cls.tmpdir_eSDKQA, 'conf', 'local.conf'), 'a+') as f: + f.write(sstate_config) + + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.tmpdir_eSDKQA) + cls.http_service.stop() + + @testcase (1471) + def test_install_libraries_headers(self): + pn_sstate = 'bc' + bitbake(pn_sstate) + cmd = "devtool sdk-install %s " % pn_sstate + oeSDKExtSelfTest.run_esdk_cmd(self.env_eSDK, self.tmpdir_eSDKQA, cmd) + + @testcase(1472) + def test_image_generation_binary_feeds(self): + image = 'core-image-minimal' + cmd = "devtool build-image %s" % image + oeSDKExtSelfTest.run_esdk_cmd(self.env_eSDK, self.tmpdir_eSDKQA, cmd) + +if __name__ == '__main__': + unittest.main() diff --git a/import-layers/yocto-poky/meta/lib/oeqa/selftest/esdk_prepare.py b/import-layers/yocto-poky/meta/lib/oeqa/selftest/esdk_prepare.py deleted file mode 100755 index 1b36a0d68..000000000 --- a/import-layers/yocto-poky/meta/lib/oeqa/selftest/esdk_prepare.py +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env python - -import shutil, tempfile -import sys -import os -import imp -import unittest -try: - from oeqa.utils.commands import get_bb_var -except ImportError: - pass - -# module under test -module_file_name = "ext-sdk-prepare.py" -module_path = "" - -class ExtSdkPrepareTest(unittest.TestCase): - - """ unit test for fix for Yocto #9019 """ - - @classmethod - def setUpClass(self): - # copy module under test to temp dir - self.test_dir = tempfile.mkdtemp() - module_dest_path = os.path.join(self.test_dir, module_file_name) - try: - shutil.copy(module_path, self.test_dir) - # load module under test - self.test_mod = imp.load_source("", module_dest_path) - except: - print "error: unable to copy or load %s [src: %s, dst: %s]" % \ - (module_file_name, module_path, module_dest_path) - sys.exit(1) - - def test_prepare_unexpected(self): - # test data - # note: pathnames have been truncated from the actual bitbake - # output as they are not important for the test. - test_data = ( - 'NOTE: Running noexec task 9 of 6539 (ID: 28, quilt/quilt-native_0.64.bb, do_build)\n' - 'NOTE: Running task 10 of 6539 (ID: 29, quilt/quilt-native_0.64.bb, do_package)\n' - 'NOTE: Running task 11 of 6539 (ID: 30, quilt/quilt-native_0.64.bb, do_rm_work)\n' - 'NOTE: Running noexec task 6402 of 6539 (ID: 1, images/core-image-sato.bb, do_patch)\n' - 'NOTE: Running task 6538 of 6539 (ID: 14, images/core-image-sato.bb, do_rm_work)\n' - ) - # expected warning output - expected = [ (' task 10 of 6539 (ID: 29, quilt/quilt-native_0.64.bb, do_package)') ] - # recipe to test, matching test input data - recipes = [ "core-image-sato.bb" ] - - # run the test - output = self.test_mod.check_unexpected(test_data, recipes) - self.assertEqual(output, expected) - - @classmethod - def tearDownClass(self): - # remove temp dir - shutil.rmtree(self.test_dir) - -if __name__ == '__main__': - # running from command line - i.e., not under oe-selftest - # directory containing module under test comes from command line - if len(sys.argv) == 2 and os.path.isdir(sys.argv[1]): - module_path = os.path.join(sys.argv[1], module_file_name) - suite = unittest.TestLoader().loadTestsFromTestCase(ExtSdkPrepareTest) - unittest.TextTestRunner().run(suite) - else: - progname = os.path.basename(sys.argv[0]) - print "%s: missing directory path" % progname - print "usage: %s /path/to/directory-of(ext-sdk-prepare.py)" % progname - sys.exit(1) -else: - # running under oe-selftest - # determine module source dir from COREBASE and expected path - module_path = os.path.join(get_bb_var("COREBASE"), "meta", "files", module_file_name) diff --git a/import-layers/yocto-poky/meta/lib/oeqa/selftest/imagefeatures.py b/import-layers/yocto-poky/meta/lib/oeqa/selftest/imagefeatures.py index 8a53899c7..d015c4908 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/selftest/imagefeatures.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/selftest/imagefeatures.py @@ -98,3 +98,30 @@ class ImageFeatures(oeSelfTest): # Build a core-image-weston bitbake('core-image-weston') + def test_bmap(self): + """ + Summary: Check bmap support + Expected: 1. core-image-minimal can be build with bmap support + 2. core-image-minimal is sparse + Product: oe-core + Author: Ed Bartosh <ed.bartosh@linux.intel.com> + """ + + features = 'IMAGE_FSTYPES += " ext4 ext4.bmap"' + self.write_config(features) + + image_name = 'core-image-minimal' + bitbake(image_name) + + deploy_dir_image = get_bb_var('DEPLOY_DIR_IMAGE') + link_name = get_bb_var('IMAGE_LINK_NAME', image_name) + image_path = os.path.join(deploy_dir_image, "%s.ext4" % link_name) + bmap_path = "%s.bmap" % image_path + + # check if result image and bmap file are in deploy directory + self.assertTrue(os.path.exists(image_path)) + self.assertTrue(os.path.exists(bmap_path)) + + # check if result image is sparse + image_stat = os.stat(image_path) + self.assertTrue(image_stat.st_size > image_stat.st_blocks * 512) diff --git a/import-layers/yocto-poky/meta/lib/oeqa/selftest/liboe.py b/import-layers/yocto-poky/meta/lib/oeqa/selftest/liboe.py new file mode 100644 index 000000000..35131eb24 --- /dev/null +++ b/import-layers/yocto-poky/meta/lib/oeqa/selftest/liboe.py @@ -0,0 +1,93 @@ +from oeqa.selftest.base import oeSelfTest +from oeqa.utils.commands import get_bb_var, bitbake, runCmd +import oe.path +import glob +import os +import os.path + +class LibOE(oeSelfTest): + def test_copy_tree_special(self): + """ + Summary: oe.path.copytree() should copy files with special character + Expected: 'test file with sp£c!al @nd spaces' should exist in + copy destination + Product: OE-Core + Author: Joshua Lock <joshua.g.lock@intel.com> + """ + tmp_dir = get_bb_var('TMPDIR') + testloc = oe.path.join(tmp_dir, 'liboetests') + src = oe.path.join(testloc, 'src') + dst = oe.path.join(testloc, 'dst') + bb.utils.mkdirhier(testloc) + bb.utils.mkdirhier(src) + testfilename = 'test file with sp£c!al @nd spaces' + + # create the test file and copy it + open(oe.path.join(src, testfilename), 'w+b').close() + oe.path.copytree(src, dst) + + # ensure path exists in dest + fileindst = os.path.isfile(oe.path.join(dst, testfilename)) + self.assertTrue(fileindst, "File with spaces doesn't exist in dst") + + oe.path.remove(testloc) + + def test_copy_tree_xattr(self): + """ + Summary: oe.path.copytree() should preserve xattr on copied files + Expected: testxattr file in destination should have user.oetest + extended attribute + Product: OE-Core + Author: Joshua Lock <joshua.g.lock@intel.com> + """ + tmp_dir = get_bb_var('TMPDIR') + testloc = oe.path.join(tmp_dir, 'liboetests') + src = oe.path.join(testloc, 'src') + dst = oe.path.join(testloc, 'dst') + bb.utils.mkdirhier(testloc) + bb.utils.mkdirhier(src) + testfilename = 'testxattr' + + # ensure we have setfattr available + bitbake("attr-native") + bindir = get_bb_var('STAGING_BINDIR_NATIVE') + + # create a file with xattr and copy it + open(oe.path.join(src, testfilename), 'w+b').close() + runCmd('%s/setfattr -n user.oetest -v "testing liboe" %s' % (bindir, oe.path.join(src, testfilename))) + oe.path.copytree(src, dst) + + # ensure file in dest has user.oetest xattr + result = runCmd('%s/getfattr -n user.oetest %s' % (bindir, oe.path.join(dst, testfilename))) + self.assertIn('user.oetest="testing liboe"', result.output, 'Extended attribute not sert in dst') + + oe.path.remove(testloc) + + def test_copy_hardlink_tree_count(self): + """ + Summary: oe.path.copyhardlinktree() shouldn't miss out files + Expected: src and dst should have the same number of files + Product: OE-Core + Author: Joshua Lock <joshua.g.lock@intel.com> + """ + tmp_dir = get_bb_var('TMPDIR') + testloc = oe.path.join(tmp_dir, 'liboetests') + src = oe.path.join(testloc, 'src') + dst = oe.path.join(testloc, 'dst') + bb.utils.mkdirhier(testloc) + bb.utils.mkdirhier(src) + testfiles = ['foo', 'bar', '.baz', 'quux'] + + def touchfile(tf): + open(oe.path.join(src, tf), 'w+b').close() + + for f in testfiles: + touchfile(f) + + oe.path.copyhardlinktree(src, dst) + + dstcnt = len(os.listdir(dst)) + srccnt = len(os.listdir(src)) + self.assertEquals(dstcnt, len(testfiles), "Number of files in dst (%s) differs from number of files in src(%s)." % (dstcnt, srccnt)) + + oe.path.remove(testloc) diff --git a/import-layers/yocto-poky/meta/lib/oeqa/selftest/lic-checksum.py b/import-layers/yocto-poky/meta/lib/oeqa/selftest/lic-checksum.py index cac6d8445..2e81373ae 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/selftest/lic-checksum.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/selftest/lic-checksum.py @@ -12,20 +12,24 @@ class LicenseTests(oeSelfTest): # the license qa to fail due to a mismatched md5sum. @testcase(1197) def test_nonmatching_checksum(self): - bitbake_cmd = '-c configure emptytest' + bitbake_cmd = '-c populate_lic emptytest' error_msg = 'emptytest: The new md5 checksum is 8d777f385d3dfec8815d20f7496026dc' lic_file, lic_path = tempfile.mkstemp() os.close(lic_file) self.track_for_cleanup(lic_path) - self.write_recipeinc('emptytest', 'INHIBIT_DEFAULT_DEPS = "1"') - self.append_recipeinc('emptytest', 'LIC_FILES_CHKSUM = "file://%s;md5=d41d8cd98f00b204e9800998ecf8427e"' % lic_path) + self.write_recipeinc('emptytest', """ +INHIBIT_DEFAULT_DEPS = "1" +LIC_FILES_CHKSUM = "file://%s;md5=d41d8cd98f00b204e9800998ecf8427e" +SRC_URI = "file://%s;md5=d41d8cd98f00b204e9800998ecf8427e" +""" % (lic_path, lic_path)) result = bitbake(bitbake_cmd) with open(lic_path, "w") as f: f.write("data") + self.write_config("INHERIT_remove = \"report-error\"") result = bitbake(bitbake_cmd, ignore_status=True) if error_msg not in result.output: raise AssertionError(result.output) diff --git a/import-layers/yocto-poky/meta/lib/oeqa/selftest/pkgdata.py b/import-layers/yocto-poky/meta/lib/oeqa/selftest/pkgdata.py index 138b03aad..5a63f89ff 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/selftest/pkgdata.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/selftest/pkgdata.py @@ -131,15 +131,15 @@ class OePkgdataUtilTests(oeSelfTest): # Test recipe-space package name result = runCmd('oe-pkgdata-util list-pkg-files zlib-dev zlib-doc') files = splitoutput(result.output) - self.assertIn('zlib-dev', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('zlib-doc', files.keys(), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib-dev', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib-doc', list(files.keys()), "listed pkgs. files: %s" %result.output) self.assertIn(os.path.join(includedir, 'zlib.h'), files['zlib-dev']) self.assertIn(os.path.join(mandir, 'man3/zlib.3'), files['zlib-doc']) # Test runtime package name result = runCmd('oe-pkgdata-util list-pkg-files -r libz1 libz-dev') files = splitoutput(result.output) - self.assertIn('libz1', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('libz-dev', files.keys(), "listed pkgs. files: %s" %result.output) + self.assertIn('libz1', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('libz-dev', list(files.keys()), "listed pkgs. files: %s" %result.output) self.assertGreater(len(files['libz1']), 1) libspec = os.path.join(base_libdir, 'libz.so.1.*') found = False @@ -152,12 +152,12 @@ class OePkgdataUtilTests(oeSelfTest): # Test recipe result = runCmd('oe-pkgdata-util list-pkg-files -p zlib') files = splitoutput(result.output) - self.assertIn('zlib-dbg', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('zlib-doc', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('zlib-dev', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('zlib-staticdev', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('zlib', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertNotIn('zlib-locale', files.keys(), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib-dbg', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib-doc', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib-dev', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib-staticdev', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertNotIn('zlib-locale', list(files.keys()), "listed pkgs. files: %s" %result.output) # (ignore ptest, might not be there depending on config) self.assertIn(os.path.join(includedir, 'zlib.h'), files['zlib-dev']) self.assertIn(os.path.join(mandir, 'man3/zlib.3'), files['zlib-doc']) @@ -165,36 +165,36 @@ class OePkgdataUtilTests(oeSelfTest): # Test recipe, runtime result = runCmd('oe-pkgdata-util list-pkg-files -p zlib -r') files = splitoutput(result.output) - self.assertIn('libz-dbg', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('libz-doc', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('libz-dev', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('libz-staticdev', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('libz1', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertNotIn('libz-locale', files.keys(), "listed pkgs. files: %s" %result.output) + self.assertIn('libz-dbg', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('libz-doc', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('libz-dev', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('libz-staticdev', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('libz1', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertNotIn('libz-locale', list(files.keys()), "listed pkgs. files: %s" %result.output) self.assertIn(os.path.join(includedir, 'zlib.h'), files['libz-dev']) self.assertIn(os.path.join(mandir, 'man3/zlib.3'), files['libz-doc']) self.assertIn(os.path.join(libdir, 'libz.a'), files['libz-staticdev']) # Test recipe, unpackaged result = runCmd('oe-pkgdata-util list-pkg-files -p zlib -u') files = splitoutput(result.output) - self.assertIn('zlib-dbg', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('zlib-doc', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('zlib-dev', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('zlib-staticdev', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('zlib', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('zlib-locale', files.keys(), "listed pkgs. files: %s" %result.output) # this is the key one + self.assertIn('zlib-dbg', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib-doc', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib-dev', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib-staticdev', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib-locale', list(files.keys()), "listed pkgs. files: %s" %result.output) # this is the key one self.assertIn(os.path.join(includedir, 'zlib.h'), files['zlib-dev']) self.assertIn(os.path.join(mandir, 'man3/zlib.3'), files['zlib-doc']) self.assertIn(os.path.join(libdir, 'libz.a'), files['zlib-staticdev']) # Test recipe, runtime, unpackaged result = runCmd('oe-pkgdata-util list-pkg-files -p zlib -r -u') files = splitoutput(result.output) - self.assertIn('libz-dbg', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('libz-doc', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('libz-dev', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('libz-staticdev', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('libz1', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('libz-locale', files.keys(), "listed pkgs. files: %s" %result.output) # this is the key one + self.assertIn('libz-dbg', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('libz-doc', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('libz-dev', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('libz-staticdev', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('libz1', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('libz-locale', list(files.keys()), "listed pkgs. files: %s" %result.output) # this is the key one self.assertIn(os.path.join(includedir, 'zlib.h'), files['libz-dev']) self.assertIn(os.path.join(mandir, 'man3/zlib.3'), files['libz-doc']) self.assertIn(os.path.join(libdir, 'libz.a'), files['libz-staticdev']) diff --git a/import-layers/yocto-poky/meta/lib/oeqa/selftest/recipetool.py b/import-layers/yocto-poky/meta/lib/oeqa/selftest/recipetool.py index e72911b0a..db1f8deeb 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/selftest/recipetool.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/selftest/recipetool.py @@ -1,7 +1,7 @@ import os import logging import tempfile -import urlparse +import urllib.parse from oeqa.utils.commands import runCmd, bitbake, get_bb_var, create_temp_layer from oeqa.utils.decorators import testcase @@ -278,7 +278,7 @@ class RecipetoolTests(RecipetoolBase): '}\n'] _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/etc/selftest-replaceme-patched', self.testfile, '', expectedlines, ['testfile']) for line in output.splitlines(): - if line.startswith('WARNING: '): + if 'WARNING: ' in line: self.assertIn('add-file.patch', line, 'Unexpected warning found in output:\n%s' % line) break else: @@ -383,13 +383,13 @@ class RecipetoolTests(RecipetoolBase): @testcase(1194) def test_recipetool_create_git(self): # Ensure we have the right data in shlibs/pkgdata - bitbake('libpng pango libx11 libxext jpeg libxsettings-client libcheck') + bitbake('libpng pango libx11 libxext jpeg libcheck') # Try adding a recipe tempsrc = os.path.join(self.tempdir, 'srctree') os.makedirs(tempsrc) recipefile = os.path.join(self.tempdir, 'libmatchbox.bb') srcuri = 'git://git.yoctoproject.org/libmatchbox' - result = runCmd('recipetool create -o %s %s -x %s' % (recipefile, srcuri, tempsrc)) + result = runCmd(['recipetool', 'create', '-o', recipefile, srcuri + ";rev=9f7cf8895ae2d39c465c04cc78e918c157420269", '-x', tempsrc]) self.assertTrue(os.path.isfile(recipefile), 'recipetool did not create recipe file; output:\n%s' % result.output) checkvars = {} checkvars['LICENSE'] = 'LGPLv2.1' @@ -397,7 +397,7 @@ class RecipetoolTests(RecipetoolBase): checkvars['S'] = '${WORKDIR}/git' checkvars['PV'] = '1.11+git${SRCPV}' checkvars['SRC_URI'] = srcuri - checkvars['DEPENDS'] = set(['libcheck', 'libjpeg-turbo', 'libpng', 'libx11', 'libxsettings-client', 'libxext', 'pango']) + checkvars['DEPENDS'] = set(['libcheck', 'libjpeg-turbo', 'libpng', 'libx11', 'libxext', 'pango']) inherits = ['autotools', 'pkgconfig'] self._test_recipe_contents(recipefile, checkvars, inherits) @@ -442,6 +442,49 @@ class RecipetoolTests(RecipetoolBase): inherits = ['cmake', 'python-dir', 'gettext', 'pkgconfig'] self._test_recipe_contents(recipefile, checkvars, inherits) + def test_recipetool_create_github(self): + # Basic test to see if github URL mangling works + temprecipe = os.path.join(self.tempdir, 'recipe') + os.makedirs(temprecipe) + recipefile = os.path.join(temprecipe, 'meson_git.bb') + srcuri = 'https://github.com/mesonbuild/meson' + result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) + self.assertTrue(os.path.isfile(recipefile)) + checkvars = {} + checkvars['LICENSE'] = set(['Apache-2.0']) + checkvars['SRC_URI'] = 'git://github.com/mesonbuild/meson;protocol=https' + inherits = ['setuptools'] + self._test_recipe_contents(recipefile, checkvars, inherits) + + def test_recipetool_create_github_tarball(self): + # Basic test to ensure github URL mangling doesn't apply to release tarballs + temprecipe = os.path.join(self.tempdir, 'recipe') + os.makedirs(temprecipe) + pv = '0.32.0' + recipefile = os.path.join(temprecipe, 'meson_%s.bb' % pv) + srcuri = 'https://github.com/mesonbuild/meson/releases/download/%s/meson-%s.tar.gz' % (pv, pv) + result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) + self.assertTrue(os.path.isfile(recipefile)) + checkvars = {} + checkvars['LICENSE'] = set(['Apache-2.0']) + checkvars['SRC_URI'] = 'https://github.com/mesonbuild/meson/releases/download/${PV}/meson-${PV}.tar.gz' + inherits = ['setuptools'] + self._test_recipe_contents(recipefile, checkvars, inherits) + + def test_recipetool_create_git_http(self): + # Basic test to check http git URL mangling works + temprecipe = os.path.join(self.tempdir, 'recipe') + os.makedirs(temprecipe) + recipefile = os.path.join(temprecipe, 'matchbox-terminal_git.bb') + srcuri = 'http://git.yoctoproject.org/git/matchbox-terminal' + result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) + self.assertTrue(os.path.isfile(recipefile)) + checkvars = {} + checkvars['LICENSE'] = set(['GPLv2']) + checkvars['SRC_URI'] = 'git://git.yoctoproject.org/git/matchbox-terminal;protocol=http' + inherits = ['pkgconfig', 'autotools'] + self._test_recipe_contents(recipefile, checkvars, inherits) + class RecipetoolAppendsrcBase(RecipetoolBase): def _try_recipetool_appendsrcfile(self, testrecipe, newfile, destfile, options, expectedlines, expectedfiles): cmd = 'recipetool appendsrcfile %s %s %s %s %s' % (options, self.templayerdir, testrecipe, newfile, destfile) @@ -471,7 +514,7 @@ class RecipetoolAppendsrcBase(RecipetoolBase): '''Return the first file:// in SRC_URI for the specified recipe.''' src_uri = get_bb_var('SRC_URI', recipe).split() for uri in src_uri: - p = urlparse.urlparse(uri) + p = urllib.parse.urlparse(uri) if p.scheme == 'file': return p.netloc + p.path diff --git a/import-layers/yocto-poky/meta/lib/oeqa/selftest/runtime-test.py b/import-layers/yocto-poky/meta/lib/oeqa/selftest/runtime-test.py new file mode 100644 index 000000000..c2d5b45a4 --- /dev/null +++ b/import-layers/yocto-poky/meta/lib/oeqa/selftest/runtime-test.py @@ -0,0 +1,105 @@ +from oeqa.selftest.base import oeSelfTest +from oeqa.utils.commands import runCmd, bitbake, get_bb_var, runqemu +from oeqa.utils.decorators import testcase +import os + +class TestExport(oeSelfTest): + + def test_testexport_basic(self): + """ + Summary: Check basic testexport functionality with only ping test enabled. + Expected: 1. testexport directory must be created. + 2. runexported.py must run without any error/exception. + 3. ping test must succeed. + Product: oe-core + Author: Mariano Lopez <mariano.lopez@intel.com> + """ + + features = 'INHERIT += "testexport"\n' + # These aren't the actual IP addresses but testexport class needs something defined + features += 'TEST_SERVER_IP = "192.168.7.1"\n' + features += 'TEST_TARGET_IP = "192.168.7.1"\n' + features += 'TEST_SUITES = "ping"\n' + self.write_config(features) + + # Build tesexport for core-image-minimal + bitbake('core-image-minimal') + bitbake('-c testexport core-image-minimal') + + # Verify if TEST_EXPORT_DIR was created + testexport_dir = get_bb_var('TEST_EXPORT_DIR', 'core-image-minimal') + isdir = os.path.isdir(testexport_dir) + self.assertEqual(True, isdir, 'Failed to create testexport dir: %s' % testexport_dir) + + with runqemu('core-image-minimal') as qemu: + # Attempt to run runexported.py to perform ping test + runexported_path = os.path.join(testexport_dir, "runexported.py") + testdata_path = os.path.join(testexport_dir, "testdata.json") + cmd = "%s -t %s -s %s %s" % (runexported_path, qemu.ip, qemu.server_ip, testdata_path) + result = runCmd(cmd) + self.assertEqual(0, result.status, 'runexported.py returned a non 0 status') + + # Verify ping test was succesful + failure = True if 'FAIL' in result.output else False + self.assertNotEqual(True, failure, 'ping test failed') + + def test_testexport_sdk(self): + """ + Summary: Check sdk functionality for testexport. + Expected: 1. testexport directory must be created. + 2. SDK tarball must exists. + 3. Uncompressing of tarball must succeed. + 4. Check if the SDK directory is added to PATH. + 5. Run tar from the SDK directory. + Product: oe-core + Author: Mariano Lopez <mariano.lopez@intel.com> + """ + + features = 'INHERIT += "testexport"\n' + # These aren't the actual IP addresses but testexport class needs something defined + features += 'TEST_SERVER_IP = "192.168.7.1"\n' + features += 'TEST_TARGET_IP = "192.168.7.1"\n' + features += 'TEST_SUITES = "ping"\n' + features += 'TEST_SUITES_TAGS = "selftest_sdk"\n' + features += 'TEST_EXPORT_SDK_ENABLED = "1"\n' + features += 'TEST_EXPORT_SDK_PACKAGES = "nativesdk-tar"\n' + self.write_config(features) + + # Build tesexport for core-image-minimal + bitbake('core-image-minimal') + bitbake('-c testexport core-image-minimal') + + # Check for SDK + testexport_dir = get_bb_var('TEST_EXPORT_DIR', 'core-image-minimal') + sdk_dir = get_bb_var('TEST_EXPORT_SDK_DIR', 'core-image-minimal') + tarball_name = "%s.sh" % get_bb_var('TEST_EXPORT_SDK_NAME', 'core-image-minimal') + tarball_path = os.path.join(testexport_dir, sdk_dir, tarball_name) + self.assertEqual(os.path.isfile(tarball_path), True, "Couldn't find SDK tarball: %s" % tarball_path) + + # Run runexported.py + runexported_path = os.path.join(testexport_dir, "runexported.py") + testdata_path = os.path.join(testexport_dir, "testdata.json") + cmd = "%s %s" % (runexported_path, testdata_path) + result = runCmd(cmd) + self.assertEqual(0, result.status, 'runexported.py returned a non 0 status') + + +class TestImage(oeSelfTest): + + def test_testimage_install(self): + """ + Summary: Check install packages functionality for testimage/testexport. + Expected: 1. Import tests from a directory other than meta. + 2. Check install/unistall of socat. + Product: oe-core + Author: Mariano Lopez <mariano.lopez@intel.com> + """ + + features = 'INHERIT += "testimage"\n' + features += 'TEST_SUITES = "ping ssh selftest"\n' + features += 'TEST_SUITES_TAGS = "selftest_package_install"\n' + self.write_config(features) + + # Build core-image-sato and testimage + bitbake('core-image-full-cmdline socat') + bitbake('-c testimage core-image-full-cmdline') diff --git a/import-layers/yocto-poky/meta/lib/oeqa/selftest/signing.py b/import-layers/yocto-poky/meta/lib/oeqa/selftest/signing.py index 1babca07d..4c12d6d94 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/selftest/signing.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/selftest/signing.py @@ -12,30 +12,22 @@ from oeqa.utils.ftools import write_file class Signing(oeSelfTest): gpg_dir = "" - pub_key_name = 'key.pub' - secret_key_name = 'key.secret' + pub_key_path = "" + secret_key_path = "" @classmethod def setUpClass(cls): - # Import the gpg keys + # Check that we can find the gpg binary and fail early if we can't + if not shutil.which("gpg"): + raise AssertionError("This test needs GnuPG") - cls.gpg_dir = os.path.join(cls.testlayer_path, 'files/signing/') + cls.gpg_home_dir = tempfile.TemporaryDirectory(prefix="oeqa-signing-") + cls.gpg_dir = cls.gpg_home_dir.name - # key.secret key.pub are located in gpg_dir - pub_key_location = cls.gpg_dir + cls.pub_key_name - secret_key_location = cls.gpg_dir + cls.secret_key_name - runCmd('gpg --homedir %s --import %s %s' % (cls.gpg_dir, pub_key_location, secret_key_location)) + cls.pub_key_path = os.path.join(cls.testlayer_path, 'files', 'signing', "key.pub") + cls.secret_key_path = os.path.join(cls.testlayer_path, 'files', 'signing', "key.secret") - @classmethod - def tearDownClass(cls): - # Delete the files generated by 'gpg --import' - - gpg_files = glob.glob(cls.gpg_dir + '*.gpg*') - random_seed_file = cls.gpg_dir + 'random_seed' - gpg_files.append(random_seed_file) - - for gpg_file in gpg_files: - runCmd('rm -f ' + gpg_file) + runCmd('gpg --homedir %s --import %s %s' % (cls.gpg_dir, cls.pub_key_path, cls.secret_key_path)) @testcase(1362) def test_signing_packages(self): @@ -57,7 +49,7 @@ class Signing(oeSelfTest): feature = 'INHERIT += "sign_rpm"\n' feature += 'RPM_GPG_PASSPHRASE = "test123"\n' feature += 'RPM_GPG_NAME = "testuser"\n' - feature += 'RPM_GPG_PUBKEY = "%s%s"\n' % (self.gpg_dir, self.pub_key_name) + feature += 'RPM_GPG_PUBKEY = "%s"\n' % self.pub_key_path feature += 'GPG_PATH = "%s"\n' % self.gpg_dir self.write_config(feature) @@ -81,8 +73,8 @@ class Signing(oeSelfTest): # Use a temporary rpmdb rpmdb = tempfile.mkdtemp(prefix='oeqa-rpmdb') - runCmd('%s/rpm --define "_dbpath %s" --import %s%s' % - (staging_bindir_native, rpmdb, self.gpg_dir, self.pub_key_name)) + runCmd('%s/rpm --define "_dbpath %s" --import %s' % + (staging_bindir_native, rpmdb, self.pub_key_path)) ret = runCmd('%s/rpm --define "_dbpath %s" --checksig %s' % (staging_bindir_native, rpmdb, pkg_deploy)) diff --git a/import-layers/yocto-poky/meta/lib/oeqa/selftest/sstatetests.py b/import-layers/yocto-poky/meta/lib/oeqa/selftest/sstatetests.py index acaf405ac..6642539eb 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/selftest/sstatetests.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/selftest/sstatetests.py @@ -237,6 +237,7 @@ TMPDIR = "${TOPDIR}/tmp-sstatesamehash" BUILD_ARCH = "x86_64" BUILD_OS = "linux" SDKMACHINE = "x86_64" +PACKAGE_CLASSES = "package_rpm package_ipk package_deb" """) self.track_for_cleanup(topdir + "/tmp-sstatesamehash") bitbake("core-image-sato -S none") @@ -246,6 +247,7 @@ TMPDIR = "${TOPDIR}/tmp-sstatesamehash2" BUILD_ARCH = "i686" BUILD_OS = "linux" SDKMACHINE = "i686" +PACKAGE_CLASSES = "package_rpm package_ipk package_deb" """) self.track_for_cleanup(topdir + "/tmp-sstatesamehash2") bitbake("core-image-sato -S none") @@ -264,7 +266,7 @@ SDKMACHINE = "i686" files2 = get_files(topdir + "/tmp-sstatesamehash2/stamps/") files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash").replace("i686-linux", "x86_64-linux").replace("i686" + targetvendor + "-linux", "x86_64" + targetvendor + "-linux", ) for x in files2] self.maxDiff = None - self.assertItemsEqual(files1, files2) + self.assertCountEqual(files1, files2) @testcase(1271) @@ -298,7 +300,7 @@ NATIVELSBSTRING = \"DistroB\" files2 = get_files(topdir + "/tmp-sstatesamehash2/stamps/") files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash") for x in files2] self.maxDiff = None - self.assertItemsEqual(files1, files2) + self.assertCountEqual(files1, files2) @testcase(1368) def test_sstate_allarch_samesigs(self): @@ -309,19 +311,48 @@ NATIVELSBSTRING = \"DistroB\" the two MACHINE values. """ + configA = """ +TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\" +MACHINE = \"qemux86-64\" +""" + configB = """ +TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" +MACHINE = \"qemuarm\" +""" + self.sstate_allarch_samesigs(configA, configB) + + def test_sstate_allarch_samesigs_multilib(self): + """ + The sstate checksums of allarch multilib packages should be independent of whichever + MACHINE is set. Check this using bitbake -S. + Also, rather than duplicate the test, check nativesdk stamps are the same between + the two MACHINE values. + """ + + configA = """ +TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\" +MACHINE = \"qemux86-64\" +require conf/multilib.conf +MULTILIBS = \"multilib:lib32\" +DEFAULTTUNE_virtclass-multilib-lib32 = \"x86\" +""" + configB = """ +TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" +MACHINE = \"qemuarm\" +require conf/multilib.conf +MULTILIBS = \"\" +""" + self.sstate_allarch_samesigs(configA, configB) + + def sstate_allarch_samesigs(self, configA, configB): + topdir = get_bb_var('TOPDIR') targetos = get_bb_var('TARGET_OS') targetvendor = get_bb_var('TARGET_VENDOR') - self.write_config(""" -TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\" -MACHINE = \"qemux86\" -""") + self.write_config(configA) self.track_for_cleanup(topdir + "/tmp-sstatesamehash") bitbake("world meta-toolchain -S none") - self.write_config(""" -TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" -MACHINE = \"qemuarm\" -""") + self.write_config(configB) self.track_for_cleanup(topdir + "/tmp-sstatesamehash2") bitbake("world meta-toolchain -S none") @@ -393,7 +424,7 @@ DEFAULTTUNE_virtclass-multilib-lib32 = "x86" files2 = get_files(topdir + "/tmp-sstatesamehash2/stamps") files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash") for x in files2] self.maxDiff = None - self.assertItemsEqual(files1, files2) + self.assertCountEqual(files1, files2) def test_sstate_noop_samesigs(self): @@ -411,9 +442,11 @@ PARALLEL_MAKE = "-j 1" DL_DIR = "${TOPDIR}/download1" TIME = "111111" DATE = "20161111" -INHERIT_remove = "buildstats-summary buildhistory" +INHERIT_remove = "buildstats-summary buildhistory uninative" +http_proxy = "" """) self.track_for_cleanup(topdir + "/tmp-sstatesamehash") + self.track_for_cleanup(topdir + "/download1") bitbake("world meta-toolchain -S none") self.write_config(""" TMPDIR = "${TOPDIR}/tmp-sstatesamehash2" @@ -422,9 +455,13 @@ PARALLEL_MAKE = "-j 2" DL_DIR = "${TOPDIR}/download2" TIME = "222222" DATE = "20161212" +# Always remove uninative as we're changing proxies +INHERIT_remove = "uninative" INHERIT += "buildstats-summary buildhistory" +http_proxy = "http://example.com/" """) self.track_for_cleanup(topdir + "/tmp-sstatesamehash2") + self.track_for_cleanup(topdir + "/download2") bitbake("world meta-toolchain -S none") def get_files(d): @@ -439,23 +476,23 @@ INHERIT += "buildstats-summary buildhistory" files1 = get_files(topdir + "/tmp-sstatesamehash/stamps/") files2 = get_files(topdir + "/tmp-sstatesamehash2/stamps/") # Remove items that are identical in both sets - for k,v in files1.viewitems() & files2.viewitems(): + for k,v in files1.items() & files2.items(): del files1[k] del files2[k] if not files1 and not files2: # No changes, so we're done return - for k in files1.viewkeys() | files2.viewkeys(): + for k in files1.keys() | files2.keys(): if k in files1 and k in files2: - print "%s differs:" % k - print subprocess.check_output(("bitbake-diffsigs", + print("%s differs:" % k) + print(subprocess.check_output(("bitbake-diffsigs", topdir + "/tmp-sstatesamehash/stamps/" + k + "." + files1[k], - topdir + "/tmp-sstatesamehash2/stamps/" + k + "." + files2[k])) + topdir + "/tmp-sstatesamehash2/stamps/" + k + "." + files2[k]))) elif k in files1 and k not in files2: - print "%s in files1" % k + print("%s in files1" % k) elif k not in files1 and k in files2: - print "%s in files2" % k + print("%s in files2" % k) else: assert "shouldn't reach here" self.fail("sstate hashes not identical.") diff --git a/import-layers/yocto-poky/meta/lib/oeqa/selftest/wic.py b/import-layers/yocto-poky/meta/lib/oeqa/selftest/wic.py index a569fbf74..faac11e21 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/selftest/wic.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/selftest/wic.py @@ -49,7 +49,7 @@ class Wic(oeSelfTest): # setUpClass being unavailable. if not Wic.image_is_ready: bitbake('syslinux syslinux-native parted-native gptfdisk-native ' - 'dosfstools-native mtools-native') + 'dosfstools-native mtools-native bmap-tools-native') bitbake('core-image-minimal') Wic.image_is_ready = True @@ -276,3 +276,26 @@ class Wic(oeSelfTest): status, output = qemu.run_serial(command) self.assertEqual(1, status, 'Failed to run command "%s": %s' % (command, output)) self.assertEqual(output, '/dev/root /\r\n/dev/vda3 /mnt') + + def test_bmap(self): + """Test generation of .bmap file""" + image = "directdisk" + status = runCmd("wic create %s -e core-image-minimal --bmap" % image).status + self.assertEqual(0, status) + self.assertEqual(1, len(glob(self.resultdir + "%s-*direct" % image))) + self.assertEqual(1, len(glob(self.resultdir + "%s-*direct.bmap" % image))) + + def test_systemd_bootdisk(self): + """Test creation of systemd-bootdisk image""" + image = "systemd-bootdisk" + self.assertEqual(0, runCmd("wic create %s -e core-image-minimal" \ + % image).status) + self.assertEqual(1, len(glob(self.resultdir + "%s-*direct" % image))) + + def test_sdimage_bootpart(self): + """Test creation of sdimage-bootpart image""" + image = "sdimage-bootpart" + self.write_config('IMAGE_BOOT_FILES = "bzImage"\n') + self.assertEqual(0, runCmd("wic create %s -e core-image-minimal" \ + % image).status) + self.assertEqual(1, len(glob(self.resultdir + "%s-*direct" % image))) diff --git a/import-layers/yocto-poky/meta/lib/oeqa/targetcontrol.py b/import-layers/yocto-poky/meta/lib/oeqa/targetcontrol.py index 5422a617c..24669f461 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/targetcontrol.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/targetcontrol.py @@ -43,9 +43,7 @@ def get_target_controller(d): return controller(d) -class BaseTarget(object): - - __metaclass__ = ABCMeta +class BaseTarget(object, metaclass=ABCMeta): supported_image_fstypes = [] @@ -68,7 +66,7 @@ class BaseTarget(object): bb.note("SSH log file: %s" % self.sshlog) @abstractmethod - def start(self, params=None, ssh=True): + def start(self, params=None, ssh=True, extra_bootparams=None): pass @abstractmethod @@ -121,12 +119,17 @@ class QemuTarget(BaseTarget): self.image_fstype = self.get_image_fstype(d) self.qemulog = os.path.join(self.testdir, "qemu_boot_log.%s" % self.datetime) - self.origrootfs = os.path.join(d.getVar("DEPLOY_DIR_IMAGE", True), d.getVar("IMAGE_LINK_NAME", True) + '.' + self.image_fstype) - self.rootfs = os.path.join(self.testdir, d.getVar("IMAGE_LINK_NAME", True) + '-testimage.' + self.image_fstype) + self.rootfs = os.path.join(d.getVar("DEPLOY_DIR_IMAGE", True), d.getVar("IMAGE_LINK_NAME", True) + '.' + self.image_fstype) self.kernel = os.path.join(d.getVar("DEPLOY_DIR_IMAGE", True), d.getVar("KERNEL_IMAGETYPE", False) + '-' + d.getVar('MACHINE', False) + '.bin') dump_target_cmds = d.getVar("testimage_dump_target", True) dump_host_cmds = d.getVar("testimage_dump_host", True) dump_dir = d.getVar("TESTIMAGE_DUMP_DIR", True) + if d.getVar("QEMU_USE_KVM", False) is not None \ + and d.getVar("QEMU_USE_KVM", False) == "True" \ + and "x86" in d.getVar("MACHINE", True): + use_kvm = True + else: + use_kvm = False # Log QemuRunner log output to a file import oe.path @@ -155,17 +158,14 @@ class QemuTarget(BaseTarget): display = d.getVar("BB_ORIGENV", False).getVar("DISPLAY", True), logfile = self.qemulog, boottime = int(d.getVar("TEST_QEMUBOOT_TIMEOUT", True)), + use_kvm = use_kvm, dump_dir = dump_dir, dump_host_cmds = d.getVar("testimage_dump_host", True)) self.target_dumper = TargetDumper(dump_target_cmds, dump_dir, self.runner) def deploy(self): - try: - bb.utils.mkdirhier(self.testdir) - shutil.copyfile(self.origrootfs, self.rootfs) - except Exception as e: - bb.fatal("Error copying rootfs: %s" % e) + bb.utils.mkdirhier(self.testdir) qemuloglink = os.path.join(self.testdir, "qemu_boot_log") if os.path.islink(qemuloglink): @@ -176,8 +176,8 @@ class QemuTarget(BaseTarget): bb.note("Qemu log file: %s" % self.qemulog) super(QemuTarget, self).deploy() - def start(self, params=None, ssh=True): - if self.runner.start(params, get_ip=ssh): + def start(self, params=None, ssh=True, extra_bootparams=None): + if self.runner.start(params, get_ip=ssh, extra_bootparams=extra_bootparams): if ssh: self.ip = self.runner.ip self.server_ip = self.runner.server_ip @@ -232,7 +232,7 @@ class SimpleRemoteTarget(BaseTarget): def deploy(self): super(SimpleRemoteTarget, self).deploy() - def start(self, params=None, ssh=True): + def start(self, params=None, ssh=True, extra_bootparams=None): if ssh: self.connection = SSHControl(self.ip, logfile=self.sshlog, port=self.port) diff --git a/import-layers/yocto-poky/meta/lib/oeqa/utils/commands.py b/import-layers/yocto-poky/meta/lib/oeqa/utils/commands.py index 48f644129..5cd0f7477 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/utils/commands.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/utils/commands.py @@ -41,7 +41,7 @@ class Command(object): self.data = data self.options = dict(self.defaultopts) - if isinstance(self.cmd, basestring): + if isinstance(self.cmd, str): self.options["shell"] = True if self.data: self.options['stdin'] = subprocess.PIPE @@ -78,7 +78,10 @@ class Command(object): self.process.kill() self.thread.join() - self.output = self.output.rstrip() + if not self.output: + self.output = "" + else: + self.output = self.output.decode("utf-8", errors='replace').rstrip() self.status = self.process.poll() self.log.debug("Command '%s' returned %d as exit code." % (self.cmd, self.status)) @@ -103,6 +106,7 @@ def runCmd(command, ignore_status=False, timeout=None, assert_error=True, **opti result.command = command result.status = cmd.status result.output = cmd.output + result.error = cmd.error result.pid = cmd.process.pid if result.status and not ignore_status: @@ -123,7 +127,7 @@ def bitbake(command, ignore_status=False, timeout=None, postconfig=None, **optio else: extra_args = "" - if isinstance(command, basestring): + if isinstance(command, str): cmd = "bitbake " + extra_args + " " + command else: cmd = [ "bitbake" ] + [a for a in (command + extra_args.split(" ")) if a not in [""]] @@ -141,22 +145,45 @@ def get_bb_env(target=None, postconfig=None): else: return bitbake("-e", postconfig=postconfig).output -def get_bb_var(var, target=None, postconfig=None): - val = None +def get_bb_vars(variables=None, target=None, postconfig=None): + """Get values of multiple bitbake variables""" bbenv = get_bb_env(target, postconfig=postconfig) + + var_re = re.compile(r'^(export )?(?P<var>\w+)="(?P<value>.*)"$') + unset_re = re.compile(r'^unset (?P<var>\w+)$') lastline = None + values = {} for line in bbenv.splitlines(): - if re.search("^(export )?%s=" % var, line): - val = line.split('=', 1)[1] - val = val.strip('\"') - break - elif re.match("unset %s$" % var, line): - # Handle [unexport] variables - if lastline.startswith('# "'): - val = lastline.split('\"')[1] - break + match = var_re.match(line) + val = None + if match: + val = match.group('value') + else: + match = unset_re.match(line) + if match: + # Handle [unexport] variables + if lastline.startswith('# "'): + val = lastline.split('"')[1] + if val: + var = match.group('var') + if variables is None: + values[var] = val + else: + if var in variables: + values[var] = val + variables.remove(var) + # Stop after all required variables have been found + if not variables: + break lastline = line - return val + if variables: + # Fill in missing values + for var in variables: + values[var] = None + return values + +def get_bb_var(var, target=None, postconfig=None): + return get_bb_vars([var], target, postconfig)[var] def get_test_layer(): layers = get_bb_var("BBLAYERS").split() @@ -196,7 +223,7 @@ def runqemu(pn, ssh=True): tinfoil.config_data.setVar("TEST_QEMUBOOT_TIMEOUT", "1000") import oe.recipeutils recipefile = oe.recipeutils.pn_to_recipe(tinfoil.cooker, pn) - recipedata = oe.recipeutils.parse_recipe(recipefile, [], tinfoil.config_data) + recipedata = oe.recipeutils.parse_recipe(tinfoil.cooker, recipefile, []) # The QemuRunner log is saved out, but we need to ensure it is at the right # log level (and then ensure that since it's a child of the BitBake logger, @@ -237,3 +264,15 @@ def runqemu(pn, ssh=True): qemu.stop() except: pass + +def updateEnv(env_file): + """ + Source a file and update environment. + """ + + cmd = ". %s; env -0" % env_file + result = runCmd(cmd) + + for line in result.output.split("\0"): + (key, _, value) = line.partition("=") + os.environ[key] = value diff --git a/import-layers/yocto-poky/meta/lib/oeqa/utils/decorators.py b/import-layers/yocto-poky/meta/lib/oeqa/utils/decorators.py index 0d79223a2..25f9c54e6 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/utils/decorators.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/utils/decorators.py @@ -57,6 +57,7 @@ class skipIfFailure(object): self.testcase = testcase def __call__(self,f): + @wraps(f) def wrapped_f(*args, **kwargs): res = getResults() if self.testcase in (res.getFailList() or res.getErrorList()): @@ -71,6 +72,7 @@ class skipIfSkipped(object): self.testcase = testcase def __call__(self,f): + @wraps(f) def wrapped_f(*args, **kwargs): res = getResults() if self.testcase in res.getSkipList(): @@ -85,6 +87,7 @@ class skipUnlessPassed(object): self.testcase = testcase def __call__(self,f): + @wraps(f) def wrapped_f(*args, **kwargs): res = getResults() if self.testcase in res.getSkipList() or \ @@ -97,11 +100,11 @@ class skipUnlessPassed(object): return wrapped_f class testcase(object): - def __init__(self, test_case): self.test_case = test_case def __call__(self, func): + @wraps(func) def wrapped_f(*args, **kwargs): return func(*args, **kwargs) wrapped_f.test_case = self.test_case @@ -112,6 +115,8 @@ class NoParsingFilter(logging.Filter): def filter(self, record): return record.levelno == 100 +import inspect + def LogResults(original_class): orig_method = original_class.run @@ -121,6 +126,19 @@ def LogResults(original_class): logfile = os.path.join(os.getcwd(),'results-'+caller+'.'+timestamp+'.log') linkfile = os.path.join(os.getcwd(),'results-'+caller+'.log') + def get_class_that_defined_method(meth): + if inspect.ismethod(meth): + for cls in inspect.getmro(meth.__self__.__class__): + if cls.__dict__.get(meth.__name__) is meth: + return cls + meth = meth.__func__ # fallback to __qualname__ parsing + if inspect.isfunction(meth): + cls = getattr(inspect.getmodule(meth), + meth.__qualname__.split('.<locals>', 1)[0].rsplit('.', 1)[0]) + if isinstance(cls, type): + return cls + return None + #rewrite the run method of unittest.TestCase to add testcase logging def run(self, result, *args, **kws): orig_method(self, result, *args, **kws) @@ -132,7 +150,7 @@ def LogResults(original_class): except AttributeError: test_case = self._testMethodName - class_name = str(testMethod.im_class).split("'")[1] + class_name = str(get_class_that_defined_method(testMethod)).split("'")[1] #create custom logging level for filtering. custom_log_level = 100 @@ -171,10 +189,24 @@ def LogResults(original_class): if passed: local_log.results("Testcase "+str(test_case)+": PASSED") + # XXX: In order to avoid race condition when test if exists the linkfile + # use bb.utils.lock, the best solution is to create a unique name for the + # link file. + try: + import bb + has_bb = True + lockfilename = linkfile + '.lock' + except ImportError: + has_bb = False + + if has_bb: + lf = bb.utils.lockfile(lockfilename, block=True) # Create symlink to the current log - if os.path.exists(linkfile): + if os.path.lexists(linkfile): os.remove(linkfile) os.symlink(logfile, linkfile) + if has_bb: + bb.utils.unlockfile(lf) original_class.run = run @@ -212,7 +244,7 @@ def tag(*args, **kwargs): def wrap_ob(ob): for name in args: setattr(ob, __tag_prefix + name, True) - for name, value in kwargs.iteritems(): + for name, value in kwargs.items(): setattr(ob, __tag_prefix + name, value) return ob return wrap_ob diff --git a/import-layers/yocto-poky/meta/lib/oeqa/utils/dump.py b/import-layers/yocto-poky/meta/lib/oeqa/utils/dump.py index 63a591d36..71422a9ae 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/utils/dump.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/utils/dump.py @@ -3,7 +3,7 @@ import sys import errno import datetime import itertools -from commands import runCmd +from .commands import runCmd def get_host_dumper(d): cmds = d.getVar("testimage_dump_host", True) diff --git a/import-layers/yocto-poky/meta/lib/oeqa/utils/git.py b/import-layers/yocto-poky/meta/lib/oeqa/utils/git.py new file mode 100644 index 000000000..ae85d2766 --- /dev/null +++ b/import-layers/yocto-poky/meta/lib/oeqa/utils/git.py @@ -0,0 +1,68 @@ +# +# Copyright (C) 2016 Intel Corporation +# +# Released under the MIT license (see COPYING.MIT) +# +"""Git repository interactions""" +import os + +from oeqa.utils.commands import runCmd + + +class GitError(Exception): + """Git error handling""" + pass + +class GitRepo(object): + """Class representing a Git repository clone""" + def __init__(self, path, is_topdir=False): + self.top_dir = self._run_git_cmd_at(['rev-parse', '--show-toplevel'], + path) + realpath = os.path.realpath(path) + if is_topdir and realpath != self.top_dir: + raise GitError("{} is not a Git top directory".format(realpath)) + + @staticmethod + def _run_git_cmd_at(git_args, cwd, **kwargs): + """Run git command at a specified directory""" + git_cmd = 'git ' if isinstance(git_args, str) else ['git'] + git_cmd += git_args + ret = runCmd(git_cmd, ignore_status=True, cwd=cwd, **kwargs) + if ret.status: + cmd_str = git_cmd if isinstance(git_cmd, str) \ + else ' '.join(git_cmd) + raise GitError("'{}' failed with exit code {}: {}".format( + cmd_str, ret.status, ret.output)) + return ret.output.strip() + + @staticmethod + def init(path): + """Initialize a new Git repository""" + GitRepo._run_git_cmd_at('init', cwd=path) + return GitRepo(path, is_topdir=True) + + def run_cmd(self, git_args, env_update=None): + """Run Git command""" + env = None + if env_update: + env = os.environ.copy() + env.update(env_update) + return self._run_git_cmd_at(git_args, self.top_dir, env=env) + + def rev_parse(self, revision): + """Do git rev-parse""" + try: + return self.run_cmd(['rev-parse', revision]) + except GitError: + # Revision does not exist + return None + + def get_current_branch(self): + """Get current branch""" + try: + # Strip 11 chars, i.e. 'refs/heads' from the beginning + return self.run_cmd(['symbolic-ref', 'HEAD'])[11:] + except GitError: + return None + + diff --git a/import-layers/yocto-poky/meta/lib/oeqa/utils/httpserver.py b/import-layers/yocto-poky/meta/lib/oeqa/utils/httpserver.py index 76518d8ef..7d1233145 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/utils/httpserver.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/utils/httpserver.py @@ -1,8 +1,9 @@ -import SimpleHTTPServer +import http.server import multiprocessing import os +from socketserver import ThreadingMixIn -class HTTPServer(SimpleHTTPServer.BaseHTTPServer.HTTPServer): +class HTTPServer(ThreadingMixIn, http.server.HTTPServer): def server_start(self, root_dir): import signal @@ -10,7 +11,7 @@ class HTTPServer(SimpleHTTPServer.BaseHTTPServer.HTTPServer): os.chdir(root_dir) self.serve_forever() -class HTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): +class HTTPRequestHandler(http.server.SimpleHTTPRequestHandler): def log_message(self, format_str, *args): pass diff --git a/import-layers/yocto-poky/meta/lib/oeqa/utils/logparser.py b/import-layers/yocto-poky/meta/lib/oeqa/utils/logparser.py index 87b50354c..b377dcd27 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/utils/logparser.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/utils/logparser.py @@ -3,7 +3,7 @@ import sys import os import re -import ftools +from . import ftools # A parser that can be used to identify weather a line is a test result or a section statement. diff --git a/import-layers/yocto-poky/meta/lib/oeqa/utils/package_manager.py b/import-layers/yocto-poky/meta/lib/oeqa/utils/package_manager.py new file mode 100644 index 000000000..099ecc972 --- /dev/null +++ b/import-layers/yocto-poky/meta/lib/oeqa/utils/package_manager.py @@ -0,0 +1,29 @@ +def get_package_manager(d, root_path): + """ + Returns an OE package manager that can install packages in root_path. + """ + from oe.package_manager import RpmPM, OpkgPM, DpkgPM + + pkg_class = d.getVar("IMAGE_PKGTYPE", True) + if pkg_class == "rpm": + pm = RpmPM(d, + root_path, + d.getVar('TARGET_VENDOR', True)) + pm.create_configs() + + elif pkg_class == "ipk": + pm = OpkgPM(d, + root_path, + d.getVar("IPKGCONF_TARGET", True), + d.getVar("ALL_MULTILIB_PACKAGE_ARCHS", True)) + + elif pkg_class == "deb": + pm = DpkgPM(d, + root_path, + d.getVar('PACKAGE_ARCHS', True), + d.getVar('DPKG_ARCH', True)) + + pm.write_index() + pm.update() + + return pm diff --git a/import-layers/yocto-poky/meta/lib/oeqa/utils/qemurunner.py b/import-layers/yocto-poky/meta/lib/oeqa/utils/qemurunner.py index 784cf964f..8f1b5b980 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/utils/qemurunner.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/utils/qemurunner.py @@ -20,16 +20,17 @@ from oeqa.utils.dump import HostDumper import logging logger = logging.getLogger("BitBake.QemuRunner") +logger.addHandler(logging.StreamHandler()) # Get Unicode non printable control chars -control_range = range(0,32)+range(127,160) -control_chars = [unichr(x) for x in control_range - if unichr(x) not in string.printable] +control_range = list(range(0,32))+list(range(127,160)) +control_chars = [chr(x) for x in control_range + if chr(x) not in string.printable] re_control_char = re.compile('[%s]' % re.escape("".join(control_chars))) class QemuRunner: - def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, boottime, dump_dir, dump_host_cmds): + def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, boottime, dump_dir, dump_host_cmds, use_kvm): # Popen object for runqemu self.runqemu = None @@ -49,6 +50,7 @@ class QemuRunner: self.boottime = boottime self.logged = False self.thread = None + self.use_kvm = use_kvm self.runqemutime = 60 self.host_dumper = HostDumper(dump_host_cmds, dump_dir) @@ -71,7 +73,8 @@ class QemuRunner: if self.logfile: # It is needed to sanitize the data received from qemu # because is possible to have control characters - msg = re_control_char.sub('', unicode(msg, 'utf-8')) + msg = msg.decode("utf-8") + msg = re_control_char.sub('', msg) with codecs.open(self.logfile, "a", encoding="utf-8") as f: f.write("%s" % msg) @@ -79,7 +82,7 @@ class QemuRunner: import fcntl fl = fcntl.fcntl(o, fcntl.F_GETFL) fcntl.fcntl(o, fcntl.F_SETFL, fl | os.O_NONBLOCK) - return os.read(o.fileno(), 1000000) + return os.read(o.fileno(), 1000000).decode("utf-8") def handleSIGCHLD(self, signum, frame): @@ -91,7 +94,7 @@ class QemuRunner: self._dump_host() raise SystemExit - def start(self, qemuparams = None, get_ip = True): + def start(self, qemuparams = None, get_ip = True, extra_bootparams = None): if self.display: os.environ["DISPLAY"] = self.display # Set this flag so that Qemu doesn't do any grabs as SDL grabs @@ -114,12 +117,16 @@ class QemuRunner: try: threadsock, threadport = self.create_socket() self.server_socket, self.serverport = self.create_socket() - except socket.error, msg: + except socket.error as msg: logger.error("Failed to create listening socket: %s" % msg[1]) return False - self.qemuparams = 'bootparams="console=tty1 console=ttyS0,115200n8 printk.time=1" qemuparams="-serial tcp:127.0.0.1:{}"'.format(threadport) + bootparams = 'console=tty1 console=ttyS0,115200n8 printk.time=1' + if extra_bootparams: + bootparams = bootparams + ' ' + extra_bootparams + + self.qemuparams = 'bootparams="{0}" qemuparams="-serial tcp:127.0.0.1:{1}"'.format(bootparams, threadport) if not self.display: self.qemuparams = 'nographic ' + self.qemuparams if qemuparams: @@ -128,7 +135,15 @@ class QemuRunner: self.origchldhandler = signal.getsignal(signal.SIGCHLD) signal.signal(signal.SIGCHLD, self.handleSIGCHLD) - launch_cmd = 'runqemu tcpserial=%s %s %s %s' % (self.serverport, self.machine, self.rootfs, self.qemuparams) + launch_cmd = 'runqemu snapshot ' + if self.use_kvm: + logger.info('Using kvm for runqemu') + launch_cmd += 'kvm ' + else: + logger.info('Not using kvm for runqemu') + launch_cmd += 'tcpserial=%s %s %s %s' % (self.serverport, self.machine, self.rootfs, self.qemuparams) + logger.info('launchcmd=%s'%(launch_cmd)) + # FIXME: We pass in stdin=subprocess.PIPE here to work around stty # blocking at the end of the runqemu script when using this within # oe-selftest (this makes stty error out immediately). There ought @@ -137,12 +152,12 @@ class QemuRunner: output = self.runqemu.stdout # - # We need the preexec_fn above so that all runqemu processes can easily be killed + # We need the preexec_fn above so that all runqemu processes can easily be killed # (by killing their process group). This presents a problem if this controlling - # process itself is killed however since those processes don't notice the death + # process itself is killed however since those processes don't notice the death # of the parent and merrily continue on. # - # Rather than hack runqemu to deal with this, we add something here instead. + # Rather than hack runqemu to deal with this, we add something here instead. # Basically we fork off another process which holds an open pipe to the parent # and also is setpgrp. If/when the pipe sees EOF from the parent dieing, it kills # the process group. This is like pctrl's PDEATHSIG but for a process group @@ -192,7 +207,7 @@ class QemuRunner: else: self.ip = ips[0] self.server_ip = ips[1] - except IndexError, ValueError: + except (IndexError, ValueError): logger.info("Couldn't get ip from qemu process arguments! Here is the qemu command line used:\n%s\nand output from runqemu:\n%s" % (cmdline, self.getOutput(output))) self._dump_host() self.stop() @@ -219,6 +234,7 @@ class QemuRunner: stopread = False qemusock = None bootlog = '' + data = b'' while time.time() < endtime and not stopread: sread, swrite, serror = select.select(socklist, [], [], 5) for sock in sread: @@ -229,14 +245,19 @@ class QemuRunner: socklist.remove(self.server_socket) logger.info("Connection from %s:%s" % addr) else: - data = sock.recv(1024) + data = data + sock.recv(1024) if data: - bootlog += data - if re.search(".* login:", bootlog): - self.server_socket = qemusock - stopread = True - reachedlogin = True - logger.info("Reached login banner") + try: + data = data.decode("utf-8", errors="surrogateescape") + bootlog += data + data = b'' + if re.search(".* login:", bootlog): + self.server_socket = qemusock + stopread = True + reachedlogin = True + logger.info("Reached login banner") + except UnicodeDecodeError: + continue else: socklist.remove(sock) sock.close() @@ -277,13 +298,14 @@ class QemuRunner: if hasattr(self, "origchldhandler"): signal.signal(signal.SIGCHLD, self.origchldhandler) if self.runqemu: - os.kill(self.monitorpid, signal.SIGKILL) - logger.info("Sending SIGTERM to runqemu") - try: - os.killpg(os.getpgid(self.runqemu.pid), signal.SIGTERM) - except OSError as e: - if e.errno != errno.ESRCH: - raise + if hasattr(self, "monitorpid"): + os.kill(self.monitorpid, signal.SIGKILL) + logger.info("Sending SIGTERM to runqemu") + try: + os.killpg(os.getpgid(self.runqemu.pid), signal.SIGTERM) + except OSError as e: + if e.errno != errno.ESRCH: + raise endtime = time.time() + self.runqemutime while self.runqemu.poll() is None and time.time() < endtime: time.sleep(1) @@ -325,7 +347,7 @@ class QemuRunner: # Walk the process tree from the process specified looking for a qemu-system. Return its [pid'cmd] # ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,command'], stdout=subprocess.PIPE).communicate()[0] - processes = ps.split('\n') + processes = ps.decode("utf-8").split('\n') nfields = len(processes[0].split()) - 1 pids = {} commands = {} @@ -354,9 +376,9 @@ class QemuRunner: if p not in parents: parents.append(p) newparents = next - #print "Children matching %s:" % str(parents) + #print("Children matching %s:" % str(parents)) for p in parents: - # Need to be careful here since runqemu-internal runs "ldd qemu-system-xxxx" + # Need to be careful here since runqemu runs "ldd qemu-system-xxxx" # Also, old versions of ldd (2.11) run "LD_XXXX qemu-system-xxxx" basecmd = commands[p].split()[0] basecmd = os.path.basename(basecmd) @@ -370,14 +392,14 @@ class QemuRunner: data = '' status = 0 - self.server_socket.sendall(command) + self.server_socket.sendall(command.encode('utf-8')) keepreading = True while keepreading: sread, _, _ = select.select([self.server_socket],[],[],5) if sread: answer = self.server_socket.recv(1024) if answer: - data += answer + data += answer.decode('utf-8') # Search the prompt to stop if re.search("[a-zA-Z0-9]+@[a-zA-Z0-9\-]+:~#", data): keepreading = False @@ -442,7 +464,7 @@ class LoggingThread(threading.Thread): def stop(self): self.logger.info("Stopping logging thread") if self.running: - os.write(self.writepipe, "stop") + os.write(self.writepipe, bytes("stop", "utf-8")) def teardown(self): self.logger.info("Tearing down logging thread") diff --git a/import-layers/yocto-poky/meta/lib/oeqa/utils/qemutinyrunner.py b/import-layers/yocto-poky/meta/lib/oeqa/utils/qemutinyrunner.py index 4f95101f3..d554f0dbc 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/utils/qemutinyrunner.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/utils/qemutinyrunner.py @@ -13,7 +13,7 @@ import re import socket import select import bb -from qemurunner import QemuRunner +from .qemurunner import QemuRunner class QemuTinyRunner(QemuRunner): @@ -50,7 +50,7 @@ class QemuTinyRunner(QemuRunner): self.server_socket.connect(self.socketfile) bb.note("Created listening socket for qemu serial console.") tries = 0 - except socket.error, msg: + except socket.error as msg: self.server_socket.close() bb.fatal("Failed to create listening socket.") tries -= 1 @@ -60,7 +60,7 @@ class QemuTinyRunner(QemuRunner): with open(self.logfile, "a") as f: f.write("%s" % msg) - def start(self, qemuparams = None): + def start(self, qemuparams = None, ssh=True, extra_bootparams=None): if self.display: os.environ["DISPLAY"] = self.display @@ -102,7 +102,7 @@ class QemuTinyRunner(QemuRunner): bb.note("Qemu pid didn't appeared in %s seconds" % self.runqemutime) output = self.runqemu.stdout self.stop() - bb.note("Output from runqemu:\n%s" % output.read()) + bb.note("Output from runqemu:\n%s" % output.read().decode("utf-8")) return False return self.is_alive() @@ -131,7 +131,7 @@ class QemuTinyRunner(QemuRunner): # Walk the process tree from the process specified looking for a qemu-system. Return its [pid'cmd] # ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,command'], stdout=subprocess.PIPE).communicate()[0] - processes = ps.split('\n') + processes = ps.decode("utf-8").split('\n') nfields = len(processes[0].split()) - 1 pids = {} commands = {} @@ -160,11 +160,11 @@ class QemuTinyRunner(QemuRunner): if p not in parents: parents.append(p) newparents = next - #print "Children matching %s:" % str(parents) + #print("Children matching %s:" % str(parents)) for p in parents: - # Need to be careful here since runqemu-internal runs "ldd qemu-system-xxxx" + # Need to be careful here since runqemu runs "ldd qemu-system-xxxx" # Also, old versions of ldd (2.11) run "LD_XXXX qemu-system-xxxx" basecmd = commands[p].split()[0] basecmd = os.path.basename(basecmd) if "qemu-system" in basecmd and "-serial unix" in commands[p]: - return [int(p),commands[p]]
\ No newline at end of file + return [int(p),commands[p]] diff --git a/import-layers/yocto-poky/meta/lib/oeqa/utils/sshcontrol.py b/import-layers/yocto-poky/meta/lib/oeqa/utils/sshcontrol.py index 165874416..05d650255 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/utils/sshcontrol.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/utils/sshcontrol.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Copyright (C) 2013 Intel Corporation # # Released under the MIT license (see COPYING.MIT) @@ -51,16 +52,19 @@ class SSHProcess(object): endtime = self.starttime + timeout eof = False while time.time() < endtime and not eof: - if select.select([self.process.stdout], [], [], 5)[0] != []: - data = os.read(self.process.stdout.fileno(), 1024) - if not data: - self.process.stdout.close() - eof = True - else: - output += data - self.log(data) - endtime = time.time() + timeout - + try: + if select.select([self.process.stdout], [], [], 5)[0] != []: + data = os.read(self.process.stdout.fileno(), 1024) + if not data: + self.process.stdout.close() + eof = True + else: + data = data.decode("utf-8") + output += data + self.log(data) + endtime = time.time() + timeout + except InterruptedError: + continue # process hasn't returned yet if not eof: @@ -145,9 +149,97 @@ class SSHControl(object): return self._internal_run(command, timeout, self.ignore_status) def copy_to(self, localpath, remotepath): - command = self.scp + [localpath, '%s@%s:%s' % (self.user, self.ip, remotepath)] - return self._internal_run(command, ignore_status=False) + if os.path.islink(localpath): + link = os.readlink(localpath) + dst_dir, dst_base = os.path.split(remotepath) + return self.run("cd %s; ln -s %s %s" % (dst_dir, link, dst_base)) + else: + command = self.scp + [localpath, '%s@%s:%s' % (self.user, self.ip, remotepath)] + return self._internal_run(command, ignore_status=False) def copy_from(self, remotepath, localpath): command = self.scp + ['%s@%s:%s' % (self.user, self.ip, remotepath), localpath] return self._internal_run(command, ignore_status=False) + + def copy_dir_to(self, localpath, remotepath): + """ + Copy recursively localpath directory to remotepath in target. + """ + + for root, dirs, files in os.walk(localpath): + # Create directories in the target as needed + for d in dirs: + tmp_dir = os.path.join(root, d).replace(localpath, "") + new_dir = os.path.join(remotepath, tmp_dir.lstrip("/")) + cmd = "mkdir -p %s" % new_dir + self.run(cmd) + + # Copy files into the target + for f in files: + tmp_file = os.path.join(root, f).replace(localpath, "") + dst_file = os.path.join(remotepath, tmp_file.lstrip("/")) + src_file = os.path.join(root, f) + self.copy_to(src_file, dst_file) + + + def delete_files(self, remotepath, files): + """ + Delete files in target's remote path. + """ + + cmd = "rm" + if not isinstance(files, list): + files = [files] + + for f in files: + cmd = "%s %s" % (cmd, os.path.join(remotepath, f)) + + self.run(cmd) + + + def delete_dir(self, remotepath): + """ + Delete remotepath directory in target. + """ + + cmd = "rmdir %s" % remotepath + self.run(cmd) + + + def delete_dir_structure(self, localpath, remotepath): + """ + Delete recursively localpath structure directory in target's remotepath. + + This function is very usefult to delete a package that is installed in + the DUT and the host running the test has such package extracted in tmp + directory. + + Example: + pwd: /home/user/tmp + tree: . + └── work + ├── dir1 + │ └── file1 + └── dir2 + + localpath = "/home/user/tmp" and remotepath = "/home/user" + + With the above variables this function will try to delete the + directory in the DUT in this order: + /home/user/work/dir1/file1 + /home/user/work/dir1 (if dir is empty) + /home/user/work/dir2 (if dir is empty) + /home/user/work (if dir is empty) + """ + + for root, dirs, files in os.walk(localpath, topdown=False): + # Delete files first + tmpdir = os.path.join(root).replace(localpath, "") + remotedir = os.path.join(remotepath, tmpdir.lstrip("/")) + self.delete_files(remotedir, files) + + # Remove dirs if empty + for d in dirs: + tmpdir = os.path.join(root, d).replace(localpath, "") + remotedir = os.path.join(remotepath, tmpdir.lstrip("/")) + self.delete_dir(remotepath) diff --git a/import-layers/yocto-poky/meta/lib/oeqa/utils/targetbuild.py b/import-layers/yocto-poky/meta/lib/oeqa/utils/targetbuild.py index f850d78df..59593f5ef 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/utils/targetbuild.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/utils/targetbuild.py @@ -10,18 +10,17 @@ import bb.utils import subprocess from abc import ABCMeta, abstractmethod -class BuildProject(): - - __metaclass__ = ABCMeta +class BuildProject(metaclass=ABCMeta): def __init__(self, d, uri, foldername=None, tmpdir="/tmp/"): self.d = d self.uri = uri self.archive = os.path.basename(uri) self.localarchive = os.path.join(tmpdir,self.archive) - self.fname = re.sub(r'.tar.bz2|tar.gz$', '', self.archive) if foldername: self.fname = foldername + else: + self.fname = re.sub(r'\.tar\.bz2$|\.tar\.gz$|\.tar\.xz$', '', self.archive) # Download self.archive to self.localarchive def _download_archive(self): @@ -118,10 +117,10 @@ class SDKBuildProject(BuildProject): subprocess.check_call(cmd, shell=True) #Change targetdir to project folder - self.targetdir = self.targetdir + self.fname + self.targetdir = os.path.join(self.targetdir, self.fname) - def run_configure(self, configure_args=''): - return super(SDKBuildProject, self).run_configure(configure_args=(configure_args or '$CONFIGURE_FLAGS'), extra_cmds=' gnu-configize; ') + def run_configure(self, configure_args='', extra_cmds=' gnu-configize; '): + return super(SDKBuildProject, self).run_configure(configure_args=(configure_args or '$CONFIGURE_FLAGS'), extra_cmds=extra_cmds) def run_install(self, install_args=''): return super(SDKBuildProject, self).run_install(install_args=(install_args or "DESTDIR=%s/../install" % self.targetdir)) @@ -134,4 +133,3 @@ class SDKBuildProject(BuildProject): def _run(self, cmd): self.log("Running . %s; " % self.sdkenv + cmd) return subprocess.call(". %s; " % self.sdkenv + cmd, shell=True) - diff --git a/import-layers/yocto-poky/meta/lib/oeqa/utils/testexport.py b/import-layers/yocto-poky/meta/lib/oeqa/utils/testexport.py index 243463bc1..57be2ca44 100644 --- a/import-layers/yocto-poky/meta/lib/oeqa/utils/testexport.py +++ b/import-layers/yocto-poky/meta/lib/oeqa/utils/testexport.py @@ -6,7 +6,7 @@ import os, re, glob as g, shutil as sh,sys from time import sleep -from commands import runCmd +from .commands import runCmd from difflib import SequenceMatcher as SM try: @@ -17,13 +17,13 @@ except ImportError: pass def plain(self, msg): if msg: - print msg + print(msg) def warn(self, msg): if msg: - print "WARNING: " + msg + print("WARNING: " + msg) def fatal(self, msg): if msg: - print "FATAL:" + msg + print("FATAL:" + msg) sys.exit(1) bb = my_log() |
