Squashed 'import-layers/yocto-poky/' changes from dc8508f6099..67491b0c104

Yocto 2.2.2 (Morty)

Change-Id: Id9a452e28940d9f166957de243d9cb1d8818704e
git-subtree-dir: import-layers/yocto-poky
git-subtree-split: 67491b0c104101bb9f366d697edd23c895be4302
Signed-off-by: Brad Bishop <bradleyb@fuzziesquirrel.com>

8 files changed, 103 insertions, 70 deletions

diff --git a/import-layers/yocto-poky/meta/lib/oe/copy_buildsystem.py b/import-layers/yocto-poky/meta/lib/oe/copy_buildsystem.py
index afaff6859..29ac6d418 100644
--- a/import-layers/yocto-poky/meta/lib/oe/copy_buildsystem.py
+++ b/import-layers/yocto-poky/meta/lib/oe/copy_buildsystem.py
@@ -4,11 +4,15 @@ import stat
 import shutil
 
 def _smart_copy(src, dest):
+    import subprocess
     # smart_copy will choose the correct function depending on whether the
     # source is a file or a directory.
     mode = os.stat(src).st_mode
     if stat.S_ISDIR(mode):
-        shutil.copytree(src, dest, symlinks=True, ignore=shutil.ignore_patterns('.git'))
+        bb.utils.mkdirhier(dest)
+        cmd = "tar --exclude='.git' --xattrs --xattrs-include='*' -chf - -C %s -p . \
+        | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dest)
+        subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
     else:
         shutil.copyfile(src, dest)
         shutil.copymode(src, dest)
diff --git a/import-layers/yocto-poky/meta/lib/oe/gpg_sign.py b/import-layers/yocto-poky/meta/lib/oe/gpg_sign.py
index 38eb0cb13..ba61f9890 100644
--- a/import-layers/yocto-poky/meta/lib/oe/gpg_sign.py
+++ b/import-layers/yocto-poky/meta/lib/oe/gpg_sign.py
@@ -10,6 +10,7 @@ class LocalSigner(object):
         self.gpg_bin = d.getVar('GPG_BIN', True) or \
                   bb.utils.which(os.getenv('PATH'), 'gpg')
         self.gpg_path = d.getVar('GPG_PATH', True)
+        self.gpg_version = self.get_gpg_version()
         self.rpm_bin = bb.utils.which(os.getenv('PATH'), "rpm")
 
     def export_pubkey(self, output_file, keyid, armor=True):
@@ -31,15 +32,18 @@ class LocalSigner(object):
 
         cmd = self.rpm_bin + " --addsign --define '_gpg_name %s'  " % keyid
         cmd += "--define '_gpg_passphrase %s' " % passphrase
+        if self.gpg_version > (2,1,):
+            cmd += "--define '_gpg_sign_cmd_extra_args --pinentry-mode=loopback' "
         if self.gpg_bin:
             cmd += "--define '%%__gpg %s' " % self.gpg_bin
         if self.gpg_path:
             cmd += "--define '_gpg_path %s' " % self.gpg_path
-        cmd += ' '.join(files)
 
-        status, output = oe.utils.getstatusoutput(cmd)
-        if status:
-            raise bb.build.FuncFailed("Failed to sign RPM packages: %s" % output)
+        # Sign in chunks of 100 packages
+        for i in range(0, len(files), 100):
+            status, output = oe.utils.getstatusoutput(cmd + ' '.join(files[i:i+100]))
+            if status:
+                raise bb.build.FuncFailed("Failed to sign RPM packages: %s" % output)
 
     def detach_sign(self, input_file, keyid, passphrase_file, passphrase=None, armor=True):
         """Create a detached signature of a file"""
@@ -58,9 +62,7 @@ class LocalSigner(object):
 
         #gpg > 2.1 supports password pipes only through the loopback interface
         #gpg < 2.1 errors out if given unknown parameters
-        dots = self.get_gpg_version().split('.')
-        assert len(dots) >= 2
-        if int(dots[0]) >= 2 and int(dots[1]) >= 1:
+        if self.gpg_version > (2,1,):
             cmd += ['--pinentry-mode', 'loopback']
 
         cmd += [input_file]
@@ -87,10 +89,11 @@ class LocalSigner(object):
 
 
     def get_gpg_version(self):
-        """Return the gpg version"""
+        """Return the gpg version as a tuple of ints"""
         import subprocess
         try:
-            return subprocess.check_output((self.gpg_bin, "--version")).split()[2].decode("utf-8")
+            ver_str = subprocess.check_output((self.gpg_bin, "--version")).split()[2].decode("utf-8")
+            return tuple([int(i) for i in ver_str.split('.')])
         except subprocess.CalledProcessError as e:
             raise bb.build.FuncFailed("Could not get gpg version: %s" % e)
 
diff --git a/import-layers/yocto-poky/meta/lib/oe/package_manager.py b/import-layers/yocto-poky/meta/lib/oe/package_manager.py
index 3cee9730a..13577b18b 100644
--- a/import-layers/yocto-poky/meta/lib/oe/package_manager.py
+++ b/import-layers/yocto-poky/meta/lib/oe/package_manager.py
@@ -1673,13 +1673,15 @@ class OpkgPM(OpkgDpkgPM):
                                         self.d.getVar('FEED_DEPLOYDIR_BASE_URI', True),
                                         arch))
 
-                        if self.opkg_dir != '/var/lib/opkg':
+                        if self.d.getVar('OPKGLIBDIR', True) != '/var/lib':
                             # There is no command line option for this anymore, we need to add
                             # info_dir and status_file to config file, if OPKGLIBDIR doesn't have
                             # the default value of "/var/lib" as defined in opkg:
-                            # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_INFO_DIR      "/var/lib/opkg/info"
-                            # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_STATUS_FILE   "/var/lib/opkg/status"
+                            # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_LISTS_DIR     VARDIR "/lib/opkg/lists"
+                            # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_INFO_DIR      VARDIR "/lib/opkg/info"
+                            # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_STATUS_FILE   VARDIR "/lib/opkg/status"
                             cfg_file.write("option info_dir     %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR', True), 'opkg', 'info'))
+                            cfg_file.write("option lists_dir    %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR', True), 'opkg', 'lists'))
                             cfg_file.write("option status_file  %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR', True), 'opkg', 'status'))
 
 
@@ -1698,13 +1700,15 @@ class OpkgPM(OpkgDpkgPM):
                     config_file.write("src oe-%s file:%s\n" %
                                       (arch, pkgs_dir))
 
-            if self.opkg_dir != '/var/lib/opkg':
+            if self.d.getVar('OPKGLIBDIR', True) != '/var/lib':
                 # There is no command line option for this anymore, we need to add
                 # info_dir and status_file to config file, if OPKGLIBDIR doesn't have
                 # the default value of "/var/lib" as defined in opkg:
-                # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_INFO_DIR      "/var/lib/opkg/info"
-                # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_STATUS_FILE   "/var/lib/opkg/status"
+                # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_LISTS_DIR     VARDIR "/lib/opkg/lists"
+                # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_INFO_DIR      VARDIR "/lib/opkg/info"
+                # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_STATUS_FILE   VARDIR "/lib/opkg/status"
                 config_file.write("option info_dir     %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR', True), 'opkg', 'info'))
+                config_file.write("option lists_dir    %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR', True), 'opkg', 'lists'))
                 config_file.write("option status_file  %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR', True), 'opkg', 'status'))
 
     def insert_feeds_uris(self):
@@ -1776,7 +1780,7 @@ class OpkgPM(OpkgDpkgPM):
 
     def remove(self, pkgs, with_dependencies=True):
         if with_dependencies:
-            cmd = "%s %s --force-depends --force-remove --force-removal-of-dependent-packages remove %s" % \
+            cmd = "%s %s --force-remove --force-removal-of-dependent-packages remove %s" % \
                 (self.opkg_cmd, self.opkg_args, ' '.join(pkgs))
         else:
             cmd = "%s %s --force-depends remove %s" % \
@@ -1860,7 +1864,10 @@ class OpkgPM(OpkgDpkgPM):
 
         # Create an temp dir as opkg root for dummy installation
         temp_rootfs = self.d.expand('${T}/opkg')
-        temp_opkg_dir = os.path.join(temp_rootfs, 'var/lib/opkg')
+        opkg_lib_dir = self.d.getVar('OPKGLIBDIR', True)
+        if opkg_lib_dir[0] == "/":
+            opkg_lib_dir = opkg_lib_dir[1:]
+        temp_opkg_dir = os.path.join(temp_rootfs, opkg_lib_dir, 'opkg')
         bb.utils.mkdirhier(temp_opkg_dir)
 
         opkg_args = "-f %s -o %s " % (self.config_file, temp_rootfs)
diff --git a/import-layers/yocto-poky/meta/lib/oe/path.py b/import-layers/yocto-poky/meta/lib/oe/path.py
index 06a5af265..ed7fd1eef 100644
--- a/import-layers/yocto-poky/meta/lib/oe/path.py
+++ b/import-layers/yocto-poky/meta/lib/oe/path.py
@@ -83,12 +83,14 @@ def copyhardlinktree(src, dst):
         if os.path.isdir(src):
             import glob
             if len(glob.glob('%s/.??*' % src)) > 0:
-                source = '%s/.??* ' % src
-            source = source + '%s/*' % src
+                source = './.??* '
+            source += './*'
+            s_dir = src
         else:
             source = src
-        cmd = 'cp -afl --preserve=xattr %s %s' % (source, dst)
-        subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
+            s_dir = os.getcwd()
+        cmd = 'cp -afl --preserve=xattr %s %s' % (source, os.path.realpath(dst))
+        subprocess.check_output(cmd, shell=True, cwd=s_dir, stderr=subprocess.STDOUT)
     else:
         copytree(src, dst)
 
diff --git a/import-layers/yocto-poky/meta/lib/oe/qa.py b/import-layers/yocto-poky/meta/lib/oe/qa.py
index fbe719d8e..22d76dcbc 100644
--- a/import-layers/yocto-poky/meta/lib/oe/qa.py
+++ b/import-layers/yocto-poky/meta/lib/oe/qa.py
@@ -1,4 +1,4 @@
-import os, struct
+import os, struct, mmap
 
 class NotELFFileError(Exception):
     pass
@@ -23,9 +23,9 @@ class ELFFile:
     EV_CURRENT   = 1
 
     # possible values for EI_DATA
-    ELFDATANONE  = 0
-    ELFDATA2LSB  = 1
-    ELFDATA2MSB  = 2
+    EI_DATA_NONE  = 0
+    EI_DATA_LSB  = 1
+    EI_DATA_MSB  = 2
 
     PT_INTERP = 3
 
@@ -34,51 +34,46 @@ class ELFFile:
             #print "'%x','%x' %s" % (ord(expectation), ord(result), self.name)
             raise NotELFFileError("%s is not an ELF" % self.name)
 
-    def __init__(self, name, bits = 0):
+    def __init__(self, name):
         self.name = name
-        self.bits = bits
         self.objdump_output = {}
 
-    def open(self):
-        if not os.path.isfile(self.name):
-            raise NotELFFileError("%s is not a normal file" % self.name)
+    # Context Manager functions to close the mmap explicitly
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.data.close()
 
+    def open(self):
         with open(self.name, "rb") as f:
-            # Read 4k which should cover most of the headers we're after
-            self.data = f.read(4096)
+            try:
+                self.data = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
+            except ValueError:
+                # This means the file is empty
+                raise NotELFFileError("%s is empty" % self.name)
 
+        # Check the file has the minimum number of ELF table entries
         if len(self.data) < ELFFile.EI_NIDENT + 4:
             raise NotELFFileError("%s is not an ELF" % self.name)
 
+        # ELF header
         self.my_assert(self.data[0], 0x7f)
         self.my_assert(self.data[1], ord('E'))
         self.my_assert(self.data[2], ord('L'))
         self.my_assert(self.data[3], ord('F'))
-        if self.bits == 0:
-            if self.data[ELFFile.EI_CLASS] == ELFFile.ELFCLASS32:
-                self.bits = 32
-            elif self.data[ELFFile.EI_CLASS] == ELFFile.ELFCLASS64:
-                self.bits = 64
-            else:
-                # Not 32-bit or 64.. lets assert
-                raise NotELFFileError("ELF but not 32 or 64 bit.")
-        elif self.bits == 32:
-            self.my_assert(self.data[ELFFile.EI_CLASS], ELFFile.ELFCLASS32)
-        elif self.bits == 64:
-            self.my_assert(self.data[ELFFile.EI_CLASS], ELFFile.ELFCLASS64)
+        if self.data[ELFFile.EI_CLASS] == ELFFile.ELFCLASS32:
+            self.bits = 32
+        elif self.data[ELFFile.EI_CLASS] == ELFFile.ELFCLASS64:
+            self.bits = 64
         else:
-            raise NotELFFileError("Must specify unknown, 32 or 64 bit size.")
+            # Not 32-bit or 64.. lets assert
+            raise NotELFFileError("ELF but not 32 or 64 bit.")
         self.my_assert(self.data[ELFFile.EI_VERSION], ELFFile.EV_CURRENT)
 
-        self.sex = self.data[ELFFile.EI_DATA]
-        if self.sex == ELFFile.ELFDATANONE:
-            raise NotELFFileError("self.sex == ELFDATANONE")
-        elif self.sex == ELFFile.ELFDATA2LSB:
-            self.sex = "<"
-        elif self.sex == ELFFile.ELFDATA2MSB:
-            self.sex = ">"
-        else:
-            raise NotELFFileError("Unknown self.sex")
+        self.endian = self.data[ELFFile.EI_DATA]
+        if self.endian not in (ELFFile.EI_DATA_LSB, ELFFile.EI_DATA_MSB):
+            raise NotELFFileError("Unexpected EI_DATA %x" % self.endian)
 
     def osAbi(self):
         return self.data[ELFFile.EI_OSABI]
@@ -90,16 +85,20 @@ class ELFFile:
         return self.bits
 
     def isLittleEndian(self):
-        return self.sex == "<"
+        return self.endian == ELFFile.EI_DATA_LSB
 
     def isBigEndian(self):
-        return self.sex == ">"
+        return self.endian == ELFFile.EI_DATA_MSB
+
+    def getStructEndian(self):
+        return {ELFFile.EI_DATA_LSB: "<",
+                ELFFile.EI_DATA_MSB: ">"}[self.endian]
 
     def getShort(self, offset):
-        return struct.unpack_from(self.sex+"H", self.data, offset)[0]
+        return struct.unpack_from(self.getStructEndian() + "H", self.data, offset)[0]
 
     def getWord(self, offset):
-        return struct.unpack_from(self.sex+"i", self.data, offset)[0]
+        return struct.unpack_from(self.getStructEndian() + "i", self.data, offset)[0]
 
     def isDynamic(self):
         """
@@ -118,7 +117,7 @@ class ELFFile:
 
     def machine(self):
         """
-        We know the sex stored in self.sex and we
+        We know the endian stored in self.endian and we
         know the position
         """
         return self.getShort(ELFFile.E_MACHINE)
@@ -166,6 +165,7 @@ def elf_machine_to_string(machine):
 
 if __name__ == "__main__":
     import sys
-    elf = ELFFile(sys.argv[1])
-    elf.open()
-    print(elf.isDynamic())
+
+    with ELFFile(sys.argv[1]) as elf:
+        elf.open()
+        print(elf.isDynamic())
diff --git a/import-layers/yocto-poky/meta/lib/oe/rootfs.py b/import-layers/yocto-poky/meta/lib/oe/rootfs.py
index a348b975c..f96788399 100644
--- a/import-layers/yocto-poky/meta/lib/oe/rootfs.py
+++ b/import-layers/yocto-poky/meta/lib/oe/rootfs.py
@@ -186,10 +186,6 @@ class Rootfs(object, metaclass=ABCMeta):
 
         shutil.copytree(postinst_intercepts_dir, intercepts_dir)
 
-        shutil.copy(self.d.expand("${COREBASE}/meta/files/deploydir_readme.txt"),
-                    self.deploydir +
-                    "/README_-_DO_NOT_DELETE_FILES_IN_THIS_DIRECTORY.txt")
-
         execute_pre_post_process(self.d, pre_process_cmds)
 
         if self.progress_reporter:
@@ -477,8 +473,6 @@ class RpmRootfs(Rootfs):
 
         execute_pre_post_process(self.d, rpm_post_process_cmds)
 
-        self._log_check()
-
         if self.inc_rpm_image_gen == "1":
             self.pm.backup_packaging_data()
 
@@ -951,7 +945,9 @@ class OpkgRootfs(DpkgOpkgRootfs):
         if self.progress_reporter:
             self.progress_reporter.next_stage()
 
-        self._setup_dbg_rootfs(['/etc', '/var/lib/opkg', '/usr/lib/ssl'])
+        opkg_lib_dir = self.d.getVar('OPKGLIBDIR', True)
+        opkg_dir = os.path.join(opkg_lib_dir, 'opkg')
+        self._setup_dbg_rootfs(['/etc', opkg_dir, '/usr/lib/ssl'])
 
         execute_pre_post_process(self.d, opkg_post_process_cmds)
 
diff --git a/import-layers/yocto-poky/meta/lib/oe/terminal.py b/import-layers/yocto-poky/meta/lib/oe/terminal.py
index 3901ad3f2..3c8ef59a4 100644
--- a/import-layers/yocto-poky/meta/lib/oe/terminal.py
+++ b/import-layers/yocto-poky/meta/lib/oe/terminal.py
@@ -227,6 +227,8 @@ def spawn(name, sh_cmd, title=None, env=None, d=None):
 
     pipe = terminal(sh_cmd, title, env, d)
     output = pipe.communicate()[0]
+    if output:
+        output = output.decode("utf-8")
     if pipe.returncode != 0:
         raise ExecutionError(sh_cmd, pipe.returncode, output)
 
diff --git a/import-layers/yocto-poky/meta/lib/oe/utils.py b/import-layers/yocto-poky/meta/lib/oe/utils.py
index d6545b197..36cf74f29 100644
--- a/import-layers/yocto-poky/meta/lib/oe/utils.py
+++ b/import-layers/yocto-poky/meta/lib/oe/utils.py
@@ -230,6 +230,25 @@ def format_pkg_list(pkg_dict, ret_format=None):
 
     return '\n'.join(output)
 
+def host_gcc_version(d):
+    import re, subprocess
+
+    compiler = d.getVar("BUILD_CC", True)
+
+    try:
+        env = os.environ.copy()
+        env["PATH"] = d.getVar("PATH", True)
+        output = subprocess.check_output("%s --version" % compiler, shell=True, env=env).decode("utf-8")
+    except subprocess.CalledProcessError as e:
+        bb.fatal("Error running %s --version: %s" % (compiler, e.output.decode("utf-8")))
+
+    match = re.match(".* (\d\.\d)\.\d.*", output.split('\n')[0])
+    if not match:
+        bb.fatal("Can't get compiler version from %s --version output" % compiler)
+
+    version = match.group(1)
+    return "-%s" % version if version in ("4.8", "4.9") else ""
+
 #
 # Python 2.7 doesn't have threaded pools (just multiprocessing)
 # so implement a version here