Yocto 2.5

Move OpenBMC to Yocto 2.5(sumo)

Signed-off-by: Brad Bishop <bradleyb@fuzziesquirrel.com>
Change-Id: I5c5ad6904a16e14c1c397f0baf10c9d465594a78

1 files changed, 194 insertions, 31 deletions

diff --git a/import-layers/yocto-poky/scripts/lib/devtool/upgrade.py b/import-layers/yocto-poky/scripts/lib/devtool/upgrade.py
index f1b3ff0a9..c3fd866ef 100644
--- a/import-layers/yocto-poky/scripts/lib/devtool/upgrade.py
+++ b/import-layers/yocto-poky/scripts/lib/devtool/upgrade.py
@@ -33,7 +33,7 @@ sys.path = sys.path + [devtool_path]
 
 import oe.recipeutils
 from devtool import standard
-from devtool import exec_build_env_command, setup_tinfoil, DevtoolError, parse_recipe, use_external_build, update_unlockedsigs
+from devtool import exec_build_env_command, setup_tinfoil, DevtoolError, parse_recipe, use_external_build, update_unlockedsigs, check_prerelease_version
 
 logger = logging.getLogger('devtool')
 
@@ -55,17 +55,6 @@ def _copy_source_code(orig, dest):
         dest_path = os.path.join(dest, path)
         shutil.move(os.path.join(orig, path), dest_path)
 
-def _get_checksums(rf):
-    import re
-    checksums = {}
-    with open(rf) as f:
-        for line in f:
-            for cs in ['md5sum', 'sha256sum']:
-                m = re.match("^SRC_URI\[%s\].*=.*\"(.*)\"" % cs, line)
-                if m:
-                    checksums[cs] = m.group(1)
-    return checksums
-
 def _remove_patch_dirs(recipefolder):
     for root, dirs, files in os.walk(recipefolder):
         for d in dirs:
@@ -202,6 +191,8 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee
         __run('git tag -f devtool-base-new')
         md5 = None
         sha256 = None
+        _, _, _, _, _, params = bb.fetch2.decodeurl(uri)
+        srcsubdir_rel = params.get('destsuffix', 'git')
         if not srcbranch:
             check_branch, check_branch_err = __run('git branch -r --contains %s' % srcrev)
             get_branch = [x.strip() for x in check_branch.splitlines()]
@@ -236,6 +227,7 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee
 
         tmpsrctree = _get_srctree(tmpdir)
         srctree = os.path.abspath(srctree)
+        srcsubdir_rel = os.path.relpath(tmpsrctree, tmpdir)
 
         # Delete all sources so we ensure no stray files are left over
         for item in os.listdir(srctree):
@@ -251,8 +243,15 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee
         _copy_source_code(tmpsrctree, srctree)
 
         (stdout,_) = __run('git ls-files --modified --others --exclude-standard')
-        for f in stdout.splitlines():
-            __run('git add "%s"' % f)
+        filelist = stdout.splitlines()
+        pbar = bb.ui.knotty.BBProgress('Adding changed files', len(filelist))
+        pbar.start()
+        batchsize = 100
+        for i in range(0, len(filelist), batchsize):
+            batch = filelist[i:i+batchsize]
+            __run('git add -A %s' % ' '.join(['"%s"' % item for item in batch]))
+            pbar.update(i)
+        pbar.finish()
 
         useroptions = []
         oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=rd)
@@ -264,10 +263,8 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee
 
     if no_patch:
         patches = oe.recipeutils.get_recipe_patches(crd)
-        if len(patches):
-            logger.warn('By user choice, the following patches will NOT be applied')
-            for patch in patches:
-                logger.warn("%s" % os.path.basename(patch))
+        if patches:
+            logger.warn('By user choice, the following patches will NOT be applied to the new source tree:\n  %s' % '\n  '.join([os.path.basename(patch) for patch in patches]))
     else:
         __run('git checkout devtool-patched -b %s' % branch)
         skiptag = False
@@ -292,9 +289,28 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee
         else:
             shutil.rmtree(tmpsrctree)
 
-    return (rev, md5, sha256, srcbranch)
+    return (rev, md5, sha256, srcbranch, srcsubdir_rel)
 
-def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, workspace, tinfoil, rd):
+def _add_license_diff_to_recipe(path, diff):
+    notice_text = """# FIXME: the LIC_FILES_CHKSUM values have been updated by 'devtool upgrade'.
+# The following is the difference between the old and the new license text.
+# Please update the LICENSE value if needed, and summarize the changes in
+# the commit message via 'License-Update:' tag.
+# (example: 'License-Update: copyright years updated.')
+#
+# The changes:
+#
+"""
+    commented_diff = "\n".join(["# {}".format(l) for l in diff.split('\n')])
+    with open(path, 'rb') as f:
+        orig_content = f.read()
+    with open(path, 'wb') as f:
+        f.write(notice_text.encode())
+        f.write(commented_diff.encode())
+        f.write("\n#\n\n".encode())
+        f.write(orig_content)
+
+def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, srcsubdir_old, srcsubdir_new, workspace, tinfoil, rd, license_diff, new_licenses):
     """Creates the new recipe under workspace"""
 
     bpn = rd.getVar('BPN')
@@ -346,9 +362,67 @@ def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, workspace, tinfoil
 
     newvalues['PR'] = None
 
+    # Work out which SRC_URI entries have changed in case the entry uses a name
+    crd = rd.createCopy()
+    crd.setVar('PV', newpv)
+    for var, value in newvalues.items():
+        crd.setVar(var, value)
+    old_src_uri = (rd.getVar('SRC_URI') or '').split()
+    new_src_uri = (crd.getVar('SRC_URI') or '').split()
+    newnames = []
+    addnames = []
+    for newentry in new_src_uri:
+        _, _, _, _, _, params = bb.fetch2.decodeurl(newentry)
+        if 'name' in params:
+            newnames.append(params['name'])
+            if newentry not in old_src_uri:
+                addnames.append(params['name'])
+    # Find what's been set in the original recipe
+    oldnames = []
+    noname = False
+    for varflag in rd.getVarFlags('SRC_URI'):
+        if varflag.endswith(('.md5sum', '.sha256sum')):
+            name = varflag.rsplit('.', 1)[0]
+            if name not in oldnames:
+                oldnames.append(name)
+        elif varflag in ['md5sum', 'sha256sum']:
+            noname = True
+    # Even if SRC_URI has named entries it doesn't have to actually use the name
+    if noname and addnames and addnames[0] not in oldnames:
+        addnames = []
+    # Drop any old names (the name actually might include ${PV})
+    for name in oldnames:
+        if name not in newnames:
+            newvalues['SRC_URI[%s.md5sum]' % name] = None
+            newvalues['SRC_URI[%s.sha256sum]' % name] = None
+
     if md5 and sha256:
-        newvalues['SRC_URI[md5sum]'] = md5
-        newvalues['SRC_URI[sha256sum]'] = sha256
+        if addnames:
+            nameprefix = '%s.' % addnames[0]
+        else:
+            nameprefix = ''
+        newvalues['SRC_URI[%smd5sum]' % nameprefix] = md5
+        newvalues['SRC_URI[%ssha256sum]' % nameprefix] = sha256
+
+    if srcsubdir_new != srcsubdir_old:
+        s_subdir_old = os.path.relpath(os.path.abspath(rd.getVar('S')), rd.getVar('WORKDIR'))
+        s_subdir_new = os.path.relpath(os.path.abspath(crd.getVar('S')), crd.getVar('WORKDIR'))
+        if srcsubdir_old == s_subdir_old and srcsubdir_new != s_subdir_new:
+            # Subdir for old extracted source matches what S points to (it should!)
+            # but subdir for new extracted source doesn't match what S will be
+            newvalues['S'] = '${WORKDIR}/%s' % srcsubdir_new.replace(newpv, '${PV}')
+            if crd.expand(newvalues['S']) == crd.expand('${WORKDIR}/${BP}'):
+                # It's the default, drop it
+                # FIXME what if S is being set in a .inc?
+                newvalues['S'] = None
+                logger.info('Source subdirectory has changed, dropping S value since it now matches the default ("${WORKDIR}/${BP}")')
+            else:
+                logger.info('Source subdirectory has changed, updating S value')
+
+    if license_diff:
+        newlicchksum = " ".join(["file://{};md5={}".format(l["path"], l["actual_md5"]) + (";beginline={}".format(l["beginline"]) if l["beginline"] else "") + (";endline={}".format(l["endline"]) if l["endline"] else "") for l in new_licenses])
+        newvalues["LIC_FILES_CHKSUM"] = newlicchksum
+        _add_license_diff_to_recipe(fullpath, license_diff)
 
     rd = tinfoil.parse_recipe_file(fullpath, False)
     oe.recipeutils.patch_recipe(rd, fullpath, newvalues)
@@ -377,14 +451,53 @@ def _check_git_config():
     if configerr:
         raise DevtoolError('Your git configuration is incomplete which will prevent rebases from working:\n' + '\n'.join(configerr))
 
+def _extract_licenses(srcpath, recipe_licenses):
+    licenses = []
+    for url in recipe_licenses.split():
+        license = {}
+        (type, host, path, user, pswd, parm) = bb.fetch.decodeurl(url)
+        license['path'] = path
+        license['md5'] = parm.get('md5', '')
+        license['beginline'], license['endline'] = 0, 0
+        if 'beginline' in parm:
+            license['beginline'] = int(parm['beginline'])
+        if 'endline' in parm:
+            license['endline'] = int(parm['endline'])
+        license['text'] = []
+        with open(os.path.join(srcpath, path), 'rb') as f:
+            import hashlib
+            actual_md5 = hashlib.md5()
+            lineno = 0
+            for line in f:
+                lineno += 1
+                if (lineno >= license['beginline']) and ((lineno <= license['endline']) or not license['endline']):
+                    license['text'].append(line.decode(errors='ignore'))
+                    actual_md5.update(line)
+        license['actual_md5'] = actual_md5.hexdigest()
+        licenses.append(license)
+    return licenses
+
+def _generate_license_diff(old_licenses, new_licenses):
+    need_diff = False
+    for l in new_licenses:
+        if l['md5'] != l['actual_md5']:
+            need_diff = True
+            break
+    if need_diff == False:
+        return None
+
+    import difflib
+    diff = ''
+    for old, new in zip(old_licenses, new_licenses):
+        for line in difflib.unified_diff(old['text'], new['text'], old['path'], new['path']):
+            diff = diff + line
+    return diff
 
 def upgrade(args, config, basepath, workspace):
     """Entry point for the devtool 'upgrade' subcommand"""
 
     if args.recipename in workspace:
         raise DevtoolError("recipe %s is already in your workspace" % args.recipename)
-    if not args.version and not args.srcrev:
-        raise DevtoolError("You must provide a version using the --version/-V option, or for recipes that fetch from an SCM such as git, the --srcrev/-S option")
     if args.srcbranch and not args.srcrev:
         raise DevtoolError("If you specify --srcbranch/-B then you must use --srcrev/-S to specify the revision" % args.recipename)
 
@@ -407,22 +520,43 @@ def upgrade(args, config, basepath, workspace):
         else:
             srctree = standard.get_default_srctree(config, pn)
 
+        # try to automatically discover latest version and revision if not provided on command line
+        if not args.version and not args.srcrev:
+            version_info = oe.recipeutils.get_recipe_upstream_version(rd)
+            if version_info['version'] and not version_info['version'].endswith("new-commits-available"):
+                args.version = version_info['version']
+            if version_info['revision']:
+                args.srcrev = version_info['revision']
+        if not args.version and not args.srcrev:
+            raise DevtoolError("Automatic discovery of latest version/revision failed - you must provide a version using the --version/-V option, or for recipes that fetch from an SCM such as git, the --srcrev/-S option.")
+
         standard._check_compatible_recipe(pn, rd)
         old_srcrev = rd.getVar('SRCREV')
         if old_srcrev == 'INVALID':
             old_srcrev = None
         if old_srcrev and not args.srcrev:
             raise DevtoolError("Recipe specifies a SRCREV value; you must specify a new one when upgrading")
-        if rd.getVar('PV') == args.version and old_srcrev == args.srcrev:
+        old_ver = rd.getVar('PV')
+        if old_ver == args.version and old_srcrev == args.srcrev:
             raise DevtoolError("Current and upgrade versions are the same version")
+        if args.version:
+            if bb.utils.vercmp_string(args.version, old_ver) < 0:
+                logger.warning('Upgrade version %s compares as less than the current version %s. If you are using a package feed for on-target upgrades or providing this recipe for general consumption, then you should increment PE in the recipe (or if there is no current PE value set, set it to "1")' % (args.version, old_ver))
+            check_prerelease_version(args.version, 'devtool upgrade')
 
         rf = None
+        license_diff = None
         try:
-            rev1 = standard._extract_source(srctree, False, 'devtool-orig', False, config, basepath, workspace, args.fixed_setup, rd, tinfoil)
-            rev2, md5, sha256, srcbranch = _extract_new_source(args.version, srctree, args.no_patch,
+            logger.info('Extracting current version source...')
+            rev1, srcsubdir1 = standard._extract_source(srctree, False, 'devtool-orig', False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides)
+            old_licenses = _extract_licenses(srctree, rd.getVar('LIC_FILES_CHKSUM'))
+            logger.info('Extracting upgraded version source...')
+            rev2, md5, sha256, srcbranch, srcsubdir2 = _extract_new_source(args.version, srctree, args.no_patch,
                                                     args.srcrev, args.srcbranch, args.branch, args.keep_temp,
                                                     tinfoil, rd)
-            rf, copied = _create_new_recipe(args.version, md5, sha256, args.srcrev, srcbranch, config.workspace_path, tinfoil, rd)
+            new_licenses = _extract_licenses(srctree, rd.getVar('LIC_FILES_CHKSUM'))
+            license_diff = _generate_license_diff(old_licenses, new_licenses)
+            rf, copied = _create_new_recipe(args.version, md5, sha256, args.srcrev, srcbranch, srcsubdir1, srcsubdir2, config.workspace_path, tinfoil, rd, license_diff, new_licenses)
         except bb.process.CmdError as e:
             _upgrade_error(e, rf, srctree)
         except DevtoolError as e:
@@ -433,10 +567,32 @@ def upgrade(args, config, basepath, workspace):
                         copied, config.workspace_path, rd)
         standard._add_md5(config, pn, af)
 
-        update_unlockedsigs(basepath, workspace, [pn], args.fixed_setup)
+        update_unlockedsigs(basepath, workspace, args.fixed_setup, [pn])
 
         logger.info('Upgraded source extracted to %s' % srctree)
         logger.info('New recipe is %s' % rf)
+        if license_diff:
+            logger.info('License checksums have been updated in the new recipe; please refer to it for the difference between the old and the new license texts.')
+    finally:
+        tinfoil.shutdown()
+    return 0
+
+def latest_version(args, config, basepath, workspace):
+    """Entry point for the devtool 'latest_version' subcommand"""
+    tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
+    try:
+        rd = parse_recipe(config, tinfoil, args.recipename, True)
+        if not rd:
+            return 1
+        version_info = oe.recipeutils.get_recipe_upstream_version(rd)
+        # "new-commits-available" is an indication that upstream never issues version tags
+        if not version_info['version'].endswith("new-commits-available"):
+            logger.info("Current version: {}".format(version_info['current_version']))
+            logger.info("Latest version: {}".format(version_info['version']))
+            if version_info['revision']:
+                logger.info("Latest version's commit: {}".format(version_info['revision']))
+        else:
+            logger.info("Latest commit: {}".format(version_info['revision']))
     finally:
         tinfoil.shutdown()
     return 0
@@ -451,13 +607,20 @@ def register_commands(subparsers, context):
                                            group='starting')
     parser_upgrade.add_argument('recipename', help='Name of recipe to upgrade (just name - no version, path or extension)')
     parser_upgrade.add_argument('srctree',  nargs='?', help='Path to where to extract the source tree. If not specified, a subdirectory of %s will be used.' % defsrctree)
-    parser_upgrade.add_argument('--version', '-V', help='Version to upgrade to (PV)')
-    parser_upgrade.add_argument('--srcrev', '-S', help='Source revision to upgrade to (required if fetching from an SCM such as git)')
+    parser_upgrade.add_argument('--version', '-V', help='Version to upgrade to (PV). If omitted, latest upstream version will be determined and used, if possible.')
+    parser_upgrade.add_argument('--srcrev', '-S', help='Source revision to upgrade to (useful when fetching from an SCM such as git)')
     parser_upgrade.add_argument('--srcbranch', '-B', help='Branch in source repository containing the revision to use (if fetching from an SCM such as git)')
     parser_upgrade.add_argument('--branch', '-b', default="devtool", help='Name for new development branch to checkout (default "%(default)s")')
     parser_upgrade.add_argument('--no-patch', action="store_true", help='Do not apply patches from the recipe to the new source code')
+    parser_upgrade.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations')
     group = parser_upgrade.add_mutually_exclusive_group()
     group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
     group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true")
     parser_upgrade.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
     parser_upgrade.set_defaults(func=upgrade, fixed_setup=context.fixed_setup)
+
+    parser_latest_version = subparsers.add_parser('latest-version', help='Report the latest version of an existing recipe',
+                                                  description='Queries the upstream server for what the latest upstream release is (for git, tags are checked, for tarballs, a list of them is obtained, and one with the highest version number is reported)',
+                                                  group='info')
+    parser_latest_version.add_argument('recipename', help='Name of recipe to query (just name - no version, path or extension)')
+    parser_latest_version.set_defaults(func=latest_version)