diff options
Diffstat (limited to 'poky/bitbake/lib/bb/fetch2/gitsm.py')
| -rw-r--r-- | poky/bitbake/lib/bb/fetch2/gitsm.py | 263 |
1 files changed, 189 insertions, 74 deletions
diff --git a/poky/bitbake/lib/bb/fetch2/gitsm.py b/poky/bitbake/lib/bb/fetch2/gitsm.py index 0aff1008e..35729dbc0 100644 --- a/poky/bitbake/lib/bb/fetch2/gitsm.py +++ b/poky/bitbake/lib/bb/fetch2/gitsm.py @@ -31,9 +31,12 @@ NOTE: Switching a SRC_URI from "git://" to "gitsm://" requires a clean of your r import os import bb +import copy from bb.fetch2.git import Git from bb.fetch2 import runfetchcmd from bb.fetch2 import logger +from bb.fetch2 import Fetch +from bb.fetch2 import BBFetchException class GitSM(Git): def supports(self, ud, d): @@ -42,94 +45,206 @@ class GitSM(Git): """ return ud.type in ['gitsm'] - def uses_submodules(self, ud, d, wd): - for name in ud.names: - try: - runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=wd) - return True - except bb.fetch.FetchError: - pass - return False + @staticmethod + def parse_gitmodules(gitmodules): + modules = {} + module = "" + for line in gitmodules.splitlines(): + if line.startswith('[submodule'): + module = line.split('"')[1] + modules[module] = {} + elif module and line.strip().startswith('path'): + path = line.split('=')[1].strip() + modules[module]['path'] = path + elif module and line.strip().startswith('url'): + url = line.split('=')[1].strip() + modules[module]['url'] = url + return modules - def _set_relative_paths(self, repopath): - """ - Fix submodule paths to be relative instead of absolute, - so that when we move the repo it doesn't break - (In Git 1.7.10+ this is done automatically) - """ + def update_submodules(self, ud, d): submodules = [] - with open(os.path.join(repopath, '.gitmodules'), 'r') as f: - for line in f.readlines(): - if line.startswith('[submodule'): - submodules.append(line.split('"')[1]) + paths = {} + uris = {} + local_paths = {} - for module in submodules: - repo_conf = os.path.join(repopath, module, '.git') - if os.path.exists(repo_conf): - with open(repo_conf, 'r') as f: - lines = f.readlines() - newpath = '' - for i, line in enumerate(lines): - if line.startswith('gitdir:'): - oldpath = line.split(': ')[-1].rstrip() - if oldpath.startswith('/'): - newpath = '../' * (module.count('/') + 1) + '.git/modules/' + module - lines[i] = 'gitdir: %s\n' % newpath - break - if newpath: - with open(repo_conf, 'w') as f: - for line in lines: - f.write(line) - - repo_conf2 = os.path.join(repopath, '.git', 'modules', module, 'config') - if os.path.exists(repo_conf2): - with open(repo_conf2, 'r') as f: - lines = f.readlines() - newpath = '' - for i, line in enumerate(lines): - if line.lstrip().startswith('worktree = '): - oldpath = line.split(' = ')[-1].rstrip() - if oldpath.startswith('/'): - newpath = '../' * (module.count('/') + 3) + module - lines[i] = '\tworktree = %s\n' % newpath - break - if newpath: - with open(repo_conf2, 'w') as f: - for line in lines: - f.write(line) + for name in ud.names: + try: + gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=ud.clonedir) + except: + # No submodules to update + continue + + for m, md in self.parse_gitmodules(gitmodules).items(): + submodules.append(m) + paths[m] = md['path'] + uris[m] = md['url'] + if uris[m].startswith('..'): + newud = copy.copy(ud) + newud.path = os.path.realpath(os.path.join(newud.path, md['url'])) + uris[m] = Git._get_repo_url(self, newud) - def update_submodules(self, ud, d): - # We have to convert bare -> full repo, do the submodule bit, then convert back - tmpclonedir = ud.clonedir + ".tmp" - gitdir = tmpclonedir + os.sep + ".git" - bb.utils.remove(tmpclonedir, True) - os.mkdir(tmpclonedir) - os.rename(ud.clonedir, gitdir) - runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*true/bare = false/'", d) - runfetchcmd(ud.basecmd + " reset --hard", d, workdir=tmpclonedir) - runfetchcmd(ud.basecmd + " checkout -f " + ud.revisions[ud.names[0]], d, workdir=tmpclonedir) - runfetchcmd(ud.basecmd + " submodule update --init --recursive", d, workdir=tmpclonedir) - self._set_relative_paths(tmpclonedir) - runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*false/bare = true/'", d, workdir=tmpclonedir) - os.rename(gitdir, ud.clonedir,) - bb.utils.remove(tmpclonedir, True) + for module in submodules: + module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], paths[module]), d, quiet=True, workdir=ud.clonedir) + module_hash = module_hash.split()[2] + + # Build new SRC_URI + proto = uris[module].split(':', 1)[0] + url = uris[module].replace('%s:' % proto, 'gitsm:', 1) + url += ';protocol=%s' % proto + url += ";name=%s" % module + url += ";bareclone=1;nocheckout=1;nobranch=1" + + ld = d.createCopy() + # Not necessary to set SRC_URI, since we're passing the URI to + # Fetch. + #ld.setVar('SRC_URI', url) + ld.setVar('SRCREV_%s' % module, module_hash) + + # Workaround for issues with SRCPV/SRCREV_FORMAT errors + # error refer to 'multiple' repositories. Only the repository + # in the original SRC_URI actually matters... + ld.setVar('SRCPV', d.getVar('SRCPV')) + ld.setVar('SRCREV_FORMAT', module) + + newfetch = Fetch([url], ld, cache=False) + newfetch.download() + local_paths[module] = newfetch.localpath(url) + + # Correct the submodule references to the local download version... + runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]}, d, workdir=ud.clonedir) + + symlink_path = os.path.join(ud.clonedir, 'modules', paths[module]) + if not os.path.exists(symlink_path): + try: + os.makedirs(os.path.dirname(symlink_path), exist_ok=True) + except OSError: + pass + os.symlink(local_paths[module], symlink_path) + + return True + + def need_update(self, ud, d): + main_repo_needs_update = Git.need_update(self, ud, d) + + # First check that the main repository has enough history fetched. If it doesn't, then we don't + # even have the .gitmodules and gitlinks for the submodules to attempt asking whether the + # submodules' histories are recent enough. + if main_repo_needs_update: + return True + + # Now check that the submodule histories are new enough. The git-submodule command doesn't have + # any clean interface for doing this aside from just attempting the checkout (with network + # fetched disabled). + return not self.update_submodules(ud, d) def download(self, ud, d): Git.download(self, ud, d) if not ud.shallow or ud.localpath != ud.fullshallow: - submodules = self.uses_submodules(ud, d, ud.clonedir) - if submodules: - self.update_submodules(ud, d) + self.update_submodules(ud, d) + + def copy_submodules(self, submodules, ud, destdir, d): + if ud.bareclone: + repo_conf = destdir + else: + repo_conf = os.path.join(destdir, '.git') + + if submodules and not os.path.exists(os.path.join(repo_conf, 'modules')): + os.mkdir(os.path.join(repo_conf, 'modules')) + + for module, md in submodules.items(): + srcpath = os.path.join(ud.clonedir, 'modules', md['path']) + modpath = os.path.join(repo_conf, 'modules', md['path']) + + if os.path.exists(srcpath): + if os.path.exists(os.path.join(srcpath, '.git')): + srcpath = os.path.join(srcpath, '.git') + + target = modpath + if os.path.exists(modpath): + target = os.path.dirname(modpath) + + os.makedirs(os.path.dirname(target), exist_ok=True) + runfetchcmd("cp -fpLR %s %s" % (srcpath, target), d) + elif os.path.exists(modpath): + # Module already exists, likely unpacked from a shallow mirror clone + pass + else: + # This is fatal, as we do NOT want git-submodule to hit the network + raise bb.fetch2.FetchError('Submodule %s does not exist in %s or %s.' % (module, srcpath, modpath)) def clone_shallow_local(self, ud, dest, d): super(GitSM, self).clone_shallow_local(ud, dest, d) - runfetchcmd('cp -fpPRH "%s/modules" "%s/"' % (ud.clonedir, os.path.join(dest, '.git')), d) + # Copy over the submodules' fetched histories too. + repo_conf = os.path.join(dest, '.git') + + submodules = [] + for name in ud.names: + try: + gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revision), d, quiet=True, workdir=dest) + except: + # No submodules to update + continue + + submodules = self.parse_gitmodules(gitmodules) + self.copy_submodules(submodules, ud, dest, d) def unpack(self, ud, destdir, d): Git.unpack(self, ud, destdir, d) - if self.uses_submodules(ud, d, ud.destdir): - runfetchcmd(ud.basecmd + " checkout " + ud.revisions[ud.names[0]], d, workdir=ud.destdir) - runfetchcmd(ud.basecmd + " submodule update --init --recursive", d, workdir=ud.destdir) + # Copy over the submodules' fetched histories too. + if ud.bareclone: + repo_conf = ud.destdir + else: + repo_conf = os.path.join(ud.destdir, '.git') + + update_submodules = False + paths = {} + uris = {} + local_paths = {} + for name in ud.names: + try: + gitmodules = runfetchcmd("%s show HEAD:.gitmodules" % (ud.basecmd), d, quiet=True, workdir=ud.destdir) + except: + # No submodules to update + continue + + submodules = self.parse_gitmodules(gitmodules) + self.copy_submodules(submodules, ud, ud.destdir, d) + + submodules_queue = [(module, os.path.join(repo_conf, 'modules', md['path'])) for module, md in submodules.items()] + while len(submodules_queue) != 0: + module, modpath = submodules_queue.pop() + + # add submodule children recursively + try: + gitmodules = runfetchcmd("%s show HEAD:.gitmodules" % (ud.basecmd), d, quiet=True, workdir=modpath) + for m, md in self.parse_gitmodules(gitmodules).items(): + submodules_queue.append([m, os.path.join(modpath, 'modules', md['path'])]) + except: + # no children + pass + + + # There are submodules to update + update_submodules = True + + # Determine (from the submodule) the correct url to reference + try: + output = runfetchcmd("%(basecmd)s config remote.origin.url" % {'basecmd': ud.basecmd}, d, workdir=modpath) + except bb.fetch2.FetchError as e: + # No remote url defined in this submodule + continue + + local_paths[module] = output + + # Setup the local URL properly (like git submodule init or sync would do...) + runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]}, d, workdir=ud.destdir) + + # Ensure the submodule repository is NOT set to bare, since we're checking it out... + runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=modpath) + + if update_submodules: + # Run submodule update, this sets up the directories -- without touching the config + runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir) |
