yocto-poky: Move to import-layers subdir

We are going to import additional layers, so create a subdir to
hold all of the layers that we import with git-subtree.

Change-Id: I6f732153a22be8ca663035c518837e3cc5ec0799
Signed-off-by: Patrick Williams <patrick@stwcx.xyz>

5 files changed, 523 insertions, 0 deletions

diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/__init__.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/__init__.py
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/__init__.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/__init__.py
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py
new file mode 100644
index 000000000..5e70437b2
--- /dev/null
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py
@@ -0,0 +1,161 @@
+from django.core.management.base import NoArgsCommand, CommandError
+from django.db import transaction
+from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException
+from bldcontrol.models import BuildRequest, BuildEnvironment, BRError
+from orm.models import ToasterSetting, Build
+import os
+import traceback
+
+def DN(path):
+    if path is None:
+        return ""
+    else:
+        return os.path.dirname(path)
+
+
+class Command(NoArgsCommand):
+    args = ""
+    help = "Verifies that the configured settings are valid and usable, or prompts the user to fix the settings."
+
+    def __init__(self, *args, **kwargs):
+        super(Command, self).__init__(*args, **kwargs)
+        self.guesspath = DN(DN(DN(DN(DN(DN(DN(__file__)))))))
+
+    def _find_first_path_for_file(self, startdirectory, filename, level=0):
+        if level < 0:
+            return None
+        dirs = []
+        for i in os.listdir(startdirectory):
+            j = os.path.join(startdirectory, i)
+            if os.path.isfile(j):
+                if i == filename:
+                    return startdirectory
+            elif os.path.isdir(j):
+                dirs.append(j)
+        for j in dirs:
+            ret = self._find_first_path_for_file(j, filename, level - 1)
+            if ret is not None:
+                return ret
+        return None
+
+    def _recursive_list_directories(self, startdirectory, level=0):
+        if level < 0:
+            return []
+        dirs = []
+        try:
+            for i in os.listdir(startdirectory):
+                j = os.path.join(startdirectory, i)
+                if os.path.isdir(j):
+                    dirs.append(j)
+        except OSError:
+            pass
+        for j in dirs:
+            dirs = dirs + self._recursive_list_directories(j, level - 1)
+        return dirs
+
+
+    def _verify_build_environment(self):
+        # provide a local build env. This will be extended later to include non local
+        if BuildEnvironment.objects.count() == 0:
+            BuildEnvironment.objects.create(betype=BuildEnvironment.TYPE_LOCAL)
+
+        # we make sure we have builddir and sourcedir for all defined build envionments
+        for be in BuildEnvironment.objects.all():
+            be.needs_import = False
+            def _verify_be():
+                is_changed = False
+
+                def _update_sourcedir():
+                    be.sourcedir = os.environ.get('TOASTER_DIR')
+                    return True
+
+                if len(be.sourcedir) == 0:
+                    print "\n -- Validation: The layers checkout directory must be set."
+                    is_changed = _update_sourcedir()
+
+                if not be.sourcedir.startswith("/"):
+                    print "\n -- Validation: The layers checkout directory must be set to an absolute path."
+                    is_changed = _update_sourcedir()
+
+                if is_changed:
+                    if be.betype == BuildEnvironment.TYPE_LOCAL:
+                        be.needs_import = True
+                    return True
+
+                def _update_builddir():
+                    be.builddir = os.environ.get('TOASTER_DIR')+"/build"
+                    return True
+
+                if len(be.builddir) == 0:
+                    print "\n -- Validation: The build directory must be set."
+                    is_changed = _update_builddir()
+
+                if not be.builddir.startswith("/"):
+                    print "\n -- Validation: The build directory must to be set to an absolute path."
+                    is_changed = _update_builddir()
+
+
+                if is_changed:
+                    print "\nBuild configuration saved"
+                    be.save()
+                    return True
+
+
+                if be.needs_import:
+                    try:
+                        config_file = os.environ.get('TOASTER_CONF')
+                        print "\nImporting file: %s" % config_file
+                        from loadconf import Command as LoadConfigCommand
+
+                        LoadConfigCommand()._import_layer_config(config_file)
+                        # we run lsupdates after config update
+                        print "\nLayer configuration imported. Updating information from the layer sources, please wait.\nYou can re-update any time later by running bitbake/lib/toaster/manage.py lsupdates"
+                        from django.core.management import call_command
+                        call_command("lsupdates")
+
+                        # we don't look for any other config files
+                        return is_changed
+                    except Exception as e:
+                        print "Failure while trying to import the toaster config file %s: %s" %\
+                            (config_file, e)
+                        traceback.print_exc(e)
+
+                return is_changed
+
+            while _verify_be():
+                pass
+        return 0
+
+    def _verify_default_settings(self):
+        # verify that default settings are there
+        if ToasterSetting.objects.filter(name='DEFAULT_RELEASE').count() != 1:
+            ToasterSetting.objects.filter(name='DEFAULT_RELEASE').delete()
+            ToasterSetting.objects.get_or_create(name='DEFAULT_RELEASE', value='')
+        return 0
+
+    def _verify_builds_in_progress(self):
+        # we are just starting up. we must not have any builds in progress, or build environments taken
+        for b in BuildRequest.objects.filter(state=BuildRequest.REQ_INPROGRESS):
+            BRError.objects.create(req=b, errtype="toaster",
+                                   errmsg=
+                                   "Toaster found this build IN PROGRESS while Toaster started up. This is an inconsistent state, and the build was marked as failed")
+
+        BuildRequest.objects.filter(state=BuildRequest.REQ_INPROGRESS).update(state=BuildRequest.REQ_FAILED)
+
+        BuildEnvironment.objects.update(lock=BuildEnvironment.LOCK_FREE)
+
+        # also mark "In Progress builds as failures"
+        from django.utils import timezone
+        Build.objects.filter(outcome=Build.IN_PROGRESS).update(outcome=Build.FAILED, completed_on=timezone.now())
+
+        return 0
+
+
+
+    def handle_noargs(self, **options):
+        retval = 0
+        retval += self._verify_build_environment()
+        retval += self._verify_default_settings()
+        retval += self._verify_builds_in_progress()
+
+        return retval
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py
new file mode 100644
index 000000000..5022b5940
--- /dev/null
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py
@@ -0,0 +1,183 @@
+from django.core.management.base import BaseCommand, CommandError
+from orm.models import LayerSource, ToasterSetting, Branch, Layer, Layer_Version
+from orm.models import BitbakeVersion, Release, ReleaseDefaultLayer, ReleaseLayerSourcePriority
+from django.db import IntegrityError
+import os
+
+from checksettings import DN
+
+import logging
+logger = logging.getLogger("toaster")
+
+def _reduce_canon_path(path):
+    components = []
+    for c in path.split("/"):
+        if c == "..":
+            del components[-1]
+        elif c == ".":
+            pass
+        else:
+            components.append(c)
+    if len(components) < 2:
+        components.append('')
+    return "/".join(components)
+
+def _get_id_for_sourcetype(s):
+    for i in LayerSource.SOURCE_TYPE:
+        if s == i[1]:
+            return i[0]
+    raise Exception("Could not find definition for sourcetype '%s'. Valid source types are %s" % (str(s), ', '.join(map(lambda x: "'%s'" % x[1], LayerSource.SOURCE_TYPE ))))
+
+class Command(BaseCommand):
+    help = "Loads a toasterconf.json file in the database"
+    args = "filepath"
+
+
+
+    def _import_layer_config(self, filepath):
+        if not os.path.exists(filepath) or not os.path.isfile(filepath):
+            raise Exception("Failed to find toaster config file %s ." % filepath)
+
+        import json
+        data = json.loads(open(filepath, "r").read())
+
+        # verify config file validity before updating settings
+        for i in ['bitbake', 'releases', 'defaultrelease', 'config', 'layersources']:
+            assert i in data
+
+        def _read_git_url_from_local_repository(address):
+            url = None
+            # we detect the remote name at runtime
+            import subprocess
+            (remote, remote_name) = address.split(":", 1)
+            cmd = subprocess.Popen("git remote -v", shell=True, cwd = os.path.dirname(filepath), stdout=subprocess.PIPE, stderr = subprocess.PIPE)
+            (out,err) = cmd.communicate()
+            if cmd.returncode != 0:
+                logging.warning("Error while importing layer vcs_url: git error: %s" % err)
+            for line in out.split("\n"):
+                try:
+                    (name, path) = line.split("\t", 1)
+                    if name == remote_name:
+                        url = path.split(" ")[0]
+                        break
+                except ValueError:
+                    pass
+            if url == None:
+                logging.warning("Error while looking for remote \"%s\" in \"%s\"" % (remote_name, out))
+            return url
+
+
+        # import bitbake data
+        for bvi in data['bitbake']:
+            bvo, created = BitbakeVersion.objects.get_or_create(name=bvi['name'])
+            if bvi['giturl'].startswith("remote:"):
+                bvo.giturl = _read_git_url_from_local_repository(bvi['giturl'])
+                if bvo.giturl is None:
+                    logger.error("The toaster config file references the local git repo, but Toaster cannot detect it.\nYour local configuration for bitbake version %s is invalid. Make sure that the toasterconf.json file is correct." % bvi['name'])
+
+            if bvo.giturl is None:
+                bvo.giturl = bvi['giturl']
+            bvo.branch = bvi['branch']
+            bvo.dirpath = bvi['dirpath']
+            bvo.save()
+
+        # set the layer sources
+        for lsi in data['layersources']:
+            assert 'sourcetype' in lsi
+            assert 'apiurl' in lsi
+            assert 'name' in lsi
+            assert 'branches' in lsi
+
+
+            if _get_id_for_sourcetype(lsi['sourcetype']) == LayerSource.TYPE_LAYERINDEX or lsi['apiurl'].startswith("/"):
+                apiurl = lsi['apiurl']
+            else:
+                apiurl = _reduce_canon_path(os.path.join(DN(os.path.abspath(filepath)), lsi['apiurl']))
+
+            assert ((_get_id_for_sourcetype(lsi['sourcetype']) == LayerSource.TYPE_LAYERINDEX) or apiurl.startswith("/")), (lsi['sourcetype'],apiurl)
+
+            try:
+                ls, created = LayerSource.objects.get_or_create(sourcetype = _get_id_for_sourcetype(lsi['sourcetype']), apiurl = apiurl)
+                ls.name = lsi['name']
+                ls.save()
+            except IntegrityError as e:
+                logger.warning("IntegrityError %s \nWhile setting name %s for layer source %s " % (e, lsi['name'], ls))
+
+
+            layerbranches = []
+            for branchname in lsi['branches']:
+                bo, created = Branch.objects.get_or_create(layer_source = ls, name = branchname)
+                layerbranches.append(bo)
+
+            if 'layers' in lsi:
+                for layerinfo in lsi['layers']:
+                    lo, created = Layer.objects.get_or_create(layer_source = ls, name = layerinfo['name'])
+                    if layerinfo['local_path'].startswith("/"):
+                        lo.local_path = layerinfo['local_path']
+                    else:
+                        lo.local_path = _reduce_canon_path(os.path.join(ls.apiurl, layerinfo['local_path']))
+
+                    if not os.path.exists(lo.local_path):
+                        logger.error("Local layer path %s must exists. Are you trying to import a layer that does not exist ? Check your local toasterconf.json" % lo.local_path)
+
+                    if layerinfo['vcs_url'].startswith("remote:"):
+                        lo.vcs_url = _read_git_url_from_local_repository(layerinfo['vcs_url'])
+                        if lo.vcs_url is None:
+                            logger.error("The toaster config file references the local git repo, but Toaster cannot detect it.\nYour local configuration for layer %s is invalid. Make sure that the toasterconf.json file is correct." % layerinfo['name'])
+
+                    if lo.vcs_url is None:
+                        lo.vcs_url = layerinfo['vcs_url']
+
+                    if 'layer_index_url' in layerinfo:
+                        lo.layer_index_url = layerinfo['layer_index_url']
+                    lo.save()
+
+                    for branch in layerbranches:
+                        lvo, created = Layer_Version.objects.get_or_create(layer_source = ls,
+                                up_branch = branch,
+                                commit = branch.name,
+                                layer = lo)
+                        lvo.dirpath = layerinfo['dirpath']
+                        lvo.save()
+        # set releases
+        for ri in data['releases']:
+            bvo = BitbakeVersion.objects.get(name = ri['bitbake'])
+            assert bvo is not None
+
+            ro, created = Release.objects.get_or_create(name = ri['name'], bitbake_version = bvo, branch_name = ri['branch'])
+            ro.description = ri['description']
+            ro.helptext = ri['helptext']
+            ro.save()
+
+            # save layer source priority for release
+            for ls_name in ri['layersourcepriority'].keys():
+                rlspo, created = ReleaseLayerSourcePriority.objects.get_or_create(release = ro, layer_source = LayerSource.objects.get(name=ls_name))
+                rlspo.priority = ri['layersourcepriority'][ls_name]
+                rlspo.save()
+
+            for dli in ri['defaultlayers']:
+                # find layers with the same name
+                ReleaseDefaultLayer.objects.get_or_create( release = ro, layer_name = dli)
+
+        # set default release
+        if ToasterSetting.objects.filter(name = "DEFAULT_RELEASE").count() > 0:
+            ToasterSetting.objects.filter(name = "DEFAULT_RELEASE").update(value = data['defaultrelease'])
+        else:
+            ToasterSetting.objects.create(name = "DEFAULT_RELEASE", value = data['defaultrelease'])
+
+        # set default config variables
+        for configname in data['config']:
+            if ToasterSetting.objects.filter(name = "DEFCONF_" + configname).count() > 0:
+                ToasterSetting.objects.filter(name = "DEFCONF_" + configname).update(value = data['config'][configname])
+            else:
+                ToasterSetting.objects.create(name = "DEFCONF_" + configname, value = data['config'][configname])
+
+
+    def handle(self, *args, **options):
+        if len(args) == 0:
+            raise CommandError("Need a path to the toasterconf.json file")
+        filepath = args[0]
+        self._import_layer_config(filepath)
+
+
+
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py
new file mode 100644
index 000000000..27289be5f
--- /dev/null
+++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py
@@ -0,0 +1,179 @@
+from django.core.management.base import NoArgsCommand, CommandError
+from django.db import transaction
+from django.db.models import Q
+
+from bldcontrol.bbcontroller import getBuildEnvironmentController
+from bldcontrol.bbcontroller import ShellCmdException, BuildSetupException
+from bldcontrol.models import BuildRequest, BuildEnvironment
+from bldcontrol.models import BRError, BRVariable
+
+from orm.models import Build, ToasterSetting, LogMessage, Target
+
+import os
+import logging
+import time
+import sys
+import traceback
+
+logger = logging.getLogger("toaster")
+
+class Command(NoArgsCommand):
+    args    = ""
+    help    = "Schedules and executes build requests as possible."
+    "Does not return (interrupt with Ctrl-C)"
+
+
+    @transaction.atomic
+    def _selectBuildEnvironment(self):
+        bec = getBuildEnvironmentController(lock = BuildEnvironment.LOCK_FREE)
+        bec.be.lock = BuildEnvironment.LOCK_LOCK
+        bec.be.save()
+        return bec
+
+    @transaction.atomic
+    def _selectBuildRequest(self):
+        br = BuildRequest.objects.filter(state=BuildRequest.REQ_QUEUED).first()
+        return br
+
+    def schedule(self):
+        try:
+            # select the build environment and the request to build
+            br = self._selectBuildRequest()
+            if br:
+                br.state = BuildRequest.REQ_INPROGRESS
+                br.save()
+            else:
+                return
+
+            try:
+                bec = self._selectBuildEnvironment()
+            except IndexError as e:
+                # we could not find a BEC; postpone the BR
+                br.state = BuildRequest.REQ_QUEUED
+                br.save()
+                logger.debug("runbuilds: No build env")
+                return
+
+            logger.debug("runbuilds: starting build %s, environment %s" % \
+                         (str(br).decode('utf-8'), bec.be))
+
+            # let the build request know where it is being executed
+            br.environment = bec.be
+            br.save()
+
+            # this triggers an async build
+            bec.triggerBuild(br.brbitbake, br.brlayer_set.all(),
+                             br.brvariable_set.all(), br.brtarget_set.all(),
+                             "%d:%d" % (br.pk, bec.be.pk))
+
+        except Exception as e:
+            logger.error("runbuilds: Error launching build %s" % e)
+            traceback.print_exc(e)
+            if "[Errno 111] Connection refused" in str(e):
+                # Connection refused, read toaster_server.out
+                errmsg = bec.readServerLogFile()
+            else:
+                errmsg = str(e)
+
+            BRError.objects.create(req = br,
+                    errtype = str(type(e)),
+                    errmsg = errmsg,
+                    traceback = traceback.format_exc(e))
+            br.state = BuildRequest.REQ_FAILED
+            br.save()
+            bec.be.lock = BuildEnvironment.LOCK_FREE
+            bec.be.save()
+
+    def archive(self):
+        for br in BuildRequest.objects.filter(state = BuildRequest.REQ_ARCHIVE):
+            if br.build == None:
+                br.state = BuildRequest.REQ_FAILED
+            else:
+                br.state = BuildRequest.REQ_COMPLETED
+            br.save()
+
+    def cleanup(self):
+        from django.utils import timezone
+        from datetime import timedelta
+        # environments locked for more than 30 seconds
+        # they should be unlocked
+        BuildEnvironment.objects.filter(
+            Q(buildrequest__state__in=[BuildRequest.REQ_FAILED,
+                                       BuildRequest.REQ_COMPLETED,
+                                       BuildRequest.REQ_CANCELLING]) &
+            Q(lock=BuildEnvironment.LOCK_LOCK) &
+            Q(updated__lt=timezone.now() - timedelta(seconds = 30))
+        ).update(lock=BuildEnvironment.LOCK_FREE)
+
+
+        # update all Builds that were in progress and failed to start
+        for br in BuildRequest.objects.filter(
+            state=BuildRequest.REQ_FAILED,
+            build__outcome=Build.IN_PROGRESS):
+            # transpose the launch errors in ToasterExceptions
+            br.build.outcome = Build.FAILED
+            for brerror in br.brerror_set.all():
+                logger.debug("Saving error %s" % brerror)
+                LogMessage.objects.create(build=br.build,
+                                          level=LogMessage.EXCEPTION,
+                                          message=brerror.errmsg)
+            br.build.save()
+
+            # we don't have a true build object here; hence, toasterui
+            # didn't have a change to release the BE lock
+            br.environment.lock = BuildEnvironment.LOCK_FREE
+            br.environment.save()
+
+
+        # update all BuildRequests without a build created
+        for br in BuildRequest.objects.filter(build = None):
+            br.build = Build.objects.create(project=br.project,
+                                            completed_on=br.updated,
+                                            started_on=br.created)
+            br.build.outcome = Build.FAILED
+            try:
+                br.build.machine = br.brvariable_set.get(name='MACHINE').value
+            except BRVariable.DoesNotExist:
+                pass
+            br.save()
+            # transpose target information
+            for brtarget in br.brtarget_set.all():
+                Target.objects.create(build=br.build,
+                                      target=brtarget.target,
+                                      task=brtarget.task)
+            # transpose the launch errors in ToasterExceptions
+            for brerror in br.brerror_set.all():
+                LogMessage.objects.create(build=br.build,
+                                          level=LogMessage.EXCEPTION,
+                                          message=brerror.errmsg)
+
+            br.build.save()
+
+        # Make sure the LOCK is removed for builds which have been fully
+        # cancelled
+        for br in BuildRequest.objects.filter(
+            Q(build__outcome=Build.CANCELLED) &
+            Q(state=BuildRequest.REQ_CANCELLING) &
+            ~Q(environment=None)):
+            br.environment.lock = BuildEnvironment.LOCK_FREE
+            br.environment.save()
+
+
+    def handle_noargs(self, **options):
+        while True:
+            try:
+                self.cleanup()
+            except Exception as e:
+                logger.warn("runbuilds: cleanup exception %s" % str(e))
+
+            try:
+                self.archive()
+            except Exception as e:
+                logger.warn("runbuilds: archive exception %s" % str(e))
+
+            try:
+                self.schedule()
+            except Exception as e:
+                logger.warn("runbuilds: schedule exception %s" % str(e))
+
+            time.sleep(1)