diff options
Diffstat (limited to 'import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol')
16 files changed, 1477 insertions, 0 deletions
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/__init__.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/__init__.py diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/admin.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/admin.py new file mode 100644 index 000000000..fcbe5f593 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/admin.py @@ -0,0 +1,8 @@ +from django.contrib import admin +from django.contrib.admin.filters import RelatedFieldListFilter +from .models import BuildEnvironment + +class BuildEnvironmentAdmin(admin.ModelAdmin): + pass + +admin.site.register(BuildEnvironment, BuildEnvironmentAdmin) diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/bbcontroller.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/bbcontroller.py new file mode 100644 index 000000000..d09ac1787 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/bbcontroller.py @@ -0,0 +1,142 @@ +# +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# BitBake Toaster Implementation +# +# Copyright (C) 2014 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import os +import sys +import re +from django.db import transaction +from django.db.models import Q +from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake + +# load Bitbake components +path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) +sys.path.insert(0, path) +import bb.server.xmlrpc + +class BitbakeController(object): + """ This is the basic class that controlls a bitbake server. + It is outside the scope of this class on how the server is started and aquired + """ + + def __init__(self, be): + self.connection = bb.server.xmlrpc._create_server(be.bbaddress, + int(be.bbport))[0] + + def _runCommand(self, command): + result, error = self.connection.runCommand(command) + if error: + raise Exception(error) + return result + + def disconnect(self): + return self.connection.removeClient() + + def setVariable(self, name, value): + return self._runCommand(["setVariable", name, value]) + + def getVariable(self, name): + return self._runCommand(["getVariable", name]) + + def triggerEvent(self, event): + return self._runCommand(["triggerEvent", event]) + + def build(self, targets, task = None): + if task is None: + task = "build" + return self._runCommand(["buildTargets", targets, task]) + + def forceShutDown(self): + return self._runCommand(["stateForceShutdown"]) + + + +def getBuildEnvironmentController(**kwargs): + """ Gets you a BuildEnvironmentController that encapsulates a build environment, + based on the query dictionary sent in. + + This is used to retrieve, for example, the currently running BE from inside + the toaster UI, or find a new BE to start a new build in it. + + The return object MUST always be a BuildEnvironmentController. + """ + + from localhostbecontroller import LocalhostBEController + + be = BuildEnvironment.objects.filter(Q(**kwargs))[0] + if be.betype == BuildEnvironment.TYPE_LOCAL: + return LocalhostBEController(be) + else: + raise Exception("FIXME: Implement BEC for type %s" % str(be.betype)) + + +class BuildEnvironmentController(object): + """ BuildEnvironmentController (BEC) is the abstract class that defines the operations that MUST + or SHOULD be supported by a Build Environment. It is used to establish the framework, and must + not be instantiated directly by the user. + + Use the "getBuildEnvironmentController()" function to get a working BEC for your remote. + + How the BuildEnvironments are discovered is outside the scope of this class. + + You must derive this class to teach Toaster how to operate in your own infrastructure. + We provide some specific BuildEnvironmentController classes that can be used either to + directly set-up Toaster infrastructure, or as a model for your own infrastructure set: + + * Localhost controller will run the Toaster BE on the same account as the web server + (current user if you are using the the Django development web server) + on the local machine, with the "build/" directory under the "poky/" source checkout directory. + Bash is expected to be available. + + """ + def __init__(self, be): + """ Takes a BuildEnvironment object as parameter that points to the settings of the BE. + """ + self.be = be + self.connection = None + + def setLayers(self, bitbake, ls): + """ Checks-out bitbake executor and layers from git repositories. + Sets the layer variables in the config file, after validating local layer paths. + bitbake must be a single BRBitbake instance + The layer paths must be in a list of BRLayer object + + a word of attention: by convention, the first layer for any build will be poky! + """ + raise NotImplementedError("FIXME: Must override setLayers") + + def getArtifact(self, path): + """ This call returns an artifact identified by the 'path'. How 'path' is interpreted as + up to the implementing BEC. The return MUST be a REST URL where a GET will actually return + the content of the artifact, e.g. for use as a "download link" in a web UI. + """ + raise NotImplementedError("Must return the REST URL of the artifact") + + def triggerBuild(self, bitbake, layers, variables, targets): + raise NotImplementedError("Must override BE release") + +class ShellCmdException(Exception): + pass + + +class BuildSetupException(Exception): + pass + diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py new file mode 100644 index 000000000..7def1f3a1 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py @@ -0,0 +1,332 @@ +# +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# BitBake Toaster Implementation +# +# Copyright (C) 2014 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import os +import sys +import re +import shutil +from django.db import transaction +from django.db.models import Q +from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake +from orm.models import CustomImageRecipe, Layer, Layer_Version, ProjectLayer +import subprocess + +from toastermain import settings + +from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, BitbakeController + +import logging +logger = logging.getLogger("toaster") + +from pprint import pprint, pformat + +class LocalhostBEController(BuildEnvironmentController): + """ Implementation of the BuildEnvironmentController for the localhost; + this controller manages the default build directory, + the server setup and system start and stop for the localhost-type build environment + + """ + + def __init__(self, be): + super(LocalhostBEController, self).__init__(be) + self.pokydirname = None + self.islayerset = False + + def _shellcmd(self, command, cwd=None, nowait=False): + if cwd is None: + cwd = self.be.sourcedir + + logger.debug("lbc_shellcmmd: (%s) %s" % (cwd, command)) + p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if nowait: + return + (out,err) = p.communicate() + p.wait() + if p.returncode: + if len(err) == 0: + err = "command: %s \n%s" % (command, out) + else: + err = "command: %s \n%s" % (command, err) + logger.warn("localhostbecontroller: shellcmd error %s" % err) + raise ShellCmdException(err) + else: + logger.debug("localhostbecontroller: shellcmd success") + return out + + def getGitCloneDirectory(self, url, branch): + """Construct unique clone directory name out of url and branch.""" + if branch != "HEAD": + return "_toaster_clones/_%s_%s" % (re.sub('[:/@%]', '_', url), branch) + + # word of attention; this is a localhost-specific issue; only on the localhost we expect to have "HEAD" releases + # which _ALWAYS_ means the current poky checkout + from os.path import dirname as DN + local_checkout_path = DN(DN(DN(DN(DN(os.path.abspath(__file__)))))) + #logger.debug("localhostbecontroller: using HEAD checkout in %s" % local_checkout_path) + return local_checkout_path + + + def setLayers(self, bitbake, layers, targets): + """ a word of attention: by convention, the first layer for any build will be poky! """ + + assert self.be.sourcedir is not None + # set layers in the layersource + + # 1. get a list of repos with branches, and map dirpaths for each layer + gitrepos = {} + + gitrepos[(bitbake.giturl, bitbake.commit)] = [] + gitrepos[(bitbake.giturl, bitbake.commit)].append( ("bitbake", bitbake.dirpath) ) + + for layer in layers: + # We don't need to git clone the layer for the CustomImageRecipe + # as it's generated by us layer on if needed + if CustomImageRecipe.LAYER_NAME in layer.name: + continue + if not (layer.giturl, layer.commit) in gitrepos: + gitrepos[(layer.giturl, layer.commit)] = [] + gitrepos[(layer.giturl, layer.commit)].append( (layer.name, layer.dirpath) ) + + + logger.debug("localhostbecontroller, our git repos are %s" % pformat(gitrepos)) + + + # 2. Note for future use if the current source directory is a + # checked-out git repos that could match a layer's vcs_url and therefore + # be used to speed up cloning (rather than fetching it again). + + cached_layers = {} + + try: + for remotes in self._shellcmd("git remote -v", self.be.sourcedir).split("\n"): + try: + remote = remotes.split("\t")[1].split(" ")[0] + if remote not in cached_layers: + cached_layers[remote] = self.be.sourcedir + except IndexError: + pass + except ShellCmdException: + # ignore any errors in collecting git remotes this is an optional + # step + pass + + logger.info("Using pre-checked out source for layer %s", cached_layers) + + layerlist = [] + + + # 3. checkout the repositories + for giturl, commit in gitrepos.keys(): + localdirname = os.path.join(self.be.sourcedir, self.getGitCloneDirectory(giturl, commit)) + logger.debug("localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname)) + + # make sure our directory is a git repository + if os.path.exists(localdirname): + localremotes = self._shellcmd("git remote -v", localdirname) + if not giturl in localremotes: + raise BuildSetupException("Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join(localremotes.split("\n")), giturl)) + else: + if giturl in cached_layers: + logger.debug("localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname)) + self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname)) + self._shellcmd("git remote remove origin", localdirname) + self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname) + else: + logger.debug("localhostbecontroller: cloning %s in %s" % (giturl, localdirname)) + self._shellcmd('git clone "%s" "%s"' % (giturl, localdirname)) + + # branch magic name "HEAD" will inhibit checkout + if commit != "HEAD": + logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname)) + ref = commit if re.match('^[a-fA-F0-9]+$', commit) else 'origin/%s' % commit + self._shellcmd('git fetch --all && git reset --hard "%s"' % ref, localdirname) + + # take the localdirname as poky dir if we can find the oe-init-build-env + if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")): + logger.debug("localhostbecontroller: selected poky dir name %s" % localdirname) + self.pokydirname = localdirname + + # make sure we have a working bitbake + if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')): + logger.debug("localhostbecontroller: checking bitbake into the poky dirname %s " % self.pokydirname) + self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbake.commit, bitbake.giturl, os.path.join(self.pokydirname, 'bitbake'))) + + # verify our repositories + for name, dirpath in gitrepos[(giturl, commit)]: + localdirpath = os.path.join(localdirname, dirpath) + logger.debug("localhostbecontroller: localdirpath expected '%s'" % localdirpath) + if not os.path.exists(localdirpath): + raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) + + if name != "bitbake": + layerlist.append(localdirpath.rstrip("/")) + + logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist)) + + # 5. create custom layer and add custom recipes to it + layerpath = os.path.join(self.be.builddir, + CustomImageRecipe.LAYER_NAME) + for target in targets: + try: + customrecipe = CustomImageRecipe.objects.get(name=target.target, + project=bitbake.req.project) + except CustomImageRecipe.DoesNotExist: + continue # not a custom recipe, skip + + # create directory structure + for name in ("conf", "recipes"): + path = os.path.join(layerpath, name) + if not os.path.isdir(path): + os.makedirs(path) + + # create layer.oonf + config = os.path.join(layerpath, "conf", "layer.conf") + if not os.path.isfile(config): + with open(config, "w") as conf: + conf.write('BBPATH .= ":${LAYERDIR}"\nBBFILES += "${LAYERDIR}/recipes/*.bb"\n') + + # Update the Layer_Version dirpath that has our base_recipe in + # to be able to read the base recipe to then generate the + # custom recipe. + br_layer_base_recipe = layers.get( + layer_version=customrecipe.base_recipe.layer_version) + + br_layer_base_dirpath = \ + os.path.join(self.be.sourcedir, + self.getGitCloneDirectory( + br_layer_base_recipe.giturl, + br_layer_base_recipe.commit), + customrecipe.base_recipe.layer_version.dirpath + ) + + customrecipe.base_recipe.layer_version.dirpath = \ + br_layer_base_dirpath + + customrecipe.base_recipe.layer_version.save() + + # create recipe + recipe_path = \ + os.path.join(layerpath, "recipes", "%s.bb" % target.target) + with open(recipe_path, "w") as recipef: + recipef.write(customrecipe.generate_recipe_file_contents()) + + # Update the layer and recipe objects + customrecipe.layer_version.dirpath = layerpath + customrecipe.layer_version.save() + + customrecipe.file_path = recipe_path + customrecipe.save() + + # create *Layer* objects needed for build machinery to work + BRLayer.objects.get_or_create(req=target.req, + name=layer.name, + dirpath=layerpath, + giturl="file://%s" % layerpath) + if os.path.isdir(layerpath): + layerlist.append(layerpath) + + self.islayerset = True + return layerlist + + def readServerLogFile(self): + return open(os.path.join(self.be.builddir, "toaster_server.log"), "r").read() + + + def triggerBuild(self, bitbake, layers, variables, targets, brbe): + layers = self.setLayers(bitbake, layers, targets) + + # init build environment from the clone + builddir = '%s-toaster-%d' % (self.be.builddir, bitbake.req.project.id) + oe_init = os.path.join(self.pokydirname, 'oe-init-build-env') + # init build environment + self._shellcmd("bash -c 'source %s %s'" % (oe_init, builddir), + self.be.sourcedir) + + # update bblayers.conf + bblconfpath = os.path.join(builddir, "conf/bblayers.conf") + conflines = open(bblconfpath, "r").readlines() + skip = False + with open(bblconfpath, 'w') as bblayers: + for line in conflines: + if line.startswith("# line added by toaster"): + skip = True + continue + if skip: + skip = False + else: + bblayers.write(line) + + bblayers.write('# line added by toaster build control\n' + 'BBLAYERS = "%s"' % ' '.join(layers)) + + # write configuration file + confpath = os.path.join(builddir, 'conf/toaster.conf') + with open(confpath, 'w') as conf: + for var in variables: + conf.write('%s="%s"\n' % (var.name, var.value)) + conf.write('INHERIT+="toaster buildhistory"') + + # run bitbake server from the clone + bitbake = os.path.join(self.pokydirname, 'bitbake', 'bin', 'bitbake') + self._shellcmd('bash -c \"source %s %s; BITBAKE_UI="" %s --read %s ' + '--server-only -t xmlrpc -B 0.0.0.0:0\"' % (oe_init, + builddir, bitbake, confpath), self.be.sourcedir) + + # read port number from bitbake.lock + self.be.bbport = "" + bblock = os.path.join(builddir, 'bitbake.lock') + with open(bblock) as fplock: + for line in fplock: + if ":" in line: + self.be.bbport = line.split(":")[-1].strip() + logger.debug("localhostbecontroller: bitbake port %s", self.be.bbport) + break + + if not self.be.bbport: + raise BuildSetupException("localhostbecontroller: can't read bitbake port from %s" % bblock) + + self.be.bbaddress = "localhost" + self.be.bbstate = BuildEnvironment.SERVER_STARTED + self.be.lock = BuildEnvironment.LOCK_RUNNING + self.be.save() + + bbtargets = '' + for target in targets: + task = target.task + if task: + if not task.startswith('do_'): + task = 'do_' + task + task = ':%s' % task + bbtargets += '%s%s ' % (target.target, task) + + # run build with local bitbake. stop the server after the build. + log = os.path.join(builddir, 'toaster_ui.log') + local_bitbake = os.path.join(os.path.dirname(os.getenv('BBBASEDIR')), + 'bitbake') + self._shellcmd(['bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:-1" ' + '%s %s -u toasterui --token="" >>%s 2>&1;' + 'BITBAKE_UI="" BBSERVER=0.0.0.0:-1 %s -m)&\"' \ + % (brbe, local_bitbake, bbtargets, log, bitbake)], + builddir, nowait=True) + + logger.debug('localhostbecontroller: Build launched, exiting. ' + 'Follow build logs at %s' % log) diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/__init__.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/__init__.py diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/__init__.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/__init__.py diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py new file mode 100644 index 000000000..5e70437b2 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py @@ -0,0 +1,161 @@ +from django.core.management.base import NoArgsCommand, CommandError +from django.db import transaction +from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException +from bldcontrol.models import BuildRequest, BuildEnvironment, BRError +from orm.models import ToasterSetting, Build +import os +import traceback + +def DN(path): + if path is None: + return "" + else: + return os.path.dirname(path) + + +class Command(NoArgsCommand): + args = "" + help = "Verifies that the configured settings are valid and usable, or prompts the user to fix the settings." + + def __init__(self, *args, **kwargs): + super(Command, self).__init__(*args, **kwargs) + self.guesspath = DN(DN(DN(DN(DN(DN(DN(__file__))))))) + + def _find_first_path_for_file(self, startdirectory, filename, level=0): + if level < 0: + return None + dirs = [] + for i in os.listdir(startdirectory): + j = os.path.join(startdirectory, i) + if os.path.isfile(j): + if i == filename: + return startdirectory + elif os.path.isdir(j): + dirs.append(j) + for j in dirs: + ret = self._find_first_path_for_file(j, filename, level - 1) + if ret is not None: + return ret + return None + + def _recursive_list_directories(self, startdirectory, level=0): + if level < 0: + return [] + dirs = [] + try: + for i in os.listdir(startdirectory): + j = os.path.join(startdirectory, i) + if os.path.isdir(j): + dirs.append(j) + except OSError: + pass + for j in dirs: + dirs = dirs + self._recursive_list_directories(j, level - 1) + return dirs + + + def _verify_build_environment(self): + # provide a local build env. This will be extended later to include non local + if BuildEnvironment.objects.count() == 0: + BuildEnvironment.objects.create(betype=BuildEnvironment.TYPE_LOCAL) + + # we make sure we have builddir and sourcedir for all defined build envionments + for be in BuildEnvironment.objects.all(): + be.needs_import = False + def _verify_be(): + is_changed = False + + def _update_sourcedir(): + be.sourcedir = os.environ.get('TOASTER_DIR') + return True + + if len(be.sourcedir) == 0: + print "\n -- Validation: The layers checkout directory must be set." + is_changed = _update_sourcedir() + + if not be.sourcedir.startswith("/"): + print "\n -- Validation: The layers checkout directory must be set to an absolute path." + is_changed = _update_sourcedir() + + if is_changed: + if be.betype == BuildEnvironment.TYPE_LOCAL: + be.needs_import = True + return True + + def _update_builddir(): + be.builddir = os.environ.get('TOASTER_DIR')+"/build" + return True + + if len(be.builddir) == 0: + print "\n -- Validation: The build directory must be set." + is_changed = _update_builddir() + + if not be.builddir.startswith("/"): + print "\n -- Validation: The build directory must to be set to an absolute path." + is_changed = _update_builddir() + + + if is_changed: + print "\nBuild configuration saved" + be.save() + return True + + + if be.needs_import: + try: + config_file = os.environ.get('TOASTER_CONF') + print "\nImporting file: %s" % config_file + from loadconf import Command as LoadConfigCommand + + LoadConfigCommand()._import_layer_config(config_file) + # we run lsupdates after config update + print "\nLayer configuration imported. Updating information from the layer sources, please wait.\nYou can re-update any time later by running bitbake/lib/toaster/manage.py lsupdates" + from django.core.management import call_command + call_command("lsupdates") + + # we don't look for any other config files + return is_changed + except Exception as e: + print "Failure while trying to import the toaster config file %s: %s" %\ + (config_file, e) + traceback.print_exc(e) + + return is_changed + + while _verify_be(): + pass + return 0 + + def _verify_default_settings(self): + # verify that default settings are there + if ToasterSetting.objects.filter(name='DEFAULT_RELEASE').count() != 1: + ToasterSetting.objects.filter(name='DEFAULT_RELEASE').delete() + ToasterSetting.objects.get_or_create(name='DEFAULT_RELEASE', value='') + return 0 + + def _verify_builds_in_progress(self): + # we are just starting up. we must not have any builds in progress, or build environments taken + for b in BuildRequest.objects.filter(state=BuildRequest.REQ_INPROGRESS): + BRError.objects.create(req=b, errtype="toaster", + errmsg= + "Toaster found this build IN PROGRESS while Toaster started up. This is an inconsistent state, and the build was marked as failed") + + BuildRequest.objects.filter(state=BuildRequest.REQ_INPROGRESS).update(state=BuildRequest.REQ_FAILED) + + BuildEnvironment.objects.update(lock=BuildEnvironment.LOCK_FREE) + + # also mark "In Progress builds as failures" + from django.utils import timezone + Build.objects.filter(outcome=Build.IN_PROGRESS).update(outcome=Build.FAILED, completed_on=timezone.now()) + + return 0 + + + + def handle_noargs(self, **options): + retval = 0 + retval += self._verify_build_environment() + retval += self._verify_default_settings() + retval += self._verify_builds_in_progress() + + return retval diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py new file mode 100644 index 000000000..5022b5940 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py @@ -0,0 +1,183 @@ +from django.core.management.base import BaseCommand, CommandError +from orm.models import LayerSource, ToasterSetting, Branch, Layer, Layer_Version +from orm.models import BitbakeVersion, Release, ReleaseDefaultLayer, ReleaseLayerSourcePriority +from django.db import IntegrityError +import os + +from checksettings import DN + +import logging +logger = logging.getLogger("toaster") + +def _reduce_canon_path(path): + components = [] + for c in path.split("/"): + if c == "..": + del components[-1] + elif c == ".": + pass + else: + components.append(c) + if len(components) < 2: + components.append('') + return "/".join(components) + +def _get_id_for_sourcetype(s): + for i in LayerSource.SOURCE_TYPE: + if s == i[1]: + return i[0] + raise Exception("Could not find definition for sourcetype '%s'. Valid source types are %s" % (str(s), ', '.join(map(lambda x: "'%s'" % x[1], LayerSource.SOURCE_TYPE )))) + +class Command(BaseCommand): + help = "Loads a toasterconf.json file in the database" + args = "filepath" + + + + def _import_layer_config(self, filepath): + if not os.path.exists(filepath) or not os.path.isfile(filepath): + raise Exception("Failed to find toaster config file %s ." % filepath) + + import json + data = json.loads(open(filepath, "r").read()) + + # verify config file validity before updating settings + for i in ['bitbake', 'releases', 'defaultrelease', 'config', 'layersources']: + assert i in data + + def _read_git_url_from_local_repository(address): + url = None + # we detect the remote name at runtime + import subprocess + (remote, remote_name) = address.split(":", 1) + cmd = subprocess.Popen("git remote -v", shell=True, cwd = os.path.dirname(filepath), stdout=subprocess.PIPE, stderr = subprocess.PIPE) + (out,err) = cmd.communicate() + if cmd.returncode != 0: + logging.warning("Error while importing layer vcs_url: git error: %s" % err) + for line in out.split("\n"): + try: + (name, path) = line.split("\t", 1) + if name == remote_name: + url = path.split(" ")[0] + break + except ValueError: + pass + if url == None: + logging.warning("Error while looking for remote \"%s\" in \"%s\"" % (remote_name, out)) + return url + + + # import bitbake data + for bvi in data['bitbake']: + bvo, created = BitbakeVersion.objects.get_or_create(name=bvi['name']) + if bvi['giturl'].startswith("remote:"): + bvo.giturl = _read_git_url_from_local_repository(bvi['giturl']) + if bvo.giturl is None: + logger.error("The toaster config file references the local git repo, but Toaster cannot detect it.\nYour local configuration for bitbake version %s is invalid. Make sure that the toasterconf.json file is correct." % bvi['name']) + + if bvo.giturl is None: + bvo.giturl = bvi['giturl'] + bvo.branch = bvi['branch'] + bvo.dirpath = bvi['dirpath'] + bvo.save() + + # set the layer sources + for lsi in data['layersources']: + assert 'sourcetype' in lsi + assert 'apiurl' in lsi + assert 'name' in lsi + assert 'branches' in lsi + + + if _get_id_for_sourcetype(lsi['sourcetype']) == LayerSource.TYPE_LAYERINDEX or lsi['apiurl'].startswith("/"): + apiurl = lsi['apiurl'] + else: + apiurl = _reduce_canon_path(os.path.join(DN(os.path.abspath(filepath)), lsi['apiurl'])) + + assert ((_get_id_for_sourcetype(lsi['sourcetype']) == LayerSource.TYPE_LAYERINDEX) or apiurl.startswith("/")), (lsi['sourcetype'],apiurl) + + try: + ls, created = LayerSource.objects.get_or_create(sourcetype = _get_id_for_sourcetype(lsi['sourcetype']), apiurl = apiurl) + ls.name = lsi['name'] + ls.save() + except IntegrityError as e: + logger.warning("IntegrityError %s \nWhile setting name %s for layer source %s " % (e, lsi['name'], ls)) + + + layerbranches = [] + for branchname in lsi['branches']: + bo, created = Branch.objects.get_or_create(layer_source = ls, name = branchname) + layerbranches.append(bo) + + if 'layers' in lsi: + for layerinfo in lsi['layers']: + lo, created = Layer.objects.get_or_create(layer_source = ls, name = layerinfo['name']) + if layerinfo['local_path'].startswith("/"): + lo.local_path = layerinfo['local_path'] + else: + lo.local_path = _reduce_canon_path(os.path.join(ls.apiurl, layerinfo['local_path'])) + + if not os.path.exists(lo.local_path): + logger.error("Local layer path %s must exists. Are you trying to import a layer that does not exist ? Check your local toasterconf.json" % lo.local_path) + + if layerinfo['vcs_url'].startswith("remote:"): + lo.vcs_url = _read_git_url_from_local_repository(layerinfo['vcs_url']) + if lo.vcs_url is None: + logger.error("The toaster config file references the local git repo, but Toaster cannot detect it.\nYour local configuration for layer %s is invalid. Make sure that the toasterconf.json file is correct." % layerinfo['name']) + + if lo.vcs_url is None: + lo.vcs_url = layerinfo['vcs_url'] + + if 'layer_index_url' in layerinfo: + lo.layer_index_url = layerinfo['layer_index_url'] + lo.save() + + for branch in layerbranches: + lvo, created = Layer_Version.objects.get_or_create(layer_source = ls, + up_branch = branch, + commit = branch.name, + layer = lo) + lvo.dirpath = layerinfo['dirpath'] + lvo.save() + # set releases + for ri in data['releases']: + bvo = BitbakeVersion.objects.get(name = ri['bitbake']) + assert bvo is not None + + ro, created = Release.objects.get_or_create(name = ri['name'], bitbake_version = bvo, branch_name = ri['branch']) + ro.description = ri['description'] + ro.helptext = ri['helptext'] + ro.save() + + # save layer source priority for release + for ls_name in ri['layersourcepriority'].keys(): + rlspo, created = ReleaseLayerSourcePriority.objects.get_or_create(release = ro, layer_source = LayerSource.objects.get(name=ls_name)) + rlspo.priority = ri['layersourcepriority'][ls_name] + rlspo.save() + + for dli in ri['defaultlayers']: + # find layers with the same name + ReleaseDefaultLayer.objects.get_or_create( release = ro, layer_name = dli) + + # set default release + if ToasterSetting.objects.filter(name = "DEFAULT_RELEASE").count() > 0: + ToasterSetting.objects.filter(name = "DEFAULT_RELEASE").update(value = data['defaultrelease']) + else: + ToasterSetting.objects.create(name = "DEFAULT_RELEASE", value = data['defaultrelease']) + + # set default config variables + for configname in data['config']: + if ToasterSetting.objects.filter(name = "DEFCONF_" + configname).count() > 0: + ToasterSetting.objects.filter(name = "DEFCONF_" + configname).update(value = data['config'][configname]) + else: + ToasterSetting.objects.create(name = "DEFCONF_" + configname, value = data['config'][configname]) + + + def handle(self, *args, **options): + if len(args) == 0: + raise CommandError("Need a path to the toasterconf.json file") + filepath = args[0] + self._import_layer_config(filepath) + + + diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py new file mode 100644 index 000000000..27289be5f --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py @@ -0,0 +1,179 @@ +from django.core.management.base import NoArgsCommand, CommandError +from django.db import transaction +from django.db.models import Q + +from bldcontrol.bbcontroller import getBuildEnvironmentController +from bldcontrol.bbcontroller import ShellCmdException, BuildSetupException +from bldcontrol.models import BuildRequest, BuildEnvironment +from bldcontrol.models import BRError, BRVariable + +from orm.models import Build, ToasterSetting, LogMessage, Target + +import os +import logging +import time +import sys +import traceback + +logger = logging.getLogger("toaster") + +class Command(NoArgsCommand): + args = "" + help = "Schedules and executes build requests as possible." + "Does not return (interrupt with Ctrl-C)" + + + @transaction.atomic + def _selectBuildEnvironment(self): + bec = getBuildEnvironmentController(lock = BuildEnvironment.LOCK_FREE) + bec.be.lock = BuildEnvironment.LOCK_LOCK + bec.be.save() + return bec + + @transaction.atomic + def _selectBuildRequest(self): + br = BuildRequest.objects.filter(state=BuildRequest.REQ_QUEUED).first() + return br + + def schedule(self): + try: + # select the build environment and the request to build + br = self._selectBuildRequest() + if br: + br.state = BuildRequest.REQ_INPROGRESS + br.save() + else: + return + + try: + bec = self._selectBuildEnvironment() + except IndexError as e: + # we could not find a BEC; postpone the BR + br.state = BuildRequest.REQ_QUEUED + br.save() + logger.debug("runbuilds: No build env") + return + + logger.debug("runbuilds: starting build %s, environment %s" % \ + (str(br).decode('utf-8'), bec.be)) + + # let the build request know where it is being executed + br.environment = bec.be + br.save() + + # this triggers an async build + bec.triggerBuild(br.brbitbake, br.brlayer_set.all(), + br.brvariable_set.all(), br.brtarget_set.all(), + "%d:%d" % (br.pk, bec.be.pk)) + + except Exception as e: + logger.error("runbuilds: Error launching build %s" % e) + traceback.print_exc(e) + if "[Errno 111] Connection refused" in str(e): + # Connection refused, read toaster_server.out + errmsg = bec.readServerLogFile() + else: + errmsg = str(e) + + BRError.objects.create(req = br, + errtype = str(type(e)), + errmsg = errmsg, + traceback = traceback.format_exc(e)) + br.state = BuildRequest.REQ_FAILED + br.save() + bec.be.lock = BuildEnvironment.LOCK_FREE + bec.be.save() + + def archive(self): + for br in BuildRequest.objects.filter(state = BuildRequest.REQ_ARCHIVE): + if br.build == None: + br.state = BuildRequest.REQ_FAILED + else: + br.state = BuildRequest.REQ_COMPLETED + br.save() + + def cleanup(self): + from django.utils import timezone + from datetime import timedelta + # environments locked for more than 30 seconds + # they should be unlocked + BuildEnvironment.objects.filter( + Q(buildrequest__state__in=[BuildRequest.REQ_FAILED, + BuildRequest.REQ_COMPLETED, + BuildRequest.REQ_CANCELLING]) & + Q(lock=BuildEnvironment.LOCK_LOCK) & + Q(updated__lt=timezone.now() - timedelta(seconds = 30)) + ).update(lock=BuildEnvironment.LOCK_FREE) + + + # update all Builds that were in progress and failed to start + for br in BuildRequest.objects.filter( + state=BuildRequest.REQ_FAILED, + build__outcome=Build.IN_PROGRESS): + # transpose the launch errors in ToasterExceptions + br.build.outcome = Build.FAILED + for brerror in br.brerror_set.all(): + logger.debug("Saving error %s" % brerror) + LogMessage.objects.create(build=br.build, + level=LogMessage.EXCEPTION, + message=brerror.errmsg) + br.build.save() + + # we don't have a true build object here; hence, toasterui + # didn't have a change to release the BE lock + br.environment.lock = BuildEnvironment.LOCK_FREE + br.environment.save() + + + # update all BuildRequests without a build created + for br in BuildRequest.objects.filter(build = None): + br.build = Build.objects.create(project=br.project, + completed_on=br.updated, + started_on=br.created) + br.build.outcome = Build.FAILED + try: + br.build.machine = br.brvariable_set.get(name='MACHINE').value + except BRVariable.DoesNotExist: + pass + br.save() + # transpose target information + for brtarget in br.brtarget_set.all(): + Target.objects.create(build=br.build, + target=brtarget.target, + task=brtarget.task) + # transpose the launch errors in ToasterExceptions + for brerror in br.brerror_set.all(): + LogMessage.objects.create(build=br.build, + level=LogMessage.EXCEPTION, + message=brerror.errmsg) + + br.build.save() + + # Make sure the LOCK is removed for builds which have been fully + # cancelled + for br in BuildRequest.objects.filter( + Q(build__outcome=Build.CANCELLED) & + Q(state=BuildRequest.REQ_CANCELLING) & + ~Q(environment=None)): + br.environment.lock = BuildEnvironment.LOCK_FREE + br.environment.save() + + + def handle_noargs(self, **options): + while True: + try: + self.cleanup() + except Exception as e: + logger.warn("runbuilds: cleanup exception %s" % str(e)) + + try: + self.archive() + except Exception as e: + logger.warn("runbuilds: archive exception %s" % str(e)) + + try: + self.schedule() + except Exception as e: + logger.warn("runbuilds: schedule exception %s" % str(e)) + + time.sleep(1) diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/migrations/0001_initial.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/migrations/0001_initial.py new file mode 100644 index 000000000..67db37856 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/migrations/0001_initial.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('orm', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='BRBitbake', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('giturl', models.CharField(max_length=254)), + ('commit', models.CharField(max_length=254)), + ('dirpath', models.CharField(max_length=254)), + ], + ), + migrations.CreateModel( + name='BRError', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('errtype', models.CharField(max_length=100)), + ('errmsg', models.TextField()), + ('traceback', models.TextField()), + ], + ), + migrations.CreateModel( + name='BRLayer', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('name', models.CharField(max_length=100)), + ('giturl', models.CharField(max_length=254)), + ('commit', models.CharField(max_length=254)), + ('dirpath', models.CharField(max_length=254)), + ('layer_version', models.ForeignKey(to='orm.Layer_Version', null=True)), + ], + ), + migrations.CreateModel( + name='BRTarget', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('target', models.CharField(max_length=100)), + ('task', models.CharField(max_length=100, null=True)), + ], + ), + migrations.CreateModel( + name='BRVariable', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('name', models.CharField(max_length=100)), + ('value', models.TextField(blank=True)), + ], + ), + migrations.CreateModel( + name='BuildEnvironment', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('address', models.CharField(max_length=254)), + ('betype', models.IntegerField(choices=[(0, b'local'), (1, b'ssh')])), + ('bbaddress', models.CharField(max_length=254, blank=True)), + ('bbport', models.IntegerField(default=-1)), + ('bbtoken', models.CharField(max_length=126, blank=True)), + ('bbstate', models.IntegerField(default=0, choices=[(0, b'stopped'), (1, b'started')])), + ('sourcedir', models.CharField(max_length=512, blank=True)), + ('builddir', models.CharField(max_length=512, blank=True)), + ('lock', models.IntegerField(default=0, choices=[(0, b'free'), (1, b'lock'), (2, b'running')])), + ('created', models.DateTimeField(auto_now_add=True)), + ('updated', models.DateTimeField(auto_now=True)), + ], + ), + migrations.CreateModel( + name='BuildRequest', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('state', models.IntegerField(default=0, choices=[(0, b'created'), (1, b'queued'), (2, b'in progress'), (3, b'completed'), (4, b'failed'), (5, b'deleted'), (6, b'archive')])), + ('created', models.DateTimeField(auto_now_add=True)), + ('updated', models.DateTimeField(auto_now=True)), + ('build', models.OneToOneField(null=True, to='orm.Build')), + ('environment', models.ForeignKey(to='bldcontrol.BuildEnvironment', null=True)), + ('project', models.ForeignKey(to='orm.Project')), + ], + ), + migrations.AddField( + model_name='brvariable', + name='req', + field=models.ForeignKey(to='bldcontrol.BuildRequest'), + ), + migrations.AddField( + model_name='brtarget', + name='req', + field=models.ForeignKey(to='bldcontrol.BuildRequest'), + ), + migrations.AddField( + model_name='brlayer', + name='req', + field=models.ForeignKey(to='bldcontrol.BuildRequest'), + ), + migrations.AddField( + model_name='brerror', + name='req', + field=models.ForeignKey(to='bldcontrol.BuildRequest'), + ), + migrations.AddField( + model_name='brbitbake', + name='req', + field=models.OneToOneField(to='bldcontrol.BuildRequest'), + ), + ] diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/migrations/0002_auto_20160120_1250.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/migrations/0002_auto_20160120_1250.py new file mode 100644 index 000000000..0c2475aba --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/migrations/0002_auto_20160120_1250.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bldcontrol', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='buildenvironment', + name='betype', + field=models.IntegerField(choices=[(0, b'local')]), + ), + ] diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/migrations/0003_add_cancelling_state.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/migrations/0003_add_cancelling_state.py new file mode 100644 index 000000000..eec9216ca --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/migrations/0003_add_cancelling_state.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bldcontrol', '0002_auto_20160120_1250'), + ] + + operations = [ + migrations.AlterField( + model_name='buildrequest', + name='state', + field=models.IntegerField(default=0, choices=[(0, b'created'), (1, b'queued'), (2, b'in progress'), (3, b'completed'), (4, b'failed'), (5, b'deleted'), (6, b'cancelling'), (7, b'archive')]), + ), + ] diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/migrations/__init__.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/migrations/__init__.py diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/models.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/models.py new file mode 100644 index 000000000..cb49a58c4 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/models.py @@ -0,0 +1,160 @@ +from __future__ import unicode_literals +from django.db import models +from django.core.validators import MaxValueValidator, MinValueValidator +from django.utils.encoding import force_bytes +from orm.models import Project, ProjectLayer, ProjectVariable, ProjectTarget, Build, Layer_Version + +import logging +logger = logging.getLogger("toaster") +# a BuildEnvironment is the equivalent of the "build/" directory on the localhost +class BuildEnvironment(models.Model): + SERVER_STOPPED = 0 + SERVER_STARTED = 1 + SERVER_STATE = ( + (SERVER_STOPPED, "stopped"), + (SERVER_STARTED, "started"), + ) + + TYPE_LOCAL = 0 + TYPE = ( + (TYPE_LOCAL, "local"), + ) + + LOCK_FREE = 0 + LOCK_LOCK = 1 + LOCK_RUNNING = 2 + LOCK_STATE = ( + (LOCK_FREE, "free"), + (LOCK_LOCK, "lock"), + (LOCK_RUNNING, "running"), + ) + + address = models.CharField(max_length = 254) + betype = models.IntegerField(choices = TYPE) + bbaddress = models.CharField(max_length = 254, blank = True) + bbport = models.IntegerField(default = -1) + bbtoken = models.CharField(max_length = 126, blank = True) + bbstate = models.IntegerField(choices = SERVER_STATE, default = SERVER_STOPPED) + sourcedir = models.CharField(max_length = 512, blank = True) + builddir = models.CharField(max_length = 512, blank = True) + lock = models.IntegerField(choices = LOCK_STATE, default = LOCK_FREE) + created = models.DateTimeField(auto_now_add = True) + updated = models.DateTimeField(auto_now = True) + + def get_artifact(self, path): + if self.betype == BuildEnvironment.TYPE_LOCAL: + return open(path, "r") + raise NotImplementedError("FIXME: artifact download not implemented "\ + "for build environment type %s" % \ + self.get_betype_display()) + + def has_artifact(self, path): + import os + if self.betype == BuildEnvironment.TYPE_LOCAL: + return os.path.exists(path) + raise NotImplementedError("FIXME: has artifact not implemented for "\ + "build environment type %s" % \ + self.get_betype_display()) + +# a BuildRequest is a request that the scheduler will build using a BuildEnvironment +# the build request queue is the table itself, ordered by state + +class BuildRequest(models.Model): + REQ_CREATED = 0 + REQ_QUEUED = 1 + REQ_INPROGRESS = 2 + REQ_COMPLETED = 3 + REQ_FAILED = 4 + REQ_DELETED = 5 + REQ_CANCELLING = 6 + REQ_ARCHIVE = 7 + + REQUEST_STATE = ( + (REQ_CREATED, "created"), + (REQ_QUEUED, "queued"), + (REQ_INPROGRESS, "in progress"), + (REQ_COMPLETED, "completed"), + (REQ_FAILED, "failed"), + (REQ_DELETED, "deleted"), + (REQ_CANCELLING, "cancelling"), + (REQ_ARCHIVE, "archive"), + ) + + search_allowed_fields = ("brtarget__target", "build__project__name") + + project = models.ForeignKey(Project) + build = models.OneToOneField(Build, null = True) # TODO: toasterui should set this when Build is created + environment = models.ForeignKey(BuildEnvironment, null = True) + state = models.IntegerField(choices = REQUEST_STATE, default = REQ_CREATED) + created = models.DateTimeField(auto_now_add = True) + updated = models.DateTimeField(auto_now = True) + + def __init__(self, *args, **kwargs): + super(BuildRequest, self).__init__(*args, **kwargs) + # Save the old state incase it's about to be modified + self.old_state = self.state + + def save(self, *args, **kwargs): + # Check that the state we're trying to set is not going backwards + # e.g. from REQ_FAILED to REQ_INPROGRESS + if self.old_state != self.state and self.old_state > self.state: + logger.warn("Invalid state change requested: " + "Cannot go from %s to %s - ignoring request" % + (BuildRequest.REQUEST_STATE[self.old_state][1], + BuildRequest.REQUEST_STATE[self.state][1]) + ) + # Set property back to the old value + self.state = self.old_state + return + + super(BuildRequest, self).save(*args, **kwargs) + + + def get_duration(self): + return (self.updated - self.created).total_seconds() + + def get_sorted_target_list(self): + tgts = self.brtarget_set.order_by( 'target' ); + return( tgts ); + + def get_machine(self): + return self.brvariable_set.get(name="MACHINE").value + + def __str__(self): + return force_bytes('%s %s' % (self.project, self.get_state_display())) + +# These tables specify the settings for running an actual build. +# They MUST be kept in sync with the tables in orm.models.Project* + +class BRLayer(models.Model): + req = models.ForeignKey(BuildRequest) + name = models.CharField(max_length = 100) + giturl = models.CharField(max_length = 254) + commit = models.CharField(max_length = 254) + dirpath = models.CharField(max_length = 254) + layer_version = models.ForeignKey(Layer_Version, null=True) + +class BRBitbake(models.Model): + req = models.OneToOneField(BuildRequest) # only one bitbake for a request + giturl = models.CharField(max_length =254) + commit = models.CharField(max_length = 254) + dirpath = models.CharField(max_length = 254) + +class BRVariable(models.Model): + req = models.ForeignKey(BuildRequest) + name = models.CharField(max_length=100) + value = models.TextField(blank = True) + +class BRTarget(models.Model): + req = models.ForeignKey(BuildRequest) + target = models.CharField(max_length=100) + task = models.CharField(max_length=100, null=True) + +class BRError(models.Model): + req = models.ForeignKey(BuildRequest) + errtype = models.CharField(max_length=100) + errmsg = models.TextField() + traceback = models.TextField() + + def __str__(self): + return "%s (%s)" % (self.errmsg, self.req) diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/tests.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/tests.py new file mode 100644 index 000000000..f20cc7d4b --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/tests.py @@ -0,0 +1,160 @@ +""" +This file demonstrates writing tests using the unittest module. These will pass +when you run "manage.py test". + +Replace this with more appropriate tests for your application. +""" + +from django.test import TestCase + +from bldcontrol.bbcontroller import BitbakeController, BuildSetupException +from bldcontrol.localhostbecontroller import LocalhostBEController +from bldcontrol.models import BuildEnvironment, BuildRequest +from bldcontrol.management.commands.runbuilds import Command + +import socket +import subprocess +import os + +# standard poky data hardcoded for testing +BITBAKE_LAYER = type('bitbake_info', (object,), { "giturl": "git://git.yoctoproject.org/poky.git", "dirpath": "", "commit": "HEAD"}) +POKY_LAYERS = [ + type('poky_info', (object,), { "name": "meta", "giturl": "git://git.yoctoproject.org/poky.git", "dirpath": "meta", "commit": "HEAD"}), + type('poky_info', (object,), { "name": "meta-yocto", "giturl": "git://git.yoctoproject.org/poky.git", "dirpath": "meta-yocto", "commit": "HEAD"}), + type('poky_info', (object,), { "name": "meta-yocto-bsp", "giturl": "git://git.yoctoproject.org/poky.git", "dirpath": "meta-yocto-bsp", "commit": "HEAD"}), + ] + + + +# we have an abstract test class designed to ensure that the controllers use a single interface +# specific controller tests only need to override the _getBuildEnvironment() method + +test_sourcedir = os.getenv("TTS_SOURCE_DIR") +test_builddir = os.getenv("TTS_BUILD_DIR") +test_address = os.getenv("TTS_TEST_ADDRESS", "localhost") + +if test_sourcedir == None or test_builddir == None or test_address == None: + raise Exception("Please set TTTS_SOURCE_DIR, TTS_BUILD_DIR and TTS_TEST_ADDRESS") + +# The bb server will expect a toaster-pre.conf file to exist. If it doesn't exit then we make +# an empty one here. +open(test_builddir + 'conf/toaster-pre.conf', 'a').close() + +class BEControllerTests(object): + + def _serverForceStop(self, bc): + err = bc._shellcmd("netstat -tapn 2>/dev/null | grep 8200 | awk '{print $7}' | sort -fu | cut -d \"/\" -f 1 | grep -v -- - | tee /dev/fd/2 | xargs -r kill") + self.assertTrue(err == '', "bitbake server pid %s not stopped" % err) + + def test_serverStartAndStop(self): + obe = self._getBuildEnvironment() + bc = self._getBEController(obe) + try: + # setting layers, skip any layer info + bc.setLayers(BITBAKE_LAYER, POKY_LAYERS) + except NotImplementedError: + print "Test skipped due to command not implemented yet" + return True + # We are ok with the exception as we're handling the git already exists + except BuildSetupException: + pass + + bc.pokydirname = test_sourcedir + bc.islayerset = True + + hostname = test_address.split("@")[-1] + + # test start server and stop + bc.startBBServer() + + self.assertFalse(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex((hostname, int(bc.be.bbport))), "Server not answering") + + self._serverForceStop(bc) + + def test_getBBController(self): + obe = self._getBuildEnvironment() + bc = self._getBEController(obe) + layerSet = False + try: + # setting layers, skip any layer info + layerSet = bc.setLayers(BITBAKE_LAYER, POKY_LAYERS) + except NotImplementedError: + print "Test skipped due to command not implemented yet" + return True + # We are ok with the exception as we're handling the git already exists + except BuildSetupException: + pass + + bc.pokydirname = test_sourcedir + bc.islayerset = True + + bbc = bc.getBBController() + self.assertTrue(isinstance(bbc, BitbakeController)) + + self._serverForceStop(bc) + +class LocalhostBEControllerTests(TestCase, BEControllerTests): + def __init__(self, *args): + super(LocalhostBEControllerTests, self).__init__(*args) + + + def _getBuildEnvironment(self): + return BuildEnvironment.objects.create( + lock = BuildEnvironment.LOCK_FREE, + betype = BuildEnvironment.TYPE_LOCAL, + address = test_address, + sourcedir = test_sourcedir, + builddir = test_builddir ) + + def _getBEController(self, obe): + return LocalhostBEController(obe) + +class RunBuildsCommandTests(TestCase): + def test_bec_select(self): + """ + Tests that we can find and lock a build environment + """ + + obe = BuildEnvironment.objects.create(lock = BuildEnvironment.LOCK_FREE, betype = BuildEnvironment.TYPE_LOCAL) + command = Command() + bec = command._selectBuildEnvironment() + + # make sure we select the object we've just built + self.assertTrue(bec.be.id == obe.id, "Environment is not properly selected") + # we have a locked environment + self.assertTrue(bec.be.lock == BuildEnvironment.LOCK_LOCK, "Environment is not locked") + # no more selections possible here + self.assertRaises(IndexError, command._selectBuildEnvironment) + + def test_br_select(self): + from orm.models import Project, Release, BitbakeVersion, Branch + p = Project.objects.create_project("test", Release.objects.get_or_create(name = "HEAD", bitbake_version = BitbakeVersion.objects.get_or_create(name="HEAD", branch=Branch.objects.get_or_create(name="HEAD"))[0])[0]) + obr = BuildRequest.objects.create(state = BuildRequest.REQ_QUEUED, project = p) + command = Command() + br = command._selectBuildRequest() + + # make sure we select the object we've just built + self.assertTrue(obr.id == br.id, "Request is not properly selected") + # we have a locked environment + self.assertTrue(br.state == BuildRequest.REQ_INPROGRESS, "Request is not updated") + # no more selections possible here + self.assertRaises(IndexError, command._selectBuildRequest) + + +class UtilityTests(TestCase): + def test_reduce_path(self): + from bldcontrol.management.commands.loadconf import _reduce_canon_path, _get_id_for_sourcetype + + self.assertTrue( _reduce_canon_path("/") == "/") + self.assertTrue( _reduce_canon_path("/home/..") == "/") + self.assertTrue( _reduce_canon_path("/home/../ana") == "/ana") + self.assertTrue( _reduce_canon_path("/home/../ana/..") == "/") + self.assertTrue( _reduce_canon_path("/home/ana/mihai/../maria") == "/home/ana/maria") + + def test_get_id_for_sorucetype(self): + from bldcontrol.management.commands.loadconf import _reduce_canon_path, _get_id_for_sourcetype + self.assertTrue( _get_id_for_sourcetype("layerindex") == 1) + self.assertTrue( _get_id_for_sourcetype("local") == 0) + self.assertTrue( _get_id_for_sourcetype("imported") == 2) + with self.assertRaises(Exception): + _get_id_for_sourcetype("unknown") diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/views.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/views.py new file mode 100644 index 000000000..60f00ef0e --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/views.py @@ -0,0 +1 @@ +# Create your views here. |
