diff options
Diffstat (limited to 'import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management')
5 files changed, 523 insertions, 0 deletions
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/__init__.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/__init__.py diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/__init__.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/__init__.py diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py new file mode 100644 index 000000000..5e70437b2 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py @@ -0,0 +1,161 @@ +from django.core.management.base import NoArgsCommand, CommandError +from django.db import transaction +from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException +from bldcontrol.models import BuildRequest, BuildEnvironment, BRError +from orm.models import ToasterSetting, Build +import os +import traceback + +def DN(path): + if path is None: + return "" + else: + return os.path.dirname(path) + + +class Command(NoArgsCommand): + args = "" + help = "Verifies that the configured settings are valid and usable, or prompts the user to fix the settings." + + def __init__(self, *args, **kwargs): + super(Command, self).__init__(*args, **kwargs) + self.guesspath = DN(DN(DN(DN(DN(DN(DN(__file__))))))) + + def _find_first_path_for_file(self, startdirectory, filename, level=0): + if level < 0: + return None + dirs = [] + for i in os.listdir(startdirectory): + j = os.path.join(startdirectory, i) + if os.path.isfile(j): + if i == filename: + return startdirectory + elif os.path.isdir(j): + dirs.append(j) + for j in dirs: + ret = self._find_first_path_for_file(j, filename, level - 1) + if ret is not None: + return ret + return None + + def _recursive_list_directories(self, startdirectory, level=0): + if level < 0: + return [] + dirs = [] + try: + for i in os.listdir(startdirectory): + j = os.path.join(startdirectory, i) + if os.path.isdir(j): + dirs.append(j) + except OSError: + pass + for j in dirs: + dirs = dirs + self._recursive_list_directories(j, level - 1) + return dirs + + + def _verify_build_environment(self): + # provide a local build env. This will be extended later to include non local + if BuildEnvironment.objects.count() == 0: + BuildEnvironment.objects.create(betype=BuildEnvironment.TYPE_LOCAL) + + # we make sure we have builddir and sourcedir for all defined build envionments + for be in BuildEnvironment.objects.all(): + be.needs_import = False + def _verify_be(): + is_changed = False + + def _update_sourcedir(): + be.sourcedir = os.environ.get('TOASTER_DIR') + return True + + if len(be.sourcedir) == 0: + print "\n -- Validation: The layers checkout directory must be set." + is_changed = _update_sourcedir() + + if not be.sourcedir.startswith("/"): + print "\n -- Validation: The layers checkout directory must be set to an absolute path." + is_changed = _update_sourcedir() + + if is_changed: + if be.betype == BuildEnvironment.TYPE_LOCAL: + be.needs_import = True + return True + + def _update_builddir(): + be.builddir = os.environ.get('TOASTER_DIR')+"/build" + return True + + if len(be.builddir) == 0: + print "\n -- Validation: The build directory must be set." + is_changed = _update_builddir() + + if not be.builddir.startswith("/"): + print "\n -- Validation: The build directory must to be set to an absolute path." + is_changed = _update_builddir() + + + if is_changed: + print "\nBuild configuration saved" + be.save() + return True + + + if be.needs_import: + try: + config_file = os.environ.get('TOASTER_CONF') + print "\nImporting file: %s" % config_file + from loadconf import Command as LoadConfigCommand + + LoadConfigCommand()._import_layer_config(config_file) + # we run lsupdates after config update + print "\nLayer configuration imported. Updating information from the layer sources, please wait.\nYou can re-update any time later by running bitbake/lib/toaster/manage.py lsupdates" + from django.core.management import call_command + call_command("lsupdates") + + # we don't look for any other config files + return is_changed + except Exception as e: + print "Failure while trying to import the toaster config file %s: %s" %\ + (config_file, e) + traceback.print_exc(e) + + return is_changed + + while _verify_be(): + pass + return 0 + + def _verify_default_settings(self): + # verify that default settings are there + if ToasterSetting.objects.filter(name='DEFAULT_RELEASE').count() != 1: + ToasterSetting.objects.filter(name='DEFAULT_RELEASE').delete() + ToasterSetting.objects.get_or_create(name='DEFAULT_RELEASE', value='') + return 0 + + def _verify_builds_in_progress(self): + # we are just starting up. we must not have any builds in progress, or build environments taken + for b in BuildRequest.objects.filter(state=BuildRequest.REQ_INPROGRESS): + BRError.objects.create(req=b, errtype="toaster", + errmsg= + "Toaster found this build IN PROGRESS while Toaster started up. This is an inconsistent state, and the build was marked as failed") + + BuildRequest.objects.filter(state=BuildRequest.REQ_INPROGRESS).update(state=BuildRequest.REQ_FAILED) + + BuildEnvironment.objects.update(lock=BuildEnvironment.LOCK_FREE) + + # also mark "In Progress builds as failures" + from django.utils import timezone + Build.objects.filter(outcome=Build.IN_PROGRESS).update(outcome=Build.FAILED, completed_on=timezone.now()) + + return 0 + + + + def handle_noargs(self, **options): + retval = 0 + retval += self._verify_build_environment() + retval += self._verify_default_settings() + retval += self._verify_builds_in_progress() + + return retval diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py new file mode 100644 index 000000000..5022b5940 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py @@ -0,0 +1,183 @@ +from django.core.management.base import BaseCommand, CommandError +from orm.models import LayerSource, ToasterSetting, Branch, Layer, Layer_Version +from orm.models import BitbakeVersion, Release, ReleaseDefaultLayer, ReleaseLayerSourcePriority +from django.db import IntegrityError +import os + +from checksettings import DN + +import logging +logger = logging.getLogger("toaster") + +def _reduce_canon_path(path): + components = [] + for c in path.split("/"): + if c == "..": + del components[-1] + elif c == ".": + pass + else: + components.append(c) + if len(components) < 2: + components.append('') + return "/".join(components) + +def _get_id_for_sourcetype(s): + for i in LayerSource.SOURCE_TYPE: + if s == i[1]: + return i[0] + raise Exception("Could not find definition for sourcetype '%s'. Valid source types are %s" % (str(s), ', '.join(map(lambda x: "'%s'" % x[1], LayerSource.SOURCE_TYPE )))) + +class Command(BaseCommand): + help = "Loads a toasterconf.json file in the database" + args = "filepath" + + + + def _import_layer_config(self, filepath): + if not os.path.exists(filepath) or not os.path.isfile(filepath): + raise Exception("Failed to find toaster config file %s ." % filepath) + + import json + data = json.loads(open(filepath, "r").read()) + + # verify config file validity before updating settings + for i in ['bitbake', 'releases', 'defaultrelease', 'config', 'layersources']: + assert i in data + + def _read_git_url_from_local_repository(address): + url = None + # we detect the remote name at runtime + import subprocess + (remote, remote_name) = address.split(":", 1) + cmd = subprocess.Popen("git remote -v", shell=True, cwd = os.path.dirname(filepath), stdout=subprocess.PIPE, stderr = subprocess.PIPE) + (out,err) = cmd.communicate() + if cmd.returncode != 0: + logging.warning("Error while importing layer vcs_url: git error: %s" % err) + for line in out.split("\n"): + try: + (name, path) = line.split("\t", 1) + if name == remote_name: + url = path.split(" ")[0] + break + except ValueError: + pass + if url == None: + logging.warning("Error while looking for remote \"%s\" in \"%s\"" % (remote_name, out)) + return url + + + # import bitbake data + for bvi in data['bitbake']: + bvo, created = BitbakeVersion.objects.get_or_create(name=bvi['name']) + if bvi['giturl'].startswith("remote:"): + bvo.giturl = _read_git_url_from_local_repository(bvi['giturl']) + if bvo.giturl is None: + logger.error("The toaster config file references the local git repo, but Toaster cannot detect it.\nYour local configuration for bitbake version %s is invalid. Make sure that the toasterconf.json file is correct." % bvi['name']) + + if bvo.giturl is None: + bvo.giturl = bvi['giturl'] + bvo.branch = bvi['branch'] + bvo.dirpath = bvi['dirpath'] + bvo.save() + + # set the layer sources + for lsi in data['layersources']: + assert 'sourcetype' in lsi + assert 'apiurl' in lsi + assert 'name' in lsi + assert 'branches' in lsi + + + if _get_id_for_sourcetype(lsi['sourcetype']) == LayerSource.TYPE_LAYERINDEX or lsi['apiurl'].startswith("/"): + apiurl = lsi['apiurl'] + else: + apiurl = _reduce_canon_path(os.path.join(DN(os.path.abspath(filepath)), lsi['apiurl'])) + + assert ((_get_id_for_sourcetype(lsi['sourcetype']) == LayerSource.TYPE_LAYERINDEX) or apiurl.startswith("/")), (lsi['sourcetype'],apiurl) + + try: + ls, created = LayerSource.objects.get_or_create(sourcetype = _get_id_for_sourcetype(lsi['sourcetype']), apiurl = apiurl) + ls.name = lsi['name'] + ls.save() + except IntegrityError as e: + logger.warning("IntegrityError %s \nWhile setting name %s for layer source %s " % (e, lsi['name'], ls)) + + + layerbranches = [] + for branchname in lsi['branches']: + bo, created = Branch.objects.get_or_create(layer_source = ls, name = branchname) + layerbranches.append(bo) + + if 'layers' in lsi: + for layerinfo in lsi['layers']: + lo, created = Layer.objects.get_or_create(layer_source = ls, name = layerinfo['name']) + if layerinfo['local_path'].startswith("/"): + lo.local_path = layerinfo['local_path'] + else: + lo.local_path = _reduce_canon_path(os.path.join(ls.apiurl, layerinfo['local_path'])) + + if not os.path.exists(lo.local_path): + logger.error("Local layer path %s must exists. Are you trying to import a layer that does not exist ? Check your local toasterconf.json" % lo.local_path) + + if layerinfo['vcs_url'].startswith("remote:"): + lo.vcs_url = _read_git_url_from_local_repository(layerinfo['vcs_url']) + if lo.vcs_url is None: + logger.error("The toaster config file references the local git repo, but Toaster cannot detect it.\nYour local configuration for layer %s is invalid. Make sure that the toasterconf.json file is correct." % layerinfo['name']) + + if lo.vcs_url is None: + lo.vcs_url = layerinfo['vcs_url'] + + if 'layer_index_url' in layerinfo: + lo.layer_index_url = layerinfo['layer_index_url'] + lo.save() + + for branch in layerbranches: + lvo, created = Layer_Version.objects.get_or_create(layer_source = ls, + up_branch = branch, + commit = branch.name, + layer = lo) + lvo.dirpath = layerinfo['dirpath'] + lvo.save() + # set releases + for ri in data['releases']: + bvo = BitbakeVersion.objects.get(name = ri['bitbake']) + assert bvo is not None + + ro, created = Release.objects.get_or_create(name = ri['name'], bitbake_version = bvo, branch_name = ri['branch']) + ro.description = ri['description'] + ro.helptext = ri['helptext'] + ro.save() + + # save layer source priority for release + for ls_name in ri['layersourcepriority'].keys(): + rlspo, created = ReleaseLayerSourcePriority.objects.get_or_create(release = ro, layer_source = LayerSource.objects.get(name=ls_name)) + rlspo.priority = ri['layersourcepriority'][ls_name] + rlspo.save() + + for dli in ri['defaultlayers']: + # find layers with the same name + ReleaseDefaultLayer.objects.get_or_create( release = ro, layer_name = dli) + + # set default release + if ToasterSetting.objects.filter(name = "DEFAULT_RELEASE").count() > 0: + ToasterSetting.objects.filter(name = "DEFAULT_RELEASE").update(value = data['defaultrelease']) + else: + ToasterSetting.objects.create(name = "DEFAULT_RELEASE", value = data['defaultrelease']) + + # set default config variables + for configname in data['config']: + if ToasterSetting.objects.filter(name = "DEFCONF_" + configname).count() > 0: + ToasterSetting.objects.filter(name = "DEFCONF_" + configname).update(value = data['config'][configname]) + else: + ToasterSetting.objects.create(name = "DEFCONF_" + configname, value = data['config'][configname]) + + + def handle(self, *args, **options): + if len(args) == 0: + raise CommandError("Need a path to the toasterconf.json file") + filepath = args[0] + self._import_layer_config(filepath) + + + diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py new file mode 100644 index 000000000..27289be5f --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py @@ -0,0 +1,179 @@ +from django.core.management.base import NoArgsCommand, CommandError +from django.db import transaction +from django.db.models import Q + +from bldcontrol.bbcontroller import getBuildEnvironmentController +from bldcontrol.bbcontroller import ShellCmdException, BuildSetupException +from bldcontrol.models import BuildRequest, BuildEnvironment +from bldcontrol.models import BRError, BRVariable + +from orm.models import Build, ToasterSetting, LogMessage, Target + +import os +import logging +import time +import sys +import traceback + +logger = logging.getLogger("toaster") + +class Command(NoArgsCommand): + args = "" + help = "Schedules and executes build requests as possible." + "Does not return (interrupt with Ctrl-C)" + + + @transaction.atomic + def _selectBuildEnvironment(self): + bec = getBuildEnvironmentController(lock = BuildEnvironment.LOCK_FREE) + bec.be.lock = BuildEnvironment.LOCK_LOCK + bec.be.save() + return bec + + @transaction.atomic + def _selectBuildRequest(self): + br = BuildRequest.objects.filter(state=BuildRequest.REQ_QUEUED).first() + return br + + def schedule(self): + try: + # select the build environment and the request to build + br = self._selectBuildRequest() + if br: + br.state = BuildRequest.REQ_INPROGRESS + br.save() + else: + return + + try: + bec = self._selectBuildEnvironment() + except IndexError as e: + # we could not find a BEC; postpone the BR + br.state = BuildRequest.REQ_QUEUED + br.save() + logger.debug("runbuilds: No build env") + return + + logger.debug("runbuilds: starting build %s, environment %s" % \ + (str(br).decode('utf-8'), bec.be)) + + # let the build request know where it is being executed + br.environment = bec.be + br.save() + + # this triggers an async build + bec.triggerBuild(br.brbitbake, br.brlayer_set.all(), + br.brvariable_set.all(), br.brtarget_set.all(), + "%d:%d" % (br.pk, bec.be.pk)) + + except Exception as e: + logger.error("runbuilds: Error launching build %s" % e) + traceback.print_exc(e) + if "[Errno 111] Connection refused" in str(e): + # Connection refused, read toaster_server.out + errmsg = bec.readServerLogFile() + else: + errmsg = str(e) + + BRError.objects.create(req = br, + errtype = str(type(e)), + errmsg = errmsg, + traceback = traceback.format_exc(e)) + br.state = BuildRequest.REQ_FAILED + br.save() + bec.be.lock = BuildEnvironment.LOCK_FREE + bec.be.save() + + def archive(self): + for br in BuildRequest.objects.filter(state = BuildRequest.REQ_ARCHIVE): + if br.build == None: + br.state = BuildRequest.REQ_FAILED + else: + br.state = BuildRequest.REQ_COMPLETED + br.save() + + def cleanup(self): + from django.utils import timezone + from datetime import timedelta + # environments locked for more than 30 seconds + # they should be unlocked + BuildEnvironment.objects.filter( + Q(buildrequest__state__in=[BuildRequest.REQ_FAILED, + BuildRequest.REQ_COMPLETED, + BuildRequest.REQ_CANCELLING]) & + Q(lock=BuildEnvironment.LOCK_LOCK) & + Q(updated__lt=timezone.now() - timedelta(seconds = 30)) + ).update(lock=BuildEnvironment.LOCK_FREE) + + + # update all Builds that were in progress and failed to start + for br in BuildRequest.objects.filter( + state=BuildRequest.REQ_FAILED, + build__outcome=Build.IN_PROGRESS): + # transpose the launch errors in ToasterExceptions + br.build.outcome = Build.FAILED + for brerror in br.brerror_set.all(): + logger.debug("Saving error %s" % brerror) + LogMessage.objects.create(build=br.build, + level=LogMessage.EXCEPTION, + message=brerror.errmsg) + br.build.save() + + # we don't have a true build object here; hence, toasterui + # didn't have a change to release the BE lock + br.environment.lock = BuildEnvironment.LOCK_FREE + br.environment.save() + + + # update all BuildRequests without a build created + for br in BuildRequest.objects.filter(build = None): + br.build = Build.objects.create(project=br.project, + completed_on=br.updated, + started_on=br.created) + br.build.outcome = Build.FAILED + try: + br.build.machine = br.brvariable_set.get(name='MACHINE').value + except BRVariable.DoesNotExist: + pass + br.save() + # transpose target information + for brtarget in br.brtarget_set.all(): + Target.objects.create(build=br.build, + target=brtarget.target, + task=brtarget.task) + # transpose the launch errors in ToasterExceptions + for brerror in br.brerror_set.all(): + LogMessage.objects.create(build=br.build, + level=LogMessage.EXCEPTION, + message=brerror.errmsg) + + br.build.save() + + # Make sure the LOCK is removed for builds which have been fully + # cancelled + for br in BuildRequest.objects.filter( + Q(build__outcome=Build.CANCELLED) & + Q(state=BuildRequest.REQ_CANCELLING) & + ~Q(environment=None)): + br.environment.lock = BuildEnvironment.LOCK_FREE + br.environment.save() + + + def handle_noargs(self, **options): + while True: + try: + self.cleanup() + except Exception as e: + logger.warn("runbuilds: cleanup exception %s" % str(e)) + + try: + self.archive() + except Exception as e: + logger.warn("runbuilds: archive exception %s" % str(e)) + + try: + self.schedule() + except Exception as e: + logger.warn("runbuilds: schedule exception %s" % str(e)) + + time.sleep(1) |
