diff options
Diffstat (limited to 'import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands')
3 files changed, 91 insertions, 276 deletions
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py index 5e70437b2..2ed994f61 100644 --- a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py @@ -1,10 +1,15 @@ from django.core.management.base import NoArgsCommand, CommandError from django.db import transaction + +from django.core.management import call_command from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException from bldcontrol.models import BuildRequest, BuildEnvironment, BRError -from orm.models import ToasterSetting, Build +from orm.models import ToasterSetting, Build, Layer + import os import traceback +import warnings + def DN(path): if path is None: @@ -21,39 +26,6 @@ class Command(NoArgsCommand): super(Command, self).__init__(*args, **kwargs) self.guesspath = DN(DN(DN(DN(DN(DN(DN(__file__))))))) - def _find_first_path_for_file(self, startdirectory, filename, level=0): - if level < 0: - return None - dirs = [] - for i in os.listdir(startdirectory): - j = os.path.join(startdirectory, i) - if os.path.isfile(j): - if i == filename: - return startdirectory - elif os.path.isdir(j): - dirs.append(j) - for j in dirs: - ret = self._find_first_path_for_file(j, filename, level - 1) - if ret is not None: - return ret - return None - - def _recursive_list_directories(self, startdirectory, level=0): - if level < 0: - return [] - dirs = [] - try: - for i in os.listdir(startdirectory): - j = os.path.join(startdirectory, i) - if os.path.isdir(j): - dirs.append(j) - except OSError: - pass - for j in dirs: - dirs = dirs + self._recursive_list_directories(j, level - 1) - return dirs - - def _verify_build_environment(self): # provide a local build env. This will be extended later to include non local if BuildEnvironment.objects.count() == 0: @@ -70,11 +42,10 @@ class Command(NoArgsCommand): return True if len(be.sourcedir) == 0: - print "\n -- Validation: The layers checkout directory must be set." is_changed = _update_sourcedir() if not be.sourcedir.startswith("/"): - print "\n -- Validation: The layers checkout directory must be set to an absolute path." + print("\n -- Validation: The layers checkout directory must be set to an absolute path.") is_changed = _update_sourcedir() if is_changed: @@ -87,38 +58,67 @@ class Command(NoArgsCommand): return True if len(be.builddir) == 0: - print "\n -- Validation: The build directory must be set." is_changed = _update_builddir() if not be.builddir.startswith("/"): - print "\n -- Validation: The build directory must to be set to an absolute path." + print("\n -- Validation: The build directory must to be set to an absolute path.") is_changed = _update_builddir() - if is_changed: - print "\nBuild configuration saved" + print("\nBuild configuration saved") be.save() return True - if be.needs_import: try: - config_file = os.environ.get('TOASTER_CONF') - print "\nImporting file: %s" % config_file - from loadconf import Command as LoadConfigCommand + print("Loading default settings") + call_command("loaddata", "settings") + template_conf = os.environ.get("TEMPLATECONF", "") + + if "poky" in template_conf: + print("Loading poky configuration") + call_command("loaddata", "poky") + else: + print("Loading OE-Core configuration") + call_command("loaddata", "oe-core") + if template_conf: + oe_core_path = os.path.realpath( + template_conf + + "/../") + else: + print("TEMPLATECONF not found. You may have to" + " manually configure layer paths") + oe_core_path = input("Please enter the path of" + " your openembedded-core " + "layer: ") + # Update the layer instances of openemebedded-core + for layer in Layer.objects.filter( + name="openembedded-core", + local_source_dir="OE-CORE-LAYER-DIR"): + layer.local_path = oe_core_path + layer.save() + + # Import the custom fixture if it's present + with warnings.catch_warnings(): + warnings.filterwarnings( + action="ignore", + message="^.*No fixture named.*$") + print("Importing custom settings if present") + call_command("loaddata", "custom") - LoadConfigCommand()._import_layer_config(config_file) # we run lsupdates after config update - print "\nLayer configuration imported. Updating information from the layer sources, please wait.\nYou can re-update any time later by running bitbake/lib/toaster/manage.py lsupdates" - from django.core.management import call_command + print("\nFetching information from the layer index, " + "please wait.\nYou can re-update any time later " + "by running bitbake/lib/toaster/manage.py " + "lsupdates\n") call_command("lsupdates") # we don't look for any other config files return is_changed except Exception as e: - print "Failure while trying to import the toaster config file %s: %s" %\ - (config_file, e) - traceback.print_exc(e) + print("Failure while trying to setup toaster: %s" + % e) + traceback.print_exc() return is_changed diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py deleted file mode 100644 index 5022b5940..000000000 --- a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py +++ /dev/null @@ -1,183 +0,0 @@ -from django.core.management.base import BaseCommand, CommandError -from orm.models import LayerSource, ToasterSetting, Branch, Layer, Layer_Version -from orm.models import BitbakeVersion, Release, ReleaseDefaultLayer, ReleaseLayerSourcePriority -from django.db import IntegrityError -import os - -from checksettings import DN - -import logging -logger = logging.getLogger("toaster") - -def _reduce_canon_path(path): - components = [] - for c in path.split("/"): - if c == "..": - del components[-1] - elif c == ".": - pass - else: - components.append(c) - if len(components) < 2: - components.append('') - return "/".join(components) - -def _get_id_for_sourcetype(s): - for i in LayerSource.SOURCE_TYPE: - if s == i[1]: - return i[0] - raise Exception("Could not find definition for sourcetype '%s'. Valid source types are %s" % (str(s), ', '.join(map(lambda x: "'%s'" % x[1], LayerSource.SOURCE_TYPE )))) - -class Command(BaseCommand): - help = "Loads a toasterconf.json file in the database" - args = "filepath" - - - - def _import_layer_config(self, filepath): - if not os.path.exists(filepath) or not os.path.isfile(filepath): - raise Exception("Failed to find toaster config file %s ." % filepath) - - import json - data = json.loads(open(filepath, "r").read()) - - # verify config file validity before updating settings - for i in ['bitbake', 'releases', 'defaultrelease', 'config', 'layersources']: - assert i in data - - def _read_git_url_from_local_repository(address): - url = None - # we detect the remote name at runtime - import subprocess - (remote, remote_name) = address.split(":", 1) - cmd = subprocess.Popen("git remote -v", shell=True, cwd = os.path.dirname(filepath), stdout=subprocess.PIPE, stderr = subprocess.PIPE) - (out,err) = cmd.communicate() - if cmd.returncode != 0: - logging.warning("Error while importing layer vcs_url: git error: %s" % err) - for line in out.split("\n"): - try: - (name, path) = line.split("\t", 1) - if name == remote_name: - url = path.split(" ")[0] - break - except ValueError: - pass - if url == None: - logging.warning("Error while looking for remote \"%s\" in \"%s\"" % (remote_name, out)) - return url - - - # import bitbake data - for bvi in data['bitbake']: - bvo, created = BitbakeVersion.objects.get_or_create(name=bvi['name']) - if bvi['giturl'].startswith("remote:"): - bvo.giturl = _read_git_url_from_local_repository(bvi['giturl']) - if bvo.giturl is None: - logger.error("The toaster config file references the local git repo, but Toaster cannot detect it.\nYour local configuration for bitbake version %s is invalid. Make sure that the toasterconf.json file is correct." % bvi['name']) - - if bvo.giturl is None: - bvo.giturl = bvi['giturl'] - bvo.branch = bvi['branch'] - bvo.dirpath = bvi['dirpath'] - bvo.save() - - # set the layer sources - for lsi in data['layersources']: - assert 'sourcetype' in lsi - assert 'apiurl' in lsi - assert 'name' in lsi - assert 'branches' in lsi - - - if _get_id_for_sourcetype(lsi['sourcetype']) == LayerSource.TYPE_LAYERINDEX or lsi['apiurl'].startswith("/"): - apiurl = lsi['apiurl'] - else: - apiurl = _reduce_canon_path(os.path.join(DN(os.path.abspath(filepath)), lsi['apiurl'])) - - assert ((_get_id_for_sourcetype(lsi['sourcetype']) == LayerSource.TYPE_LAYERINDEX) or apiurl.startswith("/")), (lsi['sourcetype'],apiurl) - - try: - ls, created = LayerSource.objects.get_or_create(sourcetype = _get_id_for_sourcetype(lsi['sourcetype']), apiurl = apiurl) - ls.name = lsi['name'] - ls.save() - except IntegrityError as e: - logger.warning("IntegrityError %s \nWhile setting name %s for layer source %s " % (e, lsi['name'], ls)) - - - layerbranches = [] - for branchname in lsi['branches']: - bo, created = Branch.objects.get_or_create(layer_source = ls, name = branchname) - layerbranches.append(bo) - - if 'layers' in lsi: - for layerinfo in lsi['layers']: - lo, created = Layer.objects.get_or_create(layer_source = ls, name = layerinfo['name']) - if layerinfo['local_path'].startswith("/"): - lo.local_path = layerinfo['local_path'] - else: - lo.local_path = _reduce_canon_path(os.path.join(ls.apiurl, layerinfo['local_path'])) - - if not os.path.exists(lo.local_path): - logger.error("Local layer path %s must exists. Are you trying to import a layer that does not exist ? Check your local toasterconf.json" % lo.local_path) - - if layerinfo['vcs_url'].startswith("remote:"): - lo.vcs_url = _read_git_url_from_local_repository(layerinfo['vcs_url']) - if lo.vcs_url is None: - logger.error("The toaster config file references the local git repo, but Toaster cannot detect it.\nYour local configuration for layer %s is invalid. Make sure that the toasterconf.json file is correct." % layerinfo['name']) - - if lo.vcs_url is None: - lo.vcs_url = layerinfo['vcs_url'] - - if 'layer_index_url' in layerinfo: - lo.layer_index_url = layerinfo['layer_index_url'] - lo.save() - - for branch in layerbranches: - lvo, created = Layer_Version.objects.get_or_create(layer_source = ls, - up_branch = branch, - commit = branch.name, - layer = lo) - lvo.dirpath = layerinfo['dirpath'] - lvo.save() - # set releases - for ri in data['releases']: - bvo = BitbakeVersion.objects.get(name = ri['bitbake']) - assert bvo is not None - - ro, created = Release.objects.get_or_create(name = ri['name'], bitbake_version = bvo, branch_name = ri['branch']) - ro.description = ri['description'] - ro.helptext = ri['helptext'] - ro.save() - - # save layer source priority for release - for ls_name in ri['layersourcepriority'].keys(): - rlspo, created = ReleaseLayerSourcePriority.objects.get_or_create(release = ro, layer_source = LayerSource.objects.get(name=ls_name)) - rlspo.priority = ri['layersourcepriority'][ls_name] - rlspo.save() - - for dli in ri['defaultlayers']: - # find layers with the same name - ReleaseDefaultLayer.objects.get_or_create( release = ro, layer_name = dli) - - # set default release - if ToasterSetting.objects.filter(name = "DEFAULT_RELEASE").count() > 0: - ToasterSetting.objects.filter(name = "DEFAULT_RELEASE").update(value = data['defaultrelease']) - else: - ToasterSetting.objects.create(name = "DEFAULT_RELEASE", value = data['defaultrelease']) - - # set default config variables - for configname in data['config']: - if ToasterSetting.objects.filter(name = "DEFCONF_" + configname).count() > 0: - ToasterSetting.objects.filter(name = "DEFCONF_" + configname).update(value = data['config'][configname]) - else: - ToasterSetting.objects.create(name = "DEFCONF_" + configname, value = data['config'][configname]) - - - def handle(self, *args, **options): - if len(args) == 0: - raise CommandError("Need a path to the toasterconf.json file") - filepath = args[0] - self._import_layer_config(filepath) - - - diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py index 27289be5f..7f7a5a955 100644 --- a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py @@ -1,31 +1,27 @@ -from django.core.management.base import NoArgsCommand, CommandError +from django.core.management.base import NoArgsCommand from django.db import transaction from django.db.models import Q from bldcontrol.bbcontroller import getBuildEnvironmentController -from bldcontrol.bbcontroller import ShellCmdException, BuildSetupException from bldcontrol.models import BuildRequest, BuildEnvironment from bldcontrol.models import BRError, BRVariable -from orm.models import Build, ToasterSetting, LogMessage, Target +from orm.models import Build, LogMessage, Target -import os import logging -import time -import sys import traceback +import signal logger = logging.getLogger("toaster") class Command(NoArgsCommand): - args = "" - help = "Schedules and executes build requests as possible." - "Does not return (interrupt with Ctrl-C)" - + args = "" + help = "Schedules and executes build requests as possible. "\ + "Does not return (interrupt with Ctrl-C)" @transaction.atomic def _selectBuildEnvironment(self): - bec = getBuildEnvironmentController(lock = BuildEnvironment.LOCK_FREE) + bec = getBuildEnvironmentController(lock=BuildEnvironment.LOCK_FREE) bec.be.lock = BuildEnvironment.LOCK_LOCK bec.be.save() return bec @@ -54,8 +50,8 @@ class Command(NoArgsCommand): logger.debug("runbuilds: No build env") return - logger.debug("runbuilds: starting build %s, environment %s" % \ - (str(br).decode('utf-8'), bec.be)) + logger.info("runbuilds: starting build %s, environment %s" % \ + (br, bec.be)) # let the build request know where it is being executed br.environment = bec.be @@ -68,24 +64,22 @@ class Command(NoArgsCommand): except Exception as e: logger.error("runbuilds: Error launching build %s" % e) - traceback.print_exc(e) + traceback.print_exc() if "[Errno 111] Connection refused" in str(e): # Connection refused, read toaster_server.out errmsg = bec.readServerLogFile() else: errmsg = str(e) - BRError.objects.create(req = br, - errtype = str(type(e)), - errmsg = errmsg, - traceback = traceback.format_exc(e)) + BRError.objects.create(req=br, errtype=str(type(e)), errmsg=errmsg, + traceback=traceback.format_exc()) br.state = BuildRequest.REQ_FAILED br.save() bec.be.lock = BuildEnvironment.LOCK_FREE bec.be.save() def archive(self): - for br in BuildRequest.objects.filter(state = BuildRequest.REQ_ARCHIVE): + for br in BuildRequest.objects.filter(state=BuildRequest.REQ_ARCHIVE): if br.build == None: br.state = BuildRequest.REQ_FAILED else: @@ -102,14 +96,13 @@ class Command(NoArgsCommand): BuildRequest.REQ_COMPLETED, BuildRequest.REQ_CANCELLING]) & Q(lock=BuildEnvironment.LOCK_LOCK) & - Q(updated__lt=timezone.now() - timedelta(seconds = 30)) + Q(updated__lt=timezone.now() - timedelta(seconds=30)) ).update(lock=BuildEnvironment.LOCK_FREE) # update all Builds that were in progress and failed to start - for br in BuildRequest.objects.filter( - state=BuildRequest.REQ_FAILED, - build__outcome=Build.IN_PROGRESS): + for br in BuildRequest.objects.filter(state=BuildRequest.REQ_FAILED, + build__outcome=Build.IN_PROGRESS): # transpose the launch errors in ToasterExceptions br.build.outcome = Build.FAILED for brerror in br.brerror_set.all(): @@ -126,7 +119,7 @@ class Command(NoArgsCommand): # update all BuildRequests without a build created - for br in BuildRequest.objects.filter(build = None): + for br in BuildRequest.objects.filter(build=None): br.build = Build.objects.create(project=br.project, completed_on=br.updated, started_on=br.created) @@ -151,29 +144,34 @@ class Command(NoArgsCommand): # Make sure the LOCK is removed for builds which have been fully # cancelled - for br in BuildRequest.objects.filter( - Q(build__outcome=Build.CANCELLED) & - Q(state=BuildRequest.REQ_CANCELLING) & - ~Q(environment=None)): + for br in BuildRequest.objects.filter(\ + Q(build__outcome=Build.CANCELLED) & + Q(state=BuildRequest.REQ_CANCELLING) & + ~Q(environment=None)): br.environment.lock = BuildEnvironment.LOCK_FREE br.environment.save() + def runbuild(self): + try: + self.cleanup() + except Exception as e: + logger.warn("runbuilds: cleanup exception %s" % str(e)) - def handle_noargs(self, **options): - while True: - try: - self.cleanup() - except Exception as e: - logger.warn("runbuilds: cleanup exception %s" % str(e)) + try: + self.archive() + except Exception as e: + logger.warn("runbuilds: archive exception %s" % str(e)) - try: - self.archive() - except Exception as e: - logger.warn("runbuilds: archive exception %s" % str(e)) + try: + self.schedule() + except Exception as e: + logger.warn("runbuilds: schedule exception %s" % str(e)) - try: - self.schedule() - except Exception as e: - logger.warn("runbuilds: schedule exception %s" % str(e)) + def handle_noargs(self, **options): + self.runbuild() + + signal.signal(signal.SIGUSR1, lambda sig, frame: None) - time.sleep(1) + while True: + signal.pause() + self.runbuild() |
