diff options
Diffstat (limited to 'import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol')
11 files changed, 250 insertions, 332 deletions
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/bbcontroller.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/bbcontroller.py index d09ac1787..912f67bf8 100644 --- a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/bbcontroller.py +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/bbcontroller.py @@ -30,7 +30,6 @@ from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, B # load Bitbake components path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) sys.path.insert(0, path) -import bb.server.xmlrpc class BitbakeController(object): """ This is the basic class that controlls a bitbake server. @@ -38,6 +37,7 @@ class BitbakeController(object): """ def __init__(self, be): + import bb.server.xmlrpc self.connection = bb.server.xmlrpc._create_server(be.bbaddress, int(be.bbport))[0] @@ -79,7 +79,7 @@ def getBuildEnvironmentController(**kwargs): The return object MUST always be a BuildEnvironmentController. """ - from localhostbecontroller import LocalhostBEController + from bldcontrol.localhostbecontroller import LocalhostBEController be = BuildEnvironment.objects.filter(Q(**kwargs))[0] if be.betype == BuildEnvironment.TYPE_LOCAL: diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py index 7def1f3a1..e5f7c988c 100644 --- a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py @@ -32,7 +32,7 @@ import subprocess from toastermain import settings -from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, BitbakeController +from bldcontrol.bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, BitbakeController import logging logger = logging.getLogger("toaster") @@ -66,16 +66,16 @@ class LocalhostBEController(BuildEnvironmentController): err = "command: %s \n%s" % (command, out) else: err = "command: %s \n%s" % (command, err) - logger.warn("localhostbecontroller: shellcmd error %s" % err) + logger.warning("localhostbecontroller: shellcmd error %s" % err) raise ShellCmdException(err) else: logger.debug("localhostbecontroller: shellcmd success") - return out + return out.decode('utf-8') def getGitCloneDirectory(self, url, branch): """Construct unique clone directory name out of url and branch.""" if branch != "HEAD": - return "_toaster_clones/_%s_%s" % (re.sub('[:/@%]', '_', url), branch) + return "_toaster_clones/_%s_%s" % (re.sub('[:/@+%]', '_', url), branch) # word of attention; this is a localhost-specific issue; only on the localhost we expect to have "HEAD" releases # which _ALWAYS_ means the current poky checkout @@ -89,19 +89,34 @@ class LocalhostBEController(BuildEnvironmentController): """ a word of attention: by convention, the first layer for any build will be poky! """ assert self.be.sourcedir is not None + + layerlist = [] + nongitlayerlist = [] + # set layers in the layersource # 1. get a list of repos with branches, and map dirpaths for each layer gitrepos = {} - gitrepos[(bitbake.giturl, bitbake.commit)] = [] - gitrepos[(bitbake.giturl, bitbake.commit)].append( ("bitbake", bitbake.dirpath) ) + # if we're using a remotely fetched version of bitbake add its git + # details to the list of repos to clone + if bitbake.giturl and bitbake.commit: + gitrepos[(bitbake.giturl, bitbake.commit)] = [] + gitrepos[(bitbake.giturl, bitbake.commit)].append( + ("bitbake", bitbake.dirpath)) for layer in layers: # We don't need to git clone the layer for the CustomImageRecipe # as it's generated by us layer on if needed if CustomImageRecipe.LAYER_NAME in layer.name: continue + + # If we have local layers then we don't need clone them + # For local layers giturl will be empty + if not layer.giturl: + nongitlayerlist.append(layer.layer_version.layer.local_source_dir) + continue + if not (layer.giturl, layer.commit) in gitrepos: gitrepos[(layer.giturl, layer.commit)] = [] gitrepos[(layer.giturl, layer.commit)].append( (layer.name, layer.dirpath) ) @@ -131,19 +146,22 @@ class LocalhostBEController(BuildEnvironmentController): logger.info("Using pre-checked out source for layer %s", cached_layers) - layerlist = [] - - # 3. checkout the repositories for giturl, commit in gitrepos.keys(): localdirname = os.path.join(self.be.sourcedir, self.getGitCloneDirectory(giturl, commit)) logger.debug("localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname)) - # make sure our directory is a git repository + # see if our directory is a git repository if os.path.exists(localdirname): - localremotes = self._shellcmd("git remote -v", localdirname) - if not giturl in localremotes: - raise BuildSetupException("Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join(localremotes.split("\n")), giturl)) + try: + localremotes = self._shellcmd("git remote -v", + localdirname) + if not giturl in localremotes and commit != 'HEAD': + raise BuildSetupException("Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join(localremotes.split("\n")), giturl)) + except ShellCmdException: + # our localdirname might not be a git repository + #- that's fine + pass else: if giturl in cached_layers: logger.debug("localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname)) @@ -245,6 +263,7 @@ class LocalhostBEController(BuildEnvironmentController): layerlist.append(layerpath) self.islayerset = True + layerlist.extend(nongitlayerlist) return layerlist def readServerLogFile(self): @@ -287,7 +306,7 @@ class LocalhostBEController(BuildEnvironmentController): # run bitbake server from the clone bitbake = os.path.join(self.pokydirname, 'bitbake', 'bin', 'bitbake') - self._shellcmd('bash -c \"source %s %s; BITBAKE_UI="" %s --read %s ' + self._shellcmd('bash -c \"source %s %s; BITBAKE_UI="knotty" %s --read %s ' '--server-only -t xmlrpc -B 0.0.0.0:0\"' % (oe_init, builddir, bitbake, confpath), self.be.sourcedir) @@ -324,7 +343,7 @@ class LocalhostBEController(BuildEnvironmentController): 'bitbake') self._shellcmd(['bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:-1" ' '%s %s -u toasterui --token="" >>%s 2>&1;' - 'BITBAKE_UI="" BBSERVER=0.0.0.0:-1 %s -m)&\"' \ + 'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:-1 %s -m)&\"' \ % (brbe, local_bitbake, bbtargets, log, bitbake)], builddir, nowait=True) diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py index 5e70437b2..2ed994f61 100644 --- a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py @@ -1,10 +1,15 @@ from django.core.management.base import NoArgsCommand, CommandError from django.db import transaction + +from django.core.management import call_command from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException from bldcontrol.models import BuildRequest, BuildEnvironment, BRError -from orm.models import ToasterSetting, Build +from orm.models import ToasterSetting, Build, Layer + import os import traceback +import warnings + def DN(path): if path is None: @@ -21,39 +26,6 @@ class Command(NoArgsCommand): super(Command, self).__init__(*args, **kwargs) self.guesspath = DN(DN(DN(DN(DN(DN(DN(__file__))))))) - def _find_first_path_for_file(self, startdirectory, filename, level=0): - if level < 0: - return None - dirs = [] - for i in os.listdir(startdirectory): - j = os.path.join(startdirectory, i) - if os.path.isfile(j): - if i == filename: - return startdirectory - elif os.path.isdir(j): - dirs.append(j) - for j in dirs: - ret = self._find_first_path_for_file(j, filename, level - 1) - if ret is not None: - return ret - return None - - def _recursive_list_directories(self, startdirectory, level=0): - if level < 0: - return [] - dirs = [] - try: - for i in os.listdir(startdirectory): - j = os.path.join(startdirectory, i) - if os.path.isdir(j): - dirs.append(j) - except OSError: - pass - for j in dirs: - dirs = dirs + self._recursive_list_directories(j, level - 1) - return dirs - - def _verify_build_environment(self): # provide a local build env. This will be extended later to include non local if BuildEnvironment.objects.count() == 0: @@ -70,11 +42,10 @@ class Command(NoArgsCommand): return True if len(be.sourcedir) == 0: - print "\n -- Validation: The layers checkout directory must be set." is_changed = _update_sourcedir() if not be.sourcedir.startswith("/"): - print "\n -- Validation: The layers checkout directory must be set to an absolute path." + print("\n -- Validation: The layers checkout directory must be set to an absolute path.") is_changed = _update_sourcedir() if is_changed: @@ -87,38 +58,67 @@ class Command(NoArgsCommand): return True if len(be.builddir) == 0: - print "\n -- Validation: The build directory must be set." is_changed = _update_builddir() if not be.builddir.startswith("/"): - print "\n -- Validation: The build directory must to be set to an absolute path." + print("\n -- Validation: The build directory must to be set to an absolute path.") is_changed = _update_builddir() - if is_changed: - print "\nBuild configuration saved" + print("\nBuild configuration saved") be.save() return True - if be.needs_import: try: - config_file = os.environ.get('TOASTER_CONF') - print "\nImporting file: %s" % config_file - from loadconf import Command as LoadConfigCommand + print("Loading default settings") + call_command("loaddata", "settings") + template_conf = os.environ.get("TEMPLATECONF", "") + + if "poky" in template_conf: + print("Loading poky configuration") + call_command("loaddata", "poky") + else: + print("Loading OE-Core configuration") + call_command("loaddata", "oe-core") + if template_conf: + oe_core_path = os.path.realpath( + template_conf + + "/../") + else: + print("TEMPLATECONF not found. You may have to" + " manually configure layer paths") + oe_core_path = input("Please enter the path of" + " your openembedded-core " + "layer: ") + # Update the layer instances of openemebedded-core + for layer in Layer.objects.filter( + name="openembedded-core", + local_source_dir="OE-CORE-LAYER-DIR"): + layer.local_path = oe_core_path + layer.save() + + # Import the custom fixture if it's present + with warnings.catch_warnings(): + warnings.filterwarnings( + action="ignore", + message="^.*No fixture named.*$") + print("Importing custom settings if present") + call_command("loaddata", "custom") - LoadConfigCommand()._import_layer_config(config_file) # we run lsupdates after config update - print "\nLayer configuration imported. Updating information from the layer sources, please wait.\nYou can re-update any time later by running bitbake/lib/toaster/manage.py lsupdates" - from django.core.management import call_command + print("\nFetching information from the layer index, " + "please wait.\nYou can re-update any time later " + "by running bitbake/lib/toaster/manage.py " + "lsupdates\n") call_command("lsupdates") # we don't look for any other config files return is_changed except Exception as e: - print "Failure while trying to import the toaster config file %s: %s" %\ - (config_file, e) - traceback.print_exc(e) + print("Failure while trying to setup toaster: %s" + % e) + traceback.print_exc() return is_changed diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py deleted file mode 100644 index 5022b5940..000000000 --- a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py +++ /dev/null @@ -1,183 +0,0 @@ -from django.core.management.base import BaseCommand, CommandError -from orm.models import LayerSource, ToasterSetting, Branch, Layer, Layer_Version -from orm.models import BitbakeVersion, Release, ReleaseDefaultLayer, ReleaseLayerSourcePriority -from django.db import IntegrityError -import os - -from checksettings import DN - -import logging -logger = logging.getLogger("toaster") - -def _reduce_canon_path(path): - components = [] - for c in path.split("/"): - if c == "..": - del components[-1] - elif c == ".": - pass - else: - components.append(c) - if len(components) < 2: - components.append('') - return "/".join(components) - -def _get_id_for_sourcetype(s): - for i in LayerSource.SOURCE_TYPE: - if s == i[1]: - return i[0] - raise Exception("Could not find definition for sourcetype '%s'. Valid source types are %s" % (str(s), ', '.join(map(lambda x: "'%s'" % x[1], LayerSource.SOURCE_TYPE )))) - -class Command(BaseCommand): - help = "Loads a toasterconf.json file in the database" - args = "filepath" - - - - def _import_layer_config(self, filepath): - if not os.path.exists(filepath) or not os.path.isfile(filepath): - raise Exception("Failed to find toaster config file %s ." % filepath) - - import json - data = json.loads(open(filepath, "r").read()) - - # verify config file validity before updating settings - for i in ['bitbake', 'releases', 'defaultrelease', 'config', 'layersources']: - assert i in data - - def _read_git_url_from_local_repository(address): - url = None - # we detect the remote name at runtime - import subprocess - (remote, remote_name) = address.split(":", 1) - cmd = subprocess.Popen("git remote -v", shell=True, cwd = os.path.dirname(filepath), stdout=subprocess.PIPE, stderr = subprocess.PIPE) - (out,err) = cmd.communicate() - if cmd.returncode != 0: - logging.warning("Error while importing layer vcs_url: git error: %s" % err) - for line in out.split("\n"): - try: - (name, path) = line.split("\t", 1) - if name == remote_name: - url = path.split(" ")[0] - break - except ValueError: - pass - if url == None: - logging.warning("Error while looking for remote \"%s\" in \"%s\"" % (remote_name, out)) - return url - - - # import bitbake data - for bvi in data['bitbake']: - bvo, created = BitbakeVersion.objects.get_or_create(name=bvi['name']) - if bvi['giturl'].startswith("remote:"): - bvo.giturl = _read_git_url_from_local_repository(bvi['giturl']) - if bvo.giturl is None: - logger.error("The toaster config file references the local git repo, but Toaster cannot detect it.\nYour local configuration for bitbake version %s is invalid. Make sure that the toasterconf.json file is correct." % bvi['name']) - - if bvo.giturl is None: - bvo.giturl = bvi['giturl'] - bvo.branch = bvi['branch'] - bvo.dirpath = bvi['dirpath'] - bvo.save() - - # set the layer sources - for lsi in data['layersources']: - assert 'sourcetype' in lsi - assert 'apiurl' in lsi - assert 'name' in lsi - assert 'branches' in lsi - - - if _get_id_for_sourcetype(lsi['sourcetype']) == LayerSource.TYPE_LAYERINDEX or lsi['apiurl'].startswith("/"): - apiurl = lsi['apiurl'] - else: - apiurl = _reduce_canon_path(os.path.join(DN(os.path.abspath(filepath)), lsi['apiurl'])) - - assert ((_get_id_for_sourcetype(lsi['sourcetype']) == LayerSource.TYPE_LAYERINDEX) or apiurl.startswith("/")), (lsi['sourcetype'],apiurl) - - try: - ls, created = LayerSource.objects.get_or_create(sourcetype = _get_id_for_sourcetype(lsi['sourcetype']), apiurl = apiurl) - ls.name = lsi['name'] - ls.save() - except IntegrityError as e: - logger.warning("IntegrityError %s \nWhile setting name %s for layer source %s " % (e, lsi['name'], ls)) - - - layerbranches = [] - for branchname in lsi['branches']: - bo, created = Branch.objects.get_or_create(layer_source = ls, name = branchname) - layerbranches.append(bo) - - if 'layers' in lsi: - for layerinfo in lsi['layers']: - lo, created = Layer.objects.get_or_create(layer_source = ls, name = layerinfo['name']) - if layerinfo['local_path'].startswith("/"): - lo.local_path = layerinfo['local_path'] - else: - lo.local_path = _reduce_canon_path(os.path.join(ls.apiurl, layerinfo['local_path'])) - - if not os.path.exists(lo.local_path): - logger.error("Local layer path %s must exists. Are you trying to import a layer that does not exist ? Check your local toasterconf.json" % lo.local_path) - - if layerinfo['vcs_url'].startswith("remote:"): - lo.vcs_url = _read_git_url_from_local_repository(layerinfo['vcs_url']) - if lo.vcs_url is None: - logger.error("The toaster config file references the local git repo, but Toaster cannot detect it.\nYour local configuration for layer %s is invalid. Make sure that the toasterconf.json file is correct." % layerinfo['name']) - - if lo.vcs_url is None: - lo.vcs_url = layerinfo['vcs_url'] - - if 'layer_index_url' in layerinfo: - lo.layer_index_url = layerinfo['layer_index_url'] - lo.save() - - for branch in layerbranches: - lvo, created = Layer_Version.objects.get_or_create(layer_source = ls, - up_branch = branch, - commit = branch.name, - layer = lo) - lvo.dirpath = layerinfo['dirpath'] - lvo.save() - # set releases - for ri in data['releases']: - bvo = BitbakeVersion.objects.get(name = ri['bitbake']) - assert bvo is not None - - ro, created = Release.objects.get_or_create(name = ri['name'], bitbake_version = bvo, branch_name = ri['branch']) - ro.description = ri['description'] - ro.helptext = ri['helptext'] - ro.save() - - # save layer source priority for release - for ls_name in ri['layersourcepriority'].keys(): - rlspo, created = ReleaseLayerSourcePriority.objects.get_or_create(release = ro, layer_source = LayerSource.objects.get(name=ls_name)) - rlspo.priority = ri['layersourcepriority'][ls_name] - rlspo.save() - - for dli in ri['defaultlayers']: - # find layers with the same name - ReleaseDefaultLayer.objects.get_or_create( release = ro, layer_name = dli) - - # set default release - if ToasterSetting.objects.filter(name = "DEFAULT_RELEASE").count() > 0: - ToasterSetting.objects.filter(name = "DEFAULT_RELEASE").update(value = data['defaultrelease']) - else: - ToasterSetting.objects.create(name = "DEFAULT_RELEASE", value = data['defaultrelease']) - - # set default config variables - for configname in data['config']: - if ToasterSetting.objects.filter(name = "DEFCONF_" + configname).count() > 0: - ToasterSetting.objects.filter(name = "DEFCONF_" + configname).update(value = data['config'][configname]) - else: - ToasterSetting.objects.create(name = "DEFCONF_" + configname, value = data['config'][configname]) - - - def handle(self, *args, **options): - if len(args) == 0: - raise CommandError("Need a path to the toasterconf.json file") - filepath = args[0] - self._import_layer_config(filepath) - - - diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py index 27289be5f..7f7a5a955 100644 --- a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py @@ -1,31 +1,27 @@ -from django.core.management.base import NoArgsCommand, CommandError +from django.core.management.base import NoArgsCommand from django.db import transaction from django.db.models import Q from bldcontrol.bbcontroller import getBuildEnvironmentController -from bldcontrol.bbcontroller import ShellCmdException, BuildSetupException from bldcontrol.models import BuildRequest, BuildEnvironment from bldcontrol.models import BRError, BRVariable -from orm.models import Build, ToasterSetting, LogMessage, Target +from orm.models import Build, LogMessage, Target -import os import logging -import time -import sys import traceback +import signal logger = logging.getLogger("toaster") class Command(NoArgsCommand): - args = "" - help = "Schedules and executes build requests as possible." - "Does not return (interrupt with Ctrl-C)" - + args = "" + help = "Schedules and executes build requests as possible. "\ + "Does not return (interrupt with Ctrl-C)" @transaction.atomic def _selectBuildEnvironment(self): - bec = getBuildEnvironmentController(lock = BuildEnvironment.LOCK_FREE) + bec = getBuildEnvironmentController(lock=BuildEnvironment.LOCK_FREE) bec.be.lock = BuildEnvironment.LOCK_LOCK bec.be.save() return bec @@ -54,8 +50,8 @@ class Command(NoArgsCommand): logger.debug("runbuilds: No build env") return - logger.debug("runbuilds: starting build %s, environment %s" % \ - (str(br).decode('utf-8'), bec.be)) + logger.info("runbuilds: starting build %s, environment %s" % \ + (br, bec.be)) # let the build request know where it is being executed br.environment = bec.be @@ -68,24 +64,22 @@ class Command(NoArgsCommand): except Exception as e: logger.error("runbuilds: Error launching build %s" % e) - traceback.print_exc(e) + traceback.print_exc() if "[Errno 111] Connection refused" in str(e): # Connection refused, read toaster_server.out errmsg = bec.readServerLogFile() else: errmsg = str(e) - BRError.objects.create(req = br, - errtype = str(type(e)), - errmsg = errmsg, - traceback = traceback.format_exc(e)) + BRError.objects.create(req=br, errtype=str(type(e)), errmsg=errmsg, + traceback=traceback.format_exc()) br.state = BuildRequest.REQ_FAILED br.save() bec.be.lock = BuildEnvironment.LOCK_FREE bec.be.save() def archive(self): - for br in BuildRequest.objects.filter(state = BuildRequest.REQ_ARCHIVE): + for br in BuildRequest.objects.filter(state=BuildRequest.REQ_ARCHIVE): if br.build == None: br.state = BuildRequest.REQ_FAILED else: @@ -102,14 +96,13 @@ class Command(NoArgsCommand): BuildRequest.REQ_COMPLETED, BuildRequest.REQ_CANCELLING]) & Q(lock=BuildEnvironment.LOCK_LOCK) & - Q(updated__lt=timezone.now() - timedelta(seconds = 30)) + Q(updated__lt=timezone.now() - timedelta(seconds=30)) ).update(lock=BuildEnvironment.LOCK_FREE) # update all Builds that were in progress and failed to start - for br in BuildRequest.objects.filter( - state=BuildRequest.REQ_FAILED, - build__outcome=Build.IN_PROGRESS): + for br in BuildRequest.objects.filter(state=BuildRequest.REQ_FAILED, + build__outcome=Build.IN_PROGRESS): # transpose the launch errors in ToasterExceptions br.build.outcome = Build.FAILED for brerror in br.brerror_set.all(): @@ -126,7 +119,7 @@ class Command(NoArgsCommand): # update all BuildRequests without a build created - for br in BuildRequest.objects.filter(build = None): + for br in BuildRequest.objects.filter(build=None): br.build = Build.objects.create(project=br.project, completed_on=br.updated, started_on=br.created) @@ -151,29 +144,34 @@ class Command(NoArgsCommand): # Make sure the LOCK is removed for builds which have been fully # cancelled - for br in BuildRequest.objects.filter( - Q(build__outcome=Build.CANCELLED) & - Q(state=BuildRequest.REQ_CANCELLING) & - ~Q(environment=None)): + for br in BuildRequest.objects.filter(\ + Q(build__outcome=Build.CANCELLED) & + Q(state=BuildRequest.REQ_CANCELLING) & + ~Q(environment=None)): br.environment.lock = BuildEnvironment.LOCK_FREE br.environment.save() + def runbuild(self): + try: + self.cleanup() + except Exception as e: + logger.warn("runbuilds: cleanup exception %s" % str(e)) - def handle_noargs(self, **options): - while True: - try: - self.cleanup() - except Exception as e: - logger.warn("runbuilds: cleanup exception %s" % str(e)) + try: + self.archive() + except Exception as e: + logger.warn("runbuilds: archive exception %s" % str(e)) - try: - self.archive() - except Exception as e: - logger.warn("runbuilds: archive exception %s" % str(e)) + try: + self.schedule() + except Exception as e: + logger.warn("runbuilds: schedule exception %s" % str(e)) - try: - self.schedule() - except Exception as e: - logger.warn("runbuilds: schedule exception %s" % str(e)) + def handle_noargs(self, **options): + self.runbuild() + + signal.signal(signal.SIGUSR1, lambda sig, frame: None) - time.sleep(1) + while True: + signal.pause() + self.runbuild() diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/migrations/0004_auto_20160523_1446.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/migrations/0004_auto_20160523_1446.py new file mode 100644 index 000000000..3d9062954 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/migrations/0004_auto_20160523_1446.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bldcontrol', '0003_add_cancelling_state'), + ] + + operations = [ + migrations.AlterField( + model_name='buildenvironment', + name='bbstate', + field=models.IntegerField(default=0, choices=[(0, 'stopped'), (1, 'started')]), + ), + migrations.AlterField( + model_name='buildenvironment', + name='betype', + field=models.IntegerField(choices=[(0, 'local')]), + ), + migrations.AlterField( + model_name='buildenvironment', + name='lock', + field=models.IntegerField(default=0, choices=[(0, 'free'), (1, 'lock'), (2, 'running')]), + ), + migrations.AlterField( + model_name='buildrequest', + name='state', + field=models.IntegerField(default=0, choices=[(0, 'created'), (1, 'queued'), (2, 'in progress'), (3, 'completed'), (4, 'failed'), (5, 'deleted'), (6, 'cancelling'), (7, 'archive')]), + ), + ] diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/migrations/0005_reorder_buildrequest_states.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/migrations/0005_reorder_buildrequest_states.py new file mode 100644 index 000000000..4bb951776 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/migrations/0005_reorder_buildrequest_states.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bldcontrol', '0004_auto_20160523_1446'), + ] + + operations = [ + migrations.AlterField( + model_name='buildrequest', + name='state', + field=models.IntegerField(choices=[(0, 'created'), (1, 'queued'), (2, 'in progress'), (3, 'failed'), (4, 'deleted'), (5, 'cancelling'), (6, 'completed'), (7, 'archive')], default=0), + ), + ] diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/migrations/0006_brlayer_local_source_dir.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/migrations/0006_brlayer_local_source_dir.py new file mode 100644 index 000000000..2460002f0 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/migrations/0006_brlayer_local_source_dir.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bldcontrol', '0005_reorder_buildrequest_states'), + ] + + operations = [ + migrations.AddField( + model_name='brlayer', + name='local_source_dir', + field=models.CharField(max_length=254, null=True), + ), + ] diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/migrations/0007_brlayers_optional_gitinfo.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/migrations/0007_brlayers_optional_gitinfo.py new file mode 100644 index 000000000..4be42a4cf --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/migrations/0007_brlayers_optional_gitinfo.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bldcontrol', '0006_brlayer_local_source_dir'), + ] + + operations = [ + migrations.AlterField( + model_name='brlayer', + name='commit', + field=models.CharField(max_length=254, null=True), + ), + migrations.AlterField( + model_name='brlayer', + name='dirpath', + field=models.CharField(max_length=254, null=True), + ), + migrations.AlterField( + model_name='brlayer', + name='giturl', + field=models.CharField(max_length=254, null=True), + ), + ] diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/models.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/models.py index cb49a58c4..409614b9e 100644 --- a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/models.py +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/models.py @@ -1,7 +1,7 @@ from __future__ import unicode_literals from django.db import models from django.core.validators import MaxValueValidator, MinValueValidator -from django.utils.encoding import force_bytes +from django.utils.encoding import force_text from orm.models import Project, ProjectLayer, ProjectVariable, ProjectTarget, Build, Layer_Version import logging @@ -63,20 +63,20 @@ class BuildRequest(models.Model): REQ_CREATED = 0 REQ_QUEUED = 1 REQ_INPROGRESS = 2 - REQ_COMPLETED = 3 - REQ_FAILED = 4 - REQ_DELETED = 5 - REQ_CANCELLING = 6 + REQ_FAILED = 3 + REQ_DELETED = 4 + REQ_CANCELLING = 5 + REQ_COMPLETED = 6 REQ_ARCHIVE = 7 REQUEST_STATE = ( (REQ_CREATED, "created"), (REQ_QUEUED, "queued"), (REQ_INPROGRESS, "in progress"), - (REQ_COMPLETED, "completed"), (REQ_FAILED, "failed"), (REQ_DELETED, "deleted"), (REQ_CANCELLING, "cancelling"), + (REQ_COMPLETED, "completed"), (REQ_ARCHIVE, "archive"), ) @@ -91,18 +91,18 @@ class BuildRequest(models.Model): def __init__(self, *args, **kwargs): super(BuildRequest, self).__init__(*args, **kwargs) - # Save the old state incase it's about to be modified + # Save the old state in case it's about to be modified self.old_state = self.state def save(self, *args, **kwargs): # Check that the state we're trying to set is not going backwards # e.g. from REQ_FAILED to REQ_INPROGRESS if self.old_state != self.state and self.old_state > self.state: - logger.warn("Invalid state change requested: " - "Cannot go from %s to %s - ignoring request" % - (BuildRequest.REQUEST_STATE[self.old_state][1], - BuildRequest.REQUEST_STATE[self.state][1]) - ) + logger.warning("Invalid state change requested: " + "Cannot go from %s to %s - ignoring request" % + (BuildRequest.REQUEST_STATE[self.old_state][1], + BuildRequest.REQUEST_STATE[self.state][1]) + ) # Set property back to the old value self.state = self.old_state return @@ -121,17 +121,19 @@ class BuildRequest(models.Model): return self.brvariable_set.get(name="MACHINE").value def __str__(self): - return force_bytes('%s %s' % (self.project, self.get_state_display())) + return force_text('%s %s' % (self.project, self.get_state_display())) # These tables specify the settings for running an actual build. # They MUST be kept in sync with the tables in orm.models.Project* + class BRLayer(models.Model): - req = models.ForeignKey(BuildRequest) - name = models.CharField(max_length = 100) - giturl = models.CharField(max_length = 254) - commit = models.CharField(max_length = 254) - dirpath = models.CharField(max_length = 254) + req = models.ForeignKey(BuildRequest) + name = models.CharField(max_length=100) + giturl = models.CharField(max_length=254, null=True) + local_source_dir = models.CharField(max_length=254, null=True) + commit = models.CharField(max_length=254, null=True) + dirpath = models.CharField(max_length=254, null=True) layer_version = models.ForeignKey(Layer_Version, null=True) class BRBitbake(models.Model): diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/tests.py b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/tests.py index f20cc7d4b..475ac0a16 100644 --- a/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/tests.py +++ b/import-layers/yocto-poky/bitbake/lib/toaster/bldcontrol/tests.py @@ -53,7 +53,7 @@ class BEControllerTests(object): # setting layers, skip any layer info bc.setLayers(BITBAKE_LAYER, POKY_LAYERS) except NotImplementedError: - print "Test skipped due to command not implemented yet" + print("Test skipped due to command not implemented yet") return True # We are ok with the exception as we're handling the git already exists except BuildSetupException: @@ -79,7 +79,7 @@ class BEControllerTests(object): # setting layers, skip any layer info layerSet = bc.setLayers(BITBAKE_LAYER, POKY_LAYERS) except NotImplementedError: - print "Test skipped due to command not implemented yet" + print("Test skipped due to command not implemented yet") return True # We are ok with the exception as we're handling the git already exists except BuildSetupException: @@ -139,22 +139,3 @@ class RunBuildsCommandTests(TestCase): self.assertTrue(br.state == BuildRequest.REQ_INPROGRESS, "Request is not updated") # no more selections possible here self.assertRaises(IndexError, command._selectBuildRequest) - - -class UtilityTests(TestCase): - def test_reduce_path(self): - from bldcontrol.management.commands.loadconf import _reduce_canon_path, _get_id_for_sourcetype - - self.assertTrue( _reduce_canon_path("/") == "/") - self.assertTrue( _reduce_canon_path("/home/..") == "/") - self.assertTrue( _reduce_canon_path("/home/../ana") == "/ana") - self.assertTrue( _reduce_canon_path("/home/../ana/..") == "/") - self.assertTrue( _reduce_canon_path("/home/ana/mihai/../maria") == "/home/ana/maria") - - def test_get_id_for_sorucetype(self): - from bldcontrol.management.commands.loadconf import _reduce_canon_path, _get_id_for_sourcetype - self.assertTrue( _get_id_for_sourcetype("layerindex") == 1) - self.assertTrue( _get_id_for_sourcetype("local") == 0) - self.assertTrue( _get_id_for_sourcetype("imported") == 2) - with self.assertRaises(Exception): - _get_id_for_sourcetype("unknown") |
