diff options
Diffstat (limited to 'poky/bitbake/lib/toaster/orm/models.py')
| -rw-r--r-- | poky/bitbake/lib/toaster/orm/models.py | 1832 |
1 files changed, 1832 insertions, 0 deletions
diff --git a/poky/bitbake/lib/toaster/orm/models.py b/poky/bitbake/lib/toaster/orm/models.py new file mode 100644 index 000000000..3a7dff8ca --- /dev/null +++ b/poky/bitbake/lib/toaster/orm/models.py @@ -0,0 +1,1832 @@ +# +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# BitBake Toaster Implementation +# +# Copyright (C) 2013 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from __future__ import unicode_literals + +from django.db import models, IntegrityError, DataError +from django.db.models import F, Q, Sum, Count +from django.utils import timezone +from django.utils.encoding import force_bytes + +from django.core.urlresolvers import reverse + +from django.core import validators +from django.conf import settings +import django.db.models.signals + +import sys +import os +import re +import itertools +from signal import SIGUSR1 + + +import logging +logger = logging.getLogger("toaster") + +if 'sqlite' in settings.DATABASES['default']['ENGINE']: + from django.db import transaction, OperationalError + from time import sleep + + _base_save = models.Model.save + def save(self, *args, **kwargs): + while True: + try: + with transaction.atomic(): + return _base_save(self, *args, **kwargs) + except OperationalError as err: + if 'database is locked' in str(err): + logger.warning("%s, model: %s, args: %s, kwargs: %s", + err, self.__class__, args, kwargs) + sleep(0.5) + continue + raise + + models.Model.save = save + + # HACK: Monkey patch Django to fix 'database is locked' issue + + from django.db.models.query import QuerySet + _base_insert = QuerySet._insert + def _insert(self, *args, **kwargs): + with transaction.atomic(using=self.db, savepoint=False): + return _base_insert(self, *args, **kwargs) + QuerySet._insert = _insert + + from django.utils import six + def _create_object_from_params(self, lookup, params): + """ + Tries to create an object using passed params. + Used by get_or_create and update_or_create + """ + try: + obj = self.create(**params) + return obj, True + except (IntegrityError, DataError): + exc_info = sys.exc_info() + try: + return self.get(**lookup), False + except self.model.DoesNotExist: + pass + six.reraise(*exc_info) + + QuerySet._create_object_from_params = _create_object_from_params + + # end of HACK + +class GitURLValidator(validators.URLValidator): + import re + regex = re.compile( + r'^(?:ssh|git|http|ftp)s?://' # http:// or https:// + r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain... + r'localhost|' # localhost... + r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|' # ...or ipv4 + r'\[?[A-F0-9]*:[A-F0-9:]+\]?)' # ...or ipv6 + r'(?::\d+)?' # optional port + r'(?:/?|[/?]\S+)$', re.IGNORECASE) + +def GitURLField(**kwargs): + r = models.URLField(**kwargs) + for i in range(len(r.validators)): + if isinstance(r.validators[i], validators.URLValidator): + r.validators[i] = GitURLValidator() + return r + + +class ToasterSetting(models.Model): + name = models.CharField(max_length=63) + helptext = models.TextField() + value = models.CharField(max_length=255) + + def __unicode__(self): + return "Setting %s = %s" % (self.name, self.value) + + +class ProjectManager(models.Manager): + def create_project(self, name, release): + if release is not None: + prj = self.model(name=name, + bitbake_version=release.bitbake_version, + release=release) + else: + prj = self.model(name=name, + bitbake_version=None, + release=None) + + prj.save() + + for defaultconf in ToasterSetting.objects.filter( + name__startswith="DEFCONF_"): + name = defaultconf.name[8:] + ProjectVariable.objects.create(project=prj, + name=name, + value=defaultconf.value) + + if release is None: + return prj + + for rdl in release.releasedefaultlayer_set.all(): + lv = Layer_Version.objects.filter( + layer__name=rdl.layer_name, + release=release).first() + + if lv: + ProjectLayer.objects.create(project=prj, + layercommit=lv, + optional=False) + else: + logger.warning("Default project layer %s not found" % + rdl.layer_name) + + return prj + + # return single object with is_default = True + def get_or_create_default_project(self): + projects = super(ProjectManager, self).filter(is_default=True) + + if len(projects) > 1: + raise Exception('Inconsistent project data: multiple ' + + 'default projects (i.e. with is_default=True)') + elif len(projects) < 1: + options = { + 'name': 'Command line builds', + 'short_description': + 'Project for builds started outside Toaster', + 'is_default': True + } + project = Project.objects.create(**options) + project.save() + + return project + else: + return projects[0] + + +class Project(models.Model): + search_allowed_fields = ['name', 'short_description', 'release__name', + 'release__branch_name'] + name = models.CharField(max_length=100) + short_description = models.CharField(max_length=50, blank=True) + bitbake_version = models.ForeignKey('BitbakeVersion', null=True) + release = models.ForeignKey("Release", null=True) + created = models.DateTimeField(auto_now_add=True) + updated = models.DateTimeField(auto_now=True) + # This is a horrible hack; since Toaster has no "User" model available when + # running in interactive mode, we can't reference the field here directly + # Instead, we keep a possible null reference to the User id, + # as not to force + # hard links to possibly missing models + user_id = models.IntegerField(null=True) + objects = ProjectManager() + + # set to True for the project which is the default container + # for builds initiated by the command line etc. + is_default= models.BooleanField(default=False) + + def __unicode__(self): + return "%s (Release %s, BBV %s)" % (self.name, self.release, self.bitbake_version) + + def get_current_machine_name(self): + try: + return self.projectvariable_set.get(name="MACHINE").value + except (ProjectVariable.DoesNotExist,IndexError): + return None; + + def get_number_of_builds(self): + """Return the number of builds which have ended""" + + return self.build_set.exclude( + Q(outcome=Build.IN_PROGRESS) | + Q(outcome=Build.CANCELLED) + ).count() + + def get_last_build_id(self): + try: + return Build.objects.filter( project = self.id ).order_by('-completed_on')[0].id + except (Build.DoesNotExist,IndexError): + return( -1 ) + + def get_last_outcome(self): + build_id = self.get_last_build_id() + if (-1 == build_id): + return( "" ) + try: + return Build.objects.filter( id = build_id )[ 0 ].outcome + except (Build.DoesNotExist,IndexError): + return( "not_found" ) + + def get_last_target(self): + build_id = self.get_last_build_id() + if (-1 == build_id): + return( "" ) + try: + return Target.objects.filter(build = build_id)[0].target + except (Target.DoesNotExist,IndexError): + return( "not_found" ) + + def get_last_errors(self): + build_id = self.get_last_build_id() + if (-1 == build_id): + return( 0 ) + try: + return Build.objects.filter(id = build_id)[ 0 ].errors.count() + except (Build.DoesNotExist,IndexError): + return( "not_found" ) + + def get_last_warnings(self): + build_id = self.get_last_build_id() + if (-1 == build_id): + return( 0 ) + try: + return Build.objects.filter(id = build_id)[ 0 ].warnings.count() + except (Build.DoesNotExist,IndexError): + return( "not_found" ) + + def get_last_build_extensions(self): + """ + Get list of file name extensions for images produced by the most + recent build + """ + last_build = Build.objects.get(pk = self.get_last_build_id()) + return last_build.get_image_file_extensions() + + def get_last_imgfiles(self): + build_id = self.get_last_build_id() + if (-1 == build_id): + return( "" ) + try: + return Variable.objects.filter(build = build_id, variable_name = "IMAGE_FSTYPES")[ 0 ].variable_value + except (Variable.DoesNotExist,IndexError): + return( "not_found" ) + + def get_all_compatible_layer_versions(self): + """ Returns Queryset of all Layer_Versions which are compatible with + this project""" + queryset = None + + # guard on release, as it can be null + if self.release: + queryset = Layer_Version.objects.filter( + (Q(release=self.release) & + Q(build=None) & + Q(project=None)) | + Q(project=self)) + else: + queryset = Layer_Version.objects.none() + + return queryset + + def get_project_layer_versions(self, pk=False): + """ Returns the Layer_Versions currently added to this project """ + layer_versions = self.projectlayer_set.all().values_list('layercommit', + flat=True) + + if pk is False: + return Layer_Version.objects.filter(pk__in=layer_versions) + else: + return layer_versions + + + def get_available_machines(self): + """ Returns QuerySet of all Machines which are provided by the + Layers currently added to the Project """ + queryset = Machine.objects.filter( + layer_version__in=self.get_project_layer_versions()) + + return queryset + + def get_all_compatible_machines(self): + """ Returns QuerySet of all the compatible machines available to the + project including ones from Layers not currently added """ + queryset = Machine.objects.filter( + layer_version__in=self.get_all_compatible_layer_versions()) + + return queryset + + def get_available_distros(self): + """ Returns QuerySet of all Distros which are provided by the + Layers currently added to the Project """ + queryset = Distro.objects.filter( + layer_version__in=self.get_project_layer_versions()) + + return queryset + + def get_all_compatible_distros(self): + """ Returns QuerySet of all the compatible Wind River distros available to the + project including ones from Layers not currently added """ + queryset = Distro.objects.filter( + layer_version__in=self.get_all_compatible_layer_versions()) + + return queryset + + def get_available_recipes(self): + """ Returns QuerySet of all the recipes that are provided by layers + added to this project """ + queryset = Recipe.objects.filter( + layer_version__in=self.get_project_layer_versions()) + + return queryset + + def get_all_compatible_recipes(self): + """ Returns QuerySet of all the compatible Recipes available to the + project including ones from Layers not currently added """ + queryset = Recipe.objects.filter( + layer_version__in=self.get_all_compatible_layer_versions()).exclude(name__exact='') + + return queryset + + def schedule_build(self): + + from bldcontrol.models import BuildRequest, BRTarget, BRLayer + from bldcontrol.models import BRBitbake, BRVariable + + try: + now = timezone.now() + build = Build.objects.create(project=self, + completed_on=now, + started_on=now) + + br = BuildRequest.objects.create(project=self, + state=BuildRequest.REQ_QUEUED, + build=build) + BRBitbake.objects.create(req=br, + giturl=self.bitbake_version.giturl, + commit=self.bitbake_version.branch, + dirpath=self.bitbake_version.dirpath) + + for t in self.projecttarget_set.all(): + BRTarget.objects.create(req=br, target=t.target, task=t.task) + Target.objects.create(build=br.build, target=t.target, + task=t.task) + # If we're about to build a custom image recipe make sure + # that layer is currently in the project before we create the + # BRLayer objects + customrecipe = CustomImageRecipe.objects.filter( + name=t.target, + project=self).first() + if customrecipe: + ProjectLayer.objects.get_or_create( + project=self, + layercommit=customrecipe.layer_version, + optional=False) + + for l in self.projectlayer_set.all().order_by("pk"): + commit = l.layercommit.get_vcs_reference() + logger.debug("Adding layer to build %s" % + l.layercommit.layer.name) + BRLayer.objects.create( + req=br, + name=l.layercommit.layer.name, + giturl=l.layercommit.layer.vcs_url, + commit=commit, + dirpath=l.layercommit.dirpath, + layer_version=l.layercommit, + local_source_dir=l.layercommit.layer.local_source_dir + ) + + for v in self.projectvariable_set.all(): + BRVariable.objects.create(req=br, name=v.name, value=v.value) + + try: + br.build.machine = self.projectvariable_set.get( + name='MACHINE').value + br.build.save() + except ProjectVariable.DoesNotExist: + pass + + br.save() + signal_runbuilds() + + except Exception: + # revert the build request creation since we're not done cleanly + br.delete() + raise + return br + +class Build(models.Model): + SUCCEEDED = 0 + FAILED = 1 + IN_PROGRESS = 2 + CANCELLED = 3 + + BUILD_OUTCOME = ( + (SUCCEEDED, 'Succeeded'), + (FAILED, 'Failed'), + (IN_PROGRESS, 'In Progress'), + (CANCELLED, 'Cancelled'), + ) + + search_allowed_fields = ['machine', 'cooker_log_path', "target__target", "target__target_image_file__file_name"] + + project = models.ForeignKey(Project) # must have a project + machine = models.CharField(max_length=100) + distro = models.CharField(max_length=100) + distro_version = models.CharField(max_length=100) + started_on = models.DateTimeField() + completed_on = models.DateTimeField() + outcome = models.IntegerField(choices=BUILD_OUTCOME, default=IN_PROGRESS) + cooker_log_path = models.CharField(max_length=500) + build_name = models.CharField(max_length=100, default='') + bitbake_version = models.CharField(max_length=50) + + # number of recipes to parse for this build + recipes_to_parse = models.IntegerField(default=1) + + # number of recipes parsed so far for this build + recipes_parsed = models.IntegerField(default=1) + + # number of repos to clone for this build + repos_to_clone = models.IntegerField(default=1) + + # number of repos cloned so far for this build (default off) + repos_cloned = models.IntegerField(default=1) + + @staticmethod + def get_recent(project=None): + """ + Return recent builds as a list; if project is set, only return + builds for that project + """ + + builds = Build.objects.all() + + if project: + builds = builds.filter(project=project) + + finished_criteria = \ + Q(outcome=Build.SUCCEEDED) | \ + Q(outcome=Build.FAILED) | \ + Q(outcome=Build.CANCELLED) + + recent_builds = list(itertools.chain( + builds.filter(outcome=Build.IN_PROGRESS).order_by("-started_on"), + builds.filter(finished_criteria).order_by("-completed_on")[:3] + )) + + # add percentage done property to each build; this is used + # to show build progress in mrb_section.html + for build in recent_builds: + build.percentDone = build.completeper() + build.outcomeText = build.get_outcome_text() + + return recent_builds + + def started(self): + """ + As build variables are only added for a build when its BuildStarted event + is received, a build with no build variables is counted as + "in preparation" and not properly started yet. This method + will return False if a build has no build variables (it never properly + started), or True otherwise. + + Note that this is a temporary workaround for the fact that we don't + have a fine-grained state variable on a build which would allow us + to record "in progress" (BuildStarted received) vs. "in preparation". + """ + variables = Variable.objects.filter(build=self) + return len(variables) > 0 + + def completeper(self): + tf = Task.objects.filter(build = self) + tfc = tf.count() + if tfc > 0: + completeper = tf.exclude(outcome=Task.OUTCOME_NA).count()*100 // tfc + else: + completeper = 0 + return completeper + + def eta(self): + eta = timezone.now() + completeper = self.completeper() + if self.completeper() > 0: + eta += ((eta - self.started_on)*(100-completeper))/completeper + return eta + + def has_images(self): + """ + Returns True if at least one of the targets for this build has an + image file associated with it, False otherwise + """ + targets = Target.objects.filter(build_id=self.id) + has_images = False + for target in targets: + if target.has_images(): + has_images = True + break + return has_images + + def has_image_recipes(self): + """ + Returns True if a build has any targets which were built from + image recipes. + """ + image_recipes = self.get_image_recipes() + return len(image_recipes) > 0 + + def get_image_file_extensions(self): + """ + Get string of file name extensions for images produced by this build; + note that this is the actual list of extensions stored on Target objects + for this build, and not the value of IMAGE_FSTYPES. + + Returns comma-separated string, e.g. "vmdk, ext4" + """ + extensions = [] + + targets = Target.objects.filter(build_id = self.id) + for target in targets: + if not target.is_image: + continue + + target_image_files = Target_Image_File.objects.filter( + target_id=target.id) + + for target_image_file in target_image_files: + extensions.append(target_image_file.suffix) + + extensions = list(set(extensions)) + extensions.sort() + + return ', '.join(extensions) + + def get_image_fstypes(self): + """ + Get the IMAGE_FSTYPES variable value for this build as a de-duplicated + list of image file suffixes. + """ + image_fstypes = Variable.objects.get( + build=self, variable_name='IMAGE_FSTYPES').variable_value + return list(set(re.split(r' {1,}', image_fstypes))) + + def get_sorted_target_list(self): + tgts = Target.objects.filter(build_id = self.id).order_by( 'target' ); + return( tgts ); + + def get_recipes(self): + """ + Get the recipes related to this build; + note that the related layer versions and layers are also prefetched + by this query, as this queryset can be sorted by these objects in the + build recipes view; prefetching them here removes the need + for another query in that view + """ + layer_versions = Layer_Version.objects.filter(build=self) + criteria = Q(layer_version__id__in=layer_versions) + return Recipe.objects.filter(criteria) \ + .select_related('layer_version', 'layer_version__layer') + + def get_image_recipes(self): + """ + Returns a list of image Recipes (custom and built-in) related to this + build, sorted by name; note that this has to be done in two steps, as + there's no way to get all the custom image recipes and image recipes + in one query + """ + custom_image_recipes = self.get_custom_image_recipes() + custom_image_recipe_names = custom_image_recipes.values_list('name', flat=True) + + not_custom_image_recipes = ~Q(name__in=custom_image_recipe_names) & \ + Q(is_image=True) + + built_image_recipes = self.get_recipes().filter(not_custom_image_recipes) + + # append to the custom image recipes and sort + customisable_image_recipes = list( + itertools.chain(custom_image_recipes, built_image_recipes) + ) + + return sorted(customisable_image_recipes, key=lambda recipe: recipe.name) + + def get_custom_image_recipes(self): + """ + Returns a queryset of CustomImageRecipes related to this build, + sorted by name + """ + built_recipe_names = self.get_recipes().values_list('name', flat=True) + criteria = Q(name__in=built_recipe_names) & Q(project=self.project) + queryset = CustomImageRecipe.objects.filter(criteria).order_by('name') + return queryset + + def get_outcome_text(self): + return Build.BUILD_OUTCOME[int(self.outcome)][1] + + @property + def failed_tasks(self): + """ Get failed tasks for the build """ + tasks = self.task_build.all() + return tasks.filter(order__gt=0, outcome=Task.OUTCOME_FAILED) + + @property + def errors(self): + return (self.logmessage_set.filter(level=LogMessage.ERROR) | + self.logmessage_set.filter(level=LogMessage.EXCEPTION) | + self.logmessage_set.filter(level=LogMessage.CRITICAL)) + + @property + def warnings(self): + return self.logmessage_set.filter(level=LogMessage.WARNING) + + @property + def timespent(self): + return self.completed_on - self.started_on + + @property + def timespent_seconds(self): + return self.timespent.total_seconds() + + @property + def target_labels(self): + """ + Sorted (a-z) "target1:task, target2, target3" etc. string for all + targets in this build + """ + targets = self.target_set.all() + target_labels = [target.target + + (':' + target.task if target.task else '') + for target in targets] + target_labels.sort() + + return target_labels + + def get_buildrequest(self): + buildrequest = None + if hasattr(self, 'buildrequest'): + buildrequest = self.buildrequest + return buildrequest + + def is_queued(self): + from bldcontrol.models import BuildRequest + buildrequest = self.get_buildrequest() + if buildrequest: + return buildrequest.state == BuildRequest.REQ_QUEUED + else: + return False + + def is_cancelling(self): + from bldcontrol.models import BuildRequest + buildrequest = self.get_buildrequest() + if buildrequest: + return self.outcome == Build.IN_PROGRESS and \ + buildrequest.state == BuildRequest.REQ_CANCELLING + else: + return False + + def is_cloning(self): + """ + True if the build is still cloning repos + """ + return self.outcome == Build.IN_PROGRESS and \ + self.repos_cloned < self.repos_to_clone + + def is_parsing(self): + """ + True if the build is still parsing recipes + """ + return self.outcome == Build.IN_PROGRESS and \ + self.recipes_parsed < self.recipes_to_parse + + def is_starting(self): + """ + True if the build has no completed tasks yet and is still just starting + tasks. + + Note that the mechanism for testing whether a Task is "done" is whether + its outcome field is set, as per the completeper() method. + """ + return self.outcome == Build.IN_PROGRESS and \ + self.task_build.exclude(outcome=Task.OUTCOME_NA).count() == 0 + + + def get_state(self): + """ + Get the state of the build; one of 'Succeeded', 'Failed', 'In Progress', + 'Cancelled' (Build outcomes); or 'Queued', 'Cancelling' (states + dependent on the BuildRequest state). + + This works around the fact that we have BuildRequest states as well + as Build states, but really we just want to know the state of the build. + """ + if self.is_cancelling(): + return 'Cancelling'; + elif self.is_queued(): + return 'Queued' + elif self.is_cloning(): + return 'Cloning' + elif self.is_parsing(): + return 'Parsing' + elif self.is_starting(): + return 'Starting' + else: + return self.get_outcome_text() + + def __str__(self): + return "%d %s %s" % (self.id, self.project, ",".join([t.target for t in self.target_set.all()])) + +class ProjectTarget(models.Model): + project = models.ForeignKey(Project) + target = models.CharField(max_length=100) + task = models.CharField(max_length=100, null=True) + +class Target(models.Model): + search_allowed_fields = ['target', 'file_name'] + build = models.ForeignKey(Build) + target = models.CharField(max_length=100) + task = models.CharField(max_length=100, null=True) + is_image = models.BooleanField(default = False) + image_size = models.IntegerField(default=0) + license_manifest_path = models.CharField(max_length=500, null=True) + package_manifest_path = models.CharField(max_length=500, null=True) + + def package_count(self): + return Target_Installed_Package.objects.filter(target_id__exact=self.id).count() + + def __unicode__(self): + return self.target + + def get_similar_targets(self): + """ + Get target sfor the same machine, task and target name + (e.g. 'core-image-minimal') from a successful build for this project + (but excluding this target). + + Note that we only look for targets built by this project because + projects can have different configurations from each other, and put + their artifacts in different directories. + + The possibility of error when retrieving candidate targets + is minimised by the fact that bitbake will rebuild artifacts if MACHINE + (or various other variables) change. In this case, there is no need to + clone artifacts from another target, as those artifacts will have + been re-generated for this target anyway. + """ + query = ~Q(pk=self.pk) & \ + Q(target=self.target) & \ + Q(build__machine=self.build.machine) & \ + Q(build__outcome=Build.SUCCEEDED) & \ + Q(build__project=self.build.project) + + return Target.objects.filter(query) + + def get_similar_target_with_image_files(self): + """ + Get the most recent similar target with Target_Image_Files associated + with it, for the purpose of cloning those files onto this target. + """ + similar_target = None + + candidates = self.get_similar_targets() + if candidates.count() == 0: + return similar_target + + task_subquery = Q(task=self.task) + + # we can look for a 'build' task if this task is a 'populate_sdk_ext' + # task, as the latter also creates images; and vice versa; note that + # 'build' targets can have their task set to ''; + # also note that 'populate_sdk' does not produce image files + image_tasks = [ + '', # aka 'build' + 'build', + 'image', + 'populate_sdk_ext' + ] + if self.task in image_tasks: + task_subquery = Q(task__in=image_tasks) + + # annotate with the count of files, to exclude any targets which + # don't have associated files + candidates = candidates.annotate(num_files=Count('target_image_file')) + + query = task_subquery & Q(num_files__gt=0) + + candidates = candidates.filter(query) + + if candidates.count() > 0: + candidates.order_by('build__completed_on') + similar_target = candidates.last() + + return similar_target + + def get_similar_target_with_sdk_files(self): + """ + Get the most recent similar target with TargetSDKFiles associated + with it, for the purpose of cloning those files onto this target. + """ + similar_target = None + + candidates = self.get_similar_targets() + if candidates.count() == 0: + return similar_target + + # annotate with the count of files, to exclude any targets which + # don't have associated files + candidates = candidates.annotate(num_files=Count('targetsdkfile')) + + query = Q(task=self.task) & Q(num_files__gt=0) + + candidates = candidates.filter(query) + + if candidates.count() > 0: + candidates.order_by('build__completed_on') + similar_target = candidates.last() + + return similar_target + + def clone_image_artifacts_from(self, target): + """ + Make clones of the Target_Image_Files and TargetKernelFile objects + associated with Target target, then associate them with this target. + + Note that for Target_Image_Files, we only want files from the previous + build whose suffix matches one of the suffixes defined in this + target's build's IMAGE_FSTYPES configuration variable. This prevents the + Target_Image_File object for an ext4 image being associated with a + target for a project which didn't produce an ext4 image (for example). + + Also sets the license_manifest_path and package_manifest_path + of this target to the same path as that of target being cloned from, as + the manifests are also build artifacts but are treated differently. + """ + + image_fstypes = self.build.get_image_fstypes() + + # filter out any image files whose suffixes aren't in the + # IMAGE_FSTYPES suffixes variable for this target's build + image_files = [target_image_file \ + for target_image_file in target.target_image_file_set.all() \ + if target_image_file.suffix in image_fstypes] + + for image_file in image_files: + image_file.pk = None + image_file.target = self + image_file.save() + + kernel_files = target.targetkernelfile_set.all() + for kernel_file in kernel_files: + kernel_file.pk = None + kernel_file.target = self + kernel_file.save() + + self.license_manifest_path = target.license_manifest_path + self.package_manifest_path = target.package_manifest_path + self.save() + + def clone_sdk_artifacts_from(self, target): + """ + Clone TargetSDKFile objects from target and associate them with this + target. + """ + sdk_files = target.targetsdkfile_set.all() + for sdk_file in sdk_files: + sdk_file.pk = None + sdk_file.target = self + sdk_file.save() + + def has_images(self): + """ + Returns True if this target has one or more image files attached to it. + """ + return self.target_image_file_set.all().count() > 0 + +# kernel artifacts for a target: bzImage and modules* +class TargetKernelFile(models.Model): + target = models.ForeignKey(Target) + file_name = models.FilePathField() + file_size = models.IntegerField() + + @property + def basename(self): + return os.path.basename(self.file_name) + +# SDK artifacts for a target: sh and manifest files +class TargetSDKFile(models.Model): + target = models.ForeignKey(Target) + file_name = models.FilePathField() + file_size = models.IntegerField() + + @property + def basename(self): + return os.path.basename(self.file_name) + +class Target_Image_File(models.Model): + # valid suffixes for image files produced by a build + SUFFIXES = { + 'btrfs', 'cpio', 'cpio.gz', 'cpio.lz4', 'cpio.lzma', 'cpio.xz', + 'cramfs', 'elf', 'ext2', 'ext2.bz2', 'ext2.gz', 'ext2.lzma', 'ext4', + 'ext4.gz', 'ext3', 'ext3.gz', 'hdddirect', 'hddimg', 'iso', 'jffs2', + 'jffs2.sum', 'multiubi', 'qcow2', 'squashfs', 'squashfs-lzo', + 'squashfs-xz', 'tar', 'tar.bz2', 'tar.gz', 'tar.lz4', 'tar.xz', 'ubi', + 'ubifs', 'vdi', 'vmdk', 'wic', 'wic.bmap', 'wic.bz2', 'wic.gz', 'wic.lzma' + } + + target = models.ForeignKey(Target) + file_name = models.FilePathField(max_length=254) + file_size = models.IntegerField() + + @property + def suffix(self): + """ + Suffix for image file, minus leading "." + """ + for suffix in Target_Image_File.SUFFIXES: + if self.file_name.endswith(suffix): + return suffix + + filename, suffix = os.path.splitext(self.file_name) + suffix = suffix.lstrip('.') + return suffix + +class Target_File(models.Model): + ITYPE_REGULAR = 1 + ITYPE_DIRECTORY = 2 + ITYPE_SYMLINK = 3 + ITYPE_SOCKET = 4 + ITYPE_FIFO = 5 + ITYPE_CHARACTER = 6 + ITYPE_BLOCK = 7 + ITYPES = ( (ITYPE_REGULAR ,'regular'), + ( ITYPE_DIRECTORY ,'directory'), + ( ITYPE_SYMLINK ,'symlink'), + ( ITYPE_SOCKET ,'socket'), + ( ITYPE_FIFO ,'fifo'), + ( ITYPE_CHARACTER ,'character'), + ( ITYPE_BLOCK ,'block'), + ) + + target = models.ForeignKey(Target) + path = models.FilePathField() + size = models.IntegerField() + inodetype = models.IntegerField(choices = ITYPES) + permission = models.CharField(max_length=16) + owner = models.CharField(max_length=128) + group = models.CharField(max_length=128) + directory = models.ForeignKey('Target_File', related_name="directory_set", null=True) + sym_target = models.ForeignKey('Target_File', related_name="symlink_set", null=True) + + +class Task(models.Model): + + SSTATE_NA = 0 + SSTATE_MISS = 1 + SSTATE_FAILED = 2 + SSTATE_RESTORED = 3 + + SSTATE_RESULT = ( + (SSTATE_NA, 'Not Applicable'), # For rest of tasks, but they still need checking. + (SSTATE_MISS, 'File not in cache'), # the sstate object was not found + (SSTATE_FAILED, 'Failed'), # there was a pkg, but the script failed + (SSTATE_RESTORED, 'Succeeded'), # successfully restored + ) + + CODING_NA = 0 + CODING_PYTHON = 2 + CODING_SHELL = 3 + + TASK_CODING = ( + (CODING_NA, 'N/A'), + (CODING_PYTHON, 'Python'), + (CODING_SHELL, 'Shell'), + ) + + OUTCOME_NA = -1 + OUTCOME_SUCCESS = 0 + OUTCOME_COVERED = 1 + OUTCOME_CACHED = 2 + OUTCOME_PREBUILT = 3 + OUTCOME_FAILED = 4 + OUTCOME_EMPTY = 5 + + TASK_OUTCOME = ( + (OUTCOME_NA, 'Not Available'), + (OUTCOME_SUCCESS, 'Succeeded'), + (OUTCOME_COVERED, 'Covered'), + (OUTCOME_CACHED, 'Cached'), + (OUTCOME_PREBUILT, 'Prebuilt'), + (OUTCOME_FAILED, 'Failed'), + (OUTCOME_EMPTY, 'Empty'), + ) + + TASK_OUTCOME_HELP = ( + (OUTCOME_SUCCESS, 'This task successfully completed'), + (OUTCOME_COVERED, 'This task did not run because its output is provided by another task'), + (OUTCOME_CACHED, 'This task restored output from the sstate-cache directory or mirrors'), + (OUTCOME_PREBUILT, 'This task did not run because its outcome was reused from a previous build'), + (OUTCOME_FAILED, 'This task did not complete'), + (OUTCOME_EMPTY, 'This task has no executable content'), + (OUTCOME_NA, ''), + ) + + search_allowed_fields = [ "recipe__name", "recipe__version", "task_name", "logfile" ] + + def __init__(self, *args, **kwargs): + super(Task, self).__init__(*args, **kwargs) + try: + self._helptext = HelpText.objects.get(key=self.task_name, area=HelpText.VARIABLE, build=self.build).text + except HelpText.DoesNotExist: + self._helptext = None + + def get_related_setscene(self): + return Task.objects.filter(task_executed=True, build = self.build, recipe = self.recipe, task_name=self.task_name+"_setscene") + + def get_outcome_text(self): + return Task.TASK_OUTCOME[int(self.outcome) + 1][1] + + def get_outcome_help(self): + return Task.TASK_OUTCOME_HELP[int(self.outcome)][1] + + def get_sstate_text(self): + if self.sstate_result==Task.SSTATE_NA: + return '' + else: + return Task.SSTATE_RESULT[int(self.sstate_result)][1] + + def get_executed_display(self): + if self.task_executed: + return "Executed" + return "Not Executed" + + def get_description(self): + return self._helptext + + build = models.ForeignKey(Build, related_name='task_build') + order = models.IntegerField(null=True) + task_executed = models.BooleanField(default=False) # True means Executed, False means Not/Executed + outcome = models.IntegerField(choices=TASK_OUTCOME, default=OUTCOME_NA) + sstate_checksum = models.CharField(max_length=100, blank=True) + path_to_sstate_obj = models.FilePathField(max_length=500, blank=True) + recipe = models.ForeignKey('Recipe', related_name='tasks') + task_name = models.CharField(max_length=100) + source_url = models.FilePathField(max_length=255, blank=True) + work_directory = models.FilePathField(max_length=255, blank=True) + script_type = models.IntegerField(choices=TASK_CODING, default=CODING_NA) + line_number = models.IntegerField(default=0) + + # start/end times + started = models.DateTimeField(null=True) + ended = models.DateTimeField(null=True) + + # in seconds; this is stored to enable sorting + elapsed_time = models.DecimalField(max_digits=8, decimal_places=2, null=True) + + # in bytes; note that disk_io is stored to enable sorting + disk_io = models.IntegerField(null=True) + disk_io_read = models.IntegerField(null=True) + disk_io_write = models.IntegerField(null=True) + + # in seconds + cpu_time_user = models.DecimalField(max_digits=8, decimal_places=2, null=True) + cpu_time_system = models.DecimalField(max_digits=8, decimal_places=2, null=True) + + sstate_result = models.IntegerField(choices=SSTATE_RESULT, default=SSTATE_NA) + message = models.CharField(max_length=240) + logfile = models.FilePathField(max_length=255, blank=True) + + outcome_text = property(get_outcome_text) + sstate_text = property(get_sstate_text) + + def __unicode__(self): + return "%d(%d) %s:%s" % (self.pk, self.build.pk, self.recipe.name, self.task_name) + + class Meta: + ordering = ('order', 'recipe' ,) + unique_together = ('build', 'recipe', 'task_name', ) + + +class Task_Dependency(models.Model): + task = models.ForeignKey(Task, related_name='task_dependencies_task') + depends_on = models.ForeignKey(Task, related_name='task_dependencies_depends') + +class Package(models.Model): + search_allowed_fields = ['name', 'version', 'revision', 'recipe__name', 'recipe__version', 'recipe__license', 'recipe__layer_version__layer__name', 'recipe__layer_version__branch', 'recipe__layer_version__commit', 'recipe__layer_version__local_path', 'installed_name'] + build = models.ForeignKey('Build', null=True) + recipe = models.ForeignKey('Recipe', null=True) + name = models.CharField(max_length=100) + installed_name = models.CharField(max_length=100, default='') + version = models.CharField(max_length=100, blank=True) + revision = models.CharField(max_length=32, blank=True) + summary = models.TextField(blank=True) + description = models.TextField(blank=True) + size = models.IntegerField(default=0) + installed_size = models.IntegerField(default=0) + section = models.CharField(max_length=80, blank=True) + license = models.CharField(max_length=80, blank=True) + + @property + def is_locale_package(self): + """ Returns True if this package is identifiable as a locale package """ + if self.name.find('locale') != -1: + return True + return False + + @property + def is_packagegroup(self): + """ Returns True is this package is identifiable as a packagegroup """ + if self.name.find('packagegroup') != -1: + return True + return False + +class CustomImagePackage(Package): + # CustomImageRecipe fields to track pacakges appended, + # included and excluded from a CustomImageRecipe + recipe_includes = models.ManyToManyField('CustomImageRecipe', + related_name='includes_set') + recipe_excludes = models.ManyToManyField('CustomImageRecipe', + related_name='excludes_set') + recipe_appends = models.ManyToManyField('CustomImageRecipe', + related_name='appends_set') + + +class Package_DependencyManager(models.Manager): + use_for_related_fields = True + TARGET_LATEST = "use-latest-target-for-target" + + def get_queryset(self): + return super(Package_DependencyManager, self).get_queryset().exclude(package_id = F('depends_on__id')) + + def for_target_or_none(self, target): + """ filter the dependencies to be displayed by the supplied target + if no dependences are found for the target then try None as the target + which will return the dependences calculated without the context of a + target e.g. non image recipes. + + returns: { size, packages } + """ + package_dependencies = self.all_depends().order_by('depends_on__name') + + if target is self.TARGET_LATEST: + installed_deps =\ + package_dependencies.filter(~Q(target__target=None)) + else: + installed_deps =\ + package_dependencies.filter(Q(target__target=target)) + + packages_list = None + total_size = 0 + + # If we have installed depdencies for this package and target then use + # these to display + if installed_deps.count() > 0: + packages_list = installed_deps + total_size = installed_deps.aggregate( + Sum('depends_on__size'))['depends_on__size__sum'] + else: + new_list = [] + package_names = [] + + # Find dependencies for the package that we know about even if + # it's not installed on a target e.g. from a non-image recipe + for p in package_dependencies.filter(Q(target=None)): + if p.depends_on.name in package_names: + continue + else: + package_names.append(p.depends_on.name) + new_list.append(p.pk) + # while we're here we may as well total up the size to + # avoid iterating again + total_size += p.depends_on.size + + # We want to return a queryset here for consistency so pick the + # deps from the new_list + packages_list = package_dependencies.filter(Q(pk__in=new_list)) + + return {'packages': packages_list, + 'size': total_size} + + def all_depends(self): + """ Returns just the depends packages and not any other dep_type + Note that this is for any target + """ + return self.filter(Q(dep_type=Package_Dependency.TYPE_RDEPENDS) | + Q(dep_type=Package_Dependency.TYPE_TRDEPENDS)) + + +class Package_Dependency(models.Model): + TYPE_RDEPENDS = 0 + TYPE_TRDEPENDS = 1 + TYPE_RRECOMMENDS = 2 + TYPE_TRECOMMENDS = 3 + TYPE_RSUGGESTS = 4 + TYPE_RPROVIDES = 5 + TYPE_RREPLACES = 6 + TYPE_RCONFLICTS = 7 + ' TODO: bpackage should be changed to remove the DEPENDS_TYPE access ' + DEPENDS_TYPE = ( + (TYPE_RDEPENDS, "depends"), + (TYPE_TRDEPENDS, "depends"), + (TYPE_TRECOMMENDS, "recommends"), + (TYPE_RRECOMMENDS, "recommends"), + (TYPE_RSUGGESTS, "suggests"), + (TYPE_RPROVIDES, "provides"), + (TYPE_RREPLACES, "replaces"), + (TYPE_RCONFLICTS, "conflicts"), + ) + """ Indexed by dep_type, in view order, key for short name and help + description which when viewed will be printf'd with the + package name. + """ + DEPENDS_DICT = { + TYPE_RDEPENDS : ("depends", "%s is required to run %s"), + TYPE_TRDEPENDS : ("depends", "%s is required to run %s"), + TYPE_TRECOMMENDS : ("recommends", "%s extends the usability of %s"), + TYPE_RRECOMMENDS : ("recommends", "%s extends the usability of %s"), + TYPE_RSUGGESTS : ("suggests", "%s is suggested for installation with %s"), + TYPE_RPROVIDES : ("provides", "%s is provided by %s"), + TYPE_RREPLACES : ("replaces", "%s is replaced by %s"), + TYPE_RCONFLICTS : ("conflicts", "%s conflicts with %s, which will not be installed if this package is not first removed"), + } + + package = models.ForeignKey(Package, related_name='package_dependencies_source') + depends_on = models.ForeignKey(Package, related_name='package_dependencies_target') # soft dependency + dep_type = models.IntegerField(choices=DEPENDS_TYPE) + target = models.ForeignKey(Target, null=True) + objects = Package_DependencyManager() + +class Target_Installed_Package(models.Model): + target = models.ForeignKey(Target) + package = models.ForeignKey(Package, related_name='buildtargetlist_package') + + +class Package_File(models.Model): + package = models.ForeignKey(Package, related_name='buildfilelist_package') + path = models.FilePathField(max_length=255, blank=True) + size = models.IntegerField() + + +class Recipe(models.Model): + search_allowed_fields = ['name', 'version', 'file_path', 'section', + 'summary', 'description', 'license', + 'layer_version__layer__name', + 'layer_version__branch', 'layer_version__commit', + 'layer_version__local_path', + 'layer_version__layer_source'] + + up_date = models.DateTimeField(null=True, default=None) + + name = models.CharField(max_length=100, blank=True) + version = models.CharField(max_length=100, blank=True) + layer_version = models.ForeignKey('Layer_Version', + related_name='recipe_layer_version') + summary = models.TextField(blank=True) + description = models.TextField(blank=True) + section = models.CharField(max_length=100, blank=True) + license = models.CharField(max_length=200, blank=True) + homepage = models.URLField(blank=True) + bugtracker = models.URLField(blank=True) + file_path = models.FilePathField(max_length=255) + pathflags = models.CharField(max_length=200, blank=True) + is_image = models.BooleanField(default=False) + + def __unicode__(self): + return "Recipe " + self.name + ":" + self.version + + def get_vcs_recipe_file_link_url(self): + return self.layer_version.get_vcs_file_link_url(self.file_path) + + def get_description_or_summary(self): + if self.description: + return self.description + elif self.summary: + return self.summary + else: + return "" + + class Meta: + unique_together = (("layer_version", "file_path", "pathflags"), ) + + +class Recipe_DependencyManager(models.Manager): + use_for_related_fields = True + + def get_queryset(self): + return super(Recipe_DependencyManager, self).get_queryset().exclude(recipe_id = F('depends_on__id')) + +class Provides(models.Model): + name = models.CharField(max_length=100) + recipe = models.ForeignKey(Recipe) + +class Recipe_Dependency(models.Model): + TYPE_DEPENDS = 0 + TYPE_RDEPENDS = 1 + + DEPENDS_TYPE = ( + (TYPE_DEPENDS, "depends"), + (TYPE_RDEPENDS, "rdepends"), + ) + recipe = models.ForeignKey(Recipe, related_name='r_dependencies_recipe') + depends_on = models.ForeignKey(Recipe, related_name='r_dependencies_depends') + via = models.ForeignKey(Provides, null=True, default=None) + dep_type = models.IntegerField(choices=DEPENDS_TYPE) + objects = Recipe_DependencyManager() + + +class Machine(models.Model): + search_allowed_fields = ["name", "description", "layer_version__layer__name"] + up_date = models.DateTimeField(null = True, default = None) + + layer_version = models.ForeignKey('Layer_Version') + name = models.CharField(max_length=255) + description = models.CharField(max_length=255) + + def get_vcs_machine_file_link_url(self): + path = 'conf/machine/'+self.name+'.conf' + + return self.layer_version.get_vcs_file_link_url(path) + + def __unicode__(self): + return "Machine " + self.name + "(" + self.description + ")" + + + + + +class BitbakeVersion(models.Model): + + name = models.CharField(max_length=32, unique = True) + giturl = GitURLField() + branch = models.CharField(max_length=32) + dirpath = models.CharField(max_length=255) + + def __unicode__(self): + return "%s (Branch: %s)" % (self.name, self.branch) + + +class Release(models.Model): + """ A release is a project template, used to pre-populate Project settings with a configuration set """ + name = models.CharField(max_length=32, unique = True) + description = models.CharField(max_length=255) + bitbake_version = models.ForeignKey(BitbakeVersion) + branch_name = models.CharField(max_length=50, default = "") + helptext = models.TextField(null=True) + + def __unicode__(self): + return "%s (%s)" % (self.name, self.branch_name) + + def __str__(self): + return self.name + +class ReleaseDefaultLayer(models.Model): + release = models.ForeignKey(Release) + layer_name = models.CharField(max_length=100, default="") + + +class LayerSource(object): + """ Where the layer metadata came from """ + TYPE_LOCAL = 0 + TYPE_LAYERINDEX = 1 + TYPE_IMPORTED = 2 + TYPE_BUILD = 3 + + SOURCE_TYPE = ( + (TYPE_LOCAL, "local"), + (TYPE_LAYERINDEX, "layerindex"), + (TYPE_IMPORTED, "imported"), + (TYPE_BUILD, "build"), + ) + + def types_dict(): + """ Turn the TYPES enums into a simple dictionary """ + dictionary = {} + for key in LayerSource.__dict__: + if "TYPE" in key: + dictionary[key] = getattr(LayerSource, key) + return dictionary + + +class Layer(models.Model): + + up_date = models.DateTimeField(null=True, default=timezone.now) + + name = models.CharField(max_length=100) + layer_index_url = models.URLField() + vcs_url = GitURLField(default=None, null=True) + local_source_dir = models.TextField(null=True, default=None) + vcs_web_url = models.URLField(null=True, default=None) + vcs_web_tree_base_url = models.URLField(null=True, default=None) + vcs_web_file_base_url = models.URLField(null=True, default=None) + + summary = models.TextField(help_text='One-line description of the layer', + null=True, default=None) + description = models.TextField(null=True, default=None) + + def __unicode__(self): + return "%s / %s " % (self.name, self.summary) + + +class Layer_Version(models.Model): + """ + A Layer_Version either belongs to a single project or no project + """ + search_allowed_fields = ["layer__name", "layer__summary", + "layer__description", "layer__vcs_url", + "dirpath", "release__name", "commit", "branch"] + + build = models.ForeignKey(Build, related_name='layer_version_build', + default=None, null=True) + + layer = models.ForeignKey(Layer, related_name='layer_version_layer') + + layer_source = models.IntegerField(choices=LayerSource.SOURCE_TYPE, + default=0) + + up_date = models.DateTimeField(null=True, default=timezone.now) + + # To which metadata release does this layer version belong to + release = models.ForeignKey(Release, null=True, default=None) + + branch = models.CharField(max_length=80) + commit = models.CharField(max_length=100) + # If the layer is in a subdir + dirpath = models.CharField(max_length=255, null=True, default=None) + + # if -1, this is a default layer + priority = models.IntegerField(default=0) + + # where this layer exists on the filesystem + local_path = models.FilePathField(max_length=1024, default="/") + + # Set if this layer is restricted to a particular project + project = models.ForeignKey('Project', null=True, default=None) + + # code lifted, with adaptations, from the layerindex-web application + # https://git.yoctoproject.org/cgit/cgit.cgi/layerindex-web/ + def _handle_url_path(self, base_url, path): + import re, posixpath + if base_url: + if self.dirpath: + if path: + extra_path = self.dirpath + '/' + path + # Normalise out ../ in path for usage URL + extra_path = posixpath.normpath(extra_path) + # Minor workaround to handle case where subdirectory has been added between branches + # (should probably support usage URL per branch to handle this... sigh...) + if extra_path.startswith('../'): + extra_path = extra_path[3:] + else: + extra_path = self.dirpath + else: + extra_path = path + branchname = self.release.name + url = base_url.replace('%branch%', branchname) + + # If there's a % in the path (e.g. a wildcard bbappend) we need to encode it + if extra_path: + extra_path = extra_path.replace('%', '%25') + + if '%path%' in base_url: + if extra_path: + url = re.sub(r'\[([^\]]*%path%[^\]]*)\]', '\\1', url) + else: + url = re.sub(r'\[([^\]]*%path%[^\]]*)\]', '', url) + return url.replace('%path%', extra_path) + else: + return url + extra_path + return None + + def get_vcs_link_url(self): + if self.layer.vcs_web_url is None: + return None + return self.layer.vcs_web_url + + def get_vcs_file_link_url(self, file_path=""): + if self.layer.vcs_web_file_base_url is None: + return None + return self._handle_url_path(self.layer.vcs_web_file_base_url, + file_path) + + def get_vcs_dirpath_link_url(self): + if self.layer.vcs_web_tree_base_url is None: + return None + return self._handle_url_path(self.layer.vcs_web_tree_base_url, '') + + def get_vcs_reference(self): + if self.commit is not None and len(self.commit) > 0: + return self.commit + if self.branch is not None and len(self.branch) > 0: + return self.branch + if self.release is not None: + return self.release.name + return 'N/A' + + def get_detailspage_url(self, project_id=None): + """ returns the url to the layer details page uses own project + field if project_id is not specified """ + + if project_id is None: + project_id = self.project.pk + + return reverse('layerdetails', args=(project_id, self.pk)) + + def get_alldeps(self, project_id): + """Get full list of unique layer dependencies.""" + def gen_layerdeps(lver, project, depth): + if depth == 0: + return + for ldep in lver.dependencies.all(): + yield ldep.depends_on + # get next level of deps recursively calling gen_layerdeps + for subdep in gen_layerdeps(ldep.depends_on, project, depth-1): + yield subdep + + project = Project.objects.get(pk=project_id) + result = [] + projectlvers = [player.layercommit for player in + project.projectlayer_set.all()] + # protect against infinite layer dependency loops + maxdepth = 20 + for dep in gen_layerdeps(self, project, maxdepth): + # filter out duplicates and layers already belonging to the project + if dep not in result + projectlvers: + result.append(dep) + + return sorted(result, key=lambda x: x.layer.name) + + def __unicode__(self): + return ("id %d belongs to layer: %s" % (self.pk, self.layer.name)) + + def __str__(self): + if self.release: + release = self.release.name + else: + release = "No release set" + + return "%d %s (%s)" % (self.pk, self.layer.name, release) + + +class LayerVersionDependency(models.Model): + + layer_version = models.ForeignKey(Layer_Version, + related_name="dependencies") + depends_on = models.ForeignKey(Layer_Version, + related_name="dependees") + +class ProjectLayer(models.Model): + project = models.ForeignKey(Project) + layercommit = models.ForeignKey(Layer_Version, null=True) + optional = models.BooleanField(default = True) + + def __unicode__(self): + return "%s, %s" % (self.project.name, self.layercommit) + + class Meta: + unique_together = (("project", "layercommit"),) + +class CustomImageRecipe(Recipe): + + # CustomImageRecipe's belong to layers called: + LAYER_NAME = "toaster-custom-images" + + search_allowed_fields = ['name'] + base_recipe = models.ForeignKey(Recipe, related_name='based_on_recipe') + project = models.ForeignKey(Project) + last_updated = models.DateTimeField(null=True, default=None) + + def get_last_successful_built_target(self): + """ Return the last successful built target object if one exists + otherwise return None """ + return Target.objects.filter(Q(build__outcome=Build.SUCCEEDED) & + Q(build__project=self.project) & + Q(target=self.name)).last() + + def update_package_list(self): + """ Update the package list from the last good build of this + CustomImageRecipe + """ + # Check if we're aldready up-to-date or not + target = self.get_last_successful_built_target() + if target == None: + # So we've never actually built this Custom recipe but what about + # the recipe it's based on? + target = \ + Target.objects.filter(Q(build__outcome=Build.SUCCEEDED) & + Q(build__project=self.project) & + Q(target=self.base_recipe.name)).last() + if target == None: + return + + if target.build.completed_on == self.last_updated: + return + + self.includes_set.clear() + + excludes_list = self.excludes_set.values_list('name', flat=True) + appends_list = self.appends_set.values_list('name', flat=True) + + built_packages_list = \ + target.target_installed_package_set.values_list('package__name', + flat=True) + for built_package in built_packages_list: + # Is the built package in the custom packages list? + if built_package in excludes_list: + continue + + if built_package in appends_list: + continue + + cust_img_p = \ + CustomImagePackage.objects.get(name=built_package) + self.includes_set.add(cust_img_p) + + + self.last_updated = target.build.completed_on + self.save() + + def get_all_packages(self): + """Get the included packages and any appended packages""" + self.update_package_list() + + return CustomImagePackage.objects.filter((Q(recipe_appends=self) | + Q(recipe_includes=self)) & + ~Q(recipe_excludes=self)) + + def get_base_recipe_file(self): + """Get the base recipe file path if it exists on the file system""" + path_schema_one = "%s/%s" % (self.base_recipe.layer_version.local_path, + self.base_recipe.file_path) + + path_schema_two = self.base_recipe.file_path + + if os.path.exists(path_schema_one): + return path_schema_one + + # The path may now be the full path if the recipe has been built + if os.path.exists(path_schema_two): + return path_schema_two + + return None + + def generate_recipe_file_contents(self): + """Generate the contents for the recipe file.""" + # If we have no excluded packages we only need to _append + if self.excludes_set.count() == 0: + packages_conf = "IMAGE_INSTALL_append = \" " + + for pkg in self.appends_set.all(): + packages_conf += pkg.name+' ' + else: + packages_conf = "IMAGE_FEATURES =\"\"\nIMAGE_INSTALL = \"" + # We add all the known packages to be built by this recipe apart + # from locale packages which are are controlled with IMAGE_LINGUAS. + for pkg in self.get_all_packages().exclude( + name__icontains="locale"): + packages_conf += pkg.name+' ' + + packages_conf += "\"" + + base_recipe_path = self.get_base_recipe_file() + if base_recipe_path: + base_recipe = open(base_recipe_path, 'r').read() + else: + raise IOError("Based on recipe file not found: %s" % + base_recipe_path) + + # Add a special case for when the recipe we have based a custom image + # recipe on requires another recipe. + # For example: + # "require core-image-minimal.bb" is changed to: + # "require recipes-core/images/core-image-minimal.bb" + + req_search = re.search(r'(require\s+)(.+\.bb\s*$)', + base_recipe, + re.MULTILINE) + if req_search: + require_filename = req_search.group(2).strip() + + corrected_location = Recipe.objects.filter( + Q(layer_version=self.base_recipe.layer_version) & + Q(file_path__icontains=require_filename)).last().file_path + + new_require_line = "require %s" % corrected_location + + base_recipe = base_recipe.replace(req_search.group(0), + new_require_line) + + info = { + "date": timezone.now().strftime("%Y-%m-%d %H:%M:%S"), + "base_recipe": base_recipe, + "recipe_name": self.name, + "base_recipe_name": self.base_recipe.name, + "license": self.license, + "summary": self.summary, + "description": self.description, + "packages_conf": packages_conf.strip() + } + + recipe_contents = ("# Original recipe %(base_recipe_name)s \n" + "%(base_recipe)s\n\n" + "# Recipe %(recipe_name)s \n" + "# Customisation Generated by Toaster on %(date)s\n" + "SUMMARY = \"%(summary)s\"\n" + "DESCRIPTION = \"%(description)s\"\n" + "LICENSE = \"%(license)s\"\n" + "%(packages_conf)s") % info + + return recipe_contents + +class ProjectVariable(models.Model): + project = models.ForeignKey(Project) + name = models.CharField(max_length=100) + value = models.TextField(blank = True) + +class Variable(models.Model): + search_allowed_fields = ['variable_name', 'variable_value', + 'vhistory__file_name', "description"] + build = models.ForeignKey(Build, related_name='variable_build') + variable_name = models.CharField(max_length=100) + variable_value = models.TextField(blank=True) + changed = models.BooleanField(default=False) + human_readable_name = models.CharField(max_length=200) + description = models.TextField(blank=True) + +class VariableHistory(models.Model): + variable = models.ForeignKey(Variable, related_name='vhistory') + value = models.TextField(blank=True) + file_name = models.FilePathField(max_length=255) + line_number = models.IntegerField(null=True) + operation = models.CharField(max_length=64) + +class HelpText(models.Model): + VARIABLE = 0 + HELPTEXT_AREA = ((VARIABLE, 'variable'), ) + + build = models.ForeignKey(Build, related_name='helptext_build') + area = models.IntegerField(choices=HELPTEXT_AREA) + key = models.CharField(max_length=100) + text = models.TextField() + +class LogMessage(models.Model): + EXCEPTION = -1 # used to signal self-toaster-exceptions + INFO = 0 + WARNING = 1 + ERROR = 2 + CRITICAL = 3 + + LOG_LEVEL = ( + (INFO, "info"), + (WARNING, "warn"), + (ERROR, "error"), + (CRITICAL, "critical"), + (EXCEPTION, "toaster exception") + ) + + build = models.ForeignKey(Build) + task = models.ForeignKey(Task, blank = True, null=True) + level = models.IntegerField(choices=LOG_LEVEL, default=INFO) + message = models.TextField(blank=True, null=True) + pathname = models.FilePathField(max_length=255, blank=True) + lineno = models.IntegerField(null=True) + + def __str__(self): + return force_bytes('%s %s %s' % (self.get_level_display(), self.message, self.build)) + +def invalidate_cache(**kwargs): + from django.core.cache import cache + try: + cache.clear() + except Exception as e: + logger.warning("Problem with cache backend: Failed to clear cache: %s" % e) + +def signal_runbuilds(): + """Send SIGUSR1 to runbuilds process""" + try: + with open(os.path.join(os.getenv('BUILDDIR', '.'), + '.runbuilds.pid')) as pidf: + os.kill(int(pidf.read()), SIGUSR1) + except FileNotFoundError: + logger.info("Stopping existing runbuilds: no current process found") + +class Distro(models.Model): + search_allowed_fields = ["name", "description", "layer_version__layer__name"] + up_date = models.DateTimeField(null = True, default = None) + + layer_version = models.ForeignKey('Layer_Version') + name = models.CharField(max_length=255) + description = models.CharField(max_length=255) + + def get_vcs_distro_file_link_url(self): + path = self.name+'.conf' + return self.layer_version.get_vcs_file_link_url(path) + + def __unicode__(self): + return "Distro " + self.name + "(" + self.description + ")" + +django.db.models.signals.post_save.connect(invalidate_cache) +django.db.models.signals.post_delete.connect(invalidate_cache) +django.db.models.signals.m2m_changed.connect(invalidate_cache) |
