diff options
Diffstat (limited to 'import-layers/yocto-poky/bitbake/lib/toaster/orm/models.py')
| -rw-r--r-- | import-layers/yocto-poky/bitbake/lib/toaster/orm/models.py | 1832 |
1 files changed, 0 insertions, 1832 deletions
diff --git a/import-layers/yocto-poky/bitbake/lib/toaster/orm/models.py b/import-layers/yocto-poky/bitbake/lib/toaster/orm/models.py deleted file mode 100644 index 3a7dff8ca..000000000 --- a/import-layers/yocto-poky/bitbake/lib/toaster/orm/models.py +++ /dev/null @@ -1,1832 +0,0 @@ -# -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -# -# BitBake Toaster Implementation -# -# Copyright (C) 2013 Intel Corporation -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -from __future__ import unicode_literals - -from django.db import models, IntegrityError, DataError -from django.db.models import F, Q, Sum, Count -from django.utils import timezone -from django.utils.encoding import force_bytes - -from django.core.urlresolvers import reverse - -from django.core import validators -from django.conf import settings -import django.db.models.signals - -import sys -import os -import re -import itertools -from signal import SIGUSR1 - - -import logging -logger = logging.getLogger("toaster") - -if 'sqlite' in settings.DATABASES['default']['ENGINE']: - from django.db import transaction, OperationalError - from time import sleep - - _base_save = models.Model.save - def save(self, *args, **kwargs): - while True: - try: - with transaction.atomic(): - return _base_save(self, *args, **kwargs) - except OperationalError as err: - if 'database is locked' in str(err): - logger.warning("%s, model: %s, args: %s, kwargs: %s", - err, self.__class__, args, kwargs) - sleep(0.5) - continue - raise - - models.Model.save = save - - # HACK: Monkey patch Django to fix 'database is locked' issue - - from django.db.models.query import QuerySet - _base_insert = QuerySet._insert - def _insert(self, *args, **kwargs): - with transaction.atomic(using=self.db, savepoint=False): - return _base_insert(self, *args, **kwargs) - QuerySet._insert = _insert - - from django.utils import six - def _create_object_from_params(self, lookup, params): - """ - Tries to create an object using passed params. - Used by get_or_create and update_or_create - """ - try: - obj = self.create(**params) - return obj, True - except (IntegrityError, DataError): - exc_info = sys.exc_info() - try: - return self.get(**lookup), False - except self.model.DoesNotExist: - pass - six.reraise(*exc_info) - - QuerySet._create_object_from_params = _create_object_from_params - - # end of HACK - -class GitURLValidator(validators.URLValidator): - import re - regex = re.compile( - r'^(?:ssh|git|http|ftp)s?://' # http:// or https:// - r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain... - r'localhost|' # localhost... - r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|' # ...or ipv4 - r'\[?[A-F0-9]*:[A-F0-9:]+\]?)' # ...or ipv6 - r'(?::\d+)?' # optional port - r'(?:/?|[/?]\S+)$', re.IGNORECASE) - -def GitURLField(**kwargs): - r = models.URLField(**kwargs) - for i in range(len(r.validators)): - if isinstance(r.validators[i], validators.URLValidator): - r.validators[i] = GitURLValidator() - return r - - -class ToasterSetting(models.Model): - name = models.CharField(max_length=63) - helptext = models.TextField() - value = models.CharField(max_length=255) - - def __unicode__(self): - return "Setting %s = %s" % (self.name, self.value) - - -class ProjectManager(models.Manager): - def create_project(self, name, release): - if release is not None: - prj = self.model(name=name, - bitbake_version=release.bitbake_version, - release=release) - else: - prj = self.model(name=name, - bitbake_version=None, - release=None) - - prj.save() - - for defaultconf in ToasterSetting.objects.filter( - name__startswith="DEFCONF_"): - name = defaultconf.name[8:] - ProjectVariable.objects.create(project=prj, - name=name, - value=defaultconf.value) - - if release is None: - return prj - - for rdl in release.releasedefaultlayer_set.all(): - lv = Layer_Version.objects.filter( - layer__name=rdl.layer_name, - release=release).first() - - if lv: - ProjectLayer.objects.create(project=prj, - layercommit=lv, - optional=False) - else: - logger.warning("Default project layer %s not found" % - rdl.layer_name) - - return prj - - # return single object with is_default = True - def get_or_create_default_project(self): - projects = super(ProjectManager, self).filter(is_default=True) - - if len(projects) > 1: - raise Exception('Inconsistent project data: multiple ' + - 'default projects (i.e. with is_default=True)') - elif len(projects) < 1: - options = { - 'name': 'Command line builds', - 'short_description': - 'Project for builds started outside Toaster', - 'is_default': True - } - project = Project.objects.create(**options) - project.save() - - return project - else: - return projects[0] - - -class Project(models.Model): - search_allowed_fields = ['name', 'short_description', 'release__name', - 'release__branch_name'] - name = models.CharField(max_length=100) - short_description = models.CharField(max_length=50, blank=True) - bitbake_version = models.ForeignKey('BitbakeVersion', null=True) - release = models.ForeignKey("Release", null=True) - created = models.DateTimeField(auto_now_add=True) - updated = models.DateTimeField(auto_now=True) - # This is a horrible hack; since Toaster has no "User" model available when - # running in interactive mode, we can't reference the field here directly - # Instead, we keep a possible null reference to the User id, - # as not to force - # hard links to possibly missing models - user_id = models.IntegerField(null=True) - objects = ProjectManager() - - # set to True for the project which is the default container - # for builds initiated by the command line etc. - is_default= models.BooleanField(default=False) - - def __unicode__(self): - return "%s (Release %s, BBV %s)" % (self.name, self.release, self.bitbake_version) - - def get_current_machine_name(self): - try: - return self.projectvariable_set.get(name="MACHINE").value - except (ProjectVariable.DoesNotExist,IndexError): - return None; - - def get_number_of_builds(self): - """Return the number of builds which have ended""" - - return self.build_set.exclude( - Q(outcome=Build.IN_PROGRESS) | - Q(outcome=Build.CANCELLED) - ).count() - - def get_last_build_id(self): - try: - return Build.objects.filter( project = self.id ).order_by('-completed_on')[0].id - except (Build.DoesNotExist,IndexError): - return( -1 ) - - def get_last_outcome(self): - build_id = self.get_last_build_id() - if (-1 == build_id): - return( "" ) - try: - return Build.objects.filter( id = build_id )[ 0 ].outcome - except (Build.DoesNotExist,IndexError): - return( "not_found" ) - - def get_last_target(self): - build_id = self.get_last_build_id() - if (-1 == build_id): - return( "" ) - try: - return Target.objects.filter(build = build_id)[0].target - except (Target.DoesNotExist,IndexError): - return( "not_found" ) - - def get_last_errors(self): - build_id = self.get_last_build_id() - if (-1 == build_id): - return( 0 ) - try: - return Build.objects.filter(id = build_id)[ 0 ].errors.count() - except (Build.DoesNotExist,IndexError): - return( "not_found" ) - - def get_last_warnings(self): - build_id = self.get_last_build_id() - if (-1 == build_id): - return( 0 ) - try: - return Build.objects.filter(id = build_id)[ 0 ].warnings.count() - except (Build.DoesNotExist,IndexError): - return( "not_found" ) - - def get_last_build_extensions(self): - """ - Get list of file name extensions for images produced by the most - recent build - """ - last_build = Build.objects.get(pk = self.get_last_build_id()) - return last_build.get_image_file_extensions() - - def get_last_imgfiles(self): - build_id = self.get_last_build_id() - if (-1 == build_id): - return( "" ) - try: - return Variable.objects.filter(build = build_id, variable_name = "IMAGE_FSTYPES")[ 0 ].variable_value - except (Variable.DoesNotExist,IndexError): - return( "not_found" ) - - def get_all_compatible_layer_versions(self): - """ Returns Queryset of all Layer_Versions which are compatible with - this project""" - queryset = None - - # guard on release, as it can be null - if self.release: - queryset = Layer_Version.objects.filter( - (Q(release=self.release) & - Q(build=None) & - Q(project=None)) | - Q(project=self)) - else: - queryset = Layer_Version.objects.none() - - return queryset - - def get_project_layer_versions(self, pk=False): - """ Returns the Layer_Versions currently added to this project """ - layer_versions = self.projectlayer_set.all().values_list('layercommit', - flat=True) - - if pk is False: - return Layer_Version.objects.filter(pk__in=layer_versions) - else: - return layer_versions - - - def get_available_machines(self): - """ Returns QuerySet of all Machines which are provided by the - Layers currently added to the Project """ - queryset = Machine.objects.filter( - layer_version__in=self.get_project_layer_versions()) - - return queryset - - def get_all_compatible_machines(self): - """ Returns QuerySet of all the compatible machines available to the - project including ones from Layers not currently added """ - queryset = Machine.objects.filter( - layer_version__in=self.get_all_compatible_layer_versions()) - - return queryset - - def get_available_distros(self): - """ Returns QuerySet of all Distros which are provided by the - Layers currently added to the Project """ - queryset = Distro.objects.filter( - layer_version__in=self.get_project_layer_versions()) - - return queryset - - def get_all_compatible_distros(self): - """ Returns QuerySet of all the compatible Wind River distros available to the - project including ones from Layers not currently added """ - queryset = Distro.objects.filter( - layer_version__in=self.get_all_compatible_layer_versions()) - - return queryset - - def get_available_recipes(self): - """ Returns QuerySet of all the recipes that are provided by layers - added to this project """ - queryset = Recipe.objects.filter( - layer_version__in=self.get_project_layer_versions()) - - return queryset - - def get_all_compatible_recipes(self): - """ Returns QuerySet of all the compatible Recipes available to the - project including ones from Layers not currently added """ - queryset = Recipe.objects.filter( - layer_version__in=self.get_all_compatible_layer_versions()).exclude(name__exact='') - - return queryset - - def schedule_build(self): - - from bldcontrol.models import BuildRequest, BRTarget, BRLayer - from bldcontrol.models import BRBitbake, BRVariable - - try: - now = timezone.now() - build = Build.objects.create(project=self, - completed_on=now, - started_on=now) - - br = BuildRequest.objects.create(project=self, - state=BuildRequest.REQ_QUEUED, - build=build) - BRBitbake.objects.create(req=br, - giturl=self.bitbake_version.giturl, - commit=self.bitbake_version.branch, - dirpath=self.bitbake_version.dirpath) - - for t in self.projecttarget_set.all(): - BRTarget.objects.create(req=br, target=t.target, task=t.task) - Target.objects.create(build=br.build, target=t.target, - task=t.task) - # If we're about to build a custom image recipe make sure - # that layer is currently in the project before we create the - # BRLayer objects - customrecipe = CustomImageRecipe.objects.filter( - name=t.target, - project=self).first() - if customrecipe: - ProjectLayer.objects.get_or_create( - project=self, - layercommit=customrecipe.layer_version, - optional=False) - - for l in self.projectlayer_set.all().order_by("pk"): - commit = l.layercommit.get_vcs_reference() - logger.debug("Adding layer to build %s" % - l.layercommit.layer.name) - BRLayer.objects.create( - req=br, - name=l.layercommit.layer.name, - giturl=l.layercommit.layer.vcs_url, - commit=commit, - dirpath=l.layercommit.dirpath, - layer_version=l.layercommit, - local_source_dir=l.layercommit.layer.local_source_dir - ) - - for v in self.projectvariable_set.all(): - BRVariable.objects.create(req=br, name=v.name, value=v.value) - - try: - br.build.machine = self.projectvariable_set.get( - name='MACHINE').value - br.build.save() - except ProjectVariable.DoesNotExist: - pass - - br.save() - signal_runbuilds() - - except Exception: - # revert the build request creation since we're not done cleanly - br.delete() - raise - return br - -class Build(models.Model): - SUCCEEDED = 0 - FAILED = 1 - IN_PROGRESS = 2 - CANCELLED = 3 - - BUILD_OUTCOME = ( - (SUCCEEDED, 'Succeeded'), - (FAILED, 'Failed'), - (IN_PROGRESS, 'In Progress'), - (CANCELLED, 'Cancelled'), - ) - - search_allowed_fields = ['machine', 'cooker_log_path', "target__target", "target__target_image_file__file_name"] - - project = models.ForeignKey(Project) # must have a project - machine = models.CharField(max_length=100) - distro = models.CharField(max_length=100) - distro_version = models.CharField(max_length=100) - started_on = models.DateTimeField() - completed_on = models.DateTimeField() - outcome = models.IntegerField(choices=BUILD_OUTCOME, default=IN_PROGRESS) - cooker_log_path = models.CharField(max_length=500) - build_name = models.CharField(max_length=100, default='') - bitbake_version = models.CharField(max_length=50) - - # number of recipes to parse for this build - recipes_to_parse = models.IntegerField(default=1) - - # number of recipes parsed so far for this build - recipes_parsed = models.IntegerField(default=1) - - # number of repos to clone for this build - repos_to_clone = models.IntegerField(default=1) - - # number of repos cloned so far for this build (default off) - repos_cloned = models.IntegerField(default=1) - - @staticmethod - def get_recent(project=None): - """ - Return recent builds as a list; if project is set, only return - builds for that project - """ - - builds = Build.objects.all() - - if project: - builds = builds.filter(project=project) - - finished_criteria = \ - Q(outcome=Build.SUCCEEDED) | \ - Q(outcome=Build.FAILED) | \ - Q(outcome=Build.CANCELLED) - - recent_builds = list(itertools.chain( - builds.filter(outcome=Build.IN_PROGRESS).order_by("-started_on"), - builds.filter(finished_criteria).order_by("-completed_on")[:3] - )) - - # add percentage done property to each build; this is used - # to show build progress in mrb_section.html - for build in recent_builds: - build.percentDone = build.completeper() - build.outcomeText = build.get_outcome_text() - - return recent_builds - - def started(self): - """ - As build variables are only added for a build when its BuildStarted event - is received, a build with no build variables is counted as - "in preparation" and not properly started yet. This method - will return False if a build has no build variables (it never properly - started), or True otherwise. - - Note that this is a temporary workaround for the fact that we don't - have a fine-grained state variable on a build which would allow us - to record "in progress" (BuildStarted received) vs. "in preparation". - """ - variables = Variable.objects.filter(build=self) - return len(variables) > 0 - - def completeper(self): - tf = Task.objects.filter(build = self) - tfc = tf.count() - if tfc > 0: - completeper = tf.exclude(outcome=Task.OUTCOME_NA).count()*100 // tfc - else: - completeper = 0 - return completeper - - def eta(self): - eta = timezone.now() - completeper = self.completeper() - if self.completeper() > 0: - eta += ((eta - self.started_on)*(100-completeper))/completeper - return eta - - def has_images(self): - """ - Returns True if at least one of the targets for this build has an - image file associated with it, False otherwise - """ - targets = Target.objects.filter(build_id=self.id) - has_images = False - for target in targets: - if target.has_images(): - has_images = True - break - return has_images - - def has_image_recipes(self): - """ - Returns True if a build has any targets which were built from - image recipes. - """ - image_recipes = self.get_image_recipes() - return len(image_recipes) > 0 - - def get_image_file_extensions(self): - """ - Get string of file name extensions for images produced by this build; - note that this is the actual list of extensions stored on Target objects - for this build, and not the value of IMAGE_FSTYPES. - - Returns comma-separated string, e.g. "vmdk, ext4" - """ - extensions = [] - - targets = Target.objects.filter(build_id = self.id) - for target in targets: - if not target.is_image: - continue - - target_image_files = Target_Image_File.objects.filter( - target_id=target.id) - - for target_image_file in target_image_files: - extensions.append(target_image_file.suffix) - - extensions = list(set(extensions)) - extensions.sort() - - return ', '.join(extensions) - - def get_image_fstypes(self): - """ - Get the IMAGE_FSTYPES variable value for this build as a de-duplicated - list of image file suffixes. - """ - image_fstypes = Variable.objects.get( - build=self, variable_name='IMAGE_FSTYPES').variable_value - return list(set(re.split(r' {1,}', image_fstypes))) - - def get_sorted_target_list(self): - tgts = Target.objects.filter(build_id = self.id).order_by( 'target' ); - return( tgts ); - - def get_recipes(self): - """ - Get the recipes related to this build; - note that the related layer versions and layers are also prefetched - by this query, as this queryset can be sorted by these objects in the - build recipes view; prefetching them here removes the need - for another query in that view - """ - layer_versions = Layer_Version.objects.filter(build=self) - criteria = Q(layer_version__id__in=layer_versions) - return Recipe.objects.filter(criteria) \ - .select_related('layer_version', 'layer_version__layer') - - def get_image_recipes(self): - """ - Returns a list of image Recipes (custom and built-in) related to this - build, sorted by name; note that this has to be done in two steps, as - there's no way to get all the custom image recipes and image recipes - in one query - """ - custom_image_recipes = self.get_custom_image_recipes() - custom_image_recipe_names = custom_image_recipes.values_list('name', flat=True) - - not_custom_image_recipes = ~Q(name__in=custom_image_recipe_names) & \ - Q(is_image=True) - - built_image_recipes = self.get_recipes().filter(not_custom_image_recipes) - - # append to the custom image recipes and sort - customisable_image_recipes = list( - itertools.chain(custom_image_recipes, built_image_recipes) - ) - - return sorted(customisable_image_recipes, key=lambda recipe: recipe.name) - - def get_custom_image_recipes(self): - """ - Returns a queryset of CustomImageRecipes related to this build, - sorted by name - """ - built_recipe_names = self.get_recipes().values_list('name', flat=True) - criteria = Q(name__in=built_recipe_names) & Q(project=self.project) - queryset = CustomImageRecipe.objects.filter(criteria).order_by('name') - return queryset - - def get_outcome_text(self): - return Build.BUILD_OUTCOME[int(self.outcome)][1] - - @property - def failed_tasks(self): - """ Get failed tasks for the build """ - tasks = self.task_build.all() - return tasks.filter(order__gt=0, outcome=Task.OUTCOME_FAILED) - - @property - def errors(self): - return (self.logmessage_set.filter(level=LogMessage.ERROR) | - self.logmessage_set.filter(level=LogMessage.EXCEPTION) | - self.logmessage_set.filter(level=LogMessage.CRITICAL)) - - @property - def warnings(self): - return self.logmessage_set.filter(level=LogMessage.WARNING) - - @property - def timespent(self): - return self.completed_on - self.started_on - - @property - def timespent_seconds(self): - return self.timespent.total_seconds() - - @property - def target_labels(self): - """ - Sorted (a-z) "target1:task, target2, target3" etc. string for all - targets in this build - """ - targets = self.target_set.all() - target_labels = [target.target + - (':' + target.task if target.task else '') - for target in targets] - target_labels.sort() - - return target_labels - - def get_buildrequest(self): - buildrequest = None - if hasattr(self, 'buildrequest'): - buildrequest = self.buildrequest - return buildrequest - - def is_queued(self): - from bldcontrol.models import BuildRequest - buildrequest = self.get_buildrequest() - if buildrequest: - return buildrequest.state == BuildRequest.REQ_QUEUED - else: - return False - - def is_cancelling(self): - from bldcontrol.models import BuildRequest - buildrequest = self.get_buildrequest() - if buildrequest: - return self.outcome == Build.IN_PROGRESS and \ - buildrequest.state == BuildRequest.REQ_CANCELLING - else: - return False - - def is_cloning(self): - """ - True if the build is still cloning repos - """ - return self.outcome == Build.IN_PROGRESS and \ - self.repos_cloned < self.repos_to_clone - - def is_parsing(self): - """ - True if the build is still parsing recipes - """ - return self.outcome == Build.IN_PROGRESS and \ - self.recipes_parsed < self.recipes_to_parse - - def is_starting(self): - """ - True if the build has no completed tasks yet and is still just starting - tasks. - - Note that the mechanism for testing whether a Task is "done" is whether - its outcome field is set, as per the completeper() method. - """ - return self.outcome == Build.IN_PROGRESS and \ - self.task_build.exclude(outcome=Task.OUTCOME_NA).count() == 0 - - - def get_state(self): - """ - Get the state of the build; one of 'Succeeded', 'Failed', 'In Progress', - 'Cancelled' (Build outcomes); or 'Queued', 'Cancelling' (states - dependent on the BuildRequest state). - - This works around the fact that we have BuildRequest states as well - as Build states, but really we just want to know the state of the build. - """ - if self.is_cancelling(): - return 'Cancelling'; - elif self.is_queued(): - return 'Queued' - elif self.is_cloning(): - return 'Cloning' - elif self.is_parsing(): - return 'Parsing' - elif self.is_starting(): - return 'Starting' - else: - return self.get_outcome_text() - - def __str__(self): - return "%d %s %s" % (self.id, self.project, ",".join([t.target for t in self.target_set.all()])) - -class ProjectTarget(models.Model): - project = models.ForeignKey(Project) - target = models.CharField(max_length=100) - task = models.CharField(max_length=100, null=True) - -class Target(models.Model): - search_allowed_fields = ['target', 'file_name'] - build = models.ForeignKey(Build) - target = models.CharField(max_length=100) - task = models.CharField(max_length=100, null=True) - is_image = models.BooleanField(default = False) - image_size = models.IntegerField(default=0) - license_manifest_path = models.CharField(max_length=500, null=True) - package_manifest_path = models.CharField(max_length=500, null=True) - - def package_count(self): - return Target_Installed_Package.objects.filter(target_id__exact=self.id).count() - - def __unicode__(self): - return self.target - - def get_similar_targets(self): - """ - Get target sfor the same machine, task and target name - (e.g. 'core-image-minimal') from a successful build for this project - (but excluding this target). - - Note that we only look for targets built by this project because - projects can have different configurations from each other, and put - their artifacts in different directories. - - The possibility of error when retrieving candidate targets - is minimised by the fact that bitbake will rebuild artifacts if MACHINE - (or various other variables) change. In this case, there is no need to - clone artifacts from another target, as those artifacts will have - been re-generated for this target anyway. - """ - query = ~Q(pk=self.pk) & \ - Q(target=self.target) & \ - Q(build__machine=self.build.machine) & \ - Q(build__outcome=Build.SUCCEEDED) & \ - Q(build__project=self.build.project) - - return Target.objects.filter(query) - - def get_similar_target_with_image_files(self): - """ - Get the most recent similar target with Target_Image_Files associated - with it, for the purpose of cloning those files onto this target. - """ - similar_target = None - - candidates = self.get_similar_targets() - if candidates.count() == 0: - return similar_target - - task_subquery = Q(task=self.task) - - # we can look for a 'build' task if this task is a 'populate_sdk_ext' - # task, as the latter also creates images; and vice versa; note that - # 'build' targets can have their task set to ''; - # also note that 'populate_sdk' does not produce image files - image_tasks = [ - '', # aka 'build' - 'build', - 'image', - 'populate_sdk_ext' - ] - if self.task in image_tasks: - task_subquery = Q(task__in=image_tasks) - - # annotate with the count of files, to exclude any targets which - # don't have associated files - candidates = candidates.annotate(num_files=Count('target_image_file')) - - query = task_subquery & Q(num_files__gt=0) - - candidates = candidates.filter(query) - - if candidates.count() > 0: - candidates.order_by('build__completed_on') - similar_target = candidates.last() - - return similar_target - - def get_similar_target_with_sdk_files(self): - """ - Get the most recent similar target with TargetSDKFiles associated - with it, for the purpose of cloning those files onto this target. - """ - similar_target = None - - candidates = self.get_similar_targets() - if candidates.count() == 0: - return similar_target - - # annotate with the count of files, to exclude any targets which - # don't have associated files - candidates = candidates.annotate(num_files=Count('targetsdkfile')) - - query = Q(task=self.task) & Q(num_files__gt=0) - - candidates = candidates.filter(query) - - if candidates.count() > 0: - candidates.order_by('build__completed_on') - similar_target = candidates.last() - - return similar_target - - def clone_image_artifacts_from(self, target): - """ - Make clones of the Target_Image_Files and TargetKernelFile objects - associated with Target target, then associate them with this target. - - Note that for Target_Image_Files, we only want files from the previous - build whose suffix matches one of the suffixes defined in this - target's build's IMAGE_FSTYPES configuration variable. This prevents the - Target_Image_File object for an ext4 image being associated with a - target for a project which didn't produce an ext4 image (for example). - - Also sets the license_manifest_path and package_manifest_path - of this target to the same path as that of target being cloned from, as - the manifests are also build artifacts but are treated differently. - """ - - image_fstypes = self.build.get_image_fstypes() - - # filter out any image files whose suffixes aren't in the - # IMAGE_FSTYPES suffixes variable for this target's build - image_files = [target_image_file \ - for target_image_file in target.target_image_file_set.all() \ - if target_image_file.suffix in image_fstypes] - - for image_file in image_files: - image_file.pk = None - image_file.target = self - image_file.save() - - kernel_files = target.targetkernelfile_set.all() - for kernel_file in kernel_files: - kernel_file.pk = None - kernel_file.target = self - kernel_file.save() - - self.license_manifest_path = target.license_manifest_path - self.package_manifest_path = target.package_manifest_path - self.save() - - def clone_sdk_artifacts_from(self, target): - """ - Clone TargetSDKFile objects from target and associate them with this - target. - """ - sdk_files = target.targetsdkfile_set.all() - for sdk_file in sdk_files: - sdk_file.pk = None - sdk_file.target = self - sdk_file.save() - - def has_images(self): - """ - Returns True if this target has one or more image files attached to it. - """ - return self.target_image_file_set.all().count() > 0 - -# kernel artifacts for a target: bzImage and modules* -class TargetKernelFile(models.Model): - target = models.ForeignKey(Target) - file_name = models.FilePathField() - file_size = models.IntegerField() - - @property - def basename(self): - return os.path.basename(self.file_name) - -# SDK artifacts for a target: sh and manifest files -class TargetSDKFile(models.Model): - target = models.ForeignKey(Target) - file_name = models.FilePathField() - file_size = models.IntegerField() - - @property - def basename(self): - return os.path.basename(self.file_name) - -class Target_Image_File(models.Model): - # valid suffixes for image files produced by a build - SUFFIXES = { - 'btrfs', 'cpio', 'cpio.gz', 'cpio.lz4', 'cpio.lzma', 'cpio.xz', - 'cramfs', 'elf', 'ext2', 'ext2.bz2', 'ext2.gz', 'ext2.lzma', 'ext4', - 'ext4.gz', 'ext3', 'ext3.gz', 'hdddirect', 'hddimg', 'iso', 'jffs2', - 'jffs2.sum', 'multiubi', 'qcow2', 'squashfs', 'squashfs-lzo', - 'squashfs-xz', 'tar', 'tar.bz2', 'tar.gz', 'tar.lz4', 'tar.xz', 'ubi', - 'ubifs', 'vdi', 'vmdk', 'wic', 'wic.bmap', 'wic.bz2', 'wic.gz', 'wic.lzma' - } - - target = models.ForeignKey(Target) - file_name = models.FilePathField(max_length=254) - file_size = models.IntegerField() - - @property - def suffix(self): - """ - Suffix for image file, minus leading "." - """ - for suffix in Target_Image_File.SUFFIXES: - if self.file_name.endswith(suffix): - return suffix - - filename, suffix = os.path.splitext(self.file_name) - suffix = suffix.lstrip('.') - return suffix - -class Target_File(models.Model): - ITYPE_REGULAR = 1 - ITYPE_DIRECTORY = 2 - ITYPE_SYMLINK = 3 - ITYPE_SOCKET = 4 - ITYPE_FIFO = 5 - ITYPE_CHARACTER = 6 - ITYPE_BLOCK = 7 - ITYPES = ( (ITYPE_REGULAR ,'regular'), - ( ITYPE_DIRECTORY ,'directory'), - ( ITYPE_SYMLINK ,'symlink'), - ( ITYPE_SOCKET ,'socket'), - ( ITYPE_FIFO ,'fifo'), - ( ITYPE_CHARACTER ,'character'), - ( ITYPE_BLOCK ,'block'), - ) - - target = models.ForeignKey(Target) - path = models.FilePathField() - size = models.IntegerField() - inodetype = models.IntegerField(choices = ITYPES) - permission = models.CharField(max_length=16) - owner = models.CharField(max_length=128) - group = models.CharField(max_length=128) - directory = models.ForeignKey('Target_File', related_name="directory_set", null=True) - sym_target = models.ForeignKey('Target_File', related_name="symlink_set", null=True) - - -class Task(models.Model): - - SSTATE_NA = 0 - SSTATE_MISS = 1 - SSTATE_FAILED = 2 - SSTATE_RESTORED = 3 - - SSTATE_RESULT = ( - (SSTATE_NA, 'Not Applicable'), # For rest of tasks, but they still need checking. - (SSTATE_MISS, 'File not in cache'), # the sstate object was not found - (SSTATE_FAILED, 'Failed'), # there was a pkg, but the script failed - (SSTATE_RESTORED, 'Succeeded'), # successfully restored - ) - - CODING_NA = 0 - CODING_PYTHON = 2 - CODING_SHELL = 3 - - TASK_CODING = ( - (CODING_NA, 'N/A'), - (CODING_PYTHON, 'Python'), - (CODING_SHELL, 'Shell'), - ) - - OUTCOME_NA = -1 - OUTCOME_SUCCESS = 0 - OUTCOME_COVERED = 1 - OUTCOME_CACHED = 2 - OUTCOME_PREBUILT = 3 - OUTCOME_FAILED = 4 - OUTCOME_EMPTY = 5 - - TASK_OUTCOME = ( - (OUTCOME_NA, 'Not Available'), - (OUTCOME_SUCCESS, 'Succeeded'), - (OUTCOME_COVERED, 'Covered'), - (OUTCOME_CACHED, 'Cached'), - (OUTCOME_PREBUILT, 'Prebuilt'), - (OUTCOME_FAILED, 'Failed'), - (OUTCOME_EMPTY, 'Empty'), - ) - - TASK_OUTCOME_HELP = ( - (OUTCOME_SUCCESS, 'This task successfully completed'), - (OUTCOME_COVERED, 'This task did not run because its output is provided by another task'), - (OUTCOME_CACHED, 'This task restored output from the sstate-cache directory or mirrors'), - (OUTCOME_PREBUILT, 'This task did not run because its outcome was reused from a previous build'), - (OUTCOME_FAILED, 'This task did not complete'), - (OUTCOME_EMPTY, 'This task has no executable content'), - (OUTCOME_NA, ''), - ) - - search_allowed_fields = [ "recipe__name", "recipe__version", "task_name", "logfile" ] - - def __init__(self, *args, **kwargs): - super(Task, self).__init__(*args, **kwargs) - try: - self._helptext = HelpText.objects.get(key=self.task_name, area=HelpText.VARIABLE, build=self.build).text - except HelpText.DoesNotExist: - self._helptext = None - - def get_related_setscene(self): - return Task.objects.filter(task_executed=True, build = self.build, recipe = self.recipe, task_name=self.task_name+"_setscene") - - def get_outcome_text(self): - return Task.TASK_OUTCOME[int(self.outcome) + 1][1] - - def get_outcome_help(self): - return Task.TASK_OUTCOME_HELP[int(self.outcome)][1] - - def get_sstate_text(self): - if self.sstate_result==Task.SSTATE_NA: - return '' - else: - return Task.SSTATE_RESULT[int(self.sstate_result)][1] - - def get_executed_display(self): - if self.task_executed: - return "Executed" - return "Not Executed" - - def get_description(self): - return self._helptext - - build = models.ForeignKey(Build, related_name='task_build') - order = models.IntegerField(null=True) - task_executed = models.BooleanField(default=False) # True means Executed, False means Not/Executed - outcome = models.IntegerField(choices=TASK_OUTCOME, default=OUTCOME_NA) - sstate_checksum = models.CharField(max_length=100, blank=True) - path_to_sstate_obj = models.FilePathField(max_length=500, blank=True) - recipe = models.ForeignKey('Recipe', related_name='tasks') - task_name = models.CharField(max_length=100) - source_url = models.FilePathField(max_length=255, blank=True) - work_directory = models.FilePathField(max_length=255, blank=True) - script_type = models.IntegerField(choices=TASK_CODING, default=CODING_NA) - line_number = models.IntegerField(default=0) - - # start/end times - started = models.DateTimeField(null=True) - ended = models.DateTimeField(null=True) - - # in seconds; this is stored to enable sorting - elapsed_time = models.DecimalField(max_digits=8, decimal_places=2, null=True) - - # in bytes; note that disk_io is stored to enable sorting - disk_io = models.IntegerField(null=True) - disk_io_read = models.IntegerField(null=True) - disk_io_write = models.IntegerField(null=True) - - # in seconds - cpu_time_user = models.DecimalField(max_digits=8, decimal_places=2, null=True) - cpu_time_system = models.DecimalField(max_digits=8, decimal_places=2, null=True) - - sstate_result = models.IntegerField(choices=SSTATE_RESULT, default=SSTATE_NA) - message = models.CharField(max_length=240) - logfile = models.FilePathField(max_length=255, blank=True) - - outcome_text = property(get_outcome_text) - sstate_text = property(get_sstate_text) - - def __unicode__(self): - return "%d(%d) %s:%s" % (self.pk, self.build.pk, self.recipe.name, self.task_name) - - class Meta: - ordering = ('order', 'recipe' ,) - unique_together = ('build', 'recipe', 'task_name', ) - - -class Task_Dependency(models.Model): - task = models.ForeignKey(Task, related_name='task_dependencies_task') - depends_on = models.ForeignKey(Task, related_name='task_dependencies_depends') - -class Package(models.Model): - search_allowed_fields = ['name', 'version', 'revision', 'recipe__name', 'recipe__version', 'recipe__license', 'recipe__layer_version__layer__name', 'recipe__layer_version__branch', 'recipe__layer_version__commit', 'recipe__layer_version__local_path', 'installed_name'] - build = models.ForeignKey('Build', null=True) - recipe = models.ForeignKey('Recipe', null=True) - name = models.CharField(max_length=100) - installed_name = models.CharField(max_length=100, default='') - version = models.CharField(max_length=100, blank=True) - revision = models.CharField(max_length=32, blank=True) - summary = models.TextField(blank=True) - description = models.TextField(blank=True) - size = models.IntegerField(default=0) - installed_size = models.IntegerField(default=0) - section = models.CharField(max_length=80, blank=True) - license = models.CharField(max_length=80, blank=True) - - @property - def is_locale_package(self): - """ Returns True if this package is identifiable as a locale package """ - if self.name.find('locale') != -1: - return True - return False - - @property - def is_packagegroup(self): - """ Returns True is this package is identifiable as a packagegroup """ - if self.name.find('packagegroup') != -1: - return True - return False - -class CustomImagePackage(Package): - # CustomImageRecipe fields to track pacakges appended, - # included and excluded from a CustomImageRecipe - recipe_includes = models.ManyToManyField('CustomImageRecipe', - related_name='includes_set') - recipe_excludes = models.ManyToManyField('CustomImageRecipe', - related_name='excludes_set') - recipe_appends = models.ManyToManyField('CustomImageRecipe', - related_name='appends_set') - - -class Package_DependencyManager(models.Manager): - use_for_related_fields = True - TARGET_LATEST = "use-latest-target-for-target" - - def get_queryset(self): - return super(Package_DependencyManager, self).get_queryset().exclude(package_id = F('depends_on__id')) - - def for_target_or_none(self, target): - """ filter the dependencies to be displayed by the supplied target - if no dependences are found for the target then try None as the target - which will return the dependences calculated without the context of a - target e.g. non image recipes. - - returns: { size, packages } - """ - package_dependencies = self.all_depends().order_by('depends_on__name') - - if target is self.TARGET_LATEST: - installed_deps =\ - package_dependencies.filter(~Q(target__target=None)) - else: - installed_deps =\ - package_dependencies.filter(Q(target__target=target)) - - packages_list = None - total_size = 0 - - # If we have installed depdencies for this package and target then use - # these to display - if installed_deps.count() > 0: - packages_list = installed_deps - total_size = installed_deps.aggregate( - Sum('depends_on__size'))['depends_on__size__sum'] - else: - new_list = [] - package_names = [] - - # Find dependencies for the package that we know about even if - # it's not installed on a target e.g. from a non-image recipe - for p in package_dependencies.filter(Q(target=None)): - if p.depends_on.name in package_names: - continue - else: - package_names.append(p.depends_on.name) - new_list.append(p.pk) - # while we're here we may as well total up the size to - # avoid iterating again - total_size += p.depends_on.size - - # We want to return a queryset here for consistency so pick the - # deps from the new_list - packages_list = package_dependencies.filter(Q(pk__in=new_list)) - - return {'packages': packages_list, - 'size': total_size} - - def all_depends(self): - """ Returns just the depends packages and not any other dep_type - Note that this is for any target - """ - return self.filter(Q(dep_type=Package_Dependency.TYPE_RDEPENDS) | - Q(dep_type=Package_Dependency.TYPE_TRDEPENDS)) - - -class Package_Dependency(models.Model): - TYPE_RDEPENDS = 0 - TYPE_TRDEPENDS = 1 - TYPE_RRECOMMENDS = 2 - TYPE_TRECOMMENDS = 3 - TYPE_RSUGGESTS = 4 - TYPE_RPROVIDES = 5 - TYPE_RREPLACES = 6 - TYPE_RCONFLICTS = 7 - ' TODO: bpackage should be changed to remove the DEPENDS_TYPE access ' - DEPENDS_TYPE = ( - (TYPE_RDEPENDS, "depends"), - (TYPE_TRDEPENDS, "depends"), - (TYPE_TRECOMMENDS, "recommends"), - (TYPE_RRECOMMENDS, "recommends"), - (TYPE_RSUGGESTS, "suggests"), - (TYPE_RPROVIDES, "provides"), - (TYPE_RREPLACES, "replaces"), - (TYPE_RCONFLICTS, "conflicts"), - ) - """ Indexed by dep_type, in view order, key for short name and help - description which when viewed will be printf'd with the - package name. - """ - DEPENDS_DICT = { - TYPE_RDEPENDS : ("depends", "%s is required to run %s"), - TYPE_TRDEPENDS : ("depends", "%s is required to run %s"), - TYPE_TRECOMMENDS : ("recommends", "%s extends the usability of %s"), - TYPE_RRECOMMENDS : ("recommends", "%s extends the usability of %s"), - TYPE_RSUGGESTS : ("suggests", "%s is suggested for installation with %s"), - TYPE_RPROVIDES : ("provides", "%s is provided by %s"), - TYPE_RREPLACES : ("replaces", "%s is replaced by %s"), - TYPE_RCONFLICTS : ("conflicts", "%s conflicts with %s, which will not be installed if this package is not first removed"), - } - - package = models.ForeignKey(Package, related_name='package_dependencies_source') - depends_on = models.ForeignKey(Package, related_name='package_dependencies_target') # soft dependency - dep_type = models.IntegerField(choices=DEPENDS_TYPE) - target = models.ForeignKey(Target, null=True) - objects = Package_DependencyManager() - -class Target_Installed_Package(models.Model): - target = models.ForeignKey(Target) - package = models.ForeignKey(Package, related_name='buildtargetlist_package') - - -class Package_File(models.Model): - package = models.ForeignKey(Package, related_name='buildfilelist_package') - path = models.FilePathField(max_length=255, blank=True) - size = models.IntegerField() - - -class Recipe(models.Model): - search_allowed_fields = ['name', 'version', 'file_path', 'section', - 'summary', 'description', 'license', - 'layer_version__layer__name', - 'layer_version__branch', 'layer_version__commit', - 'layer_version__local_path', - 'layer_version__layer_source'] - - up_date = models.DateTimeField(null=True, default=None) - - name = models.CharField(max_length=100, blank=True) - version = models.CharField(max_length=100, blank=True) - layer_version = models.ForeignKey('Layer_Version', - related_name='recipe_layer_version') - summary = models.TextField(blank=True) - description = models.TextField(blank=True) - section = models.CharField(max_length=100, blank=True) - license = models.CharField(max_length=200, blank=True) - homepage = models.URLField(blank=True) - bugtracker = models.URLField(blank=True) - file_path = models.FilePathField(max_length=255) - pathflags = models.CharField(max_length=200, blank=True) - is_image = models.BooleanField(default=False) - - def __unicode__(self): - return "Recipe " + self.name + ":" + self.version - - def get_vcs_recipe_file_link_url(self): - return self.layer_version.get_vcs_file_link_url(self.file_path) - - def get_description_or_summary(self): - if self.description: - return self.description - elif self.summary: - return self.summary - else: - return "" - - class Meta: - unique_together = (("layer_version", "file_path", "pathflags"), ) - - -class Recipe_DependencyManager(models.Manager): - use_for_related_fields = True - - def get_queryset(self): - return super(Recipe_DependencyManager, self).get_queryset().exclude(recipe_id = F('depends_on__id')) - -class Provides(models.Model): - name = models.CharField(max_length=100) - recipe = models.ForeignKey(Recipe) - -class Recipe_Dependency(models.Model): - TYPE_DEPENDS = 0 - TYPE_RDEPENDS = 1 - - DEPENDS_TYPE = ( - (TYPE_DEPENDS, "depends"), - (TYPE_RDEPENDS, "rdepends"), - ) - recipe = models.ForeignKey(Recipe, related_name='r_dependencies_recipe') - depends_on = models.ForeignKey(Recipe, related_name='r_dependencies_depends') - via = models.ForeignKey(Provides, null=True, default=None) - dep_type = models.IntegerField(choices=DEPENDS_TYPE) - objects = Recipe_DependencyManager() - - -class Machine(models.Model): - search_allowed_fields = ["name", "description", "layer_version__layer__name"] - up_date = models.DateTimeField(null = True, default = None) - - layer_version = models.ForeignKey('Layer_Version') - name = models.CharField(max_length=255) - description = models.CharField(max_length=255) - - def get_vcs_machine_file_link_url(self): - path = 'conf/machine/'+self.name+'.conf' - - return self.layer_version.get_vcs_file_link_url(path) - - def __unicode__(self): - return "Machine " + self.name + "(" + self.description + ")" - - - - - -class BitbakeVersion(models.Model): - - name = models.CharField(max_length=32, unique = True) - giturl = GitURLField() - branch = models.CharField(max_length=32) - dirpath = models.CharField(max_length=255) - - def __unicode__(self): - return "%s (Branch: %s)" % (self.name, self.branch) - - -class Release(models.Model): - """ A release is a project template, used to pre-populate Project settings with a configuration set """ - name = models.CharField(max_length=32, unique = True) - description = models.CharField(max_length=255) - bitbake_version = models.ForeignKey(BitbakeVersion) - branch_name = models.CharField(max_length=50, default = "") - helptext = models.TextField(null=True) - - def __unicode__(self): - return "%s (%s)" % (self.name, self.branch_name) - - def __str__(self): - return self.name - -class ReleaseDefaultLayer(models.Model): - release = models.ForeignKey(Release) - layer_name = models.CharField(max_length=100, default="") - - -class LayerSource(object): - """ Where the layer metadata came from """ - TYPE_LOCAL = 0 - TYPE_LAYERINDEX = 1 - TYPE_IMPORTED = 2 - TYPE_BUILD = 3 - - SOURCE_TYPE = ( - (TYPE_LOCAL, "local"), - (TYPE_LAYERINDEX, "layerindex"), - (TYPE_IMPORTED, "imported"), - (TYPE_BUILD, "build"), - ) - - def types_dict(): - """ Turn the TYPES enums into a simple dictionary """ - dictionary = {} - for key in LayerSource.__dict__: - if "TYPE" in key: - dictionary[key] = getattr(LayerSource, key) - return dictionary - - -class Layer(models.Model): - - up_date = models.DateTimeField(null=True, default=timezone.now) - - name = models.CharField(max_length=100) - layer_index_url = models.URLField() - vcs_url = GitURLField(default=None, null=True) - local_source_dir = models.TextField(null=True, default=None) - vcs_web_url = models.URLField(null=True, default=None) - vcs_web_tree_base_url = models.URLField(null=True, default=None) - vcs_web_file_base_url = models.URLField(null=True, default=None) - - summary = models.TextField(help_text='One-line description of the layer', - null=True, default=None) - description = models.TextField(null=True, default=None) - - def __unicode__(self): - return "%s / %s " % (self.name, self.summary) - - -class Layer_Version(models.Model): - """ - A Layer_Version either belongs to a single project or no project - """ - search_allowed_fields = ["layer__name", "layer__summary", - "layer__description", "layer__vcs_url", - "dirpath", "release__name", "commit", "branch"] - - build = models.ForeignKey(Build, related_name='layer_version_build', - default=None, null=True) - - layer = models.ForeignKey(Layer, related_name='layer_version_layer') - - layer_source = models.IntegerField(choices=LayerSource.SOURCE_TYPE, - default=0) - - up_date = models.DateTimeField(null=True, default=timezone.now) - - # To which metadata release does this layer version belong to - release = models.ForeignKey(Release, null=True, default=None) - - branch = models.CharField(max_length=80) - commit = models.CharField(max_length=100) - # If the layer is in a subdir - dirpath = models.CharField(max_length=255, null=True, default=None) - - # if -1, this is a default layer - priority = models.IntegerField(default=0) - - # where this layer exists on the filesystem - local_path = models.FilePathField(max_length=1024, default="/") - - # Set if this layer is restricted to a particular project - project = models.ForeignKey('Project', null=True, default=None) - - # code lifted, with adaptations, from the layerindex-web application - # https://git.yoctoproject.org/cgit/cgit.cgi/layerindex-web/ - def _handle_url_path(self, base_url, path): - import re, posixpath - if base_url: - if self.dirpath: - if path: - extra_path = self.dirpath + '/' + path - # Normalise out ../ in path for usage URL - extra_path = posixpath.normpath(extra_path) - # Minor workaround to handle case where subdirectory has been added between branches - # (should probably support usage URL per branch to handle this... sigh...) - if extra_path.startswith('../'): - extra_path = extra_path[3:] - else: - extra_path = self.dirpath - else: - extra_path = path - branchname = self.release.name - url = base_url.replace('%branch%', branchname) - - # If there's a % in the path (e.g. a wildcard bbappend) we need to encode it - if extra_path: - extra_path = extra_path.replace('%', '%25') - - if '%path%' in base_url: - if extra_path: - url = re.sub(r'\[([^\]]*%path%[^\]]*)\]', '\\1', url) - else: - url = re.sub(r'\[([^\]]*%path%[^\]]*)\]', '', url) - return url.replace('%path%', extra_path) - else: - return url + extra_path - return None - - def get_vcs_link_url(self): - if self.layer.vcs_web_url is None: - return None - return self.layer.vcs_web_url - - def get_vcs_file_link_url(self, file_path=""): - if self.layer.vcs_web_file_base_url is None: - return None - return self._handle_url_path(self.layer.vcs_web_file_base_url, - file_path) - - def get_vcs_dirpath_link_url(self): - if self.layer.vcs_web_tree_base_url is None: - return None - return self._handle_url_path(self.layer.vcs_web_tree_base_url, '') - - def get_vcs_reference(self): - if self.commit is not None and len(self.commit) > 0: - return self.commit - if self.branch is not None and len(self.branch) > 0: - return self.branch - if self.release is not None: - return self.release.name - return 'N/A' - - def get_detailspage_url(self, project_id=None): - """ returns the url to the layer details page uses own project - field if project_id is not specified """ - - if project_id is None: - project_id = self.project.pk - - return reverse('layerdetails', args=(project_id, self.pk)) - - def get_alldeps(self, project_id): - """Get full list of unique layer dependencies.""" - def gen_layerdeps(lver, project, depth): - if depth == 0: - return - for ldep in lver.dependencies.all(): - yield ldep.depends_on - # get next level of deps recursively calling gen_layerdeps - for subdep in gen_layerdeps(ldep.depends_on, project, depth-1): - yield subdep - - project = Project.objects.get(pk=project_id) - result = [] - projectlvers = [player.layercommit for player in - project.projectlayer_set.all()] - # protect against infinite layer dependency loops - maxdepth = 20 - for dep in gen_layerdeps(self, project, maxdepth): - # filter out duplicates and layers already belonging to the project - if dep not in result + projectlvers: - result.append(dep) - - return sorted(result, key=lambda x: x.layer.name) - - def __unicode__(self): - return ("id %d belongs to layer: %s" % (self.pk, self.layer.name)) - - def __str__(self): - if self.release: - release = self.release.name - else: - release = "No release set" - - return "%d %s (%s)" % (self.pk, self.layer.name, release) - - -class LayerVersionDependency(models.Model): - - layer_version = models.ForeignKey(Layer_Version, - related_name="dependencies") - depends_on = models.ForeignKey(Layer_Version, - related_name="dependees") - -class ProjectLayer(models.Model): - project = models.ForeignKey(Project) - layercommit = models.ForeignKey(Layer_Version, null=True) - optional = models.BooleanField(default = True) - - def __unicode__(self): - return "%s, %s" % (self.project.name, self.layercommit) - - class Meta: - unique_together = (("project", "layercommit"),) - -class CustomImageRecipe(Recipe): - - # CustomImageRecipe's belong to layers called: - LAYER_NAME = "toaster-custom-images" - - search_allowed_fields = ['name'] - base_recipe = models.ForeignKey(Recipe, related_name='based_on_recipe') - project = models.ForeignKey(Project) - last_updated = models.DateTimeField(null=True, default=None) - - def get_last_successful_built_target(self): - """ Return the last successful built target object if one exists - otherwise return None """ - return Target.objects.filter(Q(build__outcome=Build.SUCCEEDED) & - Q(build__project=self.project) & - Q(target=self.name)).last() - - def update_package_list(self): - """ Update the package list from the last good build of this - CustomImageRecipe - """ - # Check if we're aldready up-to-date or not - target = self.get_last_successful_built_target() - if target == None: - # So we've never actually built this Custom recipe but what about - # the recipe it's based on? - target = \ - Target.objects.filter(Q(build__outcome=Build.SUCCEEDED) & - Q(build__project=self.project) & - Q(target=self.base_recipe.name)).last() - if target == None: - return - - if target.build.completed_on == self.last_updated: - return - - self.includes_set.clear() - - excludes_list = self.excludes_set.values_list('name', flat=True) - appends_list = self.appends_set.values_list('name', flat=True) - - built_packages_list = \ - target.target_installed_package_set.values_list('package__name', - flat=True) - for built_package in built_packages_list: - # Is the built package in the custom packages list? - if built_package in excludes_list: - continue - - if built_package in appends_list: - continue - - cust_img_p = \ - CustomImagePackage.objects.get(name=built_package) - self.includes_set.add(cust_img_p) - - - self.last_updated = target.build.completed_on - self.save() - - def get_all_packages(self): - """Get the included packages and any appended packages""" - self.update_package_list() - - return CustomImagePackage.objects.filter((Q(recipe_appends=self) | - Q(recipe_includes=self)) & - ~Q(recipe_excludes=self)) - - def get_base_recipe_file(self): - """Get the base recipe file path if it exists on the file system""" - path_schema_one = "%s/%s" % (self.base_recipe.layer_version.local_path, - self.base_recipe.file_path) - - path_schema_two = self.base_recipe.file_path - - if os.path.exists(path_schema_one): - return path_schema_one - - # The path may now be the full path if the recipe has been built - if os.path.exists(path_schema_two): - return path_schema_two - - return None - - def generate_recipe_file_contents(self): - """Generate the contents for the recipe file.""" - # If we have no excluded packages we only need to _append - if self.excludes_set.count() == 0: - packages_conf = "IMAGE_INSTALL_append = \" " - - for pkg in self.appends_set.all(): - packages_conf += pkg.name+' ' - else: - packages_conf = "IMAGE_FEATURES =\"\"\nIMAGE_INSTALL = \"" - # We add all the known packages to be built by this recipe apart - # from locale packages which are are controlled with IMAGE_LINGUAS. - for pkg in self.get_all_packages().exclude( - name__icontains="locale"): - packages_conf += pkg.name+' ' - - packages_conf += "\"" - - base_recipe_path = self.get_base_recipe_file() - if base_recipe_path: - base_recipe = open(base_recipe_path, 'r').read() - else: - raise IOError("Based on recipe file not found: %s" % - base_recipe_path) - - # Add a special case for when the recipe we have based a custom image - # recipe on requires another recipe. - # For example: - # "require core-image-minimal.bb" is changed to: - # "require recipes-core/images/core-image-minimal.bb" - - req_search = re.search(r'(require\s+)(.+\.bb\s*$)', - base_recipe, - re.MULTILINE) - if req_search: - require_filename = req_search.group(2).strip() - - corrected_location = Recipe.objects.filter( - Q(layer_version=self.base_recipe.layer_version) & - Q(file_path__icontains=require_filename)).last().file_path - - new_require_line = "require %s" % corrected_location - - base_recipe = base_recipe.replace(req_search.group(0), - new_require_line) - - info = { - "date": timezone.now().strftime("%Y-%m-%d %H:%M:%S"), - "base_recipe": base_recipe, - "recipe_name": self.name, - "base_recipe_name": self.base_recipe.name, - "license": self.license, - "summary": self.summary, - "description": self.description, - "packages_conf": packages_conf.strip() - } - - recipe_contents = ("# Original recipe %(base_recipe_name)s \n" - "%(base_recipe)s\n\n" - "# Recipe %(recipe_name)s \n" - "# Customisation Generated by Toaster on %(date)s\n" - "SUMMARY = \"%(summary)s\"\n" - "DESCRIPTION = \"%(description)s\"\n" - "LICENSE = \"%(license)s\"\n" - "%(packages_conf)s") % info - - return recipe_contents - -class ProjectVariable(models.Model): - project = models.ForeignKey(Project) - name = models.CharField(max_length=100) - value = models.TextField(blank = True) - -class Variable(models.Model): - search_allowed_fields = ['variable_name', 'variable_value', - 'vhistory__file_name', "description"] - build = models.ForeignKey(Build, related_name='variable_build') - variable_name = models.CharField(max_length=100) - variable_value = models.TextField(blank=True) - changed = models.BooleanField(default=False) - human_readable_name = models.CharField(max_length=200) - description = models.TextField(blank=True) - -class VariableHistory(models.Model): - variable = models.ForeignKey(Variable, related_name='vhistory') - value = models.TextField(blank=True) - file_name = models.FilePathField(max_length=255) - line_number = models.IntegerField(null=True) - operation = models.CharField(max_length=64) - -class HelpText(models.Model): - VARIABLE = 0 - HELPTEXT_AREA = ((VARIABLE, 'variable'), ) - - build = models.ForeignKey(Build, related_name='helptext_build') - area = models.IntegerField(choices=HELPTEXT_AREA) - key = models.CharField(max_length=100) - text = models.TextField() - -class LogMessage(models.Model): - EXCEPTION = -1 # used to signal self-toaster-exceptions - INFO = 0 - WARNING = 1 - ERROR = 2 - CRITICAL = 3 - - LOG_LEVEL = ( - (INFO, "info"), - (WARNING, "warn"), - (ERROR, "error"), - (CRITICAL, "critical"), - (EXCEPTION, "toaster exception") - ) - - build = models.ForeignKey(Build) - task = models.ForeignKey(Task, blank = True, null=True) - level = models.IntegerField(choices=LOG_LEVEL, default=INFO) - message = models.TextField(blank=True, null=True) - pathname = models.FilePathField(max_length=255, blank=True) - lineno = models.IntegerField(null=True) - - def __str__(self): - return force_bytes('%s %s %s' % (self.get_level_display(), self.message, self.build)) - -def invalidate_cache(**kwargs): - from django.core.cache import cache - try: - cache.clear() - except Exception as e: - logger.warning("Problem with cache backend: Failed to clear cache: %s" % e) - -def signal_runbuilds(): - """Send SIGUSR1 to runbuilds process""" - try: - with open(os.path.join(os.getenv('BUILDDIR', '.'), - '.runbuilds.pid')) as pidf: - os.kill(int(pidf.read()), SIGUSR1) - except FileNotFoundError: - logger.info("Stopping existing runbuilds: no current process found") - -class Distro(models.Model): - search_allowed_fields = ["name", "description", "layer_version__layer__name"] - up_date = models.DateTimeField(null = True, default = None) - - layer_version = models.ForeignKey('Layer_Version') - name = models.CharField(max_length=255) - description = models.CharField(max_length=255) - - def get_vcs_distro_file_link_url(self): - path = self.name+'.conf' - return self.layer_version.get_vcs_file_link_url(path) - - def __unicode__(self): - return "Distro " + self.name + "(" + self.description + ")" - -django.db.models.signals.post_save.connect(invalidate_cache) -django.db.models.signals.post_delete.connect(invalidate_cache) -django.db.models.signals.m2m_changed.connect(invalidate_cache) |
