diff options
Diffstat (limited to 'import-layers/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py')
| -rw-r--r-- | import-layers/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py | 1521 |
1 files changed, 1521 insertions, 0 deletions
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py b/import-layers/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py new file mode 100644 index 000000000..93979054d --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py @@ -0,0 +1,1521 @@ +# +# BitBake ToasterUI Implementation +# +# Copyright (C) 2013 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import sys +import bb +import re +import os + +os.environ["DJANGO_SETTINGS_MODULE"] = "toaster.toastermain.settings" + + +import django +from django.utils import timezone + + +def _configure_toaster(): + """ Add toaster to sys path for importing modules + """ + sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'toaster')) +_configure_toaster() + +django.setup() + +from orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage, HelpText +from orm.models import Target_Image_File, BuildArtifact +from orm.models import Variable, VariableHistory +from orm.models import Package, Package_File, Target_Installed_Package, Target_File +from orm.models import Task_Dependency, Package_Dependency +from orm.models import Recipe_Dependency, Provides +from orm.models import Project, CustomImagePackage, CustomImageRecipe + +from bldcontrol.models import BuildEnvironment, BuildRequest + +from bb.msg import BBLogFormatter as formatter +from django.db import models +from pprint import pformat +import logging +from datetime import datetime, timedelta + +from django.db import transaction, connection + +# pylint: disable=invalid-name +# the logger name is standard throughout BitBake +logger = logging.getLogger("ToasterLogger") + + +class NotExisting(Exception): + pass + +class ORMWrapper(object): + """ This class creates the dictionaries needed to store information in the database + following the format defined by the Django models. It is also used to save this + information in the database. + """ + + def __init__(self): + self.layer_version_objects = [] + self.layer_version_built = [] + self.task_objects = {} + self.recipe_objects = {} + + @staticmethod + def _build_key(**kwargs): + key = "0" + for k in sorted(kwargs.keys()): + if isinstance(kwargs[k], models.Model): + key += "-%d" % kwargs[k].id + else: + key += "-%s" % str(kwargs[k]) + return key + + + def _cached_get_or_create(self, clazz, **kwargs): + """ This is a memory-cached get_or_create. We assume that the objects will not be created in the + database through any other means. + """ + + assert issubclass(clazz, models.Model), "_cached_get_or_create needs to get the class as first argument" + + key = ORMWrapper._build_key(**kwargs) + dictname = "objects_%s" % clazz.__name__ + if not dictname in vars(self).keys(): + vars(self)[dictname] = {} + + created = False + if not key in vars(self)[dictname].keys(): + vars(self)[dictname][key], created = \ + clazz.objects.get_or_create(**kwargs) + + return (vars(self)[dictname][key], created) + + + def _cached_get(self, clazz, **kwargs): + """ This is a memory-cached get. We assume that the objects will not change in the database between gets. + """ + assert issubclass(clazz, models.Model), "_cached_get needs to get the class as first argument" + + key = ORMWrapper._build_key(**kwargs) + dictname = "objects_%s" % clazz.__name__ + + if not dictname in vars(self).keys(): + vars(self)[dictname] = {} + + if not key in vars(self)[dictname].keys(): + vars(self)[dictname][key] = clazz.objects.get(**kwargs) + + return vars(self)[dictname][key] + + def _timestamp_to_datetime(self, secs): + """ + Convert timestamp in seconds to Python datetime + """ + return datetime(1970, 1, 1) + timedelta(seconds=secs) + + # pylint: disable=no-self-use + # we disable detection of no self use in functions because the methods actually work on the object + # even if they don't touch self anywhere + + # pylint: disable=bad-continuation + # we do not follow the python conventions for continuation indentation due to long lines here + + def create_build_object(self, build_info, brbe, project_id): + assert 'machine' in build_info + assert 'distro' in build_info + assert 'distro_version' in build_info + assert 'started_on' in build_info + assert 'cooker_log_path' in build_info + assert 'build_name' in build_info + assert 'bitbake_version' in build_info + + prj = None + buildrequest = None + if brbe is not None: # this build was triggered by a request from a user + logger.debug(1, "buildinfohelper: brbe is %s" % brbe) + br, _ = brbe.split(":") + buildrequest = BuildRequest.objects.get(pk = br) + prj = buildrequest.project + + elif project_id is not None: # this build was triggered by an external system for a specific project + logger.debug(1, "buildinfohelper: project is %s" % prj) + prj = Project.objects.get(pk = project_id) + + else: # this build was triggered by a legacy system, or command line interactive mode + prj = Project.objects.get_or_create_default_project() + logger.debug(1, "buildinfohelper: project is not specified, defaulting to %s" % prj) + + + if buildrequest is not None: + build = buildrequest.build + logger.info("Updating existing build, with %s", build_info) + build.project = prj + build.machine=build_info['machine'] + build.distro=build_info['distro'] + build.distro_version=build_info['distro_version'] + build.cooker_log_path=build_info['cooker_log_path'] + build.build_name=build_info['build_name'] + build.bitbake_version=build_info['bitbake_version'] + build.save() + + else: + build = Build.objects.create( + project = prj, + machine=build_info['machine'], + distro=build_info['distro'], + distro_version=build_info['distro_version'], + started_on=build_info['started_on'], + completed_on=build_info['started_on'], + cooker_log_path=build_info['cooker_log_path'], + build_name=build_info['build_name'], + bitbake_version=build_info['bitbake_version']) + + logger.debug(1, "buildinfohelper: build is created %s" % build) + + if buildrequest is not None: + buildrequest.build = build + buildrequest.save() + + return build + + @staticmethod + def get_or_create_targets(target_info): + result = [] + for target in target_info['targets']: + task = '' + if ':' in target: + target, task = target.split(':', 1) + if task.startswith('do_'): + task = task[3:] + if task == 'build': + task = '' + obj, created = Target.objects.get_or_create(build=target_info['build'], + target=target) + if created: + obj.is_image = False + if task: + obj.task = task + obj.save() + result.append(obj) + return result + + def update_build_object(self, build, errors, warnings, taskfailures): + assert isinstance(build,Build) + assert isinstance(errors, int) + assert isinstance(warnings, int) + + if build.outcome == Build.CANCELLED: + return + try: + if build.buildrequest.state == BuildRequest.REQ_CANCELLING: + return + except AttributeError: + # We may not have a buildrequest if this is a command line build + pass + + outcome = Build.SUCCEEDED + if errors or taskfailures: + outcome = Build.FAILED + + build.completed_on = timezone.now() + build.outcome = outcome + build.save() + + def update_target_set_license_manifest(self, target, license_manifest_path): + target.license_manifest_path = license_manifest_path + target.save() + + def update_task_object(self, build, task_name, recipe_name, task_stats): + """ + Find the task for build which matches the recipe and task name + to be stored + """ + task_to_update = Task.objects.get( + build = build, + task_name = task_name, + recipe__name = recipe_name + ) + + if 'started' in task_stats and 'ended' in task_stats: + task_to_update.started = self._timestamp_to_datetime(task_stats['started']) + task_to_update.ended = self._timestamp_to_datetime(task_stats['ended']) + task_to_update.elapsed_time = (task_stats['ended'] - task_stats['started']) + task_to_update.cpu_time_user = task_stats.get('cpu_time_user') + task_to_update.cpu_time_system = task_stats.get('cpu_time_system') + if 'disk_io_read' in task_stats and 'disk_io_write' in task_stats: + task_to_update.disk_io_read = task_stats['disk_io_read'] + task_to_update.disk_io_write = task_stats['disk_io_write'] + task_to_update.disk_io = task_stats['disk_io_read'] + task_stats['disk_io_write'] + + task_to_update.save() + + def get_update_task_object(self, task_information, must_exist = False): + assert 'build' in task_information + assert 'recipe' in task_information + assert 'task_name' in task_information + + # we use must_exist info for database look-up optimization + task_object, created = self._cached_get_or_create(Task, + build=task_information['build'], + recipe=task_information['recipe'], + task_name=task_information['task_name'] + ) + if created and must_exist: + task_information['debug'] = "build id %d, recipe id %d" % (task_information['build'].pk, task_information['recipe'].pk) + raise NotExisting("Task object created when expected to exist", task_information) + + object_changed = False + for v in vars(task_object): + if v in task_information.keys(): + if vars(task_object)[v] != task_information[v]: + vars(task_object)[v] = task_information[v] + object_changed = True + + # update setscene-related information if the task has a setscene + if task_object.outcome == Task.OUTCOME_COVERED and 1 == task_object.get_related_setscene().count(): + task_object.outcome = Task.OUTCOME_CACHED + object_changed = True + + outcome_task_setscene = Task.objects.get(task_executed=True, build = task_object.build, + recipe = task_object.recipe, task_name=task_object.task_name+"_setscene").outcome + if outcome_task_setscene == Task.OUTCOME_SUCCESS: + task_object.sstate_result = Task.SSTATE_RESTORED + object_changed = True + elif outcome_task_setscene == Task.OUTCOME_FAILED: + task_object.sstate_result = Task.SSTATE_FAILED + object_changed = True + + if object_changed: + task_object.save() + return task_object + + + def get_update_recipe_object(self, recipe_information, must_exist = False): + assert 'layer_version' in recipe_information + assert 'file_path' in recipe_information + assert 'pathflags' in recipe_information + + assert not recipe_information['file_path'].startswith("/") # we should have layer-relative paths at all times + + + def update_recipe_obj(recipe_object): + object_changed = False + for v in vars(recipe_object): + if v in recipe_information.keys(): + object_changed = True + vars(recipe_object)[v] = recipe_information[v] + + if object_changed: + recipe_object.save() + + recipe, created = self._cached_get_or_create(Recipe, layer_version=recipe_information['layer_version'], + file_path=recipe_information['file_path'], pathflags = recipe_information['pathflags']) + + update_recipe_obj(recipe) + + built_recipe = None + # Create a copy of the recipe for historical puposes and update it + for built_layer in self.layer_version_built: + if built_layer.layer == recipe_information['layer_version'].layer: + built_recipe, c = self._cached_get_or_create(Recipe, + layer_version=built_layer, + file_path=recipe_information['file_path'], + pathflags = recipe_information['pathflags']) + update_recipe_obj(built_recipe) + break + + + # If we're in analysis mode or if this is a custom recipe + # then we are wholly responsible for the data + # and therefore we return the 'real' recipe rather than the build + # history copy of the recipe. + if recipe_information['layer_version'].build is not None and \ + recipe_information['layer_version'].build.project == \ + Project.objects.get_or_create_default_project(): + return recipe + + if built_recipe is None: + return recipe + + return built_recipe + + def get_update_layer_version_object(self, build_obj, layer_obj, layer_version_information): + if isinstance(layer_obj, Layer_Version): + # Special case the toaster-custom-images layer which is created + # on the fly so don't update the values which may cause the layer + # to be duplicated on a future get_or_create + if layer_obj.layer.name == CustomImageRecipe.LAYER_NAME: + return layer_obj + # We already found our layer version for this build so just + # update it with the new build information + logger.debug("We found our layer from toaster") + layer_obj.local_path = layer_version_information['local_path'] + layer_obj.save() + self.layer_version_objects.append(layer_obj) + + # create a new copy of this layer version as a snapshot for + # historical purposes + layer_copy, c = Layer_Version.objects.get_or_create( + build=build_obj, + layer=layer_obj.layer, + up_branch=layer_obj.up_branch, + branch=layer_version_information['branch'], + commit=layer_version_information['commit'], + local_path=layer_version_information['local_path'], + ) + + logger.info("created new historical layer version %d", + layer_copy.pk) + + self.layer_version_built.append(layer_copy) + + return layer_obj + + assert isinstance(build_obj, Build) + assert isinstance(layer_obj, Layer) + assert 'branch' in layer_version_information + assert 'commit' in layer_version_information + assert 'priority' in layer_version_information + assert 'local_path' in layer_version_information + + # If we're doing a command line build then associate this new layer with the + # project to avoid it 'contaminating' toaster data + project = None + if build_obj.project == Project.objects.get_or_create_default_project(): + project = build_obj.project + + layer_version_object, _ = Layer_Version.objects.get_or_create( + build = build_obj, + layer = layer_obj, + branch = layer_version_information['branch'], + commit = layer_version_information['commit'], + priority = layer_version_information['priority'], + local_path = layer_version_information['local_path'], + project=project) + + self.layer_version_objects.append(layer_version_object) + + return layer_version_object + + def get_update_layer_object(self, layer_information, brbe): + assert 'name' in layer_information + assert 'layer_index_url' in layer_information + + if brbe is None: + layer_object, _ = Layer.objects.get_or_create( + name=layer_information['name'], + layer_index_url=layer_information['layer_index_url']) + return layer_object + else: + # we are under managed mode; we must match the layer used in the Project Layer + br_id, be_id = brbe.split(":") + + # find layer by checkout path; + from bldcontrol import bbcontroller + bc = bbcontroller.getBuildEnvironmentController(pk = be_id) + + # we might have a race condition here, as the project layers may change between the build trigger and the actual build execution + # but we can only match on the layer name, so the worst thing can happen is a mis-identification of the layer, not a total failure + + # note that this is different + buildrequest = BuildRequest.objects.get(pk = br_id) + for brl in buildrequest.brlayer_set.all(): + localdirname = os.path.join(bc.getGitCloneDirectory(brl.giturl, brl.commit), brl.dirpath) + # we get a relative path, unless running in HEAD mode where the path is absolute + if not localdirname.startswith("/"): + localdirname = os.path.join(bc.be.sourcedir, localdirname) + #logger.debug(1, "Localdirname %s lcal_path %s" % (localdirname, layer_information['local_path'])) + if localdirname.startswith(layer_information['local_path']): + # If the build request came from toaster this field + # should contain the information from the layer_version + # That created this build request. + if brl.layer_version: + return brl.layer_version + + # we matched the BRLayer, but we need the layer_version that generated this BR; reverse of the Project.schedule_build() + #logger.debug(1, "Matched %s to BRlayer %s" % (pformat(layer_information["local_path"]), localdirname)) + + for pl in buildrequest.project.projectlayer_set.filter(layercommit__layer__name = brl.name): + if pl.layercommit.layer.vcs_url == brl.giturl : + layer = pl.layercommit.layer + layer.save() + return layer + + raise NotExisting("Unidentified layer %s" % pformat(layer_information)) + + + def save_target_file_information(self, build_obj, target_obj, filedata): + assert isinstance(build_obj, Build) + assert isinstance(target_obj, Target) + dirs = filedata['dirs'] + files = filedata['files'] + syms = filedata['syms'] + + # always create the root directory as a special case; + # note that this is never displayed, so the owner, group, + # size, permission are irrelevant + tf_obj = Target_File.objects.create(target = target_obj, + path = '/', + size = 0, + owner = '', + group = '', + permission = '', + inodetype = Target_File.ITYPE_DIRECTORY) + tf_obj.save() + + # insert directories, ordered by name depth + for d in sorted(dirs, key=lambda x:len(x[-1].split("/"))): + (user, group, size) = d[1:4] + permission = d[0][1:] + path = d[4].lstrip(".") + + # we already created the root directory, so ignore any + # entry for it + if len(path) == 0: + continue + + parent_path = "/".join(path.split("/")[:len(path.split("/")) - 1]) + if len(parent_path) == 0: + parent_path = "/" + parent_obj = self._cached_get(Target_File, target = target_obj, path = parent_path, inodetype = Target_File.ITYPE_DIRECTORY) + tf_obj = Target_File.objects.create( + target = target_obj, + path = unicode(path, 'utf-8'), + size = size, + inodetype = Target_File.ITYPE_DIRECTORY, + permission = permission, + owner = user, + group = group, + directory = parent_obj) + + + # we insert files + for d in files: + (user, group, size) = d[1:4] + permission = d[0][1:] + path = d[4].lstrip(".") + parent_path = "/".join(path.split("/")[:len(path.split("/")) - 1]) + inodetype = Target_File.ITYPE_REGULAR + if d[0].startswith('b'): + inodetype = Target_File.ITYPE_BLOCK + if d[0].startswith('c'): + inodetype = Target_File.ITYPE_CHARACTER + if d[0].startswith('p'): + inodetype = Target_File.ITYPE_FIFO + + tf_obj = Target_File.objects.create( + target = target_obj, + path = unicode(path, 'utf-8'), + size = size, + inodetype = inodetype, + permission = permission, + owner = user, + group = group) + parent_obj = self._cached_get(Target_File, target = target_obj, path = parent_path, inodetype = Target_File.ITYPE_DIRECTORY) + tf_obj.directory = parent_obj + tf_obj.save() + + # we insert symlinks + for d in syms: + (user, group, size) = d[1:4] + permission = d[0][1:] + path = d[4].lstrip(".") + filetarget_path = d[6] + + parent_path = "/".join(path.split("/")[:len(path.split("/")) - 1]) + if not filetarget_path.startswith("/"): + # we have a relative path, get a normalized absolute one + filetarget_path = parent_path + "/" + filetarget_path + fcp = filetarget_path.split("/") + fcpl = [] + for i in fcp: + if i == "..": + fcpl.pop() + else: + fcpl.append(i) + filetarget_path = "/".join(fcpl) + + try: + filetarget_obj = Target_File.objects.get( + target = target_obj, + path = unicode(filetarget_path, 'utf-8')) + except Target_File.DoesNotExist: + # we might have an invalid link; no way to detect this. just set it to None + filetarget_obj = None + + parent_obj = Target_File.objects.get(target = target_obj, path = parent_path, inodetype = Target_File.ITYPE_DIRECTORY) + + tf_obj = Target_File.objects.create( + target = target_obj, + path = unicode(path, 'utf-8'), + size = size, + inodetype = Target_File.ITYPE_SYMLINK, + permission = permission, + owner = user, + group = group, + directory = parent_obj, + sym_target = filetarget_obj) + + + def save_target_package_information(self, build_obj, target_obj, packagedict, pkgpnmap, recipes, built_package=False): + assert isinstance(build_obj, Build) + assert isinstance(target_obj, Target) + + errormsg = "" + for p in packagedict: + # Search name swtiches round the installed name vs package name + # by default installed name == package name + searchname = p + if p not in pkgpnmap: + logger.warning("Image packages list contains %p, but is" + " missing from all packages list where the" + " metadata comes from. Skipping...", p) + continue + + if 'OPKGN' in pkgpnmap[p].keys(): + searchname = pkgpnmap[p]['OPKGN'] + + built_recipe = recipes[pkgpnmap[p]['PN']] + + if built_package: + packagedict[p]['object'], created = Package.objects.get_or_create( build = build_obj, name = searchname ) + recipe = built_recipe + else: + packagedict[p]['object'], created = \ + CustomImagePackage.objects.get_or_create(name=searchname) + # Clear the Package_Dependency objects as we're going to update + # the CustomImagePackage with the latest dependency information + packagedict[p]['object'].package_dependencies_target.all().delete() + packagedict[p]['object'].package_dependencies_source.all().delete() + try: + recipe = self._cached_get( + Recipe, + name=built_recipe.name, + layer_version__build=None, + layer_version__up_branch= + built_recipe.layer_version.up_branch, + file_path=built_recipe.file_path, + version=built_recipe.version + ) + except (Recipe.DoesNotExist, + Recipe.MultipleObjectsReturned) as e: + logger.info("We did not find one recipe for the" + "configuration data package %s %s" % (p, e)) + continue + + if created or packagedict[p]['object'].size == -1: # save the data anyway we can, not just if it was not created here; bug [YOCTO #6887] + # fill in everything we can from the runtime-reverse package data + try: + packagedict[p]['object'].recipe = recipe + packagedict[p]['object'].version = pkgpnmap[p]['PV'] + packagedict[p]['object'].installed_name = p + packagedict[p]['object'].revision = pkgpnmap[p]['PR'] + packagedict[p]['object'].license = pkgpnmap[p]['LICENSE'] + packagedict[p]['object'].section = pkgpnmap[p]['SECTION'] + packagedict[p]['object'].summary = pkgpnmap[p]['SUMMARY'] + packagedict[p]['object'].description = pkgpnmap[p]['DESCRIPTION'] + packagedict[p]['object'].size = int(pkgpnmap[p]['PKGSIZE']) + + # no files recorded for this package, so save files info + packagefile_objects = [] + for targetpath in pkgpnmap[p]['FILES_INFO']: + targetfilesize = pkgpnmap[p]['FILES_INFO'][targetpath] + packagefile_objects.append(Package_File( package = packagedict[p]['object'], + path = targetpath, + size = targetfilesize)) + if len(packagefile_objects): + Package_File.objects.bulk_create(packagefile_objects) + except KeyError as e: + errormsg += " stpi: Key error, package %s key %s \n" % ( p, e ) + + # save disk installed size + packagedict[p]['object'].installed_size = packagedict[p]['size'] + packagedict[p]['object'].save() + + if built_package: + Target_Installed_Package.objects.create(target = target_obj, package = packagedict[p]['object']) + + packagedeps_objs = [] + for p in packagedict: + for (px,deptype) in packagedict[p]['depends']: + if deptype == 'depends': + tdeptype = Package_Dependency.TYPE_TRDEPENDS + elif deptype == 'recommends': + tdeptype = Package_Dependency.TYPE_TRECOMMENDS + + try: + packagedeps_objs.append(Package_Dependency( + package = packagedict[p]['object'], + depends_on = packagedict[px]['object'], + dep_type = tdeptype, + target = target_obj)) + except KeyError as e: + logger.warn("Could not add dependency to the package %s " + "because %s is an unknown package", p, px) + + if len(packagedeps_objs) > 0: + Package_Dependency.objects.bulk_create(packagedeps_objs) + else: + logger.info("No package dependencies created") + + if len(errormsg) > 0: + logger.warn("buildinfohelper: target_package_info could not identify recipes: \n%s", errormsg) + + def save_target_image_file_information(self, target_obj, file_name, file_size): + Target_Image_File.objects.create( target = target_obj, + file_name = file_name, + file_size = file_size) + + def save_artifact_information(self, build_obj, file_name, file_size): + # we skip the image files from other builds + if Target_Image_File.objects.filter(file_name = file_name).count() > 0: + return + + # do not update artifacts found in other builds + if BuildArtifact.objects.filter(file_name = file_name).count() > 0: + return + + BuildArtifact.objects.create(build = build_obj, file_name = file_name, file_size = file_size) + + def create_logmessage(self, log_information): + assert 'build' in log_information + assert 'level' in log_information + assert 'message' in log_information + + log_object = LogMessage.objects.create( + build = log_information['build'], + level = log_information['level'], + message = log_information['message']) + + for v in vars(log_object): + if v in log_information.keys(): + vars(log_object)[v] = log_information[v] + + return log_object.save() + + + def save_build_package_information(self, build_obj, package_info, recipes, + built_package): + # assert isinstance(build_obj, Build) + + # create and save the object + pname = package_info['PKG'] + built_recipe = recipes[package_info['PN']] + if 'OPKGN' in package_info.keys(): + pname = package_info['OPKGN'] + + if built_package: + bp_object, _ = Package.objects.get_or_create( build = build_obj, + name = pname ) + recipe = built_recipe + else: + bp_object, created = \ + CustomImagePackage.objects.get_or_create(name=pname) + try: + recipe = self._cached_get(Recipe, + name=built_recipe.name, + layer_version__build=None, + file_path=built_recipe.file_path, + version=built_recipe.version) + + except (Recipe.DoesNotExist, Recipe.MultipleObjectsReturned): + logger.debug("We did not find one recipe for the configuration" + "data package %s" % pname) + return + + bp_object.installed_name = package_info['PKG'] + bp_object.recipe = recipe + bp_object.version = package_info['PKGV'] + bp_object.revision = package_info['PKGR'] + bp_object.summary = package_info['SUMMARY'] + bp_object.description = package_info['DESCRIPTION'] + bp_object.size = int(package_info['PKGSIZE']) + bp_object.section = package_info['SECTION'] + bp_object.license = package_info['LICENSE'] + bp_object.save() + + # save any attached file information + packagefile_objects = [] + for path in package_info['FILES_INFO']: + packagefile_objects.append(Package_File( package = bp_object, + path = path, + size = package_info['FILES_INFO'][path] )) + if len(packagefile_objects): + Package_File.objects.bulk_create(packagefile_objects) + + def _po_byname(p): + if built_package: + pkg, created = Package.objects.get_or_create(build=build_obj, + name=p) + else: + pkg, created = CustomImagePackage.objects.get_or_create(name=p) + + if created: + pkg.size = -1 + pkg.save() + return pkg + + packagedeps_objs = [] + # save soft dependency information + if 'RDEPENDS' in package_info and package_info['RDEPENDS']: + for p in bb.utils.explode_deps(package_info['RDEPENDS']): + packagedeps_objs.append(Package_Dependency( package = bp_object, + depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RDEPENDS)) + if 'RPROVIDES' in package_info and package_info['RPROVIDES']: + for p in bb.utils.explode_deps(package_info['RPROVIDES']): + packagedeps_objs.append(Package_Dependency( package = bp_object, + depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RPROVIDES)) + if 'RRECOMMENDS' in package_info and package_info['RRECOMMENDS']: + for p in bb.utils.explode_deps(package_info['RRECOMMENDS']): + packagedeps_objs.append(Package_Dependency( package = bp_object, + depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RRECOMMENDS)) + if 'RSUGGESTS' in package_info and package_info['RSUGGESTS']: + for p in bb.utils.explode_deps(package_info['RSUGGESTS']): + packagedeps_objs.append(Package_Dependency( package = bp_object, + depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RSUGGESTS)) + if 'RREPLACES' in package_info and package_info['RREPLACES']: + for p in bb.utils.explode_deps(package_info['RREPLACES']): + packagedeps_objs.append(Package_Dependency( package = bp_object, + depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RREPLACES)) + if 'RCONFLICTS' in package_info and package_info['RCONFLICTS']: + for p in bb.utils.explode_deps(package_info['RCONFLICTS']): + packagedeps_objs.append(Package_Dependency( package = bp_object, + depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RCONFLICTS)) + + if len(packagedeps_objs) > 0: + Package_Dependency.objects.bulk_create(packagedeps_objs) + + return bp_object + + def save_build_variables(self, build_obj, vardump): + assert isinstance(build_obj, Build) + + for k in vardump: + desc = vardump[k]['doc'] + if desc is None: + var_words = [word for word in k.split('_')] + root_var = "_".join([word for word in var_words if word.isupper()]) + if root_var and root_var != k and root_var in vardump: + desc = vardump[root_var]['doc'] + if desc is None: + desc = '' + if len(desc): + HelpText.objects.get_or_create(build=build_obj, + area=HelpText.VARIABLE, + key=k, text=desc) + if not bool(vardump[k]['func']): + value = vardump[k]['v'] + if value is None: + value = '' + variable_obj = Variable.objects.create( build = build_obj, + variable_name = k, + variable_value = value, + description = desc) + + varhist_objects = [] + for vh in vardump[k]['history']: + if not 'documentation.conf' in vh['file']: + varhist_objects.append(VariableHistory( variable = variable_obj, + file_name = vh['file'], + line_number = vh['line'], + operation = vh['op'])) + if len(varhist_objects): + VariableHistory.objects.bulk_create(varhist_objects) + + +class MockEvent(object): + """ This object is used to create event, for which normal event-processing methods can + be used, out of data that is not coming via an actual event + """ + def __init__(self): + self.msg = None + self.levelno = None + self.taskname = None + self.taskhash = None + self.pathname = None + self.lineno = None + + +class BuildInfoHelper(object): + """ This class gathers the build information from the server and sends it + towards the ORM wrapper for storing in the database + It is instantiated once per build + Keeps in memory all data that needs matching before writing it to the database + """ + + # pylint: disable=protected-access + # the code will look into the protected variables of the event; no easy way around this + # pylint: disable=bad-continuation + # we do not follow the python conventions for continuation indentation due to long lines here + + def __init__(self, server, has_build_history = False, brbe = None): + self.internal_state = {} + self.internal_state['taskdata'] = {} + self.internal_state['targets'] = [] + self.task_order = 0 + self.autocommit_step = 1 + self.server = server + # we use manual transactions if the database doesn't autocommit on us + if not connection.features.autocommits_when_autocommit_is_off: + transaction.set_autocommit(False) + self.orm_wrapper = ORMWrapper() + self.has_build_history = has_build_history + self.tmp_dir = self.server.runCommand(["getVariable", "TMPDIR"])[0] + + # this is set for Toaster-triggered builds by localhostbecontroller + # via toasterui + self.brbe = brbe + + self.project = None + + logger.debug(1, "buildinfohelper: Build info helper inited %s" % vars(self)) + + + ################### + ## methods to convert event/external info into objects that the ORM layer uses + + + def _get_build_information(self, build_log_path): + build_info = {} + build_info['machine'] = self.server.runCommand(["getVariable", "MACHINE"])[0] + build_info['distro'] = self.server.runCommand(["getVariable", "DISTRO"])[0] + build_info['distro_version'] = self.server.runCommand(["getVariable", "DISTRO_VERSION"])[0] + build_info['started_on'] = timezone.now() + build_info['completed_on'] = timezone.now() + build_info['cooker_log_path'] = build_log_path + build_info['build_name'] = self.server.runCommand(["getVariable", "BUILDNAME"])[0] + build_info['bitbake_version'] = self.server.runCommand(["getVariable", "BB_VERSION"])[0] + build_info['project'] = self.project = self.server.runCommand(["getVariable", "TOASTER_PROJECT"])[0] + return build_info + + def _get_task_information(self, event, recipe): + assert 'taskname' in vars(event) + + task_information = {} + task_information['build'] = self.internal_state['build'] + task_information['outcome'] = Task.OUTCOME_NA + task_information['recipe'] = recipe + task_information['task_name'] = event.taskname + try: + # some tasks don't come with a hash. and that's ok + task_information['sstate_checksum'] = event.taskhash + except AttributeError: + pass + return task_information + + def _get_layer_version_for_path(self, path): + assert path.startswith("/") + assert 'build' in self.internal_state + + def _slkey_interactive(layer_version): + assert isinstance(layer_version, Layer_Version) + return len(layer_version.local_path) + + # Heuristics: we always match recipe to the deepest layer path in the discovered layers + for lvo in sorted(self.orm_wrapper.layer_version_objects, reverse=True, key=_slkey_interactive): + # we can match to the recipe file path + if path.startswith(lvo.local_path): + return lvo + + #if we get here, we didn't read layers correctly; dump whatever information we have on the error log + logger.warn("Could not match layer version for recipe path %s : %s", path, self.orm_wrapper.layer_version_objects) + + #mockup the new layer + unknown_layer, _ = Layer.objects.get_or_create(name="Unidentified layer", layer_index_url="") + unknown_layer_version_obj, _ = Layer_Version.objects.get_or_create(layer = unknown_layer, build = self.internal_state['build']) + + # append it so we don't run into this error again and again + self.orm_wrapper.layer_version_objects.append(unknown_layer_version_obj) + + return unknown_layer_version_obj + + def _get_recipe_information_from_taskfile(self, taskfile): + localfilepath = taskfile.split(":")[-1] + filepath_flags = ":".join(sorted(taskfile.split(":")[:-1])) + layer_version_obj = self._get_layer_version_for_path(localfilepath) + + + + recipe_info = {} + recipe_info['layer_version'] = layer_version_obj + recipe_info['file_path'] = localfilepath + recipe_info['pathflags'] = filepath_flags + + if recipe_info['file_path'].startswith(recipe_info['layer_version'].local_path): + recipe_info['file_path'] = recipe_info['file_path'][len(recipe_info['layer_version'].local_path):].lstrip("/") + else: + raise RuntimeError("Recipe file path %s is not under layer version at %s" % (recipe_info['file_path'], recipe_info['layer_version'].local_path)) + + return recipe_info + + def _get_path_information(self, task_object): + assert isinstance(task_object, Task) + build_stats_format = "{tmpdir}/buildstats/{buildname}/{package}/" + build_stats_path = [] + + for t in self.internal_state['targets']: + buildname = self.internal_state['build'].build_name + pe, pv = task_object.recipe.version.split(":",1) + if len(pe) > 0: + package = task_object.recipe.name + "-" + pe + "_" + pv + else: + package = task_object.recipe.name + "-" + pv + + build_stats_path.append(build_stats_format.format(tmpdir=self.tmp_dir, + buildname=buildname, + package=package)) + + return build_stats_path + + + ################################ + ## external available methods to store information + @staticmethod + def _get_data_from_event(event): + evdata = None + if '_localdata' in vars(event): + evdata = event._localdata + elif 'data' in vars(event): + evdata = event.data + else: + raise Exception("Event with neither _localdata or data properties") + return evdata + + def store_layer_info(self, event): + layerinfos = BuildInfoHelper._get_data_from_event(event) + self.internal_state['lvs'] = {} + for layer in layerinfos: + try: + self.internal_state['lvs'][self.orm_wrapper.get_update_layer_object(layerinfos[layer], self.brbe)] = layerinfos[layer]['version'] + self.internal_state['lvs'][self.orm_wrapper.get_update_layer_object(layerinfos[layer], self.brbe)]['local_path'] = layerinfos[layer]['local_path'] + except NotExisting as nee: + logger.warn("buildinfohelper: cannot identify layer exception:%s ", nee) + + + def store_started_build(self, event, build_log_path): + assert '_pkgs' in vars(event) + build_information = self._get_build_information(build_log_path) + + # Update brbe and project as they can be changed for every build + self.project = build_information['project'] + + build_obj = self.orm_wrapper.create_build_object(build_information, self.brbe, self.project) + + self.internal_state['build'] = build_obj + + # save layer version information for this build + if not 'lvs' in self.internal_state: + logger.error("Layer version information not found; Check if the bitbake server was configured to inherit toaster.bbclass.") + else: + for layer_obj in self.internal_state['lvs']: + self.orm_wrapper.get_update_layer_version_object(build_obj, layer_obj, self.internal_state['lvs'][layer_obj]) + + del self.internal_state['lvs'] + + # create target information + target_information = {} + target_information['targets'] = event._pkgs + target_information['build'] = build_obj + + self.internal_state['targets'] = self.orm_wrapper.get_or_create_targets(target_information) + + # Save build configuration + data = self.server.runCommand(["getAllKeysWithFlags", ["doc", "func"]])[0] + + # convert the paths from absolute to relative to either the build directory or layer checkouts + path_prefixes = [] + + if self.brbe is not None: + _, be_id = self.brbe.split(":") + be = BuildEnvironment.objects.get(pk = be_id) + path_prefixes.append(be.builddir) + + for layer in sorted(self.orm_wrapper.layer_version_objects, key = lambda x:len(x.local_path), reverse=True): + path_prefixes.append(layer.local_path) + + # we strip the prefixes + for k in data: + if not bool(data[k]['func']): + for vh in data[k]['history']: + if not 'documentation.conf' in vh['file']: + abs_file_name = vh['file'] + for pp in path_prefixes: + if abs_file_name.startswith(pp + "/"): + vh['file']=abs_file_name[len(pp + "/"):] + break + + # save the variables + self.orm_wrapper.save_build_variables(build_obj, data) + + return self.brbe + + + def update_target_image_file(self, event): + evdata = BuildInfoHelper._get_data_from_event(event) + + for t in self.internal_state['targets']: + if t.is_image == True: + output_files = list(evdata.viewkeys()) + for output in output_files: + if t.target in output and 'rootfs' in output and not output.endswith(".manifest"): + self.orm_wrapper.save_target_image_file_information(t, output, evdata[output]) + + def update_artifact_image_file(self, event): + evdata = BuildInfoHelper._get_data_from_event(event) + for artifact_path in evdata.keys(): + self.orm_wrapper.save_artifact_information(self.internal_state['build'], artifact_path, evdata[artifact_path]) + + def update_build_information(self, event, errors, warnings, taskfailures): + if 'build' in self.internal_state: + self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures) + + + def store_license_manifest_path(self, event): + deploy_dir = BuildInfoHelper._get_data_from_event(event)['deploy_dir'] + image_name = BuildInfoHelper._get_data_from_event(event)['image_name'] + path = deploy_dir + "/licenses/" + image_name + "/license.manifest" + for target in self.internal_state['targets']: + if target.target in image_name: + self.orm_wrapper.update_target_set_license_manifest(target, path) + + + def store_started_task(self, event): + assert isinstance(event, (bb.runqueue.sceneQueueTaskStarted, bb.runqueue.runQueueTaskStarted, bb.runqueue.runQueueTaskSkipped)) + assert 'taskfile' in vars(event) + localfilepath = event.taskfile.split(":")[-1] + assert localfilepath.startswith("/") + + identifier = event.taskfile + ":" + event.taskname + + recipe_information = self._get_recipe_information_from_taskfile(event.taskfile) + recipe = self.orm_wrapper.get_update_recipe_object(recipe_information, True) + + task_information = self._get_task_information(event, recipe) + task_information['outcome'] = Task.OUTCOME_NA + + if isinstance(event, bb.runqueue.runQueueTaskSkipped): + assert 'reason' in vars(event) + task_information['task_executed'] = False + if event.reason == "covered": + task_information['outcome'] = Task.OUTCOME_COVERED + if event.reason == "existing": + task_information['outcome'] = Task.OUTCOME_PREBUILT + else: + task_information['task_executed'] = True + if 'noexec' in vars(event) and event.noexec == True: + task_information['task_executed'] = False + task_information['outcome'] = Task.OUTCOME_EMPTY + task_information['script_type'] = Task.CODING_NA + + # do not assign order numbers to scene tasks + if not isinstance(event, bb.runqueue.sceneQueueTaskStarted): + self.task_order += 1 + task_information['order'] = self.task_order + + self.orm_wrapper.get_update_task_object(task_information) + + self.internal_state['taskdata'][identifier] = { + 'outcome': task_information['outcome'], + } + + + def store_tasks_stats(self, event): + task_data = BuildInfoHelper._get_data_from_event(event) + + for (task_file, task_name, task_stats, recipe_name) in task_data: + build = self.internal_state['build'] + self.orm_wrapper.update_task_object(build, task_name, recipe_name, task_stats) + + def update_and_store_task(self, event): + assert 'taskfile' in vars(event) + localfilepath = event.taskfile.split(":")[-1] + assert localfilepath.startswith("/") + + identifier = event.taskfile + ":" + event.taskname + if not identifier in self.internal_state['taskdata']: + if isinstance(event, bb.build.TaskBase): + # we do a bit of guessing + candidates = [x for x in self.internal_state['taskdata'].keys() if x.endswith(identifier)] + if len(candidates) == 1: + identifier = candidates[0] + + assert identifier in self.internal_state['taskdata'] + identifierlist = identifier.split(":") + realtaskfile = ":".join(identifierlist[0:len(identifierlist)-1]) + recipe_information = self._get_recipe_information_from_taskfile(realtaskfile) + recipe = self.orm_wrapper.get_update_recipe_object(recipe_information, True) + task_information = self._get_task_information(event,recipe) + + task_information['outcome'] = self.internal_state['taskdata'][identifier]['outcome'] + + if 'logfile' in vars(event): + task_information['logfile'] = event.logfile + + if '_message' in vars(event): + task_information['message'] = event._message + + if 'taskflags' in vars(event): + # with TaskStarted, we get even more information + if 'python' in event.taskflags.keys() and event.taskflags['python'] == '1': + task_information['script_type'] = Task.CODING_PYTHON + else: + task_information['script_type'] = Task.CODING_SHELL + + if task_information['outcome'] == Task.OUTCOME_NA: + if isinstance(event, (bb.runqueue.runQueueTaskCompleted, bb.runqueue.sceneQueueTaskCompleted)): + task_information['outcome'] = Task.OUTCOME_SUCCESS + del self.internal_state['taskdata'][identifier] + + if isinstance(event, (bb.runqueue.runQueueTaskFailed, bb.runqueue.sceneQueueTaskFailed)): + task_information['outcome'] = Task.OUTCOME_FAILED + del self.internal_state['taskdata'][identifier] + + if not connection.features.autocommits_when_autocommit_is_off: + # we force a sync point here, to get the progress bar to show + if self.autocommit_step % 3 == 0: + transaction.set_autocommit(True) + transaction.set_autocommit(False) + self.autocommit_step += 1 + + self.orm_wrapper.get_update_task_object(task_information, True) # must exist + + + def store_missed_state_tasks(self, event): + for (fn, taskname, taskhash, sstatefile) in BuildInfoHelper._get_data_from_event(event)['missed']: + + # identifier = fn + taskname + "_setscene" + recipe_information = self._get_recipe_information_from_taskfile(fn) + recipe = self.orm_wrapper.get_update_recipe_object(recipe_information) + mevent = MockEvent() + mevent.taskname = taskname + mevent.taskhash = taskhash + task_information = self._get_task_information(mevent,recipe) + + task_information['start_time'] = timezone.now() + task_information['outcome'] = Task.OUTCOME_NA + task_information['sstate_checksum'] = taskhash + task_information['sstate_result'] = Task.SSTATE_MISS + task_information['path_to_sstate_obj'] = sstatefile + + self.orm_wrapper.get_update_task_object(task_information) + + for (fn, taskname, taskhash, sstatefile) in BuildInfoHelper._get_data_from_event(event)['found']: + + # identifier = fn + taskname + "_setscene" + recipe_information = self._get_recipe_information_from_taskfile(fn) + recipe = self.orm_wrapper.get_update_recipe_object(recipe_information) + mevent = MockEvent() + mevent.taskname = taskname + mevent.taskhash = taskhash + task_information = self._get_task_information(mevent,recipe) + + task_information['path_to_sstate_obj'] = sstatefile + + self.orm_wrapper.get_update_task_object(task_information) + + + def store_target_package_data(self, event): + # for all image targets + for target in self.internal_state['targets']: + if target.is_image: + pkgdata = BuildInfoHelper._get_data_from_event(event)['pkgdata'] + imgdata = BuildInfoHelper._get_data_from_event(event)['imgdata'].get(target.target, {}) + filedata = BuildInfoHelper._get_data_from_event(event)['filedata'].get(target.target, {}) + + try: + self.orm_wrapper.save_target_package_information(self.internal_state['build'], target, imgdata, pkgdata, self.internal_state['recipes'], built_package=True) + self.orm_wrapper.save_target_package_information(self.internal_state['build'], target, imgdata.copy(), pkgdata, self.internal_state['recipes'], built_package=False) + except KeyError as e: + logger.warn("KeyError in save_target_package_information" + "%s ", e) + + try: + self.orm_wrapper.save_target_file_information(self.internal_state['build'], target, filedata) + except KeyError as e: + logger.warn("KeyError in save_target_file_information" + "%s ", e) + + + + + def store_dependency_information(self, event): + assert '_depgraph' in vars(event) + assert 'layer-priorities' in event._depgraph + assert 'pn' in event._depgraph + assert 'tdepends' in event._depgraph + + errormsg = "" + + # save layer version priorities + if 'layer-priorities' in event._depgraph.keys(): + for lv in event._depgraph['layer-priorities']: + (_, path, _, priority) = lv + layer_version_obj = self._get_layer_version_for_path(path[1:]) # paths start with a ^ + assert layer_version_obj is not None + layer_version_obj.priority = priority + layer_version_obj.save() + + # save recipe information + self.internal_state['recipes'] = {} + for pn in event._depgraph['pn']: + + file_name = event._depgraph['pn'][pn]['filename'].split(":")[-1] + pathflags = ":".join(sorted(event._depgraph['pn'][pn]['filename'].split(":")[:-1])) + layer_version_obj = self._get_layer_version_for_path(file_name) + + assert layer_version_obj is not None + + recipe_info = {} + recipe_info['name'] = pn + recipe_info['layer_version'] = layer_version_obj + + if 'version' in event._depgraph['pn'][pn]: + recipe_info['version'] = event._depgraph['pn'][pn]['version'].lstrip(":") + + if 'summary' in event._depgraph['pn'][pn]: + recipe_info['summary'] = event._depgraph['pn'][pn]['summary'] + + if 'license' in event._depgraph['pn'][pn]: + recipe_info['license'] = event._depgraph['pn'][pn]['license'] + + if 'description' in event._depgraph['pn'][pn]: + recipe_info['description'] = event._depgraph['pn'][pn]['description'] + + if 'section' in event._depgraph['pn'][pn]: + recipe_info['section'] = event._depgraph['pn'][pn]['section'] + + if 'homepage' in event._depgraph['pn'][pn]: + recipe_info['homepage'] = event._depgraph['pn'][pn]['homepage'] + + if 'bugtracker' in event._depgraph['pn'][pn]: + recipe_info['bugtracker'] = event._depgraph['pn'][pn]['bugtracker'] + + recipe_info['file_path'] = file_name + recipe_info['pathflags'] = pathflags + + if recipe_info['file_path'].startswith(recipe_info['layer_version'].local_path): + recipe_info['file_path'] = recipe_info['file_path'][len(recipe_info['layer_version'].local_path):].lstrip("/") + else: + raise RuntimeError("Recipe file path %s is not under layer version at %s" % (recipe_info['file_path'], recipe_info['layer_version'].local_path)) + + recipe = self.orm_wrapper.get_update_recipe_object(recipe_info) + recipe.is_image = False + if 'inherits' in event._depgraph['pn'][pn].keys(): + for cls in event._depgraph['pn'][pn]['inherits']: + if cls.endswith('/image.bbclass'): + recipe.is_image = True + recipe_info['is_image'] = True + # Save the is_image state to the relevant recipe objects + self.orm_wrapper.get_update_recipe_object(recipe_info) + break + if recipe.is_image: + for t in self.internal_state['targets']: + if pn == t.target: + t.is_image = True + t.save() + self.internal_state['recipes'][pn] = recipe + + # we'll not get recipes for key w/ values listed in ASSUME_PROVIDED + + assume_provided = self.server.runCommand(["getVariable", "ASSUME_PROVIDED"])[0].split() + + # save recipe dependency + # buildtime + recipedeps_objects = [] + for recipe in event._depgraph['depends']: + target = self.internal_state['recipes'][recipe] + for dep in event._depgraph['depends'][recipe]: + if dep in assume_provided: + continue + via = None + if 'providermap' in event._depgraph and dep in event._depgraph['providermap']: + deprecipe = event._depgraph['providermap'][dep][0] + dependency = self.internal_state['recipes'][deprecipe] + via = Provides.objects.get_or_create(name=dep, + recipe=dependency)[0] + elif dep in self.internal_state['recipes']: + dependency = self.internal_state['recipes'][dep] + else: + errormsg += " stpd: KeyError saving recipe dependency for %s, %s \n" % (recipe, dep) + continue + recipe_dep = Recipe_Dependency(recipe=target, + depends_on=dependency, + via=via, + dep_type=Recipe_Dependency.TYPE_DEPENDS) + recipedeps_objects.append(recipe_dep) + + Recipe_Dependency.objects.bulk_create(recipedeps_objects) + + # save all task information + def _save_a_task(taskdesc): + spec = re.split(r'\.', taskdesc) + pn = ".".join(spec[0:-1]) + taskname = spec[-1] + e = event + e.taskname = pn + recipe = self.internal_state['recipes'][pn] + task_info = self._get_task_information(e, recipe) + task_info['task_name'] = taskname + task_obj = self.orm_wrapper.get_update_task_object(task_info) + return task_obj + + # create tasks + tasks = {} + for taskdesc in event._depgraph['tdepends']: + tasks[taskdesc] = _save_a_task(taskdesc) + + # create dependencies between tasks + taskdeps_objects = [] + for taskdesc in event._depgraph['tdepends']: + target = tasks[taskdesc] + for taskdep in event._depgraph['tdepends'][taskdesc]: + if taskdep not in tasks: + # Fetch tasks info is not collected previously + dep = _save_a_task(taskdep) + else: + dep = tasks[taskdep] + taskdeps_objects.append(Task_Dependency( task = target, depends_on = dep )) + Task_Dependency.objects.bulk_create(taskdeps_objects) + + if len(errormsg) > 0: + logger.warn("buildinfohelper: dependency info not identify recipes: \n%s", errormsg) + + + def store_build_package_information(self, event): + package_info = BuildInfoHelper._get_data_from_event(event) + self.orm_wrapper.save_build_package_information( + self.internal_state['build'], + package_info, + self.internal_state['recipes'], + built_package=True) + + self.orm_wrapper.save_build_package_information( + self.internal_state['build'], + package_info, + self.internal_state['recipes'], + built_package=False) + + def _store_build_done(self, errorcode): + logger.info("Build exited with errorcode %d", errorcode) + br_id, be_id = self.brbe.split(":") + be = BuildEnvironment.objects.get(pk = be_id) + be.lock = BuildEnvironment.LOCK_LOCK + be.save() + br = BuildRequest.objects.get(pk = br_id) + + # if we're 'done' because we got cancelled update the build outcome + if br.state == BuildRequest.REQ_CANCELLING: + logger.info("Build cancelled") + br.build.outcome = Build.CANCELLED + br.build.save() + self.internal_state['build'] = br.build + errorcode = 0 + + if errorcode == 0: + # request archival of the project artifacts + br.state = BuildRequest.REQ_COMPLETED + else: + br.state = BuildRequest.REQ_FAILED + br.save() + + + def store_log_error(self, text): + mockevent = MockEvent() + mockevent.levelno = formatter.ERROR + mockevent.msg = text + mockevent.pathname = '-- None' + mockevent.lineno = LogMessage.ERROR + self.store_log_event(mockevent) + + def store_log_exception(self, text, backtrace = ""): + mockevent = MockEvent() + mockevent.levelno = -1 + mockevent.msg = text + mockevent.pathname = backtrace + mockevent.lineno = -1 + self.store_log_event(mockevent) + + + def store_log_event(self, event): + if event.levelno < formatter.WARNING: + return + + if 'args' in vars(event): + event.msg = event.msg % event.args + + if not 'build' in self.internal_state: + if self.brbe is None: + if not 'backlog' in self.internal_state: + self.internal_state['backlog'] = [] + self.internal_state['backlog'].append(event) + return + else: # we're under Toaster control, the build is already created + br, _ = self.brbe.split(":") + buildrequest = BuildRequest.objects.get(pk = br) + self.internal_state['build'] = buildrequest.build + + if 'build' in self.internal_state and 'backlog' in self.internal_state: + # if we have a backlog of events, do our best to save them here + if len(self.internal_state['backlog']): + tempevent = self.internal_state['backlog'].pop() + logger.debug(1, "buildinfohelper: Saving stored event %s " % tempevent) + self.store_log_event(tempevent) + else: + logger.info("buildinfohelper: All events saved") + del self.internal_state['backlog'] + + log_information = {} + log_information['build'] = self.internal_state['build'] + if event.levelno == formatter.CRITICAL: + log_information['level'] = LogMessage.CRITICAL + elif event.levelno == formatter.ERROR: + log_information['level'] = LogMessage.ERROR + elif event.levelno == formatter.WARNING: + log_information['level'] = LogMessage.WARNING + elif event.levelno == -2: # toaster self-logging + log_information['level'] = -2 + else: + log_information['level'] = LogMessage.INFO + + log_information['message'] = event.msg + log_information['pathname'] = event.pathname + log_information['lineno'] = event.lineno + logger.info("Logging error 2: %s", log_information) + + self.orm_wrapper.create_logmessage(log_information) + + def close(self, errorcode): + if self.brbe is not None: + self._store_build_done(errorcode) + + if 'backlog' in self.internal_state: + if 'build' in self.internal_state: + # we save missed events in the database for the current build + tempevent = self.internal_state['backlog'].pop() + self.store_log_event(tempevent) + else: + # we have no build, and we still have events; something amazingly wrong happend + for event in self.internal_state['backlog']: + logger.error("UNSAVED log: %s", event.msg) + + if not connection.features.autocommits_when_autocommit_is_off: + transaction.set_autocommit(True) + + # unset the brbe; this is to prevent subsequent command-line builds + # being incorrectly attached to the previous Toaster-triggered build; + # see https://bugzilla.yoctoproject.org/show_bug.cgi?id=9021 + self.brbe = None |
