diff options
Diffstat (limited to 'import-layers/yocto-poky/scripts/lib/wic')
37 files changed, 1456 insertions, 2998 deletions
diff --git a/import-layers/yocto-poky/scripts/lib/wic/__init__.py b/import-layers/yocto-poky/scripts/lib/wic/__init__.py index 63c1d9c84..85876b138 100644 --- a/import-layers/yocto-poky/scripts/lib/wic/__init__.py +++ b/import-layers/yocto-poky/scripts/lib/wic/__init__.py @@ -1,4 +1,20 @@ -import os, sys +#!/usr/bin/env python -tt +# +# Copyright (c) 2007 Red Hat, Inc. +# Copyright (c) 2011 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. -cur_path = os.path.dirname(__file__) or '.' -sys.path.insert(0, cur_path + '/3rdparty') +class WicError(Exception): + pass diff --git a/import-layers/yocto-poky/scripts/lib/wic/__version__.py b/import-layers/yocto-poky/scripts/lib/wic/__version__.py deleted file mode 100644 index 5452a4671..000000000 --- a/import-layers/yocto-poky/scripts/lib/wic/__version__.py +++ /dev/null @@ -1 +0,0 @@ -VERSION = "2.00" diff --git a/import-layers/yocto-poky/scripts/lib/wic/canned-wks/directdisk-bootloader-config.cfg b/import-layers/yocto-poky/scripts/lib/wic/canned-wks/directdisk-bootloader-config.cfg index a16bd6ac6..d5a07d204 100644 --- a/import-layers/yocto-poky/scripts/lib/wic/canned-wks/directdisk-bootloader-config.cfg +++ b/import-layers/yocto-poky/scripts/lib/wic/canned-wks/directdisk-bootloader-config.cfg @@ -1,11 +1,27 @@ # This is an example configuration file for syslinux. -PROMPT 0 -TIMEOUT 10 - +TIMEOUT 50 ALLOWOPTIONS 1 SERIAL 0 115200 +PROMPT 0 + +UI vesamenu.c32 +menu title Select boot options +menu tabmsg Press [Tab] to edit, [Return] to select + +DEFAULT Graphics console boot + +LABEL Graphics console boot +KERNEL /vmlinuz +APPEND label=boot root=/dev/sda2 rootwait + +LABEL Serial console boot +KERNEL /vmlinuz +APPEND label=boot root=/dev/sda2 rootwait console=ttyS0,115200 + +LABEL Graphics console install +KERNEL /vmlinuz +APPEND label=install root=/dev/sda2 rootwait -DEFAULT boot -LABEL boot +LABEL Serial console install KERNEL /vmlinuz -APPEND label=boot root=/dev/sda2 rootwait rootfstype=ext4 video=vesafb vga=0x318 console=tty0 +APPEND label=install root=/dev/sda2 rootwait console=ttyS0,115200 diff --git a/import-layers/yocto-poky/scripts/lib/wic/canned-wks/mkgummidisk.wks b/import-layers/yocto-poky/scripts/lib/wic/canned-wks/mkgummidisk.wks deleted file mode 100644 index f3ae09099..000000000 --- a/import-layers/yocto-poky/scripts/lib/wic/canned-wks/mkgummidisk.wks +++ /dev/null @@ -1,11 +0,0 @@ -# short-description: Create an EFI disk image -# long-description: Creates a partitioned EFI disk image that the user -# can directly dd to boot media. - -part /boot --source bootimg-efi --sourceparams="loader=gummiboot" --ondisk sda --label msdos --active --align 1024 - -part / --source rootfs --ondisk sda --fstype=ext4 --label platform --align 1024 - -part swap --ondisk sda --size 44 --label swap1 --fstype=swap - -bootloader --ptable gpt --timeout=5 --append="rootwait rootfstype=ext4 console=ttyS0,115200 console=tty0" diff --git a/import-layers/yocto-poky/scripts/lib/wic/canned-wks/qemux86-directdisk.wks b/import-layers/yocto-poky/scripts/lib/wic/canned-wks/qemux86-directdisk.wks index a6518a0f4..db30bbced 100644 --- a/import-layers/yocto-poky/scripts/lib/wic/canned-wks/qemux86-directdisk.wks +++ b/import-layers/yocto-poky/scripts/lib/wic/canned-wks/qemux86-directdisk.wks @@ -4,5 +4,5 @@ include common.wks.inc -bootloader --timeout=0 --append="vga=0 uvesafb.mode_option=640x480-32 root=/dev/vda2 rw mem=256M ip=192.168.7.2::192.168.7.1:255.255.255.0 oprofile.timer=1 rootfstype=ext4 " +bootloader --timeout=0 --append="vga=0 uvesafb.mode_option=640x480-32 root=/dev/sda2 rw mem=256M ip=192.168.7.2::192.168.7.1:255.255.255.0 oprofile.timer=1 rootfstype=ext4 " diff --git a/import-layers/yocto-poky/scripts/lib/wic/conf.py b/import-layers/yocto-poky/scripts/lib/wic/conf.py deleted file mode 100644 index 070ec3096..000000000 --- a/import-layers/yocto-poky/scripts/lib/wic/conf.py +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env python -tt -# -# Copyright (c) 2011 Intel, Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., 59 -# Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -import os - -from wic.ksparser import KickStart, KickStartError -from wic import msger -from wic.utils import misc - - -def get_siteconf(): - wic_path = os.path.dirname(__file__) - eos = wic_path.find('scripts') + len('scripts') - scripts_path = wic_path[:eos] - - return scripts_path + "/lib/image/config/wic.conf" - -class ConfigMgr(object): - DEFAULTS = { - 'common': { - "distro_name": "Default Distribution", - "plugin_dir": "/usr/lib/wic/plugins"}, # TODO use prefix also? - 'create': { - "tmpdir": '/var/tmp/wic', - "outdir": './wic-output', - "release": None, - "logfile": None, - "name_prefix": None, - "name_suffix": None} - } - - # make the manager class as singleton - _instance = None - def __new__(cls, *args, **kwargs): - if not cls._instance: - cls._instance = super(ConfigMgr, cls).__new__(cls, *args, **kwargs) - - return cls._instance - - def __init__(self, ksconf=None, siteconf=None): - # reset config options - self.reset() - - if not siteconf: - siteconf = get_siteconf() - - # initial options from siteconf - self._siteconf = siteconf - - if ksconf: - self._ksconf = ksconf - - def reset(self): - self.__ksconf = None - self.__siteconf = None - self.create = {} - - # initialize the values with defaults - for sec, vals in self.DEFAULTS.items(): - setattr(self, sec, vals) - - def __set_ksconf(self, ksconf): - if not os.path.isfile(ksconf): - msger.error('Cannot find ks file: %s' % ksconf) - - self.__ksconf = ksconf - self._parse_kickstart(ksconf) - def __get_ksconf(self): - return self.__ksconf - _ksconf = property(__get_ksconf, __set_ksconf) - - def _parse_kickstart(self, ksconf=None): - if not ksconf: - return - - try: - ksobj = KickStart(ksconf) - except KickStartError as err: - msger.error(str(err)) - - self.create['ks'] = ksobj - self.create['name'] = os.path.splitext(os.path.basename(ksconf))[0] - - self.create['name'] = misc.build_name(ksconf, - self.create['release'], - self.create['name_prefix'], - self.create['name_suffix']) - -configmgr = ConfigMgr() diff --git a/import-layers/yocto-poky/scripts/lib/wic/config/wic.conf b/import-layers/yocto-poky/scripts/lib/wic/config/wic.conf deleted file mode 100644 index a51bcb55e..000000000 --- a/import-layers/yocto-poky/scripts/lib/wic/config/wic.conf +++ /dev/null @@ -1,6 +0,0 @@ -[common] -; general settings -distro_name = OpenEmbedded - -[create] -; settings for create subcommand diff --git a/import-layers/yocto-poky/scripts/lib/wic/creator.py b/import-layers/yocto-poky/scripts/lib/wic/creator.py deleted file mode 100644 index 8f7d1503f..000000000 --- a/import-layers/yocto-poky/scripts/lib/wic/creator.py +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/env python -tt -# -# Copyright (c) 2011 Intel, Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., 59 -# Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -import os, sys -from optparse import OptionParser, SUPPRESS_HELP - -from wic import msger -from wic.utils import errors -from wic.conf import configmgr -from wic.plugin import pluginmgr - - -class Creator(): - """${name}: create an image - - Usage: - ${name} SUBCOMMAND <ksfile> [OPTS] - - ${command_list} - ${option_list} - """ - - name = 'wic create(cr)' - - def __init__(self, *args, **kwargs): - self._subcmds = {} - - # get cmds from pluginmgr - # mix-in do_subcmd interface - for subcmd, klass in pluginmgr.get_plugins('imager').items(): - if not hasattr(klass, 'do_create'): - msger.warning("Unsupported subcmd: %s" % subcmd) - continue - - func = getattr(klass, 'do_create') - self._subcmds[subcmd] = func - - def get_optparser(self): - optparser = OptionParser() - optparser.add_option('-d', '--debug', action='store_true', - dest='debug', - help=SUPPRESS_HELP) - optparser.add_option('-v', '--verbose', action='store_true', - dest='verbose', - help=SUPPRESS_HELP) - optparser.add_option('', '--logfile', type='string', dest='logfile', - default=None, - help='Path of logfile') - optparser.add_option('-c', '--config', type='string', dest='config', - default=None, - help='Specify config file for wic') - optparser.add_option('-o', '--outdir', type='string', action='store', - dest='outdir', default=None, - help='Output directory') - optparser.add_option('', '--tmpfs', action='store_true', dest='enabletmpfs', - help='Setup tmpdir as tmpfs to accelerate, experimental' - ' feature, use it if you have more than 4G memory') - optparser.add_option('', '--bmap', action='store_true', help='generate .bmap') - return optparser - - def postoptparse(self, options): - abspath = lambda pth: os.path.abspath(os.path.expanduser(pth)) - - if options.verbose: - msger.set_loglevel('verbose') - if options.debug: - msger.set_loglevel('debug') - - if options.logfile: - logfile_abs_path = abspath(options.logfile) - if os.path.isdir(logfile_abs_path): - raise errors.Usage("logfile's path %s should be file" - % options.logfile) - if not os.path.exists(os.path.dirname(logfile_abs_path)): - os.makedirs(os.path.dirname(logfile_abs_path)) - msger.set_interactive(False) - msger.set_logfile(logfile_abs_path) - configmgr.create['logfile'] = options.logfile - - if options.config: - configmgr.reset() - configmgr._siteconf = options.config - - if options.outdir is not None: - configmgr.create['outdir'] = abspath(options.outdir) - - cdir = 'outdir' - if os.path.exists(configmgr.create[cdir]) \ - and not os.path.isdir(configmgr.create[cdir]): - msger.error('Invalid directory specified: %s' \ - % configmgr.create[cdir]) - - if options.enabletmpfs: - configmgr.create['enabletmpfs'] = options.enabletmpfs - - def main(self, argv=None): - if argv is None: - argv = sys.argv - else: - argv = argv[:] # don't modify caller's list - - pname = argv[0] - if pname not in self._subcmds: - msger.error('Unknown plugin: %s' % pname) - - optparser = self.get_optparser() - options, args = optparser.parse_args(argv) - - self.postoptparse(options) - - return self._subcmds[pname](options, *args[1:]) diff --git a/import-layers/yocto-poky/scripts/lib/wic/engine.py b/import-layers/yocto-poky/scripts/lib/wic/engine.py index 5b104631c..f59821fea 100644 --- a/import-layers/yocto-poky/scripts/lib/wic/engine.py +++ b/import-layers/yocto-poky/scripts/lib/wic/engine.py @@ -28,14 +28,14 @@ # Tom Zanussi <tom.zanussi (at] linux.intel.com> # +import logging import os -import sys -from wic import msger, creator -from wic.utils import misc -from wic.plugin import pluginmgr -from wic.utils.oe import misc +from wic import WicError +from wic.pluginbase import PluginMgr +from wic.utils.misc import get_bitbake_var +logger = logging.getLogger('wic') def verify_build_env(): """ @@ -44,23 +44,25 @@ def verify_build_env(): Returns True if it is, false otherwise """ if not os.environ.get("BUILDDIR"): - print("BUILDDIR not found, exiting. (Did you forget to source oe-init-build-env?)") - sys.exit(1) + raise WicError("BUILDDIR not found, exiting. (Did you forget to source oe-init-build-env?)") return True CANNED_IMAGE_DIR = "lib/wic/canned-wks" # relative to scripts SCRIPTS_CANNED_IMAGE_DIR = "scripts/" + CANNED_IMAGE_DIR +WIC_DIR = "wic" def build_canned_image_list(path): - layers_path = misc.get_bitbake_var("BBLAYERS") + layers_path = get_bitbake_var("BBLAYERS") canned_wks_layer_dirs = [] if layers_path is not None: for layer_path in layers_path.split(): - cpath = os.path.join(layer_path, SCRIPTS_CANNED_IMAGE_DIR) - canned_wks_layer_dirs.append(cpath) + for wks_path in (WIC_DIR, SCRIPTS_CANNED_IMAGE_DIR): + cpath = os.path.join(layer_path, wks_path) + if os.path.isdir(cpath): + canned_wks_layer_dirs.append(cpath) cpath = os.path.join(path, CANNED_IMAGE_DIR) canned_wks_layer_dirs.append(cpath) @@ -137,26 +139,24 @@ def list_source_plugins(): """ List the available source plugins i.e. plugins available for --source. """ - plugins = pluginmgr.get_source_plugins() + plugins = PluginMgr.get_plugins('source') for plugin in plugins: print(" %s" % plugin) def wic_create(wks_file, rootfs_dir, bootimg_dir, kernel_dir, - native_sysroot, scripts_path, image_output_dir, - compressor, bmap, debug): - """Create image + native_sysroot, options): + """ + Create image wks_file - user-defined OE kickstart file rootfs_dir - absolute path to the build's /rootfs dir bootimg_dir - absolute path to the build's boot artifacts directory kernel_dir - absolute path to the build's kernel directory native_sysroot - absolute path to the build's native sysroots dir - scripts_path - absolute path to /scripts dir image_output_dir - dirname to create for image - compressor - compressor utility to compress the image - bmap - enable generation of .bmap + options - wic command line options (debug, bmap, etc) Normally, the values for the build artifacts values are determined by 'wic -e' from the output of the 'bitbake -e' command given an @@ -179,22 +179,22 @@ def wic_create(wks_file, rootfs_dir, bootimg_dir, kernel_dir, try: oe_builddir = os.environ["BUILDDIR"] except KeyError: - print("BUILDDIR not found, exiting. (Did you forget to source oe-init-build-env?)") - sys.exit(1) + raise WicError("BUILDDIR not found, exiting. (Did you forget to source oe-init-build-env?)") - if debug: - msger.set_loglevel('debug') + if not os.path.exists(options.outdir): + os.makedirs(options.outdir) - crobj = creator.Creator() + pname = 'direct' + plugin_class = PluginMgr.get_plugins('imager').get(pname) + if not plugin_class: + raise WicError('Unknown plugin: %s' % pname) - cmdline = ["direct", native_sysroot, kernel_dir, bootimg_dir, rootfs_dir, - wks_file, image_output_dir, oe_builddir, compressor or ""] - if bmap: - cmdline.append('--bmap') + plugin = plugin_class(wks_file, rootfs_dir, bootimg_dir, kernel_dir, + native_sysroot, oe_builddir, options) - crobj.main(cmdline) + plugin.do_create() - print("\nThe image(s) were created using OE kickstart file:\n %s" % wks_file) + logger.info("The image(s) were created using OE kickstart file:\n %s", wks_file) def wic_list(args, scripts_path): @@ -214,12 +214,44 @@ def wic_list(args, scripts_path): wks_file = args[0] fullpath = find_canned_image(scripts_path, wks_file) if not fullpath: - print("No image named %s found, exiting. "\ - "(Use 'wic list images' to list available images, or "\ - "specify a fully-qualified OE kickstart (.wks) "\ - "filename)\n" % wks_file) - sys.exit(1) + raise WicError("No image named %s found, exiting. " + "(Use 'wic list images' to list available images, " + "or specify a fully-qualified OE kickstart (.wks) " + "filename)" % wks_file) + list_canned_image_help(scripts_path, fullpath) return True return False + +def find_canned(scripts_path, file_name): + """ + Find a file either by its path or by name in the canned files dir. + + Return None if not found + """ + if os.path.exists(file_name): + return file_name + + layers_canned_wks_dir = build_canned_image_list(scripts_path) + for canned_wks_dir in layers_canned_wks_dir: + for root, dirs, files in os.walk(canned_wks_dir): + for fname in files: + if fname == file_name: + fullpath = os.path.join(canned_wks_dir, fname) + return fullpath + +def get_custom_config(boot_file): + """ + Get the custom configuration to be used for the bootloader. + + Return None if the file can't be found. + """ + # Get the scripts path of poky + scripts_path = os.path.abspath("%s/../.." % os.path.dirname(__file__)) + + cfg_file = find_canned(scripts_path, boot_file) + if cfg_file: + with open(cfg_file, "r") as f: + config = f.read() + return config diff --git a/import-layers/yocto-poky/scripts/lib/wic/filemap.py b/import-layers/yocto-poky/scripts/lib/wic/filemap.py index 162603ed0..1f1aacc52 100644 --- a/import-layers/yocto-poky/scripts/lib/wic/filemap.py +++ b/import-layers/yocto-poky/scripts/lib/wic/filemap.py @@ -530,13 +530,16 @@ def filemap(image, log=None): except ErrorNotSupp: return FilemapSeek(image, log) -def sparse_copy(src_fname, dst_fname, offset=0, skip=0): +def sparse_copy(src_fname, dst_fname, offset=0, skip=0, api=None): """Efficiently copy sparse file to or into another file.""" - fmap = filemap(src_fname) + if not api: + api = filemap + fmap = api(src_fname) try: dst_file = open(dst_fname, 'r+b') except IOError: dst_file = open(dst_fname, 'wb') + dst_file.truncate(os.path.getsize(src_fname)) for first, last in fmap.get_mapped_ranges(0, fmap.blocks_cnt): start = first * fmap.block_size diff --git a/import-layers/yocto-poky/scripts/lib/wic/help.py b/import-layers/yocto-poky/scripts/lib/wic/help.py index e5347ec4b..d6e027d25 100644 --- a/import-layers/yocto-poky/scripts/lib/wic/help.py +++ b/import-layers/yocto-poky/scripts/lib/wic/help.py @@ -28,10 +28,12 @@ import subprocess import logging -from wic.plugin import pluginmgr, PLUGIN_TYPES +from wic.pluginbase import PluginMgr, PLUGIN_TYPES + +logger = logging.getLogger('wic') def subcommand_error(args): - logging.info("invalid subcommand %s" % args[0]) + logger.info("invalid subcommand %s", args[0]) def display_help(subcommand, subcommands): @@ -66,7 +68,7 @@ def get_wic_plugins_help(): result = wic_plugins_help for plugin_type in PLUGIN_TYPES: result += '\n\n%s PLUGINS\n\n' % plugin_type.upper() - for name, plugin in pluginmgr.get_plugins(plugin_type).items(): + for name, plugin in PluginMgr.get_plugins(plugin_type).items(): result += "\n %s plugin:\n" % name if plugin.__doc__: result += plugin.__doc__ @@ -81,13 +83,13 @@ def invoke_subcommand(args, parser, main_command_usage, subcommands): Should use argparse, but has to work in 2.6. """ if not args: - logging.error("No subcommand specified, exiting") + logger.error("No subcommand specified, exiting") parser.print_help() return 1 elif args[0] == "help": wic_help(args, main_command_usage, subcommands) elif args[0] not in subcommands: - logging.error("Unsupported subcommand %s, exiting\n" % (args[0])) + logger.error("Unsupported subcommand %s, exiting\n", args[0]) parser.print_help() return 1 else: @@ -371,12 +373,7 @@ DESCRIPTION This scheme is extensible - adding more hooks is a simple matter of adding more plugin methods to SourcePlugin and derived classes. - The code that then needs to call the plugin methods uses - plugin.get_source_plugin_methods() to find the method(s) needed by - the call; this is done by filling up a dict with keys containing - the method names of interest - on success, these will be filled in - with the actual methods. Please see the implementation for - examples and details. + Please see the implementation for details. """ wic_overview_help = """ @@ -646,6 +643,12 @@ DESCRIPTION not specified, the size is in MB. You do not need this option if you use --source. + --fixed-size: Exact partition size. Value format is the same + as for --size option. This option cannot be + specified along with --size. If partition data + is larger than --fixed-size and error will be + raised when assembling disk image. + --source: This option is a wic-specific option that names the source of the data that will populate the partition. The most common value for this option @@ -684,6 +687,8 @@ DESCRIPTION apply to partitions created using '--source rootfs' (see --source above). Valid values are: + vfat + msdos ext2 ext3 ext4 @@ -715,17 +720,25 @@ DESCRIPTION partition table. It may be useful for bootloaders. + --exclude-path: This option is specific to wic. It excludes the given + relative path from the resulting image. If the path + ends with a slash, only the content of the directory + is omitted, not the directory itself. This option only + has an effect with the rootfs source plugin. + --extra-space: This option is specific to wic. It adds extra space after the space filled by the content of the partition. The final size can go beyond the size specified by --size. - By default, 10MB. + By default, 10MB. This option cannot be used + with --fixed-size option. --overhead-factor: This option is specific to wic. The size of the partition is multiplied by this factor. It has to be greater than or - equal to 1. - The default value is 1.3. + equal to 1. The default value is 1.3. + This option cannot be used with --fixed-size + option. --part-type: This option is specific to wic. It specifies partition type GUID for GPT partitions. diff --git a/import-layers/yocto-poky/scripts/lib/wic/imager/__init__.py b/import-layers/yocto-poky/scripts/lib/wic/imager/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/import-layers/yocto-poky/scripts/lib/wic/imager/__init__.py +++ /dev/null diff --git a/import-layers/yocto-poky/scripts/lib/wic/imager/baseimager.py b/import-layers/yocto-poky/scripts/lib/wic/imager/baseimager.py deleted file mode 100644 index 1a52dd8b4..000000000 --- a/import-layers/yocto-poky/scripts/lib/wic/imager/baseimager.py +++ /dev/null @@ -1,191 +0,0 @@ -#!/usr/bin/env python -tt -# -# Copyright (c) 2007 Red Hat Inc. -# Copyright (c) 2009, 2010, 2011 Intel, Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., 59 -# Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -import os -import tempfile -import shutil - -from wic import msger -from wic.utils.errors import CreatorError -from wic.utils import runner - -class BaseImageCreator(): - """Base class for image creation. - - BaseImageCreator is the simplest creator class available; it will - create a system image according to the supplied kickstart file. - - e.g. - - import wic.imgcreate as imgcreate - ks = imgcreate.read_kickstart("foo.ks") - imgcreate.ImageCreator(ks, "foo").create() - """ - - def __del__(self): - self.cleanup() - - def __init__(self, createopts=None): - """Initialize an ImageCreator instance. - - ks -- a pykickstart.KickstartParser instance; this instance will be - used to drive the install by e.g. providing the list of packages - to be installed, the system configuration and %post scripts - - name -- a name for the image; used for e.g. image filenames or - filesystem labels - """ - - self.__builddir = None - - self.ks = None - self.name = "target" - self.tmpdir = "/var/tmp/wic" - self.workdir = "/var/tmp/wic/build" - - # setup tmpfs tmpdir when enabletmpfs is True - self.enabletmpfs = False - - if createopts: - # Mapping table for variables that have different names. - optmap = {"outdir" : "destdir", - } - - # update setting from createopts - for key in createopts: - if key in optmap: - option = optmap[key] - else: - option = key - setattr(self, option, createopts[key]) - - self.destdir = os.path.abspath(os.path.expanduser(self.destdir)) - - self._dep_checks = ["ls", "bash", "cp", "echo"] - - # Output image file names - self.outimage = [] - - # No ks provided when called by convertor, so skip the dependency check - if self.ks: - # If we have btrfs partition we need to check necessary tools - for part in self.ks.partitions: - if part.fstype and part.fstype == "btrfs": - self._dep_checks.append("mkfs.btrfs") - break - - # make sure the specified tmpdir and cachedir exist - if not os.path.exists(self.tmpdir): - os.makedirs(self.tmpdir) - - - # - # Hooks for subclasses - # - def _create(self): - """Create partitions for the disk image(s) - - This is the hook where subclasses may create the partitions - that will be assembled into disk image(s). - - There is no default implementation. - """ - pass - - def _cleanup(self): - """Undo anything performed in _create(). - - This is the hook where subclasses must undo anything which was - done in _create(). - - There is no default implementation. - - """ - pass - - # - # Actual implementation - # - def __ensure_builddir(self): - if not self.__builddir is None: - return - - try: - self.workdir = os.path.join(self.tmpdir, "build") - if not os.path.exists(self.workdir): - os.makedirs(self.workdir) - self.__builddir = tempfile.mkdtemp(dir=self.workdir, - prefix="imgcreate-") - except OSError as err: - raise CreatorError("Failed create build directory in %s: %s" % - (self.tmpdir, err)) - - def __setup_tmpdir(self): - if not self.enabletmpfs: - return - - runner.show('mount -t tmpfs -o size=4G tmpfs %s' % self.workdir) - - def __clean_tmpdir(self): - if not self.enabletmpfs: - return - - runner.show('umount -l %s' % self.workdir) - - def create(self): - """Create partitions for the disk image(s) - - Create the partitions that will be assembled into disk - image(s). - """ - self.__setup_tmpdir() - self.__ensure_builddir() - - self._create() - - def cleanup(self): - """Undo anything performed in create(). - - Note, make sure to call this method once finished with the creator - instance in order to ensure no stale files are left on the host e.g.: - - creator = ImageCreator(ks, name) - try: - creator.create() - finally: - creator.cleanup() - - """ - if not self.__builddir: - return - - self._cleanup() - - shutil.rmtree(self.__builddir, ignore_errors=True) - self.__builddir = None - - self.__clean_tmpdir() - - - def print_outimage_info(self): - msg = "The new image can be found here:\n" - self.outimage.sort() - for path in self.outimage: - msg += ' %s\n' % os.path.abspath(path) - - msger.info(msg) diff --git a/import-layers/yocto-poky/scripts/lib/wic/imager/direct.py b/import-layers/yocto-poky/scripts/lib/wic/imager/direct.py deleted file mode 100644 index 4c547e04a..000000000 --- a/import-layers/yocto-poky/scripts/lib/wic/imager/direct.py +++ /dev/null @@ -1,419 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -# -# Copyright (c) 2013, Intel Corporation. -# All rights reserved. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# DESCRIPTION -# This implements the 'direct' image creator class for 'wic' -# -# AUTHORS -# Tom Zanussi <tom.zanussi (at] linux.intel.com> -# - -import os -import shutil -import uuid - -from wic import msger -from wic.utils.oe.misc import get_bitbake_var -from wic.utils.partitionedfs import Image -from wic.utils.errors import CreatorError, ImageError -from wic.imager.baseimager import BaseImageCreator -from wic.plugin import pluginmgr -from wic.utils.oe.misc import exec_cmd, exec_native_cmd - -disk_methods = { - "do_install_disk":None, -} - -class DiskImage(): - """ - A Disk backed by a file. - """ - def __init__(self, device, size): - self.size = size - self.device = device - self.created = False - - def exists(self): - return os.path.exists(self.device) - - def create(self): - if self.created: - return - # create sparse disk image - cmd = "truncate %s -s %s" % (self.device, self.size) - exec_cmd(cmd) - self.created = True - -class DirectImageCreator(BaseImageCreator): - """ - Installs a system into a file containing a partitioned disk image. - - DirectImageCreator is an advanced ImageCreator subclass; an image - file is formatted with a partition table, each partition created - from a rootfs or other OpenEmbedded build artifact and dd'ed into - the virtual disk. The disk image can subsequently be dd'ed onto - media and used on actual hardware. - """ - - def __init__(self, oe_builddir, image_output_dir, rootfs_dir, bootimg_dir, - kernel_dir, native_sysroot, compressor, creatoropts=None, - bmap=False): - """ - Initialize a DirectImageCreator instance. - - This method takes the same arguments as ImageCreator.__init__() - """ - BaseImageCreator.__init__(self, creatoropts) - - self.__image = None - self.__disks = {} - self.__disk_format = "direct" - self._disk_names = [] - self.ptable_format = self.ks.bootloader.ptable - - self.oe_builddir = oe_builddir - if image_output_dir: - self.tmpdir = image_output_dir - self.rootfs_dir = rootfs_dir - self.bootimg_dir = bootimg_dir - self.kernel_dir = kernel_dir - self.native_sysroot = native_sysroot - self.compressor = compressor - self.bmap = bmap - - def __get_part_num(self, num, parts): - """calculate the real partition number, accounting for partitions not - in the partition table and logical partitions - """ - realnum = 0 - for pnum, part in enumerate(parts, 1): - if not part.no_table: - realnum += 1 - if pnum == num: - if part.no_table: - return 0 - if self.ptable_format == 'msdos' and realnum > 3 and len(parts) > 4: - # account for logical partition numbering, ex. sda5.. - return realnum + 1 - return realnum - - def _write_fstab(self, image_rootfs): - """overriden to generate fstab (temporarily) in rootfs. This is called - from _create, make sure it doesn't get called from - BaseImage.create() - """ - if not image_rootfs: - return - - fstab_path = image_rootfs + "/etc/fstab" - if not os.path.isfile(fstab_path): - return - - with open(fstab_path) as fstab: - fstab_lines = fstab.readlines() - - if self._update_fstab(fstab_lines, self._get_parts()): - shutil.copyfile(fstab_path, fstab_path + ".orig") - - with open(fstab_path, "w") as fstab: - fstab.writelines(fstab_lines) - - return fstab_path - - def _update_fstab(self, fstab_lines, parts): - """Assume partition order same as in wks""" - updated = False - for num, part in enumerate(parts, 1): - pnum = self.__get_part_num(num, parts) - if not pnum or not part.mountpoint \ - or part.mountpoint in ("/", "/boot"): - continue - - # mmc device partitions are named mmcblk0p1, mmcblk0p2.. - prefix = 'p' if part.disk.startswith('mmcblk') else '' - device_name = "/dev/%s%s%d" % (part.disk, prefix, pnum) - - opts = part.fsopts if part.fsopts else "defaults" - line = "\t".join([device_name, part.mountpoint, part.fstype, - opts, "0", "0"]) + "\n" - - fstab_lines.append(line) - updated = True - - return updated - - def set_bootimg_dir(self, bootimg_dir): - """ - Accessor for bootimg_dir, the actual location used for the source - of the bootimg. Should be set by source plugins (only if they - change the default bootimg source) so the correct info gets - displayed for print_outimage_info(). - """ - self.bootimg_dir = bootimg_dir - - def _get_parts(self): - if not self.ks: - raise CreatorError("Failed to get partition info, " - "please check your kickstart setting.") - - # Set a default partition if no partition is given out - if not self.ks.partitions: - partstr = "part / --size 1900 --ondisk sda --fstype=ext3" - args = partstr.split() - part = self.ks.parse(args[1:]) - if part not in self.ks.partitions: - self.ks.partitions.append(part) - - # partitions list from kickstart file - return self.ks.partitions - - def get_disk_names(self): - """ Returns a list of physical target disk names (e.g., 'sdb') which - will be created. """ - - if self._disk_names: - return self._disk_names - - #get partition info from ks handler - parts = self._get_parts() - - for i in range(len(parts)): - if parts[i].disk: - disk_name = parts[i].disk - else: - raise CreatorError("Failed to create disks, no --ondisk " - "specified in partition line of ks file") - - if parts[i].mountpoint and not parts[i].fstype: - raise CreatorError("Failed to create disks, no --fstype " - "specified for partition with mountpoint " - "'%s' in the ks file") - - self._disk_names.append(disk_name) - - return self._disk_names - - def _full_name(self, name, extention): - """ Construct full file name for a file we generate. """ - return "%s-%s.%s" % (self.name, name, extention) - - def _full_path(self, path, name, extention): - """ Construct full file path to a file we generate. """ - return os.path.join(path, self._full_name(name, extention)) - - def get_default_source_plugin(self): - """ - The default source plugin i.e. the plugin that's consulted for - overall image generation tasks outside of any particular - partition. For convenience, we just hang it off the - bootloader handler since it's the one non-partition object in - any setup. By default the default plugin is set to the same - plugin as the /boot partition; since we hang it off the - bootloader object, the default can be explicitly set using the - --source bootloader param. - """ - return self.ks.bootloader.source - - # - # Actual implemention - # - def _create(self): - """ - For 'wic', we already have our build artifacts - we just create - filesystems from the artifacts directly and combine them into - a partitioned image. - """ - parts = self._get_parts() - - self.__image = Image(self.native_sysroot) - - disk_ids = {} - for num, part in enumerate(parts, 1): - # as a convenience, set source to the boot partition source - # instead of forcing it to be set via bootloader --source - if not self.ks.bootloader.source and part.mountpoint == "/boot": - self.ks.bootloader.source = part.source - - # generate parition UUIDs - if not part.uuid and part.use_uuid: - if self.ptable_format == 'gpt': - part.uuid = str(uuid.uuid4()) - else: # msdos partition table - if part.disk not in disk_ids: - disk_ids[part.disk] = int.from_bytes(os.urandom(4), 'little') - disk_id = disk_ids[part.disk] - part.uuid = '%0x-%02d' % (disk_id, self.__get_part_num(num, parts)) - - fstab_path = self._write_fstab(self.rootfs_dir.get("ROOTFS_DIR")) - - shutil.rmtree(self.workdir) - os.mkdir(self.workdir) - - for part in parts: - # get rootfs size from bitbake variable if it's not set in .ks file - if not part.size: - # and if rootfs name is specified for the partition - image_name = part.rootfs_dir - if image_name: - # Bitbake variable ROOTFS_SIZE is calculated in - # Image._get_rootfs_size method from meta/lib/oe/image.py - # using IMAGE_ROOTFS_SIZE, IMAGE_ROOTFS_ALIGNMENT, - # IMAGE_OVERHEAD_FACTOR and IMAGE_ROOTFS_EXTRA_SPACE - rsize_bb = get_bitbake_var('ROOTFS_SIZE', image_name) - if rsize_bb: - part.size = int(round(float(rsize_bb))) - # need to create the filesystems in order to get their - # sizes before we can add them and do the layout. - # Image.create() actually calls __format_disks() to create - # the disk images and carve out the partitions, then - # self.assemble() calls Image.assemble() which calls - # __write_partitition() for each partition to dd the fs - # into the partitions. - part.prepare(self, self.workdir, self.oe_builddir, self.rootfs_dir, - self.bootimg_dir, self.kernel_dir, self.native_sysroot) - - - self.__image.add_partition(int(part.size), - part.disk, - part.mountpoint, - part.source_file, - part.fstype, - part.label, - fsopts=part.fsopts, - boot=part.active, - align=part.align, - no_table=part.no_table, - part_type=part.part_type, - uuid=part.uuid, - system_id=part.system_id) - - if fstab_path: - shutil.move(fstab_path + ".orig", fstab_path) - - self.__image.layout_partitions(self.ptable_format) - - self.__imgdir = self.workdir - for disk_name, disk in self.__image.disks.items(): - full_path = self._full_path(self.__imgdir, disk_name, "direct") - msger.debug("Adding disk %s as %s with size %s bytes" \ - % (disk_name, full_path, disk['min_size'])) - disk_obj = DiskImage(full_path, disk['min_size']) - self.__disks[disk_name] = disk_obj - self.__image.add_disk(disk_name, disk_obj, disk_ids.get(disk_name)) - - self.__image.create() - - def assemble(self): - """ - Assemble partitions into disk image(s) - """ - for disk_name, disk in self.__image.disks.items(): - full_path = self._full_path(self.__imgdir, disk_name, "direct") - msger.debug("Assembling disk %s as %s with size %s bytes" \ - % (disk_name, full_path, disk['min_size'])) - self.__image.assemble(full_path) - - def finalize(self): - """ - Finalize the disk image. - - For example, prepare the image to be bootable by e.g. - creating and installing a bootloader configuration. - - """ - source_plugin = self.get_default_source_plugin() - if source_plugin: - self._source_methods = pluginmgr.get_source_plugin_methods(source_plugin, disk_methods) - for disk_name, disk in self.__image.disks.items(): - self._source_methods["do_install_disk"](disk, disk_name, self, - self.workdir, - self.oe_builddir, - self.bootimg_dir, - self.kernel_dir, - self.native_sysroot) - - for disk_name, disk in self.__image.disks.items(): - full_path = self._full_path(self.__imgdir, disk_name, "direct") - # Generate .bmap - if self.bmap: - msger.debug("Generating bmap file for %s" % disk_name) - exec_native_cmd("bmaptool create %s -o %s.bmap" % (full_path, full_path), - self.native_sysroot) - # Compress the image - if self.compressor: - msger.debug("Compressing disk %s with %s" % (disk_name, self.compressor)) - exec_cmd("%s %s" % (self.compressor, full_path)) - - def print_outimage_info(self): - """ - Print the image(s) and artifacts used, for the user. - """ - msg = "The new image(s) can be found here:\n" - - parts = self._get_parts() - - for disk_name in self.__image.disks: - extension = "direct" + {"gzip": ".gz", - "bzip2": ".bz2", - "xz": ".xz", - "": ""}.get(self.compressor) - full_path = self._full_path(self.__imgdir, disk_name, extension) - msg += ' %s\n\n' % full_path - - msg += 'The following build artifacts were used to create the image(s):\n' - for part in parts: - if part.rootfs_dir is None: - continue - if part.mountpoint == '/': - suffix = ':' - else: - suffix = '["%s"]:' % (part.mountpoint or part.label) - msg += ' ROOTFS_DIR%s%s\n' % (suffix.ljust(20), part.rootfs_dir) - - msg += ' BOOTIMG_DIR: %s\n' % self.bootimg_dir - msg += ' KERNEL_DIR: %s\n' % self.kernel_dir - msg += ' NATIVE_SYSROOT: %s\n' % self.native_sysroot - - msger.info(msg) - - @property - def rootdev(self): - """ - Get root device name to use as a 'root' parameter - in kernel command line. - - Assume partition order same as in wks - """ - parts = self._get_parts() - for num, part in enumerate(parts, 1): - if part.mountpoint == "/": - if part.uuid: - return "PARTUUID=%s" % part.uuid - else: - suffix = 'p' if part.disk.startswith('mmcblk') else '' - pnum = self.__get_part_num(num, parts) - return "/dev/%s%s%-d" % (part.disk, suffix, pnum) - - def _cleanup(self): - if not self.__image is None: - try: - self.__image.cleanup() - except ImageError as err: - msger.warning("%s" % err) - diff --git a/import-layers/yocto-poky/scripts/lib/wic/ksparser.py b/import-layers/yocto-poky/scripts/lib/wic/ksparser.py index 0894e2b19..d026caad0 100644 --- a/import-layers/yocto-poky/scripts/lib/wic/ksparser.py +++ b/import-layers/yocto-poky/scripts/lib/wic/ksparser.py @@ -27,11 +27,14 @@ import os import shlex +import logging + from argparse import ArgumentParser, ArgumentError, ArgumentTypeError -from wic import msger +from wic.engine import find_canned from wic.partition import Partition -from wic.utils.misc import find_canned + +logger = logging.getLogger('wic') class KickStartError(Exception): """Custom exception.""" @@ -113,6 +116,9 @@ def systemidtype(arg): class KickStart(): """"Kickstart parser implementation.""" + DEFAULT_EXTRA_SPACE = 10*1024 + DEFAULT_OVERHEAD_FACTOR = 1.3 + def __init__(self, confpath): self.partitions = [] @@ -127,16 +133,27 @@ class KickStart(): part.add_argument('mountpoint', nargs='?') part.add_argument('--active', action='store_true') part.add_argument('--align', type=int) - part.add_argument("--extra-space", type=sizetype, default=10*1024) + part.add_argument('--exclude-path', nargs='+') + part.add_argument("--extra-space", type=sizetype) part.add_argument('--fsoptions', dest='fsopts') - part.add_argument('--fstype') + part.add_argument('--fstype', default='vfat', + choices=('ext2', 'ext3', 'ext4', 'btrfs', + 'squashfs', 'vfat', 'msdos', 'swap')) part.add_argument('--label') part.add_argument('--no-table', action='store_true') - part.add_argument('--ondisk', '--ondrive', dest='disk') - part.add_argument("--overhead-factor", type=overheadtype, default=1.3) + part.add_argument('--ondisk', '--ondrive', dest='disk', default='sda') + part.add_argument("--overhead-factor", type=overheadtype) part.add_argument('--part-type') part.add_argument('--rootfs-dir') - part.add_argument('--size', type=sizetype, default=0) + + # --size and --fixed-size cannot be specified together; options + # ----extra-space and --overhead-factor should also raise a parser + # --error, but since nesting mutually exclusive groups does not work, + # ----extra-space/--overhead-factor are handled later + sizeexcl = part.add_mutually_exclusive_group() + sizeexcl.add_argument('--size', type=sizetype, default=0) + sizeexcl.add_argument('--fixed-size', type=sizetype, default=0) + part.add_argument('--source') part.add_argument('--sourceparams') part.add_argument('--system-id', type=systemidtype) @@ -156,7 +173,7 @@ class KickStart(): self._parse(parser, confpath) if not self.bootloader: - msger.warning('bootloader config not specified, using defaults') + logger.warning('bootloader config not specified, using defaults\n') self.bootloader = bootloader.parse_args([]) def _parse(self, parser, confpath): @@ -170,11 +187,33 @@ class KickStart(): lineno += 1 if line and line[0] != '#': try: - parsed = parser.parse_args(shlex.split(line)) + line_args = shlex.split(line) + parsed = parser.parse_args(line_args) except ArgumentError as err: raise KickStartError('%s:%d: %s' % \ (confpath, lineno, err)) if line.startswith('part'): + # using ArgumentParser one cannot easily tell if option + # was passed as argument, if said option has a default + # value; --overhead-factor/--extra-space cannot be used + # with --fixed-size, so at least detect when these were + # passed with non-0 values ... + if parsed.fixed_size: + if parsed.overhead_factor or parsed.extra_space: + err = "%s:%d: arguments --overhead-factor and --extra-space not "\ + "allowed with argument --fixed-size" \ + % (confpath, lineno) + raise KickStartError(err) + else: + # ... and provide defaults if not using + # --fixed-size iff given option was not used + # (again, one cannot tell if option was passed but + # with value equal to 0) + if '--overhead-factor' not in line_args: + parsed.overhead_factor = self.DEFAULT_OVERHEAD_FACTOR + if '--extra-space' not in line_args: + parsed.extra_space = self.DEFAULT_EXTRA_SPACE + self.partnum += 1 self.partitions.append(Partition(parsed, self.partnum)) elif line.startswith('include'): diff --git a/import-layers/yocto-poky/scripts/lib/wic/msger.py b/import-layers/yocto-poky/scripts/lib/wic/msger.py deleted file mode 100644 index fb8336d94..000000000 --- a/import-layers/yocto-poky/scripts/lib/wic/msger.py +++ /dev/null @@ -1,235 +0,0 @@ -#!/usr/bin/env python -tt -# vim: ai ts=4 sts=4 et sw=4 -# -# Copyright (c) 2009, 2010, 2011 Intel, Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., 59 -# Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -import os -import sys -import re -import time - -__ALL__ = ['get_loglevel', - 'set_loglevel', - 'set_logfile', - 'debug', - 'verbose', - 'info', - 'warning', - 'error', - ] - -# COLORs in ANSI -INFO_COLOR = 32 # green -WARN_COLOR = 33 # yellow -ERR_COLOR = 31 # red -ASK_COLOR = 34 # blue -NO_COLOR = 0 - -PREFIX_RE = re.compile('^<(.*?)>\s*(.*)', re.S) - -INTERACTIVE = True - -LOG_LEVEL = 1 -LOG_LEVELS = { - 'quiet': 0, - 'normal': 1, - 'verbose': 2, - 'debug': 3, - 'never': 4, -} - -LOG_FILE_FP = None -LOG_CONTENT = '' -CATCHERR_BUFFILE_FD = -1 -CATCHERR_BUFFILE_PATH = None -CATCHERR_SAVED_2 = -1 - -def _general_print(head, color, msg=None, stream=None, level='normal'): - global LOG_CONTENT - if not stream: - stream = sys.stdout - - if LOG_LEVELS[level] > LOG_LEVEL: - # skip - return - - errormsg = '' - if CATCHERR_BUFFILE_FD > 0: - size = os.lseek(CATCHERR_BUFFILE_FD, 0, os.SEEK_END) - os.lseek(CATCHERR_BUFFILE_FD, 0, os.SEEK_SET) - errormsg = os.read(CATCHERR_BUFFILE_FD, size) - os.ftruncate(CATCHERR_BUFFILE_FD, 0) - - # append error msg to LOG - if errormsg: - LOG_CONTENT += errormsg - - # append normal msg to LOG - save_msg = msg.strip() if msg else None - if save_msg: - timestr = time.strftime("[%m/%d %H:%M:%S %Z] ", time.localtime()) - LOG_CONTENT += timestr + save_msg + '\n' - - if errormsg: - _color_print('', NO_COLOR, errormsg, stream, level) - - _color_print(head, color, msg, stream, level) - -def _color_print(head, color, msg, stream, level): - colored = True - if color == NO_COLOR or \ - not stream.isatty() or \ - os.getenv('ANSI_COLORS_DISABLED') is not None: - colored = False - - if head.startswith('\r'): - # need not \n at last - newline = False - else: - newline = True - - if colored: - head = '\033[%dm%s:\033[0m ' %(color, head) - if not newline: - # ESC cmd to clear line - head = '\033[2K' + head - else: - if head: - head += ': ' - if head.startswith('\r'): - head = head.lstrip() - newline = True - - if msg is not None: - stream.write('%s%s' % (head, msg)) - if newline: - stream.write('\n') - - stream.flush() - -def _color_perror(head, color, msg, level='normal'): - if CATCHERR_BUFFILE_FD > 0: - _general_print(head, color, msg, sys.stdout, level) - else: - _general_print(head, color, msg, sys.stderr, level) - -def _split_msg(head, msg): - if isinstance(msg, list): - msg = '\n'.join(map(str, msg)) - - if msg.startswith('\n'): - # means print \n at first - msg = msg.lstrip() - head = '\n' + head - - elif msg.startswith('\r'): - # means print \r at first - msg = msg.lstrip() - head = '\r' + head - - match = PREFIX_RE.match(msg) - if match: - head += ' <%s>' % match.group(1) - msg = match.group(2) - - return head, msg - -def get_loglevel(): - return next((k for k, v in LOG_LEVELS.items() if v == LOG_LEVEL)) - -def set_loglevel(level): - global LOG_LEVEL - if level not in LOG_LEVELS: - # no effect - return - - LOG_LEVEL = LOG_LEVELS[level] - -def set_interactive(mode=True): - global INTERACTIVE - if mode: - INTERACTIVE = True - else: - INTERACTIVE = False - -def log(msg=''): - # log msg to LOG_CONTENT then save to logfile - global LOG_CONTENT - if msg: - LOG_CONTENT += msg - -def info(msg): - head, msg = _split_msg('Info', msg) - _general_print(head, INFO_COLOR, msg) - -def verbose(msg): - head, msg = _split_msg('Verbose', msg) - _general_print(head, INFO_COLOR, msg, level='verbose') - -def warning(msg): - head, msg = _split_msg('Warning', msg) - _color_perror(head, WARN_COLOR, msg) - -def debug(msg): - head, msg = _split_msg('Debug', msg) - _color_perror(head, ERR_COLOR, msg, level='debug') - -def error(msg): - head, msg = _split_msg('Error', msg) - _color_perror(head, ERR_COLOR, msg) - sys.exit(1) - -def set_logfile(fpath): - global LOG_FILE_FP - - def _savelogf(): - if LOG_FILE_FP: - with open(LOG_FILE_FP, 'w') as log: - log.write(LOG_CONTENT) - - if LOG_FILE_FP is not None: - warning('duplicate log file configuration') - - LOG_FILE_FP = fpath - - import atexit - atexit.register(_savelogf) - -def enable_logstderr(fpath): - global CATCHERR_BUFFILE_FD - global CATCHERR_BUFFILE_PATH - global CATCHERR_SAVED_2 - - if os.path.exists(fpath): - os.remove(fpath) - CATCHERR_BUFFILE_PATH = fpath - CATCHERR_BUFFILE_FD = os.open(CATCHERR_BUFFILE_PATH, os.O_RDWR|os.O_CREAT) - CATCHERR_SAVED_2 = os.dup(2) - os.dup2(CATCHERR_BUFFILE_FD, 2) - -def disable_logstderr(): - global CATCHERR_BUFFILE_FD - global CATCHERR_BUFFILE_PATH - global CATCHERR_SAVED_2 - - raw(msg=None) # flush message buffer and print it. - os.dup2(CATCHERR_SAVED_2, 2) - os.close(CATCHERR_SAVED_2) - os.close(CATCHERR_BUFFILE_FD) - os.unlink(CATCHERR_BUFFILE_PATH) - CATCHERR_BUFFILE_FD = -1 - CATCHERR_BUFFILE_PATH = None - CATCHERR_SAVED_2 = -1 diff --git a/import-layers/yocto-poky/scripts/lib/wic/partition.py b/import-layers/yocto-poky/scripts/lib/wic/partition.py index ec3aa6622..939e66731 100644 --- a/import-layers/yocto-poky/scripts/lib/wic/partition.py +++ b/import-layers/yocto-poky/scripts/lib/wic/partition.py @@ -24,18 +24,15 @@ # Tom Zanussi <tom.zanussi (at] linux.intel.com> # Ed Bartosh <ed.bartosh> (at] linux.intel.com> +import logging import os import tempfile -from wic.utils.oe.misc import msger, parse_sourceparams -from wic.utils.oe.misc import exec_cmd, exec_native_cmd -from wic.plugin import pluginmgr +from wic import WicError +from wic.utils.misc import exec_cmd, exec_native_cmd, get_bitbake_var +from wic.pluginbase import PluginMgr -partition_methods = { - "do_stage_partition":None, - "do_prepare_partition":None, - "do_configure_partition":None, -} +logger = logging.getLogger('wic') class Partition(): @@ -44,16 +41,20 @@ class Partition(): self.active = args.active self.align = args.align self.disk = args.disk + self.device = None self.extra_space = args.extra_space + self.exclude_path = args.exclude_path self.fsopts = args.fsopts self.fstype = args.fstype self.label = args.label self.mountpoint = args.mountpoint self.no_table = args.no_table + self.num = None self.overhead_factor = args.overhead_factor self.part_type = args.part_type self.rootfs_dir = args.rootfs_dir self.size = args.size + self.fixed_size = args.fixed_size self.source = args.source self.sourceparams = args.sourceparams self.system_id = args.system_id @@ -71,100 +72,128 @@ class Partition(): number of (1k) blocks we need to add to get to --size, 0 if we're already there or beyond. """ - msger.debug("Requested partition size for %s: %d" % \ - (self.mountpoint, self.size)) + logger.debug("Requested partition size for %s: %d", + self.mountpoint, self.size) if not self.size: return 0 requested_blocks = self.size - msger.debug("Requested blocks %d, current_blocks %d" % \ - (requested_blocks, current_blocks)) + logger.debug("Requested blocks %d, current_blocks %d", + requested_blocks, current_blocks) if requested_blocks > current_blocks: return requested_blocks - current_blocks else: return 0 + def get_rootfs_size(self, actual_rootfs_size=0): + """ + Calculate the required size of rootfs taking into consideration + --size/--fixed-size flags as well as overhead and extra space, as + specified in kickstart file. Raises an error if the + `actual_rootfs_size` is larger than fixed-size rootfs. + + """ + if self.fixed_size: + rootfs_size = self.fixed_size + if actual_rootfs_size > rootfs_size: + raise WicError("Actual rootfs size (%d kB) is larger than " + "allowed size %d kB" % + (actual_rootfs_size, rootfs_size)) + else: + extra_blocks = self.get_extra_block_count(actual_rootfs_size) + if extra_blocks < self.extra_space: + extra_blocks = self.extra_space + + rootfs_size = actual_rootfs_size + extra_blocks + rootfs_size *= self.overhead_factor + + logger.debug("Added %d extra blocks to %s to get to %d total blocks", + extra_blocks, self.mountpoint, rootfs_size) + + return rootfs_size + + @property + def disk_size(self): + """ + Obtain on-disk size of partition taking into consideration + --size/--fixed-size options. + + """ + return self.fixed_size if self.fixed_size else self.size + def prepare(self, creator, cr_workdir, oe_builddir, rootfs_dir, bootimg_dir, kernel_dir, native_sysroot): """ Prepare content for individual partitions, depending on partition command parameters. """ - if self.sourceparams: - self.sourceparams_dict = parse_sourceparams(self.sourceparams) - if not self.source: - if not self.size: - msger.error("The %s partition has a size of zero. Please " - "specify a non-zero --size for that partition." % \ - self.mountpoint) - if self.fstype and self.fstype == "swap": + if not self.size and not self.fixed_size: + raise WicError("The %s partition has a size of zero. Please " + "specify a non-zero --size/--fixed-size for that " + "partition." % self.mountpoint) + + if self.fstype == "swap": self.prepare_swap_partition(cr_workdir, oe_builddir, native_sysroot) self.source_file = "%s/fs.%s" % (cr_workdir, self.fstype) - elif self.fstype: + else: + if self.fstype == 'squashfs': + raise WicError("It's not possible to create empty squashfs " + "partition '%s'" % (self.mountpoint)) + rootfs = "%s/fs_%s.%s.%s" % (cr_workdir, self.label, self.lineno, self.fstype) if os.path.isfile(rootfs): os.remove(rootfs) - for prefix in ("ext", "btrfs", "vfat", "squashfs"): - if self.fstype.startswith(prefix): - method = getattr(self, - "prepare_empty_partition_" + prefix) - method(rootfs, oe_builddir, native_sysroot) - self.source_file = rootfs - break + + prefix = "ext" if self.fstype.startswith("ext") else self.fstype + method = getattr(self, "prepare_empty_partition_" + prefix) + method(rootfs, oe_builddir, native_sysroot) + self.source_file = rootfs return - plugins = pluginmgr.get_source_plugins() + plugins = PluginMgr.get_plugins('source') if self.source not in plugins: - msger.error("The '%s' --source specified for %s doesn't exist.\n\t" - "See 'wic list source-plugins' for a list of available" - " --sources.\n\tSee 'wic help source-plugins' for " - "details on adding a new source plugin." % \ - (self.source, self.mountpoint)) - - self._source_methods = pluginmgr.get_source_plugin_methods(\ - self.source, partition_methods) - self._source_methods["do_configure_partition"](self, self.sourceparams_dict, - creator, cr_workdir, - oe_builddir, - bootimg_dir, - kernel_dir, - native_sysroot) - self._source_methods["do_stage_partition"](self, self.sourceparams_dict, - creator, cr_workdir, - oe_builddir, - bootimg_dir, kernel_dir, - native_sysroot) - self._source_methods["do_prepare_partition"](self, self.sourceparams_dict, - creator, cr_workdir, - oe_builddir, - bootimg_dir, kernel_dir, rootfs_dir, - native_sysroot) + raise WicError("The '%s' --source specified for %s doesn't exist.\n\t" + "See 'wic list source-plugins' for a list of available" + " --sources.\n\tSee 'wic help source-plugins' for " + "details on adding a new source plugin." % + (self.source, self.mountpoint)) + + srcparams_dict = {} + if self.sourceparams: + # Split sourceparams string of the form key1=val1[,key2=val2,...] + # into a dict. Also accepts valueless keys i.e. without = + splitted = self.sourceparams.split(',') + srcparams_dict = dict(par.split('=') for par in splitted if par) + + plugin = PluginMgr.get_plugins('source')[self.source] + plugin.do_configure_partition(self, srcparams_dict, creator, + cr_workdir, oe_builddir, bootimg_dir, + kernel_dir, native_sysroot) + plugin.do_stage_partition(self, srcparams_dict, creator, + cr_workdir, oe_builddir, bootimg_dir, + kernel_dir, native_sysroot) + plugin.do_prepare_partition(self, srcparams_dict, creator, + cr_workdir, oe_builddir, bootimg_dir, + kernel_dir, rootfs_dir, native_sysroot) + # further processing required Partition.size to be an integer, make # sure that it is one - if type(self.size) is not int: - msger.error("Partition %s internal size is not an integer. " \ - "This a bug in source plugin %s and needs to be fixed." \ - % (self.mountpoint, self.source)) + if not isinstance(self.size, int): + raise WicError("Partition %s internal size is not an integer. " + "This a bug in source plugin %s and needs to be fixed." % + (self.mountpoint, self.source)) - def prepare_rootfs_from_fs_image(self, cr_workdir, oe_builddir, - rootfs_dir): - """ - Handle an already-created partition e.g. xxx.ext3 - """ - rootfs = oe_builddir - du_cmd = "du -Lbks %s" % rootfs - out = exec_cmd(du_cmd) - rootfs_size = out.split()[0] - - self.size = int(rootfs_size) - self.source_file = rootfs + if self.fixed_size and self.size > self.fixed_size: + raise WicError("File system image of partition %s is " + "larger (%d kB) than its allowed size %d kB" % + (self.mountpoint, self.size, self.fixed_size)) def prepare_rootfs(self, cr_workdir, oe_builddir, rootfs_dir, native_sysroot): @@ -183,30 +212,36 @@ class Partition(): pseudo += "export PSEUDO_LOCALSTATEDIR=%s;" % p_localstatedir pseudo += "export PSEUDO_PASSWD=%s;" % p_passwd pseudo += "export PSEUDO_NOSYMLINKEXP=%s;" % p_nosymlinkexp - pseudo += "%s/usr/bin/pseudo " % native_sysroot + pseudo += "%s " % get_bitbake_var("FAKEROOTCMD") rootfs = "%s/rootfs_%s.%s.%s" % (cr_workdir, self.label, self.lineno, self.fstype) if os.path.isfile(rootfs): os.remove(rootfs) - if not self.fstype: - msger.error("File system for partition %s not specified in kickstart, " \ - "use --fstype option" % (self.mountpoint)) - - for prefix in ("ext", "btrfs", "vfat", "squashfs"): - if self.fstype.startswith(prefix): - method = getattr(self, "prepare_rootfs_" + prefix) - method(rootfs, oe_builddir, rootfs_dir, native_sysroot, pseudo) - - self.source_file = rootfs - - # get the rootfs size in the right units for kickstart (kB) - du_cmd = "du -Lbks %s" % rootfs - out = exec_cmd(du_cmd) - self.size = int(out.split()[0]) + # Get rootfs size from bitbake variable if it's not set in .ks file + if not self.size: + # Bitbake variable ROOTFS_SIZE is calculated in + # Image._get_rootfs_size method from meta/lib/oe/image.py + # using IMAGE_ROOTFS_SIZE, IMAGE_ROOTFS_ALIGNMENT, + # IMAGE_OVERHEAD_FACTOR and IMAGE_ROOTFS_EXTRA_SPACE + rsize_bb = get_bitbake_var('ROOTFS_SIZE') + if rsize_bb: + logger.warning('overhead-factor was specified, but size was not,' + ' so bitbake variables will be used for the size.' + ' In this case both IMAGE_OVERHEAD_FACTOR and ' + '--overhead-factor will be applied') + self.size = int(round(float(rsize_bb))) + + prefix = "ext" if self.fstype.startswith("ext") else self.fstype + method = getattr(self, "prepare_rootfs_" + prefix) + method(rootfs, oe_builddir, rootfs_dir, native_sysroot, pseudo) + self.source_file = rootfs - break + # get the rootfs size in the right units for kickstart (kB) + du_cmd = "du -Lbks %s" % rootfs + out = exec_cmd(du_cmd) + self.size = int(out.split()[0]) def prepare_rootfs_ext(self, rootfs, oe_builddir, rootfs_dir, native_sysroot, pseudo): @@ -217,17 +252,10 @@ class Partition(): out = exec_cmd(du_cmd) actual_rootfs_size = int(out.split()[0]) - extra_blocks = self.get_extra_block_count(actual_rootfs_size) - if extra_blocks < self.extra_space: - extra_blocks = self.extra_space - - rootfs_size = actual_rootfs_size + extra_blocks - rootfs_size *= self.overhead_factor - - msger.debug("Added %d extra blocks to %s to get to %d total blocks" % \ - (extra_blocks, self.mountpoint, rootfs_size)) + rootfs_size = self.get_rootfs_size(actual_rootfs_size) - exec_cmd("truncate %s -s %d" % (rootfs, rootfs_size * 1024)) + with open(rootfs, 'w') as sparse: + os.ftruncate(sparse.fileno(), rootfs_size * 1024) extra_imagecmd = "-i 8192" @@ -239,7 +267,7 @@ class Partition(): (self.fstype, extra_imagecmd, rootfs, label_str, rootfs_dir) exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo) - mkfs_cmd = "fsck.%s -pvfD %s || [ $? -le 3 ]" % (self.fstype, rootfs) + mkfs_cmd = "fsck.%s -pvfD %s" % (self.fstype, rootfs) exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo) def prepare_rootfs_btrfs(self, rootfs, oe_builddir, rootfs_dir, @@ -253,17 +281,10 @@ class Partition(): out = exec_cmd(du_cmd) actual_rootfs_size = int(out.split()[0]) - extra_blocks = self.get_extra_block_count(actual_rootfs_size) - if extra_blocks < self.extra_space: - extra_blocks = self.extra_space + rootfs_size = self.get_rootfs_size(actual_rootfs_size) - rootfs_size = actual_rootfs_size + extra_blocks - rootfs_size *= self.overhead_factor - - msger.debug("Added %d extra blocks to %s to get to %d total blocks" % \ - (extra_blocks, self.mountpoint, rootfs_size)) - - exec_cmd("truncate %s -s %d" % (rootfs, rootfs_size * 1024)) + with open(rootfs, 'w') as sparse: + os.ftruncate(sparse.fileno(), rootfs_size * 1024) label_str = "" if self.label: @@ -273,29 +294,27 @@ class Partition(): (self.fstype, rootfs_size * 1024, rootfs_dir, label_str, rootfs) exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo) - def prepare_rootfs_vfat(self, rootfs, oe_builddir, rootfs_dir, - native_sysroot, pseudo): + def prepare_rootfs_msdos(self, rootfs, oe_builddir, rootfs_dir, + native_sysroot, pseudo): """ - Prepare content for a vfat rootfs partition. + Prepare content for a msdos/vfat rootfs partition. """ du_cmd = "du -bks %s" % rootfs_dir out = exec_cmd(du_cmd) blocks = int(out.split()[0]) - extra_blocks = self.get_extra_block_count(blocks) - if extra_blocks < self.extra_space: - extra_blocks = self.extra_space - - blocks += extra_blocks - - msger.debug("Added %d extra blocks to %s to get to %d total blocks" % \ - (extra_blocks, self.mountpoint, blocks)) + rootfs_size = self.get_rootfs_size(blocks) label_str = "-n boot" if self.label: label_str = "-n %s" % self.label - dosfs_cmd = "mkdosfs %s -S 512 -C %s %d" % (label_str, rootfs, blocks) + size_str = "" + if self.fstype == 'msdos': + size_str = "-F 16" # FAT 16 + + dosfs_cmd = "mkdosfs %s -S 512 %s -C %s %d" % (label_str, size_str, + rootfs, rootfs_size) exec_native_cmd(dosfs_cmd, native_sysroot) mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (rootfs, rootfs_dir) @@ -304,6 +323,8 @@ class Partition(): chmod_cmd = "chmod 644 %s" % rootfs exec_cmd(chmod_cmd) + prepare_rootfs_vfat = prepare_rootfs_msdos + def prepare_rootfs_squashfs(self, rootfs, oe_builddir, rootfs_dir, native_sysroot, pseudo): """ @@ -318,7 +339,9 @@ class Partition(): """ Prepare an empty ext2/3/4 partition. """ - exec_cmd("truncate %s -s %d" % (rootfs, self.size * 1024)) + size = self.disk_size + with open(rootfs, 'w') as sparse: + os.ftruncate(sparse.fileno(), size * 1024) extra_imagecmd = "-i 8192" @@ -335,7 +358,9 @@ class Partition(): """ Prepare an empty btrfs partition. """ - exec_cmd("truncate %s -s %d" % (rootfs, self.size * 1024)) + size = self.disk_size + with open(rootfs, 'w') as sparse: + os.ftruncate(sparse.fileno(), size * 1024) label_str = "" if self.label: @@ -345,50 +370,29 @@ class Partition(): (self.fstype, self.size * 1024, label_str, rootfs) exec_native_cmd(mkfs_cmd, native_sysroot) - def prepare_empty_partition_vfat(self, rootfs, oe_builddir, - native_sysroot): + def prepare_empty_partition_msdos(self, rootfs, oe_builddir, + native_sysroot): """ Prepare an empty vfat partition. """ - blocks = self.size + blocks = self.disk_size label_str = "-n boot" if self.label: label_str = "-n %s" % self.label - dosfs_cmd = "mkdosfs %s -S 512 -C %s %d" % (label_str, rootfs, blocks) + size_str = "" + if self.fstype == 'msdos': + size_str = "-F 16" # FAT 16 + + dosfs_cmd = "mkdosfs %s -S 512 %s -C %s %d" % (label_str, size_str, + rootfs, blocks) exec_native_cmd(dosfs_cmd, native_sysroot) chmod_cmd = "chmod 644 %s" % rootfs exec_cmd(chmod_cmd) - def prepare_empty_partition_squashfs(self, cr_workdir, oe_builddir, - native_sysroot): - """ - Prepare an empty squashfs partition. - """ - msger.warning("Creating of an empty squashfs %s partition was attempted. " \ - "Proceeding as requested." % self.mountpoint) - - path = "%s/fs_%s.%s" % (cr_workdir, self.label, self.fstype) - os.path.isfile(path) and os.remove(path) - - # it is not possible to create a squashfs without source data, - # thus prepare an empty temp dir that is used as source - tmpdir = tempfile.mkdtemp() - - squashfs_cmd = "mksquashfs %s %s -noappend" % \ - (tmpdir, path) - exec_native_cmd(squashfs_cmd, native_sysroot) - - os.rmdir(tmpdir) - - # get the rootfs size in the right units for kickstart (kB) - du_cmd = "du -Lbks %s" % path - out = exec_cmd(du_cmd) - fs_size = out.split()[0] - - self.size = int(fs_size) + prepare_empty_partition_vfat = prepare_empty_partition_msdos def prepare_swap_partition(self, cr_workdir, oe_builddir, native_sysroot): """ @@ -396,7 +400,8 @@ class Partition(): """ path = "%s/fs.%s" % (cr_workdir, self.fstype) - exec_cmd("truncate %s -s %d" % (path, self.size * 1024)) + with open(path, 'w') as sparse: + os.ftruncate(sparse.fileno(), self.size * 1024) import uuid label_str = "" @@ -404,4 +409,3 @@ class Partition(): label_str = "-L %s" % self.label mkswap_cmd = "mkswap %s -U %s %s" % (label_str, str(uuid.uuid1()), path) exec_native_cmd(mkswap_cmd, native_sysroot) - diff --git a/import-layers/yocto-poky/scripts/lib/wic/plugin.py b/import-layers/yocto-poky/scripts/lib/wic/plugin.py deleted file mode 100644 index 306b32437..000000000 --- a/import-layers/yocto-poky/scripts/lib/wic/plugin.py +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env python -tt -# -# Copyright (c) 2011 Intel, Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., 59 -# Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -import os, sys - -from wic import msger -from wic import pluginbase -from wic.utils import errors -from wic.utils.oe.misc import get_bitbake_var - -__ALL__ = ['PluginMgr', 'pluginmgr'] - -PLUGIN_TYPES = ["imager", "source"] - -PLUGIN_DIR = "/lib/wic/plugins" # relative to scripts -SCRIPTS_PLUGIN_DIR = "scripts" + PLUGIN_DIR - -class PluginMgr(): - plugin_dirs = {} - - # make the manager class as singleton - _instance = None - def __new__(cls, *args, **kwargs): - if not cls._instance: - cls._instance = super(PluginMgr, cls).__new__(cls, *args, **kwargs) - - return cls._instance - - def __init__(self): - wic_path = os.path.dirname(__file__) - eos = wic_path.rfind('scripts') + len('scripts') - scripts_path = wic_path[:eos] - self.scripts_path = scripts_path - self.plugin_dir = scripts_path + PLUGIN_DIR - self.layers_path = None - - def _build_plugin_dir_list(self, plugin_dir, ptype): - if self.layers_path is None: - self.layers_path = get_bitbake_var("BBLAYERS") - layer_dirs = [] - - if self.layers_path is not None: - for layer_path in self.layers_path.split(): - path = os.path.join(layer_path, SCRIPTS_PLUGIN_DIR, ptype) - layer_dirs.append(path) - - path = os.path.join(plugin_dir, ptype) - layer_dirs.append(path) - - return layer_dirs - - def append_dirs(self, dirs): - for path in dirs: - self._add_plugindir(path) - - # load all the plugins AGAIN - self._load_all() - - def _add_plugindir(self, path): - path = os.path.abspath(os.path.expanduser(path)) - - if not os.path.isdir(path): - return - - if path not in self.plugin_dirs: - self.plugin_dirs[path] = False - # the value True/False means "loaded" - - def _load_all(self): - for (pdir, loaded) in self.plugin_dirs.items(): - if loaded: - continue - - sys.path.insert(0, pdir) - for mod in [x[:-3] for x in os.listdir(pdir) if x.endswith(".py")]: - if mod and mod != '__init__': - if mod in sys.modules: - #self.plugin_dirs[pdir] = True - msger.warning("Module %s already exists, skip" % mod) - else: - try: - pymod = __import__(mod) - self.plugin_dirs[pdir] = True - msger.debug("Plugin module %s:%s imported"\ - % (mod, pymod.__file__)) - except ImportError as err: - msg = 'Failed to load plugin %s/%s: %s' \ - % (os.path.basename(pdir), mod, err) - msger.warning(msg) - - del sys.path[0] - - def get_plugins(self, ptype): - """ the return value is dict of name:class pairs """ - - if ptype not in PLUGIN_TYPES: - raise errors.CreatorError('%s is not valid plugin type' % ptype) - - plugins_dir = self._build_plugin_dir_list(self.plugin_dir, ptype) - - self.append_dirs(plugins_dir) - - return pluginbase.get_plugins(ptype) - - def get_source_plugins(self): - """ - Return list of available source plugins. - """ - plugins_dir = self._build_plugin_dir_list(self.plugin_dir, 'source') - - self.append_dirs(plugins_dir) - - return self.get_plugins('source') - - - def get_source_plugin_methods(self, source_name, methods): - """ - The methods param is a dict with the method names to find. On - return, the dict values will be filled in with pointers to the - corresponding methods. If one or more methods are not found, - None is returned. - """ - return_methods = None - for _source_name, klass in self.get_plugins('source').items(): - if _source_name == source_name: - for _method_name in methods: - if not hasattr(klass, _method_name): - msger.warning("Unimplemented %s source interface for: %s"\ - % (_method_name, _source_name)) - return None - func = getattr(klass, _method_name) - methods[_method_name] = func - return_methods = methods - return return_methods - -pluginmgr = PluginMgr() diff --git a/import-layers/yocto-poky/scripts/lib/wic/pluginbase.py b/import-layers/yocto-poky/scripts/lib/wic/pluginbase.py index e737dee7b..fb3d179c2 100644 --- a/import-layers/yocto-poky/scripts/lib/wic/pluginbase.py +++ b/import-layers/yocto-poky/scripts/lib/wic/pluginbase.py @@ -15,26 +15,74 @@ # with this program; if not, write to the Free Software Foundation, Inc., 59 # Temple Place - Suite 330, Boston, MA 02111-1307, USA. -__all__ = ['ImagerPlugin', 'SourcePlugin', 'get_plugins'] +__all__ = ['ImagerPlugin', 'SourcePlugin'] + +import os +import logging -import sys from collections import defaultdict +from importlib.machinery import SourceFileLoader + +from wic import WicError +from wic.utils.misc import get_bitbake_var + +PLUGIN_TYPES = ["imager", "source"] + +SCRIPTS_PLUGIN_DIR = "scripts/lib/wic/plugins" + +logger = logging.getLogger('wic') + +PLUGINS = defaultdict(dict) -from wic import msger +class PluginMgr: + _plugin_dirs = [] + + @classmethod + def get_plugins(cls, ptype): + """Get dictionary of <plugin_name>:<class> pairs.""" + if ptype not in PLUGIN_TYPES: + raise WicError('%s is not valid plugin type' % ptype) + + # collect plugin directories + if not cls._plugin_dirs: + cls._plugin_dirs = [os.path.join(os.path.dirname(__file__), 'plugins')] + layers = get_bitbake_var("BBLAYERS") or '' + for layer_path in layers.split(): + path = os.path.join(layer_path, SCRIPTS_PLUGIN_DIR) + path = os.path.abspath(os.path.expanduser(path)) + if path not in cls._plugin_dirs and os.path.isdir(path): + cls._plugin_dirs.insert(0, path) + + if ptype not in PLUGINS: + # load all ptype plugins + for pdir in cls._plugin_dirs: + ppath = os.path.join(pdir, ptype) + if os.path.isdir(ppath): + for fname in os.listdir(ppath): + if fname.endswith('.py'): + mname = fname[:-3] + mpath = os.path.join(ppath, fname) + logger.debug("loading plugin module %s", mpath) + SourceFileLoader(mname, mpath).load_module() + + return PLUGINS.get(ptype) class PluginMeta(type): - plugins = defaultdict(dict) def __new__(cls, name, bases, attrs): class_type = type.__new__(cls, name, bases, attrs) if 'name' in attrs: - cls.plugins[class_type.wic_plugin_type][attrs['name']] = class_type + PLUGINS[class_type.wic_plugin_type][attrs['name']] = class_type return class_type -class ImagerPlugin(PluginMeta("Plugin", (), {})): +class ImagerPlugin(metaclass=PluginMeta): wic_plugin_type = "imager" -class SourcePlugin(PluginMeta("Plugin", (), {})): + def do_create(self): + raise WicError("Method %s.do_create is not implemented" % + self.__class__.__name__) + +class SourcePlugin(metaclass=PluginMeta): wic_plugin_type = "source" """ The methods that can be implemented by --source plugins. @@ -50,7 +98,7 @@ class SourcePlugin(PluginMeta("Plugin", (), {})): disk image. This provides a hook to allow finalization of a disk image e.g. to write an MBR to it. """ - msger.debug("SourcePlugin: do_install_disk: disk: %s" % disk_name) + logger.debug("SourcePlugin: do_install_disk: disk: %s", disk_name) @classmethod def do_stage_partition(cls, part, source_params, creator, cr_workdir, @@ -67,7 +115,7 @@ class SourcePlugin(PluginMeta("Plugin", (), {})): Not that get_bitbake_var() allows you to acces non-standard variables that you might want to use for this. """ - msger.debug("SourcePlugin: do_stage_partition: part: %s" % part) + logger.debug("SourcePlugin: do_stage_partition: part: %s", part) @classmethod def do_configure_partition(cls, part, source_params, creator, cr_workdir, @@ -78,7 +126,7 @@ class SourcePlugin(PluginMeta("Plugin", (), {})): custom configuration files for a partition, for example syslinux or grub config files. """ - msger.debug("SourcePlugin: do_configure_partition: part: %s" % part) + logger.debug("SourcePlugin: do_configure_partition: part: %s", part) @classmethod def do_prepare_partition(cls, part, source_params, creator, cr_workdir, @@ -88,7 +136,5 @@ class SourcePlugin(PluginMeta("Plugin", (), {})): Called to do the actual content population for a partition i.e. it 'prepares' the partition to be incorporated into the image. """ - msger.debug("SourcePlugin: do_prepare_partition: part: %s" % part) + logger.debug("SourcePlugin: do_prepare_partition: part: %s", part) -def get_plugins(typen): - return PluginMeta.plugins.get(typen) diff --git a/import-layers/yocto-poky/scripts/lib/wic/plugins/imager/direct.py b/import-layers/yocto-poky/scripts/lib/wic/plugins/imager/direct.py new file mode 100644 index 000000000..f2e612733 --- /dev/null +++ b/import-layers/yocto-poky/scripts/lib/wic/plugins/imager/direct.py @@ -0,0 +1,561 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (c) 2013, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# DESCRIPTION +# This implements the 'direct' imager plugin class for 'wic' +# +# AUTHORS +# Tom Zanussi <tom.zanussi (at] linux.intel.com> +# + +import logging +import os +import shutil +import tempfile +import uuid + +from time import strftime + +from wic import WicError +from wic.filemap import sparse_copy +from wic.ksparser import KickStart, KickStartError +from wic.pluginbase import PluginMgr, ImagerPlugin +from wic.utils.misc import get_bitbake_var, exec_cmd, exec_native_cmd + +logger = logging.getLogger('wic') + +class DirectPlugin(ImagerPlugin): + """ + Install a system into a file containing a partitioned disk image. + + An image file is formatted with a partition table, each partition + created from a rootfs or other OpenEmbedded build artifact and dd'ed + into the virtual disk. The disk image can subsequently be dd'ed onto + media and used on actual hardware. + """ + name = 'direct' + + def __init__(self, wks_file, rootfs_dir, bootimg_dir, kernel_dir, + native_sysroot, oe_builddir, options): + try: + self.ks = KickStart(wks_file) + except KickStartError as err: + raise WicError(str(err)) + + # parse possible 'rootfs=name' items + self.rootfs_dir = dict(rdir.split('=') for rdir in rootfs_dir.split(' ')) + self.bootimg_dir = bootimg_dir + self.kernel_dir = kernel_dir + self.native_sysroot = native_sysroot + self.oe_builddir = oe_builddir + + self.outdir = options.outdir + self.compressor = options.compressor + self.bmap = options.bmap + + self.name = "%s-%s" % (os.path.splitext(os.path.basename(wks_file))[0], + strftime("%Y%m%d%H%M")) + self.workdir = tempfile.mkdtemp(dir=self.outdir, prefix='tmp.wic.') + self._image = None + self.ptable_format = self.ks.bootloader.ptable + self.parts = self.ks.partitions + + # as a convenience, set source to the boot partition source + # instead of forcing it to be set via bootloader --source + for part in self.parts: + if not self.ks.bootloader.source and part.mountpoint == "/boot": + self.ks.bootloader.source = part.source + break + + image_path = self._full_path(self.workdir, self.parts[0].disk, "direct") + self._image = PartitionedImage(image_path, self.ptable_format, + self.parts, self.native_sysroot) + + def do_create(self): + """ + Plugin entry point. + """ + try: + self.create() + self.assemble() + self.finalize() + self.print_info() + finally: + self.cleanup() + + def _write_fstab(self, image_rootfs): + """overriden to generate fstab (temporarily) in rootfs. This is called + from _create, make sure it doesn't get called from + BaseImage.create() + """ + if not image_rootfs: + return + + fstab_path = image_rootfs + "/etc/fstab" + if not os.path.isfile(fstab_path): + return + + with open(fstab_path) as fstab: + fstab_lines = fstab.readlines() + + if self._update_fstab(fstab_lines, self.parts): + shutil.copyfile(fstab_path, fstab_path + ".orig") + + with open(fstab_path, "w") as fstab: + fstab.writelines(fstab_lines) + + return fstab_path + + def _update_fstab(self, fstab_lines, parts): + """Assume partition order same as in wks""" + updated = False + for part in parts: + if not part.realnum or not part.mountpoint \ + or part.mountpoint in ("/", "/boot"): + continue + + # mmc device partitions are named mmcblk0p1, mmcblk0p2.. + prefix = 'p' if part.disk.startswith('mmcblk') else '' + device_name = "/dev/%s%s%d" % (part.disk, prefix, part.realnum) + + opts = part.fsopts if part.fsopts else "defaults" + line = "\t".join([device_name, part.mountpoint, part.fstype, + opts, "0", "0"]) + "\n" + + fstab_lines.append(line) + updated = True + + return updated + + def _full_path(self, path, name, extention): + """ Construct full file path to a file we generate. """ + return os.path.join(path, "%s-%s.%s" % (self.name, name, extention)) + + # + # Actual implemention + # + def create(self): + """ + For 'wic', we already have our build artifacts - we just create + filesystems from the artifacts directly and combine them into + a partitioned image. + """ + fstab_path = self._write_fstab(self.rootfs_dir.get("ROOTFS_DIR")) + + for part in self.parts: + # get rootfs size from bitbake variable if it's not set in .ks file + if not part.size: + # and if rootfs name is specified for the partition + image_name = self.rootfs_dir.get(part.rootfs_dir) + if image_name and os.path.sep not in image_name: + # Bitbake variable ROOTFS_SIZE is calculated in + # Image._get_rootfs_size method from meta/lib/oe/image.py + # using IMAGE_ROOTFS_SIZE, IMAGE_ROOTFS_ALIGNMENT, + # IMAGE_OVERHEAD_FACTOR and IMAGE_ROOTFS_EXTRA_SPACE + rsize_bb = get_bitbake_var('ROOTFS_SIZE', image_name) + if rsize_bb: + part.size = int(round(float(rsize_bb))) + + self._image.prepare(self) + + if fstab_path: + shutil.move(fstab_path + ".orig", fstab_path) + + self._image.layout_partitions() + self._image.create() + + def assemble(self): + """ + Assemble partitions into disk image + """ + self._image.assemble() + + def finalize(self): + """ + Finalize the disk image. + + For example, prepare the image to be bootable by e.g. + creating and installing a bootloader configuration. + """ + source_plugin = self.ks.bootloader.source + disk_name = self.parts[0].disk + if source_plugin: + plugin = PluginMgr.get_plugins('source')[source_plugin] + plugin.do_install_disk(self._image, disk_name, self, self.workdir, + self.oe_builddir, self.bootimg_dir, + self.kernel_dir, self.native_sysroot) + + full_path = self._image.path + # Generate .bmap + if self.bmap: + logger.debug("Generating bmap file for %s", disk_name) + exec_native_cmd("bmaptool create %s -o %s.bmap" % (full_path, full_path), + self.native_sysroot) + # Compress the image + if self.compressor: + logger.debug("Compressing disk %s with %s", disk_name, self.compressor) + exec_cmd("%s %s" % (self.compressor, full_path)) + + def print_info(self): + """ + Print the image(s) and artifacts used, for the user. + """ + msg = "The new image(s) can be found here:\n" + + extension = "direct" + {"gzip": ".gz", + "bzip2": ".bz2", + "xz": ".xz", + None: ""}.get(self.compressor) + full_path = self._full_path(self.outdir, self.parts[0].disk, extension) + msg += ' %s\n\n' % full_path + + msg += 'The following build artifacts were used to create the image(s):\n' + for part in self.parts: + if part.rootfs_dir is None: + continue + if part.mountpoint == '/': + suffix = ':' + else: + suffix = '["%s"]:' % (part.mountpoint or part.label) + msg += ' ROOTFS_DIR%s%s\n' % (suffix.ljust(20), part.rootfs_dir) + + msg += ' BOOTIMG_DIR: %s\n' % self.bootimg_dir + msg += ' KERNEL_DIR: %s\n' % self.kernel_dir + msg += ' NATIVE_SYSROOT: %s\n' % self.native_sysroot + + logger.info(msg) + + @property + def rootdev(self): + """ + Get root device name to use as a 'root' parameter + in kernel command line. + + Assume partition order same as in wks + """ + for part in self.parts: + if part.mountpoint == "/": + if part.uuid: + return "PARTUUID=%s" % part.uuid + else: + suffix = 'p' if part.disk.startswith('mmcblk') else '' + return "/dev/%s%s%-d" % (part.disk, suffix, part.realnum) + + def cleanup(self): + if self._image: + self._image.cleanup() + + # Move results to the output dir + if not os.path.exists(self.outdir): + os.makedirs(self.outdir) + + for fname in os.listdir(self.workdir): + path = os.path.join(self.workdir, fname) + if os.path.isfile(path): + shutil.move(path, os.path.join(self.outdir, fname)) + + # remove work directory + shutil.rmtree(self.workdir, ignore_errors=True) + +# Overhead of the MBR partitioning scheme (just one sector) +MBR_OVERHEAD = 1 + +# Overhead of the GPT partitioning scheme +GPT_OVERHEAD = 34 + +# Size of a sector in bytes +SECTOR_SIZE = 512 + +class PartitionedImage(): + """ + Partitioned image in a file. + """ + + def __init__(self, path, ptable_format, partitions, native_sysroot=None): + self.path = path # Path to the image file + self.numpart = 0 # Number of allocated partitions + self.realpart = 0 # Number of partitions in the partition table + self.offset = 0 # Offset of next partition (in sectors) + self.min_size = 0 # Minimum required disk size to fit + # all partitions (in bytes) + self.ptable_format = ptable_format # Partition table format + # Disk system identifier + self.identifier = int.from_bytes(os.urandom(4), 'little') + + self.partitions = partitions + self.partimages = [] + # Size of a sector used in calculations + self.sector_size = SECTOR_SIZE + self.native_sysroot = native_sysroot + + # calculate the real partition number, accounting for partitions not + # in the partition table and logical partitions + realnum = 0 + for part in self.partitions: + if part.no_table: + part.realnum = 0 + else: + realnum += 1 + if self.ptable_format == 'msdos' and realnum > 3: + part.realnum = realnum + 1 + continue + part.realnum = realnum + + # generate parition UUIDs + for part in self.partitions: + if not part.uuid and part.use_uuid: + if self.ptable_format == 'gpt': + part.uuid = str(uuid.uuid4()) + else: # msdos partition table + part.uuid = '%08x-%02d' % (self.identifier, part.realnum) + + def prepare(self, imager): + """Prepare an image. Call prepare method of all image partitions.""" + for part in self.partitions: + # need to create the filesystems in order to get their + # sizes before we can add them and do the layout. + part.prepare(imager, imager.workdir, imager.oe_builddir, + imager.rootfs_dir, imager.bootimg_dir, + imager.kernel_dir, imager.native_sysroot) + + # Converting kB to sectors for parted + part.size_sec = part.disk_size * 1024 // self.sector_size + + def layout_partitions(self): + """ Layout the partitions, meaning calculate the position of every + partition on the disk. The 'ptable_format' parameter defines the + partition table format and may be "msdos". """ + + logger.debug("Assigning %s partitions to disks", self.ptable_format) + + # The number of primary and logical partitions. Extended partition and + # partitions not listed in the table are not included. + num_real_partitions = len([p for p in self.partitions if not p.no_table]) + + # Go through partitions in the order they are added in .ks file + for num in range(len(self.partitions)): + part = self.partitions[num] + + if self.ptable_format == 'msdos' and part.part_type: + # The --part-type can also be implemented for MBR partitions, + # in which case it would map to the 1-byte "partition type" + # filed at offset 3 of the partition entry. + raise WicError("setting custom partition type is not " \ + "implemented for msdos partitions") + + # Get the disk where the partition is located + self.numpart += 1 + if not part.no_table: + self.realpart += 1 + + if self.numpart == 1: + if self.ptable_format == "msdos": + overhead = MBR_OVERHEAD + elif self.ptable_format == "gpt": + overhead = GPT_OVERHEAD + + # Skip one sector required for the partitioning scheme overhead + self.offset += overhead + + if self.realpart > 3 and num_real_partitions > 4: + # Reserve a sector for EBR for every logical partition + # before alignment is performed. + if self.ptable_format == "msdos": + self.offset += 1 + + if part.align: + # If not first partition and we do have alignment set we need + # to align the partition. + # FIXME: This leaves a empty spaces to the disk. To fill the + # gaps we could enlargea the previous partition? + + # Calc how much the alignment is off. + align_sectors = self.offset % (part.align * 1024 // self.sector_size) + + if align_sectors: + # If partition is not aligned as required, we need + # to move forward to the next alignment point + align_sectors = (part.align * 1024 // self.sector_size) - align_sectors + + logger.debug("Realignment for %s%s with %s sectors, original" + " offset %s, target alignment is %sK.", + part.disk, self.numpart, align_sectors, + self.offset, part.align) + + # increase the offset so we actually start the partition on right alignment + self.offset += align_sectors + + part.start = self.offset + self.offset += part.size_sec + + part.type = 'primary' + if not part.no_table: + part.num = self.realpart + else: + part.num = 0 + + if self.ptable_format == "msdos": + # only count the partitions that are in partition table + if num_real_partitions > 4: + if self.realpart > 3: + part.type = 'logical' + part.num = self.realpart + 1 + + logger.debug("Assigned %s to %s%d, sectors range %d-%d size %d " + "sectors (%d bytes).", part.mountpoint, part.disk, + part.num, part.start, self.offset - 1, part.size_sec, + part.size_sec * self.sector_size) + + # Once all the partitions have been layed out, we can calculate the + # minumim disk size + self.min_size = self.offset + if self.ptable_format == "gpt": + self.min_size += GPT_OVERHEAD + + self.min_size *= self.sector_size + + def _create_partition(self, device, parttype, fstype, start, size): + """ Create a partition on an image described by the 'device' object. """ + + # Start is included to the size so we need to substract one from the end. + end = start + size - 1 + logger.debug("Added '%s' partition, sectors %d-%d, size %d sectors", + parttype, start, end, size) + + cmd = "parted -s %s unit s mkpart %s" % (device, parttype) + if fstype: + cmd += " %s" % fstype + cmd += " %d %d" % (start, end) + + return exec_native_cmd(cmd, self.native_sysroot) + + def create(self): + logger.debug("Creating sparse file %s", self.path) + with open(self.path, 'w') as sparse: + os.ftruncate(sparse.fileno(), self.min_size) + + logger.debug("Initializing partition table for %s", self.path) + exec_native_cmd("parted -s %s mklabel %s" % + (self.path, self.ptable_format), self.native_sysroot) + + logger.debug("Set disk identifier %x", self.identifier) + with open(self.path, 'r+b') as img: + img.seek(0x1B8) + img.write(self.identifier.to_bytes(4, 'little')) + + logger.debug("Creating partitions") + + for part in self.partitions: + if part.num == 0: + continue + + if self.ptable_format == "msdos" and part.num == 5: + # Create an extended partition (note: extended + # partition is described in MBR and contains all + # logical partitions). The logical partitions save a + # sector for an EBR just before the start of a + # partition. The extended partition must start one + # sector before the start of the first logical + # partition. This way the first EBR is inside of the + # extended partition. Since the extended partitions + # starts a sector before the first logical partition, + # add a sector at the back, so that there is enough + # room for all logical partitions. + self._create_partition(self.path, "extended", + None, part.start - 1, + self.offset - part.start + 1) + + if part.fstype == "swap": + parted_fs_type = "linux-swap" + elif part.fstype == "vfat": + parted_fs_type = "fat32" + elif part.fstype == "msdos": + parted_fs_type = "fat16" + if not part.system_id: + part.system_id = '0x6' # FAT16 + else: + # Type for ext2/ext3/ext4/btrfs + parted_fs_type = "ext2" + + # Boot ROM of OMAP boards require vfat boot partition to have an + # even number of sectors. + if part.mountpoint == "/boot" and part.fstype in ["vfat", "msdos"] \ + and part.size_sec % 2: + logger.debug("Subtracting one sector from '%s' partition to " + "get even number of sectors for the partition", + part.mountpoint) + part.size_sec -= 1 + + self._create_partition(self.path, part.type, + parted_fs_type, part.start, part.size_sec) + + if part.part_type: + logger.debug("partition %d: set type UID to %s", + part.num, part.part_type) + exec_native_cmd("sgdisk --typecode=%d:%s %s" % \ + (part.num, part.part_type, + self.path), self.native_sysroot) + + if part.uuid and self.ptable_format == "gpt": + logger.debug("partition %d: set UUID to %s", + part.num, part.uuid) + exec_native_cmd("sgdisk --partition-guid=%d:%s %s" % \ + (part.num, part.uuid, self.path), + self.native_sysroot) + + if part.label and self.ptable_format == "gpt": + logger.debug("partition %d: set name to %s", + part.num, part.label) + exec_native_cmd("parted -s %s name %d %s" % \ + (self.path, part.num, part.label), + self.native_sysroot) + + if part.active: + flag_name = "legacy_boot" if self.ptable_format == 'gpt' else "boot" + logger.debug("Set '%s' flag for partition '%s' on disk '%s'", + flag_name, part.num, self.path) + exec_native_cmd("parted -s %s set %d %s on" % \ + (self.path, part.num, flag_name), + self.native_sysroot) + if part.system_id: + exec_native_cmd("sfdisk --part-type %s %s %s" % \ + (self.path, part.num, part.system_id), + self.native_sysroot) + + def cleanup(self): + # remove partition images + for image in set(self.partimages): + os.remove(image) + + def assemble(self): + logger.debug("Installing partitions") + + for part in self.partitions: + source = part.source_file + if source: + # install source_file contents into a partition + sparse_copy(source, self.path, part.start * self.sector_size) + + logger.debug("Installed %s in partition %d, sectors %d-%d, " + "size %d sectors", source, part.num, part.start, + part.start + part.size_sec - 1, part.size_sec) + + partimage = self.path + '.p%d' % part.num + os.rename(source, partimage) + self.partimages.append(partimage) diff --git a/import-layers/yocto-poky/scripts/lib/wic/plugins/imager/direct_plugin.py b/import-layers/yocto-poky/scripts/lib/wic/plugins/imager/direct_plugin.py deleted file mode 100644 index 8fe393080..000000000 --- a/import-layers/yocto-poky/scripts/lib/wic/plugins/imager/direct_plugin.py +++ /dev/null @@ -1,103 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -# -# Copyright (c) 2013, Intel Corporation. -# All rights reserved. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# DESCRIPTION -# This implements the 'direct' imager plugin class for 'wic' -# -# AUTHORS -# Tom Zanussi <tom.zanussi (at] linux.intel.com> -# - -from wic.utils import errors -from wic.conf import configmgr - -import wic.imager.direct as direct -from wic.pluginbase import ImagerPlugin - -class DirectPlugin(ImagerPlugin): - """ - Install a system into a file containing a partitioned disk image. - - An image file is formatted with a partition table, each partition - created from a rootfs or other OpenEmbedded build artifact and dd'ed - into the virtual disk. The disk image can subsequently be dd'ed onto - media and used on actual hardware. - """ - - name = 'direct' - - @classmethod - def __rootfs_dir_to_dict(cls, rootfs_dirs): - """ - Gets a string that contain 'connection=dir' splitted by - space and return a dict - """ - krootfs_dir = {} - for rootfs_dir in rootfs_dirs.split(' '): - key, val = rootfs_dir.split('=') - krootfs_dir[key] = val - - return krootfs_dir - - @classmethod - def do_create(cls, opts, *args): - """ - Create direct image, called from creator as 'direct' cmd - """ - if len(args) != 8: - raise errors.Usage("Extra arguments given") - - native_sysroot = args[0] - kernel_dir = args[1] - bootimg_dir = args[2] - rootfs_dir = args[3] - - creatoropts = configmgr.create - ksconf = args[4] - - image_output_dir = args[5] - oe_builddir = args[6] - compressor = args[7] - - krootfs_dir = cls.__rootfs_dir_to_dict(rootfs_dir) - - configmgr._ksconf = ksconf - - creator = direct.DirectImageCreator(oe_builddir, - image_output_dir, - krootfs_dir, - bootimg_dir, - kernel_dir, - native_sysroot, - compressor, - creatoropts, - opts.bmap) - - try: - creator.create() - creator.assemble() - creator.finalize() - creator.print_outimage_info() - - except errors.CreatorError: - raise - finally: - creator.cleanup() - - return 0 diff --git a/import-layers/yocto-poky/scripts/lib/wic/plugins/source/bootimg-efi.py b/import-layers/yocto-poky/scripts/lib/wic/plugins/source/bootimg-efi.py index 4adb80bec..9879cb9fc 100644 --- a/import-layers/yocto-poky/scripts/lib/wic/plugins/source/bootimg-efi.py +++ b/import-layers/yocto-poky/scripts/lib/wic/plugins/source/bootimg-efi.py @@ -24,25 +24,28 @@ # Tom Zanussi <tom.zanussi (at] linux.intel.com> # +import logging import os import shutil -from wic import msger +from wic import WicError +from wic.engine import get_custom_config from wic.pluginbase import SourcePlugin -from wic.utils.misc import get_custom_config -from wic.utils.oe.misc import exec_cmd, exec_native_cmd, get_bitbake_var, \ - BOOTDD_EXTRA_SPACE +from wic.utils.misc import (exec_cmd, exec_native_cmd, get_bitbake_var, + BOOTDD_EXTRA_SPACE) + +logger = logging.getLogger('wic') class BootimgEFIPlugin(SourcePlugin): """ Create EFI boot partition. - This plugin supports GRUB 2 and gummiboot bootloaders. + This plugin supports GRUB 2 and systemd-boot bootloaders. """ name = 'bootimg-efi' @classmethod - def do_configure_grubefi(cls, hdddir, creator, cr_workdir): + def do_configure_grubefi(cls, creator, cr_workdir): """ Create loader-specific (grub-efi) config """ @@ -53,11 +56,11 @@ class BootimgEFIPlugin(SourcePlugin): if custom_cfg: # Use a custom configuration for grub grubefi_conf = custom_cfg - msger.debug("Using custom configuration file " - "%s for grub.cfg" % configfile) + logger.debug("Using custom configuration file " + "%s for grub.cfg", configfile) else: - msger.error("configfile is specified but failed to " - "get it from %s." % configfile) + raise WicError("configfile is specified but failed to " + "get it from %s." % configfile) if not custom_cfg: # Create grub configuration using parameters from wks file @@ -75,14 +78,14 @@ class BootimgEFIPlugin(SourcePlugin): % (kernel, creator.rootdev, bootloader.append) grubefi_conf += "}\n" - msger.debug("Writing grubefi config %s/hdd/boot/EFI/BOOT/grub.cfg" \ - % cr_workdir) + logger.debug("Writing grubefi config %s/hdd/boot/EFI/BOOT/grub.cfg", + cr_workdir) cfg = open("%s/hdd/boot/EFI/BOOT/grub.cfg" % cr_workdir, "w") cfg.write(grubefi_conf) cfg.close() @classmethod - def do_configure_gummiboot(cls, hdddir, creator, cr_workdir): + def do_configure_systemdboot(cls, hdddir, creator, cr_workdir, source_params): """ Create loader-specific systemd-boot/gummiboot config """ @@ -98,8 +101,21 @@ class BootimgEFIPlugin(SourcePlugin): loader_conf += "default boot\n" loader_conf += "timeout %d\n" % bootloader.timeout - msger.debug("Writing gummiboot config %s/hdd/boot/loader/loader.conf" \ - % cr_workdir) + initrd = source_params.get('initrd') + + if initrd: + # obviously we need to have a common common deploy var + bootimg_dir = get_bitbake_var("DEPLOY_DIR_IMAGE") + if not bootimg_dir: + raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting") + + cp_cmd = "cp %s/%s %s" % (bootimg_dir, initrd, hdddir) + exec_cmd(cp_cmd, True) + else: + logger.debug("Ignoring missing initrd") + + logger.debug("Writing systemd-boot config " + "%s/hdd/boot/loader/loader.conf", cr_workdir) cfg = open("%s/hdd/boot/loader/loader.conf" % cr_workdir, "w") cfg.write(loader_conf) cfg.close() @@ -109,16 +125,16 @@ class BootimgEFIPlugin(SourcePlugin): if configfile: custom_cfg = get_custom_config(configfile) if custom_cfg: - # Use a custom configuration for gummiboot + # Use a custom configuration for systemd-boot boot_conf = custom_cfg - msger.debug("Using custom configuration file " - "%s for gummiboots's boot.conf" % configfile) + logger.debug("Using custom configuration file " + "%s for systemd-boots's boot.conf", configfile) else: - msger.error("configfile is specified but failed to " - "get it from %s." % configfile) + raise WicError("configfile is specified but failed to " + "get it from %s.", configfile) if not custom_cfg: - # Create gummiboot configuration using parameters from wks file + # Create systemd-boot configuration using parameters from wks file kernel = "/bzImage" boot_conf = "" @@ -127,8 +143,11 @@ class BootimgEFIPlugin(SourcePlugin): boot_conf += "options LABEL=Boot root=%s %s\n" % \ (creator.rootdev, bootloader.append) - msger.debug("Writing gummiboot config %s/hdd/boot/loader/entries/boot.conf" \ - % cr_workdir) + if initrd: + boot_conf += "initrd /%s\n" % initrd + + logger.debug("Writing systemd-boot config " + "%s/hdd/boot/loader/entries/boot.conf", cr_workdir) cfg = open("%s/hdd/boot/loader/entries/boot.conf" % cr_workdir, "w") cfg.write(boot_conf) cfg.close() @@ -148,14 +167,13 @@ class BootimgEFIPlugin(SourcePlugin): try: if source_params['loader'] == 'grub-efi': - cls.do_configure_grubefi(hdddir, creator, cr_workdir) - elif source_params['loader'] == 'gummiboot' \ - or source_params['loader'] == 'systemd-boot': - cls.do_configure_gummiboot(hdddir, creator, cr_workdir) + cls.do_configure_grubefi(creator, cr_workdir) + elif source_params['loader'] == 'systemd-boot': + cls.do_configure_systemdboot(hdddir, creator, cr_workdir, source_params) else: - msger.error("unrecognized bootimg-efi loader: %s" % source_params['loader']) + raise WicError("unrecognized bootimg-efi loader: %s" % source_params['loader']) except KeyError: - msger.error("bootimg-efi requires a loader, none specified") + raise WicError("bootimg-efi requires a loader, none specified") @classmethod @@ -167,12 +185,10 @@ class BootimgEFIPlugin(SourcePlugin): 'prepares' the partition to be incorporated into the image. In this case, prepare content for an EFI (grub) boot partition. """ - if not bootimg_dir: - bootimg_dir = get_bitbake_var("HDDDIR") - if not bootimg_dir: - msger.error("Couldn't find HDDDIR, exiting\n") - # just so the result notes display it - creator.set_bootimg_dir(bootimg_dir) + if not kernel_dir: + kernel_dir = get_bitbake_var("DEPLOY_DIR_IMAGE") + if not kernel_dir: + raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting") staging_kernel_dir = kernel_dir @@ -182,24 +198,27 @@ class BootimgEFIPlugin(SourcePlugin): (staging_kernel_dir, hdddir) exec_cmd(install_cmd) + try: if source_params['loader'] == 'grub-efi': shutil.copyfile("%s/hdd/boot/EFI/BOOT/grub.cfg" % cr_workdir, "%s/grub.cfg" % cr_workdir) - cp_cmd = "cp %s/EFI/BOOT/* %s/EFI/BOOT" % (bootimg_dir, hdddir) - exec_cmd(cp_cmd, True) + for mod in [x for x in os.listdir(kernel_dir) if x.startswith("grub-efi-")]: + cp_cmd = "cp %s/%s %s/EFI/BOOT/%s" % (kernel_dir, mod, hdddir, mod[9:]) + exec_cmd(cp_cmd, True) shutil.move("%s/grub.cfg" % cr_workdir, "%s/hdd/boot/EFI/BOOT/grub.cfg" % cr_workdir) - elif source_params['loader'] == 'gummiboot' \ - or source_params['loader'] == 'systemd-boot': - cp_cmd = "cp %s/EFI/BOOT/* %s/EFI/BOOT" % (bootimg_dir, hdddir) - exec_cmd(cp_cmd, True) + elif source_params['loader'] == 'systemd-boot': + for mod in [x for x in os.listdir(kernel_dir) if x.startswith("systemd-")]: + cp_cmd = "cp %s/%s %s/EFI/BOOT/%s" % (kernel_dir, mod, hdddir, mod[8:]) + exec_cmd(cp_cmd, True) else: - msger.error("unrecognized bootimg-efi loader: %s" % source_params['loader']) + raise WicError("unrecognized bootimg-efi loader: %s" % + source_params['loader']) except KeyError: - msger.error("bootimg-efi requires a loader, none specified") + raise WicError("bootimg-efi requires a loader, none specified") - startup = os.path.join(bootimg_dir, "startup.nsh") + startup = os.path.join(kernel_dir, "startup.nsh") if os.path.exists(startup): cp_cmd = "cp %s %s/" % (startup, hdddir) exec_cmd(cp_cmd, True) @@ -215,8 +234,8 @@ class BootimgEFIPlugin(SourcePlugin): blocks += extra_blocks - msger.debug("Added %d extra blocks to %s to get to %d total blocks" % \ - (extra_blocks, part.mountpoint, blocks)) + logger.debug("Added %d extra blocks to %s to get to %d total blocks", + extra_blocks, part.mountpoint, blocks) # dosfs image, created by mkdosfs bootimg = "%s/boot.img" % cr_workdir diff --git a/import-layers/yocto-poky/scripts/lib/wic/plugins/source/bootimg-partition.py b/import-layers/yocto-poky/scripts/lib/wic/plugins/source/bootimg-partition.py index b76c1211a..13fddbd47 100644 --- a/import-layers/yocto-poky/scripts/lib/wic/plugins/source/bootimg-partition.py +++ b/import-layers/yocto-poky/scripts/lib/wic/plugins/source/bootimg-partition.py @@ -23,14 +23,18 @@ # Maciej Borzecki <maciej.borzecki (at] open-rnd.pl> # +import logging import os import re -from wic import msger -from wic.pluginbase import SourcePlugin -from wic.utils.oe.misc import exec_cmd, get_bitbake_var from glob import glob +from wic import WicError +from wic.pluginbase import SourcePlugin +from wic.utils.misc import exec_cmd, get_bitbake_var + +logger = logging.getLogger('wic') + class BootimgPartitionPlugin(SourcePlugin): """ Create an image of boot partition, copying over files @@ -40,26 +44,6 @@ class BootimgPartitionPlugin(SourcePlugin): name = 'bootimg-partition' @classmethod - def do_install_disk(cls, disk, disk_name, cr, workdir, oe_builddir, - bootimg_dir, kernel_dir, native_sysroot): - """ - Called after all partitions have been prepared and assembled into a - disk image. Do nothing. - """ - pass - - @classmethod - def do_configure_partition(cls, part, source_params, cr, cr_workdir, - oe_builddir, bootimg_dir, kernel_dir, - native_sysroot): - """ - Called before do_prepare_partition(). Possibly prepare - configuration files of some sort. - - """ - pass - - @classmethod def do_prepare_partition(cls, part, source_params, cr, cr_workdir, oe_builddir, bootimg_dir, kernel_dir, rootfs_dir, native_sysroot): @@ -74,19 +58,19 @@ class BootimgPartitionPlugin(SourcePlugin): install_cmd = "install -d %s" % hdddir exec_cmd(install_cmd) - if not bootimg_dir: - bootimg_dir = get_bitbake_var("DEPLOY_DIR_IMAGE") - if not bootimg_dir: - msger.error("Couldn't find DEPLOY_DIR_IMAGE, exiting\n") + if not kernel_dir: + kernel_dir = get_bitbake_var("DEPLOY_DIR_IMAGE") + if not kernel_dir: + raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting") - msger.debug('Bootimg dir: %s' % bootimg_dir) + logger.debug('Kernel dir: %s', bootimg_dir) boot_files = get_bitbake_var("IMAGE_BOOT_FILES") if not boot_files: - msger.error('No boot files defined, IMAGE_BOOT_FILES unset') + raise WicError('No boot files defined, IMAGE_BOOT_FILES unset') - msger.debug('Boot files: %s' % boot_files) + logger.debug('Boot files: %s', boot_files) # list of tuples (src_name, dst_name) deploy_files = [] @@ -94,11 +78,11 @@ class BootimgPartitionPlugin(SourcePlugin): if ';' in src_entry: dst_entry = tuple(src_entry.split(';')) if not dst_entry[0] or not dst_entry[1]: - msger.error('Malformed boot file entry: %s' % (src_entry)) + raise WicError('Malformed boot file entry: %s' % src_entry) else: dst_entry = (src_entry, src_entry) - msger.debug('Destination entry: %r' % (dst_entry,)) + logger.debug('Destination entry: %r', dst_entry) deploy_files.append(dst_entry) for deploy_entry in deploy_files: @@ -114,27 +98,26 @@ class BootimgPartitionPlugin(SourcePlugin): os.path.join(dst, os.path.basename(name)) - srcs = glob(os.path.join(bootimg_dir, src)) + srcs = glob(os.path.join(kernel_dir, src)) - msger.debug('Globbed sources: %s' % (', '.join(srcs))) + logger.debug('Globbed sources: %s', ', '.join(srcs)) for entry in srcs: entry_dst_name = entry_name_fn(entry) install_task.append((entry, os.path.join(hdddir, entry_dst_name))) else: - install_task = [(os.path.join(bootimg_dir, src), + install_task = [(os.path.join(kernel_dir, src), os.path.join(hdddir, dst))] for task in install_task: src_path, dst_path = task - msger.debug('Install %s as %s' % (os.path.basename(src_path), - dst_path)) + logger.debug('Install %s as %s', + os.path.basename(src_path), dst_path) install_cmd = "install -m 0644 -D %s %s" \ % (src_path, dst_path) exec_cmd(install_cmd) - msger.debug('Prepare boot partition using rootfs in %s' % (hdddir)) + logger.debug('Prepare boot partition using rootfs in %s', hdddir) part.prepare_rootfs(cr_workdir, oe_builddir, hdddir, native_sysroot) - diff --git a/import-layers/yocto-poky/scripts/lib/wic/plugins/source/bootimg-pcbios.py b/import-layers/yocto-poky/scripts/lib/wic/plugins/source/bootimg-pcbios.py index f204daa32..5890c1267 100644 --- a/import-layers/yocto-poky/scripts/lib/wic/plugins/source/bootimg-pcbios.py +++ b/import-layers/yocto-poky/scripts/lib/wic/plugins/source/bootimg-pcbios.py @@ -24,15 +24,17 @@ # Tom Zanussi <tom.zanussi (at] linux.intel.com> # +import logging import os -from wic.utils.errors import ImageError -from wic import msger +from wic import WicError +from wic.engine import get_custom_config from wic.utils import runner -from wic.utils.misc import get_custom_config from wic.pluginbase import SourcePlugin -from wic.utils.oe.misc import exec_cmd, exec_native_cmd, \ - get_bitbake_var, BOOTDD_EXTRA_SPACE +from wic.utils.misc import (exec_cmd, exec_native_cmd, + get_bitbake_var, BOOTDD_EXTRA_SPACE) + +logger = logging.getLogger('wic') class BootimgPcbiosPlugin(SourcePlugin): """ @@ -42,33 +44,45 @@ class BootimgPcbiosPlugin(SourcePlugin): name = 'bootimg-pcbios' @classmethod + def _get_bootimg_dir(cls, bootimg_dir, dirname): + """ + Check if dirname exists in default bootimg_dir or + in wic-tools STAGING_DIR. + """ + for result in (bootimg_dir, get_bitbake_var("STAGING_DATADIR", "wic-tools")): + if os.path.exists("%s/%s" % (result, dirname)): + return result + + raise WicError("Couldn't find correct bootimg_dir, exiting") + + @classmethod def do_install_disk(cls, disk, disk_name, creator, workdir, oe_builddir, bootimg_dir, kernel_dir, native_sysroot): """ Called after all partitions have been prepared and assembled into a disk image. In this case, we install the MBR. """ + bootimg_dir = cls._get_bootimg_dir(bootimg_dir, 'syslinux') mbrfile = "%s/syslinux/" % bootimg_dir if creator.ptable_format == 'msdos': mbrfile += "mbr.bin" elif creator.ptable_format == 'gpt': mbrfile += "gptmbr.bin" else: - msger.error("Unsupported partition table: %s" % creator.ptable_format) + raise WicError("Unsupported partition table: %s" % + creator.ptable_format) if not os.path.exists(mbrfile): - msger.error("Couldn't find %s. If using the -e option, do you " - "have the right MACHINE set in local.conf? If not, " - "is the bootimg_dir path correct?" % mbrfile) + raise WicError("Couldn't find %s. If using the -e option, do you " + "have the right MACHINE set in local.conf? If not, " + "is the bootimg_dir path correct?" % mbrfile) full_path = creator._full_path(workdir, disk_name, "direct") - msger.debug("Installing MBR on disk %s as %s with size %s bytes" \ - % (disk_name, full_path, disk['min_size'])) + logger.debug("Installing MBR on disk %s as %s with size %s bytes", + disk_name, full_path, disk.min_size) - rcode = runner.show(['dd', 'if=%s' % mbrfile, - 'of=%s' % full_path, 'conv=notrunc']) - if rcode != 0: - raise ImageError("Unable to set MBR to %s" % full_path) + dd_cmd = "dd if=%s of=%s conv=notrunc" % (mbrfile, full_path) + exec_cmd(dd_cmd, native_sysroot) @classmethod def do_configure_partition(cls, part, source_params, creator, cr_workdir, @@ -90,11 +104,11 @@ class BootimgPcbiosPlugin(SourcePlugin): if custom_cfg: # Use a custom configuration for grub syslinux_conf = custom_cfg - msger.debug("Using custom configuration file " - "%s for syslinux.cfg" % bootloader.configfile) + logger.debug("Using custom configuration file %s " + "for syslinux.cfg", bootloader.configfile) else: - msger.error("configfile is specified but failed to " - "get it from %s." % bootloader.configfile) + raise WicError("configfile is specified but failed to " + "get it from %s." % bootloader.configfile) if not custom_cfg: # Create syslinux configuration using parameters from wks file @@ -122,8 +136,8 @@ class BootimgPcbiosPlugin(SourcePlugin): syslinux_conf += "APPEND label=boot root=%s %s\n" % \ (creator.rootdev, bootloader.append) - msger.debug("Writing syslinux config %s/hdd/boot/syslinux.cfg" \ - % cr_workdir) + logger.debug("Writing syslinux config %s/hdd/boot/syslinux.cfg", + cr_workdir) cfg = open("%s/hdd/boot/syslinux.cfg" % cr_workdir, "w") cfg.write(syslinux_conf) cfg.close() @@ -137,33 +151,25 @@ class BootimgPcbiosPlugin(SourcePlugin): 'prepares' the partition to be incorporated into the image. In this case, prepare content for legacy bios boot partition. """ - def _has_syslinux(dirname): - if dirname: - syslinux = "%s/syslinux" % dirname - if os.path.exists(syslinux): - return True - return False - - if not _has_syslinux(bootimg_dir): - bootimg_dir = get_bitbake_var("STAGING_DATADIR") - if not bootimg_dir: - msger.error("Couldn't find STAGING_DATADIR, exiting\n") - if not _has_syslinux(bootimg_dir): - msger.error("Please build syslinux first\n") - # just so the result notes display it - creator.set_bootimg_dir(bootimg_dir) + bootimg_dir = cls._get_bootimg_dir(bootimg_dir, 'syslinux') staging_kernel_dir = kernel_dir hdddir = "%s/hdd/boot" % cr_workdir - install_cmd = "install -m 0644 %s/bzImage %s/vmlinuz" \ - % (staging_kernel_dir, hdddir) - exec_cmd(install_cmd) + cmds = ("install -m 0644 %s/bzImage %s/vmlinuz" % + (staging_kernel_dir, hdddir), + "install -m 444 %s/syslinux/ldlinux.sys %s/ldlinux.sys" % + (bootimg_dir, hdddir), + "install -m 0644 %s/syslinux/vesamenu.c32 %s/vesamenu.c32" % + (bootimg_dir, hdddir), + "install -m 444 %s/syslinux/libcom32.c32 %s/libcom32.c32" % + (bootimg_dir, hdddir), + "install -m 444 %s/syslinux/libutil.c32 %s/libutil.c32" % + (bootimg_dir, hdddir)) - install_cmd = "install -m 444 %s/syslinux/ldlinux.sys %s/ldlinux.sys" \ - % (bootimg_dir, hdddir) - exec_cmd(install_cmd) + for install_cmd in cmds: + exec_cmd(install_cmd) du_cmd = "du -bks %s" % hdddir out = exec_cmd(du_cmd) @@ -176,8 +182,8 @@ class BootimgPcbiosPlugin(SourcePlugin): blocks += extra_blocks - msger.debug("Added %d extra blocks to %s to get to %d total blocks" % \ - (extra_blocks, part.mountpoint, blocks)) + logger.debug("Added %d extra blocks to %s to get to %d total blocks", + extra_blocks, part.mountpoint, blocks) # dosfs image, created by mkdosfs bootimg = "%s/boot.img" % cr_workdir @@ -198,7 +204,5 @@ class BootimgPcbiosPlugin(SourcePlugin): out = exec_cmd(du_cmd) bootimg_size = out.split()[0] - part.size = int(out.split()[0]) + part.size = int(bootimg_size) part.source_file = bootimg - - diff --git a/import-layers/yocto-poky/scripts/lib/wic/plugins/source/fsimage.py b/import-layers/yocto-poky/scripts/lib/wic/plugins/source/fsimage.py deleted file mode 100644 index f894e8936..000000000 --- a/import-layers/yocto-poky/scripts/lib/wic/plugins/source/fsimage.py +++ /dev/null @@ -1,73 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# - -import os - -from wic import msger -from wic.pluginbase import SourcePlugin -from wic.utils.oe.misc import get_bitbake_var - -class FSImagePlugin(SourcePlugin): - """ - Add an already existing filesystem image to the partition layout. - """ - - name = 'fsimage' - - @classmethod - def do_install_disk(cls, disk, disk_name, cr, workdir, oe_builddir, - bootimg_dir, kernel_dir, native_sysroot): - """ - Called after all partitions have been prepared and assembled into a - disk image. Do nothing. - """ - pass - - @classmethod - def do_configure_partition(cls, part, source_params, cr, cr_workdir, - oe_builddir, bootimg_dir, kernel_dir, - native_sysroot): - """ - Called before do_prepare_partition(). Possibly prepare - configuration files of some sort. - """ - pass - - @classmethod - def do_prepare_partition(cls, part, source_params, cr, cr_workdir, - oe_builddir, bootimg_dir, kernel_dir, - rootfs_dir, native_sysroot): - """ - Called to do the actual content population for a partition i.e. it - 'prepares' the partition to be incorporated into the image. - """ - if not bootimg_dir: - bootimg_dir = get_bitbake_var("DEPLOY_DIR_IMAGE") - if not bootimg_dir: - msger.error("Couldn't find DEPLOY_DIR_IMAGE, exiting\n") - - msger.debug('Bootimg dir: %s' % bootimg_dir) - - if 'file' not in source_params: - msger.error("No file specified\n") - return - - src = os.path.join(bootimg_dir, source_params['file']) - - - msger.debug('Preparing partition using image %s' % (src)) - part.prepare_rootfs_from_fs_image(cr_workdir, src, "") diff --git a/import-layers/yocto-poky/scripts/lib/wic/plugins/source/isoimage-isohybrid.py b/import-layers/yocto-poky/scripts/lib/wic/plugins/source/isoimage-isohybrid.py index 3858fd439..1ceba62be 100644 --- a/import-layers/yocto-poky/scripts/lib/wic/plugins/source/isoimage-isohybrid.py +++ b/import-layers/yocto-poky/scripts/lib/wic/plugins/source/isoimage-isohybrid.py @@ -20,15 +20,18 @@ # AUTHORS # Mihaly Varga <mihaly.varga (at] ni.com> +import glob +import logging import os import re import shutil -import glob -from wic import msger +from wic import WicError +from wic.engine import get_custom_config from wic.pluginbase import SourcePlugin -from wic.utils.misc import get_custom_config -from wic.utils.oe.misc import exec_cmd, exec_native_cmd, get_bitbake_var +from wic.utils.misc import exec_cmd, exec_native_cmd, get_bitbake_var + +logger = logging.getLogger('wic') class IsoImagePlugin(SourcePlugin): """ @@ -85,8 +88,9 @@ class IsoImagePlugin(SourcePlugin): syslinux_conf += "APPEND initrd=/initrd LABEL=boot %s\n" \ % bootloader.append - msger.debug("Writing syslinux config %s/ISO/isolinux/isolinux.cfg" \ - % cr_workdir) + logger.debug("Writing syslinux config %s/ISO/isolinux/isolinux.cfg", + cr_workdir) + with open("%s/ISO/isolinux/isolinux.cfg" % cr_workdir, "w") as cfg: cfg.write(syslinux_conf) @@ -99,11 +103,11 @@ class IsoImagePlugin(SourcePlugin): if configfile: grubefi_conf = get_custom_config(configfile) if grubefi_conf: - msger.debug("Using custom configuration file " - "%s for grub.cfg" % configfile) + logger.debug("Using custom configuration file %s for grub.cfg", + configfile) else: - msger.error("configfile is specified but failed to " - "get it from %s." % configfile) + raise WicError("configfile is specified " + "but failed to get it from %s", configfile) else: splash = os.path.join(cr_workdir, "EFI/boot/splash.jpg") if os.path.exists(splash): @@ -133,8 +137,8 @@ class IsoImagePlugin(SourcePlugin): if splashline: grubefi_conf += "%s\n" % splashline - msger.debug("Writing grubefi config %s/EFI/BOOT/grub.cfg" \ - % cr_workdir) + logger.debug("Writing grubefi config %s/EFI/BOOT/grub.cfg", cr_workdir) + with open("%s/EFI/BOOT/grub.cfg" % cr_workdir, "w") as cfg: cfg.write(grubefi_conf) @@ -144,25 +148,25 @@ class IsoImagePlugin(SourcePlugin): Create path for initramfs image """ - initrd = get_bitbake_var("INITRD") + initrd = get_bitbake_var("INITRD_LIVE") or get_bitbake_var("INITRD") if not initrd: initrd_dir = get_bitbake_var("DEPLOY_DIR_IMAGE") if not initrd_dir: - msger.error("Couldn't find DEPLOY_DIR_IMAGE, exiting.\n") + raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting.") image_name = get_bitbake_var("IMAGE_BASENAME") if not image_name: - msger.error("Couldn't find IMAGE_BASENAME, exiting.\n") + raise WicError("Couldn't find IMAGE_BASENAME, exiting.") image_type = get_bitbake_var("INITRAMFS_FSTYPES") if not image_type: - msger.error("Couldn't find INITRAMFS_FSTYPES, exiting.\n") + raise WicError("Couldn't find INITRAMFS_FSTYPES, exiting.") - machine_arch = get_bitbake_var("MACHINE_ARCH") - if not machine_arch: - msger.error("Couldn't find MACHINE_ARCH, exiting.\n") + target_arch = get_bitbake_var("TRANSLATED_TARGET_ARCH") + if not target_arch: + raise WicError("Couldn't find TRANSLATED_TARGET_ARCH, exiting.") - initrd = glob.glob('%s/%s*%s.%s' % (initrd_dir, image_name, machine_arch, image_type))[0] + initrd = glob.glob('%s/%s*%s.%s' % (initrd_dir, image_name, target_arch, image_type))[0] if not os.path.exists(initrd): # Create initrd from rootfs directory @@ -183,7 +187,7 @@ class IsoImagePlugin(SourcePlugin): os.symlink(os.readlink("%s/sbin/init" % rootfs_dir), \ "%s/init" % initrd_dir) else: - msger.error("Couldn't find or build initrd, exiting.\n") + raise WicError("Couldn't find or build initrd, exiting.") exec_cmd("cd %s && find . | cpio -o -H newc -R +0:+0 >./initrd.cpio " \ % initrd_dir, as_shell=True) @@ -194,55 +198,6 @@ class IsoImagePlugin(SourcePlugin): return initrd @classmethod - def do_stage_partition(cls, part, source_params, creator, cr_workdir, - oe_builddir, bootimg_dir, kernel_dir, - native_sysroot): - """ - Special content staging called before do_prepare_partition(). - It cheks if all necessary tools are available, if not - tries to instal them. - """ - # Make sure parted is available in native sysroot - if not os.path.isfile("%s/usr/sbin/parted" % native_sysroot): - msger.info("Building parted-native...\n") - exec_cmd("bitbake parted-native") - - # Make sure mkfs.ext2/3/4 is available in native sysroot - if not os.path.isfile("%s/sbin/mkfs.ext2" % native_sysroot): - msger.info("Building e2fsprogs-native...\n") - exec_cmd("bitbake e2fsprogs-native") - - # Make sure syslinux is available in sysroot and in native sysroot - syslinux_dir = get_bitbake_var("STAGING_DATADIR") - if not syslinux_dir: - msger.error("Couldn't find STAGING_DATADIR, exiting.\n") - if not os.path.exists("%s/syslinux" % syslinux_dir): - msger.info("Building syslinux...\n") - exec_cmd("bitbake syslinux") - if not os.path.exists("%s/syslinux" % syslinux_dir): - msger.error("Please build syslinux first\n") - - # Make sure syslinux is available in native sysroot - if not os.path.exists("%s/usr/bin/syslinux" % native_sysroot): - msger.info("Building syslinux-native...\n") - exec_cmd("bitbake syslinux-native") - - #Make sure mkisofs is available in native sysroot - if not os.path.isfile("%s/usr/bin/mkisofs" % native_sysroot): - msger.info("Building cdrtools-native...\n") - exec_cmd("bitbake cdrtools-native") - - # Make sure mkfs.vfat is available in native sysroot - if not os.path.isfile("%s/sbin/mkfs.vfat" % native_sysroot): - msger.info("Building dosfstools-native...\n") - exec_cmd("bitbake dosfstools-native") - - # Make sure mtools is available in native sysroot - if not os.path.isfile("%s/usr/bin/mcopy" % native_sysroot): - msger.info("Building mtools-native...\n") - exec_cmd("bitbake mtools-native") - - @classmethod def do_configure_partition(cls, part, source_params, creator, cr_workdir, oe_builddir, bootimg_dir, kernel_dir, native_sysroot): @@ -258,11 +213,11 @@ class IsoImagePlugin(SourcePlugin): exec_cmd(install_cmd) # Overwrite the name of the created image - msger.debug("%s" % source_params) + logger.debug(source_params) if 'image_name' in source_params and \ source_params['image_name'].strip(): creator.name = source_params['image_name'].strip() - msger.debug("The name of the image is: %s" % creator.name) + logger.debug("The name of the image is: %s", creator.name) @classmethod def do_prepare_partition(cls, part, source_params, creator, cr_workdir, @@ -278,7 +233,7 @@ class IsoImagePlugin(SourcePlugin): if part.rootfs_dir is None: if not 'ROOTFS_DIR' in rootfs_dir: - msger.error("Couldn't find --rootfs-dir, exiting.\n") + raise WicError("Couldn't find --rootfs-dir, exiting.") rootfs_dir = rootfs_dir['ROOTFS_DIR'] else: if part.rootfs_dir in rootfs_dir: @@ -286,24 +241,21 @@ class IsoImagePlugin(SourcePlugin): elif part.rootfs_dir: rootfs_dir = part.rootfs_dir else: - msg = "Couldn't find --rootfs-dir=%s connection " - msg += "or it is not a valid path, exiting.\n" - msger.error(msg % part.rootfs_dir) + raise WicError("Couldn't find --rootfs-dir=%s connection " + "or it is not a valid path, exiting." % + part.rootfs_dir) if not os.path.isdir(rootfs_dir): rootfs_dir = get_bitbake_var("IMAGE_ROOTFS") if not os.path.isdir(rootfs_dir): - msger.error("Couldn't find IMAGE_ROOTFS, exiting.\n") + raise WicError("Couldn't find IMAGE_ROOTFS, exiting.") part.rootfs_dir = rootfs_dir # Prepare rootfs.img - hdd_dir = get_bitbake_var("HDDDIR") + deploy_dir = get_bitbake_var("DEPLOY_DIR_IMAGE") img_iso_dir = get_bitbake_var("ISODIR") - - rootfs_img = "%s/rootfs.img" % hdd_dir - if not os.path.isfile(rootfs_img): - rootfs_img = "%s/rootfs.img" % img_iso_dir + rootfs_img = "%s/rootfs.img" % img_iso_dir if not os.path.isfile(rootfs_img): # check if rootfs.img is in deploydir deploy_dir = get_bitbake_var("DEPLOY_DIR_IMAGE") @@ -331,15 +283,22 @@ class IsoImagePlugin(SourcePlugin): if os.path.isfile(part.source_file): os.remove(part.source_file) - # Prepare initial ramdisk - initrd = "%s/initrd" % hdd_dir - if not os.path.isfile(initrd): - initrd = "%s/initrd" % img_iso_dir - if not os.path.isfile(initrd): - initrd = cls._build_initramfs_path(rootfs_dir, cr_workdir) - - install_cmd = "install -m 0644 %s %s/initrd" \ - % (initrd, isodir) + # Support using a different initrd other than default + if source_params.get('initrd'): + initrd = source_params['initrd'] + if not deploy_dir: + raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting") + cp_cmd = "cp %s/%s %s" % (deploy_dir, initrd, cr_workdir) + exec_cmd(cp_cmd) + else: + # Prepare initial ramdisk + initrd = "%s/initrd" % deploy_dir + if not os.path.isfile(initrd): + initrd = "%s/initrd" % img_iso_dir + if not os.path.isfile(initrd): + initrd = cls._build_initramfs_path(rootfs_dir, cr_workdir) + + install_cmd = "install -m 0644 %s %s/initrd" % (initrd, isodir) exec_cmd(install_cmd) # Remove the temporary file created by _build_initramfs_path function @@ -348,7 +307,7 @@ class IsoImagePlugin(SourcePlugin): # Install bzImage install_cmd = "install -m 0644 %s/bzImage %s/bzImage" % \ - (kernel_dir, isodir) + (kernel_dir, isodir) exec_cmd(install_cmd) #Create bootloader for efi boot @@ -371,7 +330,7 @@ class IsoImagePlugin(SourcePlugin): # didn't contains it target_arch = get_bitbake_var("TARGET_SYS") if not target_arch: - msger.error("Coludn't find target architecture\n") + raise WicError("Coludn't find target architecture") if re.match("x86_64", target_arch): grub_target = 'x86_64-efi' @@ -380,21 +339,18 @@ class IsoImagePlugin(SourcePlugin): grub_target = 'i386-efi' grub_image = "bootia32.efi" else: - msger.error("grub-efi is incompatible with target %s\n" \ - % target_arch) + raise WicError("grub-efi is incompatible with target %s" % + target_arch) if not os.path.isfile("%s/EFI/BOOT/%s" \ % (bootimg_dir, grub_image)): - grub_path = get_bitbake_var("STAGING_LIBDIR") + grub_path = get_bitbake_var("STAGING_LIBDIR", "wic-tools") if not grub_path: - msger.error("Couldn't find STAGING_LIBDIR, exiting.\n") + raise WicError("Couldn't find STAGING_LIBDIR, exiting.") grub_core = "%s/grub/%s" % (grub_path, grub_target) if not os.path.exists(grub_core): - msger.info("Building grub-efi...\n") - exec_cmd("bitbake grub-efi") - if not os.path.exists(grub_core): - msger.error("Please build grub-efi first\n") + raise WicError("Please build grub-efi first") grub_cmd = "grub-mkimage -p '/EFI/BOOT' " grub_cmd += "-d %s " % grub_core @@ -410,11 +366,10 @@ class IsoImagePlugin(SourcePlugin): exec_native_cmd(grub_cmd, native_sysroot) else: - # TODO: insert gummiboot stuff - msger.error("unrecognized bootimg-efi loader: %s" \ - % source_params['loader']) + raise WicError("unrecognized bootimg-efi loader: %s" % + source_params['loader']) except KeyError: - msger.error("bootimg-efi requires a loader, none specified") + raise WicError("bootimg-efi requires a loader, none specified") if os.path.exists("%s/EFI/BOOT" % isodir): shutil.rmtree("%s/EFI/BOOT" % isodir) @@ -437,9 +392,8 @@ class IsoImagePlugin(SourcePlugin): blocks = int(out.split()[0]) # Add some extra space for file system overhead blocks += 100 - msg = "Added 100 extra blocks to %s to get to %d total blocks" \ - % (part.mountpoint, blocks) - msger.debug(msg) + logger.debug("Added 100 extra blocks to %s to get to %d " + "total blocks", part.mountpoint, blocks) # dosfs image for EFI boot bootimg = "%s/efi.img" % isodir @@ -459,9 +413,9 @@ class IsoImagePlugin(SourcePlugin): exec_cmd(chmod_cmd) # Prepare files for legacy boot - syslinux_dir = get_bitbake_var("STAGING_DATADIR") + syslinux_dir = get_bitbake_var("STAGING_DATADIR", "wic-tools") if not syslinux_dir: - msger.error("Couldn't find STAGING_DATADIR, exiting.\n") + raise WicError("Couldn't find STAGING_DATADIR, exiting.") if os.path.exists("%s/isolinux" % isodir): shutil.rmtree("%s/isolinux" % isodir) @@ -501,7 +455,7 @@ class IsoImagePlugin(SourcePlugin): mkisofs_cmd += "-eltorito-platform 0xEF -eltorito-boot %s " % efi_img mkisofs_cmd += "-no-emul-boot %s " % isodir - msger.debug("running command: %s" % mkisofs_cmd) + logger.debug("running command: %s", mkisofs_cmd) exec_native_cmd(mkisofs_cmd, native_sysroot) shutil.rmtree(isodir) @@ -522,23 +476,19 @@ class IsoImagePlugin(SourcePlugin): utility for booting via BIOS from disk storage devices. """ + iso_img = "%s.p1" % disk.path full_path = creator._full_path(workdir, disk_name, "direct") - iso_img = "%s.p1" % full_path full_path_iso = creator._full_path(workdir, disk_name, "iso") isohybrid_cmd = "isohybrid -u %s" % iso_img - msger.debug("running command: %s" % \ - isohybrid_cmd) + logger.debug("running command: %s", isohybrid_cmd) exec_native_cmd(isohybrid_cmd, native_sysroot) # Replace the image created by direct plugin with the one created by # mkisofs command. This is necessary because the iso image created by # mkisofs has a very specific MBR is system area of the ISO image, and # direct plugin adds and configures an another MBR. - msger.debug("Replaceing the image created by direct plugin\n") - os.remove(full_path) + logger.debug("Replaceing the image created by direct plugin\n") + os.remove(disk.path) shutil.copy2(iso_img, full_path_iso) shutil.copy2(full_path_iso, full_path) - - # Remove temporary ISO file - os.remove(iso_img) diff --git a/import-layers/yocto-poky/scripts/lib/wic/plugins/source/rawcopy.py b/import-layers/yocto-poky/scripts/lib/wic/plugins/source/rawcopy.py index 618dd4475..e1c4f5e7d 100644 --- a/import-layers/yocto-poky/scripts/lib/wic/plugins/source/rawcopy.py +++ b/import-layers/yocto-poky/scripts/lib/wic/plugins/source/rawcopy.py @@ -15,13 +15,16 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # +import logging import os -from wic import msger +from wic import WicError from wic.pluginbase import SourcePlugin -from wic.utils.oe.misc import exec_cmd, get_bitbake_var +from wic.utils.misc import exec_cmd, get_bitbake_var from wic.filemap import sparse_copy +logger = logging.getLogger('wic') + class RawCopyPlugin(SourcePlugin): """ Populate partition content from raw image file. @@ -30,25 +33,6 @@ class RawCopyPlugin(SourcePlugin): name = 'rawcopy' @classmethod - def do_install_disk(cls, disk, disk_name, cr, workdir, oe_builddir, - bootimg_dir, kernel_dir, native_sysroot): - """ - Called after all partitions have been prepared and assembled into a - disk image. Do nothing. - """ - pass - - @classmethod - def do_configure_partition(cls, part, source_params, cr, cr_workdir, - oe_builddir, bootimg_dir, kernel_dir, - native_sysroot): - """ - Called before do_prepare_partition(). Possibly prepare - configuration files of some sort. - """ - pass - - @classmethod def do_prepare_partition(cls, part, source_params, cr, cr_workdir, oe_builddir, bootimg_dir, kernel_dir, rootfs_dir, native_sysroot): @@ -56,18 +40,17 @@ class RawCopyPlugin(SourcePlugin): Called to do the actual content population for a partition i.e. it 'prepares' the partition to be incorporated into the image. """ - if not bootimg_dir: - bootimg_dir = get_bitbake_var("DEPLOY_DIR_IMAGE") - if not bootimg_dir: - msger.error("Couldn't find DEPLOY_DIR_IMAGE, exiting\n") + if not kernel_dir: + kernel_dir = get_bitbake_var("DEPLOY_DIR_IMAGE") + if not kernel_dir: + raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting") - msger.debug('Bootimg dir: %s' % bootimg_dir) + logger.debug('Kernel dir: %s', kernel_dir) if 'file' not in source_params: - msger.error("No file specified\n") - return + raise WicError("No file specified") - src = os.path.join(bootimg_dir, source_params['file']) + src = os.path.join(kernel_dir, source_params['file']) dst = os.path.join(cr_workdir, "%s.%s" % (source_params['file'], part.lineno)) if 'skip' in source_params: @@ -84,4 +67,3 @@ class RawCopyPlugin(SourcePlugin): part.size = filesize part.source_file = dst - diff --git a/import-layers/yocto-poky/scripts/lib/wic/plugins/source/rootfs.py b/import-layers/yocto-poky/scripts/lib/wic/plugins/source/rootfs.py index 425da8b22..f2e2ca8a2 100644 --- a/import-layers/yocto-poky/scripts/lib/wic/plugins/source/rootfs.py +++ b/import-layers/yocto-poky/scripts/lib/wic/plugins/source/rootfs.py @@ -25,11 +25,17 @@ # Joao Henrique Ferreira de Freitas <joaohf (at] gmail.com> # +import logging import os +import shutil -from wic import msger +from oe.path import copyhardlinktree + +from wic import WicError from wic.pluginbase import SourcePlugin -from wic.utils.oe.misc import get_bitbake_var +from wic.utils.misc import get_bitbake_var, exec_cmd + +logger = logging.getLogger('wic') class RootfsPlugin(SourcePlugin): """ @@ -45,10 +51,9 @@ class RootfsPlugin(SourcePlugin): image_rootfs_dir = get_bitbake_var("IMAGE_ROOTFS", rootfs_dir) if not os.path.isdir(image_rootfs_dir): - msg = "No valid artifact IMAGE_ROOTFS from image named" - msg += " %s has been found at %s, exiting.\n" % \ - (rootfs_dir, image_rootfs_dir) - msger.error(msg) + raise WicError("No valid artifact IMAGE_ROOTFS from image " + "named %s has been found at %s, exiting." % + (rootfs_dir, image_rootfs_dir)) return image_rootfs_dir @@ -63,8 +68,8 @@ class RootfsPlugin(SourcePlugin): """ if part.rootfs_dir is None: if not 'ROOTFS_DIR' in krootfs_dir: - msg = "Couldn't find --rootfs-dir, exiting" - msger.error(msg) + raise WicError("Couldn't find --rootfs-dir, exiting") + rootfs_dir = krootfs_dir['ROOTFS_DIR'] else: if part.rootfs_dir in krootfs_dir: @@ -72,12 +77,49 @@ class RootfsPlugin(SourcePlugin): elif part.rootfs_dir: rootfs_dir = part.rootfs_dir else: - msg = "Couldn't find --rootfs-dir=%s connection" - msg += " or it is not a valid path, exiting" - msger.error(msg % part.rootfs_dir) + raise WicError("Couldn't find --rootfs-dir=%s connection or " + "it is not a valid path, exiting" % part.rootfs_dir) real_rootfs_dir = cls.__get_rootfs_dir(rootfs_dir) - part.rootfs_dir = real_rootfs_dir - part.prepare_rootfs(cr_workdir, oe_builddir, real_rootfs_dir, native_sysroot) + # Handle excluded paths. + if part.exclude_path is not None: + # We need a new rootfs directory we can delete files from. Copy to + # workdir. + new_rootfs = os.path.realpath(os.path.join(cr_workdir, "rootfs")) + + if os.path.lexists(new_rootfs): + shutil.rmtree(os.path.join(new_rootfs)) + + copyhardlinktree(real_rootfs_dir, new_rootfs) + real_rootfs_dir = new_rootfs + + for orig_path in part.exclude_path: + path = orig_path + if os.path.isabs(path): + msger.error("Must be relative: --exclude-path=%s" % orig_path) + + full_path = os.path.realpath(os.path.join(new_rootfs, path)) + + # Disallow climbing outside of parent directory using '..', + # because doing so could be quite disastrous (we will delete the + # directory). + if not full_path.startswith(new_rootfs): + msger.error("'%s' points to a path outside the rootfs" % orig_path) + + if path.endswith(os.sep): + # Delete content only. + for entry in os.listdir(full_path): + full_entry = os.path.join(full_path, entry) + if os.path.isdir(full_entry) and not os.path.islink(full_entry): + shutil.rmtree(full_entry) + else: + os.remove(full_entry) + else: + # Delete whole directory. + shutil.rmtree(full_path) + + part.rootfs_dir = real_rootfs_dir + part.prepare_rootfs(cr_workdir, oe_builddir, + real_rootfs_dir, native_sysroot) diff --git a/import-layers/yocto-poky/scripts/lib/wic/plugins/source/rootfs_pcbios_ext.py b/import-layers/yocto-poky/scripts/lib/wic/plugins/source/rootfs_pcbios_ext.py deleted file mode 100644 index 3d60e6f0f..000000000 --- a/import-layers/yocto-poky/scripts/lib/wic/plugins/source/rootfs_pcbios_ext.py +++ /dev/null @@ -1,177 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -# -# This program is free software; you can distribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for mo details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# AUTHOR -# Adrian Freihofer <adrian.freihofer (at] neratec.com> -# - -import os -from wic import msger -from wic.utils import syslinux -from wic.utils import runner -from wic.utils.oe import misc -from wic.utils.errors import ImageError -from wic.pluginbase import SourcePlugin - - -# pylint: disable=no-init -class RootfsPlugin(SourcePlugin): - """ - Create root partition and install syslinux bootloader - - This plugin creates a disk image containing a bootable root partition with - syslinux installed. The filesystem is ext2/3/4, no extra boot partition is - required. - - Example kickstart file: - part / --source rootfs-pcbios-ext --ondisk sda --fstype=ext4 --label rootfs --align 1024 - bootloader --source rootfs-pcbios-ext --timeout=0 --append="rootwait rootfstype=ext4" - - The first line generates a root file system including a syslinux.cfg file - The "--source rootfs-pcbios-ext" in the second line triggers the installation - of ldlinux.sys into the image. - """ - - name = 'rootfs-pcbios-ext' - - @staticmethod - def _get_rootfs_dir(rootfs_dir): - """ - Find rootfs pseudo dir - - If rootfs_dir is a directory consider it as rootfs directory. - Otherwise ask bitbake about the IMAGE_ROOTFS directory. - """ - if os.path.isdir(rootfs_dir): - return rootfs_dir - - image_rootfs_dir = misc.get_bitbake_var("IMAGE_ROOTFS", rootfs_dir) - if not os.path.isdir(image_rootfs_dir): - msg = "No valid artifact IMAGE_ROOTFS from image named" - msg += " %s has been found at %s, exiting.\n" % \ - (rootfs_dir, image_rootfs_dir) - msger.error(msg) - - return image_rootfs_dir - - # pylint: disable=unused-argument - @classmethod - def do_configure_partition(cls, part, source_params, image_creator, - image_creator_workdir, oe_builddir, bootimg_dir, - kernel_dir, native_sysroot): - """ - Creates syslinux config in rootfs directory - - Called before do_prepare_partition() - """ - bootloader = image_creator.ks.bootloader - - syslinux_conf = "" - syslinux_conf += "PROMPT 0\n" - - syslinux_conf += "TIMEOUT " + str(bootloader.timeout) + "\n" - syslinux_conf += "ALLOWOPTIONS 1\n" - - # Derive SERIAL... line from from kernel boot parameters - syslinux_conf += syslinux.serial_console_form_kargs(options) + "\n" - - syslinux_conf += "DEFAULT linux\n" - syslinux_conf += "LABEL linux\n" - syslinux_conf += " KERNEL /boot/bzImage\n" - - syslinux_conf += " APPEND label=boot root=%s %s\n" % \ - (image_creator.rootdev, bootloader.append) - - syslinux_cfg = os.path.join(image_creator.rootfs_dir['ROOTFS_DIR'], "boot", "syslinux.cfg") - msger.debug("Writing syslinux config %s" % syslinux_cfg) - with open(syslinux_cfg, "w") as cfg: - cfg.write(syslinux_conf) - - @classmethod - def do_prepare_partition(cls, part, source_params, image_creator, - image_creator_workdir, oe_builddir, bootimg_dir, - kernel_dir, krootfs_dir, native_sysroot): - """ - Creates partition out of rootfs directory - - Prepare content for a rootfs partition i.e. create a partition - and fill it from a /rootfs dir. - Install syslinux bootloader into root partition image file - """ - def is_exe(exepath): - """Verify exepath is an executable file""" - return os.path.isfile(exepath) and os.access(exepath, os.X_OK) - - # Make sure syslinux-nomtools is available in native sysroot or fail - native_syslinux_nomtools = os.path.join(native_sysroot, "usr/bin/syslinux-nomtools") - if not is_exe(native_syslinux_nomtools): - msger.info("building syslinux-native...") - misc.exec_cmd("bitbake syslinux-native") - if not is_exe(native_syslinux_nomtools): - msger.error("Couldn't find syslinux-nomtools (%s), exiting\n" % - native_syslinux_nomtools) - - if part.rootfs is None: - if 'ROOTFS_DIR' not in krootfs_dir: - msger.error("Couldn't find --rootfs-dir, exiting") - rootfs_dir = krootfs_dir['ROOTFS_DIR'] - else: - if part.rootfs in krootfs_dir: - rootfs_dir = krootfs_dir[part.rootfs] - elif part.rootfs: - rootfs_dir = part.rootfs - else: - msg = "Couldn't find --rootfs-dir=%s connection" - msg += " or it is not a valid path, exiting" - msger.error(msg % part.rootfs) - - real_rootfs_dir = cls._get_rootfs_dir(rootfs_dir) - - part.rootfs_dir = real_rootfs_dir - part.prepare_rootfs(image_creator_workdir, oe_builddir, real_rootfs_dir, native_sysroot) - - # install syslinux into rootfs partition - syslinux_cmd = "syslinux-nomtools -d /boot -i %s" % part.source_file - misc.exec_native_cmd(syslinux_cmd, native_sysroot) - - @classmethod - def do_install_disk(cls, disk, disk_name, image_creator, workdir, oe_builddir, - bootimg_dir, kernel_dir, native_sysroot): - """ - Assemble partitions to disk image - - Called after all partitions have been prepared and assembled into a - disk image. In this case, we install the MBR. - """ - mbrfile = os.path.join(native_sysroot, "usr/share/syslinux/") - if image_creator.ptable_format == 'msdos': - mbrfile += "mbr.bin" - elif image_creator.ptable_format == 'gpt': - mbrfile += "gptmbr.bin" - else: - msger.error("Unsupported partition table: %s" % \ - image_creator.ptable_format) - - if not os.path.exists(mbrfile): - msger.error("Couldn't find %s. Has syslinux-native been baked?" % mbrfile) - - full_path = disk['disk'].device - msger.debug("Installing MBR on disk %s as %s with size %s bytes" \ - % (disk_name, full_path, disk['min_size'])) - - ret_code = runner.show(['dd', 'if=%s' % mbrfile, 'of=%s' % full_path, 'conv=notrunc']) - if ret_code != 0: - raise ImageError("Unable to set MBR to %s" % full_path) diff --git a/import-layers/yocto-poky/scripts/lib/wic/test b/import-layers/yocto-poky/scripts/lib/wic/test deleted file mode 100644 index 9daeafb98..000000000 --- a/import-layers/yocto-poky/scripts/lib/wic/test +++ /dev/null @@ -1 +0,0 @@ -test diff --git a/import-layers/yocto-poky/scripts/lib/wic/utils/errors.py b/import-layers/yocto-poky/scripts/lib/wic/utils/errors.py deleted file mode 100644 index d1b514dd9..000000000 --- a/import-layers/yocto-poky/scripts/lib/wic/utils/errors.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python -tt -# -# Copyright (c) 2007 Red Hat, Inc. -# Copyright (c) 2011 Intel, Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., 59 -# Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -class WicError(Exception): - pass - -class CreatorError(WicError): - pass - -class Usage(WicError): - pass - -class ImageError(WicError): - pass diff --git a/import-layers/yocto-poky/scripts/lib/wic/utils/misc.py b/import-layers/yocto-poky/scripts/lib/wic/utils/misc.py index 1415ae906..37e0ad6a3 100644 --- a/import-layers/yocto-poky/scripts/lib/wic/utils/misc.py +++ b/import-layers/yocto-poky/scripts/lib/wic/utils/misc.py @@ -1,95 +1,230 @@ -#!/usr/bin/env python -tt +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- # -# Copyright (c) 2010, 2011 Intel Inc. +# Copyright (c) 2013, Intel Corporation. +# All rights reserved. # -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. # -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., 59 -# Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# DESCRIPTION +# This module provides a place to collect various wic-related utils +# for the OpenEmbedded Image Tools. +# +# AUTHORS +# Tom Zanussi <tom.zanussi (at] linux.intel.com> +# +"""Miscellaneous functions.""" +import logging import os -import time -import wic.engine - -def build_name(kscfg, release=None, prefix=None, suffix=None): - """Construct and return an image name string. +import re + +from collections import defaultdict +from distutils import spawn + +from wic import WicError +from wic.utils import runner + +logger = logging.getLogger('wic') + +# executable -> recipe pairs for exec_native_cmd +NATIVE_RECIPES = {"bmaptool": "bmap-tools", + "grub-mkimage": "grub-efi", + "isohybrid": "syslinux", + "mcopy": "mtools", + "mkdosfs": "dosfstools", + "mkisofs": "cdrtools", + "mkfs.btrfs": "btrfs-tools", + "mkfs.ext2": "e2fsprogs", + "mkfs.ext3": "e2fsprogs", + "mkfs.ext4": "e2fsprogs", + "mkfs.vfat": "dosfstools", + "mksquashfs": "squashfs-tools", + "mkswap": "util-linux", + "mmd": "syslinux", + "parted": "parted", + "sfdisk": "util-linux", + "sgdisk": "gptfdisk", + "syslinux": "syslinux" + } + +def _exec_cmd(cmd_and_args, as_shell=False): + """ + Execute command, catching stderr, stdout - This is a utility function to help create sensible name and fslabel - strings. The name is constructed using the sans-prefix-and-extension - kickstart filename and the supplied prefix and suffix. + Need to execute as_shell if the command uses wildcards + """ + logger.debug("_exec_cmd: %s", cmd_and_args) + args = cmd_and_args.split() + logger.debug(args) - kscfg -- a path to a kickstart file - release -- a replacement to suffix for image release - prefix -- a prefix to prepend to the name; defaults to None, which causes - no prefix to be used - suffix -- a suffix to append to the name; defaults to None, which causes - a YYYYMMDDHHMM suffix to be used + if as_shell: + ret, out = runner.runtool(cmd_and_args) + else: + ret, out = runner.runtool(args) + out = out.strip() + if ret != 0: + raise WicError("_exec_cmd: %s returned '%s' instead of 0\noutput: %s" % \ + (cmd_and_args, ret, out)) - Note, if maxlen is less then the len(suffix), you get to keep both pieces. + logger.debug("_exec_cmd: output for %s (rc = %d): %s", + cmd_and_args, ret, out) - """ - name = os.path.basename(kscfg) - idx = name.rfind('.') - if idx >= 0: - name = name[:idx] + return ret, out - if release is not None: - suffix = "" - if prefix is None: - prefix = "" - if suffix is None: - suffix = time.strftime("%Y%m%d%H%M") - if name.startswith(prefix): - name = name[len(prefix):] +def exec_cmd(cmd_and_args, as_shell=False): + """ + Execute command, return output + """ + return _exec_cmd(cmd_and_args, as_shell)[1] - prefix = "%s-" % prefix if prefix else "" - suffix = "-%s" % suffix if suffix else "" - ret = prefix + name + suffix +def exec_native_cmd(cmd_and_args, native_sysroot, pseudo=""): + """ + Execute native command, catching stderr, stdout - return ret + Need to execute as_shell if the command uses wildcards -def find_canned(scripts_path, file_name): + Always need to execute native commands as_shell """ - Find a file either by its path or by name in the canned files dir. - - Return None if not found + # The reason -1 is used is because there may be "export" commands. + args = cmd_and_args.split(';')[-1].split() + logger.debug(args) + + if pseudo: + cmd_and_args = pseudo + cmd_and_args + + wtools_sysroot = get_bitbake_var("RECIPE_SYSROOT_NATIVE", "wic-tools") + + native_paths = \ + "%s/sbin:%s/usr/sbin:%s/usr/bin:%s/sbin:%s/usr/sbin:%s/usr/bin" % \ + (wtools_sysroot, wtools_sysroot, wtools_sysroot, + native_sysroot, native_sysroot, native_sysroot) + native_cmd_and_args = "export PATH=%s:$PATH;%s" % \ + (native_paths, cmd_and_args) + logger.debug("exec_native_cmd: %s", native_cmd_and_args) + + # If the command isn't in the native sysroot say we failed. + if spawn.find_executable(args[0], native_paths): + ret, out = _exec_cmd(native_cmd_and_args, True) + else: + ret = 127 + out = "can't find native executable %s in %s" % (args[0], native_paths) + + prog = args[0] + # shell command-not-found + if ret == 127 \ + or (pseudo and ret == 1 and out == "Can't find '%s' in $PATH." % prog): + msg = "A native program %s required to build the image "\ + "was not found (see details above).\n\n" % prog + recipe = NATIVE_RECIPES.get(prog) + if recipe: + msg += "Please make sure wic-tools have %s-native in its DEPENDS, bake it with 'bitbake wic-tools' "\ + "and try again.\n" % recipe + else: + msg += "Wic failed to find a recipe to build native %s. Please "\ + "file a bug against wic.\n" % prog + raise WicError(msg) + + return ret, out + +BOOTDD_EXTRA_SPACE = 16384 + +class BitbakeVars(defaultdict): """ - if os.path.exists(file_name): - return file_name - - layers_canned_wks_dir = wic.engine.build_canned_image_list(scripts_path) - for canned_wks_dir in layers_canned_wks_dir: - for root, dirs, files in os.walk(canned_wks_dir): - for fname in files: - if fname == file_name: - fullpath = os.path.join(canned_wks_dir, fname) - return fullpath - -def get_custom_config(boot_file): + Container for Bitbake variables. """ - Get the custom configuration to be used for the bootloader. - - Return None if the file can't be found. + def __init__(self): + defaultdict.__init__(self, dict) + + # default_image and vars_dir attributes should be set from outside + self.default_image = None + self.vars_dir = None + + def _parse_line(self, line, image, matcher=re.compile(r"^([a-zA-Z0-9\-_+./~]+)=(.*)")): + """ + Parse one line from bitbake -e output or from .env file. + Put result key-value pair into the storage. + """ + if "=" not in line: + return + match = matcher.match(line) + if not match: + return + key, val = match.groups() + self[image][key] = val.strip('"') + + def get_var(self, var, image=None, cache=True): + """ + Get bitbake variable from 'bitbake -e' output or from .env file. + This is a lazy method, i.e. it runs bitbake or parses file only when + only when variable is requested. It also caches results. + """ + if not image: + image = self.default_image + + if image not in self: + if image and self.vars_dir: + fname = os.path.join(self.vars_dir, image + '.env') + if os.path.isfile(fname): + # parse .env file + with open(fname) as varsfile: + for line in varsfile: + self._parse_line(line, image) + else: + print("Couldn't get bitbake variable from %s." % fname) + print("File %s doesn't exist." % fname) + return + else: + # Get bitbake -e output + cmd = "bitbake -e" + if image: + cmd += " %s" % image + + log_level = logger.getEffectiveLevel() + logger.setLevel(logging.INFO) + ret, lines = _exec_cmd(cmd) + logger.setLevel(log_level) + + if ret: + logger.error("Couldn't get '%s' output.", cmd) + logger.error("Bitbake failed with error:\n%s\n", lines) + return + + # Parse bitbake -e output + for line in lines.split('\n'): + self._parse_line(line, image) + + # Make first image a default set of variables + if cache: + images = [key for key in self if key] + if len(images) == 1: + self[None] = self[image] + + result = self[image].get(var) + if not cache: + self.pop(image, None) + + return result + +# Create BB_VARS singleton +BB_VARS = BitbakeVars() + +def get_bitbake_var(var, image=None, cache=True): + """ + Provide old get_bitbake_var API by wrapping + get_var method of BB_VARS singleton. """ - scripts_path = os.path.abspath(os.path.dirname(__file__)) - # Get the scripts path of poky - for x in range(0, 3): - scripts_path = os.path.dirname(scripts_path) - - cfg_file = find_canned(scripts_path, boot_file) - if cfg_file: - with open(cfg_file, "r") as f: - config = f.read() - return config - - return None + return BB_VARS.get_var(var, image, cache) diff --git a/import-layers/yocto-poky/scripts/lib/wic/utils/oe/__init__.py b/import-layers/yocto-poky/scripts/lib/wic/utils/oe/__init__.py deleted file mode 100644 index 0a81575a7..000000000 --- a/import-layers/yocto-poky/scripts/lib/wic/utils/oe/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# -# OpenEmbedded wic utils library -# -# Copyright (c) 2013, Intel Corporation. -# All rights reserved. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# AUTHORS -# Tom Zanussi <tom.zanussi (at] linux.intel.com> -# diff --git a/import-layers/yocto-poky/scripts/lib/wic/utils/oe/misc.py b/import-layers/yocto-poky/scripts/lib/wic/utils/oe/misc.py deleted file mode 100644 index fe188c9d2..000000000 --- a/import-layers/yocto-poky/scripts/lib/wic/utils/oe/misc.py +++ /dev/null @@ -1,246 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -# -# Copyright (c) 2013, Intel Corporation. -# All rights reserved. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# DESCRIPTION -# This module provides a place to collect various wic-related utils -# for the OpenEmbedded Image Tools. -# -# AUTHORS -# Tom Zanussi <tom.zanussi (at] linux.intel.com> -# -"""Miscellaneous functions.""" - -import os -from collections import defaultdict -from distutils import spawn - -from wic import msger -from wic.utils import runner - -# executable -> recipe pairs for exec_native_cmd -NATIVE_RECIPES = {"bmaptool": "bmap-tools", - "mcopy": "mtools", - "mkdosfs": "dosfstools", - "mkfs.btrfs": "btrfs-tools", - "mkfs.ext2": "e2fsprogs", - "mkfs.ext3": "e2fsprogs", - "mkfs.ext4": "e2fsprogs", - "mkfs.vfat": "dosfstools", - "mksquashfs": "squashfs-tools", - "mkswap": "util-linux", - "parted": "parted", - "sfdisk": "util-linux", - "sgdisk": "gptfdisk", - "syslinux": "syslinux" - } - -def _exec_cmd(cmd_and_args, as_shell=False, catch=3): - """ - Execute command, catching stderr, stdout - - Need to execute as_shell if the command uses wildcards - """ - msger.debug("_exec_cmd: %s" % cmd_and_args) - args = cmd_and_args.split() - msger.debug(args) - - if as_shell: - ret, out = runner.runtool(cmd_and_args, catch) - else: - ret, out = runner.runtool(args, catch) - out = out.strip() - msger.debug("_exec_cmd: output for %s (rc = %d): %s" % \ - (cmd_and_args, ret, out)) - - return (ret, out) - - -def exec_cmd(cmd_and_args, as_shell=False, catch=3): - """ - Execute command, catching stderr, stdout - - Exits if rc non-zero - """ - ret, out = _exec_cmd(cmd_and_args, as_shell, catch) - - if ret != 0: - msger.error("exec_cmd: %s returned '%s' instead of 0" % \ - (cmd_and_args, ret)) - - return out - -def exec_native_cmd(cmd_and_args, native_sysroot, catch=3, pseudo=""): - """ - Execute native command, catching stderr, stdout - - Need to execute as_shell if the command uses wildcards - - Always need to execute native commands as_shell - """ - # The reason -1 is used is because there may be "export" commands. - args = cmd_and_args.split(';')[-1].split() - msger.debug(args) - - if pseudo: - cmd_and_args = pseudo + cmd_and_args - native_paths = \ - "%s/sbin:%s/usr/sbin:%s/usr/bin" % \ - (native_sysroot, native_sysroot, native_sysroot) - native_cmd_and_args = "export PATH=%s:$PATH;%s" % \ - (native_paths, cmd_and_args) - msger.debug("exec_native_cmd: %s" % cmd_and_args) - - # If the command isn't in the native sysroot say we failed. - if spawn.find_executable(args[0], native_paths): - ret, out = _exec_cmd(native_cmd_and_args, True, catch) - else: - ret = 127 - - prog = args[0] - # shell command-not-found - if ret == 127 \ - or (pseudo and ret == 1 and out == "Can't find '%s' in $PATH." % prog): - msg = "A native program %s required to build the image "\ - "was not found (see details above).\n\n" % prog - recipe = NATIVE_RECIPES.get(prog) - if recipe: - msg += "Please bake it with 'bitbake %s-native' "\ - "and try again.\n" % recipe - else: - msg += "Wic failed to find a recipe to build native %s. Please "\ - "file a bug against wic.\n" % prog - msger.error(msg) - if out: - msger.debug('"%s" output: %s' % (args[0], out)) - - if ret != 0: - msger.error("exec_cmd: '%s' returned '%s' instead of 0" % \ - (cmd_and_args, ret)) - - return ret, out - -BOOTDD_EXTRA_SPACE = 16384 - -class BitbakeVars(defaultdict): - """ - Container for Bitbake variables. - """ - def __init__(self): - defaultdict.__init__(self, dict) - - # default_image and vars_dir attributes should be set from outside - self.default_image = None - self.vars_dir = None - - def _parse_line(self, line, image): - """ - Parse one line from bitbake -e output or from .env file. - Put result key-value pair into the storage. - """ - if "=" not in line: - return - try: - key, val = line.split("=") - except ValueError: - return - key = key.strip() - val = val.strip() - if key.replace('_', '').isalnum(): - self[image][key] = val.strip('"') - - def get_var(self, var, image=None): - """ - Get bitbake variable from 'bitbake -e' output or from .env file. - This is a lazy method, i.e. it runs bitbake or parses file only when - only when variable is requested. It also caches results. - """ - if not image: - image = self.default_image - - if image not in self: - if image and self.vars_dir: - fname = os.path.join(self.vars_dir, image + '.env') - if os.path.isfile(fname): - # parse .env file - with open(fname) as varsfile: - for line in varsfile: - self._parse_line(line, image) - else: - print("Couldn't get bitbake variable from %s." % fname) - print("File %s doesn't exist." % fname) - return - else: - # Get bitbake -e output - cmd = "bitbake -e" - if image: - cmd += " %s" % image - - log_level = msger.get_loglevel() - msger.set_loglevel('normal') - ret, lines = _exec_cmd(cmd) - msger.set_loglevel(log_level) - - if ret: - print("Couldn't get '%s' output." % cmd) - print("Bitbake failed with error:\n%s\n" % lines) - return - - # Parse bitbake -e output - for line in lines.split('\n'): - self._parse_line(line, image) - - # Make first image a default set of variables - images = [key for key in self if key] - if len(images) == 1: - self[None] = self[image] - - return self[image].get(var) - -# Create BB_VARS singleton -BB_VARS = BitbakeVars() - -def get_bitbake_var(var, image=None): - """ - Provide old get_bitbake_var API by wrapping - get_var method of BB_VARS singleton. - """ - return BB_VARS.get_var(var, image) - -def parse_sourceparams(sourceparams): - """ - Split sourceparams string of the form key1=val1[,key2=val2,...] - into a dict. Also accepts valueless keys i.e. without =. - - Returns dict of param key/val pairs (note that val may be None). - """ - params_dict = {} - - params = sourceparams.split(',') - if params: - for par in params: - if not par: - continue - if not '=' in par: - key = par - val = None - else: - key, val = par.split('=') - params_dict[key] = val - - return params_dict diff --git a/import-layers/yocto-poky/scripts/lib/wic/utils/partitionedfs.py b/import-layers/yocto-poky/scripts/lib/wic/utils/partitionedfs.py deleted file mode 100644 index 9ea4a30cb..000000000 --- a/import-layers/yocto-poky/scripts/lib/wic/utils/partitionedfs.py +++ /dev/null @@ -1,379 +0,0 @@ -#!/usr/bin/env python -tt -# -# Copyright (c) 2009, 2010, 2011 Intel, Inc. -# Copyright (c) 2007, 2008 Red Hat, Inc. -# Copyright (c) 2008 Daniel P. Berrange -# Copyright (c) 2008 David P. Huff -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., 59 -# Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -import os -from wic import msger -from wic.utils.errors import ImageError -from wic.utils.oe.misc import exec_cmd, exec_native_cmd -from wic.filemap import sparse_copy - -# Overhead of the MBR partitioning scheme (just one sector) -MBR_OVERHEAD = 1 - -# Overhead of the GPT partitioning scheme -GPT_OVERHEAD = 34 - -# Size of a sector in bytes -SECTOR_SIZE = 512 - -class Image(): - """ - Generic base object for an image. - - An Image is a container for a set of DiskImages and associated - partitions. - """ - def __init__(self, native_sysroot=None): - self.disks = {} - self.partitions = [] - self.partimages = [] - # Size of a sector used in calculations - self.sector_size = SECTOR_SIZE - self._partitions_layed_out = False - self.native_sysroot = native_sysroot - - def __add_disk(self, disk_name): - """ Add a disk 'disk_name' to the internal list of disks. Note, - 'disk_name' is the name of the disk in the target system - (e.g., sdb). """ - - if disk_name in self.disks: - # We already have this disk - return - - assert not self._partitions_layed_out - - self.disks[disk_name] = \ - {'disk': None, # Disk object - 'numpart': 0, # Number of allocate partitions - 'realpart': 0, # Number of partitions in the partition table - 'partitions': [], # Indexes to self.partitions - 'offset': 0, # Offset of next partition (in sectors) - # Minimum required disk size to fit all partitions (in bytes) - 'min_size': 0, - 'ptable_format': "msdos", # Partition table format - 'identifier': None} # Disk system identifier - - def add_disk(self, disk_name, disk_obj, identifier): - """ Add a disk object which have to be partitioned. More than one disk - can be added. In case of multiple disks, disk partitions have to be - added for each disk separately with 'add_partition()". """ - - self.__add_disk(disk_name) - self.disks[disk_name]['disk'] = disk_obj - self.disks[disk_name]['identifier'] = identifier - - def __add_partition(self, part): - """ This is a helper function for 'add_partition()' which adds a - partition to the internal list of partitions. """ - - assert not self._partitions_layed_out - - self.partitions.append(part) - self.__add_disk(part['disk_name']) - - def add_partition(self, size, disk_name, mountpoint, source_file=None, fstype=None, - label=None, fsopts=None, boot=False, align=None, no_table=False, - part_type=None, uuid=None, system_id=None): - """ Add the next partition. Partitions have to be added in the - first-to-last order. """ - - ks_pnum = len(self.partitions) - - # Converting kB to sectors for parted - size = size * 1024 // self.sector_size - - part = {'ks_pnum': ks_pnum, # Partition number in the KS file - 'size': size, # In sectors - 'mountpoint': mountpoint, # Mount relative to chroot - 'source_file': source_file, # partition contents - 'fstype': fstype, # Filesystem type - 'fsopts': fsopts, # Filesystem mount options - 'label': label, # Partition label - 'disk_name': disk_name, # physical disk name holding partition - 'device': None, # kpartx device node for partition - 'num': None, # Partition number - 'boot': boot, # Bootable flag - 'align': align, # Partition alignment - 'no_table' : no_table, # Partition does not appear in partition table - 'part_type' : part_type, # Partition type - 'uuid': uuid, # Partition UUID - 'system_id': system_id} # Partition system id - - self.__add_partition(part) - - def layout_partitions(self, ptable_format="msdos"): - """ Layout the partitions, meaning calculate the position of every - partition on the disk. The 'ptable_format' parameter defines the - partition table format and may be "msdos". """ - - msger.debug("Assigning %s partitions to disks" % ptable_format) - - if self._partitions_layed_out: - return - - self._partitions_layed_out = True - - # Go through partitions in the order they are added in .ks file - for num in range(len(self.partitions)): - part = self.partitions[num] - - if part['disk_name'] not in self.disks: - raise ImageError("No disk %s for partition %s" \ - % (part['disk_name'], part['mountpoint'])) - - if ptable_format == 'msdos' and part['part_type']: - # The --part-type can also be implemented for MBR partitions, - # in which case it would map to the 1-byte "partition type" - # filed at offset 3 of the partition entry. - raise ImageError("setting custom partition type is not " \ - "implemented for msdos partitions") - - # Get the disk where the partition is located - disk = self.disks[part['disk_name']] - disk['numpart'] += 1 - if not part['no_table']: - disk['realpart'] += 1 - disk['ptable_format'] = ptable_format - - if disk['numpart'] == 1: - if ptable_format == "msdos": - overhead = MBR_OVERHEAD - elif ptable_format == "gpt": - overhead = GPT_OVERHEAD - - # Skip one sector required for the partitioning scheme overhead - disk['offset'] += overhead - - if disk['realpart'] > 3: - # Reserve a sector for EBR for every logical partition - # before alignment is performed. - if ptable_format == "msdos": - disk['offset'] += 1 - - - if part['align']: - # If not first partition and we do have alignment set we need - # to align the partition. - # FIXME: This leaves a empty spaces to the disk. To fill the - # gaps we could enlargea the previous partition? - - # Calc how much the alignment is off. - align_sectors = disk['offset'] % (part['align'] * 1024 // self.sector_size) - - if align_sectors: - # If partition is not aligned as required, we need - # to move forward to the next alignment point - align_sectors = (part['align'] * 1024 // self.sector_size) - align_sectors - - msger.debug("Realignment for %s%s with %s sectors, original" - " offset %s, target alignment is %sK." % - (part['disk_name'], disk['numpart'], align_sectors, - disk['offset'], part['align'])) - - # increase the offset so we actually start the partition on right alignment - disk['offset'] += align_sectors - - part['start'] = disk['offset'] - disk['offset'] += part['size'] - - part['type'] = 'primary' - if not part['no_table']: - part['num'] = disk['realpart'] - else: - part['num'] = 0 - - if disk['ptable_format'] == "msdos": - if len(self.partitions) > 4: - if disk['realpart'] > 3: - part['type'] = 'logical' - part['num'] = disk['realpart'] + 1 - - disk['partitions'].append(num) - msger.debug("Assigned %s to %s%d, sectors range %d-%d size %d " - "sectors (%d bytes)." \ - % (part['mountpoint'], part['disk_name'], part['num'], - part['start'], part['start'] + part['size'] - 1, - part['size'], part['size'] * self.sector_size)) - - # Once all the partitions have been layed out, we can calculate the - # minumim disk sizes. - for disk in self.disks.values(): - disk['min_size'] = disk['offset'] - if disk['ptable_format'] == "gpt": - disk['min_size'] += GPT_OVERHEAD - - disk['min_size'] *= self.sector_size - - def __create_partition(self, device, parttype, fstype, start, size): - """ Create a partition on an image described by the 'device' object. """ - - # Start is included to the size so we need to substract one from the end. - end = start + size - 1 - msger.debug("Added '%s' partition, sectors %d-%d, size %d sectors" % - (parttype, start, end, size)) - - cmd = "parted -s %s unit s mkpart %s" % (device, parttype) - if fstype: - cmd += " %s" % fstype - cmd += " %d %d" % (start, end) - - return exec_native_cmd(cmd, self.native_sysroot) - - def __format_disks(self): - self.layout_partitions() - - for dev in self.disks: - disk = self.disks[dev] - msger.debug("Initializing partition table for %s" % \ - (disk['disk'].device)) - exec_native_cmd("parted -s %s mklabel %s" % \ - (disk['disk'].device, disk['ptable_format']), - self.native_sysroot) - - if disk['identifier']: - msger.debug("Set disk identifier %x" % disk['identifier']) - with open(disk['disk'].device, 'r+b') as img: - img.seek(0x1B8) - img.write(disk['identifier'].to_bytes(4, 'little')) - - msger.debug("Creating partitions") - - for part in self.partitions: - if part['num'] == 0: - continue - - disk = self.disks[part['disk_name']] - if disk['ptable_format'] == "msdos" and part['num'] == 5: - # Create an extended partition (note: extended - # partition is described in MBR and contains all - # logical partitions). The logical partitions save a - # sector for an EBR just before the start of a - # partition. The extended partition must start one - # sector before the start of the first logical - # partition. This way the first EBR is inside of the - # extended partition. Since the extended partitions - # starts a sector before the first logical partition, - # add a sector at the back, so that there is enough - # room for all logical partitions. - self.__create_partition(disk['disk'].device, "extended", - None, part['start'] - 1, - disk['offset'] - part['start'] + 1) - - if part['fstype'] == "swap": - parted_fs_type = "linux-swap" - elif part['fstype'] == "vfat": - parted_fs_type = "fat32" - elif part['fstype'] == "msdos": - parted_fs_type = "fat16" - elif part['fstype'] == "ontrackdm6aux3": - parted_fs_type = "ontrackdm6aux3" - else: - # Type for ext2/ext3/ext4/btrfs - parted_fs_type = "ext2" - - # Boot ROM of OMAP boards require vfat boot partition to have an - # even number of sectors. - if part['mountpoint'] == "/boot" and part['fstype'] in ["vfat", "msdos"] \ - and part['size'] % 2: - msger.debug("Subtracting one sector from '%s' partition to " \ - "get even number of sectors for the partition" % \ - part['mountpoint']) - part['size'] -= 1 - - self.__create_partition(disk['disk'].device, part['type'], - parted_fs_type, part['start'], part['size']) - - if part['part_type']: - msger.debug("partition %d: set type UID to %s" % \ - (part['num'], part['part_type'])) - exec_native_cmd("sgdisk --typecode=%d:%s %s" % \ - (part['num'], part['part_type'], - disk['disk'].device), self.native_sysroot) - - if part['uuid'] and disk['ptable_format'] == "gpt": - msger.debug("partition %d: set UUID to %s" % \ - (part['num'], part['uuid'])) - exec_native_cmd("sgdisk --partition-guid=%d:%s %s" % \ - (part['num'], part['uuid'], disk['disk'].device), - self.native_sysroot) - - if part['boot']: - flag_name = "legacy_boot" if disk['ptable_format'] == 'gpt' else "boot" - msger.debug("Set '%s' flag for partition '%s' on disk '%s'" % \ - (flag_name, part['num'], disk['disk'].device)) - exec_native_cmd("parted -s %s set %d %s on" % \ - (disk['disk'].device, part['num'], flag_name), - self.native_sysroot) - if part['system_id']: - exec_native_cmd("sfdisk --part-type %s %s %s" % \ - (disk['disk'].device, part['num'], part['system_id']), - self.native_sysroot) - - # Parted defaults to enabling the lba flag for fat16 partitions, - # which causes compatibility issues with some firmware (and really - # isn't necessary). - if parted_fs_type == "fat16": - if disk['ptable_format'] == 'msdos': - msger.debug("Disable 'lba' flag for partition '%s' on disk '%s'" % \ - (part['num'], disk['disk'].device)) - exec_native_cmd("parted -s %s set %d lba off" % \ - (disk['disk'].device, part['num']), - self.native_sysroot) - - def cleanup(self): - if self.disks: - for dev in self.disks: - disk = self.disks[dev] - try: - disk['disk'].cleanup() - except: - pass - # remove partition images - for image in self.partimages: - if os.path.isfile(image): - os.remove(image) - - def assemble(self, image_file): - msger.debug("Installing partitions") - - for part in self.partitions: - source = part['source_file'] - if source: - # install source_file contents into a partition - sparse_copy(source, image_file, part['start'] * self.sector_size) - - msger.debug("Installed %s in partition %d, sectors %d-%d, " - "size %d sectors" % \ - (source, part['num'], part['start'], - part['start'] + part['size'] - 1, part['size'])) - - partimage = image_file + '.p%d' % part['num'] - os.rename(source, partimage) - self.partimages.append(partimage) - - def create(self): - for dev in self.disks: - disk = self.disks[dev] - disk['disk'].create() - - self.__format_disks() - - return diff --git a/import-layers/yocto-poky/scripts/lib/wic/utils/runner.py b/import-layers/yocto-poky/scripts/lib/wic/utils/runner.py index db536ba58..4aa00fbe2 100644 --- a/import-layers/yocto-poky/scripts/lib/wic/utils/runner.py +++ b/import-layers/yocto-poky/scripts/lib/wic/utils/runner.py @@ -14,29 +14,17 @@ # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., 59 # Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -import os import subprocess -from wic import msger +from wic import WicError -def runtool(cmdln_or_args, catch=1): +def runtool(cmdln_or_args): """ wrapper for most of the subprocess calls input: cmdln_or_args: can be both args and cmdln str (shell=True) - catch: 0, quitely run - 1, only STDOUT - 2, only STDERR - 3, both STDOUT and STDERR return: - (rc, output) - if catch==0: the output will always None + rc, output """ - - if catch not in (0, 1, 2, 3): - # invalid catch selection, will cause exception, that's good - return None - if isinstance(cmdln_or_args, list): cmd = cmdln_or_args[0] shell = False @@ -45,66 +33,20 @@ def runtool(cmdln_or_args, catch=1): cmd = shlex.split(cmdln_or_args)[0] shell = True - if catch != 3: - dev_null = os.open("/dev/null", os.O_WRONLY) - - if catch == 0: - sout = dev_null - serr = dev_null - elif catch == 1: - sout = subprocess.PIPE - serr = dev_null - elif catch == 2: - sout = dev_null - serr = subprocess.PIPE - elif catch == 3: - sout = subprocess.PIPE - serr = subprocess.STDOUT + sout = subprocess.PIPE + serr = subprocess.STDOUT try: process = subprocess.Popen(cmdln_or_args, stdout=sout, stderr=serr, shell=shell) - (sout, serr) = process.communicate() + sout, serr = process.communicate() # combine stdout and stderr, filter None out and decode out = ''.join([out.decode('utf-8') for out in [sout, serr] if out]) except OSError as err: if err.errno == 2: # [Errno 2] No such file or directory - msger.error('Cannot run command: %s, lost dependency?' % cmd) + raise WicError('Cannot run command: %s, lost dependency?' % cmd) else: raise # relay - finally: - if catch != 3: - os.close(dev_null) - - return (process.returncode, out) - -def show(cmdln_or_args): - # show all the message using msger.verbose - - rcode, out = runtool(cmdln_or_args, catch=3) - - if isinstance(cmdln_or_args, list): - cmd = ' '.join(cmdln_or_args) - else: - cmd = cmdln_or_args - - msg = 'running command: "%s"' % cmd - if out: - out = out.strip() - if out: - msg += ', with output::' - msg += '\n +----------------' - for line in out.splitlines(): - msg += '\n | %s' % line - msg += '\n +----------------' - - msger.verbose(msg) - return rcode - -def outs(cmdln_or_args, catch=1): - # get the outputs of tools - return runtool(cmdln_or_args, catch)[1].strip() -def quiet(cmdln_or_args): - return runtool(cmdln_or_args, catch=0)[0] + return process.returncode, out diff --git a/import-layers/yocto-poky/scripts/lib/wic/utils/syslinux.py b/import-layers/yocto-poky/scripts/lib/wic/utils/syslinux.py deleted file mode 100644 index aace2863c..000000000 --- a/import-layers/yocto-poky/scripts/lib/wic/utils/syslinux.py +++ /dev/null @@ -1,58 +0,0 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., 59 -# Temple Place - Suite 330, Boston, MA 02111-1307, USA. -# -# AUTHOR -# Adrian Freihofer <adrian.freihofer (at] neratec.com> - - -import re -from wic import msger - - -def serial_console_form_kargs(kernel_args): - """ - Create SERIAL... line from kernel parameters - - syslinux needs a line SERIAL port [baudrate [flowcontrol]] - in the syslinux.cfg file. The config line is generated based - on kernel boot parameters. The the parameters of the first - ttyS console are considered for syslinux config. - @param kernel_args kernel command line - @return line for syslinux config file e.g. "SERIAL 0 115200" - """ - syslinux_conf = "" - for param in kernel_args.split(): - param_match = re.match("console=ttyS([0-9]+),?([0-9]*)([noe]?)([0-9]?)(r?)", param) - if param_match: - syslinux_conf += "SERIAL " + param_match.group(1) - # baudrate - if param_match.group(2): - syslinux_conf += " " + param_match.group(2) - # parity - if param_match.group(3) and param_match.group(3) != 'n': - msger.warning("syslinux does not support parity for console. {} is ignored." - .format(param_match.group(3))) - # number of bits - if param_match.group(4) and param_match.group(4) != '8': - msger.warning("syslinux supports 8 bit console configuration only. {} is ignored." - .format(param_match.group(4))) - # flow control - if param_match.group(5) and param_match.group(5) != '': - msger.warning("syslinux console flowcontrol configuration. {} is ignored." - .format(param_match.group(5))) - break - - return syslinux_conf |
