diff options
Diffstat (limited to 'tools/buildman')
| -rw-r--r-- | tools/buildman/README | 142 | ||||
| -rw-r--r-- | tools/buildman/bsettings.py | 11 | ||||
| -rw-r--r-- | tools/buildman/builder.py | 21 | ||||
| -rw-r--r-- | tools/buildman/builderthread.py | 7 | ||||
| -rw-r--r-- | tools/buildman/cmdline.py | 10 | ||||
| -rw-r--r-- | tools/buildman/control.py | 58 | ||||
| -rw-r--r-- | tools/buildman/test.py | 75 | ||||
| -rw-r--r-- | tools/buildman/toolchain.py | 270 |
8 files changed, 529 insertions, 65 deletions
diff --git a/tools/buildman/README b/tools/buildman/README index 0f8ea200f5..cf7bf5c17e 100644 --- a/tools/buildman/README +++ b/tools/buildman/README @@ -141,8 +141,8 @@ $ git clone git://git.denx.de/u-boot.git . $ git checkout -b my-branch origin/master $ # Add some commits to the branch, reading for testing -2. Create ~/.buildman to tell buildman where to find tool chains. As an -example: +2. Create ~/.buildman to tell buildman where to find tool chains (see 'The +.buildman file' later for details). As an example: # Buildman settings file @@ -171,7 +171,16 @@ The toolchain-alias section indicates that the i386 toolchain should be used to build x86 commits. -2. Check the available toolchains +3. Make sure you have the require Python pre-requisites + +Buildman uses multiprocessing, Queue, shutil, StringIO, ConfigParser and +urllib2. These should normally be available, but if you get an error like +this then you will need to obtain those modules: + + ImportError: No module named multiprocessing + + +4. Check the available toolchains Run this check to make sure that you have a toolchain for every architecture. @@ -301,6 +310,47 @@ You can see that everything is covered, even some strange ones that won't be used (c88 and c99). This is a feature. +5. Install new toolchains if needed + +You can download toolchains and update the [toolchain] section of the +settings file to find them. + +To make this easier, buildman can automatically download and install +toolchains from kernel.org. First list the available architectures: + +$ ./tools/buildman/buildman sandbox --fetch-arch list +Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.6.3/ +Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.6.2/ +Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.5.1/ +Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.2.4/ +Available architectures: alpha am33_2.0 arm avr32 bfin cris crisv32 frv h8300 +hppa hppa64 i386 ia64 m32r m68k mips mips64 or32 powerpc powerpc64 s390x sh4 +sparc sparc64 tilegx x86_64 xtensa + +Then pick one and download it: + +$ ./tools/buildman/buildman sandbox --fetch-arch or32 +Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.6.3/ +Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.6.2/ +Checking: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.5.1/ +Downloading: https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.5.1//x86_64-gcc-4.5.1-nolibc_or32-linux.tar.xz +Unpacking to: /home/sjg/.buildman-toolchains +Testing + - looking in '/home/sjg/.buildman-toolchains/gcc-4.5.1-nolibc/or32-linux/.' + - looking in '/home/sjg/.buildman-toolchains/gcc-4.5.1-nolibc/or32-linux/bin' + - found '/home/sjg/.buildman-toolchains/gcc-4.5.1-nolibc/or32-linux/bin/or32-linux-gcc' +Tool chain test: OK + +Buildman should now be set up to use your new toolchain. + +At the time of writing, U-Boot has these architectures: + + arc, arm, avr32, blackfin, m68k, microblaze, mips, nds32, nios2, openrisc + powerpc, sandbox, sh, sparc, x86 + +Of these, only arc, microblaze and nds32 are not available at kernel.org.. + + How to run it ============= @@ -310,8 +360,9 @@ branch with a valid upstream) $ ./tools/buildman/buildman -b <branch> -n If it can't detect the upstream branch, try checking out the branch, and -doing something like 'git branch --set-upstream <branch> upstream/master' -or something similar. +doing something like 'git branch --set-upstream-to upstream/master' +or something similar. Buildman will try to guess a suitable upstream branch +if it can't find one (you will see a message like" Guessing upstream as ...). As an example: @@ -665,28 +716,62 @@ It is common when refactoring code for the rodata to decrease as the text size increases, and vice versa. -Providing 'make' flags -====================== +The .buildman file +================== + +The .buildman file provides information about the available toolchains and +also allows build flags to be passed to 'make'. It consists of several +sections, with the section name in square brackets. Within each section are +a set of (tag, value) pairs. + +'[toolchain]' section + + This lists the available toolchains. The tag here doesn't matter, but + make sure it is unique. The value is the path to the toolchain. Buildman + will look in that path for a file ending in 'gcc'. It will then execute + it to check that it is a C compiler, passing only the --version flag to + it. If the return code is 0, buildman assumes that it is a valid C + compiler. It uses the first part of the name as the architecture and + strips off the last part when setting the CROSS_COMPILE environment + variable (parts are delimited with a hyphen). + + For example powerpc-linux-gcc will be noted as a toolchain for 'powerpc' + and CROSS_COMPILE will be set to powerpc-linux- when using it. + +'[toolchain-alias]' section + + This converts toolchain architecture names to U-Boot names. For example, + if an x86 toolchains is called i386-linux-gcc it will not normally be + used for architecture 'x86'. Adding 'x86: i386 x86_64' to this section + will tell buildman that the i386 and x86_64 toolchains can be used for + the x86 architecture. + +'[make-flags]' section -U-Boot's build system supports a few flags (such as BUILD_TAG) which affect -the build product. These flags can be specified in the buildman settings -file. They can also be useful when building U-Boot against other open source -software. + U-Boot's build system supports a few flags (such as BUILD_TAG) which + affect the build product. These flags can be specified in the buildman + settings file. They can also be useful when building U-Boot against other + open source software. -[make-flags] -at91-boards=ENABLE_AT91_TEST=1 -snapper9260=${at91-boards} BUILD_TAG=442 -snapper9g45=${at91-boards} BUILD_TAG=443 + [make-flags] + at91-boards=ENABLE_AT91_TEST=1 + snapper9260=${at91-boards} BUILD_TAG=442 + snapper9g45=${at91-boards} BUILD_TAG=443 -This will use 'make ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260 -and 'make ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45. A special -variable ${target} is available to access the target name (snapper9260 and -snapper9g20 in this case). Variables are resolved recursively. Note that -variables can only contain the characters A-Z, a-z, 0-9, hyphen (-) and -underscore (_). + This will use 'make ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260 + and 'make ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45. A special + variable ${target} is available to access the target name (snapper9260 + and snapper9g20 in this case). Variables are resolved recursively. Note + that variables can only contain the characters A-Z, a-z, 0-9, hyphen (-) + and underscore (_). -It is expected that any variables added are dealt with in U-Boot's -config.mk file and documented in the README. + It is expected that any variables added are dealt with in U-Boot's + config.mk file and documented in the README. + + Note that you can pass ad-hoc options to the build using environment + variables, for example: + + SOME_OPTION=1234 ./tools/buildman/buildman my_board Quick Sanity Check @@ -698,6 +783,17 @@ build the selected boards and display build status as it runs (i.e. -v is enabled automatically). Use -e to see errors/warnings as well. +Building Ranges +=============== + +You can build a range of commits by specifying a range instead of a branch +when using the -b flag. For example: + + upstream/master..us-buildman + +will build commits in us-buildman that are not in upstream/master. + + Other options ============= diff --git a/tools/buildman/bsettings.py b/tools/buildman/bsettings.py index fdd875b073..b361469180 100644 --- a/tools/buildman/bsettings.py +++ b/tools/buildman/bsettings.py @@ -40,7 +40,16 @@ def GetItems(section): try: return settings.items(section) except ConfigParser.NoSectionError as e: - print e return [] except: raise + +def SetItem(section, tag, value): + """Set an item and write it back to the settings file""" + global settings + global config_fname + + settings.set(section, tag, value) + if config_fname is not None: + with open(config_fname, 'w') as fd: + settings.write(fd) diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py index 7002034221..1b0ad99275 100644 --- a/tools/buildman/builder.py +++ b/tools/buildman/builder.py @@ -174,7 +174,8 @@ class Builder: self.func_sizes = func_sizes def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, - gnu_make='make', checkout=True, show_unknown=True, step=1): + gnu_make='make', checkout=True, show_unknown=True, step=1, + no_subdirs=False, full_path=False, verbose_build=False): """Create a new Builder object Args: @@ -188,6 +189,11 @@ class Builder: This is used for testing. show_unknown: Show unknown boards (those not built) in summary step: 1 to process every commit, n to process every nth commit + no_subdirs: Don't create subdirectories when building current + source for a single board + full_path: Return the full path in CROSS_COMPILE and don't set + PATH + verbose_build: Run build with V=1 and don't use 'make -s' """ self.toolchains = toolchains self.base_dir = base_dir @@ -213,6 +219,9 @@ class Builder: self._step = step self.in_tree = False self._error_lines = 0 + self.no_subdirs = no_subdirs + self.full_path = full_path + self.verbose_build = verbose_build self.col = terminal.Color() @@ -392,15 +401,17 @@ class Builder: Args: commit_upto: Commit number to use (0..self.count-1) """ + commit_dir = None if self.commits: commit = self.commits[commit_upto] subject = commit.subject.translate(trans_valid_chars) commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, self.commit_count, commit.hash, subject[:20])) - else: + elif not self.no_subdirs: commit_dir = 'current' - output_dir = os.path.join(self.base_dir, commit_dir) - return output_dir + if not commit_dir: + return self.base_dir + return os.path.join(self.base_dir, commit_dir) def GetBuildDir(self, commit_upto, target): """Get the name of the build directory for a commit number @@ -1115,6 +1126,8 @@ class Builder: create. Having left over directories is confusing when the user wants to check the output manually. """ + if not self.commits: + return dir_list = [] for commit_upto in range(self.commit_count): dir_list.append(self._GetOutputDir(commit_upto)) diff --git a/tools/buildman/builderthread.py b/tools/buildman/builderthread.py index bc4541cb3e..efb62f16d7 100644 --- a/tools/buildman/builderthread.py +++ b/tools/buildman/builderthread.py @@ -177,7 +177,7 @@ class BuilderThread(threading.Thread): commit = 'current' # Set up the environment and command line - env = self.toolchain.MakeEnvironment() + env = self.toolchain.MakeEnvironment(self.builder.full_path) Mkdir(out_dir) args = [] cwd = work_dir @@ -197,7 +197,8 @@ class BuilderThread(threading.Thread): src_dir = os.getcwd() else: args.append('O=build') - args.append('-s') + if not self.builder.verbose_build: + args.append('-s') if self.builder.num_jobs is not None: args.extend(['-j', str(self.builder.num_jobs)]) config_args = ['%s_defconfig' % brd.target] @@ -284,7 +285,7 @@ class BuilderThread(threading.Thread): print >>fd, 'path', result.toolchain.path # Write out the image and function size information and an objdump - env = result.toolchain.MakeEnvironment() + env = result.toolchain.MakeEnvironment(self.builder.full_path) lines = [] for fname in ['u-boot', 'spl/u-boot-spl']: cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname] diff --git a/tools/buildman/cmdline.py b/tools/buildman/cmdline.py index 27d3c708e6..e8a6dadd1c 100644 --- a/tools/buildman/cmdline.py +++ b/tools/buildman/cmdline.py @@ -36,6 +36,10 @@ def ParseArgs(): parser.add_option('-F', '--force-build-failures', dest='force_build_failures', action='store_true', default=False, help='Force build of previously-failed build') + parser.add_option('--fetch-arch', type='string', + help="Fetch a toolchain for architecture FETCH_ARCH ('list' to list)." + ' You can also fetch several toolchains separate by comma, or' + " 'all' to download all") parser.add_option('-g', '--git', type='string', help='Git repo containing branch to build', default='.') parser.add_option('-G', '--config-file', type='string', @@ -55,11 +59,15 @@ def ParseArgs(): help='List available tool chains') parser.add_option('-n', '--dry-run', action='store_true', dest='dry_run', default=False, help="Do a dry run (describe actions, but do nothing)") + parser.add_option('-N', '--no-subdirs', action='store_true', dest='no_subdirs', + default=False, help="Don't create subdirectories when building current source for a single board") parser.add_option('-o', '--output-dir', type='string', dest='output_dir', default='..', help='Directory where all builds happen and buildman has its workspace (default is ../)') parser.add_option('-Q', '--quick', action='store_true', default=False, help='Do a rough build, with limited warning resolution') + parser.add_option('-p', '--full-path', action='store_true', + default=False, help="Use full toolchain path in CROSS_COMPILE") parser.add_option('-s', '--summary', action='store_true', default=False, help='Show a build summary') parser.add_option('-S', '--show-sizes', action='store_true', @@ -74,6 +82,8 @@ def ParseArgs(): default=False, help='Show boards with unknown build result') parser.add_option('-v', '--verbose', action='store_true', default=False, help='Show build results while the build progresses') + parser.add_option('-V', '--verbose-build', action='store_true', + default=False, help='Run make with V=1, showing all output') parser.add_option('-x', '--exclude', dest='exclude', type='string', action='append', help='Specify a list of boards to exclude, separated by comma') diff --git a/tools/buildman/control.py b/tools/buildman/control.py index fb79a1ecfe..720b978b23 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -118,21 +118,45 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, print return 0 + if options.fetch_arch: + if options.fetch_arch == 'list': + sorted_list = toolchains.ListArchs() + print 'Available architectures: %s\n' % ' '.join(sorted_list) + return 0 + else: + fetch_arch = options.fetch_arch + if fetch_arch == 'all': + fetch_arch = ','.join(toolchains.ListArchs()) + print 'Downloading toolchains: %s\n' % fetch_arch + for arch in fetch_arch.split(','): + ret = toolchains.FetchAndInstall(arch) + if ret: + return ret + return 0 + # Work out how many commits to build. We want to build everything on the # branch. We also build the upstream commit as a control so we can see # problems introduced by the first commit on the branch. col = terminal.Color() count = options.count + has_range = options.branch and '..' in options.branch if count == -1: if not options.branch: count = 1 else: - count = gitutil.CountCommitsInBranch(options.git_dir, - options.branch) + if has_range: + count, msg = gitutil.CountCommitsInRange(options.git_dir, + options.branch) + else: + count, msg = gitutil.CountCommitsInBranch(options.git_dir, + options.branch) if count is None: - str = ("Branch '%s' not found or has no upstream" % - options.branch) - sys.exit(col.Color(col.RED, str)) + sys.exit(col.Color(col.RED, msg)) + elif count == 0: + sys.exit(col.Color(col.RED, "Range '%s' has no commits" % + options.branch)) + if msg: + print col.Color(col.YELLOW, msg) count += 1 # Build upstream commit also if not count: @@ -172,8 +196,11 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, # to overwrite earlier ones by setting allow_overwrite=True if options.branch: if count == -1: - range_expr = gitutil.GetRangeInBranch(options.git_dir, - options.branch) + if has_range: + range_expr = options.branch + else: + range_expr = gitutil.GetRangeInBranch(options.git_dir, + options.branch) upstream_commit = gitutil.GetUpstream(options.git_dir, options.branch) series = patchstream.GetMetaDataForList(upstream_commit, @@ -207,17 +234,22 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, if not gnu_make: sys.exit('GNU Make not found') - # Create a new builder with the selected options + # Create a new builder with the selected options. + output_dir = options.output_dir if options.branch: dirname = options.branch.replace('/', '_') - else: - dirname = 'current' - output_dir = os.path.join(options.output_dir, dirname) - if clean_dir and os.path.exists(output_dir): + # As a special case allow the board directory to be placed in the + # output directory itself rather than any subdirectory. + if not options.no_subdirs: + output_dir = os.path.join(options.output_dir, dirname) + if (clean_dir and output_dir != options.output_dir and + os.path.exists(output_dir)): shutil.rmtree(output_dir) builder = Builder(toolchains, output_dir, options.git_dir, options.threads, options.jobs, gnu_make=gnu_make, checkout=True, - show_unknown=options.show_unknown, step=options.step) + show_unknown=options.show_unknown, step=options.step, + no_subdirs=options.no_subdirs, full_path=options.full_path, + verbose_build=options.verbose_build) builder.force_config_on_failure = not options.quick if make_func: builder.do_make = make_func diff --git a/tools/buildman/test.py b/tools/buildman/test.py index a2a85ac9ce..c0ad5d027d 100644 --- a/tools/buildman/test.py +++ b/tools/buildman/test.py @@ -24,6 +24,16 @@ import commit import terminal import toolchain +settings_data = ''' +# Buildman settings file + +[toolchain] +main: /usr/sbin + +[toolchain-alias] +x86: i386 x86_64 +''' + errors = [ '''main.c: In function 'main_loop': main.c:260:6: warning: unused variable 'joe' [-Wunused-variable] @@ -83,6 +93,8 @@ boards = [ ['Active', 'sandbox', 'sandbox', '', 'Tester', 'Sandbox board', 'board4', ''], ] +BASE_DIR = 'base' + class Options: """Class that holds build options""" pass @@ -111,8 +123,11 @@ class TestBuild(unittest.TestCase): self.boards.AddBoard(board.Board(*brd)) self.boards.SelectBoards([]) + # Add some test settings + bsettings.Setup(None) + bsettings.AddFile(settings_data) + # Set up the toolchains - bsettings.Setup() self.toolchains = toolchain.Toolchains() self.toolchains.Add('arm-linux-gcc', test=False) self.toolchains.Add('sparc-linux-gcc', test=False) @@ -341,6 +356,64 @@ class TestBuild(unittest.TestCase): self.assertEqual(self.boards.SelectBoards(['sandbox sandbox', 'sandbox']), {'all': 1, 'sandbox': 1}) + def CheckDirs(self, build, dirname): + self.assertEqual('base%s' % dirname, build._GetOutputDir(1)) + self.assertEqual('base%s/fred' % dirname, + build.GetBuildDir(1, 'fred')) + self.assertEqual('base%s/fred/done' % dirname, + build.GetDoneFile(1, 'fred')) + self.assertEqual('base%s/fred/u-boot.sizes' % dirname, + build.GetFuncSizesFile(1, 'fred', 'u-boot')) + self.assertEqual('base%s/fred/u-boot.objdump' % dirname, + build.GetObjdumpFile(1, 'fred', 'u-boot')) + self.assertEqual('base%s/fred/err' % dirname, + build.GetErrFile(1, 'fred')) + + def testOutputDir(self): + build = builder.Builder(self.toolchains, BASE_DIR, None, 1, 2, + checkout=False, show_unknown=False) + build.commits = self.commits + build.commit_count = len(self.commits) + subject = self.commits[1].subject.translate(builder.trans_valid_chars) + dirname ='/%02d_of_%02d_g%s_%s' % (2, build.commit_count, commits[1][0], + subject[:20]) + self.CheckDirs(build, dirname) + + def testOutputDirCurrent(self): + build = builder.Builder(self.toolchains, BASE_DIR, None, 1, 2, + checkout=False, show_unknown=False) + build.commits = None + build.commit_count = 0 + self.CheckDirs(build, '/current') + + def testOutputDirNoSubdirs(self): + build = builder.Builder(self.toolchains, BASE_DIR, None, 1, 2, + checkout=False, show_unknown=False, + no_subdirs=True) + build.commits = None + build.commit_count = 0 + self.CheckDirs(build, '') + + def testToolchainAliases(self): + self.assertTrue(self.toolchains.Select('arm') != None) + with self.assertRaises(ValueError): + self.toolchains.Select('no-arch') + with self.assertRaises(ValueError): + self.toolchains.Select('x86') + + self.toolchains = toolchain.Toolchains() + self.toolchains.Add('x86_64-linux-gcc', test=False) + self.assertTrue(self.toolchains.Select('x86') != None) + + self.toolchains = toolchain.Toolchains() + self.toolchains.Add('i386-linux-gcc', test=False) + self.assertTrue(self.toolchains.Select('x86') != None) + + def testToolchainDownload(self): + """Test that we can download toolchains""" + self.assertEqual('https://www.kernel.org/pub/tools/crosstool/files/bin/x86_64/4.6.3/x86_64-gcc-4.6.3-nolibc_arm-unknown-linux-gnueabi.tar.xz', + self.toolchains.LocateArchUrl('arm')) + if __name__ == "__main__": unittest.main() diff --git a/tools/buildman/toolchain.py b/tools/buildman/toolchain.py index 27dc31889b..d4c5d4a11e 100644 --- a/tools/buildman/toolchain.py +++ b/tools/buildman/toolchain.py @@ -5,11 +5,42 @@ import re import glob +from HTMLParser import HTMLParser import os +import sys +import tempfile +import urllib2 import bsettings import command +# Simple class to collect links from a page +class MyHTMLParser(HTMLParser): + def __init__(self, arch): + """Create a new parser + + After the parser runs, self.links will be set to a list of the links + to .xz archives found in the page, and self.arch_link will be set to + the one for the given architecture (or None if not found). + + Args: + arch: Architecture to search for + """ + HTMLParser.__init__(self) + self.arch_link = None + self.links = [] + self._match = '_%s-' % arch + + def handle_starttag(self, tag, attrs): + if tag == 'a': + for tag, value in attrs: + if tag == 'href': + if value and value.endswith('.xz'): + self.links.append(value) + if self._match in value: + self.arch_link = value + + class Toolchain: """A single toolchain @@ -20,7 +51,6 @@ class Toolchain: arch: Architecture of toolchain as determined from the first component of the filename. E.g. arm-linux-gcc becomes arm """ - def __init__(self, fname, test, verbose=False): """Create a new toolchain object. @@ -30,11 +60,18 @@ class Toolchain: """ self.gcc = fname self.path = os.path.dirname(fname) - self.cross = os.path.basename(fname)[:-3] + + # Find the CROSS_COMPILE prefix to use for U-Boot. For example, + # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'. + basename = os.path.basename(fname) + pos = basename.rfind('-') + self.cross = basename[:pos + 1] if pos != -1 else '' + + # The architecture is the first part of the name pos = self.cross.find('-') self.arch = self.cross[:pos] if pos != -1 else 'sandbox' - env = self.MakeEnvironment() + env = self.MakeEnvironment(False) # As a basic sanity check, run the C compiler with --version cmd = [fname, '--version'] @@ -74,15 +111,23 @@ class Toolchain: return prio return prio - def MakeEnvironment(self): + def MakeEnvironment(self, full_path): """Returns an environment for using the toolchain. - Thie takes the current environment, adds CROSS_COMPILE and - augments PATH so that the toolchain will operate correctly. + Thie takes the current environment and adds CROSS_COMPILE so that + the tool chain will operate correctly. + + Args: + full_path: Return the full path in CROSS_COMPILE and don't set + PATH """ env = dict(os.environ) - env['CROSS_COMPILE'] = self.cross - env['PATH'] += (':' + self.path) + if full_path: + env['CROSS_COMPILE'] = os.path.join(self.path, self.cross) + else: + env['CROSS_COMPILE'] = self.cross + env['PATH'] = self.path + ':' + env['PATH'] + return env @@ -101,18 +146,29 @@ class Toolchains: self.paths = [] self._make_flags = dict(bsettings.GetItems('make-flags')) - def GetSettings(self): + def GetPathList(self): + """Get a list of available toolchain paths + + Returns: + List of strings, each a path to a toolchain mentioned in the + [toolchain] section of the settings file. + """ toolchains = bsettings.GetItems('toolchain') if not toolchains: print ("Warning: No tool chains - please add a [toolchain] section" " to your buildman config file %s. See README for details" % bsettings.config_fname) + paths = [] for name, value in toolchains: if '*' in value: - self.paths += glob.glob(value) + paths += glob.glob(value) else: - self.paths.append(value) + paths.append(value) + return paths + + def GetSettings(self): + self.paths += self.GetPathList() def Add(self, fname, test=True, verbose=False): """Add a toolchain to our list @@ -132,6 +188,24 @@ class Toolchains: if add_it: self.toolchains[toolchain.arch] = toolchain + def ScanPath(self, path, verbose): + """Scan a path for a valid toolchain + + Args: + path: Path to scan + verbose: True to print out progress information + Returns: + Filename of C compiler if found, else None + """ + for subdir in ['.', 'bin', 'usr/bin']: + dirname = os.path.join(path, subdir) + if verbose: print " - looking in '%s'" % dirname + for fname in glob.glob(dirname + '/*gcc'): + if verbose: print " - found '%s'" % fname + return fname + return None + + def Scan(self, verbose): """Scan for available toolchains and select the best for each arch. @@ -145,12 +219,9 @@ class Toolchains: if verbose: print 'Scanning for tool chains' for path in self.paths: if verbose: print " - scanning path '%s'" % path - for subdir in ['.', 'bin', 'usr/bin']: - dirname = os.path.join(path, subdir) - if verbose: print " - looking in '%s'" % dirname - for fname in glob.glob(dirname + '/*gcc'): - if verbose: print " - found '%s'" % fname - self.Add(fname, True, verbose) + fname = self.ScanPath(path, verbose) + if fname: + self.Add(fname, True, verbose) def List(self): """List out the selected toolchains for each architecture""" @@ -170,9 +241,11 @@ class Toolchains: returns: toolchain object, or None if none found """ - for name, value in bsettings.GetItems('toolchain-alias'): - if arch == name: - arch = value + for tag, value in bsettings.GetItems('toolchain-alias'): + if arch == tag: + for alias in value.split(): + if alias in self.toolchains: + return self.toolchains[alias] if not arch in self.toolchains: raise ValueError, ("No tool chain found for arch '%s'" % arch) @@ -247,3 +320,160 @@ class Toolchains: else: i += 1 return args + + def LocateArchUrl(self, fetch_arch): + """Find a toolchain available online + + Look in standard places for available toolchains. At present the + only standard place is at kernel.org. + + Args: + arch: Architecture to look for, or 'list' for all + Returns: + If fetch_arch is 'list', a tuple: + Machine architecture (e.g. x86_64) + List of toolchains + else + URL containing this toolchain, if avaialble, else None + """ + arch = command.OutputOneLine('uname', '-m') + base = 'https://www.kernel.org/pub/tools/crosstool/files/bin' + versions = ['4.6.3', '4.6.2', '4.5.1', '4.2.4'] + links = [] + for version in versions: + url = '%s/%s/%s/' % (base, arch, version) + print 'Checking: %s' % url + response = urllib2.urlopen(url) + html = response.read() + parser = MyHTMLParser(fetch_arch) + parser.feed(html) + if fetch_arch == 'list': + links += parser.links + elif parser.arch_link: + return url + parser.arch_link + if fetch_arch == 'list': + return arch, links + return None + + def Download(self, url): + """Download a file to a temporary directory + + Args: + url: URL to download + Returns: + Tuple: + Temporary directory name + Full path to the downloaded archive file in that directory, + or None if there was an error while downloading + """ + print "Downloading: %s" % url + leaf = url.split('/')[-1] + tmpdir = tempfile.mkdtemp('.buildman') + response = urllib2.urlopen(url) + fname = os.path.join(tmpdir, leaf) + fd = open(fname, 'wb') + meta = response.info() + size = int(meta.getheaders("Content-Length")[0]) + done = 0 + block_size = 1 << 16 + status = '' + + # Read the file in chunks and show progress as we go + while True: + buffer = response.read(block_size) + if not buffer: + print chr(8) * (len(status) + 1), '\r', + break + + done += len(buffer) + fd.write(buffer) + status = r"%10d MiB [%3d%%]" % (done / 1024 / 1024, + done * 100 / size) + status = status + chr(8) * (len(status) + 1) + print status, + sys.stdout.flush() + fd.close() + if done != size: + print 'Error, failed to download' + os.remove(fname) + fname = None + return tmpdir, fname + + def Unpack(self, fname, dest): + """Unpack a tar file + + Args: + fname: Filename to unpack + dest: Destination directory + Returns: + Directory name of the first entry in the archive, without the + trailing / + """ + stdout = command.Output('tar', 'xvfJ', fname, '-C', dest) + return stdout.splitlines()[0][:-1] + + def TestSettingsHasPath(self, path): + """Check if builmand will find this toolchain + + Returns: + True if the path is in settings, False if not + """ + paths = self.GetPathList() + return path in paths + + def ListArchs(self): + """List architectures with available toolchains to download""" + host_arch, archives = self.LocateArchUrl('list') + re_arch = re.compile('[-a-z0-9.]*_([^-]*)-.*') + arch_set = set() + for archive in archives: + # Remove the host architecture from the start + arch = re_arch.match(archive[len(host_arch):]) + if arch: + arch_set.add(arch.group(1)) + return sorted(arch_set) + + def FetchAndInstall(self, arch): + """Fetch and install a new toolchain + + arch: + Architecture to fetch, or 'list' to list + """ + # Fist get the URL for this architecture + url = self.LocateArchUrl(arch) + if not url: + print ("Cannot find toolchain for arch '%s' - use 'list' to list" % + arch) + return 2 + home = os.environ['HOME'] + dest = os.path.join(home, '.buildman-toolchains') + if not os.path.exists(dest): + os.mkdir(dest) + + # Download the tar file for this toolchain and unpack it + tmpdir, tarfile = self.Download(url) + if not tarfile: + return 1 + print 'Unpacking to: %s' % dest, + sys.stdout.flush() + path = self.Unpack(tarfile, dest) + os.remove(tarfile) + os.rmdir(tmpdir) + print + + # Check that the toolchain works + print 'Testing' + dirpath = os.path.join(dest, path) + compiler_fname = self.ScanPath(dirpath, True) + if not compiler_fname: + print 'Could not locate C compiler - fetch failed.' + return 1 + toolchain = Toolchain(compiler_fname, True, True) + + # Make sure that it will be found by buildman + if not self.TestSettingsHasPath(dirpath): + print ("Adding 'download' to config file '%s'" % + bsettings.config_fname) + tools_dir = os.path.dirname(dirpath) + bsettings.SetItem('toolchain', 'download', '%s/*' % tools_dir) + return 0 |
