diff options
Diffstat (limited to 'tools/genboardscfg.py')
| -rwxr-xr-x | tools/genboardscfg.py | 301 |
1 files changed, 210 insertions, 91 deletions
diff --git a/tools/genboardscfg.py b/tools/genboardscfg.py index 734d90b5e5..e6870f5bba 100755 --- a/tools/genboardscfg.py +++ b/tools/genboardscfg.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # # Author: Masahiro Yamada <yamada.m@jp.panasonic.com> # @@ -11,6 +11,8 @@ Converter from Kconfig and MAINTAINERS to boards.cfg Run 'tools/genboardscfg.py' to create boards.cfg file. Run 'tools/genboardscfg.py -h' for available options. + +This script only works on python 2.6 or later, but not python 3.x. """ import errno @@ -30,13 +32,13 @@ CONFIG_DIR = 'configs' REFORMAT_CMD = [os.path.join('tools', 'reformat.py'), '-i', '-d', '-', '-s', '8'] SHOW_GNU_MAKE = 'scripts/show-gnu-make' -SLEEP_TIME=0.03 +SLEEP_TIME=0.003 COMMENT_BLOCK = '''# # List of boards # Automatically generated by %s: don't edit # -# Status, Arch, CPU(:SPLCPU), SoC, Vendor, Board, Target, Options, Maintainers +# Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers ''' % __file__ @@ -58,8 +60,6 @@ def get_terminal_columns(): try: ret = fcntl.ioctl(sys.stdout.fileno(), termios.TIOCGWINSZ, arg) except IOError as exception: - if exception.errno != errno.ENOTTY: - raise # If 'Inappropriate ioctl for device' error occurs, # stdout is probably redirected. Return 0. return 0 @@ -77,18 +77,62 @@ def check_top_directory(): """Exit if we are not at the top of source directory.""" for f in ('README', 'Licenses'): if not os.path.exists(f): - print >> sys.stderr, 'Please run at the top of source directory.' - sys.exit(1) + sys.exit('Please run at the top of source directory.') def get_make_cmd(): """Get the command name of GNU Make.""" process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) ret = process.communicate() if process.returncode: - print >> sys.stderr, 'GNU Make not found' - sys.exit(1) + sys.exit('GNU Make not found') return ret[0].rstrip() +def output_is_new(): + """Check if the boards.cfg file is up to date. + + Returns: + True if the boards.cfg file exists and is newer than any of + *_defconfig, MAINTAINERS and Kconfig*. False otherwise. + """ + try: + ctime = os.path.getctime(BOARD_FILE) + except OSError as exception: + if exception.errno == errno.ENOENT: + # return False on 'No such file or directory' error + return False + else: + raise + + for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR): + for filename in fnmatch.filter(filenames, '*_defconfig'): + if fnmatch.fnmatch(filename, '.*'): + continue + filepath = os.path.join(dirpath, filename) + if ctime < os.path.getctime(filepath): + return False + + for (dirpath, dirnames, filenames) in os.walk('.'): + for filename in filenames: + if (fnmatch.fnmatch(filename, '*~') or + not fnmatch.fnmatch(filename, 'Kconfig*') and + not filename == 'MAINTAINERS'): + continue + filepath = os.path.join(dirpath, filename) + if ctime < os.path.getctime(filepath): + return False + + # Detect a board that has been removed since the current boards.cfg + # was generated + with open(BOARD_FILE) as f: + for line in f: + if line[0] == '#' or line == '\n': + continue + defconfig = line.split()[6] + '_defconfig' + if not os.path.exists(os.path.join(CONFIG_DIR, defconfig)): + return False + + return True + ### classes ### class MaintainersDatabase: @@ -104,13 +148,19 @@ class MaintainersDatabase: Returns: Either 'Active' or 'Orphan' """ + if not target in self.database: + print >> sys.stderr, "WARNING: no status info for '%s'" % target + return '-' + tmp = self.database[target][0] if tmp.startswith('Maintained'): return 'Active' elif tmp.startswith('Orphan'): return 'Orphan' else: - print >> sys.stderr, 'Error: %s: unknown status' % tmp + print >> sys.stderr, ("WARNING: %s: unknown status for '%s'" % + (tmp, target)) + return '-' def get_maintainers(self, target): """Return the maintainers of the given board. @@ -118,6 +168,10 @@ class MaintainersDatabase: If the board has two or more maintainers, they are separated with colons. """ + if not target in self.database: + print >> sys.stderr, "WARNING: no maintainers for '%s'" % target + return '' + return ':'.join(self.database[target][1]) def parse_file(self, file): @@ -146,7 +200,7 @@ class MaintainersDatabase: targets.append(front) elif tag == 'S:': status = rest - elif line == '\n' and targets: + elif line == '\n': for target in targets: self.database[target] = (status, maintainers) targets = [] @@ -209,16 +263,15 @@ class DotConfigParser: # sanity check of '.config' file for field in self.must_fields: if not field in fields: - print >> sys.stderr, 'Error: %s is not defined in %s' % \ - (field, defconfig) - sys.exit(1) + print >> sys.stderr, ( + "WARNING: '%s' is not defined in '%s'. Skip." % + (field, defconfig)) + return - # fix-up for aarch64 and tegra + # fix-up for aarch64 if fields['arch'] == 'arm' and 'cpu' in fields: if fields['cpu'] == 'armv8': fields['arch'] = 'aarch64' - if 'soc' in fields and re.match('tegra[0-9]*$', fields['soc']): - fields['cpu'] += ':arm720t' target, match, rear = defconfig.partition('_defconfig') assert match and not rear, \ @@ -261,16 +314,26 @@ class Slot: Arguments: output: File object which the result is written to maintainers_database: An instance of class MaintainersDatabase + devnull: file object of 'dev/null' + make_cmd: the command name of Make """ - self.occupied = False self.build_dir = tempfile.mkdtemp() self.devnull = devnull - self.make_cmd = make_cmd + self.ps = subprocess.Popen([make_cmd, 'O=' + self.build_dir, + 'allnoconfig'], stdout=devnull) + self.occupied = True self.parser = DotConfigParser(self.build_dir, output, maintainers_database) + self.env = os.environ.copy() + self.env['srctree'] = os.getcwd() + self.env['UBOOTVERSION'] = 'dummy' + self.env['KCONFIG_OBJDIR'] = '' def __del__(self): """Delete the working directory""" + if not self.occupied: + while self.ps.poll() == None: + pass shutil.rmtree(self.build_dir) def add(self, defconfig): @@ -287,13 +350,31 @@ class Slot: """ if self.occupied: return False - o = 'O=' + self.build_dir - self.ps = subprocess.Popen([self.make_cmd, o, defconfig], - stdout=self.devnull) + + with open(os.path.join(self.build_dir, '.tmp_defconfig'), 'w') as f: + for line in open(os.path.join(CONFIG_DIR, defconfig)): + colon = line.find(':CONFIG_') + if colon == -1: + f.write(line) + else: + f.write(line[colon + 1:]) + + self.ps = subprocess.Popen([os.path.join('scripts', 'kconfig', 'conf'), + '--defconfig=.tmp_defconfig', 'Kconfig'], + stdout=self.devnull, + cwd=self.build_dir, + env=self.env) + self.defconfig = defconfig self.occupied = True return True + def wait(self): + """Wait until the current subprocess finishes.""" + while self.occupied and self.ps.poll() == None: + time.sleep(SLEEP_TIME) + self.occupied = False + def poll(self): """Check if the subprocess is running and invoke the .config parser if the subprocess is terminated. @@ -305,7 +386,11 @@ class Slot: return True if self.ps.poll() == None: return False - self.parser.parse(self.defconfig) + if self.ps.poll() == 0: + self.parser.parse(self.defconfig) + else: + print >> sys.stderr, ("WARNING: failed to process '%s'. skip." % + self.defconfig) self.occupied = False return True @@ -327,6 +412,8 @@ class Slots: for i in range(jobs): self.slots.append(Slot(output, maintainers_database, devnull, make_cmd)) + for slot in self.slots: + slot.wait() def add(self, defconfig): """Add a new subprocess if a vacant slot is available. @@ -401,64 +488,97 @@ class Indicator: sys.stdout.write('\r' + msg) sys.stdout.flush() -def __gen_boards_cfg(jobs): - """Generate boards.cfg file. +class BoardsFileGenerator: - Arguments: - jobs: The number of jobs to run simultaneously + """Generator of boards.cfg.""" - Note: - The incomplete boards.cfg is left over when an error (including - the termination by the keyboard interrupt) occurs on the halfway. - """ - check_top_directory() - print 'Generating %s ... (jobs: %d)' % (BOARD_FILE, jobs) + def __init__(self): + """Prepare basic things for generating boards.cfg.""" + # All the defconfig files to be processed + defconfigs = [] + for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR): + dirpath = dirpath[len(CONFIG_DIR) + 1:] + for filename in fnmatch.filter(filenames, '*_defconfig'): + if fnmatch.fnmatch(filename, '.*'): + continue + defconfigs.append(os.path.join(dirpath, filename)) + self.defconfigs = defconfigs + self.indicator = Indicator(len(defconfigs)) + + # Parse all the MAINTAINERS files + maintainers_database = MaintainersDatabase() + for (dirpath, dirnames, filenames) in os.walk('.'): + if 'MAINTAINERS' in filenames: + maintainers_database.parse_file(os.path.join(dirpath, + 'MAINTAINERS')) + self.maintainers_database = maintainers_database - # All the defconfig files to be processed - defconfigs = [] - for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR): - dirpath = dirpath[len(CONFIG_DIR) + 1:] - for filename in fnmatch.filter(filenames, '*_defconfig'): - defconfigs.append(os.path.join(dirpath, filename)) + def __del__(self): + """Delete the incomplete boards.cfg - # Parse all the MAINTAINERS files - maintainers_database = MaintainersDatabase() - for (dirpath, dirnames, filenames) in os.walk('.'): - if 'MAINTAINERS' in filenames: - maintainers_database.parse_file(os.path.join(dirpath, - 'MAINTAINERS')) - - # Output lines should be piped into the reformat tool - reformat_process = subprocess.Popen(REFORMAT_CMD, stdin=subprocess.PIPE, - stdout=open(BOARD_FILE, 'w')) - pipe = reformat_process.stdin - pipe.write(COMMENT_BLOCK) - - indicator = Indicator(len(defconfigs)) - slots = Slots(jobs, pipe, maintainers_database) - - # Main loop to process defconfig files: - # Add a new subprocess into a vacant slot. - # Sleep if there is no available slot. - for defconfig in defconfigs: - while not slots.add(defconfig): - while not slots.available(): - # No available slot: sleep for a while - time.sleep(SLEEP_TIME) - indicator.inc() - - # wait until all the subprocesses finish - while not slots.empty(): - time.sleep(SLEEP_TIME) - print '' - - # wait until the reformat tool finishes - reformat_process.communicate() - if reformat_process.returncode != 0: - print >> sys.stderr, '"%s" failed' % REFORMAT_CMD[0] - sys.exit(1) - -def gen_boards_cfg(jobs): + This destructor deletes boards.cfg if the private member 'in_progress' + is defined as True. The 'in_progress' member is set to True at the + beginning of the generate() method and set to False at its end. + So, in_progress==True means generating boards.cfg was terminated + on the way. + """ + + if hasattr(self, 'in_progress') and self.in_progress: + try: + os.remove(BOARD_FILE) + except OSError as exception: + # Ignore 'No such file or directory' error + if exception.errno != errno.ENOENT: + raise + print 'Removed incomplete %s' % BOARD_FILE + + def generate(self, jobs): + """Generate boards.cfg + + This method sets the 'in_progress' member to True at the beginning + and sets it to False on success. The boards.cfg should not be + touched before/after this method because 'in_progress' is used + to detect the incomplete boards.cfg. + + Arguments: + jobs: The number of jobs to run simultaneously + """ + + self.in_progress = True + print 'Generating %s ... (jobs: %d)' % (BOARD_FILE, jobs) + + # Output lines should be piped into the reformat tool + reformat_process = subprocess.Popen(REFORMAT_CMD, + stdin=subprocess.PIPE, + stdout=open(BOARD_FILE, 'w')) + pipe = reformat_process.stdin + pipe.write(COMMENT_BLOCK) + + slots = Slots(jobs, pipe, self.maintainers_database) + + # Main loop to process defconfig files: + # Add a new subprocess into a vacant slot. + # Sleep if there is no available slot. + for defconfig in self.defconfigs: + while not slots.add(defconfig): + while not slots.available(): + # No available slot: sleep for a while + time.sleep(SLEEP_TIME) + self.indicator.inc() + + # wait until all the subprocesses finish + while not slots.empty(): + time.sleep(SLEEP_TIME) + print '' + + # wait until the reformat tool finishes + reformat_process.communicate() + if reformat_process.returncode != 0: + sys.exit('"%s" failed' % REFORMAT_CMD[0]) + + self.in_progress = False + +def gen_boards_cfg(jobs=1, force=False): """Generate boards.cfg file. The incomplete boards.cfg is deleted if an error (including @@ -467,30 +587,28 @@ def gen_boards_cfg(jobs): Arguments: jobs: The number of jobs to run simultaneously """ - try: - __gen_boards_cfg(jobs) - except: - # We should remove incomplete boards.cfg - try: - os.remove(BOARD_FILE) - except OSError as exception: - # Ignore 'No such file or directory' error - if exception.errno != errno.ENOENT: - raise - raise + check_top_directory() + if not force and output_is_new(): + print "%s is up to date. Nothing to do." % BOARD_FILE + sys.exit(0) + + generator = BoardsFileGenerator() + generator.generate(jobs) def main(): parser = optparse.OptionParser() # Add options here parser.add_option('-j', '--jobs', help='the number of jobs to run simultaneously') + parser.add_option('-f', '--force', action="store_true", default=False, + help='regenerate the output even if it is new') (options, args) = parser.parse_args() + if options.jobs: try: jobs = int(options.jobs) except ValueError: - print >> sys.stderr, 'Option -j (--jobs) takes a number' - sys.exit(1) + sys.exit('Option -j (--jobs) takes a number') else: try: jobs = int(subprocess.Popen(['getconf', '_NPROCESSORS_ONLN'], @@ -498,7 +616,8 @@ def main(): except (OSError, ValueError): print 'info: failed to get the number of CPUs. Set jobs to 1' jobs = 1 - gen_boards_cfg(jobs) + + gen_boards_cfg(jobs, force=options.force) if __name__ == '__main__': main() |
