diff options
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/Makefile | 2 | ||||
| -rw-r--r-- | tools/buildman/board.py | 2 | ||||
| -rw-r--r-- | tools/buildman/builder.py | 36 | ||||
| -rwxr-xr-x | tools/buildman/buildman.py | 6 | ||||
| -rw-r--r-- | tools/buildman/control.py | 21 | ||||
| -rw-r--r-- | tools/env/Makefile | 2 | ||||
| -rwxr-xr-x | tools/genboardscfg.py | 504 | ||||
| -rw-r--r-- | tools/patman/gitutil.py | 11 |
8 files changed, 569 insertions, 15 deletions
diff --git a/tools/Makefile b/tools/Makefile index 61b2048..90e966d 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -193,7 +193,7 @@ endif # !LOGO_BMP # Define _GNU_SOURCE to obtain the getline prototype from stdio.h # HOST_EXTRACFLAGS += -include $(srctree)/include/libfdt_env.h \ - $(patsubst -I%,-idirafter%, $(UBOOTINCLUDE)) \ + $(patsubst -I%,-idirafter%, $(filter -I%, $(UBOOTINCLUDE))) \ -I$(srctree)/lib/libfdt \ -I$(srctree)/tools \ -DCONFIG_SYS_TEXT_BASE=$(CONFIG_SYS_TEXT_BASE) \ diff --git a/tools/buildman/board.py b/tools/buildman/board.py index 5172a47..7bcc932 100644 --- a/tools/buildman/board.py +++ b/tools/buildman/board.py @@ -17,7 +17,7 @@ class Board: soc: Name of SOC, or '' if none (e.g. mx31) vendor: Name of vendor (e.g. armltd) board_name: Name of board (e.g. integrator) - target: Target name (use make <target>_config to configure) + target: Target name (use make <target>_defconfig to configure) options: board-specific options (e.g. integratorcp:CM1136) """ self.target = target diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py index 39a6e8a..48408ff 100644 --- a/tools/buildman/builder.py +++ b/tools/buildman/builder.py @@ -199,7 +199,7 @@ class BuilderThread(threading.Thread): commit_upto: Commit number to build (0...n-1) brd: Board object to build work_dir: Directory to which the source will be checked out - do_config: True to run a make <board>_config on the source + do_config: True to run a make <board>_defconfig on the source force_build: Force a build even if one was previously done force_build_failures: Force a bulid if the previous result showed failure @@ -213,7 +213,10 @@ class BuilderThread(threading.Thread): # self.Make() below, in the event that we do a build. result = command.CommandResult() result.return_code = 0 - out_dir = os.path.join(work_dir, 'build') + if self.builder.in_tree: + out_dir = work_dir + else: + out_dir = os.path.join(work_dir, 'build') # Check if the job was already completed last time done_file = self.builder.GetDoneFile(commit_upto, brd.target) @@ -257,10 +260,13 @@ class BuilderThread(threading.Thread): # Set up the environment and command line env = self.toolchain.MakeEnvironment() Mkdir(out_dir) - args = ['O=build', '-s'] + args = [] + if not self.builder.in_tree: + args.append('O=build') + args.append('-s') if self.builder.num_jobs is not None: args.extend(['-j', str(self.builder.num_jobs)]) - config_args = ['%s_config' % brd.target] + config_args = ['%s_defconfig' % brd.target] config_out = '' args.extend(self.builder.toolchains.GetMakeArguments(brd)) @@ -413,7 +419,7 @@ class BuilderThread(threading.Thread): work_dir = self.builder.GetThreadDir(self.thread_num) self.toolchain = None if job.commits: - # Run 'make board_config' on the first commit + # Run 'make board_defconfig' on the first commit do_config = True commit_upto = 0 force_build = False @@ -431,7 +437,8 @@ class BuilderThread(threading.Thread): result, request_config = self.RunCommit(commit_upto, brd, work_dir, True, True, False) did_config = True - do_config = request_config + if not self.builder.force_reconfig: + do_config = request_config # If we built that commit, then config is done. But if we got # an warning, reconfig next time to force it to build the same @@ -524,6 +531,15 @@ class Builder: toolchains: Toolchains object to use for building upto: Current commit number we are building (0.count-1) warned: Number of builds that produced at least one warning + force_reconfig: Reconfigure U-Boot on each comiit. This disables + incremental building, where buildman reconfigures on the first + commit for a baord, and then just does an incremental build for + the following commits. In fact buildman will reconfigure and + retry for any failing commits, so generally the only effect of + this option is to slow things down. + in_tree: Build U-Boot in-tree instead of specifying an output + directory separate from the source code. This option is really + only useful for testing in-tree builds. Private members: _base_board_dict: Last-summarised Dict of boards @@ -560,7 +576,7 @@ class Builder: self.func_sizes = func_sizes def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, - checkout=True, show_unknown=True, step=1): + gnu_make='make', checkout=True, show_unknown=True, step=1): """Create a new Builder object Args: @@ -569,6 +585,7 @@ class Builder: git_dir: Git directory containing source repository num_threads: Number of builder threads to run num_jobs: Number of jobs to run at once (passed to make as -j) + gnu_make: the command name of GNU Make. checkout: True to check out source, False to skip that step. This is used for testing. show_unknown: Show unknown boards (those not built) in summary @@ -580,6 +597,7 @@ class Builder: self.threads = [] self.active = True self.do_make = self.Make + self.gnu_make = gnu_make self.checkout = checkout self.num_threads = num_threads self.num_jobs = num_jobs @@ -593,7 +611,9 @@ class Builder: self._next_delay_update = datetime.now() self.force_config_on_failure = True self.force_build_failures = False + self.force_reconfig = False self._step = step + self.in_tree = False self.col = terminal.Color() @@ -682,7 +702,7 @@ class Builder: args: Arguments to pass to make kwargs: Arguments to pass to command.RunPipe() """ - cmd = ['make'] + list(args) + cmd = [self.gnu_make] + list(args) result = command.RunPipe([cmd], capture=True, capture_stderr=True, cwd=cwd, raise_on_error=False, **kwargs) return result diff --git a/tools/buildman/buildman.py b/tools/buildman/buildman.py index 0da6797..42847ac 100755 --- a/tools/buildman/buildman.py +++ b/tools/buildman/buildman.py @@ -67,6 +67,9 @@ parser.add_option('-B', '--bloat', dest='show_bloat', help='Show changes in function code size for each board') parser.add_option('-c', '--count', dest='count', type='int', default=-1, help='Run build on the top n commits') +parser.add_option('-C', '--force-reconfig', dest='force_reconfig', + action='store_true', default=False, + help='Reconfigure for every commit (disable incremental build)') parser.add_option('-e', '--show_errors', action='store_true', default=False, help='Show errors and warnings') parser.add_option('-f', '--force-build', dest='force_build', @@ -82,6 +85,9 @@ parser.add_option('-g', '--git', type='string', help='Git repo containing branch to build', default='.') parser.add_option('-H', '--full-help', action='store_true', dest='full_help', default=False, help='Display the README file') +parser.add_option('-i', '--in-tree', dest='in_tree', + action='store_true', default=False, + help='Build in the source tree instead of a separate directory') parser.add_option('-j', '--jobs', dest='jobs', type='int', default=None, help='Number of jobs to run at once (passed to make)') parser.add_option('-k', '--keep-outputs', action='store_true', diff --git a/tools/buildman/control.py b/tools/buildman/control.py index cfad535..75b6498 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -14,6 +14,8 @@ import gitutil import patchstream import terminal import toolchain +import command +import subprocess def GetPlural(count): """Returns a plural 's' if count is not 1""" @@ -108,6 +110,15 @@ def DoBuildman(options, args): sys.exit(1) # Work out what subset of the boards we are building + board_file = os.path.join(options.git, 'boards.cfg') + if not os.path.exists(board_file): + print 'Could not find %s' % board_file + status = subprocess.call([os.path.join(options.git, + 'tools/genboardscfg.py')]) + if status != 0: + print >> sys.stderr, "Failed to generate boards.cfg" + sys.exit(1) + boards = board.Boards() boards.ReadBoards(os.path.join(options.git, 'boards.cfg')) why_selected = boards.SelectBoards(args) @@ -144,10 +155,16 @@ def DoBuildman(options, args): if not options.step: options.step = len(series.commits) - 1 + gnu_make = command.Output(os.path.join(options.git, + 'scripts/show-gnu-make')).rstrip() + if not gnu_make: + print >> sys.stderr, 'GNU Make not found' + sys.exit(1) + # Create a new builder with the selected options output_dir = os.path.join(options.output_dir, options.branch) builder = Builder(toolchains, output_dir, options.git_dir, - options.threads, options.jobs, checkout=True, + options.threads, options.jobs, gnu_make=gnu_make, checkout=True, show_unknown=options.show_unknown, step=options.step) builder.force_config_on_failure = not options.quick @@ -157,6 +174,8 @@ def DoBuildman(options, args): else: builder.force_build = options.force_build builder.force_build_failures = options.force_build_failures + builder.force_reconfig = options.force_reconfig + builder.in_tree = options.in_tree # Work out which boards to build board_selected = boards.GetSelectedDict() diff --git a/tools/env/Makefile b/tools/env/Makefile index f5368bc..4927489 100644 --- a/tools/env/Makefile +++ b/tools/env/Makefile @@ -11,7 +11,7 @@ HOSTCC = $(CC) # Compile for a hosted environment on the target -HOST_EXTRACFLAGS = $(patsubst -I%,-idirafter%, $(UBOOTINCLUDE)) \ +HOST_EXTRACFLAGS = $(patsubst -I%,-idirafter%, $(filter -I%, $(UBOOTINCLUDE))) \ -idirafter $(srctree)/tools/env \ -DUSE_HOSTCC \ -DTEXT_BASE=$(TEXT_BASE) diff --git a/tools/genboardscfg.py b/tools/genboardscfg.py new file mode 100755 index 0000000..734d90b --- /dev/null +++ b/tools/genboardscfg.py @@ -0,0 +1,504 @@ +#!/usr/bin/env python +# +# Author: Masahiro Yamada <yamada.m@jp.panasonic.com> +# +# SPDX-License-Identifier: GPL-2.0+ +# + +""" +Converter from Kconfig and MAINTAINERS to boards.cfg + +Run 'tools/genboardscfg.py' to create boards.cfg file. + +Run 'tools/genboardscfg.py -h' for available options. +""" + +import errno +import fnmatch +import glob +import optparse +import os +import re +import shutil +import subprocess +import sys +import tempfile +import time + +BOARD_FILE = 'boards.cfg' +CONFIG_DIR = 'configs' +REFORMAT_CMD = [os.path.join('tools', 'reformat.py'), + '-i', '-d', '-', '-s', '8'] +SHOW_GNU_MAKE = 'scripts/show-gnu-make' +SLEEP_TIME=0.03 + +COMMENT_BLOCK = '''# +# List of boards +# Automatically generated by %s: don't edit +# +# Status, Arch, CPU(:SPLCPU), SoC, Vendor, Board, Target, Options, Maintainers + +''' % __file__ + +### helper functions ### +def get_terminal_columns(): + """Get the width of the terminal. + + Returns: + The width of the terminal, or zero if the stdout is not + associated with tty. + """ + try: + return shutil.get_terminal_size().columns # Python 3.3~ + except AttributeError: + import fcntl + import termios + import struct + arg = struct.pack('hhhh', 0, 0, 0, 0) + try: + ret = fcntl.ioctl(sys.stdout.fileno(), termios.TIOCGWINSZ, arg) + except IOError as exception: + if exception.errno != errno.ENOTTY: + raise + # If 'Inappropriate ioctl for device' error occurs, + # stdout is probably redirected. Return 0. + return 0 + return struct.unpack('hhhh', ret)[1] + +def get_devnull(): + """Get the file object of '/dev/null' device.""" + try: + devnull = subprocess.DEVNULL # py3k + except AttributeError: + devnull = open(os.devnull, 'wb') + return devnull + +def check_top_directory(): + """Exit if we are not at the top of source directory.""" + for f in ('README', 'Licenses'): + if not os.path.exists(f): + print >> sys.stderr, 'Please run at the top of source directory.' + sys.exit(1) + +def get_make_cmd(): + """Get the command name of GNU Make.""" + process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) + ret = process.communicate() + if process.returncode: + print >> sys.stderr, 'GNU Make not found' + sys.exit(1) + return ret[0].rstrip() + +### classes ### +class MaintainersDatabase: + + """The database of board status and maintainers.""" + + def __init__(self): + """Create an empty database.""" + self.database = {} + + def get_status(self, target): + """Return the status of the given board. + + Returns: + Either 'Active' or 'Orphan' + """ + tmp = self.database[target][0] + if tmp.startswith('Maintained'): + return 'Active' + elif tmp.startswith('Orphan'): + return 'Orphan' + else: + print >> sys.stderr, 'Error: %s: unknown status' % tmp + + def get_maintainers(self, target): + """Return the maintainers of the given board. + + If the board has two or more maintainers, they are separated + with colons. + """ + return ':'.join(self.database[target][1]) + + def parse_file(self, file): + """Parse the given MAINTAINERS file. + + This method parses MAINTAINERS and add board status and + maintainers information to the database. + + Arguments: + file: MAINTAINERS file to be parsed + """ + targets = [] + maintainers = [] + status = '-' + for line in open(file): + tag, rest = line[:2], line[2:].strip() + if tag == 'M:': + maintainers.append(rest) + elif tag == 'F:': + # expand wildcard and filter by 'configs/*_defconfig' + for f in glob.glob(rest): + front, match, rear = f.partition('configs/') + if not front and match: + front, match, rear = rear.rpartition('_defconfig') + if match and not rear: + targets.append(front) + elif tag == 'S:': + status = rest + elif line == '\n' and targets: + for target in targets: + self.database[target] = (status, maintainers) + targets = [] + maintainers = [] + status = '-' + if targets: + for target in targets: + self.database[target] = (status, maintainers) + +class DotConfigParser: + + """A parser of .config file. + + Each line of the output should have the form of: + Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers + Most of them are extracted from .config file. + MAINTAINERS files are also consulted for Status and Maintainers fields. + """ + + re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') + re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') + re_soc = re.compile(r'CONFIG_SYS_SOC="(.*)"') + re_vendor = re.compile(r'CONFIG_SYS_VENDOR="(.*)"') + re_board = re.compile(r'CONFIG_SYS_BOARD="(.*)"') + re_config = re.compile(r'CONFIG_SYS_CONFIG_NAME="(.*)"') + re_options = re.compile(r'CONFIG_SYS_EXTRA_OPTIONS="(.*)"') + re_list = (('arch', re_arch), ('cpu', re_cpu), ('soc', re_soc), + ('vendor', re_vendor), ('board', re_board), + ('config', re_config), ('options', re_options)) + must_fields = ('arch', 'config') + + def __init__(self, build_dir, output, maintainers_database): + """Create a new .config perser. + + Arguments: + build_dir: Build directory where .config is located + output: File object which the result is written to + maintainers_database: An instance of class MaintainersDatabase + """ + self.dotconfig = os.path.join(build_dir, '.config') + self.output = output + self.database = maintainers_database + + def parse(self, defconfig): + """Parse .config file and output one-line database for the given board. + + Arguments: + defconfig: Board (defconfig) name + """ + fields = {} + for line in open(self.dotconfig): + if not line.startswith('CONFIG_SYS_'): + continue + for (key, pattern) in self.re_list: + m = pattern.match(line) + if m and m.group(1): + fields[key] = m.group(1) + break + + # sanity check of '.config' file + for field in self.must_fields: + if not field in fields: + print >> sys.stderr, 'Error: %s is not defined in %s' % \ + (field, defconfig) + sys.exit(1) + + # fix-up for aarch64 and tegra + if fields['arch'] == 'arm' and 'cpu' in fields: + if fields['cpu'] == 'armv8': + fields['arch'] = 'aarch64' + if 'soc' in fields and re.match('tegra[0-9]*$', fields['soc']): + fields['cpu'] += ':arm720t' + + target, match, rear = defconfig.partition('_defconfig') + assert match and not rear, \ + '%s : invalid defconfig file name' % defconfig + + fields['status'] = self.database.get_status(target) + fields['maintainers'] = self.database.get_maintainers(target) + + if 'options' in fields: + options = fields['config'] + ':' + \ + fields['options'].replace(r'\"', '"') + elif fields['config'] != target: + options = fields['config'] + else: + options = '-' + + self.output.write((' '.join(['%s'] * 9) + '\n') % + (fields['status'], + fields['arch'], + fields.get('cpu', '-'), + fields.get('soc', '-'), + fields.get('vendor', '-'), + fields.get('board', '-'), + target, + options, + fields['maintainers'])) + +class Slot: + + """A slot to store a subprocess. + + Each instance of this class handles one subprocess. + This class is useful to control multiple processes + for faster processing. + """ + + def __init__(self, output, maintainers_database, devnull, make_cmd): + """Create a new slot. + + Arguments: + output: File object which the result is written to + maintainers_database: An instance of class MaintainersDatabase + """ + self.occupied = False + self.build_dir = tempfile.mkdtemp() + self.devnull = devnull + self.make_cmd = make_cmd + self.parser = DotConfigParser(self.build_dir, output, + maintainers_database) + + def __del__(self): + """Delete the working directory""" + shutil.rmtree(self.build_dir) + + def add(self, defconfig): + """Add a new subprocess to the slot. + + Fails if the slot is occupied, that is, the current subprocess + is still running. + + Arguments: + defconfig: Board (defconfig) name + + Returns: + Return True on success or False on fail + """ + if self.occupied: + return False + o = 'O=' + self.build_dir + self.ps = subprocess.Popen([self.make_cmd, o, defconfig], + stdout=self.devnull) + self.defconfig = defconfig + self.occupied = True + return True + + def poll(self): + """Check if the subprocess is running and invoke the .config + parser if the subprocess is terminated. + + Returns: + Return True if the subprocess is terminated, False otherwise + """ + if not self.occupied: + return True + if self.ps.poll() == None: + return False + self.parser.parse(self.defconfig) + self.occupied = False + return True + +class Slots: + + """Controller of the array of subprocess slots.""" + + def __init__(self, jobs, output, maintainers_database): + """Create a new slots controller. + + Arguments: + jobs: A number of slots to instantiate + output: File object which the result is written to + maintainers_database: An instance of class MaintainersDatabase + """ + self.slots = [] + devnull = get_devnull() + make_cmd = get_make_cmd() + for i in range(jobs): + self.slots.append(Slot(output, maintainers_database, + devnull, make_cmd)) + + def add(self, defconfig): + """Add a new subprocess if a vacant slot is available. + + Arguments: + defconfig: Board (defconfig) name + + Returns: + Return True on success or False on fail + """ + for slot in self.slots: + if slot.add(defconfig): + return True + return False + + def available(self): + """Check if there is a vacant slot. + + Returns: + Return True if a vacant slot is found, False if all slots are full + """ + for slot in self.slots: + if slot.poll(): + return True + return False + + def empty(self): + """Check if all slots are vacant. + + Returns: + Return True if all slots are vacant, False if at least one slot + is running + """ + ret = True + for slot in self.slots: + if not slot.poll(): + ret = False + return ret + +class Indicator: + + """A class to control the progress indicator.""" + + MIN_WIDTH = 15 + MAX_WIDTH = 70 + + def __init__(self, total): + """Create an instance. + + Arguments: + total: A number of boards + """ + self.total = total + self.cur = 0 + width = get_terminal_columns() + width = min(width, self.MAX_WIDTH) + width -= self.MIN_WIDTH + if width > 0: + self.enabled = True + else: + self.enabled = False + self.width = width + + def inc(self): + """Increment the counter and show the progress bar.""" + if not self.enabled: + return + self.cur += 1 + arrow_len = self.width * self.cur // self.total + msg = '%4d/%d [' % (self.cur, self.total) + msg += '=' * arrow_len + '>' + ' ' * (self.width - arrow_len) + ']' + sys.stdout.write('\r' + msg) + sys.stdout.flush() + +def __gen_boards_cfg(jobs): + """Generate boards.cfg file. + + Arguments: + jobs: The number of jobs to run simultaneously + + Note: + The incomplete boards.cfg is left over when an error (including + the termination by the keyboard interrupt) occurs on the halfway. + """ + check_top_directory() + print 'Generating %s ... (jobs: %d)' % (BOARD_FILE, jobs) + + # All the defconfig files to be processed + defconfigs = [] + for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR): + dirpath = dirpath[len(CONFIG_DIR) + 1:] + for filename in fnmatch.filter(filenames, '*_defconfig'): + defconfigs.append(os.path.join(dirpath, filename)) + + # Parse all the MAINTAINERS files + maintainers_database = MaintainersDatabase() + for (dirpath, dirnames, filenames) in os.walk('.'): + if 'MAINTAINERS' in filenames: + maintainers_database.parse_file(os.path.join(dirpath, + 'MAINTAINERS')) + + # Output lines should be piped into the reformat tool + reformat_process = subprocess.Popen(REFORMAT_CMD, stdin=subprocess.PIPE, + stdout=open(BOARD_FILE, 'w')) + pipe = reformat_process.stdin + pipe.write(COMMENT_BLOCK) + + indicator = Indicator(len(defconfigs)) + slots = Slots(jobs, pipe, maintainers_database) + + # Main loop to process defconfig files: + # Add a new subprocess into a vacant slot. + # Sleep if there is no available slot. + for defconfig in defconfigs: + while not slots.add(defconfig): + while not slots.available(): + # No available slot: sleep for a while + time.sleep(SLEEP_TIME) + indicator.inc() + + # wait until all the subprocesses finish + while not slots.empty(): + time.sleep(SLEEP_TIME) + print '' + + # wait until the reformat tool finishes + reformat_process.communicate() + if reformat_process.returncode != 0: + print >> sys.stderr, '"%s" failed' % REFORMAT_CMD[0] + sys.exit(1) + +def gen_boards_cfg(jobs): + """Generate boards.cfg file. + + The incomplete boards.cfg is deleted if an error (including + the termination by the keyboard interrupt) occurs on the halfway. + + Arguments: + jobs: The number of jobs to run simultaneously + """ + try: + __gen_boards_cfg(jobs) + except: + # We should remove incomplete boards.cfg + try: + os.remove(BOARD_FILE) + except OSError as exception: + # Ignore 'No such file or directory' error + if exception.errno != errno.ENOENT: + raise + raise + +def main(): + parser = optparse.OptionParser() + # Add options here + parser.add_option('-j', '--jobs', + help='the number of jobs to run simultaneously') + (options, args) = parser.parse_args() + if options.jobs: + try: + jobs = int(options.jobs) + except ValueError: + print >> sys.stderr, 'Option -j (--jobs) takes a number' + sys.exit(1) + else: + try: + jobs = int(subprocess.Popen(['getconf', '_NPROCESSORS_ONLN'], + stdout=subprocess.PIPE).communicate()[0]) + except (OSError, ValueError): + print 'info: failed to get the number of CPUs. Set jobs to 1' + jobs = 1 + gen_boards_cfg(jobs) + +if __name__ == '__main__': + main() diff --git a/tools/patman/gitutil.py b/tools/patman/gitutil.py index 7b75c83..65754f5 100644 --- a/tools/patman/gitutil.py +++ b/tools/patman/gitutil.py @@ -377,9 +377,14 @@ def EmailPatches(series, cover_fname, args, dry_run, raise_on_error, cc_fname, """ to = BuildEmailList(series.get('to'), '--to', alias, raise_on_error) if not to: - print ("No recipient, please add something like this to a commit\n" - "Series-to: Fred Bloggs <f.blogs@napier.co.nz>") - return + git_config_to = command.Output('git', 'config', 'sendemail.to') + if not git_config_to: + print ("No recipient.\n" + "Please add something like this to a commit\n" + "Series-to: Fred Bloggs <f.blogs@napier.co.nz>\n" + "Or do something like this\n" + "git config sendemail.to u-boot@lists.denx.de") + return cc = BuildEmailList(series.get('cc'), '--cc', alias, raise_on_error) if self_only: to = BuildEmailList([os.getenv('USER')], '--to', alias, raise_on_error) |