Merge branch 'master' of git://git.denx.de/u-boot-arm

Conflicts:
	boards.cfg

Signed-off-by: Stefano Babic <sbabic@denx.de>

1 files changed, 504 insertions, 0 deletions

diff --git a/tools/genboardscfg.py b/tools/genboardscfg.py
new file mode 100755
index 0000000..734d90b
--- /dev/null
+++ b/tools/genboardscfg.py
@@ -0,0 +1,504 @@
+#!/usr/bin/env python
+#
+# Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
+#
+# SPDX-License-Identifier:	GPL-2.0+
+#
+
+"""
+Converter from Kconfig and MAINTAINERS to boards.cfg
+
+Run 'tools/genboardscfg.py' to create boards.cfg file.
+
+Run 'tools/genboardscfg.py -h' for available options.
+"""
+
+import errno
+import fnmatch
+import glob
+import optparse
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+
+BOARD_FILE = 'boards.cfg'
+CONFIG_DIR = 'configs'
+REFORMAT_CMD = [os.path.join('tools', 'reformat.py'),
+                '-i', '-d', '-', '-s', '8']
+SHOW_GNU_MAKE = 'scripts/show-gnu-make'
+SLEEP_TIME=0.03
+
+COMMENT_BLOCK = '''#
+# List of boards
+#   Automatically generated by %s: don't edit
+#
+# Status, Arch, CPU(:SPLCPU), SoC, Vendor, Board, Target, Options, Maintainers
+
+''' % __file__
+
+### helper functions ###
+def get_terminal_columns():
+    """Get the width of the terminal.
+
+    Returns:
+      The width of the terminal, or zero if the stdout is not
+      associated with tty.
+    """
+    try:
+        return shutil.get_terminal_size().columns # Python 3.3~
+    except AttributeError:
+        import fcntl
+        import termios
+        import struct
+        arg = struct.pack('hhhh', 0, 0, 0, 0)
+        try:
+            ret = fcntl.ioctl(sys.stdout.fileno(), termios.TIOCGWINSZ, arg)
+        except IOError as exception:
+            if exception.errno != errno.ENOTTY:
+                raise
+            # If 'Inappropriate ioctl for device' error occurs,
+            # stdout is probably redirected. Return 0.
+            return 0
+        return struct.unpack('hhhh', ret)[1]
+
+def get_devnull():
+    """Get the file object of '/dev/null' device."""
+    try:
+        devnull = subprocess.DEVNULL # py3k
+    except AttributeError:
+        devnull = open(os.devnull, 'wb')
+    return devnull
+
+def check_top_directory():
+    """Exit if we are not at the top of source directory."""
+    for f in ('README', 'Licenses'):
+        if not os.path.exists(f):
+            print >> sys.stderr, 'Please run at the top of source directory.'
+            sys.exit(1)
+
+def get_make_cmd():
+    """Get the command name of GNU Make."""
+    process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE)
+    ret = process.communicate()
+    if process.returncode:
+        print >> sys.stderr, 'GNU Make not found'
+        sys.exit(1)
+    return ret[0].rstrip()
+
+### classes ###
+class MaintainersDatabase:
+
+    """The database of board status and maintainers."""
+
+    def __init__(self):
+        """Create an empty database."""
+        self.database = {}
+
+    def get_status(self, target):
+        """Return the status of the given board.
+
+        Returns:
+          Either 'Active' or 'Orphan'
+        """
+        tmp = self.database[target][0]
+        if tmp.startswith('Maintained'):
+            return 'Active'
+        elif tmp.startswith('Orphan'):
+            return 'Orphan'
+        else:
+            print >> sys.stderr, 'Error: %s: unknown status' % tmp
+
+    def get_maintainers(self, target):
+        """Return the maintainers of the given board.
+
+        If the board has two or more maintainers, they are separated
+        with colons.
+        """
+        return ':'.join(self.database[target][1])
+
+    def parse_file(self, file):
+        """Parse the given MAINTAINERS file.
+
+        This method parses MAINTAINERS and add board status and
+        maintainers information to the database.
+
+        Arguments:
+          file: MAINTAINERS file to be parsed
+        """
+        targets = []
+        maintainers = []
+        status = '-'
+        for line in open(file):
+            tag, rest = line[:2], line[2:].strip()
+            if tag == 'M:':
+                maintainers.append(rest)
+            elif tag == 'F:':
+                # expand wildcard and filter by 'configs/*_defconfig'
+                for f in glob.glob(rest):
+                    front, match, rear = f.partition('configs/')
+                    if not front and match:
+                        front, match, rear = rear.rpartition('_defconfig')
+                        if match and not rear:
+                            targets.append(front)
+            elif tag == 'S:':
+                status = rest
+            elif line == '\n' and targets:
+                for target in targets:
+                    self.database[target] = (status, maintainers)
+                targets = []
+                maintainers = []
+                status = '-'
+        if targets:
+            for target in targets:
+                self.database[target] = (status, maintainers)
+
+class DotConfigParser:
+
+    """A parser of .config file.
+
+    Each line of the output should have the form of:
+    Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
+    Most of them are extracted from .config file.
+    MAINTAINERS files are also consulted for Status and Maintainers fields.
+    """
+
+    re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"')
+    re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"')
+    re_soc = re.compile(r'CONFIG_SYS_SOC="(.*)"')
+    re_vendor = re.compile(r'CONFIG_SYS_VENDOR="(.*)"')
+    re_board = re.compile(r'CONFIG_SYS_BOARD="(.*)"')
+    re_config = re.compile(r'CONFIG_SYS_CONFIG_NAME="(.*)"')
+    re_options = re.compile(r'CONFIG_SYS_EXTRA_OPTIONS="(.*)"')
+    re_list = (('arch', re_arch), ('cpu', re_cpu), ('soc', re_soc),
+               ('vendor', re_vendor), ('board', re_board),
+               ('config', re_config), ('options', re_options))
+    must_fields = ('arch', 'config')
+
+    def __init__(self, build_dir, output, maintainers_database):
+        """Create a new .config perser.
+
+        Arguments:
+          build_dir: Build directory where .config is located
+          output: File object which the result is written to
+          maintainers_database: An instance of class MaintainersDatabase
+        """
+        self.dotconfig = os.path.join(build_dir, '.config')
+        self.output = output
+        self.database = maintainers_database
+
+    def parse(self, defconfig):
+        """Parse .config file and output one-line database for the given board.
+
+        Arguments:
+          defconfig: Board (defconfig) name
+        """
+        fields = {}
+        for line in open(self.dotconfig):
+            if not line.startswith('CONFIG_SYS_'):
+                continue
+            for (key, pattern) in self.re_list:
+                m = pattern.match(line)
+                if m and m.group(1):
+                    fields[key] = m.group(1)
+                    break
+
+        # sanity check of '.config' file
+        for field in self.must_fields:
+            if not field in fields:
+                print >> sys.stderr, 'Error: %s is not defined in %s' % \
+                                                            (field, defconfig)
+                sys.exit(1)
+
+        # fix-up for aarch64 and tegra
+        if fields['arch'] == 'arm' and 'cpu' in fields:
+            if fields['cpu'] == 'armv8':
+                fields['arch'] = 'aarch64'
+            if 'soc' in fields and re.match('tegra[0-9]*$', fields['soc']):
+                fields['cpu'] += ':arm720t'
+
+        target, match, rear = defconfig.partition('_defconfig')
+        assert match and not rear, \
+                                '%s : invalid defconfig file name' % defconfig
+
+        fields['status'] = self.database.get_status(target)
+        fields['maintainers'] = self.database.get_maintainers(target)
+
+        if 'options' in fields:
+            options = fields['config'] + ':' + \
+                      fields['options'].replace(r'\"', '"')
+        elif fields['config'] != target:
+            options = fields['config']
+        else:
+            options = '-'
+
+        self.output.write((' '.join(['%s'] * 9) + '\n')  %
+                          (fields['status'],
+                           fields['arch'],
+                           fields.get('cpu', '-'),
+                           fields.get('soc', '-'),
+                           fields.get('vendor', '-'),
+                           fields.get('board', '-'),
+                           target,
+                           options,
+                           fields['maintainers']))
+
+class Slot:
+
+    """A slot to store a subprocess.
+
+    Each instance of this class handles one subprocess.
+    This class is useful to control multiple processes
+    for faster processing.
+    """
+
+    def __init__(self, output, maintainers_database, devnull, make_cmd):
+        """Create a new slot.
+
+        Arguments:
+          output: File object which the result is written to
+          maintainers_database: An instance of class MaintainersDatabase
+        """
+        self.occupied = False
+        self.build_dir = tempfile.mkdtemp()
+        self.devnull = devnull
+        self.make_cmd = make_cmd
+        self.parser = DotConfigParser(self.build_dir, output,
+                                      maintainers_database)
+
+    def __del__(self):
+        """Delete the working directory"""
+        shutil.rmtree(self.build_dir)
+
+    def add(self, defconfig):
+        """Add a new subprocess to the slot.
+
+        Fails if the slot is occupied, that is, the current subprocess
+        is still running.
+
+        Arguments:
+          defconfig: Board (defconfig) name
+
+        Returns:
+          Return True on success or False on fail
+        """
+        if self.occupied:
+            return False
+        o = 'O=' + self.build_dir
+        self.ps = subprocess.Popen([self.make_cmd, o, defconfig],
+                                   stdout=self.devnull)
+        self.defconfig = defconfig
+        self.occupied = True
+        return True
+
+    def poll(self):
+        """Check if the subprocess is running and invoke the .config
+        parser if the subprocess is terminated.
+
+        Returns:
+          Return True if the subprocess is terminated, False otherwise
+        """
+        if not self.occupied:
+            return True
+        if self.ps.poll() == None:
+            return False
+        self.parser.parse(self.defconfig)
+        self.occupied = False
+        return True
+
+class Slots:
+
+    """Controller of the array of subprocess slots."""
+
+    def __init__(self, jobs, output, maintainers_database):
+        """Create a new slots controller.
+
+        Arguments:
+          jobs: A number of slots to instantiate
+          output: File object which the result is written to
+          maintainers_database: An instance of class MaintainersDatabase
+        """
+        self.slots = []
+        devnull = get_devnull()
+        make_cmd = get_make_cmd()
+        for i in range(jobs):
+            self.slots.append(Slot(output, maintainers_database,
+                                   devnull, make_cmd))
+
+    def add(self, defconfig):
+        """Add a new subprocess if a vacant slot is available.
+
+        Arguments:
+          defconfig: Board (defconfig) name
+
+        Returns:
+          Return True on success or False on fail
+        """
+        for slot in self.slots:
+            if slot.add(defconfig):
+                return True
+        return False
+
+    def available(self):
+        """Check if there is a vacant slot.
+
+        Returns:
+          Return True if a vacant slot is found, False if all slots are full
+        """
+        for slot in self.slots:
+            if slot.poll():
+                return True
+        return False
+
+    def empty(self):
+        """Check if all slots are vacant.
+
+        Returns:
+          Return True if all slots are vacant, False if at least one slot
+          is running
+        """
+        ret = True
+        for slot in self.slots:
+            if not slot.poll():
+                ret = False
+        return ret
+
+class Indicator:
+
+    """A class to control the progress indicator."""
+
+    MIN_WIDTH = 15
+    MAX_WIDTH = 70
+
+    def __init__(self, total):
+        """Create an instance.
+
+        Arguments:
+          total: A number of boards
+        """
+        self.total = total
+        self.cur = 0
+        width = get_terminal_columns()
+        width = min(width, self.MAX_WIDTH)
+        width -= self.MIN_WIDTH
+        if width > 0:
+            self.enabled = True
+        else:
+            self.enabled = False
+        self.width = width
+
+    def inc(self):
+        """Increment the counter and show the progress bar."""
+        if not self.enabled:
+            return
+        self.cur += 1
+        arrow_len = self.width * self.cur // self.total
+        msg = '%4d/%d [' % (self.cur, self.total)
+        msg += '=' * arrow_len + '>' + ' ' * (self.width - arrow_len) + ']'
+        sys.stdout.write('\r' + msg)
+        sys.stdout.flush()
+
+def __gen_boards_cfg(jobs):
+    """Generate boards.cfg file.
+
+    Arguments:
+      jobs: The number of jobs to run simultaneously
+
+    Note:
+      The incomplete boards.cfg is left over when an error (including 
+      the termination by the keyboard interrupt) occurs on the halfway.
+    """
+    check_top_directory()
+    print 'Generating %s ...  (jobs: %d)' % (BOARD_FILE, jobs)
+
+    # All the defconfig files to be processed
+    defconfigs = []
+    for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
+        dirpath = dirpath[len(CONFIG_DIR) + 1:]
+        for filename in fnmatch.filter(filenames, '*_defconfig'):
+            defconfigs.append(os.path.join(dirpath, filename))
+
+    # Parse all the MAINTAINERS files
+    maintainers_database = MaintainersDatabase()
+    for (dirpath, dirnames, filenames) in os.walk('.'):
+        if 'MAINTAINERS' in filenames:
+            maintainers_database.parse_file(os.path.join(dirpath,
+                                                         'MAINTAINERS'))
+
+    # Output lines should be piped into the reformat tool
+    reformat_process = subprocess.Popen(REFORMAT_CMD, stdin=subprocess.PIPE,
+                                        stdout=open(BOARD_FILE, 'w'))
+    pipe = reformat_process.stdin
+    pipe.write(COMMENT_BLOCK)
+
+    indicator = Indicator(len(defconfigs))
+    slots = Slots(jobs, pipe, maintainers_database)
+
+    # Main loop to process defconfig files:
+    #  Add a new subprocess into a vacant slot.
+    #  Sleep if there is no available slot.
+    for defconfig in defconfigs:
+        while not slots.add(defconfig):
+            while not slots.available():
+                # No available slot: sleep for a while
+                time.sleep(SLEEP_TIME)
+        indicator.inc()
+
+    # wait until all the subprocesses finish
+    while not slots.empty():
+        time.sleep(SLEEP_TIME)
+    print ''
+
+    # wait until the reformat tool finishes
+    reformat_process.communicate()
+    if reformat_process.returncode != 0:
+        print >> sys.stderr, '"%s" failed' % REFORMAT_CMD[0]
+        sys.exit(1)
+
+def gen_boards_cfg(jobs):
+    """Generate boards.cfg file.
+
+    The incomplete boards.cfg is deleted if an error (including
+    the termination by the keyboard interrupt) occurs on the halfway.
+
+    Arguments:
+      jobs: The number of jobs to run simultaneously
+    """
+    try:
+        __gen_boards_cfg(jobs)
+    except:
+        # We should remove incomplete boards.cfg
+        try:
+            os.remove(BOARD_FILE)
+        except OSError as exception:
+            # Ignore 'No such file or directory' error
+            if exception.errno != errno.ENOENT:
+                raise
+        raise
+
+def main():
+    parser = optparse.OptionParser()
+    # Add options here
+    parser.add_option('-j', '--jobs',
+                      help='the number of jobs to run simultaneously')
+    (options, args) = parser.parse_args()
+    if options.jobs:
+        try:
+            jobs = int(options.jobs)
+        except ValueError:
+            print >> sys.stderr, 'Option -j (--jobs) takes a number'
+            sys.exit(1)
+    else:
+        try:
+            jobs = int(subprocess.Popen(['getconf', '_NPROCESSORS_ONLN'],
+                                     stdout=subprocess.PIPE).communicate()[0])
+        except (OSError, ValueError):
+            print 'info: failed to get the number of CPUs. Set jobs to 1'
+            jobs = 1
+    gen_boards_cfg(jobs)
+
+if __name__ == '__main__':
+    main()