buildman: Add a functional test

Buildman currently lacks testing in many areas, including its use of git,
make and many command-line flags.

Add a functional test which covers some of these areas. So far it does
a fake 'build' of all boards for the current source tree.

This version reads the real ~/.buildman and boards.cfg files. Future work
will improve this.

Signed-off-by: Simon Glass <sjg@chromium.org>

4 files changed, 206 insertions, 18 deletions

diff --git a/tools/buildman/buildman.py b/tools/buildman/buildman.py
index 70c2142..6771c86 100755
--- a/tools/buildman/buildman.py
+++ b/tools/buildman/buildman.py
@@ -30,27 +30,20 @@ import terminal
 import toolchain
 
 def RunTests():
+    import func_test
     import test
     import doctest
 
     result = unittest.TestResult()
-    for module in ['toolchain']:
+    for module in ['toolchain', 'gitutil']:
         suite = doctest.DocTestSuite(module)
         suite.run(result)
 
-    # TODO: Surely we can just 'print' result?
-    print result
-    for test, err in result.errors:
-        print err
-    for test, err in result.failures:
-        print err
-
     sys.argv = [sys.argv[0]]
-    suite = unittest.TestLoader().loadTestsFromTestCase(test.TestBuild)
-    result = unittest.TestResult()
-    suite.run(result)
+    for module in (test.TestBuild, func_test.TestFunctional):
+        suite = unittest.TestLoader().loadTestsFromTestCase(module)
+        suite.run(result)
 
-    # TODO: Surely we can just 'print' result?
     print result
     for test, err in result.errors:
         print err
diff --git a/tools/buildman/control.py b/tools/buildman/control.py
index 408d9b1..213e235 100644
--- a/tools/buildman/control.py
+++ b/tools/buildman/control.py
@@ -13,6 +13,7 @@ from builder import Builder
 import gitutil
 import patchstream
 import terminal
+from terminal import Print
 import toolchain
 import command
 import subprocess
@@ -77,12 +78,18 @@ def ShowActions(series, why_selected, boards_selected, builder, options):
     print ('Total boards to build for each commit: %d\n' %
             why_selected['all'])
 
-def DoBuildman(options, args):
+def DoBuildman(options, args, toolchains=None, make_func=None):
     """The main control code for buildman
 
     Args:
         options: Command line options object
         args: Command line arguments (list of strings)
+        toolchains: Toolchains to use - this should be a Toolchains()
+                object. If None, then it will be created and scanned
+        make_func: Make function to use for the builder. This is called
+                to execute 'make'. If this is None, the normal function
+                will be used, which calls the 'make' tool with suitable
+                arguments. This setting is useful for tests.
     """
     if options.full_help:
         pager = os.getenv('PAGER')
@@ -97,8 +104,10 @@ def DoBuildman(options, args):
     bsettings.Setup(options.config_file)
     options.git_dir = os.path.join(options.git, '.git')
 
-    toolchains = toolchain.Toolchains()
-    toolchains.Scan(options.list_tool_chains)
+    if not toolchains:
+        toolchains = toolchain.Toolchains()
+        toolchains.GetSettings()
+        toolchains.Scan(options.list_tool_chains)
     if options.list_tool_chains:
         toolchains.List()
         print
@@ -202,6 +211,8 @@ def DoBuildman(options, args):
             options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
             show_unknown=options.show_unknown, step=options.step)
     builder.force_config_on_failure = not options.quick
+    if make_func:
+        builder.do_make = make_func
 
     # For a dry run, just show our actions as a sanity check
     if options.dry_run:
@@ -220,8 +231,8 @@ def DoBuildman(options, args):
         else:
             commits = None
 
-        print GetActionSummary(options.summary, commits, board_selected,
-                               options)
+        Print(GetActionSummary(options.summary, commits, board_selected,
+                                options))
 
         builder.SetDisplayOptions(options.show_errors, options.show_sizes,
                                   options.show_detail, options.show_bloat,
diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py
new file mode 100644
index 0000000..8711f9c
--- /dev/null
+++ b/tools/buildman/func_test.py
@@ -0,0 +1,182 @@
+#
+# Copyright (c) 2014 Google, Inc
+#
+# SPDX-License-Identifier:      GPL-2.0+
+#
+
+import os
+import shutil
+import sys
+import tempfile
+import unittest
+
+import cmdline
+import command
+import control
+import gitutil
+import terminal
+import toolchain
+
+class TestFunctional(unittest.TestCase):
+    """Functional test for buildman.
+
+    This aims to test from just below the invocation of buildman (parsing
+    of arguments) to 'make' and 'git' invocation. It is not a true
+    emd-to-end test, as it mocks git, make and the tool chain. But this
+    makes it easier to detect when the builder is doing the wrong thing,
+    since in many cases this test code will fail. For example, only a
+    very limited subset of 'git' arguments is supported - anything
+    unexpected will fail.
+    """
+    def setUp(self):
+        self._base_dir = tempfile.mkdtemp()
+        self._git_dir = os.path.join(self._base_dir, 'src')
+        self._buildman_pathname = sys.argv[0]
+        self._buildman_dir = os.path.dirname(sys.argv[0])
+        command.test_result = self._HandleCommand
+        self._toolchains = toolchain.Toolchains()
+        self._toolchains.Add('gcc', test=False)
+
+    def tearDown(self):
+        shutil.rmtree(self._base_dir)
+
+    def _RunBuildman(self, *args):
+        return command.RunPipe([[self._buildman_pathname] + list(args)],
+                capture=True, capture_stderr=True)
+
+    def _RunControl(self, *args):
+        sys.argv = [sys.argv[0]] + list(args)
+        options, args = cmdline.ParseArgs()
+        return control.DoBuildman(options, args, toolchains=self._toolchains,
+                make_func=self._HandleMake)
+
+    def testFullHelp(self):
+        command.test_result = None
+        result = self._RunBuildman('-H')
+        help_file = os.path.join(self._buildman_dir, 'README')
+        self.assertEqual(len(result.stdout), os.path.getsize(help_file))
+        self.assertEqual(0, len(result.stderr))
+        self.assertEqual(0, result.return_code)
+
+    def testHelp(self):
+        command.test_result = None
+        result = self._RunBuildman('-h')
+        help_file = os.path.join(self._buildman_dir, 'README')
+        self.assertTrue(len(result.stdout) > 1000)
+        self.assertEqual(0, len(result.stderr))
+        self.assertEqual(0, result.return_code)
+
+    def testGitSetup(self):
+        """Test gitutils.Setup(), from outside the module itself"""
+        command.test_result = command.CommandResult(return_code=1)
+        gitutil.Setup()
+        self.assertEqual(gitutil.use_no_decorate, False)
+
+        command.test_result = command.CommandResult(return_code=0)
+        gitutil.Setup()
+        self.assertEqual(gitutil.use_no_decorate, True)
+
+    def _HandleCommandGitLog(self, args):
+        if '-n0' in args:
+            return command.CommandResult(return_code=0)
+
+        # Not handled, so abort
+        print 'git log', args
+        sys.exit(1)
+
+    def _HandleCommandGit(self, in_args):
+        """Handle execution of a git command
+
+        This uses a hacked-up parser.
+
+        Args:
+            in_args: Arguments after 'git' from the command line
+        """
+        git_args = []           # Top-level arguments to git itself
+        sub_cmd = None          # Git sub-command selected
+        args = []               # Arguments to the git sub-command
+        for arg in in_args:
+            if sub_cmd:
+                args.append(arg)
+            elif arg[0] == '-':
+                git_args.append(arg)
+            else:
+                sub_cmd = arg
+        if sub_cmd == 'config':
+            return command.CommandResult(return_code=0)
+        elif sub_cmd == 'log':
+            return self._HandleCommandGitLog(args)
+
+        # Not handled, so abort
+        print 'git', git_args, sub_cmd, args
+        sys.exit(1)
+
+    def _HandleCommandNm(self, args):
+        return command.CommandResult(return_code=0)
+
+    def _HandleCommandObjdump(self, args):
+        return command.CommandResult(return_code=0)
+
+    def _HandleCommandSize(self, args):
+        return command.CommandResult(return_code=0)
+
+    def _HandleCommand(self, **kwargs):
+        """Handle a command execution.
+
+        The command is in kwargs['pipe-list'], as a list of pipes, each a
+        list of commands. The command should be emulated as required for
+        testing purposes.
+
+        Returns:
+            A CommandResult object
+        """
+        pipe_list = kwargs['pipe_list']
+        if len(pipe_list) != 1:
+            print 'invalid pipe', kwargs
+            sys.exit(1)
+        cmd = pipe_list[0][0]
+        args = pipe_list[0][1:]
+        if cmd == 'git':
+            return self._HandleCommandGit(args)
+        elif cmd == './scripts/show-gnu-make':
+            return command.CommandResult(return_code=0, stdout='make')
+        elif cmd == 'nm':
+            return self._HandleCommandNm(args)
+        elif cmd == 'objdump':
+            return self._HandleCommandObjdump(args)
+        elif cmd == 'size':
+            return self._HandleCommandSize(args)
+
+        # Not handled, so abort
+        print 'unknown command', kwargs
+        sys.exit(1)
+        return command.CommandResult(return_code=0)
+
+    def _HandleMake(self, commit, brd, stage, cwd, *args, **kwargs):
+        """Handle execution of 'make'
+
+        Args:
+            commit: Commit object that is being built
+            brd: Board object that is being built
+            stage: Stage that we are at (mrproper, config, build)
+            cwd: Directory where make should be run
+            args: Arguments to pass to make
+            kwargs: Arguments to pass to command.RunPipe()
+        """
+        if stage == 'mrproper':
+            return command.CommandResult(return_code=0)
+        elif stage == 'config':
+            return command.CommandResult(return_code=0,
+                    combined='Test configuration complete')
+        elif stage == 'build':
+            return command.CommandResult(return_code=0)
+
+        # Not handled, so abort
+        print 'make', stage
+        sys.exit(1)
+
+    def testCurrentSource(self):
+        """Very simple test to invoke buildman on the current source"""
+        self._RunControl()
+        lines = terminal.GetPrintTestLines()
+        self.assertTrue(lines[0].text.startswith('Building current source'))
diff --git a/tools/buildman/toolchain.py b/tools/buildman/toolchain.py
index 0e91294..27dc318 100644
--- a/tools/buildman/toolchain.py
+++ b/tools/buildman/toolchain.py
@@ -99,6 +99,9 @@ class Toolchains:
     def __init__(self):
         self.toolchains = {}
         self.paths = []
+        self._make_flags = dict(bsettings.GetItems('make-flags'))
+
+    def GetSettings(self):
         toolchains = bsettings.GetItems('toolchain')
         if not toolchains:
             print ("Warning: No tool chains - please add a [toolchain] section"
@@ -110,7 +113,6 @@ class Toolchains:
                 self.paths += glob.glob(value)
             else:
                 self.paths.append(value)
-        self._make_flags = dict(bsettings.GetItems('make-flags'))
 
     def Add(self, fname, test=True, verbose=False):
         """Add a toolchain to our list