From bbd01435b9f3f1f60355b95f157170ec52c6353d Mon Sep 17 00:00:00 2001
From: Simon Glass <sjg@chromium.org>
Date: Sat, 15 Dec 2012 10:42:01 +0000
Subject: patman: Use ANSI colours only when outputting to a terminal

It is easy to detect whether or not the process is connected to a terminal,
or piped to a file. Disable ANSI colours automatically when output is
not to a terminal.

Signed-off-by: Simon Glass <sjg@chromium.org>
---
 tools/patman/terminal.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

(limited to 'tools/patman')

diff --git a/tools/patman/terminal.py b/tools/patman/terminal.py
index 838c828..8fad06e 100644
--- a/tools/patman/terminal.py
+++ b/tools/patman/terminal.py
@@ -24,6 +24,12 @@
 This module handles terminal interaction including ANSI color codes.
 """
 
+import os
+import sys
+
+# Selection of when we want our output to be colored
+COLOR_IF_TERMINAL, COLOR_ALWAYS, COLOR_NEVER = range(3)
+
 class Color(object):
   """Conditionally wraps text in ANSI color escape sequences."""
   BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8)
@@ -32,14 +38,15 @@ class Color(object):
   BOLD_START = '\033[1m'
   RESET = '\033[0m'
 
-  def __init__(self, enabled=True):
+  def __init__(self, colored=COLOR_IF_TERMINAL):
     """Create a new Color object, optionally disabling color output.
 
     Args:
       enabled: True if color output should be enabled. If False then this
         class will not add color codes at all.
     """
-    self._enabled = enabled
+    self._enabled = (colored == COLOR_ALWAYS or
+        (colored == COLOR_IF_TERMINAL and os.isatty(sys.stdout.fileno())))
 
   def Start(self, color):
     """Returns a start color code.
-- 
cgit v1.1


From 43bca004d698a2c6f457b32efeaa796e7751a72b Mon Sep 17 00:00:00 2001
From: Simon Glass <sjg@chromium.org>
Date: Sat, 15 Dec 2012 10:42:02 +0000
Subject: patman: Use bright ANSI colours by default

Rather than the rather dull colours, use bright versions which normally
look better and are easier to read.

Signed-off-by: Simon Glass <sjg@chromium.org>
---
 tools/patman/terminal.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

(limited to 'tools/patman')

diff --git a/tools/patman/terminal.py b/tools/patman/terminal.py
index 8fad06e..337a2a4 100644
--- a/tools/patman/terminal.py
+++ b/tools/patman/terminal.py
@@ -34,7 +34,8 @@ class Color(object):
   """Conditionally wraps text in ANSI color escape sequences."""
   BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8)
   BOLD = -1
-  COLOR_START = '\033[1;%dm'
+  BRIGHT_START = '\033[1;%dm'
+  NORMAL_START = '\033[22;%dm'
   BOLD_START = '\033[1m'
   RESET = '\033[0m'
 
@@ -48,7 +49,7 @@ class Color(object):
     self._enabled = (colored == COLOR_ALWAYS or
         (colored == COLOR_IF_TERMINAL and os.isatty(sys.stdout.fileno())))
 
-  def Start(self, color):
+  def Start(self, color, bright=True):
     """Returns a start color code.
 
     Args:
@@ -59,7 +60,8 @@ class Color(object):
       otherwise returns empty string
     """
     if self._enabled:
-      return self.COLOR_START % (color + 30)
+        base = self.BRIGHT_START if bright else self.NORMAL_START
+        return base % (color + 30)
     return ''
 
   def Stop(self):
@@ -70,10 +72,10 @@ class Color(object):
       returns empty string
     """
     if self._enabled:
-      return self.RESET
+        return self.RESET
     return ''
 
-  def Color(self, color, text):
+  def Color(self, color, text, bright=True):
     """Returns text with conditionally added color escape sequences.
 
     Keyword arguments:
@@ -85,9 +87,10 @@ class Color(object):
       returns text with color escape sequences based on the value of color.
     """
     if not self._enabled:
-      return text
+        return text
     if color == self.BOLD:
-      start = self.BOLD_START
+        start = self.BOLD_START
     else:
-      start = self.COLOR_START % (color + 30)
+        base = self.BRIGHT_START if bright else self.NORMAL_START
+        start = base % (color + 30)
     return start + text + self.RESET
-- 
cgit v1.1


From 71162e3cae29832192497d4a1b966336b638df01 Mon Sep 17 00:00:00 2001
From: Simon Glass <sjg@chromium.org>
Date: Sat, 15 Dec 2012 10:42:03 +0000
Subject: patman: Add cros_subprocess library to manage subprocesses

This adds a new library on top of subprocess which permits access to
the subprocess output as it is being generated. We can therefore
give the illusion that a process is running independently, but still
monitor its output so that we know what is going on.

It is possible to display output on a terminal as it is generated
(a little like tee). The supplied output function is called with all
stdout/stderr data as it arrives.

Signed-off-by: Simon Glass <sjg@chromium.org>
---
 tools/patman/cros_subprocess.py | 397 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 397 insertions(+)
 create mode 100644 tools/patman/cros_subprocess.py

(limited to 'tools/patman')

diff --git a/tools/patman/cros_subprocess.py b/tools/patman/cros_subprocess.py
new file mode 100644
index 0000000..0fc4a06
--- /dev/null
+++ b/tools/patman/cros_subprocess.py
@@ -0,0 +1,397 @@
+# Copyright (c) 2012 The Chromium OS Authors.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+#
+# Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se>
+# Licensed to PSF under a Contributor Agreement.
+# See http://www.python.org/2.4/license for licensing details.
+
+"""Subprocress execution
+
+This module holds a subclass of subprocess.Popen with our own required
+features, mainly that we get access to the subprocess output while it
+is running rather than just at the end. This makes it easiler to show
+progress information and filter output in real time.
+"""
+
+import errno
+import os
+import pty
+import select
+import subprocess
+import sys
+import unittest
+
+
+# Import these here so the caller does not need to import subprocess also.
+PIPE = subprocess.PIPE
+STDOUT = subprocess.STDOUT
+PIPE_PTY = -3     # Pipe output through a pty
+stay_alive = True
+
+
+class Popen(subprocess.Popen):
+    """Like subprocess.Popen with ptys and incremental output
+
+    This class deals with running a child process and filtering its output on
+    both stdout and stderr while it is running. We do this so we can monitor
+    progress, and possibly relay the output to the user if requested.
+
+    The class is similar to subprocess.Popen, the equivalent is something like:
+
+        Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+    But this class has many fewer features, and two enhancement:
+
+    1. Rather than getting the output data only at the end, this class sends it
+         to a provided operation as it arrives.
+    2. We use pseudo terminals so that the child will hopefully flush its output
+         to us as soon as it is produced, rather than waiting for the end of a
+         line.
+
+    Use CommunicateFilter() to handle output from the subprocess.
+
+    """
+
+    def __init__(self, args, stdin=None, stdout=PIPE_PTY, stderr=PIPE_PTY,
+                 shell=False, cwd=None, env=None, **kwargs):
+        """Cut-down constructor
+
+        Args:
+            args: Program and arguments for subprocess to execute.
+            stdin: See subprocess.Popen()
+            stdout: See subprocess.Popen(), except that we support the sentinel
+                    value of cros_subprocess.PIPE_PTY.
+            stderr: See subprocess.Popen(), except that we support the sentinel
+                    value of cros_subprocess.PIPE_PTY.
+            shell: See subprocess.Popen()
+            cwd: Working directory to change to for subprocess, or None if none.
+            env: Environment to use for this subprocess, or None to inherit parent.
+            kwargs: No other arguments are supported at the moment.    Passing other
+                    arguments will cause a ValueError to be raised.
+        """
+        stdout_pty = None
+        stderr_pty = None
+
+        if stdout == PIPE_PTY:
+            stdout_pty = pty.openpty()
+            stdout = os.fdopen(stdout_pty[1])
+        if stderr == PIPE_PTY:
+            stderr_pty = pty.openpty()
+            stderr = os.fdopen(stderr_pty[1])
+
+        super(Popen, self).__init__(args, stdin=stdin,
+                stdout=stdout, stderr=stderr, shell=shell, cwd=cwd, env=env,
+                **kwargs)
+
+        # If we're on a PTY, we passed the slave half of the PTY to the subprocess.
+        # We want to use the master half on our end from now on.    Setting this here
+        # does make some assumptions about the implementation of subprocess, but
+        # those assumptions are pretty minor.
+
+        # Note that if stderr is STDOUT, then self.stderr will be set to None by
+        # this constructor.
+        if stdout_pty is not None:
+            self.stdout = os.fdopen(stdout_pty[0])
+        if stderr_pty is not None:
+            self.stderr = os.fdopen(stderr_pty[0])
+
+        # Insist that unit tests exist for other arguments we don't support.
+        if kwargs:
+            raise ValueError("Unit tests do not test extra args - please add tests")
+
+    def CommunicateFilter(self, output):
+        """Interact with process: Read data from stdout and stderr.
+
+        This method runs until end-of-file is reached, then waits for the
+        subprocess to terminate.
+
+        The output function is sent all output from the subprocess and must be
+        defined like this:
+
+            def Output([self,] stream, data)
+            Args:
+                stream: the stream the output was received on, which will be
+                        sys.stdout or sys.stderr.
+                data: a string containing the data
+
+        Note: The data read is buffered in memory, so do not use this
+        method if the data size is large or unlimited.
+
+        Args:
+            output: Function to call with each fragment of output.
+
+        Returns:
+            A tuple (stdout, stderr, combined) which is the data received on
+            stdout, stderr and the combined data (interleaved stdout and stderr).
+
+            Note that the interleaved output will only be sensible if you have
+            set both stdout and stderr to PIPE or PIPE_PTY. Even then it depends on
+            the timing of the output in the subprocess. If a subprocess flips
+            between stdout and stderr quickly in succession, by the time we come to
+            read the output from each we may see several lines in each, and will read
+            all the stdout lines, then all the stderr lines. So the interleaving
+            may not be correct. In this case you might want to pass
+            stderr=cros_subprocess.STDOUT to the constructor.
+
+            This feature is still useful for subprocesses where stderr is
+            rarely used and indicates an error.
+
+            Note also that if you set stderr to STDOUT, then stderr will be empty
+            and the combined output will just be the same as stdout.
+        """
+
+        read_set = []
+        write_set = []
+        stdout = None # Return
+        stderr = None # Return
+
+        if self.stdin:
+            # Flush stdio buffer.    This might block, if the user has
+            # been writing to .stdin in an uncontrolled fashion.
+            self.stdin.flush()
+            if input:
+                write_set.append(self.stdin)
+            else:
+                self.stdin.close()
+        if self.stdout:
+            read_set.append(self.stdout)
+            stdout = []
+        if self.stderr and self.stderr != self.stdout:
+            read_set.append(self.stderr)
+            stderr = []
+        combined = []
+
+        input_offset = 0
+        while read_set or write_set:
+            try:
+                rlist, wlist, _ = select.select(read_set, write_set, [], 0.2)
+            except select.error, e:
+                if e.args[0] == errno.EINTR:
+                    continue
+                raise
+
+            if not stay_alive:
+                    self.terminate()
+
+            if self.stdin in wlist:
+                # When select has indicated that the file is writable,
+                # we can write up to PIPE_BUF bytes without risk
+                # blocking.    POSIX defines PIPE_BUF >= 512
+                chunk = input[input_offset : input_offset + 512]
+                bytes_written = os.write(self.stdin.fileno(), chunk)
+                input_offset += bytes_written
+                if input_offset >= len(input):
+                    self.stdin.close()
+                    write_set.remove(self.stdin)
+
+            if self.stdout in rlist:
+                data = ""
+                # We will get an error on read if the pty is closed
+                try:
+                    data = os.read(self.stdout.fileno(), 1024)
+                except OSError:
+                    pass
+                if data == "":
+                    self.stdout.close()
+                    read_set.remove(self.stdout)
+                else:
+                    stdout.append(data)
+                    combined.append(data)
+                    if output:
+                        output(sys.stdout, data)
+            if self.stderr in rlist:
+                data = ""
+                # We will get an error on read if the pty is closed
+                try:
+                    data = os.read(self.stderr.fileno(), 1024)
+                except OSError:
+                    pass
+                if data == "":
+                    self.stderr.close()
+                    read_set.remove(self.stderr)
+                else:
+                    stderr.append(data)
+                    combined.append(data)
+                    if output:
+                        output(sys.stderr, data)
+
+        # All data exchanged.    Translate lists into strings.
+        if stdout is not None:
+            stdout = ''.join(stdout)
+        else:
+            stdout = ''
+        if stderr is not None:
+            stderr = ''.join(stderr)
+        else:
+            stderr = ''
+        combined = ''.join(combined)
+
+        # Translate newlines, if requested.    We cannot let the file
+        # object do the translation: It is based on stdio, which is
+        # impossible to combine with select (unless forcing no
+        # buffering).
+        if self.universal_newlines and hasattr(file, 'newlines'):
+            if stdout:
+                stdout = self._translate_newlines(stdout)
+            if stderr:
+                stderr = self._translate_newlines(stderr)
+
+        self.wait()
+        return (stdout, stderr, combined)
+
+
+# Just being a unittest.TestCase gives us 14 public methods.    Unless we
+# disable this, we can only have 6 tests in a TestCase.    That's not enough.
+#
+# pylint: disable=R0904
+
+class TestSubprocess(unittest.TestCase):
+    """Our simple unit test for this module"""
+
+    class MyOperation:
+        """Provides a operation that we can pass to Popen"""
+        def __init__(self, input_to_send=None):
+            """Constructor to set up the operation and possible input.
+
+            Args:
+                input_to_send: a text string to send when we first get input. We will
+                    add \r\n to the string.
+            """
+            self.stdout_data = ''
+            self.stderr_data = ''
+            self.combined_data = ''
+            self.stdin_pipe = None
+            self._input_to_send = input_to_send
+            if input_to_send:
+                pipe = os.pipe()
+                self.stdin_read_pipe = pipe[0]
+                self._stdin_write_pipe = os.fdopen(pipe[1], 'w')
+
+        def Output(self, stream, data):
+            """Output handler for Popen. Stores the data for later comparison"""
+            if stream == sys.stdout:
+                self.stdout_data += data
+            if stream == sys.stderr:
+                self.stderr_data += data
+            self.combined_data += data
+
+            # Output the input string if we have one.
+            if self._input_to_send:
+                self._stdin_write_pipe.write(self._input_to_send + '\r\n')
+                self._stdin_write_pipe.flush()
+
+    def _BasicCheck(self, plist, oper):
+        """Basic checks that the output looks sane."""
+        self.assertEqual(plist[0], oper.stdout_data)
+        self.assertEqual(plist[1], oper.stderr_data)
+        self.assertEqual(plist[2], oper.combined_data)
+
+        # The total length of stdout and stderr should equal the combined length
+        self.assertEqual(len(plist[0]) + len(plist[1]), len(plist[2]))
+
+    def test_simple(self):
+        """Simple redirection: Get process list"""
+        oper = TestSubprocess.MyOperation()
+        plist = Popen(['ps']).CommunicateFilter(oper.Output)
+        self._BasicCheck(plist, oper)
+
+    def test_stderr(self):
+        """Check stdout and stderr"""
+        oper = TestSubprocess.MyOperation()
+        cmd = 'echo fred >/dev/stderr && false || echo bad'
+        plist = Popen([cmd], shell=True).CommunicateFilter(oper.Output)
+        self._BasicCheck(plist, oper)
+        self.assertEqual(plist [0], 'bad\r\n')
+        self.assertEqual(plist [1], 'fred\r\n')
+
+    def test_shell(self):
+        """Check with and without shell works"""
+        oper = TestSubprocess.MyOperation()
+        cmd = 'echo test >/dev/stderr'
+        self.assertRaises(OSError, Popen, [cmd], shell=False)
+        plist = Popen([cmd], shell=True).CommunicateFilter(oper.Output)
+        self._BasicCheck(plist, oper)
+        self.assertEqual(len(plist [0]), 0)
+        self.assertEqual(plist [1], 'test\r\n')
+
+    def test_list_args(self):
+        """Check with and without shell works using list arguments"""
+        oper = TestSubprocess.MyOperation()
+        cmd = ['echo', 'test', '>/dev/stderr']
+        plist = Popen(cmd, shell=False).CommunicateFilter(oper.Output)
+        self._BasicCheck(plist, oper)
+        self.assertEqual(plist [0], ' '.join(cmd[1:]) + '\r\n')
+        self.assertEqual(len(plist [1]), 0)
+
+        oper = TestSubprocess.MyOperation()
+
+        # this should be interpreted as 'echo' with the other args dropped
+        cmd = ['echo', 'test', '>/dev/stderr']
+        plist = Popen(cmd, shell=True).CommunicateFilter(oper.Output)
+        self._BasicCheck(plist, oper)
+        self.assertEqual(plist [0], '\r\n')
+
+    def test_cwd(self):
+        """Check we can change directory"""
+        for shell in (False, True):
+            oper = TestSubprocess.MyOperation()
+            plist = Popen('pwd', shell=shell, cwd='/tmp').CommunicateFilter(oper.Output)
+            self._BasicCheck(plist, oper)
+            self.assertEqual(plist [0], '/tmp\r\n')
+
+    def test_env(self):
+        """Check we can change environment"""
+        for add in (False, True):
+            oper = TestSubprocess.MyOperation()
+            env = os.environ
+            if add:
+                env ['FRED'] = 'fred'
+            cmd = 'echo $FRED'
+            plist = Popen(cmd, shell=True, env=env).CommunicateFilter(oper.Output)
+            self._BasicCheck(plist, oper)
+            self.assertEqual(plist [0], add and 'fred\r\n' or '\r\n')
+
+    def test_extra_args(self):
+        """Check we can't add extra arguments"""
+        self.assertRaises(ValueError, Popen, 'true', close_fds=False)
+
+    def test_basic_input(self):
+        """Check that incremental input works
+
+        We set up a subprocess which will prompt for name. When we see this prompt
+        we send the name as input to the process. It should then print the name
+        properly to stdout.
+        """
+        oper = TestSubprocess.MyOperation('Flash')
+        prompt = 'What is your name?: '
+        cmd = 'echo -n "%s"; read name; echo Hello $name' % prompt
+        plist = Popen([cmd], stdin=oper.stdin_read_pipe,
+                shell=True).CommunicateFilter(oper.Output)
+        self._BasicCheck(plist, oper)
+        self.assertEqual(len(plist [1]), 0)
+        self.assertEqual(plist [0], prompt + 'Hello Flash\r\r\n')
+
+    def test_isatty(self):
+        """Check that ptys appear as terminals to the subprocess"""
+        oper = TestSubprocess.MyOperation()
+        cmd = ('if [ -t %d ]; then echo "terminal %d" >&%d; '
+                'else echo "not %d" >&%d; fi;')
+        both_cmds = ''
+        for fd in (1, 2):
+            both_cmds += cmd % (fd, fd, fd, fd, fd)
+        plist = Popen(both_cmds, shell=True).CommunicateFilter(oper.Output)
+        self._BasicCheck(plist, oper)
+        self.assertEqual(plist [0], 'terminal 1\r\n')
+        self.assertEqual(plist [1], 'terminal 2\r\n')
+
+        # Now try with PIPE and make sure it is not a terminal
+        oper = TestSubprocess.MyOperation()
+        plist = Popen(both_cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                shell=True).CommunicateFilter(oper.Output)
+        self._BasicCheck(plist, oper)
+        self.assertEqual(plist [0], 'not 1\n')
+        self.assertEqual(plist [1], 'not 2\n')
+
+if __name__ == '__main__':
+    unittest.main()
-- 
cgit v1.1


From a10fd93cbc2ddf1da1f8b6d915f4bc3276ff7731 Mon Sep 17 00:00:00 2001
From: Simon Glass <sjg@chromium.org>
Date: Sat, 15 Dec 2012 10:42:04 +0000
Subject: patman: Make command methods return a CommandResult

Rather than returning a list of things, return an object. That makes it
easier to access the returned items, and easier to extend the return
value later.

Signed-off-by: Simon Glass <sjg@chromium.org>
---
 tools/patman/command.py     | 84 +++++++++++++++++++++++++++++++++------------
 tools/patman/gitutil.py     |  2 +-
 tools/patman/patchstream.py |  2 +-
 3 files changed, 64 insertions(+), 24 deletions(-)

(limited to 'tools/patman')

diff --git a/tools/patman/command.py b/tools/patman/command.py
index 4b00250..fc085f2 100644
--- a/tools/patman/command.py
+++ b/tools/patman/command.py
@@ -20,53 +20,93 @@
 #
 
 import os
-import subprocess
+import cros_subprocess
 
 """Shell command ease-ups for Python."""
 
-def RunPipe(pipeline, infile=None, outfile=None,
-            capture=False, oneline=False, hide_stderr=False):
+class CommandResult:
+    """A class which captures the result of executing a command.
+
+    Members:
+        stdout: stdout obtained from command, as a string
+        stderr: stderr obtained from command, as a string
+        return_code: Return code from command
+        exception: Exception received, or None if all ok
+    """
+    def __init__(self):
+        self.stdout = None
+        self.stderr = None
+        self.return_code = None
+        self.exception = None
+
+
+def RunPipe(pipe_list, infile=None, outfile=None,
+            capture=False, capture_stderr=False, oneline=False,
+            cwd=None, **kwargs):
     """
     Perform a command pipeline, with optional input/output filenames.
 
-    hide_stderr     Don't allow output of stderr (default False)
+    Args:
+        pipe_list: List of command lines to execute. Each command line is
+            piped into the next, and is itself a list of strings. For
+            example [ ['ls', '.git'] ['wc'] ] will pipe the output of
+            'ls .git' into 'wc'.
+        infile: File to provide stdin to the pipeline
+        outfile: File to store stdout
+        capture: True to capture output
+        capture_stderr: True to capture stderr
+        oneline: True to strip newline chars from output
+        kwargs: Additional keyword arguments to cros_subprocess.Popen()
+    Returns:
+        CommandResult object
     """
+    result = CommandResult()
     last_pipe = None
+    pipeline = list(pipe_list)
     while pipeline:
         cmd = pipeline.pop(0)
-        kwargs = {}
         if last_pipe is not None:
             kwargs['stdin'] = last_pipe.stdout
         elif infile:
             kwargs['stdin'] = open(infile, 'rb')
         if pipeline or capture:
-            kwargs['stdout'] = subprocess.PIPE
+            kwargs['stdout'] = cros_subprocess.PIPE
         elif outfile:
             kwargs['stdout'] = open(outfile, 'wb')
-        if hide_stderr:
-            kwargs['stderr'] = open('/dev/null', 'wb')
+        if capture_stderr:
+            kwargs['stderr'] = cros_subprocess.PIPE
 
-        last_pipe = subprocess.Popen(cmd, **kwargs)
+        try:
+            last_pipe = cros_subprocess.Popen(cmd, cwd=cwd, **kwargs)
+        except Exception, err:
+            result.exception = err
+            print 'exception', pipe_list, err
+            raise Exception("Error running '%s': %s" % (pipe_list, str))
 
     if capture:
-        ret = last_pipe.communicate()[0]
-        if not ret:
-            return None
-        elif oneline:
-            return ret.rstrip('\r\n')
-        else:
-            return ret
+        result.stdout, result.stderr, result.combined = (
+                last_pipe.CommunicateFilter(None))
+        if result.stdout and oneline:
+            result.output = result.stdout.rstrip('\r\n')
+        result.return_code = last_pipe.wait()
     else:
-        return os.waitpid(last_pipe.pid, 0)[1] == 0
+        result.return_code = os.waitpid(last_pipe.pid, 0)[1]
+    if result.return_code:
+        raise Exception("Error running '%s'" % pipe_list)
+    return result
 
 def Output(*cmd):
-    return RunPipe([cmd], capture=True)
+    return RunPipe([cmd], capture=True).stdout
 
-def OutputOneLine(*cmd):
-    return RunPipe([cmd], capture=True, oneline=True)
+def OutputOneLine(*cmd, **kwargs):
+    return (RunPipe([cmd], capture=True, oneline=True,
+            **kwargs).stdout.strip())
 
 def Run(*cmd, **kwargs):
-    return RunPipe([cmd], **kwargs)
+    return RunPipe([cmd], **kwargs).stdout
 
 def RunList(cmd):
-    return RunPipe([cmd], capture=True)
+    return RunPipe([cmd], capture=True).stdout
+
+def StopAll():
+    cros_subprocess.stay_alive = False
diff --git a/tools/patman/gitutil.py b/tools/patman/gitutil.py
index ca3ba4a..77907c1 100644
--- a/tools/patman/gitutil.py
+++ b/tools/patman/gitutil.py
@@ -40,7 +40,7 @@ def CountCommitsToBranch():
     """
     pipe = [['git', 'log', '--no-color', '--oneline', '@{upstream}..'],
             ['wc', '-l']]
-    stdout = command.RunPipe(pipe, capture=True, oneline=True)
+    stdout = command.RunPipe(pipe, capture=True, oneline=True).stdout
     patch_count = int(stdout)
     return patch_count
 
diff --git a/tools/patman/patchstream.py b/tools/patman/patchstream.py
index f7ee75a..1e4a36f 100644
--- a/tools/patman/patchstream.py
+++ b/tools/patman/patchstream.py
@@ -346,7 +346,7 @@ def GetMetaData(start, count):
     """
     pipe = [['git', 'log', '--no-color', '--reverse', 'HEAD~%d' % start,
 	'-n%d' % count]]
-    stdout = command.RunPipe(pipe, capture=True)
+    stdout = command.RunPipe(pipe, capture=True).stdout
     series = Series()
     ps = PatchStream(series, is_log=True)
     for line in stdout.splitlines():
-- 
cgit v1.1


From dc191505b903220a8581303efef0a1ead0e06532 Mon Sep 17 00:00:00 2001
From: Simon Glass <sjg@chromium.org>
Date: Sat, 15 Dec 2012 10:42:05 +0000
Subject: patman: Allow commands to raise on error, or not

Make raise_on_error a parameter so that we can control which commands
raise and which do not. If we get an error reading the alias file, just
continue.

Signed-off-by: Simon Glass <sjg@chromium.org>
---
 tools/patman/command.py | 17 +++++++++++------
 tools/patman/gitutil.py |  3 ++-
 2 files changed, 13 insertions(+), 7 deletions(-)

(limited to 'tools/patman')

diff --git a/tools/patman/command.py b/tools/patman/command.py
index fc085f2..e6af6ed 100644
--- a/tools/patman/command.py
+++ b/tools/patman/command.py
@@ -42,7 +42,7 @@ class CommandResult:
 
 def RunPipe(pipe_list, infile=None, outfile=None,
             capture=False, capture_stderr=False, oneline=False,
-            cwd=None, **kwargs):
+            raise_on_error=True, cwd=None, **kwargs):
     """
     Perform a command pipeline, with optional input/output filenames.
 
@@ -63,6 +63,7 @@ def RunPipe(pipe_list, infile=None, outfile=None,
     result = CommandResult()
     last_pipe = None
     pipeline = list(pipe_list)
+    user_pipestr =  '|'.join([' '.join(pipe) for pipe in pipe_list])
     while pipeline:
         cmd = pipeline.pop(0)
         if last_pipe is not None:
@@ -80,8 +81,10 @@ def RunPipe(pipe_list, infile=None, outfile=None,
             last_pipe = cros_subprocess.Popen(cmd, cwd=cwd, **kwargs)
         except Exception, err:
             result.exception = err
-            print 'exception', pipe_list, err
-            raise Exception("Error running '%s': %s" % (pipe_list, str))
+            if raise_on_error:
+                raise Exception("Error running '%s': %s" % (user_pipestr, str))
+            result.return_code = 255
+            return result
 
     if capture:
         result.stdout, result.stderr, result.combined = (
@@ -91,15 +94,17 @@ def RunPipe(pipe_list, infile=None, outfile=None,
         result.return_code = last_pipe.wait()
     else:
         result.return_code = os.waitpid(last_pipe.pid, 0)[1]
-    if result.return_code:
-        raise Exception("Error running '%s'" % pipe_list)
+    if raise_on_error and result.return_code:
+        raise Exception("Error running '%s'" % user_pipestr)
     return result
 
 def Output(*cmd):
-    return RunPipe([cmd], capture=True).stdout
+    return RunPipe([cmd], capture=True, raise_on_error=False).stdout
 
 def OutputOneLine(*cmd, **kwargs):
+    raise_on_error = kwargs.pop('raise_on_error', True)
     return (RunPipe([cmd], capture=True, oneline=True,
+            raise_on_error=raise_on_error,
             **kwargs).stdout.strip())
 
 def Run(*cmd, **kwargs):
diff --git a/tools/patman/gitutil.py b/tools/patman/gitutil.py
index 77907c1..33743dd 100644
--- a/tools/patman/gitutil.py
+++ b/tools/patman/gitutil.py
@@ -359,7 +359,8 @@ def GetAliasFile():
     Returns:
         Filename of git alias file, or None if none
     """
-    fname = command.OutputOneLine('git', 'config', 'sendemail.aliasesfile')
+    fname = command.OutputOneLine('git', 'config', 'sendemail.aliasesfile',
+            raise_on_error=False)
     if fname:
         fname = os.path.join(GetTopLevel(), fname.strip())
     return fname
-- 
cgit v1.1


From e62f905e1cbe55efd7438d4ef6c5d349373f2314 Mon Sep 17 00:00:00 2001
From: Simon Glass <sjg@chromium.org>
Date: Sat, 15 Dec 2012 10:42:06 +0000
Subject: patman: Allow reading metadata from a list of commits

We normally read from the current branch, but buildman will need to look
at commits from another branch. Allow the metadata to be read from any
list of commits, to provide this flexibility.

Signed-off-by: Simon Glass <sjg@chromium.org>
---
 tools/patman/patchstream.py | 36 +++++++++++++++++++++++++++++-------
 1 file changed, 29 insertions(+), 7 deletions(-)

(limited to 'tools/patman')

diff --git a/tools/patman/patchstream.py b/tools/patman/patchstream.py
index 1e4a36f..bed921d 100644
--- a/tools/patman/patchstream.py
+++ b/tools/patman/patchstream.py
@@ -237,7 +237,8 @@ class PatchStream:
         # Detect the start of a new commit
         elif commit_match:
             self.CloseCommit()
-            self.commit = commit.Commit(commit_match.group(1)[:7])
+            # TODO: We should store the whole hash, and just display a subset
+            self.commit = commit.Commit(commit_match.group(1)[:8])
 
         # Detect tags in the commit message
         elif tag_match:
@@ -334,26 +335,47 @@ class PatchStream:
         self.Finalize()
 
 
-def GetMetaData(start, count):
+def GetMetaDataForList(commit_range, git_dir=None, count=None,
+                       series = Series()):
     """Reads out patch series metadata from the commits
 
     This does a 'git log' on the relevant commits and pulls out the tags we
     are interested in.
 
     Args:
-        start: Commit to start from: 0=HEAD, 1=next one, etc.
-        count: Number of commits to list
+        commit_range: Range of commits to count (e.g. 'HEAD..base')
+        git_dir: Path to git repositiory (None to use default)
+        count: Number of commits to list, or None for no limit
+        series: Series object to add information into. By default a new series
+            is started.
+    Returns:
+        A Series object containing information about the commits.
     """
-    pipe = [['git', 'log', '--no-color', '--reverse', 'HEAD~%d' % start,
-	'-n%d' % count]]
+    params = ['git', 'log', '--no-color', '--reverse', commit_range]
+    if count is not None:
+        params[2:2] = ['-n%d' % count]
+    if git_dir:
+        params[1:1] = ['--git-dir', git_dir]
+    pipe = [params]
     stdout = command.RunPipe(pipe, capture=True).stdout
-    series = Series()
     ps = PatchStream(series, is_log=True)
     for line in stdout.splitlines():
         ps.ProcessLine(line)
     ps.Finalize()
     return series
 
+def GetMetaData(start, count):
+    """Reads out patch series metadata from the commits
+
+    This does a 'git log' on the relevant commits and pulls out the tags we
+    are interested in.
+
+    Args:
+        start: Commit to start from: 0=HEAD, 1=next one, etc.
+        count: Number of commits to list
+    """
+    return GetMetaDataForList('HEAD~%d' % start, None, count)
+
 def FixPatch(backup_dir, fname, series, commit):
     """Fix up a patch file, by adding/removing as required.
 
-- 
cgit v1.1


From 5f6a1c420070221e2fdbe81d9a8af8bcd3c05fbb Mon Sep 17 00:00:00 2001
From: Simon Glass <sjg@chromium.org>
Date: Sat, 15 Dec 2012 10:42:07 +0000
Subject: patman: Add additional git utilties

Add methods to find out the commits in a branch, clone a repo and
fetch from a repo.

Signed-off-by: Simon Glass <sjg@chromium.org>
---
 tools/patman/gitutil.py | 124 +++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 123 insertions(+), 1 deletion(-)

(limited to 'tools/patman')

diff --git a/tools/patman/gitutil.py b/tools/patman/gitutil.py
index 33743dd..99fac79 100644
--- a/tools/patman/gitutil.py
+++ b/tools/patman/gitutil.py
@@ -23,11 +23,12 @@ import command
 import re
 import os
 import series
-import settings
 import subprocess
 import sys
 import terminal
 
+import settings
+
 
 def CountCommitsToBranch():
     """Returns number of commits between HEAD and the tracking branch.
@@ -44,6 +45,119 @@ def CountCommitsToBranch():
     patch_count = int(stdout)
     return patch_count
 
+def GetUpstream(git_dir, branch):
+    """Returns the name of the upstream for a branch
+
+    Args:
+        git_dir: Git directory containing repo
+        branch: Name of branch
+
+    Returns:
+        Name of upstream branch (e.g. 'upstream/master') or None if none
+    """
+    remote = command.OutputOneLine('git', '--git-dir', git_dir, 'config',
+            'branch.%s.remote' % branch)
+    merge = command.OutputOneLine('git', '--git-dir', git_dir, 'config',
+            'branch.%s.merge' % branch)
+    if remote == '.':
+        return merge
+    elif remote and merge:
+        leaf = merge.split('/')[-1]
+        return '%s/%s' % (remote, leaf)
+    else:
+        raise ValueError, ("Cannot determine upstream branch for branch "
+                "'%s' remote='%s', merge='%s'" % (branch, remote, merge))
+
+
+def GetRangeInBranch(git_dir, branch, include_upstream=False):
+    """Returns an expression for the commits in the given branch.
+
+    Args:
+        git_dir: Directory containing git repo
+        branch: Name of branch
+    Return:
+        Expression in the form 'upstream..branch' which can be used to
+        access the commits.
+    """
+    upstream = GetUpstream(git_dir, branch)
+    return '%s%s..%s' % (upstream, '~' if include_upstream else '', branch)
+
+def CountCommitsInBranch(git_dir, branch, include_upstream=False):
+    """Returns the number of commits in the given branch.
+
+    Args:
+        git_dir: Directory containing git repo
+        branch: Name of branch
+    Return:
+        Number of patches that exist on top of the branch
+    """
+    range_expr = GetRangeInBranch(git_dir, branch, include_upstream)
+    pipe = [['git', '--git-dir', git_dir, 'log', '--oneline', range_expr],
+            ['wc', '-l']]
+    result = command.RunPipe(pipe, capture=True, oneline=True)
+    patch_count = int(result.stdout)
+    return patch_count
+
+def CountCommits(commit_range):
+    """Returns the number of commits in the given range.
+
+    Args:
+        commit_range: Range of commits to count (e.g. 'HEAD..base')
+    Return:
+        Number of patches that exist on top of the branch
+    """
+    pipe = [['git', 'log', '--oneline', commit_range],
+            ['wc', '-l']]
+    stdout = command.RunPipe(pipe, capture=True, oneline=True).stdout
+    patch_count = int(stdout)
+    return patch_count
+
+def Checkout(commit_hash, git_dir=None, work_tree=None, force=False):
+    """Checkout the selected commit for this build
+
+    Args:
+        commit_hash: Commit hash to check out
+    """
+    pipe = ['git']
+    if git_dir:
+        pipe.extend(['--git-dir', git_dir])
+    if work_tree:
+        pipe.extend(['--work-tree', work_tree])
+    pipe.append('checkout')
+    if force:
+        pipe.append('-f')
+    pipe.append(commit_hash)
+    result = command.RunPipe([pipe], capture=True, raise_on_error=False)
+    if result.return_code != 0:
+        raise OSError, 'git checkout (%s): %s' % (pipe, result.stderr)
+
+def Clone(git_dir, output_dir):
+    """Checkout the selected commit for this build
+
+    Args:
+        commit_hash: Commit hash to check out
+    """
+    pipe = ['git', 'clone', git_dir, '.']
+    result = command.RunPipe([pipe], capture=True, cwd=output_dir)
+    if result.return_code != 0:
+        raise OSError, 'git clone: %s' % result.stderr
+
+def Fetch(git_dir=None, work_tree=None):
+    """Fetch from the origin repo
+
+    Args:
+        commit_hash: Commit hash to check out
+    """
+    pipe = ['git']
+    if git_dir:
+        pipe.extend(['--git-dir', git_dir])
+    if work_tree:
+        pipe.extend(['--work-tree', work_tree])
+    pipe.append('fetch')
+    result = command.RunPipe([pipe], capture=True)
+    if result.return_code != 0:
+        raise OSError, 'git fetch: %s' % result.stderr
+
 def CreatePatches(start, count, series):
     """Create a series of patches from the top of the current branch.
 
@@ -390,6 +504,14 @@ def Setup():
     if alias_fname:
         settings.ReadGitAliases(alias_fname)
 
+def GetHead():
+    """Get the hash of the current HEAD
+
+    Returns:
+        Hash of HEAD
+    """
+    return command.OutputOneLine('git', 'show', '-s', '--pretty=format:%H')
+
 if __name__ == "__main__":
     import doctest
 
-- 
cgit v1.1


From 28b3594eb9b6d20ffab482fe37427502536c2bb6 Mon Sep 17 00:00:00 2001
From: Doug Anderson <dianders@chromium.org>
Date: Fri, 15 Mar 2013 13:24:05 +0000
Subject: patman: Make "Reviewed-by" an important tag

Although "Reviewed-by:" is a tag that gerrit adds, it's also a tag
used by upstream.  Stripping it is undesirable.  In fact, we should
treat it as important.

Signed-off-by: Doug Anderson <dianders@chromium.org>
Reviewed-by: Otavio Salvador <otavio@ossystems.com.br>
Acked-by: Simon Glass <sjg@chromium.org>
---
 tools/patman/README         | 4 ++--
 tools/patman/patchstream.py | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

(limited to 'tools/patman')

diff --git a/tools/patman/README b/tools/patman/README
index 1832ebd..86d366f 100644
--- a/tools/patman/README
+++ b/tools/patman/README
@@ -198,8 +198,9 @@ END
 	override the default signoff that patman automatically adds.
 
  Tested-by: Their Name <email>
+ Reviewed-by: Their Name <email>
  Acked-by: Their Name <email>
-	These indicate that someone has acked or tested your patch.
+	These indicate that someone has tested/reviewed/acked your patch.
 	When you get this reply on the mailing list, you can add this
 	tag to the relevant commit and the script will include it when
 	you send out the next version. If 'Tested-by:' is set to
@@ -231,7 +232,6 @@ TEST=...
 Change-Id:
 Review URL:
 Reviewed-on:
-Reviewed-by:
 
 
 Exercise for the reader: Try adding some tags to one of your current
diff --git a/tools/patman/patchstream.py b/tools/patman/patchstream.py
index bed921d..91542ad 100644
--- a/tools/patman/patchstream.py
+++ b/tools/patman/patchstream.py
@@ -31,7 +31,7 @@ from series import Series
 
 # Tags that we detect and remove
 re_remove = re.compile('^BUG=|^TEST=|^BRANCH=|^Change-Id:|^Review URL:'
-    '|Reviewed-on:|Reviewed-by:|Commit-Ready:')
+    '|Reviewed-on:|Commit-Ready:')
 
 # Lines which are allowed after a TEST= line
 re_allowed_after_test = re.compile('^Signed-off-by:')
@@ -46,7 +46,7 @@ re_cover = re.compile('^Cover-letter:')
 re_series = re.compile('^Series-(\w*): *(.*)')
 
 # Commit tags that we want to collect and keep
-re_tag = re.compile('^(Tested-by|Acked-by|Cc): (.*)')
+re_tag = re.compile('^(Tested-by|Acked-by|Reviewed-by|Cc): (.*)')
 
 # The start of a new commit in the git log
 re_commit = re.compile('^commit (.*)')
-- 
cgit v1.1


From 6d819925d0781b1fa8e5526f73cd277449dc08e1 Mon Sep 17 00:00:00 2001
From: Doug Anderson <dianders@chromium.org>
Date: Sun, 17 Mar 2013 10:31:04 +0000
Subject: patman: Allow specifying the message ID your series is in reply to

Some versions of git don't seem to prompt you for the message ID that
your series is in reply to.  Allow specifying this from the command
line.

Signed-off-by: Doug Anderson <dianders@chromium.org>
Acked-by: Simon Glass <sjg@chromium.org>
---
 tools/patman/gitutil.py | 7 ++++++-
 tools/patman/patman.py  | 4 +++-
 2 files changed, 9 insertions(+), 2 deletions(-)

(limited to 'tools/patman')

diff --git a/tools/patman/gitutil.py b/tools/patman/gitutil.py
index 99fac79..4e21d4c 100644
--- a/tools/patman/gitutil.py
+++ b/tools/patman/gitutil.py
@@ -317,7 +317,7 @@ def BuildEmailList(in_list, tag=None, alias=None):
     return result
 
 def EmailPatches(series, cover_fname, args, dry_run, cc_fname,
-        self_only=False, alias=None):
+        self_only=False, alias=None, in_reply_to=None):
     """Email a patch series.
 
     Args:
@@ -327,6 +327,8 @@ def EmailPatches(series, cover_fname, args, dry_run, cc_fname,
         dry_run: Just return the command that would be run
         cc_fname: Filename of Cc file for per-commit Cc
         self_only: True to just email to yourself as a test
+        in_reply_to: If set we'll pass this to git as --in-reply-to.
+            Should be a message ID that this is in reply to.
 
     Returns:
         Git command that was/would be run
@@ -376,6 +378,9 @@ def EmailPatches(series, cover_fname, args, dry_run, cc_fname,
         to = BuildEmailList([os.getenv('USER')], '--to', alias)
         cc = []
     cmd = ['git', 'send-email', '--annotate']
+    if in_reply_to:
+        cmd.append('--in-reply-to="%s"' % in_reply_to)
+
     cmd += to
     cmd += cc
     cmd += ['--cc-cmd', '"%s --cc-cmd %s"' % (sys.argv[0], cc_fname)]
diff --git a/tools/patman/patman.py b/tools/patman/patman.py
index e049081..377408d 100755
--- a/tools/patman/patman.py
+++ b/tools/patman/patman.py
@@ -53,6 +53,8 @@ parser.add_option('-n', '--dry-run', action='store_true', dest='dry_run',
 parser.add_option('-p', '--project', default=project.DetectProject(),
                   help="Project name; affects default option values and "
                   "aliases [default: %default]")
+parser.add_option('-r', '--in-reply-to', type='string', action='store',
+                  help="Message ID that this series is in reply to")
 parser.add_option('-s', '--start', dest='start', type='int',
        default=0, help='Commit to start creating patches from (0 = HEAD)')
 parser.add_option('-t', '--test', action='store_true', dest='test',
@@ -163,7 +165,7 @@ else:
     cmd = ''
     if ok or options.ignore_errors:
         cmd = gitutil.EmailPatches(series, cover_fname, args,
-                options.dry_run, cc_file)
+                options.dry_run, cc_file, in_reply_to=options.in_reply_to)
 
     # For a dry run, just show our actions as a sanity check
     if options.dry_run:
-- 
cgit v1.1


From fe2f8d9e2f1bc00f143a22191a1d7076667ff436 Mon Sep 17 00:00:00 2001
From: Simon Glass <sjg@chromium.org>
Date: Wed, 20 Mar 2013 16:43:00 +0000
Subject: patman: Add Cover-letter-cc tag to Cc cover letter to people

The cover letter is sent to everyone who is on the Cc list for any of
the patches in the series. Sometimes it is useful to send just the cover
letter to additional people, so that they are aware of the series, but
don't need to wade through all the individual patches.

Add a new Cover-letter-cc tag for this purpose.

Signed-off-by: Simon Glass <sjg@chromium.org>
Reviewed-by: Doug Anderson <dianders@chromium.org>
---
 tools/patman/README         | 12 +++++++++++-
 tools/patman/patchstream.py |  8 ++++++++
 tools/patman/series.py      | 11 ++++++++---
 3 files changed, 27 insertions(+), 4 deletions(-)

(limited to 'tools/patman')

diff --git a/tools/patman/README b/tools/patman/README
index 86d366f..9922f2a 100644
--- a/tools/patman/README
+++ b/tools/patman/README
@@ -182,6 +182,10 @@ END
 	Sets the cover letter contents for the series. The first line
 	will become the subject of the cover letter
 
+Cover-letter-cc: email / alias
+	Additional email addresses / aliases to send cover letter to (you
+	can add this multiple times)
+
 Series-notes:
 blah blah
 blah blah
@@ -263,7 +267,13 @@ will create a patch which is copied to x86, arm, sandbox, mikef, ag and
 afleming.
 
 If you have a cover letter it will get sent to the union of the CC lists of
-all of the other patches.
+all of the other patches. If you want to sent it to additional people you
+can add a tag:
+
+Cover-letter-cc: <list of addresses>
+
+These people will get the cover letter even if they are not on the To/Cc
+list for any of the patches.
 
 
 Example Work Flow
diff --git a/tools/patman/patchstream.py b/tools/patman/patchstream.py
index 91542ad..5c2d3bc 100644
--- a/tools/patman/patchstream.py
+++ b/tools/patman/patchstream.py
@@ -42,6 +42,9 @@ re_signoff = re.compile('^Signed-off-by:')
 # The start of the cover letter
 re_cover = re.compile('^Cover-letter:')
 
+# A cover letter Cc
+re_cover_cc = re.compile('^Cover-letter-cc: *(.*)')
+
 # Patch series tag
 re_series = re.compile('^Series-(\w*): *(.*)')
 
@@ -153,6 +156,7 @@ class PatchStream:
         # Handle state transition and skipping blank lines
         series_match = re_series.match(line)
         commit_match = re_commit.match(line) if self.is_log else None
+        cover_cc_match = re_cover_cc.match(line)
         tag_match = None
         if self.state == STATE_PATCH_HEADER:
             tag_match = re_tag.match(line)
@@ -205,6 +209,10 @@ class PatchStream:
             self.in_section = 'cover'
             self.skip_blank = False
 
+        elif cover_cc_match:
+            value = cover_cc_match.group(1)
+            self.AddToSeries(line, 'cover-cc', value)
+
         # If we are in a change list, key collected lines until a blank one
         elif self.in_change:
             if is_blank:
diff --git a/tools/patman/series.py b/tools/patman/series.py
index 6c5c570..44ad931 100644
--- a/tools/patman/series.py
+++ b/tools/patman/series.py
@@ -27,7 +27,8 @@ import gitutil
 import terminal
 
 # Series-xxx tags that we understand
-valid_series = ['to', 'cc', 'version', 'changes', 'prefix', 'notes', 'name'];
+valid_series = ['to', 'cc', 'version', 'changes', 'prefix', 'notes', 'name',
+                'cover-cc']
 
 class Series(dict):
     """Holds information about a patch series, including all tags.
@@ -43,6 +44,7 @@ class Series(dict):
     def __init__(self):
         self.cc = []
         self.to = []
+        self.cover_cc = []
         self.commits = []
         self.cover = None
         self.notes = []
@@ -69,6 +71,7 @@ class Series(dict):
             value: Tag value (part after 'Series-xxx: ')
         """
         # If we already have it, then add to our list
+        name = name.replace('-', '_')
         if name in self:
             values = value.split(',')
             values = [str.strip() for str in values]
@@ -140,7 +143,8 @@ class Series(dict):
         print 'Prefix:\t ', self.get('prefix')
         if self.cover:
             print 'Cover: %d lines' % len(self.cover)
-            all_ccs = itertools.chain(*self._generated_cc.values())
+            cover_cc = gitutil.BuildEmailList(self.get('cover_cc', ''))
+            all_ccs = itertools.chain(cover_cc, *self._generated_cc.values())
             for email in set(all_ccs):
                     print '      Cc: ',email
         if cmd:
@@ -232,7 +236,8 @@ class Series(dict):
             self._generated_cc[commit.patch] = list
 
         if cover_fname:
-            print >>fd, cover_fname, ', '.join(set(all_ccs))
+            cover_cc = gitutil.BuildEmailList(self.get('cover_cc', ''))
+            print >>fd, cover_fname, ', '.join(set(cover_cc + all_ccs))
 
         fd.close()
         return fname
-- 
cgit v1.1


From d29fe6e2d2cbcbfd1bfaaf886818d84edde0fc0d Mon Sep 17 00:00:00 2001
From: Simon Glass <sjg@chromium.org>
Date: Tue, 26 Mar 2013 13:09:39 +0000
Subject: patman: Fix up checkpatch parsing to deal with 'CHECK' lines

checkpatch has a new type of warning, a 'CHECK'. At present patman fails
with these, which makes it less than useful.

Add support for checks, making it backwards compatible with the old
checkpatch.

At the same time, clean up formatting of the CheckPatches() output,
fix erroneous "internal error" if multiple patches have warnings and
be more robust to new types of problems.

Signed-off-by: Simon Glass <sjg@chromium.org>
Reviewed-by: Doug Anderson <dianders@chromium.org>
---
 tools/patman/checkpatch.py | 110 ++++++++++++++++++++++++++++-----------------
 tools/patman/test.py       |  64 +++++++++++++++++---------
 2 files changed, 112 insertions(+), 62 deletions(-)

(limited to 'tools/patman')

diff --git a/tools/patman/checkpatch.py b/tools/patman/checkpatch.py
index d3a0477..83aaf71 100644
--- a/tools/patman/checkpatch.py
+++ b/tools/patman/checkpatch.py
@@ -19,6 +19,7 @@
 # MA 02111-1307 USA
 #
 
+import collections
 import command
 import gitutil
 import os
@@ -57,63 +58,86 @@ def CheckPatch(fname, verbose=False):
     """Run checkpatch.pl on a file.
 
     Returns:
-        4-tuple containing:
-            result: False=failure, True=ok
+        namedtuple containing:
+            ok: False=failure, True=ok
             problems: List of problems, each a dict:
                 'type'; error or warning
                 'msg': text message
                 'file' : filename
                 'line': line number
+            errors: Number of errors
+            warnings: Number of warnings
+            checks: Number of checks
             lines: Number of lines
+            stdout: Full output of checkpatch
     """
-    result = False
-    error_count, warning_count, lines = 0, 0, 0
-    problems = []
+    fields = ['ok', 'problems', 'errors', 'warnings', 'checks', 'lines',
+              'stdout']
+    result = collections.namedtuple('CheckPatchResult', fields)
+    result.ok = False
+    result.errors, result.warning, result.checks = 0, 0, 0
+    result.lines = 0
+    result.problems = []
     chk = FindCheckPatch()
     item = {}
-    stdout = command.Output(chk, '--no-tree', fname)
+    result.stdout = command.Output(chk, '--no-tree', fname)
     #pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE)
     #stdout, stderr = pipe.communicate()
 
     # total: 0 errors, 0 warnings, 159 lines checked
+    # or:
+    # total: 0 errors, 2 warnings, 7 checks, 473 lines checked
     re_stats = re.compile('total: (\\d+) errors, (\d+) warnings, (\d+)')
+    re_stats_full = re.compile('total: (\\d+) errors, (\d+) warnings, (\d+)'
+                               ' checks, (\d+)')
     re_ok = re.compile('.*has no obvious style problems')
     re_bad = re.compile('.*has style problems, please review')
     re_error = re.compile('ERROR: (.*)')
     re_warning = re.compile('WARNING: (.*)')
+    re_check = re.compile('CHECK: (.*)')
     re_file = re.compile('#\d+: FILE: ([^:]*):(\d+):')
 
-    for line in stdout.splitlines():
+    for line in result.stdout.splitlines():
         if verbose:
             print line
 
         # A blank line indicates the end of a message
         if not line and item:
-            problems.append(item)
+            result.problems.append(item)
             item = {}
-        match = re_stats.match(line)
+        match = re_stats_full.match(line)
+        if not match:
+            match = re_stats.match(line)
         if match:
-            error_count = int(match.group(1))
-            warning_count = int(match.group(2))
-            lines = int(match.group(3))
+            result.errors = int(match.group(1))
+            result.warnings = int(match.group(2))
+            if len(match.groups()) == 4:
+                result.checks = int(match.group(3))
+                result.lines = int(match.group(4))
+            else:
+                result.lines = int(match.group(3))
         elif re_ok.match(line):
-            result = True
+            result.ok = True
         elif re_bad.match(line):
-            result = False
-        match = re_error.match(line)
-        if match:
-            item['msg'] = match.group(1)
+            result.ok = False
+        err_match = re_error.match(line)
+        warn_match = re_warning.match(line)
+        file_match = re_file.match(line)
+        check_match = re_check.match(line)
+        if err_match:
+            item['msg'] = err_match.group(1)
             item['type'] = 'error'
-        match = re_warning.match(line)
-        if match:
-            item['msg'] = match.group(1)
+        elif warn_match:
+            item['msg'] = warn_match.group(1)
             item['type'] = 'warning'
-        match = re_file.match(line)
-        if match:
-            item['file'] = match.group(1)
-            item['line'] = int(match.group(2))
+        elif check_match:
+            item['msg'] = check_match.group(1)
+            item['type'] = 'check'
+        elif file_match:
+            item['file'] = file_match.group(1)
+            item['line'] = int(file_match.group(2))
 
-    return result, problems, error_count, warning_count, lines, stdout
+    return result
 
 def GetWarningMsg(col, msg_type, fname, line, msg):
     '''Create a message for a given file/line
@@ -128,37 +152,39 @@ def GetWarningMsg(col, msg_type, fname, line, msg):
         msg_type = col.Color(col.YELLOW, msg_type)
     elif msg_type == 'error':
         msg_type = col.Color(col.RED, msg_type)
+    elif msg_type == 'check':
+        msg_type = col.Color(col.MAGENTA, msg_type)
     return '%s: %s,%d: %s' % (msg_type, fname, line, msg)
 
 def CheckPatches(verbose, args):
     '''Run the checkpatch.pl script on each patch'''
-    error_count = 0
-    warning_count = 0
+    error_count, warning_count, check_count = 0, 0, 0
     col = terminal.Color()
 
     for fname in args:
-        ok, problems, errors, warnings, lines, stdout = CheckPatch(fname,
-                verbose)
-        if not ok:
-            error_count += errors
-            warning_count += warnings
-            print '%d errors, %d warnings for %s:' % (errors,
-                    warnings, fname)
-            if len(problems) != error_count + warning_count:
+        result = CheckPatch(fname, verbose)
+        if not result.ok:
+            error_count += result.errors
+            warning_count += result.warnings
+            check_count += result.checks
+            print '%d errors, %d warnings, %d checks for %s:' % (result.errors,
+                    result.warnings, result.checks, col.Color(col.BLUE, fname))
+            if (len(result.problems) != result.errors + result.warnings +
+                    result.checks):
                 print "Internal error: some problems lost"
-            for item in problems:
-                print GetWarningMsg(col, item['type'],
+            for item in result.problems:
+                print GetWarningMsg(col, item.get('type', '<unknown>'),
                         item.get('file', '<unknown>'),
-                        item.get('line', 0), item['msg'])
+                        item.get('line', 0), item.get('msg', 'message'))
+            print
             #print stdout
-    if error_count != 0 or warning_count != 0:
-        str = 'checkpatch.pl found %d error(s), %d warning(s)' % (
-            error_count, warning_count)
+    if error_count or warning_count or check_count:
+        str = 'checkpatch.pl found %d error(s), %d warning(s), %d checks(s)'
         color = col.GREEN
         if warning_count:
             color = col.YELLOW
         if error_count:
             color = col.RED
-        print col.Color(color, str)
+        print col.Color(color, str % (error_count, warning_count, check_count))
         return False
     return True
diff --git a/tools/patman/test.py b/tools/patman/test.py
index f801ced..8cd2647 100644
--- a/tools/patman/test.py
+++ b/tools/patman/test.py
@@ -190,6 +190,11 @@ index 0000000..2234c87
 +		rec->time_us = (uint32_t)timer_get_us();
 +		rec->name = name;
 +	}
++	if (!rec->name &&
++	%ssomething_else) {
++		rec->time_us = (uint32_t)timer_get_us();
++		rec->name = name;
++	}
 +%sreturn rec->time_us;
 +}
 --
@@ -197,15 +202,18 @@ index 0000000..2234c87
 '''
         signoff = 'Signed-off-by: Simon Glass <sjg@chromium.org>\n'
         tab = '	'
+        indent = '    '
         if data_type == 'good':
             pass
         elif data_type == 'no-signoff':
             signoff = ''
         elif data_type == 'spaces':
             tab = '   '
+        elif data_type == 'indent':
+            indent = tab
         else:
             print 'not implemented'
-        return data % (signoff, tab, tab)
+        return data % (signoff, tab, indent, tab)
 
     def SetupData(self, data_type):
         inhandle, inname = tempfile.mkstemp()
@@ -215,33 +223,49 @@ index 0000000..2234c87
         infd.close()
         return inname
 
-    def testCheckpatch(self):
+    def testGood(self):
         """Test checkpatch operation"""
         inf = self.SetupData('good')
-        result, problems, err, warn, lines, stdout = checkpatch.CheckPatch(inf)
-        self.assertEqual(result, True)
-        self.assertEqual(problems, [])
-        self.assertEqual(err, 0)
-        self.assertEqual(warn, 0)
-        self.assertEqual(lines, 67)
+        result = checkpatch.CheckPatch(inf)
+        self.assertEqual(result.ok, True)
+        self.assertEqual(result.problems, [])
+        self.assertEqual(result.errors, 0)
+        self.assertEqual(result.warnings, 0)
+        self.assertEqual(result.checks, 0)
+        self.assertEqual(result.lines, 67)
         os.remove(inf)
 
+    def testNoSignoff(self):
         inf = self.SetupData('no-signoff')
-        result, problems, err, warn, lines, stdout = checkpatch.CheckPatch(inf)
-        self.assertEqual(result, False)
-        self.assertEqual(len(problems), 1)
-        self.assertEqual(err, 1)
-        self.assertEqual(warn, 0)
-        self.assertEqual(lines, 67)
+        result = checkpatch.CheckPatch(inf)
+        self.assertEqual(result.ok, False)
+        self.assertEqual(len(result.problems), 1)
+        self.assertEqual(result.errors, 1)
+        self.assertEqual(result.warnings, 0)
+        self.assertEqual(result.checks, 0)
+        self.assertEqual(result.lines, 67)
         os.remove(inf)
 
+    def testSpaces(self):
         inf = self.SetupData('spaces')
-        result, problems, err, warn, lines, stdout = checkpatch.CheckPatch(inf)
-        self.assertEqual(result, False)
-        self.assertEqual(len(problems), 2)
-        self.assertEqual(err, 0)
-        self.assertEqual(warn, 2)
-        self.assertEqual(lines, 67)
+        result = checkpatch.CheckPatch(inf)
+        self.assertEqual(result.ok, False)
+        self.assertEqual(len(result.problems), 1)
+        self.assertEqual(result.errors, 0)
+        self.assertEqual(result.warnings, 1)
+        self.assertEqual(result.checks, 0)
+        self.assertEqual(result.lines, 67)
+        os.remove(inf)
+
+    def testIndent(self):
+        inf = self.SetupData('indent')
+        result = checkpatch.CheckPatch(inf)
+        self.assertEqual(result.ok, False)
+        self.assertEqual(len(result.problems), 1)
+        self.assertEqual(result.errors, 0)
+        self.assertEqual(result.warnings, 0)
+        self.assertEqual(result.checks, 1)
+        self.assertEqual(result.lines, 67)
         os.remove(inf)
 
 
-- 
cgit v1.1


From ed9222752d148c6aa655cc189b623e73ede1b62d Mon Sep 17 00:00:00 2001
From: Simon Glass <sjg@chromium.org>
Date: Tue, 26 Mar 2013 13:09:40 +0000
Subject: patman: Don't allow spaces in tags

At present something like:

   Revert "arm: Add cache operations"

will try to use

   Revert "arm

as a tag. Clearly this is wrong, so fix it.

If the revert is intended to be tagged, then the tag can come before
the revert, perhaps. Alternatively the 'Cc' tag can be used in the commit
messages.

Signed-off-by: Simon Glass <sjg@chromium.org>
Reviewed-by: Doug Anderson <dianders@chromium.org>
---
 tools/patman/commit.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'tools/patman')

diff --git a/tools/patman/commit.py b/tools/patman/commit.py
index 7144e54..cbfbc46 100644
--- a/tools/patman/commit.py
+++ b/tools/patman/commit.py
@@ -22,7 +22,7 @@
 import re
 
 # Separates a tag: at the beginning of the subject from the rest of it
-re_subject_tag = re.compile('([^:]*):\s*(.*)')
+re_subject_tag = re.compile('([^:\s]*):\s*(.*)')
 
 class Commit:
     """Holds information about a single commit/patch in the series.
-- 
cgit v1.1


From 0d99fe0fd83d0a523ad2d48647bcc3e04293b0fd Mon Sep 17 00:00:00 2001
From: Simon Glass <sjg@chromium.org>
Date: Tue, 26 Mar 2013 13:09:41 +0000
Subject: patman: Fix the comment in CheckTags to mention multiple tags

This comment is less than helpful. Since multiple tags are supported, add
an example of how multiple tags work.

Signed-off-by: Simon Glass <sjg@chromium.org>
Reviewed-by: Doug Anderson <dianders@chromium.org>
---
 tools/patman/commit.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'tools/patman')

diff --git a/tools/patman/commit.py b/tools/patman/commit.py
index cbfbc46..468b50c 100644
--- a/tools/patman/commit.py
+++ b/tools/patman/commit.py
@@ -61,9 +61,10 @@ class Commit:
 
         Subject tags look like this:
 
-            propounder: Change the widget to propound correctly
+            propounder: fort: Change the widget to propound correctly
 
-        Multiple tags are supported. The list is updated in self.tag
+        Here the tags are propounder and fort. Multiple tags are supported.
+        The list is updated in self.tag.
 
         Returns:
             None if ok, else the name of a tag with no email alias
-- 
cgit v1.1


From ca706e768d01ee30bb0e6f6af7ca65b5a244b56d Mon Sep 17 00:00:00 2001
From: Simon Glass <sjg@chromium.org>
Date: Tue, 26 Mar 2013 13:09:45 +0000
Subject: patman: Minor help message/README fixes

A few of the help messages are not quite right, and there is a typo
in the README. Fix these.

Signed-off-by: Simon Glass <sjg@chromium.org>
Reviewed-by: Doug Anderson <dianders@chromium.org>
---
 tools/patman/README    | 2 +-
 tools/patman/patman.py | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

(limited to 'tools/patman')

diff --git a/tools/patman/README b/tools/patman/README
index 9922f2a..d4e7f36 100644
--- a/tools/patman/README
+++ b/tools/patman/README
@@ -68,7 +68,7 @@ will get a consistent result each time.
 How to configure it
 ===================
 
-For most cases of using patman for U-Boot developement patman will
+For most cases of using patman for U-Boot development, patman will
 locate and use the file 'doc/git-mailrc' in your U-Boot directory.
 This contains most of the aliases you will need.
 
diff --git a/tools/patman/patman.py b/tools/patman/patman.py
index 377408d..5000b7d 100755
--- a/tools/patman/patman.py
+++ b/tools/patman/patman.py
@@ -49,7 +49,7 @@ parser.add_option('-i', '--ignore-errors', action='store_true',
        dest='ignore_errors', default=False,
        help='Send patches email even if patch errors are found')
 parser.add_option('-n', '--dry-run', action='store_true', dest='dry_run',
-       default=False, help="Do a try run (create but don't email patches)")
+       default=False, help="Do a dry run (create but don't email patches)")
 parser.add_option('-p', '--project', default=project.DetectProject(),
                   help="Project name; affects default option values and "
                   "aliases [default: %default]")
@@ -72,7 +72,7 @@ parser.add_option('--no-tags', action='store_false', dest='process_tags',
 parser.usage = """patman [options]
 
 Create patches from commits in a branch, check them and email them as
-specified by tags you place in the commits. Use -n to """
+specified by tags you place in the commits. Use -n to do a dry run first."""
 
 
 # Parse options twice: first to get the project and second to handle
-- 
cgit v1.1


From 3fefd5efa664adc06ed49547e817f3e328266a15 Mon Sep 17 00:00:00 2001
From: Simon Glass <sjg@chromium.org>
Date: Wed, 3 Apr 2013 11:01:39 +0000
Subject: patman: Ignore all Gerrit Commit-* tags

These tags are used by Gerrit, so let's ignore all of them.

Signed-off-by: Simon Glass <sjg@chromium.org>
Reviewed-by: Doug Anderson <dianders@chromium.org>
---
 tools/patman/patchstream.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'tools/patman')

diff --git a/tools/patman/patchstream.py b/tools/patman/patchstream.py
index 5c2d3bc..4fda852 100644
--- a/tools/patman/patchstream.py
+++ b/tools/patman/patchstream.py
@@ -31,7 +31,7 @@ from series import Series
 
 # Tags that we detect and remove
 re_remove = re.compile('^BUG=|^TEST=|^BRANCH=|^Change-Id:|^Review URL:'
-    '|Reviewed-on:|Commit-Ready:')
+    '|Reviewed-on:|Commit-\w*:')
 
 # Lines which are allowed after a TEST= line
 re_allowed_after_test = re.compile('^Signed-off-by:')
-- 
cgit v1.1