Parallelize cpplint in presubmit and fix usage of DISALLOW_* macros.

tools/presubmit.py

 #!/usr/bin/env python
 #
-# Copyright 2011 the V8 project authors. All rights reserved.
+# Copyright 2012 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
 #       with the distribution.
@@ skipping 21 matching lines @@
   md5er = md5.new
 
 
 import optparse
 import os
 from os.path import abspath, join, dirname, basename, exists
 import pickle
 import re
 import sys
 import subprocess
+import multiprocessing
 from subprocess import PIPE
 
 # Disabled LINT rules and reason.
 # build/include_what_you_use: Started giving false positives for variables
 #  named "string" and "map" assuming that you needed to include STL headers.
 
 ENABLED_LINT_RULES = """
 build/class
 build/deprecated
 build/endif_comment
@@ skipping 39 matching lines @@
 whitespace/labels
 whitespace/line_length
 whitespace/newline
 whitespace/operators
 whitespace/parens
 whitespace/tab
 whitespace/todo
 """.split()
 
 
+LINT_OUTPUT_PATTERN = re.compile(r'^.+[:(]\d+[:)]|^Done processing')
+
+
+def CppLintWorker(command):
+  try:
+    process = subprocess.Popen(command, stderr=subprocess.PIPE)
+    process.wait()
+    out_lines = ""
+    error_count = -1
+    while True:
+      out_line = process.stderr.readline()
+      if out_line == '' and process.poll() != None:
+        break
+      m = LINT_OUTPUT_PATTERN.match(out_line)
+      if m:
+        out_lines += out_line
+        error_count += 1
+    sys.stderr.write(out_lines)
+    return error_count
+  except KeyboardInterrupt:
+    child.kill()

[Jakob Kummerow, 2012/01/20 14:56:22]

I think you meant s/child/process/

[Yang, 2012/01/20 16:18:31]

This is actually interesting. Previously the script only reacted to ctrl+c
because child.kill() results in a syntax error and get the worker killed.

But of course your solution is the better one.

+  except:
+    print('Error running cpplint.py. Please make sure you have depot_tools' +
+          ' in your $PATH. Lint check skipped.')
+    child.kill()

[Jakob Kummerow, 2012/01/20 14:56:22]

same here.

[Yang, 2012/01/20 16:18:31]

Done.

+
+
 class FileContentsCache(object):
 
   def __init__(self, sums_file_name):
     self.sums = {}
     self.sums_file_name = sums_file_name
 
   def Load(self):
     try:
       sums_file = None
       try:
@@ skipping 85 matching lines @@
 
   def ProcessFiles(self, files, path):
     good_files_cache = FileContentsCache('.cpplint-cache')
     good_files_cache.Load()
     files = good_files_cache.FilterUnchangedFiles(files)
     if len(files) == 0:
       print 'No changes in files detected. Skipping cpplint check.'
       return True
 
     filt = '-,' + ",".join(['+' + n for n in ENABLED_LINT_RULES])
-    command = ['cpplint.py', '--filter', filt] + join(files)
+    command = ['cpplint.py', '--filter', filt]
     local_cpplint = join(path, "tools", "cpplint.py")
     if exists(local_cpplint):
-      command = ['python', local_cpplint, '--filter', filt] + join(files)
+      command = ['python', local_cpplint, '--filter', filt]
 
+    commands = join([command + [file] for file in files])
+    count = multiprocessing.cpu_count()
+    pool = multiprocessing.Pool(count)
     try:
-      process = subprocess.Popen(command, stderr=subprocess.PIPE)
-    except:
-      print('Error running cpplint.py. Please make sure you have depot_tools' +
-            ' in your $PATH. Lint check skipped.')
-      return True
-    LINT_ERROR_PATTERN = re.compile(r'^(.+)[:(]\d+[:)]')
-    while True:
-      out_line = process.stderr.readline()
-      if out_line == '' and process.poll() != None:
-        break
-      sys.stderr.write(out_line)
-      m = LINT_ERROR_PATTERN.match(out_line)
-      if m:
-        good_files_cache.RemoveFile(m.group(1))
+      results = pool.map(CppLintWorker, commands)

[Jakob Kummerow, 2012/01/20 14:56:22]

I need the following patch to make this react properly to Ctrl+C:
-      results = pool.map(CppLintWorker, commands)
+      # workaround for http://bugs.python.org/issue8296
+      results = pool.map_async(CppLintWorker, commands).get(999999)

[Yang, 2012/01/20 16:18:31]

Done.

+    except KeyboardInterrupt:
+      print "\nCaught KeyboardInterrupt, terminating workers."
+      sys.exit(1)
 
+    for i in range(len(files)):
+      if results[i] > 0:
+        good_files_cache.RemoveFile(files[i])
+
+    print "Total errors found: %d" % sum(results)
     good_files_cache.Save()
-    return process.returncode == 0

[Jakob Kummerow, 2012/01/20 14:56:22]

I think we still want a return value here. IIUC, it should be:
return sum(results) == 0

[Yang, 2012/01/20 16:18:31]

Done.

 
 
 COPYRIGHT_HEADER_PATTERN = re.compile(
     r'Copyright [\d-]*20[0-1][0-9] the V8 project authors. All rights reserved.')
 
 class SourceProcessor(SourceFileProcessor):
   """
   Check that all files include a copyright notice and no trailing whitespaces.
   """
 
@@ skipping 106 matching lines @@
   print "Running copyright header and trailing whitespaces check..."
   success = SourceProcessor().Run(workspace) and success
   if success:
     return 0
   else:
     return 1
 
 
 if __name__ == '__main__':
   sys.exit(Main())