Parallelize pylint PRESUBMIT checks.

presubmit_canned_checks.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Generic presubmit checks that can be reused by other presubmit checks."""
 
 import os as _os
 _HERE = _os.path.dirname(_os.path.abspath(__file__))
 
 # Justifications for each filter:
@@ skipping 770 matching lines @@
       description = '%s files' % len(files)
 
     return input_api.Command(
         name='Pylint (%s)' % description,
         cmd=[input_api.python_executable,
              input_api.os_path.join(_HERE, 'third_party', 'pylint.py'),
              '--args-on-stdin'],
         kwargs={'env': env, 'stdin': '\n'.join(files + extra_args)},
         message=error_type)
 
-  # Always run pylint and pass it all the py files at once.
-  # Passing py files one at time is slower and can produce
-  # different results.  input_api.verbose used to be used
-  # to enable this behaviour but differing behaviour in
-  # verbose mode is not desirable.
-  # Leave this unreachable code in here so users can make
-  # a quick local edit to diagnose pylint issues more
-  # easily.
+  # Always run pylint and pass it all the (sharded) py files at once. Passing py
+  # files one at time is slower and can produce different results.
+  # input_api.verbose used to be used to enable this behaviour but differing
+  # behaviour in verbose mode is not desirable.
+  #
+  # We shard the pylint invocations by the number of CPUs, since they tend to
+  # saturate a CPU entirely (but never more than 100%, thanks to the GIL).
   if True:
-    return [GetPylintCmd(files)]
+    # number of files to allow in one command
+    limit = max(len(files) / input_api.cpu_count, 1)
+    ret = []
+    while files:
+      ret.append(GetPylintCmd(files[:limit]))
+      files = files[limit:]

[nednguyen, 2015/06/12 18:06:14]

If we shard the files arbitrarily, how does this affects the effectiveness of
the linting jobs and some lint check like "import-error" requires global state
of all files?

+    return ret
   else:
+    # Leave this unreachable code in here so users can make a quick local edit
+    # to diagnose pylint issues more easily.
     return map(lambda x: GetPylintCmd([x]), files)
 
 
 def RunPylint(input_api, *args, **kwargs):
   """Legacy presubmit function.
 
   For better performance, get all tests and then pass to
   input_api.RunTests.
   """
   return input_api.RunTests(GetPylint(input_api, *args, **kwargs), False)
@@ skipping 317 matching lines @@
   for f in affected_files:
     cmd = ['gn', 'format', '--dry-run', f.AbsoluteLocalPath()]
     rc = gn.main(cmd)
     if rc == 2:
       warnings.append(output_api.PresubmitPromptWarning(
           '%s requires formatting. Please run `gn format --in-place %s`.' % (
               f.AbsoluteLocalPath(), f.LocalPath())))
   # It's just a warning, so ignore other types of failures assuming they'll be
   # caught elsewhere.
   return warnings