run_test_cases.py: Enable use of RUN_TEST_CASES_RESULT_FILE as environment variable.

tools/isolate/fix_test_cases.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Scripts to run a test, grab the failures and trace them."""
 
+import json
 import os
 import subprocess
 import sys
+import tempfile
 
 import run_test_cases
 
 
 if sys.platform == 'win32':
   import msvcrt  # pylint: disable=F0401
 
   def get_keyboard():
     """Returns a letter from the keyboard if any.
 
@@ skipping 20 matching lines @@
   """Traces a single test case and merges the result back into .isolate."""
   subprocess.call(
       [
         sys.executable, 'isolate.py', 'trace', '-r', result,
         '--', '--gtest_filter=' + test,
       ])
   return not subprocess.call(
       [sys.executable, 'isolate.py', 'merge', '-r', result])
 
 
+def run_all(result):
+  """Runs all the tests. Returns the tests that failed or None on failure.
+
+  Assumes run_test_cases.py is implicitly called.
+  """
+  handle, result_file = tempfile.mkstemp(prefix='run_test_cases')
+  os.close(handle)
+  env = os.environ.copy()
+  env['RUN_TEST_CASES_RESULT_FILE'] = result_file
+  subprocess.call(
+      [sys.executable, 'isolate.py', 'run', '-r', result], env=env)
+  if not os.path.isfile(result_file):
+    print >> sys.stderr, 'Failed to find %s' % result_file
+    return None
+  with open(result_file) as f:
+    data = json.load(f)
+  os.remove(result_file)
+  return [
+    test for test, runs in data.iteritems()
+    if not any(not run['returncode'] for run in runs)
+  ]
+
+
 def run(result, test):
   """Runs a single test case in an isolated environment.
 
   Returns True if the test passed.
   """
   return not subprocess.call([
     sys.executable, 'isolate.py', 'run', '-r', result,
-    '--', '--gtest_filter=' + test
+    '--', '--gtest_filter=' + test,
   ])
 
 
 def diff_and_commit(test):
   """Prints the diff and commit."""
   subprocess.call(['git', 'diff'])
   subprocess.call(['git', 'commit', '-a', '-m', test])
 
 
 def trace_and_verify(result, test):
   """Traces a test case, updates .isolate and makes sure it passes afterward.
 
   Return None if the test was already passing,  True on success.
   """
   trace_and_merge(result, test)
   diff_and_commit(test)
   return run(result, test)
 
 
-def run_all(result, executable):
+def fix_all(result):
   """Runs all the test cases in a gtest executable and trace the failing tests.
 
-  Then make sure the test passes afterward.
+  Returns True on success.
+
+  Makes sure the test passes afterward.
   """
-  test_cases = run_test_cases.list_test_cases(
-      executable, 0, 0, False, False, False)
-  print 'Found %d test cases.' % len(test_cases)
+  # These could have adverse side-effects.
+  # TODO(maruel): Be more intelligent about it, for now be safe.
+  for i in run_test_cases.KNOWN_GTEST_ENV_VARS:
+    if i in os.environ:
+      print >> 'Please unset %s' % i
+      return False
+
+  test_cases = run_all(result)
+  if test_cases is None:
+    return False
+
+  print '\nFound %d broken test cases.' % len(test_cases)
+  if not test_cases:
+    return True
+
   failures = []
   fixed_tests = []
   try:
     for index, test_case in enumerate(test_cases):
       if get_keyboard():
         # Return early.
         return True
 
       try:
         if run(result, test_case):
@@ skipping 32 matching lines @@
     executable += '.exe'
   if not os.path.isfile(executable):
     print >> sys.stderr, (
         '%s doesn\'t exist, please build %s_run' % (executable, basename))
     return 1
   if not os.path.isfile(result):
     print >> sys.stderr, (
         '%s doesn\'t exist, please build %s_run' % (result, basename))
     return 1
 
-  return not run_all(result, executable)
+  return not fix_all(result)
 
 
 if __name__ == '__main__':
   sys.exit(main())