Repeat Failed Tests in Serial

tools/isolate/run_test_cases.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Runs each test cases as a single shard, single process execution.
 
 Similar to sharding_supervisor.py but finer grained. Runs multiple instances in
 parallel.
 """
@@ skipping 436 matching lines @@
 
 def list_test_cases(executable, index, shards, disabled, fails, flaky):
   """Returns the list of test cases according to the specified criterias."""
   tests = parse_gtest_cases(gtest_list_tests(executable))
   if shards:
     tests = filter_shards(tests, index, shards)
   return filter_bad_tests(tests, disabled, fails, flaky)
 
 
 class Runner(object):
-  def __init__(self, executable, cwd_dir, timeout, progress):
+  def __init__(self, executable, cwd_dir, timeout, progress, retry_count=3):
     # Constants
     self.executable = executable
     self.cwd_dir = cwd_dir
     self.timeout = timeout
     self.progress = progress
-    self.retry_count = 3
+    self.retry_count = retry_count
     # It is important to remove the shard environment variables since it could
     # conflict with --gtest_filter.
     self.env = os.environ.copy()
     for name in GTEST_ENV_VARS_TO_REMOVE:
       self.env.pop(name, None)
     # Forcibly enable color by default, if not already disabled.
     self.env.setdefault('GTEST_COLOR', 'on')
 
   def map(self, test_case):
     """Traces a single test case and returns its output."""
@@ skipping 65 matching lines @@
     ]
   if whitelist:
     tests = [
       t for t in tests if any(fnmatch.fnmatch(t, s) for s in whitelist)
     ]
   logging.info(
       'Found %d test cases in %s' % (len(tests), os.path.basename(executable)))
   return tests
 
 
+def LogResults(result_file, results):
+  """Write the results out to a file if one is given."""
+  if not result_file:
+    return
+  with open(result_file, 'wb') as f:
+    json.dump(results, f, sort_keys=True, indent=2)
+
+
 def run_test_cases(executable, test_cases, jobs, timeout, result_file):
   """Traces test cases one by one."""
   progress = Progress(len(test_cases))
   with ThreadPool(jobs) as pool:
     function = Runner(executable, os.getcwd(), timeout, progress).map
     for test_case in test_cases:
       pool.add_task(function, test_case)
     results = pool.join(progress, 0.1)
     duration = time.time() - progress.start
   results = dict((item[0]['test_case'], item) for item in results)
-  if result_file:
-    with open(result_file, 'wb') as f:
-      json.dump(results, f, sort_keys=True, indent=2)
+  LogResults(result_file, results)
   sys.stdout.write('\n')
   total = len(results)
   if not total:
     return 1
 
   # Classify the results
   success = []
   flaky = []
   fail = []
   nb_runs = 0
   for test_case in sorted(results):
     items = results[test_case]
     nb_runs += len(items)
     if not any(not i['returncode'] for i in items):
       fail.append(test_case)
     elif len(items) > 1 and any(not i['returncode'] for i in items):
       flaky.append(test_case)
     elif len(items) == 1 and items[0]['returncode'] == 0:
       success.append(test_case)
     else:
       assert False, items
 
+  # Retry all the failures serially to see if they are just flaky when
+  # run at the same time.
+  if fail:
+    print 'Retrying failed tests serially.'
+    progress = Progress(len(fail))
+    function = Runner(
+        executable, os.getcwd(), timeout, progress, retry_count=1).map
+    test_cases_retry = fail[:]
+
+    for test_case in test_cases_retry:
+      output = function(test_case)
+      progress.print_update()
+      results[output[0]['test_case']].append(output)
+      if not output[0]['returncode']:
+        fail.remove(test_case)
+        flaky.append(test_case)
+
+    LogResults(result_file, results)
+    sys.stdout.write('\n')
+
+  print 'Summary:'
   for test_case in sorted(flaky):
     items = results[test_case]
     print '%s is flaky (tried %d times)' % (test_case, len(items))
 
+  for test_case in sorted(fail):
+    print '%s failed' % (test_case)
+
   print 'Success: %4d %5.2f%%' % (len(success), len(success) * 100. / total)
   print 'Flaky:   %4d %5.2f%%' % (len(flaky), len(flaky) * 100. / total)
   print 'Fail:    %4d %5.2f%%' % (len(fail), len(fail) * 100. / total)
   print '%.1fs Done running %d tests with %d executions. %.1f test/s' % (
       duration,
       len(results),
       nb_runs,
       nb_runs / duration)
   return int(bool(fail))
 
@@ skipping 124 matching lines @@
   return run_test_cases(
       executable,
       test_cases,
       options.jobs,
       options.timeout,
       result_file)
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv[1:]))