Android: Make number of test retries configurable.

build/android/pylib/base/shard.py

 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Implements test sharding logic."""
 
 import logging
 import threading
 
 from pylib import android_commands
@@ skipping 98 matching lines @@
 
   def __iter__(self):
     """Iterate through tests in the collection until all have been handled."""
     while True:
       r = self._pop()
       if r is None:
         break
       yield r
 
 
-def _RunTestsFromQueue(runner, test_collection, out_results, watcher):
+def _RunTestsFromQueue(runner, test_collection, out_results, watcher,
+    num_retries):
   """Runs tests from the test_collection until empty using the given runner.
 
   Adds TestRunResults objects to the out_results list and may add tests to the
   out_retry list.
 
   Args:
     runner: A TestRunner object used to run the tests.
     test_collection: A _TestCollection from which to get _Test objects to run.
     out_results: A list to add TestRunResults to.
     watcher: A watchdog_timer.WatchdogTimer object, used as a shared timeout.
+    num_retries: Number of retries for a test.
   """
   for test in test_collection:
     watcher.Reset()
     try:
       if not android_commands.IsDeviceAttached(runner.device):
         # Device is unresponsive, stop handling tests on this device.
         msg = 'Device %s is unresponsive.' % runner.device
         logging.warning(msg)
         raise android_commands.errors.DeviceUnresponsiveError(msg)
       result, retry = runner.RunTest(test.test)
       test.tries += 1
-      if retry and test.tries <= 3:
+      if retry and test.tries <= num_retries:
         # Retry non-passing results, only record passing results.
         pass_results = base_test_result.TestRunResults()
         pass_results.AddResults(result.GetPass())
         out_results.append(pass_results)
         logging.warning('Will retry test, try #%s.' % test.tries)
         test_collection.add(_Test(test=retry, tries=test.tries))
       else:
         # All tests passed or retry limit reached. Either way, record results.
         out_results.append(result)
     except android_commands.errors.DeviceUnresponsiveError:
@@ skipping 28 matching lines @@
   try:
     index = threadsafe_counter.GetAndIncrement()
     logging.warning('Creating shard %s for device %s.', index, device)
     runner = runner_factory(device, index)
     runner.SetUp()
     out_runners.append(runner)
   except android_commands.errors.DeviceUnresponsiveError as e:
     logging.warning('Failed to create shard for %s: [%s]', device, e)
 
 
-def _RunAllTests(runners, tests, timeout=None):
+def _RunAllTests(runners, tests, num_retries, timeout=None):
   """Run all tests using the given TestRunners.
 
   Args:
     runners: a list of TestRunner objects.
     tests: a list of Tests to run using the given TestRunners.
+    num_retries: number of retries for a test.
     timeout: watchdog timeout in seconds, defaults to the default timeout.
 
   Returns:
     A TestRunResults object.
   """
   logging.warning('Running %s tests with %s test runners.' %
                   (len(tests), len(runners)))
   tests_collection = _TestCollection([_Test(t) for t in tests])
   results = []
   watcher = watchdog_timer.WatchdogTimer(timeout)
   workers = reraiser_thread.ReraiserThreadGroup(
-      [reraiser_thread.ReraiserThread(_RunTestsFromQueue,
-                                      [r, tests_collection, results, watcher],
-                                      name=r.device[-4:])
+      [reraiser_thread.ReraiserThread(
+          _RunTestsFromQueue,
+          [r, tests_collection, results, watcher, num_retries],
+          name=r.device[-4:])
        for r in runners])
   workers.StartAll()
   workers.JoinAll(watcher)
   run_results = base_test_result.TestRunResults()
   for r in results:
     run_results.AddTestRunResults(r)
   return run_results
 
 
 def _CreateRunners(runner_factory, devices, timeout=None):
@@ skipping 32 matching lines @@
   """
   threads = reraiser_thread.ReraiserThreadGroup(
       [reraiser_thread.ReraiserThread(r.TearDown, name=r.device[-4:])
        for r in runners])
   threads.StartAll()
   threads.JoinAll(watchdog_timer.WatchdogTimer(timeout))
 
 
 def ShardAndRunTests(runner_factory, devices, tests, build_type='Debug',
                      test_timeout=DEFAULT_TIMEOUT,
-                     setup_timeout=DEFAULT_TIMEOUT):
+                     setup_timeout=DEFAULT_TIMEOUT,
+                     num_retries=2):
   """Run all tests on attached devices, retrying tests that don't pass.
 
   Args:
     runner_factory: callable that takes a device and index and returns a
       TestRunner object.
     devices: list of attached device serial numbers as strings.
     tests: list of tests to run.
     build_type: either 'Debug' or 'Release'.
     test_timeout: watchdog timeout in seconds for running tests, defaults to the
       default timeout.
     setup_timeout: watchdog timeout in seconds for creating and cleaning up
       test runners, defaults to the default timeout.
+    num_retries: number of retries for a test.
 
   Returns:
     A base_test_result.TestRunResults object.
   """
   forwarder.Forwarder.KillHost(build_type)
   runners = _CreateRunners(runner_factory, devices, setup_timeout)
   try:
-    return _RunAllTests(runners, tests, test_timeout)
+    return _RunAllTests(runners, tests, num_retries, test_timeout)
   finally:
     try:
       _TearDownRunners(runners, setup_timeout)
     except android_commands.errors.DeviceUnresponsiveError as e:
       logging.warning('Device unresponsive during TearDown: [%s]', e)
     finally:
       forwarder.Forwarder.KillHost(build_type)