Retry tests on other bots if the device is unresponsive/offline

build/android/pylib/base_test_sharder.py

Index: build/android/pylib/base_test_sharder.py
diff --git a/build/android/pylib/base_test_sharder.py b/build/android/pylib/base_test_sharder.py
index 48206c202ce1774a402da8bbc98f2aa9a5269491..5306769155050ba032364650daeecaa1581e609b 100644
--- a/build/android/pylib/base_test_sharder.py
+++ b/build/android/pylib/base_test_sharder.py
@@ -3,6 +3,7 @@
 # found in the LICENSE file.
 
 
+import android_commands
 import logging
 import multiprocessing
 
@@ -96,8 +97,17 @@ class BaseTestSharder(object):
       # So use map_async instead.
       async_results = pool.map_async(_ShardedTestRunnable, test_runners)
       results_lists = async_results.get(999999)
+
       test_results = TestResults.FromTestResults(results_lists)
-      if retry == self.retries - 1:
+      # Re-check the attached devices for some devices may
+      # become offline
+      retry_devices = set(android_commands.GetAttachedDevices())
+      # Remove devices that had exceptions.
+      retry_devices -= TestResults.DeviceExceptions(results_lists)
+      # Retry on devices that didn't have any exception.
+      self.attached_devices = list(retry_devices)
+      if (retry == self.retries - 1 or
+          len(self.attached_devices) == 0):
         all_passed = final_results.ok + test_results.ok
         final_results = test_results
         final_results.ok = all_passed