Make device_status_check.py email clank-infra if devices go offline.

build/android/device_status_check.py

Index: build/android/device_status_check.py
diff --git a/build/android/device_status_check.py b/build/android/device_status_check.py
index b3447cae0f42bbb3e6bd1521baf13c48e5b7b8e8..3c93e0a8570637ccea3a4528b9488cf85c07c438 100755
--- a/build/android/device_status_check.py
+++ b/build/android/device_status_check.py
@@ -8,6 +8,7 @@
 
 import optparse
 import os
+import smtplib
 import sys
 
 from pylib import buildbot_report
@@ -59,27 +60,69 @@ def CheckForMissingDevices(options, adb_online_devs):
                      and online attached devices.
   """
   out_dir = os.path.abspath(options.out_dir)
+
+  def ReadDeviceList(file_name):
+    devices_path = os.path.join(out_dir, file_name)
+    devices = []
+    try:
+      with open(devices_path) as f:
+        devices = f.read().splitlines()
+    except IOError:
+      # Ignore error, file might not exist
+      pass
+    return devices
+
+  def WriteDeviceList(file_name, device_list):
+    path = os.path.join(out_dir, file_name)
+    if not os.path.exists(out_dir):
+      os.makedirs(out_dir)
+    with open(path, 'w') as f:
+      # Write devices currently visible plus devices previously seen.
+      f.write('\n'.join(set(device_list)))
+
   last_devices_path = os.path.join(out_dir, '.last_devices')
-  last_devices = []
-  try:
-    with open(last_devices_path) as f:
-      last_devices = f.read().splitlines()
-  except IOError:
-    # Ignore error, file might not exist
-    pass
+  last_devices = ReadDeviceList('.last_devices')
 
   missing_devs = list(set(last_devices) - set(adb_online_devs))
   if missing_devs:
+    from_address = 'buildbot@chromium.org'
+    to_address = 'chromium-android-device-alerts@google.com'
+    bot_name = os.environ['BUILDBOT_BUILDERNAME']
+    slave_name = os.environ['BUILDBOT_SLAVENAME']
+    subject = 'Device not detected on %s, %s.' % (slave_name, bot_name)

[Isaac (away), 2012/10/01 22:35:45]

I would like len(adb_online_devs) in the subject, which will let us know the
urgency of the email.

[navabi1, 2012/10/02 01:21:22]

Done.

+
     buildbot_report.PrintWarning()
-    buildbot_report.PrintSummaryText(
-        '%d devices not detected.' % len(missing_devs))
-    print 'Current online devices: %s' % adb_online_devs
-    print '%s are no longer visible. Were they removed?\n' % missing_devs
-    print 'SHERIFF: See go/chrome_device_monitor'
-    print 'Cache file: %s\n\n' % last_devices_path
-    print 'adb devices'
-    print GetCmdOutput(['adb', 'devices'])
+    devices_missing_msg = '%d devices not detected.' % len(missing_devs)
+    buildbot_report.PrintSummaryText(devices_missing_msg)
+
+    body = '\r\n'.join(
+        ['Current online devices: %s' % adb_online_devs,
+         '%s are no longer visible. Were they removed?\n' % missing_devs,
+         'SHERIFF: See go/clank/engineering/buildbots/troubleshooting',
+         'Cache file: %s\n\n' % last_devices_path,
+         'adb devices: %s' % GetCmdOutput(['adb', 'devices'])])
+
+    msg_body = '\r\n'.join(
+        ['From: %s' % from_address,
+         'To: %s' % to_address,
+         'Subject: %s' % subject,
+         '', body])
+
+    print body
+
+    last_missing = ReadDeviceList('.last_missing')

[Isaac (away), 2012/10/01 22:35:45]

Move line 113 to line 134.  See comment below

[navabi1, 2012/10/02 01:21:22]

Done.

+
+    # Only send email if the first time a particular device goes offline
+    new_missing_devs = list(set(missing_devs) - set(last_missing))
+    WriteDeviceList('.last_missing', missing_devs)
+
+    if new_missing_devs:
+      server = smtplib.SMTP('localhost')
+      server.sendmail(from_address, [to_address], msg_body)
+      server.quit()
   else:

[Isaac (away), 2012/10/01 22:35:45]

If a device goes offline immediately after we restart phones, this list will
never get reset and we won't receive emails from the bot as devices drop offline
again.

How about always writing missing_devs to .last_missing and email if there are
elements of missing devices not in .last_missing.  The means if a offline device
ever comes online we will get an email if it goes offline again.

[navabi1, 2012/10/02 01:21:22]

Done.

+    # No devices missing. Clear out .last_missing list.
+    WriteDeviceList('.last_missing', [])
     new_devs = set(adb_online_devs) - set(last_devices)
     if new_devs and os.path.exists(last_devices_path):
       buildbot_report.PrintWarning()
@@ -87,12 +130,7 @@ def CheckForMissingDevices(options, adb_online_devs):
           '%d new devices detected' % len(new_devs))
       print ('New devices detected %s. And now back to your '
              'regularly scheduled program.' % list(new_devs))
-
-  if not os.path.exists(out_dir):
-    os.makedirs(out_dir)
-  with open(last_devices_path, 'w') as f:
-    # Write devices currently visible plus devices previously seen.
-    f.write('\n'.join(set(adb_online_devs + last_devices)))
+    WriteDeviceList('.last_devices', (adb_online_devs + last_devices))
 
 
 def main():