[Android] Refactor the buildbot scripts to prepare for downstream usage.

build/android/buildbot/bb_device_steps.py

 #!/usr/bin/env python
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import collections
 import glob
 import multiprocessing
 import os
 import shutil
 import sys
 
 import bb_utils
 
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
 from pylib import android_commands
 from pylib import buildbot_report
 from pylib import constants
 from pylib.gtest import gtest_config
 
 sys.path.append(os.path.join(
     constants.DIR_SOURCE_ROOT, 'third_party', 'android_testrunner'))
 import errors
 
 
 CHROME_SRC = constants.DIR_SOURCE_ROOT
+LOGCAT_DIR = os.path.join(CHROME_SRC, 'out', 'logcat')
 
 # Describes an instrumation test suite:
 #   test: Name of test we're running.
 #   apk: apk to be installed.
 #   apk_package: package for the apk to be installed.
 #   test_apk: apk to run tests on.
 #   test_data: data folder in format destination:source.
+#   host_driven_root: The python test root directory.
+#   annotation: Annotation of the tests to include.
+#   exclude_annotation: The annotation of the tests to exclude.
 I_TEST = collections.namedtuple('InstrumentationTest', [
-    'name', 'apk', 'apk_package', 'test_apk', 'test_data', 'host_driven_root'])
+    'name', 'apk', 'apk_package', 'test_apk', 'test_data', 'host_driven_root',
+    'annotation', 'exclude_annotation', 'extra_flags'])
+
+def I(name, apk, apk_package, test_apk, test_data, host_driven_root=None,
+      annotation=None, exclude_annotation=None, extra_flags=None):
+  return I_TEST(name, apk, apk_package, test_apk, test_data, host_driven_root,
+                annotation, exclude_annotation, extra_flags)
 
 INSTRUMENTATION_TESTS = dict((suite.name, suite) for suite in [
-    I_TEST('ContentShell',
-           'ContentShell.apk',
-           'org.chromium.content_shell_apk',
-           'ContentShellTest',
-           'content:content/test/data/android/device_files',
-           None),
-    I_TEST('ChromiumTestShell',
-           'ChromiumTestShell.apk',
-           'org.chromium.chrome.testshell',
-           'ChromiumTestShellTest',
-           'chrome:chrome/test/data/android/device_files',
-           constants.CHROMIUM_TEST_SHELL_HOST_DRIVEN_DIR),
-    I_TEST('AndroidWebView',
-           'AndroidWebView.apk',
-           'org.chromium.android_webview.shell',
-           'AndroidWebViewTest',
-           'webview:android_webview/test/data/device_files',
-           None),
+    I('ContentShell',
+      'ContentShell.apk',
+      'org.chromium.content_shell_apk',
+      'ContentShellTest',
+      'content:content/test/data/android/device_files'),
+    I('ChromiumTestShell',
+      'ChromiumTestShell.apk',
+      'org.chromium.chrome.testshell',
+      'ChromiumTestShellTest',
+      'chrome:chrome/test/data/android/device_files',
+      constants.CHROMIUM_TEST_SHELL_HOST_DRIVEN_DIR),
+    I('AndroidWebView',
+      'AndroidWebView.apk',
+      'org.chromium.android_webview.shell',
+      'AndroidWebViewTest',
+      'webview:android_webview/test/data/device_files'),
     ])
 
 VALID_TESTS = set(['chromedriver', 'ui', 'unit', 'webkit', 'webkit_layout'])
 
 RunCmd = bb_utils.RunCmd
 
 
 # multiprocessing map_async requires a top-level function for pickle library.
 def RebootDeviceSafe(device):
   """Reboot a device, wait for it to start, and squelch timeout exceptions."""
@@ skipping 51 matching lines @@
     options: options object.
   """
   args = ['--verbose', '--num_retries=1']
   if options.target == 'Release':
     args.append('--release')
   if options.asan:
     args.append('--tool=asan')
   buildbot_report.PrintNamedStep(constants.BROWSERTEST_SUITE_NAME)
   RunCmd(['build/android/run_browser_tests.py'] + args)
 
-def RunChromeDriverTests():
+def RunChromeDriverTests(_):
   """Run all the steps for running chromedriver tests."""
   buildbot_report.PrintNamedStep('chromedriver_annotation')
   RunCmd(['chrome/test/chromedriver/run_buildbot_steps.py',
           '--android-package=%s' % constants.CHROMIUM_TEST_SHELL_PACKAGE])
 
 def InstallApk(options, test, print_step=False):
   """Install an apk to all phones.
 
   Args:
     options: options object
@@ skipping 23 matching lines @@
           '--verbose', '-I']
   if options.target == 'Release':
     args.append('--release')
   if options.asan:
     args.append('--tool=asan')
   if options.upload_to_flakiness_server:
     args.append('--flakiness-dashboard-server=%s' %
                 constants.UPSTREAM_FLAKINESS_SERVER)
   if test.host_driven_root:
     args.append('--python_test_root=%s' % test.host_driven_root)
+  if test.annotation:
+    args.extend(['-A', test.annotation])
+  if test.exclude_annotation:
+    args.extend(['-E', test.exclude_annotation])
+  if test.extra_flags:
+    args.extend(test.extra_flags)
 
   RunCmd(['build/android/run_instrumentation_tests.py'] + args)
 
 
 def RunWebkitLint(target):
   """Lint WebKit's TestExpectation files."""
   buildbot_report.PrintNamedStep('webkit_lint')
   RunCmd(['webkit/tools/layout_tests/run_webkit_tests.py',
           '--lint-test-files',
           '--chromium',
@@ skipping 31 matching lines @@
   # TODO(dpranke): Remove this block after
   # https://codereview.chromium.org/12927002/ lands.
   for f in options.factory_properties.get('additional_expectations_files', []):
     cmd_args.extend(
         ['--additional-expectations=%s' % os.path.join(CHROME_SRC, *f)])
 
   RunCmd(['webkit/tools/layout_tests/run_webkit_tests.py'] + cmd_args,
          flunk_on_failure=False)
 
 
-def MainTestWrapper(options):
+def SpawnLogcatMonitor():
+  shutil.rmtree(LOGCAT_DIR, ignore_errors=True)
+  bb_utils.SpawnCmd([
+      os.path.join(CHROME_SRC, 'build', 'android', 'adb_logcat_monitor.py'),
+      LOGCAT_DIR])
+
+  # Wait for logcat_monitor to pull existing logcat
+  RunCmd(['sleep', '5'])
+
+def ProvisionDevices(options):
   # Restart adb to work around bugs, sleep to wait for usb discovery.
   RunCmd(['adb', 'kill-server'])
   RunCmd(['adb', 'start-server'])
   RunCmd(['sleep', '1'])
 
-  # Spawn logcat monitor
-  logcat_dir = os.path.join(CHROME_SRC, 'out/logcat')
-  shutil.rmtree(logcat_dir, ignore_errors=True)
-  bb_utils.SpawnCmd(['build/android/adb_logcat_monitor.py', logcat_dir])
-
-  # Wait for logcat_monitor to pull existing logcat
-  RunCmd(['sleep', '5'])
-
-  # Provision devices
   buildbot_report.PrintNamedStep('provision_devices')
   if options.reboot:
     RebootDevices()
   RunCmd(['build/android/provision_devices.py', '-t', options.target])
 
-  # Device check and alert emails
+
+def DeviceStatusCheck(_):
   buildbot_report.PrintNamedStep('device_status_check')
   RunCmd(['build/android/device_status_check.py'], halt_on_failure=True)
 
-  if options.install:
-    test_obj = INSTRUMENTATION_TESTS[options.install]
-    InstallApk(options, test_obj, print_step=True)
 
-  if 'chromedriver' in options.test_filter:
-    RunChromeDriverTests()
-  if 'unit' in options.test_filter:
-    RunTestSuites(options, gtest_config.STABLE_TEST_SUITES)
-  if 'ui' in options.test_filter:
-    for test in INSTRUMENTATION_TESTS.itervalues():
-      RunInstrumentationSuite(options, test)
-  if 'webkit' in options.test_filter:
-    RunTestSuites(options, [
-        gtest_config.Apk('webkit_unit_tests'),
-    ])
-    RunWebkitLint(options.target)
-  if 'webkit_layout' in options.test_filter:
-    RunWebkitLayoutTests(options)
+def GetDeviceSetupStepCmds():
+  return [
+    ('provision_devices', ProvisionDevices),
+    ('device_status_check', DeviceStatusCheck)
+  ]
 
-  if options.experimental:
-    RunTestSuites(options, gtest_config.EXPERIMENTAL_TEST_SUITES)
-    RunBrowserTestSuite(options)
 
+def RunUnitTests(options):
+  RunTestSuites(options, gtest_config.STABLE_TEST_SUITES)
+
+
+def RunInstrumentationTests(options):
+  for test in INSTRUMENTATION_TESTS.itervalues():
+    RunInstrumentationSuite(options, test)
+
+
+def RunWebkitTests(options):
+  RunTestSuites(options, [gtest_config.Apk('webkit_unit_tests')])
+  RunWebkitLint(options.target)
+
+
+def GetTestStepCmds():
+  return [
+      ('chromedriver', RunChromeDriverTests),
+      ('unit', RunUnitTests),
+      ('ui', RunInstrumentationTests),
+      ('webkit', RunWebkitTests),
+      ('webkit_layout', RunWebkitLayoutTests)
+  ]
+
+
+def LogcatDump(options):
   # Print logcat, kill logcat monitor
   buildbot_report.PrintNamedStep('logcat_dump')
-  RunCmd(['build/android/adb_logcat_printer.py', logcat_dir])
+  cmd_str = 'build/android/adb_logcat_printer.py ' + LOGCAT_DIR
+  if options.logcat_dump_output:
+    cmd_str += ' | tee %s' % options.logcat_dump_output

[Isaac (away), 2013/06/29 09:55:39]

I'm not sure I like using bash here -- could we always save to file and then cat
back?  We also don't necessarily need to take the logcat_dump_output option --
WDYT about always saving in the same place?

You'd need to add a 'stdout' option to run command and then do something like:

with open(<logcat file path>, 'w') as f:
   RunCmd([adb_logcat_printer.py, LOGCAT_DIR], stdout=f)

[Siva Chandra, 2013/07/01 23:58:53]

Done.

+  RunCmd(['bash', '-c', cmd_str])
 
+
+def GenerateTestReport(options):
   buildbot_report.PrintNamedStep('test_report')
   for report in glob.glob(
       os.path.join(CHROME_SRC, 'out', options.target, 'test_logs', '*.log')):
     RunCmd(['cat', report])
     os.remove(report)
 
 
-def main(argv):
+def GetPostTestStepCmds():
+  return [
+      ('logcat_dump', LogcatDump),
+      ('test_report', GenerateTestReport)
+  ]
+
+
+def MainTestWrapper(options):
+  # Spawn logcat monitor
+  SpawnLogcatMonitor()
+
+  bb_utils.RunAllSteps(GetDeviceSetupStepCmds(), options)
+
+  if options.install:
+    test_obj = INSTRUMENTATION_TESTS[options.install]
+    InstallApk(options, test_obj, print_step=True)
+
+  if options.test_filter:
+    bb_utils.RunSteps(options.test_filter, GetTestStepCmds(), options)
+
+  if options.experimental:
+    RunTestSuites(options, gtest_config.EXPERIMENTAL_TEST_SUITES)
+    RunBrowserTestSuite(options)
+
+  bb_utils.RunAllSteps(GetPostTestStepCmds(), options)
+
+
+def GetDeviceStepsOptParser():
   parser = bb_utils.GetParser()
   parser.add_option('--experimental', action='store_true',
                     help='Run experiemental tests')
   parser.add_option('-f', '--test-filter', metavar='<filter>', default=[],
                     action='append',
                     help=('Run a test suite. Test suites: "%s"' %
                           '", "'.join(VALID_TESTS)))
   parser.add_option('--asan', action='store_true', help='Run tests with asan.')
   parser.add_option('--install', metavar='<apk name>',
                     help='Install an apk by name')
   parser.add_option('--reboot', action='store_true',
                     help='Reboot devices before running tests')
   parser.add_option('--upload-to-flakiness-server', action='store_true',
                     help='Upload the results to the flakiness dashboard.')
   parser.add_option(
       '--auto-reconnect', action='store_true',
       help='Push script to device which restarts adbd on disconnections.')
+  parser.add_option(
+      '--logcat-dump-output',
+      help='The logcat dump output will be "tee"-ed into this file')
+
+  return parser
+
+
+def main(argv):
+  parser = GetDeviceStepsOptParser()
   options, args = parser.parse_args(argv[1:])
 
   if args:
     return sys.exit('Unused args %s' % args)
 
   unknown_tests = set(options.test_filter) - VALID_TESTS
   if unknown_tests:
     return sys.exit('Unknown tests %s' % list(unknown_tests))
 
   setattr(options, 'target', options.factory_properties.get('target', 'Debug'))
 
   MainTestWrapper(options)
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv))