Add a new script bb_host_steps.py which handles all host side steps.

build/android/buildbot/bb_device_steps.py

 #!/usr/bin/env python
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import collections
 import glob
-import json
 import multiprocessing
-import optparse
 import os
-import pipes
 import shutil
-import subprocess
 import sys
 
+import bb_utils
+
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
 from pylib import android_commands
 from pylib import buildbot_report
 from pylib import constants
 from pylib.gtest import gtest_config
 
 sys.path.append(os.path.join(
     constants.CHROME_DIR, 'third_party', 'android_testrunner'))
 import errors
 
 
-TESTING = 'BUILDBOT_TESTING' in os.environ
-
 CHROME_SRC = constants.CHROME_DIR
 
 # Describes an instrumation test suite:
 #   test: Name of test we're running.
 #   apk: apk to be installed.
 #   apk_package: package for the apk to be installed.
 #   test_apk: apk to run tests on.
 #   test_data: data folder in format destination:source.
 I_TEST = collections.namedtuple('InstrumentationTest', [
     'name', 'apk', 'apk_package', 'test_apk', 'test_data', 'host_driven_root'])
@@ skipping 14 matching lines @@
     I_TEST('AndroidWebView',
            'AndroidWebView.apk',
            'org.chromium.android_webview.shell',
            'AndroidWebViewTest',
            'webview:android_webview/test/data/device_files',
            None),
     ])
 
 VALID_TESTS = set(['chromedriver', 'ui', 'unit', 'webkit', 'webkit_layout'])
 
-
-def SpawnCmd(command):
-  """Spawn a process without waiting for termination."""
-  print '>', ' '.join(map(pipes.quote, command))
-  sys.stdout.flush()
-  if TESTING:
-    class MockPopen(object):
-      @staticmethod
-      def wait():
-        return 0
-    return MockPopen()
-
-  return subprocess.Popen(command, cwd=CHROME_SRC)
-
-def RunCmd(command, flunk_on_failure=True, halt_on_failure=False):
-  """Run a command relative to the chrome source root."""
-  code = SpawnCmd(command).wait()
-  print '<', ' '.join(map(pipes.quote, command))
-  if code != 0:
-    print 'ERROR: process exited with code %d' % code
-    if flunk_on_failure:
-      buildbot_report.PrintError()
-    else:
-      buildbot_report.PrintWarning()
-    # Allow steps to have both halting (i.e. 1) and non-halting exit codes.
-    if code != 0 and code != 88 and halt_on_failure:
-      raise OSError()
-  return code
+R = bb_utils.RunCmd

[Isaac (away), 2013/06/04 04:34:12]

This is fine, but how about we name this RunCmd?  Makes it a little easier to
parse the lines below for me.

[Siva Chandra, 2013/06/04 19:42:40]

Done.

 
 
 # multiprocessing map_async requires a top-level function for pickle library.
 def RebootDeviceSafe(device):
   """Reboot a device, wait for it to start, and squelch timeout exceptions."""
   try:
     android_commands.AndroidCommands(device).Reboot(True)
   except errors.DeviceUnresponsiveError as e:
     return e
 
 
 def RebootDevices():
   """Reboot all attached and online devices."""
   buildbot_report.PrintNamedStep('Reboot devices')
   # Early return here to avoid presubmit dependence on adb,
   # which might not exist in this checkout.
-  if TESTING:
+  if bb_utils.TESTING:
     return
   devices = android_commands.GetAttachedDevices()
   print 'Rebooting: %s' % devices
   if devices:
     pool = multiprocessing.Pool(len(devices))
     results = pool.map_async(RebootDeviceSafe, devices).get(99999)
 
     for device, result in zip(devices, results):
       if result:
         print '%s failed to startup.' % device
@@ skipping 14 matching lines @@
   args = ['--verbose']
   if options.target == 'Release':
     args.append('--release')
   if options.asan:
     args.append('--tool=asan')
   for suite in suites:
     buildbot_report.PrintNamedStep(suite.name)
     cmd = ['build/android/run_tests.py', '-s', suite.name] + args
     if suite.is_suite_exe:
       cmd.append('--exe')
-    RunCmd(cmd)
+    R(cmd)
 
 def RunBrowserTestSuite(options):
   """Manages an invocation of run_browser_tests.py.
 
   Args:
     options: options object.
   """
   args = ['--verbose']
   if options.target == 'Release':
     args.append('--release')
   if options.asan:
     args.append('--tool=asan')
   buildbot_report.PrintNamedStep(constants.BROWSERTEST_SUITE_NAME)
-  RunCmd(['build/android/run_browser_tests.py'] + args)
+  R(['build/android/run_browser_tests.py'] + args)
 
 def RunChromeDriverTests():
   """Run all the steps for running chromedriver tests."""
   buildbot_report.PrintNamedStep('chromedriver_annotation')
-  RunCmd(['chrome/test/chromedriver/run_buildbot_steps.py',
+  R(['chrome/test/chromedriver/run_buildbot_steps.py',
           '--android-package=%s' % constants.CHROMIUM_TEST_SHELL_PACKAGE])
 
 
 def CheckInstall():
   """Build bot step to see if adb install works on attached devices. """
   buildbot_report.PrintNamedStep('Check device install')
   # This step checks if apks can be installed on the devices.
   args = ['--apk', 'build/android/CheckInstallApk-debug.apk']
-  RunCmd(['build/android/adb_install_apk.py'] + args, halt_on_failure=True)
+  R(['build/android/adb_install_apk.py'] + args, halt_on_failure=True)
 
 
 def InstallApk(options, test, print_step=False):
   """Install an apk to all phones.
 
   Args:
     options: options object
     test: An I_TEST namedtuple
     print_step: Print a buildbot step
   """
   if print_step:
     buildbot_report.PrintNamedStep('install_%s' % test.name.lower())
   args = ['--apk', test.apk, '--apk_package', test.apk_package]
   if options.target == 'Release':
     args.append('--release')
 
-  RunCmd(['build/android/adb_install_apk.py'] + args, halt_on_failure=True)
+  R(['build/android/adb_install_apk.py'] + args, halt_on_failure=True)
 
 
 def RunInstrumentationSuite(options, test):
   """Manages an invocation of run_instrumentaiton_tests.py.
 
   Args:
     options: options object
     test: An I_TEST namedtuple
   """
   buildbot_report.PrintNamedStep('%s_instrumentation_tests' % test.name.lower())
 
   InstallApk(options, test)
   args = ['--test-apk', test.test_apk, '--test_data', test.test_data,
           '--verbose', '-I']
   if options.target == 'Release':
     args.append('--release')
   if options.asan:
     args.append('--tool=asan')
   if options.upload_to_flakiness_server:
     args.append('--flakiness-dashboard-server=%s' %
                 constants.UPSTREAM_FLAKINESS_SERVER)
   if test.host_driven_root:
     args.append('--python_test_root=%s' % test.host_driven_root)
 
-  RunCmd(['build/android/run_instrumentation_tests.py'] + args)
+  R(['build/android/run_instrumentation_tests.py'] + args)
 
 
 def RunWebkitLint(target):
   """Lint WebKit's TestExpectation files."""
   buildbot_report.PrintNamedStep('webkit_lint')
-  RunCmd(['webkit/tools/layout_tests/run_webkit_tests.py',
+  R(['webkit/tools/layout_tests/run_webkit_tests.py',
           '--lint-test-files',
           '--chromium',
           '--target', target])
 
 
 def RunWebkitLayoutTests(options):
   """Run layout tests on an actual device."""
   buildbot_report.PrintNamedStep('webkit_tests')
   cmd_args = [
         '--no-show-results',
@@ skipping 19 matching lines @@
   for f in options.factory_properties.get('additional_expectations', []):
     cmd_args.extend(
         ['--additional-expectations=%s' % os.path.join(CHROME_SRC, *f)])
 
   # TODO(dpranke): Remove this block after
   # https://codereview.chromium.org/12927002/ lands.
   for f in options.factory_properties.get('additional_expectations_files', []):
     cmd_args.extend(
         ['--additional-expectations=%s' % os.path.join(CHROME_SRC, *f)])
 
-  RunCmd(['webkit/tools/layout_tests/run_webkit_tests.py'] + cmd_args,
+  R(['webkit/tools/layout_tests/run_webkit_tests.py'] + cmd_args,
          flunk_on_failure=False)
 
 
 def MainTestWrapper(options):
   # Restart adb to work around bugs, sleep to wait for usb discovery.
-  RunCmd(['adb', 'kill-server'])
-  RunCmd(['adb', 'start-server'])
-  RunCmd(['sleep', '1'])
+  R(['adb', 'kill-server'])
+  R(['adb', 'start-server'])
+  R(['sleep', '1'])
 
   # Spawn logcat monitor
   logcat_dir = os.path.join(CHROME_SRC, 'out/logcat')
   shutil.rmtree(logcat_dir, ignore_errors=True)
-  SpawnCmd(['build/android/adb_logcat_monitor.py', logcat_dir])
+  bb_utils.SpawnCmd(['build/android/adb_logcat_monitor.py', logcat_dir])
 
   # Wait for logcat_monitor to pull existing logcat
-  RunCmd(['sleep', '5'])
+  R(['sleep', '5'])
 
   if options.reboot:
     RebootDevices()
 
   # Device check and alert emails
   buildbot_report.PrintNamedStep('device_status_check')
-  RunCmd(['build/android/device_status_check.py'], halt_on_failure=True)
+  R(['build/android/device_status_check.py'], halt_on_failure=True)
 
   # Provision devices
   buildbot_report.PrintNamedStep('provision_devices')
   target = options.factory_properties.get('target', 'Debug')
-  RunCmd(['build/android/provision_devices.py', '-t', target])
+  R(['build/android/provision_devices.py', '-t', target])
 
   # Check to see if devices can install apks.
   CheckInstall()
 
   if options.install:
     test_obj = INSTRUMENTATION_TESTS[options.install]
     InstallApk(options, test_obj, print_step=True)
 
   if 'chromedriver' in options.test_filter:
     RunChromeDriverTests()
   if 'unit' in options.test_filter:
     RunTestSuites(options, gtest_config.STABLE_TEST_SUITES)
   if 'ui' in options.test_filter:
     for test in INSTRUMENTATION_TESTS.itervalues():
       RunInstrumentationSuite(options, test)
   if 'webkit' in options.test_filter:
     RunTestSuites(options, [
         gtest_config.Apk('webkit_unit_tests'),
     ])
     RunWebkitLint(options.target)
   if 'webkit_layout' in options.test_filter:
     RunWebkitLayoutTests(options)
 
   if options.experimental:
     RunTestSuites(options, gtest_config.EXPERIMENTAL_TEST_SUITES)
     RunBrowserTestSuite(options)
 
   # Print logcat, kill logcat monitor
   buildbot_report.PrintNamedStep('logcat_dump')
-  RunCmd(['build/android/adb_logcat_printer.py', logcat_dir])
+  R(['build/android/adb_logcat_printer.py', logcat_dir])
 
   buildbot_report.PrintNamedStep('test_report')
   for report in glob.glob(
       os.path.join(CHROME_SRC, 'out', options.target, 'test_logs', '*.log')):
-    RunCmd(['cat', report])
+    R(['cat', report])
     os.remove(report)
 
 
 def main(argv):
-  parser = optparse.OptionParser()
+  buildbot_report.PrintNamedStep('Run tests')
 
-  def convert_json(option, _, value, parser):
-    setattr(parser.values, option.dest, json.loads(value))
-
-  parser.add_option('--build-properties', action='callback',
-                    callback=convert_json, type='string', default={},
-                    help='build properties in JSON format')
-  parser.add_option('--factory-properties', action='callback',
-                    callback=convert_json, type='string', default={},
-                    help='factory properties in JSON format')
-  parser.add_option('--slave-properties', action='callback',
-                    callback=convert_json, type='string', default={},
-                    help='Properties set by slave script in JSON format')
+  parser = bb_utils.GetParser()
   parser.add_option('--experimental', action='store_true',
                     help='Run experiemental tests')
   parser.add_option('-f', '--test-filter', metavar='<filter>', default=[],
                     action='append',
                     help=('Run a test suite. Test suites: "%s"' %
                           '", "'.join(VALID_TESTS)))
   parser.add_option('--asan', action='store_true', help='Run tests with asan.')
   parser.add_option('--install', metavar='<apk name>',
                     help='Install an apk by name')
   parser.add_option('--reboot', action='store_true',
                     help='Reboot devices before running tests')
   parser.add_option('--upload-to-flakiness-server', action='store_true',
                     help='Upload the results to the flakiness dashboard.')
   parser.add_option(
       '--auto-reconnect', action='store_true',
       help='Push script to device which restarts adbd on disconnections.')
   options, args = parser.parse_args(argv[1:])
 
-  def ParserError(msg):
-    """We avoid parser.error because it calls sys.exit."""
-    parser.print_help()
-    print >> sys.stderr, '\nERROR:', msg
-    return 1
-
   if args:
-    return ParserError('Unused args %s' % args)
+    return parser.Error('Unused args %s' % args)
 
   unknown_tests = set(options.test_filter) - VALID_TESTS
   if unknown_tests:
-    return ParserError('Unknown tests %s' % list(unknown_tests))
+    return parser.Error('Unknown tests %s' % list(unknown_tests))
 
   setattr(options, 'target', options.factory_properties.get('target', 'Debug'))
 
   # Add adb binary and chromium-source platform-tools to tip of PATH variable.
   android_paths = [os.path.join(constants.ANDROID_SDK_ROOT, 'platform-tools')]
 
   # Bots checkout chrome in /b/build/slave/<name>/build/src
   build_internal_android = os.path.abspath(os.path.join(
       CHROME_SRC, '..', '..', '..', '..', '..', 'build_internal', 'scripts',
       'slave', 'android'))
   if os.path.exists(build_internal_android):
     android_paths.insert(0, build_internal_android)
   os.environ['PATH'] = os.pathsep.join(android_paths + [os.environ['PATH']])
 
   MainTestWrapper(options)
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv))