Add a new script bb_host_steps.py which handles all host side steps.

build/android/buildbot/bb_run_bot.py

 #!/usr/bin/env python
 #
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import collections
 import copy
 import json
 import optparse
 import os
 import pipes
 import subprocess
 import sys
 
-sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
-from pylib import buildbot_report
-
 CHROME_SRC = os.path.abspath(
     os.path.join(os.path.dirname(__file__), '..', '..', '..'))
 
 GLOBAL_SLAVE_PROPS = {}
 
 BotConfig = collections.namedtuple(
-    'BotConfig', ['bot_id', 'bash_funs', 'test_obj', 'slave_props'])
+    'BotConfig', ['bot_id', 'host_opts', 'test_obj', 'slave_props'])
 TestConfig = collections.namedtuple('Tests', ['tests', 'extra_args'])
 Command = collections.namedtuple(
     'Command', ['step_name', 'command', 'testing_cmd'])
 
 
 def GetCommands(options, bot_config):
   """Get a formatted list of commands.
 
   Args:
     options: Options object.
     bot_config: A BotConfig named tuple.
   Returns:
     list of Command objects.
   """
   slave_props = dict(GLOBAL_SLAVE_PROPS)
   if bot_config.slave_props:
     slave_props.update(bot_config.slave_props)
 
   property_args = [
       '--factory-properties=%s' % json.dumps(options.factory_properties),
       '--build-properties=%s' % json.dumps(options.build_properties),
       '--slave-properties=%s' % json.dumps(slave_props)]
 
-  commands = []
-  def WrapWithBash(command):
-    """Wrap a bash command string with envsetup scripts."""
-    return ['bash', '-exc', '; '.join([
-        '. build/android/buildbot/buildbot_functions.sh',
-        'bb_baseline_setup %s %s' % (
-             CHROME_SRC,
-             ' '.join(map(pipes.quote, property_args))),
-        command])
-    ]
+  common_cmd = ['; '.join(['. build/android/buildbot/buildbot_functions.sh',
+                           'bb_baseline_setup %s %s' % (
+                               CHROME_SRC,
+                               ' '.join(map(pipes.quote, property_args)))])]
 
-  if bot_config.bash_funs:
-    # bash_funs command does not have a testing mode.
-    commands.append(
-        Command(None, WrapWithBash('; '.join(bot_config.bash_funs)), None))
+  commands = [Command('Preparation', common_cmd, None)]

[Isaac (away), 2013/06/05 03:03:07]

Why are you changing this?

[Siva Chandra, 2013/06/05 21:04:28]

This reply is for this comment and for another comment below.

We had a function 'WrapWithBash' which was being called to wrap bash functions
as well the invocation of bb_device_steps.py. Because of this, the step
'Environment Setup' was being executed twice:
http://build.chromium.org/p/chromium.linux/builders/Android%20Tests%20%28dbg%...

If I were to WrapWithBash even the invocation of bb_host_steps.py, the step
'Environment Setup' will be run thrice on builder-tests bots. Hence, what I have
done is not to wrap each script invocation with bash, but string together a
single command and run just that command. The stringing is done in the main
function below.

+
+  host_opts = bot_config.host_opts
+  if host_opts:
+    host_script = 'build/android/buildbot/bb_host_steps.py'
+    host_cmd = [host_script] + host_opts + property_args[0:2]
+    cmd = ' '.join(map(pipes.quote, host_cmd))
+    commands.append(Command('Host steps', [cmd], [cmd]))
 
   test_obj = bot_config.test_obj
   if test_obj:
     run_test_cmd = [
         'build/android/buildbot/bb_device_steps.py', '--reboot'] + property_args
     for test in test_obj.tests:
       run_test_cmd.extend(['-f', test])
     if test_obj.extra_args:
       run_test_cmd.extend(test_obj.extra_args)
-    commands.append(Command(
-        'Run tests',
-        WrapWithBash(' '.join(map(pipes.quote, run_test_cmd))), run_test_cmd))
+    cmd = ' '.join(map(pipes.quote, run_test_cmd))
+    commands.append(Command('Run tests', [cmd], [cmd]))
 
   return commands
 
 
 def GetBotStepMap():
-  compile_step = ['bb_compile']
-  std_build_steps = ['bb_compile', 'bb_zip_build']
-  std_test_steps = ['bb_extract_build']
+  compile_opt = ['--compile']
+  std_host_tests = ['--host-tests=check_webview_licenses,findbugs']
+  std_build_opts = ['--compile', '--zipbuild']
+  std_test_opts = ['--extract-build']
   std_tests = ['ui', 'unit']
   flakiness_server = '--upload-to-flakiness-server'
   extra_gyp = 'extra_gyp_defines'
 
   def B(bot_id, bash_funs, test_obj=None, slave_props=None):
     return BotConfig(bot_id, bash_funs, test_obj, slave_props)
 
   def T(tests, extra_args=None):
     return TestConfig(tests, extra_args)
 
   bot_configs = [
       # Main builders
-      B('main-builder-dbg',
-        ['bb_check_webview_licenses', 'bb_compile', 'bb_run_findbugs',
-         'bb_zip_build']),
-      B('main-builder-rel', ['bb_compile', 'bb_zip_build']),
-      B('main-clang-builder', compile_step, slave_props={extra_gyp: 'clang=1'}),
-      B('main-clobber', compile_step),
-      B('main-tests', std_test_steps, T(std_tests, [flakiness_server])),
+      B('main-builder-dbg', std_build_opts + std_host_tests),
+      B('main-builder-rel', std_build_opts),
+      B('main-clang-builder', compile_opt),
+      B('main-clobber', compile_opt),
+      B('main-tests', std_test_opts, T(std_tests, [flakiness_server])),
 
       # Other waterfalls
-      B('asan-builder-tests', compile_step + ['bb_asan_tests_setup'],
+      B('asan-builder-tests', compile_opt + ['--asan-tests-setup'],
         T(std_tests, ['--asan']), {extra_gyp: 'asan=1'}),
-      B('chromedriver-fyi-tests-dbg', std_test_steps,
+      B('chromedriver-fyi-tests-dbg', std_test_opts,
         T(['chromedriver'], ['--install=ChromiumTestShell'])),
       B('fyi-builder-dbg',
-        ['bb_check_webview_licenses', 'bb_compile', 'bb_compile_experimental',
-         'bb_run_findbugs', 'bb_zip_build']),
-      B('fyi-builder-rel',
-        ['bb_compile', 'bb_compile_experimental', 'bb_zip_build']),
-      B('fyi-tests-dbg-ics-gn', ['bb_compile', 'bb_compile_experimental'],
+        std_build_opts + std_host_tests + ['--experimental']),
+      B('fyi-builder-rel', std_build_opts + ['--experimental']),
+      B('fyi-tests-dbg-ics-gn', compile_opt + [ '--experimental'],
         T(std_tests, ['--experimental', flakiness_server])),
-      B('fyi-tests', std_test_steps,
+      B('fyi-tests', std_test_opts,
         T(std_tests, ['--experimental', flakiness_server])),
-      B('fyi-component-builder-tests-dbg', compile_step,
-        T(std_tests, ['--experimental', flakiness_server]),
-        {extra_gyp: 'component=shared_library'}),
-      B('perf-tests-rel', std_test_steps, T([], ['--install=ContentShell'])),
-      B('webkit-latest-webkit-tests', std_test_steps,
+      B('fyi-component-builder-tests-dbg', compile_opt,
+        T(std_tests, ['--experimental', flakiness_server])),
+      B('perf-tests-rel', std_test_opts, T([], ['--install=ContentShell'])),
+      B('webkit-latest-webkit-tests', std_test_opts,
         T(['webkit_layout', 'webkit'])),
-      B('webkit-latest-contentshell', compile_step, T(['webkit_layout'])),
-      B('builder-unit-tests', compile_step, T(['unit'])),
+      B('webkit-latest-contentshell', compile_opt, T(['webkit_layout'])),
+      B('builder-unit-tests', compile_opt, T(['unit'])),
 
       # Generic builder config (for substring match).
-      B('builder', std_build_steps),
+      B('builder', std_build_opts),
   ]
 
   bot_map = dict((config.bot_id, config) for config in bot_configs)
 
   # These bots have identical configuration to ones defined earlier.
   copy_map = [
       ('lkgr-clobber', 'main-clobber'),
       ('try-builder-dbg', 'main-builder-dbg'),
       ('try-builder-rel', 'main-builder-rel'),
       ('try-clang-builder', 'main-clang-builder'),
@@ skipping 51 matching lines @@
     if substring_matches:
       max_id = max(substring_matches, key=len)
       print 'Using config from id="%s" (substring match).' % max_id
       bot_config = bot_map[max_id]
   if not bot_config:
     print 'Error: config for id="%s" cannot be inferred.' % bot_id
     return 1
 
   print 'Using config:', bot_config
 
   def CommandToString(command):

[Isaac (away), 2013/06/05 03:03:07]

move this function to bb_utils

[Siva Chandra, 2013/06/05 21:04:28]

Done.

     """Returns quoted command that can be run in bash shell."""
     return ' '.join(map(pipes.quote, command))
 
   command_objs = GetCommands(options, bot_config)
   for command_obj in command_objs:
-    print 'Will run:', CommandToString(command_obj.command)
+    print 'Will run "%s" step: %s' % (command_obj.step_name,
+                                      CommandToString(command_obj.command))
 
+  commands = []
   for command_obj in command_objs:
-    if command_obj.step_name:

[Isaac (away), 2013/06/05 03:03:07]

why are you removing this?

-      buildbot_report.PrintNamedStep(command_obj.step_name)
-    command = command_obj.command
-    print CommandToString(command)
-    sys.stdout.flush()
-    env = None
     if options.TESTING:
-      if not command_obj.testing_cmd:
-        continue
-      return_code = subprocess.call(
-          command_obj.testing_cmd,
-          cwd=CHROME_SRC,
-          env=dict(os.environ, BUILDBOT_TESTING='1'))
+      if command_obj.testing_cmd:
+        commands += command_obj.testing_cmd
     else:
-      return_code = subprocess.call(command, cwd=CHROME_SRC, env=env)
-    if return_code != 0:
-      return return_code
+      commands += command_obj.command
+  if options.TESTING:
+    return_code = subprocess.call(
+        ['bash', '-exc', '; '.join(commands)], cwd=CHROME_SRC,

[Isaac (away), 2013/06/05 03:03:07]

why are you combining the commands?

+         env=dict(os.environ, BUILDBOT_TESTING='1'))
+  else:
+    return_code = subprocess.call(
+        ['bash', '-exc', '; '.join(commands)], cwd=CHROME_SRC)
+
+  if return_code != 0:
+    return return_code
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv))