Add LogDog bootstrapping to `annotated_run.py`.

scripts/slave/annotated_run.py

 #!/usr/bin/env python
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import argparse
+import collections
 import contextlib
+import datetime
+import hashlib
+import itertools
 import json
-import optparse
+import logging
 import os
+import platform
 import shutil
 import socket
 import subprocess
 import sys
 import tempfile
-import traceback
-
+
+
+# Install Infra build environment.
 BUILD_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(
-    os.path.abspath(__file__))))
-sys.path.append(os.path.join(BUILD_ROOT, 'scripts'))
-sys.path.append(os.path.join(BUILD_ROOT, 'third_party'))
+                             os.path.abspath(__file__))))
+sys.path.insert(0, os.path.join(BUILD_ROOT, 'scripts'))
+import common.env
+common.env.Install()
 
 from common import annotator
 from common import chromium_utils
 from common import master_cfg_utils
-
-SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__))
-BUILD_LIMITED_ROOT = os.path.join(
-    os.path.dirname(BUILD_ROOT), 'build_internal', 'scripts', 'slave')
-
-PACKAGE_CFG = os.path.join(
-    os.path.dirname(os.path.dirname(SCRIPT_PATH)),
-    'infra', 'config', 'recipes.cfg')
-
-if sys.platform.startswith('win'):
-  # TODO(pgervais): add windows support
-  # QQ: Where is infra/run.py on windows machines?
-  RUN_CMD = None
-else:
-  RUN_CMD = os.path.join('/', 'opt', 'infra-python', 'run.py')
-
-@contextlib.contextmanager
-def namedTempFile():
-  fd, name = tempfile.mkstemp()
-  os.close(fd)  # let the exceptions fly
-  try:
-    yield name
-  finally:
+from gerrit_util import GceAuthenticator
+
+SCRIPT_PATH = os.path.join(common.env.Build, 'scripts', 'slave')
+BUILD_LIMITED_ROOT = os.path.join(common.env.BuildInternal, 'scripts', 'slave')
+
+# Logging instance.
+LOGGER = logging.getLogger('annotated_run')
+
+# Return codes used by Butler/Annotee to indicate their failure (as opposed to
+# a forwarded return code from the underlying process).
+LOGDOG_ERROR_RETURNCODES = (

[iannucci, 2015/12/01 02:38:01]

I think the logdog stuff should be in a different FILE, but should be called
from annotated_run. That's what I meant before, but maybe I wasn't clear.

+    # Butler runtime error.
+    250,
+    # Annotee runtime error.
+    251,
+)
+
+# Whitelist of {master}=>[{builder}|WHITELIST_ALL] whitelisting specific masters

[iannucci, 2015/12/01 02:38:01]

is {builder} a set?

[dnj, 2015/12/01 03:36:04]

I was using that notation to mean "key", but I can update to not intersect with
Python.

+# and builders for experimental LogDog/Annotee export.
+LOGDOG_WHITELIST_MASTER_BUILDERS = {
+}
+
+# Sentinel value that, if present in master config, matches all builders
+# underneath that master.
+WHITELIST_ALL = '*'
+
+# Configuration for a Pub/Sub topic.
+PubSubConfig = collections.namedtuple('PubSubConfig', ('project', 'topic'))
+
+# RecipeRuntime will probe this for values.
+# - First, (system, platform)
+# - Then, (system,)
+# - Finally, (),
+PLATFORM_CONFIG = {
+  # All systems.
+  (): {
+    'logdog_pubsub': PubSubConfig(
+        project='luci-logdog',
+        topic='chrome-infra-beta',
+    ),
+  },
+
+  # Linux
+  ('Linux',): {
+    'run_cmd': '/opt/infra-python/run.py',
+    'cipd_static_paths': (
+      # XXX: Get this right?
+      '/opt/chrome-infra',
+    ),
+    'credential_paths': (
+      # XXX: Get this right?
+      '/opt/infra/service_accounts',
+    ),
+    'logdog_butler_streamserver_gen': lambda rt: os.path.join(rt.workdir,
+                                                              'butler.sock'),
+  },
+  ('Linux', 'x86_64'): {
+    'logdog_cipd_packages': {
+      'infra/tools/luci/logdog/butler/linux-amd64': 'testing',
+      'infra/tools/luci/logdog/annotee/linux-amd64': 'testing',
+    },
+  },
+
+  # Mac OSX
+  ('Darwin',): {
+    'run_cmd': '/opt/infra-python/run.py',
+  },
+
+  # Windows
+  ('Windows',): {},
+}
+
+
+class LogDogNotBootstrapped(Exception):
+  pass
+
+
+class LogDogBootstrapError(Exception):
+  pass
+
+
+def is_executable(path):
+  return os.path.isfile(path) and os.access(path, os.X_OK)
+
+
+def ensure_directory(*path):
+  path = os.path.join(*path)
+  if not os.path.isdir(path):
+    os.makedirs(path)
+  return path
+
+
+def _run_command(cmd, **kwargs):
+  dry_run = kwargs.pop('dry_run', False)
+
+  LOGGER.debug('Executing command: %s', cmd)
+  if dry_run:
+    LOGGER.info('(Dry Run) Not executing command.')
+    return 0, ''
+  proc = subprocess.Popen(cmd, stderr=subprocess.STDOUT)
+  stdout, _ = proc.communicate()
+
+  LOGGER.debug('Process [%s] returned [%d] with output:\n%s',
+               cmd, proc.returncode, stdout)
+  return proc.returncode, stdout
+
+
+def _check_command(*args, **kwargs):
+  rv, stdout = _run_command(args, **kwargs)
+  if rv != 0:
+    raise ValueError('Process exited with non-zero return code (%d)' % (rv,))
+  return stdout
+
+
+class RecipeRuntime(object):
+  """RecipeRuntime is the platform-specific runtime enviornment."""
+
+  _SENTINEL = object()
+
+  def __init__(self, **kwargs):
+    self._fields = kwargs
+
+  @classmethod
+  @contextlib.contextmanager
+  def enter(cls, leak, **kw):

[iannucci, 2015/12/01 02:38:01]

what is leak? docstring?

[dnj, 2015/12/01 03:36:04]

Done.

+    # Build our platform fields.
+    p = (platform.system(), platform.processor())
+    fields = {}
+    for i in xrange(len(p)+1):
+      fields.update(PLATFORM_CONFIG.get(p[:i], {}))
+    fields.update(kw)

[iannucci, 2015/12/01 02:38:01]

why not put this stuff in __init__? why is there `fields` and also
`self._fields`? What are fields?

[dnj, 2015/12/01 03:36:04]

In general, I don't like __init__ doing actual work, so a classmethod is better
here. Added doc, renamed to "attrs".

+
+    basedir = ensure_directory(os.getcwd(), '.recipe_runtime')
     try:
-      os.remove(name)
-    except OSError as e:
-      print >> sys.stderr, "LEAK: %s: %s" % (name, e)
+      tdir = tempfile.mkdtemp(dir=basedir)
+      LOGGER.debug('Using temporary directory [%s].', tdir)
+
+      fields['workdir'] = tdir
+      yield cls(**fields)
+    finally:
+      if basedir and os.path.isdir(basedir):
+        if not leak:
+          LOGGER.debug('Cleaning up temporary directory [%s].', basedir)
+          try:
+            # TODO(pgervais): use infra_libs.rmtree instead.
+            shutil.rmtree(basedir)
+          except Exception:
+            LOGGER.exception('Failed to clean up temporary directory [%s].',
+                             basedir)
+        else:
+          LOGGER.warning('(--leak) Leaking temporary directory [%s].', basedir)
+
+  def __getattr__(self, key):

[iannucci, 2015/12/01 02:38:01]

this seems unnecessarily fancy... why not just use get() everywhere? Or
__getitem__ if you must.

[dnj, 2015/12/01 03:36:04]

I dunno, I think this usage looks way better.

+    # Class methods/variables.
+    value = getattr(super(RecipeRuntime, self), key, self._SENTINEL)
+    if value is not self._SENTINEL:
+      return value
+
+    value = getattr(self, 'get')(key, self._SENTINEL)
+    if value is not self._SENTINEL:
+      return value
+    raise KeyError(key)
+
+  def get(self, key, default=None):
+    value = self._fields.get(key, self._SENTINEL)
+    if value is not self._SENTINEL:
+      return value
+    return default
+
+  def __str__(self):
+    return str(self._fields)
+
+
+class CIPD(object):

[iannucci, 2015/12/01 02:38:01]

separate file?

[dnj, 2015/12/01 03:36:04]

I wasn't intending to make this a first-class library, or annotated_run.py for
that matter. Do you really think that's warranted here?

+  _CIPD_NAME = 'cipd'
+
+  def __init__(self, path, root):
+    self._cipd_path = path
+    self._root = root
+
+  @classmethod
+  def find(cls, rt, rootdir):
+    for p in itertools.chain(
+        iter(os.environ.get('PATH').split(os.pathsep)),
+        rt.cipd_static_paths):
+      candidate = os.path.join(p, cls._CIPD_NAME)
+      if os.path.isfile(candidate) and os.access(candidate, os.X_OK):
+        return cls(candidate, rootdir)
+    return None
+
+  def __call__(self, *args):
+    cmd = [self._cipd_path]
+    cmd.extend(args)
+    _check_command(*cmd)
+
+  def path(self, *components):
+    return os.path.join(self._root, *components)
+
+  def ensure(self, **packages):
+    if len(packages) == 0:
+      return
+
+    # Emit package list.
+    package_list = self.path('package_list.txt')
+    lines = [
+      '# Automatically generated CIPD package list (launcher.py)',
+      '# Generated at: %s' % (datetime.datetime.now().isoformat(),),
+      '',
+    ]
+    for pkg, version in sorted(packages.iteritems()):
+      lines.append('%s %s' % (pkg, version))
+
+    ensure_directory(self._root)
+    with open(package_list, 'w+') as fd:
+      fd.write('\n'.join(lines))
+
+    # Ensure against the package list.
+    args = [
+      'ensure',
+      '-root', self._root,
+      '-list', package_list,
+    ]
+    self(*args)
+
+
+def _get_service_account_json(rt, opts):
+  """Returns (str/None): If specified, the path to the service account JSON.
+
+  This method probes the local environemnt and returns a (possibly empty) list
+  of arguments to add to the Butler command line for authentication.
+
+  If we're running on a GCE instance, no arguments will be returned, as GCE
+  service account is implicitly authenticated. If we're running on Baremetal,
+  a path to those credentials will be returned.
+
+  Args:
+    rt (RecipeRuntime): The runtime environment.
+  Raises:
+    |LogDogBootstrapError| if no credentials could be found.
+  """
+  path = opts.get('service_account_json')
+  if path:
+    return path
+
+  if GceAuthenticator.is_gce():
+    LOGGER.info('Running on GCE. No credentials necessary.')
+    return None
+
+  for credential_path in rt.get('credential_paths', ()):
+    candidate = os.path.join(credential_path, 'logdog_service_account.json')
+    if os.path.isfile(candidate):
+      return candidate
+
+  raise LogDogBootstrapError('Could not find service account credentials.')
+
+
+def _logdog_bootstrap(rt, opts, cmd):

[iannucci, 2015/12/01 02:38:01]

need docstrings for these suckers

[dnj, 2015/12/01 03:36:04]

Done.

+  bootstrap_dir = ensure_directory(rt.tempdir, 'logdog_bootstrap')
+  butler, annotee = opts.logdog_butler_path, opts.logdog_annotee_path
+  if not (butler and annotee):
+    # Load packages via CIPD.
+    cipd = CIPD.find(rt, os.path.join(bootstrap_dir, 'cipd_root'))
+    if rt.logdog_cipd_packages:
+      if not cipd:
+        raise LogDogBootstrapError('Could not find CIPD binary.')
+      cipd.ensure(**rt.logdog_cipd_packages)
+    if not butler:
+      butler = cipd.path('logdog_butler')
+    if not annotee:
+      annotee = cipd.path('logdog_annotee')
+
+  if not is_executable(annotee):
+    raise LogDogNotBootstrapped('Annotee is not executable: %s' % (annotee,))
+  if not is_executable(butler):
+    raise LogDogNotBootstrapped('Butler is not executable: %s' % (butler,))
+
+  # Determine LogDog verbosity.
+  logdog_verbose = []
+  if opts.logdog_verbose == 0:
+    pass
+  elif opts.logdog_verbose == 1:
+    logdog_verbose.extend('-log_level=info')
+  else:
+    logdog_verbose.extend('-log_level=debug')
+
+  service_account_args = []
+  service_account_json = _get_service_account_json(rt, opts)
+  if service_account_json:
+    service_account_args += ['-service-account-json', service_account_json]
+
+  streamserver_uri_gen = rt.logdog_butler_streamserver_gen
+  if not streamserver_uri_gen:
+    raise LogDogBootstrapError('No streamserver URI generator.')
+  streamserver_uri = streamserver_uri_gen(rt.workdir)
+
+  # Dump Annotee command to JSON.
+  cmd_json = os.path.join(bootstrap_dir, 'annotee_cmd.json')
+  with open(cmd_json, 'w') as fd:
+    json.dump(cmd, fd)
+
+  cmd = [
+      # Butler Command.
+      butler,
+      ] + logdog_verbose + service_account_args + [
+      '-output', 'gcps,project="%s",topic="%s"' % (rt.logdog_pubsub.project,
+                                                   rt.logdog_pubsub.topic),
+      'run',
+      '-streamserver-uri', streamserver_uri,
+      '--',
+
+      # Annotee Command.
+      annotee,
+      ] + logdog_verbose + [
+      '-json-args-path', cmd_json,
+      ]
+  rv, _ = _run_command(cmd, dry_run=opts.dry_run)
+  if rv in LOGDOG_ERROR_RETURNCODES:
+    raise LogDogBootstrapError('LogDog Error (%d)' % (rv,))
+  return rv
+
+
+def _assert_logdog_whitelisted(rt):
+  """Asserts that the runtime environment is whitelisted for LogDog bootstrap.
+
+  Args:
+    rt (RecipeRuntime): The runtime to test.
+  Raises:
+    LogDogNotBootstrapped: if the runtime is not whitelisted.
+  """
+  mastername, buildername = rt.get('mastername'), rt.get('buildername')
+  if not all((mastername, buildername)):
+    raise LogDogNotBootstrapped('Required mastername/buildernmae is not set.')
+
+  # Key on mastername.
+  bdict = LOGDOG_WHITELIST_MASTER_BUILDERS.get(mastername)
+  if bdict is not None:
+    # Key on buildername.
+    if WHITELIST_ALL in bdict or buildername in bdict:
+      LOGGER.info('Whitelisted master %s, builder %s.',
+                  mastername, buildername)
+      return
+  raise LogDogNotBootstrapped('Master %s, builder %s is not whitelisted.' % (
+                              mastername, buildername))
 
 
 def get_recipe_properties(build_properties, use_factory_properties_from_disk):
   """Constructs the recipe's properties from buildbot's properties.
 
   This retrieves the current factory properties from the master_config
   in the slave's checkout (no factory properties are handed to us from the
   master), and merges in the build properties.
 
   Using the values from the checkout allows us to do things like change
@@ skipping 44 matching lines @@
     for name, value in factory_properties.items():
       if not build_properties.has_key(name):
         s.set_build_property(name, json.dumps(value))
 
     # Build properties override factory properties.
     properties = factory_properties.copy()
     properties.update(build_properties)
     return properties
 
 
-def get_factory_properties_from_disk(mastername, buildername):
+def get_factory_properties_from_disk(rt):
   master_list = master_cfg_utils.GetMasters()
   master_path = None
   for name, path in master_list:
-    if name == mastername:
+    if name == rt.mastername:
       master_path = path
 
   if not master_path:
-    raise LookupError('master "%s" not found.' % mastername)
+    raise LookupError('master "%s" not found.' % rt.mastername)
 
   script_path = os.path.join(BUILD_ROOT, 'scripts', 'tools',
                              'dump_master_cfg.py')
 
-  with namedTempFile() as fname:
-    dump_cmd = [sys.executable,
-                script_path,
-                master_path, fname]
-    proc = subprocess.Popen(dump_cmd, cwd=BUILD_ROOT, stdout=subprocess.PIPE,
-                            stderr=subprocess.PIPE)
-    out, err = proc.communicate()
-    exit_code = proc.returncode
+  master_json = os.path.join(rt.workdir, 'dump_master_cfg.json')
+  dump_cmd = [sys.executable,
+              script_path,
+              master_path, master_json]
+  proc = subprocess.Popen(dump_cmd, cwd=BUILD_ROOT, stdout=subprocess.PIPE,
+                          stderr=subprocess.PIPE)
+  out, err = proc.communicate()
+  if proc.returncode:
+    raise LookupError('Failed to get the master config; dump_master_cfg %s'
+                      'returned %d):\n%s\n%s\n'% (
+                      rt.mastername, proc.returncode, out, err))
 
-    if exit_code:
-      raise LookupError('Failed to get the master config; dump_master_cfg %s'
-                        'returned %d):\n%s\n%s\n'% (
-                        mastername, exit_code, out, err))
-
-    with open(fname, 'rU') as f:
-      config = json.load(f)
+  with open(master_json, 'rU') as f:
+    config = json.load(f)
 
   # Now extract just the factory properties for the requested builder
   # from the master config.
   props = {}
   found = False
   for builder_dict in config['builders']:
-    if builder_dict['name'] == buildername:
+    if builder_dict['name'] == rt.buildername:
       found = True
       factory_properties = builder_dict['factory']['properties']
       for name, (value, _) in factory_properties.items():
         props[name] = value
 
   if not found:
     raise LookupError('builder "%s" not found on in master "%s"' %
-                      (buildername, mastername))
+                      (rt.buildername, rt.mastername))
 
   if 'recipe' not in props:
     raise LookupError('Cannot find recipe for %s on %s' %
-                      (buildername, mastername))
+                      (rt.buildername, rt.mastername))
 
   return props
 
 
 def get_args(argv):
   """Process command-line arguments."""
+  parser = argparse.ArgumentParser(

[iannucci, 2015/12/01 02:38:01]

separate CL please

[dnj, 2015/12/01 03:36:04]

mmk

+      description='Entry point for annotated builds.')
+  parser.add_argument('-v', '--verbose',
+      action='count', default=0,
+      help='Increase verbosity. This can be specified multiple times.')
+  parser.add_argument('-d', '--dry-run', action='store_true',
+      help='Perform the setup, but refrain from executing the recipe.')
+  parser.add_argument('-l', '--leak', action='store_true',
+      help="Refrain from cleaning up generated artifacts.")
+  parser.add_argument('--build-properties',
+      type=json.loads, default={},
+      help='build properties in JSON format')
+  parser.add_argument('--factory-properties',
+      type=json.loads, default={},
+      help='factory properties in JSON format')
+  parser.add_argument('--build-properties-gz', dest='build_properties',
+      type=chromium_utils.convert_gz_json_type, default={},
+      help='build properties in b64 gz JSON format')
+  parser.add_argument('--factory-properties-gz', dest='factory_properties',
+      type=chromium_utils.convert_gz_json_type, default={},
+      help='factory properties in b64 gz JSON format')
+  parser.add_argument('--keep-stdin', action='store_true', default=False,
+      help='don\'t close stdin when running recipe steps')
+  parser.add_argument('--master-overrides-slave', action='store_true',
+      help='use the property values given on the command line from the master, '
+           'not the ones looked up on the slave')
+  parser.add_argument('--use-factory-properties-from-disk',
+      action='store_true', default=False,
+      help='use factory properties loaded from disk on the slave')
 
-  parser = optparse.OptionParser(
-      description='Entry point for annotated builds.')
-  parser.add_option('--build-properties',
-                    action='callback', callback=chromium_utils.convert_json,
-                    type='string', default={},
-                    help='build properties in JSON format')
-  parser.add_option('--factory-properties',
-                    action='callback', callback=chromium_utils.convert_json,
-                    type='string', default={},
-                    help='factory properties in JSON format')
-  parser.add_option('--build-properties-gz',
-                    action='callback', callback=chromium_utils.convert_gz_json,
-                    type='string', default={}, dest='build_properties',
-                    help='build properties in b64 gz JSON format')
-  parser.add_option('--factory-properties-gz',
-                    action='callback', callback=chromium_utils.convert_gz_json,
-                    type='string', default={}, dest='factory_properties',
-                    help='factory properties in b64 gz JSON format')
-  parser.add_option('--keep-stdin', action='store_true', default=False,
-                    help='don\'t close stdin when running recipe steps')
-  parser.add_option('--master-overrides-slave', action='store_true',
-                    help='use the property values given on the command line '
-                         'from the master, not the ones looked up on the slave')
-  parser.add_option('--use-factory-properties-from-disk',
-                    action='store_true', default=False,
-                    help='use factory properties loaded from disk on the slave')
+  group = parser.add_argument_group('LogDog Bootstrap')
+  group.add_argument('-V', '--logdog-verbose',
+      action='count', default=0,
+      help='Increase LogDog verbosity. This can be specified multiple times.')
+  group.add_argument('-f', '--logdog-force', action='store_true',
+      help='Force LogDog bootstrapping, even if the system is not configured.')
+  group.add_argument('--logdog-butler-path',
+      help='Path to the LogDog Butler. If empty, one will be probed/downloaded '
+           'from CIPD.')
+  group.add_argument('--logdog-annotee-path',
+      help='Path to the LogDog Annotee. If empty, one will be '
+           'probed/downloaded from CIPD.')
+  group.add_argument('--logdog-service-account-json',
+      help='Path to the service account JSON. If one is not provided, the '
+           'local system credentials will be used.')
   return parser.parse_args(argv)
 
 
 def update_scripts():
-  if os.environ.get('RUN_SLAVE_UPDATED_SCRIPTS'):
-    os.environ.pop('RUN_SLAVE_UPDATED_SCRIPTS')
+  if os.environ.pop('RUN_SLAVE_UPDATED_SCRIPTS', None) is None:

[iannucci, 2015/12/01 02:38:01]

technically this is a semantic change: before it would only be popped if it was
set and non-empty. Now it will be popped if it's empty. Not sure if that's
desired or not.

[dnj, 2015/12/01 03:36:04]

Good point, I'll go ahead and revert this.

     return False
 
   stream = annotator.StructuredAnnotationStream()
 
   with stream.step('update_scripts') as s:
     gclient_name = 'gclient'
     if sys.platform.startswith('win'):
       gclient_name += '.bat'
     gclient_path = os.path.join(BUILD_ROOT, '..', 'depot_tools', gclient_name)
     gclient_cmd = [gclient_path, 'sync', '--force', '--verbose']
     try:
       fd, output_json = tempfile.mkstemp()
       os.close(fd)
       gclient_cmd += ['--output-json', output_json]
     except Exception:
       # Super paranoia try block.
       output_json = None
     cmd_dict = {
         'name': 'update_scripts',
         'cmd': gclient_cmd,
         'cwd': BUILD_ROOT,
     }
     annotator.print_step(cmd_dict, os.environ, stream)
-    if subprocess.call(gclient_cmd, cwd=BUILD_ROOT) != 0:
+    rv, _ = _run_command(gclient_cmd, cwd=BUILD_ROOT)
+    if rv != 0:
       s.step_text('gclient sync failed!')
       s.step_warnings()
     elif output_json:
       try:
         with open(output_json, 'r') as f:
           gclient_json = json.load(f)
         for line in json.dumps(
             gclient_json, sort_keys=True,
             indent=4, separators=(',', ': ')).splitlines():
           s.step_log_line('gclient_json', line)
         s.step_log_end('gclient_json')
         revision = gclient_json['solutions']['build/']['revision']
         scm = gclient_json['solutions']['build/']['scm']
         s.step_text('%s - %s' % (scm, revision))
         s.set_build_property('build_scm', json.dumps(scm))
         s.set_build_property('build_revision', json.dumps(revision))
       except Exception as e:
         s.step_text('Unable to process gclient JSON %s' % repr(e))
         s.step_warnings()
       finally:
         try:
           os.remove(output_json)
         except Exception as e:
-          print >> sys.stderr, "LEAKED:", output_json, e
+          LOGGER.warning("LEAKED: %s", output_json, exc_info=True)
     else:
       s.step_text('Unable to get SCM data')
       s.step_warnings()
 
     os.environ['RUN_SLAVE_UPDATED_SCRIPTS'] = '1'
 
     # After running update scripts, set PYTHONIOENCODING=UTF-8 for the real
     # annotated_run.
     os.environ['PYTHONIOENCODING'] = 'UTF-8'
 
     return True
 
 
 def clean_old_recipe_engine():
   """Clean stale pycs from the old location of recipe_engine.
 
   This function should only be needed for a little while after the recipe
   packages rollout (2015-09-16).
   """
   for (dirpath, _, filenames) in os.walk(
       os.path.join(BUILD_ROOT, 'third_party', 'recipe_engine')):
     for filename in filenames:
       if filename.endswith('.pyc'):
-        path = os.path.join(dirpath, filename)
-        os.remove(path)
+        os.remove(os.path.join(dirpath, filename))
 
 
-@contextlib.contextmanager
-def build_data_directory():
-  """Context manager that creates a build-specific directory.
+def write_monitoring_event(rt, outdir):
+  if not (rt.run_cmd and os.path.exists(rt.run_cmd)):
+    LOGGER.warning('Unable to find run.py at %s, no events will be sent.',
+                   rt.run_cmd)
+    return
 
-  The directory is wiped when exiting.
+  hostname = socket.getfqdn()
+  if hostname:  # just in case getfqdn() returns None.
+    hostname = hostname.split('.')[0]
+  else:
+    hostname = None
 
-  Yields:
-    build_data (str or None): full path to a writeable directory. Return None if
-        no directory can be found or if it's not writeable.
-  """
-  prefix = 'build_data'
-
-  # TODO(pgervais): import that from infra_libs.logs instead
-  if sys.platform.startswith('win'):  # pragma: no cover
-    DEFAULT_LOG_DIRECTORIES = [
-      'E:\\chrome-infra-logs',
-      'C:\\chrome-infra-logs',
-    ]
-  else:
-    DEFAULT_LOG_DIRECTORIES = ['/var/log/chrome-infra']
-
-  build_data_dir = None
-  for candidate in DEFAULT_LOG_DIRECTORIES:
-    if os.path.isdir(candidate):
-      build_data_dir = os.path.join(candidate, prefix)
-      break
-
-  # Remove any leftovers and recreate the dir.
-  if build_data_dir:
-    print >> sys.stderr, "Creating directory"
-    # TODO(pgervais): use infra_libs.rmtree instead.
-    if os.path.exists(build_data_dir):
-      try:
-        shutil.rmtree(build_data_dir)
-      except Exception as exc:
-        # Catching everything: we don't want to break any builds for that reason
-        print >> sys.stderr, (
-          "FAILURE: path can't be deleted: %s.\n%s" % (build_data_dir, str(exc))
-        )
-    print >> sys.stderr, "Creating directory"
-
-    if not os.path.exists(build_data_dir):
-      try:
-        os.mkdir(build_data_dir)
-      except Exception as exc:
-        print >> sys.stderr, (
-          "FAILURE: directory can't be created: %s.\n%s" %
-          (build_data_dir, str(exc))
-        )
-        build_data_dir = None
-
-  # Under this line build_data_dir should point to an existing empty dir
-  # or be None.
-  yield build_data_dir
-
-  # Clean up after ourselves
-  if build_data_dir:
-    # TODO(pgervais): use infra_libs.rmtree instead.
-    try:
-      shutil.rmtree(build_data_dir)
-    except Exception as exc:
-      # Catching everything: we don't want to break any builds for that reason.
-      print >> sys.stderr, (
-        "FAILURE: path can't be deleted: %s.\n%s" % (build_data_dir, str(exc))
-      )
+  try:
+    cmd = [rt.run_cmd, 'infra.tools.send_monitoring_event',
+       '--event-mon-output-file',
+           os.path.join(outdir, 'log_request_proto'),
+       '--event-mon-run-type', 'file',
+       '--event-mon-service-name',
+           'buildbot/master/master.%s'
+           % rt.get('mastername', 'UNKNOWN'),
+       '--build-event-build-name',
+           rt.get('buildername', 'UNKNOWN'),
+       '--build-event-build-number',
+           str(rt.get('buildnumber', 0)),
+       '--build-event-build-scheduling-time',
+           str(1000*int(rt.get('requestedAt', 0))),
+       '--build-event-type', 'BUILD',
+       '--event-mon-timestamp-kind', 'POINT',
+       # And use only defaults for credentials.
+     ]
+    # Add this conditionally so that we get an error in
+    # send_monitoring_event log files in case it isn't present.
+    if hostname:
+      cmd += ['--build-event-hostname', hostname]
+    _check_command(cmd)
+  except Exception:
+    LOGGER.warning("Failed to send monitoring event.", exc_info=True)
 
 
 def main(argv):
-  opts, _ = get_args(argv)
+  opts = get_args(argv)
+  if opts.verbose == 0:
+    level = logging.INFO
+  else:
+    level = logging.DEBUG
+  logging.getLogger().setLevel(level)
+
   # TODO(crbug.com/551165): remove flag "factory_properties".
   use_factory_properties_from_disk = (opts.use_factory_properties_from_disk or
                                       bool(opts.factory_properties))
   properties = get_recipe_properties(
       opts.build_properties, use_factory_properties_from_disk)
 
   clean_old_recipe_engine()
 
-  # Find out if the recipe we intend to run is in build_internal's recipes. If
-  # so, use recipes.py from there, otherwise use the one from build.
-  recipe_file = properties['recipe'].replace('/', os.path.sep) + '.py'
-  if os.path.exists(os.path.join(BUILD_LIMITED_ROOT, 'recipes', recipe_file)):
-    recipe_runner = os.path.join(BUILD_LIMITED_ROOT, 'recipes.py')
-  else:
-    recipe_runner = os.path.join(SCRIPT_PATH, 'recipes.py')
+  # Enter our runtime enviornment.
+  with RecipeRuntime.enter(opts.leak, **properties) as rt:
+    LOGGER.debug('Loaded runtime: %s', rt)
 
-  with build_data_directory() as build_data_dir:
-    # Create a LogRequestLite proto containing this build's information.
-    if build_data_dir:
-      properties['build_data_dir'] = build_data_dir
+    # Find out if the recipe we intend to run is in build_internal's recipes. If
+    # so, use recipes.py from there, otherwise use the one from build.
+    recipe_file = properties['recipe'].replace('/', os.path.sep) + '.py'
+    if os.path.exists(os.path.join(BUILD_LIMITED_ROOT, 'recipes', recipe_file)):
+      recipe_runner = os.path.join(BUILD_LIMITED_ROOT, 'recipes.py')
+    else:
+      recipe_runner = os.path.join(SCRIPT_PATH, 'recipes.py')
 
-      hostname = socket.getfqdn()
-      if hostname:  # just in case getfqdn() returns None.
-        hostname = hostname.split('.')[0]
-      else:
-        hostname = None
+    # Setup monitoring directory and send a monitoring event.
+    build_data_dir = ensure_directory(rt.workdir, 'build_data')
+    properties['build_data_dir'] = build_data_dir
 
-      if RUN_CMD and os.path.exists(RUN_CMD):
-        try:
-          cmd = [RUN_CMD, 'infra.tools.send_monitoring_event',
-             '--event-mon-output-file',
-                 os.path.join(build_data_dir, 'log_request_proto'),
-             '--event-mon-run-type', 'file',
-             '--event-mon-service-name',
-                 'buildbot/master/master.%s'
-                 % properties.get('mastername', 'UNKNOWN'),
-             '--build-event-build-name',
-                 properties.get('buildername', 'UNKNOWN'),
-             '--build-event-build-number',
-                 str(properties.get('buildnumber', 0)),
-             '--build-event-build-scheduling-time',
-                 str(1000*int(properties.get('requestedAt', 0))),
-             '--build-event-type', 'BUILD',
-             '--event-mon-timestamp-kind', 'POINT',
-             # And use only defaults for credentials.
-           ]
-          # Add this conditionally so that we get an error in
-          # send_monitoring_event log files in case it isn't present.
-          if hostname:
-            cmd += ['--build-event-hostname', hostname]
-          subprocess.call(cmd)
-        except Exception:
-          print >> sys.stderr, traceback.format_exc()
+    # Write our annotated_run.py monitoring event.
+    write_monitoring_event(rt, build_data_dir)
 
-      else:
-        print >> sys.stderr, (
-          'WARNING: Unable to find run.py at %r, no events will be sent.'
-          % str(RUN_CMD)
-        )
+    # Dump properties to JSON and build recipe command.
+    props_file = os.path.join(rt.workdir, 'recipe_properties.json')
+    with open(props_file, 'w') as fh:
+      json.dump(properties, fh)
+    cmd = [
+        sys.executable, '-u', recipe_runner,
+        'run',
+        '--workdir=%s' % os.getcwd(),
+        '--properties-file=%s' % props_file,
+        rt.recipe,
+    ]
 
-    with namedTempFile() as props_file:
-      with open(props_file, 'w') as fh:
-        fh.write(json.dumps(properties))
-      cmd = [
-          sys.executable, '-u', recipe_runner,
-          'run',
-          '--workdir=%s' % os.getcwd(),
-          '--properties-file=%s' % props_file,
-          properties['recipe'] ]
-      status = subprocess.call(cmd)
+    status = None
+    try:
+      if not opts.logdog_force:
+        _assert_logdog_whitelisted(rt)

[iannucci, 2015/12/01 02:38:01]

why not just have this return a status code? I don't see the benefit of the
raise/except switch statement here?

edit: oh, I see... _logdog_bootstrap can raise too... mumble :/

[dnj, 2015/12/01 03:36:04]

I chose exceptions here because I'm already surrounding the bootstrap in a
failsafe try...finally block, so they fit.

 
-    # TODO(pgervais): Send events from build_data_dir to the endpoint.
+      status = _logdog_bootstrap(rt, opts, cmd)

[iannucci, 2015/12/01 02:38:01]

So IIUC, on windows this will now look like

annotated_run.py    # does update scripts
  annotated_run.py  # checks to see if we're logdogging 
    butler          # logdog's
      annotated_run.py  # has logdogging code disabled and actually does work

?

*sigh*.

What's the mechanism by which the 3rd annotated_run doesn't re-run logdogging
code? I'm not seeing it :/

[dnj, 2015/12/01 03:36:04]

It should be:
annotated_run.py
  annotated_run.py
    butler
      annotee
        recipes.py

If butler/annotee fails or is disabled, it will follow up with the standard:
annotated_run.py
  annotated_run.py
    recipes.py

+    except LogDogNotBootstrapped as e:
+      LOGGER.info('Not bootstrapped: %s', e.message)
+    except LogDogBootstrapError as e:
+      LOGGER.warning('Could not bootstrap LogDog: %s', e.message)
+    except Exception:
+      LOGGER.exception('Exception while bootstrapping LogDog.')
+    finally:
+      if status is None:
+        LOGGER.info('Not using LogDog. Invoking `annotated_run.py` directly.')
+        status, _ = _run_command(cmd, dry_run=opts.dry_run)
+
+  # TODO(pgervais): Send events from build_data_dir to the endpoint.
   return status
 
+
 def shell_main(argv):
   if update_scripts():
-    return subprocess.call([sys.executable] + argv)
+    # Re-execute with the updated annotated_run.py.
+    rv, _ = _run_command([sys.executable] + argv)
+    return rv
   else:
-    return main(argv)
+    return main(argv[1:])
 
 
 if __name__ == '__main__':
+  logging.basicConfig(level=logging.INFO)
   sys.exit(shell_main(sys.argv))