Build scripts refactor to support multislave

scripts/common/chromium_utils.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """ Set of basic operations/utilities that are used by the build. """
 
 import copy
 import errno
 import fnmatch
 import glob
 import math
 import os
 import shutil
+import socket
 import stat
 import string  # pylint: disable=W0402
 import subprocess
 import sys
 import threading
 import time
 import urllib
 import zipfile
 
 try:
   import json  # pylint: disable=F0401
 except ImportError:
   import simplejson as json
 
 
+BUILD_DIR = os.path.realpath(os.path.join(
+    os.path.dirname(__file__), os.pardir, os.pardir))

[Mike Stip (use stip instead), 2013/02/20 18:42:07]

didn't know os.pardir was a thing. neat!

+
+
 # Local errors.
 class MissingArgument(Exception): pass
 class PathNotFound(Exception): pass
 class ExternalError(Exception): pass
 
 def IsWindows():
   return sys.platform == 'cygwin' or sys.platform.startswith('win')
 
 def IsLinux():
   return sys.platform.startswith('linux')
@@ skipping 933 matching lines @@
   command = ['scp', '-r', '-p', srctree, host + ':' + dst]
   result = RunCommand(command)
   if result:
     raise ExternalError('Failed to scp "%s" to "%s" (%s)' %
                         (srctree, host + ':' + dst, result))
 
 
 def ListMasters(cue='master.cfg', include_public=True, include_internal=True):
   """Returns all the masters found."""
   # Look for "internal" masters first.
-  path_internal = os.path.join(os.path.dirname(__file__), '..', '..', '..',
-      'build_internal', 'masters/*/' + cue)
-  path = os.path.join(os.path.dirname(__file__), '..', '..',
-      'masters/*/' + cue)
+  path_internal = os.path.join(
+      BUILD_DIR, os.pardir, 'build_internal', 'masters/*/' + cue)
+  path = os.path.join(BUILD_DIR, 'masters/*/' + cue)
   filenames = []
   if include_public:
     filenames += glob.glob(path)
   if include_internal:
     filenames += glob.glob(path_internal)
   return [os.path.abspath(os.path.dirname(f)) for f in filenames]
 
 
+def GetAllSlaves():
+  """Return all slave objects from masters."""
+  slaves = []
+  for master in ListMasters(cue='slaves.cfg'):
+    cur_slaves = RunSlavesCfg(os.path.join(master, 'slaves.cfg'))
+    for slave in cur_slaves:
+      slave['mastername'] = os.path.basename(master)
+    slaves.extend(cur_slaves)
+  return slaves
+
+
+def GetActiveSlavename():
+  testing_slavename = os.getenv('TESTING_SLAVENAME')
+  if testing_slavename:
+    return testing_slavename
+  return socket.getfqdn().split('.', 1)[0].lower()
+
+
+def EntryToSlaveName(entry):
+  """Produces slave name from the slaves config dict."""
+  return entry.get('slavename') or entry.get('hostname')
+
+
+def GetActiveMaster(slavename=None):
+  """Parses all the slaves.cfg and returns the name of the active master
+  determined by the hostname. Returns None otherwise.
+
+  It will be matched against *both* the 'slavename' and 'hostname' fields
+  in slaves.cfg.
+  """
+  slavename = slavename or GetActiveSlavename()
+  for slave in GetAllSlaves():
+    if slavename == EntryToSlaveName(slave):
+      return slave['master']
+
+
+def ParsePythonCfg(cfg_filepath):
+  """Retrieves data from a python config file."""
+  if not os.path.exists(cfg_filepath):
+    return None
+  base_path = os.path.dirname(os.path.abspath(cfg_filepath))
+  old_sys_path = sys.path
+  sys.path = sys.path + [base_path]
+  old_path = os.getcwd()
+  try:
+    os.chdir(base_path)
+    local_vars = {}
+    execfile(os.path.join(cfg_filepath), local_vars)
+    return local_vars
+  finally:
+    os.chdir(old_path)
+    sys.path = old_sys_path
+
+
 def RunSlavesCfg(slaves_cfg):
   """Runs slaves.cfg in a consistent way."""
-  if not os.path.exists(slaves_cfg):
-    return []
-  slaves_path = os.path.dirname(os.path.abspath(slaves_cfg))
-  old_sys_path = sys.path
-  sys.path = sys.path + [slaves_path]
-  try:
-    old_path = os.getcwd()
-    try:
-      os.chdir(slaves_path)
-      local_vars = {}
-      execfile(os.path.join(slaves_cfg), local_vars)
-      return local_vars['slaves']
-    finally:
-      os.chdir(old_path)
-  finally:
-    sys.path = old_sys_path
+  slave_config = ParsePythonCfg(slaves_cfg) or {}
+  return slave_config.get('slaves', [])
 
 
 def convert_json(option, opt, value, parser):
   """Provide an OptionParser callback to unmarshal a JSON string."""
   setattr(parser.values, option.dest, json.loads(value))
 
 
 def SafeTranslate(inputstr):
   """Convert a free form string to one that can be used in a path.
 
@@ skipping 32 matching lines @@
     output, error = proc.communicate()
     if proc.returncode != 0:
       raise ExternalError('%s failed with error %s\n' % (config_path, error))
 
     config_list = json.loads(output).values()  # pylint: disable=E1103
     config_list.sort(key=lambda cfg: cfg['display_position'])
     return config_list
   except ImportError:
     # To get around CQ pylint failures, because CQ doesn't check out chromite.
     # TODO(maruel): Remove this try block when this issue is resolved.
-    print 'cbuildbot_chromite not found!  Returning empty config dictionary.'
     return {}
 
 
 def AddPropertiesOptions(option_parser):
   """Registers command line options for parsing build and factory properties.
 
   After parsing, the options object will have the 'build_properties' and
   'factory_properties' attributes. The corresponding values will be python
   dictionaries containing the properties. If the options are not given on
   the command line, the dictionaries will be empty.
@@ skipping 11 matching lines @@
                            nargs=1, default={},
                            help='factory properties in JSON format')
 
 
 def AddThirdPartyLibToPath(lib, override=False):
   """Adds the specified dir in build/third_party to sys.path.
 
   Setting 'override' to true will place the directory in the beginning of
   sys.path, useful for overriding previously set packages.
   """
-  libpath = os.path.abspath(os.path.join(os.path.dirname(__file__),
-                                         '..', '..', 'third_party', lib))
+  libpath = os.path.abspath(os.path.join(BUILD_DIR, 'third_party', lib))
   if override:
     sys.path.insert(0, libpath)
   else:
     sys.path.append(libpath)
 
 
 def GetLKGR():
   """Connect to chromium LKGR server and get LKGR revision.
 
   On success, returns the LKGR and 'ok'. On error, returns None and the text of
@@ skipping 16 matching lines @@
     conn.close()
 
   return rev, 'ok'
 
 
 def AbsoluteCanonicalPath(*path):
   """Return the most canonical path Python can provide."""
 
   file_path = os.path.join(*path)
   return os.path.realpath(os.path.abspath(os.path.expanduser(file_path)))