Selective build clobbering feature (landmines.py and android build scripts).

build/landmines.py

+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+This file holds a list of reasons why a particular build needs to be clobbered
+(or a list of 'landmines').
+
+This script runs every build as a hook. If it detects that the build should
+be clobbered, it will touch the file <build_dir>/.landmine_triggered. The
+various build scripts will then check for the presence of this file and clobber
+accordingly. The script will also emit the reasons for the clobber to stdout.
+
+A landmine is tripped when a builder checks out a different revision, and the
+diff between the new landmines and the old ones is non-null. At this point, the
+build is clobbered.
+"""
+
+import difflib
+import functools
+import gyp_helper
+import os
+import shlex
+import sys
+import time
+
+
+def memoize(default=None):
+  """This decorator caches the return value of a parameterless pure function"""
+  def memoizer(func):
+    val = []
+    @functools.wraps(func)
+    def inner(*args, **kwargs):
+      if not val:
+        ret = func(*args, **kwargs)
+        val.append(ret if ret is not None else default)
+        print '%s -> %r' % (func.__name__, val[0])
+      return val[0]
+    return inner
+  return memoizer
+
+
+@memoize()
+def IsWindows():
+  return sys.platform.startswith('win') or sys.platform == 'cygwin'
+
+
+@memoize()
+def IsLinux():
+  return sys.platform.startswith('linux')
+
+
+@memoize()
+def IsMac():
+  return sys.platform.startswith('darwin')
+
+
+@memoize()
+def gyp_defines():
+  """Parses and returns GYP_DEFINES env var as a dictionary."""
+  return dict(arg.split('=', 1)
+      for arg in shlex.split(os.environ.get('GYP_DEFINES', '')))
+
+
+@memoize()
+def distributor():
+  """
+  Returns a string which is the distributed build engine in use (if any).
+  Possible values: 'goma', 'ib', ''
+  """
+  if 'goma' in gyp_defines():
+    return 'goma'
+  elif IsWindows():
+    if 'CHROME_HEADLESS' in os.environ:
+      return 'ib' # use (win and !goma and headless) as approximation of ib
+
+
+@memoize()
+def platform():
+  """
+  Returns a string representing the platform this build is targetted for.
+  Possible values: 'win', 'mac', 'linux', 'ios', 'android'
+  """
+  if 'OS' in gyp_defines():
+    if 'android' in gyp_defines()['OS']:
+      return 'android'
+    else:
+      return gyp_defines()['OS']
+  elif IsWindows():
+    return 'win'
+  elif IsLinux():
+    return 'linux'
+  else:
+    return 'mac'
+
+
+@memoize()
+def builder():
+  """
+  Returns a string representing the build engine (not compiler) to use.
+  Possible values: 'make', 'ninja', 'xcode', 'msvs', 'scons'
+  """
+  if 'GYP_GENERATORS' in os.environ:
+    # for simplicity, only support the first explicit generator
+    generator = os.environ['GYP_GENERATORS'].split(',')[0]
+    if generator.endswith('-android'):
+      return generator.split('-')[0]
+    else:
+      return generator
+  else:
+    if platform() == 'android':
+      # Good enough for now? Do any android bots use make?
+      return 'ninja'
+    elif platform() == 'ios':
+      return 'xcode'
+    elif IsWindows():
+      return 'msvs'
+    elif IsLinux():
+      return 'make'
+    elif IsMac():
+      return 'xcode'
+    else:
+      assert False, 'Don\'t know what builder we\'re using!'
+
+
+def get_landmines(target):
+  """
+  ALL LANDMINES ARE DEFINED HERE.
+  target is 'Release' or 'Debug'
+  """
+  landmines = []
+  add = lambda item: landmines.append(item + '\n')
+
+  if (distributor() == 'goma' and platform() == 'win32' and
+      builder() == 'ninja'):
+    add('Need to clobber winja goma due to backend cwd cache fix.')
+
+  return landmines
+
+
+def get_target_build_dir(build_tool, src_dir, target, is_iphone=False):
+  """
+  Get absolute path to the output directory dependant on build and target.

[M-A Ruel, 2012/10/29 15:10:43]

Returns output directory absolute path dependent on build and targets.

http://www.grammar-monster.com/easily_confused/dependant_dependent.htm

[iannucci, 2012/10/29 23:56:39]

Done.

+  Examples:
+    r'c:\b\build\slave\win\build\src\out\Release'
+    '/mnt/data/b/build/slave/linux/build/src/out/Debug'
+    '/b/build/slave/ios_rel_device/build/src/xcodebuild/Release-iphoneos'
+  """
+  ret = None
+  if build_tool == 'xcode':

[M-A Ruel, 2012/10/29 15:10:43]

My feeling: you should destroy them all. Much simpler.

[iannucci, 2012/10/29 23:56:39]

No support for anything else? I still need xcode in any event (for iphone).

The only things that would die would be scons, since we still use make and msvs
(at least for now, bwahahahah).

[M-A Ruel, 2012/10/30 00:53:09]

Sorry I wasn't clear, I meant: delete all of these directories unconditionally.
It's safer. For example, I may well build *both* with msvs and ninja.

+    ret = os.path.join(src_dir, 'xcodebuild',
+        target + ('-iphoneos' if is_iphone else ''))
+  elif build_tool == 'make':
+    ret = os.path.join(src_dir, 'out', target)
+  elif build_tool == 'ninja':
+    ret = os.path.join(src_dir, 'out', target)
+  elif build_tool == 'msvs':
+    ret = os.path.join(src_dir, 'build', target)
+  elif build_tool == 'scons':
+    ret = os.path.join(src_dir, 'sconsbuild', target)

[M-A Ruel, 2012/10/29 15:10:43]

else:
  raise NotImplementError()

[iannucci, 2012/10/29 23:56:39]

Done.

+  return os.path.abspath(ret)
+
+
+def main():

[M-A Ruel, 2012/10/29 15:10:43]

if len(sys.argv) > 1:
  print('Unknown argument %s' % sys.argv[1:])
  return 1

[iannucci, 2012/10/29 23:56:39]

A lot of words for:

  assert len(sys.argv) == 1, 'Unknown argument %s' % sys.argv[1:]

;). Done.

+  gyp_helper.apply_chromium_gyp_env()
+  src_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))

[M-A Ruel, 2012/10/29 15:10:43]

Use a global variable.

[iannucci, 2012/10/29 23:56:39]

Done.

+
+  for target in ('Debug', 'Release'):
+    out_dir = get_target_build_dir(builder(), src_dir, target,
+                                   platform() == 'ios')
+
+    landmines_path = os.path.join(out_dir, '.landmines')
+    if not os.path.exists(out_dir):
+      os.makedirs(out_dir)
+
+    new_landmines = get_landmines(target)
+
+    if not os.path.exists(landmines_path):
+      with open(landmines_path, 'w') as f:
+        f.writelines(new_landmines)
+    else:
+      triggered = os.path.join(out_dir, '.landmines_triggered')
+      with open(landmines_path, 'r') as f:
+        old_landmines = f.readlines()
+      if old_landmines != new_landmines:
+        old_date = time.ctime(os.stat(landmines_path).st_ctime)
+        diff = difflib.unified_diff(old_landmines, new_landmines,
+            fromfile='old_landmines', tofile='new_landmines',
+            fromfiledate=old_date, tofiledate=time.ctime(), n=0)
+
+        with open(triggered, 'w') as f:
+          f.writelines(diff)
+      elif os.path.exists(triggered):
+        # Remove false triggered landmines.
+        os.remove(triggered)
+
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(main())