Initial commit of all previous Chrome build scripts.

chromium/scripts/generate_gyp.py

+#!/usr/bin/python
+#
+# Copyright (c) 2011 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Creates a GYP include file for building FFmpeg from source.
+
+The way this works is a bit silly but it's easier than reverse engineering
+FFmpeg's configure scripts and Makefiles. It scans through build directories for
+object files then does a reverse lookup against the FFmpeg source tree to find
+the corresponding C or assembly file.
+
+Running build_ffmpeg.sh for ia32, x64, arm, and arm-neon platforms is required
+prior to running this script. The arm and arm-neon platforms assume a
+Chromium OS build environment.
+
+Step 1: Have a Chromium OS checkout (refer to http://dev.chromium.org)
+  mkdir chromeos
+  repo init ...
+  repo sync
+
+Step 2: Check out deps/third_party inside Chromium OS
+  cd path/to/chromeos
+  mkdir deps
+  cd deps
+  svn co svn://svn.chromium.org/chrome/trunk/deps/third_party/ffmpeg
+  svn co svn://svn.chromium.org/chrome/trunk/deps/third_party/libvpx
+
+Step 3: Build for ia32/x64 platforms outside chroot
+  cd path/to/chromeos/deps/ffmpeg
+  ./build_ffmpeg.sh linux ia32 path/to/chromeos/deps
+  ./build_ffmpeg.sh linux x64 path/to/chromeos/deps
+
+Step 4: Build and enter Chromium OS chroot:
+  cd path/to/chromeos/src/scripts
+  ./make_chroot
+  ./enter_chroot.sh
+
+Step 5: Setup build environment for ARM:
+  ./setup_board --board arm-generic
+
+Step 6: Build for arm/arm-neon platforms inside chroot
+  ./build_ffmpeg.sh linux arm path/to/chromeos/deps
+  ./build_ffmpeg.sh linux arm-neon path/to/chromeos/deps
+
+Step 7: Exit chroot and generate gyp file
+  exit
+  cd path/to/chromeos/deps/ffmpeg
+  ./generate_gyp.py --source_dir=source --build_dir=.
+
+Phew!
+
+While this seems insane, reverse engineering and maintaining a gyp file by hand
+is significantly more painful.
+"""
+
+__author__ = 'scherkus@chromium.org (Andrew Scherkus)'
+
+import datetime 
+import itertools 
+import optparse
+import os
+import string
+import sys
+
+GYP_HEADER = """# Copyright (c) %d The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# NOTE: this file is autogenerated by deps/third_party/ffmpeg/generate_gyp.py
+
+{
+  'conditions': [
+""" % (datetime.datetime.now().year)
+
+GYP_FOOTER = """  ],  # conditions
+}
+"""
+
+GYP_CONDITIONAL_SOURCE_STANZA_BEGIN = """    ['%s', {
+      'sources': [
+"""
+GYP_CONDITIONAL_SOURCE_STANZA_ITEM = """        'patched-ffmpeg/%s',
+"""
+GYP_CONDITIONAL_SOURCE_STANZA_END = """      ],
+    }],  # %s
+"""
+
+# Controls GYP conditional stanza generation.
+SUPPORTED_ARCHITECTURES = ['ia32', 'x64', 'arm', 'arm-neon']
+SUPPORTED_TARGETS = ['Chromium', 'Chrome', 'ChromiumOS', 'ChromeOS']
+
+def NormalizeFilename(name):
+  """ Removes leading path separators in an attempt to normalize paths."""
+  return string.lstrip(name, os.sep)
+
+
+def CleanObjectFiles(object_files):
+  """Removes unneeded object files due to linker errors, binary size, etc...
+ 
+  Args:
+    object_files: List of object files that needs cleaning.
+  """
+  blacklist = [
+      'libavcodec/inverse.o',  # Includes libavutil/inverse.c
+
+      # The following files are removed to trim down on binary size.
+      # TODO(ihf): Warning, it is *easy* right now to remove more files
+      # than is healthy and end up with a library that the linker does
+      # not complain about but that can't be loaded. Add some verification!
+      'libavcodec/dirac.o',
+      'libavcodec/resample.o',
+      'libavcodec/resample2.o',
+      'libavcodec/x86/dnxhd_mmx.o',
+      'libavformat/os_support.o',
+      'libavformat/sdp.o',
+      'libavutil/adler32.o',
+      'libavutil/aes.o',
+      'libavutil/des.o',
+      'libavutil/error.o',
+      'libavutil/file.o',
+      'libavutil/lls.o',
+      'libavutil/rc4.o',
+      'libavutil/sha.o',
+      'libavutil/tree.o',
+  ]
+  for name in blacklist:
+    if name in object_files:
+      object_files.remove(name)
+  return object_files
+
+
+def GetSourceFiles(source_dir):
+  """Returns a list of source files for the given source directory.
+
+  Args:
+    source_dir: Path to build a source mapping for.
+
+  Returns:
+    A python list of source file paths.
+  """
+
+  def IsSourceDir(d):
+    return d not in ['.git', '.svn']
+
+  def IsSourceFile(f):
+    _, ext = os.path.splitext(f)
+    return ext in ['.c', '.S', '.asm']
+
+  source_files = []
+  for root, dirs, files in os.walk(source_dir):
+    dirs = filter(IsSourceDir, dirs)
+    files = filter(IsSourceFile, files)
+
+    # Strip leading source_dir from root.
+    root = root[len(source_dir):]
+    source_files.extend([NormalizeFilename(os.path.join(root, name)) for name in
+                         files])
+  return source_files 
+
+
+def GetObjectFiles(build_dir):
+  """Returns a list of object files for the given build directory.
+
+  Args:
+    build_dir: Path to build an object file list for.
+
+  Returns:
+    A python list of object files paths.
+  """
+  object_files = []
+  for root, dirs, files in os.walk(build_dir):
+    # Strip leading build_dir from root.
+    root = root[len(build_dir):]
+
+    for name in files:
+      _, ext = os.path.splitext(name)
+      if ext == '.o':
+        name = NormalizeFilename(os.path.join(root, name))
+        object_files.append(name)
+  CleanObjectFiles(object_files)
+  return object_files
+
+
+def GetObjectToSourceMapping(source_files):
+  """Returns a map of object file paths to source file paths.
+
+  Args:
+    source_files: List of source file paths.
+
+  Returns:
+    Map with object file paths as keys and source file paths as values.
+  """
+  object_to_sources = {}
+  for name in source_files:
+    basename, ext = os.path.splitext(name)
+    key = basename + '.o'
+    object_to_sources[key] = name
+  return object_to_sources
+
+
+def GetSourceFileSet(object_to_sources, object_files):
+  """Determines set of source files given object files.
+
+  Args:
+    object_to_sources: A dictionary of object to source file paths.
+    object_files: A list of object file paths.
+
+  Returns:
+    A python set of source files requried to build said objects.
+  """
+  source_set = set()
+  for name in object_files:
+    # Intentially raise a KeyError if lookup fails since something is messed
+    # up with our source and object lists.
+    source_set.add(object_to_sources[name])
+  return source_set 
+
+
+class SourceSet(object):
+  """A SourceSet represents a set of source files that are built on the given
+  set of architectures and targets.
+  """
+
+  def __init__(self, sources, architectures, targets):
+    """Creates a SourceSet.
+
+    Args:
+      sources: a python set of source files
+      architectures: a python set of architectures (i.e., arm, x64)
+      targets: a python set of targets (i.e., Chromium, Chrome)
+    """
+    self.sources = sources
+    self.architectures = architectures
+    self.targets = targets
+
+  def __repr__(self):
+    return '{%s, %s, %s}' % (self.sources, self.architectures, self.targets)
+
+  def __eq__(self, other):
+    return (self.sources == other.sources and
+            self.architectures == other.architectures and
+            self.targets == other.targets)
+
+  def Intersect(self, other):
+    """Return a new SourceSet containing the set of source files common to both
+    this and the other SourceSet.
+
+    The resulting SourceSet represents the union of the architectures and
+    targets of this and the other SourceSet.
+    """
+    return SourceSet(self.sources & other.sources,
+                     self.architectures | other.architectures,
+                     self.targets | other.targets)
+
+  def Difference(self, other):
+    """Return a new SourceSet containing the set of source files not present in
+    the other SourceSet.
+
+    The resulting SourceSet represents the intersection of the architectures and
+    targets of this and the other SourceSet.
+    """
+    return SourceSet(self.sources - other.sources,
+                     self.architectures & other.architectures,
+                     self.targets & other.targets)
+
+  def IsEmpty(self):
+    """An empty SourceSet is defined as containing no source files or no
+    architecture/target (i.e., a set of files that aren't built on anywhere).
+    """
+    return (len(self.sources) == 0 or len(self.architectures) == 0 or
+            len(self.targets) == 0)
+
+  def GenerateGypStanza(self):
+    """Generates a gyp conditional stanza representing this source set.
+
+    TODO(scherkus): Having all this special case condition optimizing logic in
+    here feels a bit dirty, but hey it works. Perhaps refactor if it starts
+    getting out of hand.
+
+    Returns:
+      A string of gyp code.
+    """
+
+    # Only build a non-trivial conditional if it's a subset of all supported
+    # architectures.
+    arch_conditions = []
+    if self.architectures == set(SUPPORTED_ARCHITECTURES):
+      arch_conditions.append('1')
+    else:
+      for arch in self.architectures:
+        if arch == 'arm-neon':
+          arch_conditions.append('(target_arch == "arm" and arm_neon == 1)')
+        else:
+          arch_conditions.append('target_arch == "%s"' % arch)
+
+
+    # Only build a non-trivial conditional if it's a subset of all supported
+    # targets.
+    branding_conditions = []
+    if self.targets == set(SUPPORTED_TARGETS):
+      branding_conditions.append('1')
+    else:
+      for branding in self.targets:
+        branding_conditions.append('ffmpeg_branding == "%s"' % branding)
+
+    conditions = '(%s) and (%s)' % (' or '.join(arch_conditions),
+                                    ' or '.join(branding_conditions))
+
+    stanza = []
+    stanza += GYP_CONDITIONAL_SOURCE_STANZA_BEGIN % (conditions)
+    for name in sorted(self.sources):
+      stanza += GYP_CONDITIONAL_SOURCE_STANZA_ITEM % (name)
+    stanza += GYP_CONDITIONAL_SOURCE_STANZA_END % (conditions)
+    return ''.join(stanza)
+
+
+def CreatePairwiseDisjointSets(sets):
+  """ Given a list of SourceSet objects, returns the pairwise disjoint sets.
+
+  NOTE: This isn't the most efficient algorithm, but given how infrequent we
+  need to run this and how small the input size is we'll leave it as is.
+  """
+
+  disjoint_sets = list(sets)
+
+  new_sets = True
+  while new_sets:
+    new_sets = False
+    for pair in itertools.combinations(disjoint_sets, 2):
+      intersection = pair[0].Intersect(pair[1])
+
+      # Both pairs are already disjoint, nothing to do.
+      if intersection.IsEmpty():
+        continue
+
+      # Add the resulting intersection set.
+      new_sets = True
+      disjoint_sets.append(intersection)
+
+      # Calculate the resulting differences for this pair of sets.
+      #
+      # If the differences are an empty set, remove them from the list of sets,
+      # otherwise update the set itself.
+      for p in pair:
+        i = disjoint_sets.index(p)
+        difference = p.Difference(intersection)
+        if difference.IsEmpty():
+          del disjoint_sets[i]
+        else:
+          disjoint_sets[i] = difference
+
+      # Restart the calculation since the list of disjoint sets has changed.
+      break
+
+  return disjoint_sets
+
+
+def ParseOptions():
+  """Parses the options and terminates program if they are not sane.
+
+  Returns:
+    The pair (optparse.OptionValues, [string]), that is the output of
+    a successful call to parser.parse_args().
+  """
+  parser = optparse.OptionParser(
+      usage='usage: %prog [options] DIR')
+
+  parser.add_option('-s',
+                    '--source_dir',
+                    dest='source_dir',
+                    default=None,
+                    metavar='DIR',
+                    help='FFmpeg source directory containing patched-ffmpeg')
+
+  parser.add_option('-b',
+                    '--build_dir',
+                    dest='build_dir',
+                    default='.',
+                    metavar='DIR',
+                    help='Build root containing build.x64.linux, etc...')
+
+  options, args = parser.parse_args()
+
+  if not options.source_dir:
+    parser.error('No FFmpeg source directory specified')
+
+  if not os.path.exists(os.path.join(options.source_dir, 'patched-ffmpeg')):
+    parser.error('FFmpeg source directory does not contain patched-ffmpeg')
+
+  if not options.build_dir:
+    parser.error('No build root directory specified')
+
+  return options, args
+
+
+def main():
+  options, args = ParseOptions()
+
+  # Generate map of FFmpeg source files.
+  source_dir = os.path.join(options.source_dir, 'patched-ffmpeg')
+  source_files = GetSourceFiles(source_dir)
+  object_to_sources = GetObjectToSourceMapping(source_files)
+
+  # Open for writing.
+  output_name = os.path.join(options.source_dir, 'ffmpeg_generated.gypi')
+  fd = open(output_name, 'w')
+  fd.write(GYP_HEADER)
+
+  sets = []
+
+  for arch in SUPPORTED_ARCHITECTURES:
+    for target in SUPPORTED_TARGETS:
+      # Construct build directory in the form of build.$arch.linux/$target.
+      #
+      # NOTE: FFmpeg doesn't have any mac-specific files, so instead of
+      # complicating the entire process worrying about multiple OSes, we'll
+      # assume that linux is good enough for now.
+      name = ''.join(['build.', arch, '.linux'])
+      build_dir = os.path.join(options.build_dir, name, target)
+      if not os.path.exists(build_dir):
+        print "Build directory not found: %s" % build_dir
+        sys.exit(1)
+
+      object_files = GetObjectFiles(build_dir)
+
+      # Generate the set of source files to build said target.
+      s = GetSourceFileSet(object_to_sources, object_files)
+      sets.append(SourceSet(s, set([arch]), set([target])))
+
+  # Generate conditional stanza for each disjoint source set.
+  for s in CreatePairwiseDisjointSets(sets):
+    fd.write(s.GenerateGypStanza())
+
+  fd.write(GYP_FOOTER)
+  fd.close()
+
+if __name__ == '__main__':
+  main()