[telemetry] Script for finding all Telemetry dependencies.

tools/telemetry_tools/find_dependencies

+#!/usr/bin/env python
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.

[tonyg, 2014/01/29 05:10:08]

2014

[dtu, 2014/02/13 21:49:35]

Done. Fun fact, Ctrl+A in Vim increments the first number on the current line.

+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import collections
+import fnmatch
+import imp
+import logging
+import modulefinder
+import optparse
+import os
+import sys
+import zipfile
+
+sys.path.append(os.path.join(
+    os.path.dirname(os.path.realpath(__file__)), os.pardir, 'telemetry'))
+from telemetry import test
+from telemetry.core import discover
+from telemetry.core import util
+from telemetry.page import cloud_storage
+
+import path_set
+import telemetry_bootstrap
+
+
+DEPS_FILE = 'bootstrap_deps'
+
+
+def _InDirectory(subdirectory, directory):
+  subdirectory = os.path.realpath(subdirectory)
+  directory = os.path.realpath(directory)
+  common_prefix = os.path.commonprefix([subdirectory, directory])

[tonyg, 2014/01/29 05:10:08]

This has bitten me before. It does string comparison instead of path. So for
example:
'/foo'

Is that what you are expecting?

[dtu, 2014/02/13 21:49:35]

Yup! It's okay in this particular use.

+  return common_prefix == directory
+
+
+def FindBootstrapDependencies(base_dir):
+  deps_file = os.path.join(base_dir, DEPS_FILE)
+  if not os.path.exists(deps_file):
+    return []
+  deps_paths = telemetry_bootstrap.ListAllDepsPaths(deps_file)
+  return set(
+      os.path.realpath(os.path.join(util.GetChromiumSrcDir(), os.pardir, path))
+      for path in deps_paths)
+
+
+def FindPythonDependencies(module_path):
+  logging.info('Finding Python dependencies of %s' % module_path)
+
+  # Load the module to inherit its sys.path modifications.
+  imp.load_source(
+      os.path.splitext(os.path.basename(module_path))[0], module_path)
+
+  # Analyze the module for its imports.
+  finder = modulefinder.ModuleFinder()
+  finder.run_script(module_path)
+
+  # Filter for only imports in Chromium.
+  for module in finder.modules.itervalues():
+    # If it's an __init__.py, module.__path__ gives the package's folder.
+    module_path = module.__path__[0] if module.__path__ else module.__file__
+    if not module_path:
+      continue
+
+    module_path = os.path.realpath(module_path)
+    if not _InDirectory(module_path, util.GetChromiumSrcDir()):
+      continue
+
+    yield module_path
+
+
+def FindPageSetDependencies(base_dir):
+  logging.info('Finding page sets in %s' % base_dir)
+
+  # Add base_dir to path so our imports relative to base_dir will work.
+  sys.path.append(base_dir)
+  tests = discover.DiscoverClasses(base_dir, base_dir, test.Test,
+                                   index_by_class_name=True)
+
+  for test_class in tests.itervalues():
+    test_obj = test_class()
+
+    # Ensure the test's default options are set if needed.
+    parser = optparse.OptionParser()
+    test_obj.AddTestCommandLineOptions(parser)
+    options = optparse.Values()
+    for k, v in parser.get_default_values().__dict__.iteritems():
+      options.ensure_value(k, v)
+
+    # Page set paths are relative to their runner script, not relative to us.
+    util.GetBaseDir = lambda: base_dir
+    # Loading the page set will automatically download its Cloud Storage deps.
+    page_set = test_obj.CreatePageSet(options)
+
+    # Add all of its serving_dirs as dependencies.
+    for serving_dir in page_set.serving_dirs:
+      yield serving_dir
+    for page in page_set:
+      if page.is_file:
+        yield page.serving_dir
+
+
+def FindCloudStorageFiles(files):
+  for path in files:
+    data_path, extension = os.path.splitext(path)
+    if extension != '.sha1':
+      continue
+
+    if (cloud_storage.GetIfChanged(cloud_storage.PUBLIC_BUCKET, data_path) or
+        cloud_storage.GetIfChanged(cloud_storage.INTERNAL_BUCKET, data_path)):
+      yield data_path
+
+
+def FindExcludedFiles(files, options):
+  def CheckConditions(path, conditions):
+    for condition in conditions:
+      if condition(path):
+        return True
+    return False
+
+  # Define some filters for files.
+  def IsHidden(path):
+    for pathname_component in path.split(os.sep):
+      if pathname_component.startswith('.'):
+        return True
+    return False
+  def IsPyc(path):
+    return os.path.splitext(path)[1] == '.pyc'
+  def IsInCloudStorage(path):
+    return os.path.exists(path + '.sha1')
+  def MatchesExcludeOptions(path):
+    for pattern in options.exclude:
+      if (fnmatch.fnmatch(path, pattern) or
+          fnmatch.fnmatch(os.path.basename(path), pattern)):
+        return True
+    return False
+  def MatchesConditions(path, conditions):
+    for condition in conditions:
+      if condition(path):
+        return True
+    return False
+
+  # Collect filters we're going to use to exclude files.
+  exclude_conditions = [MatchesExcludeOptions, IsHidden, IsPyc]
+  if not options.include_cloud_storage_data:
+    exclude_conditions.append(IsInCloudStorage)
+
+  # Check all the files against the filters.
+  for path in files:
+    if MatchesConditions(path, exclude_conditions):
+      yield path
+
+
+def FindDependencies(paths, options):
+  # Verify arguments.
+  for path in paths:
+    if not os.path.exists(path):
+      raise ValueError('Path does not exist: %s' % path)
+
+  dependencies = path_set.PathSet()
+  dependencies |= FindPythonDependencies(os.path.realpath(__file__))
+
+  # Add dependencies.
+  for path in paths:
+    base_dir = os.path.dirname(os.path.realpath(path))
+
+    dependencies.add(base_dir)
+    dependencies |= FindBootstrapDependencies(base_dir)
+    dependencies |= FindPythonDependencies(path)
+    if options.include_page_set_data:
+      dependencies |= FindPageSetDependencies(base_dir)
+
+  # Find the Cloud Storage files in the existing paths.
+  if options.include_cloud_storage_data:
+    dependencies |= FindCloudStorageFiles(set(dependencies))
+
+  # Remove excluded files.
+  dependencies -= FindExcludedFiles(set(dependencies), options)
+
+  return dependencies
+
+
+def ZipDependencies(paths, dependencies, options):
+  base_dir = os.path.dirname(os.path.realpath(util.GetChromiumSrcDir()))
+
+  with zipfile.ZipFile(options.zip, 'w', zipfile.ZIP_DEFLATED) as zip_file:
+    # Add dependencies to archive.
+    for path in dependencies:
+      path_in_archive = os.path.join(
+          'telemetry', os.path.relpath(path, base_dir))
+      zip_file.write(path, path_in_archive)
+
+    # Add symlinks to executable paths, for ease of use.
+    for path in paths:
+      relative_path = os.path.relpath(path, base_dir)
+      link_info = zipfile.ZipInfo(
+          os.path.join('telemetry', os.path.basename(path)))
+      link_info.create_system = 3  # Unix attributes.
+      # 012 is symlink, 0777 is the permission bits rwxrwxrwx.
+      link_info.external_attr = 0120777 << 16  # Octal.
+      zip_file.writestr(link_info, relative_path)
+
+    # Add gsutil to the archive, if it's available. The gsutil in
+    # depot_tools is modified to allow authentication using prodaccess.
+    gsutil_path = cloud_storage.FindGsutil()
+    if cloud_storage.SupportsProdaccess(gsutil_path):
+      gsutil_dependencies = path_set.PathSet()
+      gsutil_dependencies.add(os.path.dirname(gsutil_path))
+      gsutil_dependencies -= FindExcludedFiles(
+          set(gsutil_dependencies), options)
+
+      gsutil_base_dir = os.path.join(os.path.dirname(gsutil_path), os.pardir)
+      for path in gsutil_dependencies:
+        path_in_archive = os.path.join(
+            'telemetry', os.path.relpath(util.GetTelemetryDir(), base_dir),
+            'third_party', os.path.relpath(path, gsutil_base_dir))
+        zip_file.write(path, path_in_archive)
+
+
+def ParseCommandLine():
+  parser = optparse.OptionParser()
+  parser.add_option(
+      '-v', '--verbose', action='count', dest='verbosity',
+      help='Increase verbosity level (repeat as needed).')
+
+  parser.add_option(
+      '-p', '--include-page-set-data', action='store_true', default=False,

[tonyg, 2014/01/29 05:10:08]

Shouldn't this default to true?

[dtu, 2014/02/13 21:49:35]

No, for the bots we're actually going to do False, since we can't yet
differentiate between src and src-internal files. This flag is referring to e.g.
third_party/WebKit/PerformanceTests, not to the page set .json and .sha1 files.

+      help='Scan tests for page set data and include them.')
+  parser.add_option(
+      '-c', '--include-cloud-storage-data', action='store_true', default=False,

[tonyg, 2014/01/29 05:10:08]

This seems dangerous and I can't think why we'd ever set it. Do we need it?

[dtu, 2014/02/13 21:49:35]

Guess not. Done.

+      help='Scan paths for data in Cloud Storage. Download and include them.')
+
+  parser.add_option(
+      '-e', '--exclude', action='append', default=[],
+      help='Exclude paths matching EXCLUDE. Can be used multiple times.')
+  parser.add_option(
+      '-E', '--exclude-file', action='append',
+      help='Exclude all files in EXCLUDE_FILE. EXCLUDE_FILE must be a '
+      'newline-separated list of paths. Can be used multiple times.')
+
+  parser.add_option(
+      '-z', '--zip',
+      help='Store files in a zip archive at ZIP.')
+
+  options, args = parser.parse_args()
+
+  if options.verbosity >= 2:
+    logging.getLogger().setLevel(logging.DEBUG)
+  elif options.verbosity:
+    logging.getLogger().setLevel(logging.INFO)
+  else:
+    logging.getLogger().setLevel(logging.WARNING)
+
+  return options, args
+
+
+def Main():

[tonyg, 2014/01/29 05:10:08]

Can this thing generate bootstrap_deps now? Since CrOS depends on that, it would
serve as another form of regression test for this.

[dtu, 2014/02/13 21:49:35]

Unfortunately, this script can't find some dependencies which are manual in
bootstrap_deps. E.g., profile data, JS dependencies of
html_page_measurement_results. Also I've had issues modifying the bootstrap_deps
in the past and now I'm scared :)

+  options, paths = ParseCommandLine()
+
+  dependencies = FindDependencies(paths, options)
+
+  if options.zip:
+    ZipDependencies(paths, dependencies, options)
+    print 'Zip archive written to %s.' % options.zip
+  else:
+    print '\n'.join(sorted(dependencies))
+
+
+if __name__ == '__main__':
+  Main()