'recipes.py isolate' command to build an isolate of the current package's toolchain

recipe_engine/package.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import ast
 import collections
 import contextlib
 import copy
 import functools
 import itertools
 import logging
 import os
+import re
 import subprocess
 import sys
 import tempfile
 
 from .third_party.google.protobuf import text_format
 from . import package_pb2
 
 
 class UncleanFilesystemError(Exception):
   pass
@@ skipping 325 matching lines @@
       return False
     return self._proto_file == other._proto_file
 
 
 class Package(object):
   """Package represents a loaded package, and contains path and dependency
   information.
 
   This is accessed by loader.py through RecipeDeps.get_package.
   """
-  def __init__(self, name, repo_spec, deps, recipes_dir):
+  def __init__(self, name, repo_spec, deps, repo_root, relative_recipes_dir):
     self.name = name
     self.repo_spec = repo_spec
     self.deps = deps
-    self.recipes_dir = recipes_dir
+    self.relative_recipes_dir = relative_recipes_dir
+    self._repo_root = repo_root
+
+  @property
+  def recipes_dir(self):
+    """Returns the root of the recipes area (the part containing recipes/ and
+    recipe_modules/
+    """
+    return os.path.join(self._repo_root, self.relative_recipes_dir)
 
   @property
   def recipe_dirs(self):
+    """Returns a collection of directories containing recipes."""

[M-A Ruel, 2016/01/09 00:51:59]

Returns the list of

[luqui, 2016/01/15 23:11:27]

Done.

     return [os.path.join(self.recipes_dir, 'recipes')]
 
   @property
   def module_dirs(self):
+    """Returns a collection of directories containing recipe modules."""

[M-A Ruel, 2016/01/09 00:51:59]

same 

[luqui, 2016/01/15 23:11:27]

Done.

     return [os.path.join(self.recipes_dir, 'recipe_modules')]
 
   def find_dep(self, dep_name):
     assert dep_name in self.deps, (
         '%s does not exist or is not declared as a dependency of %s' % (
             dep_name, self.name))
     return self.deps[dep_name]
 
   def module_path(self, module_name):
     return os.path.join(self.recipes_dir, 'recipe_modules', module_name)
 
   def loop_over_recipe_modules():
     for path in self.module_dirs:
       if os.path.isdir(path):
         for item in os.listdir(path):
           subpath = os.path.join(path, item)
           if _is_recipe_module_dir(subpath):
             yield subpath
 
+  def all_files(self, context, seen=None):
+    """Returns a generator listing all file paths this package needs to run."""

[M-A Ruel, 2016/01/09 00:51:59]

Yields all file ...

[luqui, 2016/01/15 23:11:27]

Done.

+    if seen is None:
+      seen = set()
+    if self in seen:
+      return
+    seen.add(self)
+
+    EXCLUDE = re.compile('\.pyc$')  # pycs are not portable
+
+    dirs = list(self.recipe_dirs) + list(self.module_dirs)

[M-A Ruel, 2016/01/09 00:51:59]

The the two properties return a list, I don't think it's needed to create 2
temporary copies

[luqui, 2016/01/15 23:11:27]

Good obs. One upon a time they were generators.

+    recipes_extra = os.path.join(self.recipes_dir, 'recipes.extra')
+    if os.path.exists(recipes_extra):
+      yield recipes_extra
+      with open(recipes_extra, 'r') as fh:
+        for fname in itertools.imap(self._normalize_extra_line, fh.readlines()):
+          f = os.path.join(self.recipes_dir, fname)
+          if os.path.isdir(f):
+            dirs.append(f)
+          else:
+            yield f
+
+    yield self.repo_spec.proto_file(context).path
+    yield os.path.join(self.recipes_dir, 'recipes.py')
+    for basedir in dirs:
+      for root, dirs, files in os.walk(basedir):
+        for fname in files:
+          if not EXCLUDE.search(fname):

[M-A Ruel, 2016/01/09 00:51:59]

a regexp is overkill here when this would be fine:
if not fname.endswith('.pyc'):

[luqui, 2016/01/15 23:11:27]

I want the idea of EXCLUDE to be more visible than 3 levels nested in a loop, in
case people are wondering why their pycs aren't getting included.

+            yield os.path.join(root, fname)
+
+    for dep in self.deps.values():

[M-A Ruel, 2016/01/09 00:51:59]

itervalues

[luqui, 2016/01/15 23:11:27]

Done.

+      for f in dep.all_files(context, seen):
+        yield f
+
+  def _normalize_extra_line(self, line):

[M-A Ruel, 2016/01/09 00:51:59]

This doesn't need to be a method. IMHO, it'd be more readable if you inlined it.

[luqui, 2016/01/15 23:11:27]

Ah yeah, and get rid of the imap.  Agreed, Done.

+    return re.sub(r'#.*', '', line).replace('/', os.sep).strip()
+
 
 class PackageSpec(object):
   API_VERSION = 1
 
   def __init__(self, project_id, recipes_path, deps):
     self._project_id = project_id
     self._recipes_path = recipes_path
     self._deps = deps
 
   @classmethod
@@ skipping 147 matching lines @@
     return not self.__eq__(other)
 
 
 class PackageDeps(object):
   """An object containing all the transitive dependencies of the root package.
   """
   def __init__(self, context, overrides=None):
     self._context = context
     self._packages = {}
     self._overrides = overrides or {}
+    self._root_package = None
 
   @classmethod
   def create(cls, repo_root, proto_file, allow_fetch=False, overrides=None):
     """Creates a PackageDeps object.
 
     Arguments:
       repo_root: the root of the repository containing this package.
       proto_file: a ProtoFile object corresponding to the repos recipes.cfg
       allow_fetch: whether to fetch dependencies rather than just checking for
                    them.
       overrides: if not None, a dictionary of project overrides. Dictionary keys
                  are the `project_id` field to override, and dictionary values
                  are the override path.
     """
     context = PackageContext.from_proto_file(repo_root, proto_file)
     if overrides:
       overrides = {project_id: PathRepoSpec(path)
                    for project_id, path in overrides.iteritems()}
     package_deps = cls(context, overrides=overrides)
 
-    package_deps._create_package(RootRepoSpec(proto_file), allow_fetch)
+    package_deps._root_package = package_deps._create_package(
+        RootRepoSpec(proto_file), allow_fetch)
     return package_deps
 
   def _create_package(self, repo_spec, allow_fetch):
     if allow_fetch:
       repo_spec.checkout(self._context)
     else:
       try:
         repo_spec.check_checkout(self._context)
       except UncleanFilesystemError as e:
         logging.warn(
@@ skipping 16 matching lines @@
             'Package specs do not match: %s vs %s' %
             (repo_spec, self._packages[project_id].repo_spec))
     self._packages[project_id] = None
 
     deps = {}
     for dep, dep_repo in sorted(package_spec.deps.items()):
       deps[dep] = self._create_package(dep_repo, allow_fetch)
 
     package = Package(
         project_id, repo_spec, deps,
-        os.path.join(repo_spec.repo_root(self._context),
-                     package_spec.recipes_path))
+        repo_root=repo_spec.repo_root(self._context),
+        relative_recipes_dir=package_spec.recipes_path)
 
     self._packages[project_id] = package
     return package
 
   # TODO(luqui): Remove this, so all accesses to packages are done
   # via other packages with properly scoped deps.
   def get_package(self, package_id):
     return self._packages[package_id]
 
   @property
+  def root_package(self):
+    """Returns the Package at the front of the dependency graph."""
+    return self._root_package
+
+  @property
   def packages(self):
     for p in self._packages.values():
       yield p
 
   @property
   def engine_recipes_py(self):
     return os.path.join(self._context.repo_root, 'recipes.py')
 
 
 def _run_cmd(cmd, cwd=None):
@@ skipping 70 matching lines @@
   >>> d = { 'x': 1, 'y': 2 }
   >>> sorted(_updated(d, { 'y': 3, 'z': 4 }).items())
   [('x', 1), ('y', 3), ('z', 4)]
   >>> sorted(d.items())
   [('x', 1), ('y', 2)]
   """
 
   d = copy.copy(d)
   d.update(updates)
   return d