'recipes.py isolate' command to build an isolate of the current package's toolchain

recipe_engine/package.py

Index: recipe_engine/package.py
diff --git a/recipe_engine/package.py b/recipe_engine/package.py
index 34b8727fac93ff63be81814436c7824dbc59baa5..69a2f6d232453d742b1cf93d429af7ea5d4f20bf 100644
--- a/recipe_engine/package.py
+++ b/recipe_engine/package.py
@@ -10,6 +10,7 @@ import functools
 import itertools
 import logging
 import os
+import re
 import subprocess
 import sys
 import tempfile
@@ -355,18 +356,28 @@ class Package(object):
 
   This is accessed by loader.py through RecipeDeps.get_package.
   """
-  def __init__(self, name, repo_spec, deps, recipes_dir):
+  def __init__(self, name, repo_spec, deps, repo_root, relative_recipes_dir):
     self.name = name
     self.repo_spec = repo_spec
     self.deps = deps
-    self.recipes_dir = recipes_dir
+    self.relative_recipes_dir = relative_recipes_dir
+    self._repo_root = repo_root
+
+  @property
+  def recipes_dir(self):
+    """Returns the root of the recipes area (the part containing recipes/ and
+    recipe_modules/
+    """
+    return os.path.join(self._repo_root, self.relative_recipes_dir)
 
   @property
   def recipe_dirs(self):
+    """Returns a collection of directories containing recipes."""

[M-A Ruel, 2016/01/09 00:51:59]

Returns the list of

[luqui, 2016/01/15 23:11:27]

Done.

     return [os.path.join(self.recipes_dir, 'recipes')]
 
   @property
   def module_dirs(self):
+    """Returns a collection of directories containing recipe modules."""

[M-A Ruel, 2016/01/09 00:51:59]

same 

[luqui, 2016/01/15 23:11:27]

Done.

     return [os.path.join(self.recipes_dir, 'recipe_modules')]
 
   def find_dep(self, dep_name):
@@ -386,6 +397,43 @@ class Package(object):
           if _is_recipe_module_dir(subpath):
             yield subpath
 
+  def all_files(self, context, seen=None):
+    """Returns a generator listing all file paths this package needs to run."""

[M-A Ruel, 2016/01/09 00:51:59]

Yields all file ...

[luqui, 2016/01/15 23:11:27]

Done.

+    if seen is None:
+      seen = set()
+    if self in seen:
+      return
+    seen.add(self)
+
+    EXCLUDE = re.compile('\.pyc$')  # pycs are not portable
+
+    dirs = list(self.recipe_dirs) + list(self.module_dirs)

[M-A Ruel, 2016/01/09 00:51:59]

The the two properties return a list, I don't think it's needed to create 2
temporary copies

[luqui, 2016/01/15 23:11:27]

Good obs. One upon a time they were generators.

+    recipes_extra = os.path.join(self.recipes_dir, 'recipes.extra')
+    if os.path.exists(recipes_extra):
+      yield recipes_extra
+      with open(recipes_extra, 'r') as fh:
+        for fname in itertools.imap(self._normalize_extra_line, fh.readlines()):
+          f = os.path.join(self.recipes_dir, fname)
+          if os.path.isdir(f):
+            dirs.append(f)
+          else:
+            yield f
+
+    yield self.repo_spec.proto_file(context).path
+    yield os.path.join(self.recipes_dir, 'recipes.py')
+    for basedir in dirs:
+      for root, dirs, files in os.walk(basedir):
+        for fname in files:
+          if not EXCLUDE.search(fname):

[M-A Ruel, 2016/01/09 00:51:59]

a regexp is overkill here when this would be fine:
if not fname.endswith('.pyc'):

[luqui, 2016/01/15 23:11:27]

I want the idea of EXCLUDE to be more visible than 3 levels nested in a loop, in
case people are wondering why their pycs aren't getting included.

+            yield os.path.join(root, fname)
+
+    for dep in self.deps.values():

[M-A Ruel, 2016/01/09 00:51:59]

itervalues

[luqui, 2016/01/15 23:11:27]

Done.

+      for f in dep.all_files(context, seen):
+        yield f
+
+  def _normalize_extra_line(self, line):

[M-A Ruel, 2016/01/09 00:51:59]

This doesn't need to be a method. IMHO, it'd be more readable if you inlined it.

[luqui, 2016/01/15 23:11:27]

Ah yeah, and get rid of the imap.  Agreed, Done.

+    return re.sub(r'#.*', '', line).replace('/', os.sep).strip()
+
 
 class PackageSpec(object):
   API_VERSION = 1
@@ -553,6 +601,7 @@ class PackageDeps(object):
     self._context = context
     self._packages = {}
     self._overrides = overrides or {}
+    self._root_package = None
 
   @classmethod
   def create(cls, repo_root, proto_file, allow_fetch=False, overrides=None):
@@ -573,7 +622,8 @@ class PackageDeps(object):
                    for project_id, path in overrides.iteritems()}
     package_deps = cls(context, overrides=overrides)
 
-    package_deps._create_package(RootRepoSpec(proto_file), allow_fetch)
+    package_deps._root_package = package_deps._create_package(
+        RootRepoSpec(proto_file), allow_fetch)
     return package_deps
 
   def _create_package(self, repo_spec, allow_fetch):
@@ -610,8 +660,8 @@ class PackageDeps(object):
 
     package = Package(
         project_id, repo_spec, deps,
-        os.path.join(repo_spec.repo_root(self._context),
-                     package_spec.recipes_path))
+        repo_root=repo_spec.repo_root(self._context),
+        relative_recipes_dir=package_spec.recipes_path)
 
     self._packages[project_id] = package
     return package
@@ -622,6 +672,11 @@ class PackageDeps(object):
     return self._packages[package_id]
 
   @property
+  def root_package(self):
+    """Returns the Package at the front of the dependency graph."""
+    return self._root_package
+
+  @property
   def packages(self):
     for p in self._packages.values():
       yield p