Initial refactoring to eventually support touch-only access.

tools/isolate/trace_inputs.py

 #!/usr/bin/env python
 # coding=utf-8
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Traces an executable and its child processes and extract the files accessed
 by them.
 
 The implementation uses OS-specific API. The native Kernel logger and the ETL
@@ skipping 551 matching lines @@
         json.dump(data, f, separators=(',',':'))
       else:
         json.dump(data, f, sort_keys=True, indent=2)
 
 
 class Results(object):
   """Results of a trace session."""
 
   class _TouchedObject(object):
     """Something, a file or a directory, that was accessed."""
-    def __init__(self, root, path, tainted):
-      logging.debug('%s(%s, %s)' % (self.__class__.__name__, root, path))
+    def __init__(self, root, path, tainted, size, nb_files):
+      logging.debug(
+          '%s(%s, %s, %s, %s, %s)' %
+          (self.__class__.__name__, root, path, tainted, size, nb_files))
       self.root = root
       self.path = path
       self.tainted = tainted
+      self.nb_files = nb_files
+      # Can be used as a cache or a default value, depending on context.
+      self._size = size
       # These are cache only.
       self._real_path = None
-      self._size = None
 
       # Check internal consistency.
       assert path, path
       assert tainted or bool(root) != bool(isabs(path)), (root, path)
       assert tainted or (
           not os.path.exists(self.full_path) or
           (self.full_path == get_native_path_case(self.full_path))), (
               tainted, self.full_path, get_native_path_case(self.full_path))
 
     @property
@@ skipping 17 matching lines @@
     def size(self):
       """File's size. -1 is not existent."""
       if self._size is None and not self.tainted:
         try:
           self._size = os.stat(self.full_path).st_size
         except OSError:
           self._size = -1
       return self._size
 
     def flatten(self):
-      """Returns a dict representing this object."""
+      """Returns a dict representing this object.
+
+      A 'size' of 0 means the file was only touched and not read.
+      """
       return {
         'path': self.path,
         'size': self.size,
       }
 
     def replace_variables(self, variables):
       """Replaces the root of this File with one of the variables if it matches.
 
       If a variable replacement occurs, the cloned object becomes tainted.
       """
@@ skipping 15 matching lines @@
           return None
         path = self.real_path
       else:
         path = self.full_path
       return self._clone(root, path[len(root):], self.tainted)
 
     def _clone(self, new_root, new_path, tainted):
       raise NotImplementedError(self.__class__.__name__)
 
   class File(_TouchedObject):
-    """A file that was accessed.
+    """A file that was accessed. May not be present anymore.
 
     If tainted is true, it means it is not a real path anymore as a variable
     replacement occured.
+
+    If touched_only is True, this means the file was probed for existence, and
+    it is existent, but was never _opened_. If touched_only is True, the file
+    must have existed.
     """
-    def __init__(self, root, path, tainted):
-      """Represents a file accessed. May not be present anymore."""
-      super(Results.File, self).__init__(root, path, tainted)
-      # For compatibility with Directory object interface.
-      # Shouldn't be used normally, only exists to simplify algorithms.
-      self.nb_files = 1
+    def __init__(self, root, path, tainted, size):
+      super(Results.File, self).__init__(root, path, tainted, size, 1)
 
     def _clone(self, new_root, new_path, tainted):
       """Clones itself keeping meta-data."""
-      out = self.__class__(new_root, new_path, tainted)
-      # Keep the cache for performance reason. It is also important when the
-      # file becomes tainted (with a variable instead of the real path) since
-      # self.path is not an on-disk path anymore so out._size cannot be updated.
-      out._size = self.size
+      # Keep the self.size and self._real_path caches for performance reason. It
+      # is also important when the file becomes tainted (with a variable instead
+      # of the real path) since self.path is not an on-disk path anymore so
+      # out._size cannot be updated.
+      out = self.__class__(new_root, new_path, tainted, self.size)
       out._real_path = self._real_path
       return out
 
   class Directory(_TouchedObject):
     """A directory of files. Must exist."""
     def __init__(self, root, path, tainted, size, nb_files):
       """path='.' is a valid value and must be handled appropriately."""
       assert not path.endswith(os.path.sep), path
-      super(Results.Directory, self).__init__(root, path + os.path.sep, tainted)
-      self.nb_files = nb_files
+      super(Results.Directory, self).__init__(
+          root, path + os.path.sep, tainted, size, nb_files)
       # In that case, it's not a cache, it's an actual value that is never
-      # modified.
+      # modified and represents the total size of the files contained in this
+      # directory.
       assert size
-      self._size = size
 
     def flatten(self):
       out = super(Results.Directory, self).flatten()
       out['nb_files'] = self.nb_files
       return out
 
     def _clone(self, new_root, new_path, tainted):
       """Clones itself keeping meta-data."""
       out = self.__class__(
           new_root,
           new_path.rstrip(os.path.sep),
           tainted,
           self.size,
           self.nb_files)
       out._real_path = self._real_path
       return out
 
   class Process(object):
     """A process that was traced.
 
     Contains references to the files accessed by this process and its children.
     """
-    def __init__(
-        self, pid, files, executable, command, initial_cwd, children):
+    def __init__(self, pid, files, executable, command, initial_cwd, children):
       logging.debug('Process(%s, %d, ...)' % (pid, len(files)))
       self.pid = pid
-      self.files = sorted(
-          (Results.File(None, f, False) for f in files), key=lambda x: x.path)
+      self.files = sorted(files, key=lambda x: x.path)
       self.children = children
       self.executable = executable
       self.command = command
       self.initial_cwd = initial_cwd
 
       # Check internal consistency.
-      assert len(set(f.path for f in self.files)) == len(self.files), [
-          f.path for f in self.files]
+      assert len(set(f.path for f in self.files)) == len(self.files), sorted(
+          f.path for f in self.files)
       assert isinstance(self.children, list)
       assert isinstance(self.files, list)
 
     @property
     def all(self):
       for child in self.children:
         for i in child.all:
           yield i
       yield self
 
     def flatten(self):
       return {
         'children': [c.flatten() for c in self.children],
         'command': self.command,
         'executable': self.executable,
         'files': [f.flatten() for f in self.files],
         'initial_cwd': self.initial_cwd,
         'pid': self.pid,
       }
 
     def strip_root(self, root):
       assert isabs(root) and root.endswith(os.path.sep), root
+      # Loads the files after since they are constructed as objects.
       out = self.__class__(
           self.pid,
-          [],
+          filter(None, (f.strip_root(root) for f in self.files)),
           self.executable,
           self.command,
           self.initial_cwd,
           [c.strip_root(root) for c in self.children])
-      # Override the files property.
-      out.files = filter(None, (f.strip_root(root) for f in self.files))
       logging.debug(
           'strip_root(%s) %d -> %d' % (root, len(self.files), len(out.files)))
       return out
 
-
   def __init__(self, process):
     self.process = process
     # Cache.
     self._files = None
 
   def flatten(self):
     return {
       'root': self.process.flatten(),
     }
 
@@ skipping 40 matching lines @@
         assert isinstance(root, ApiBase.Context)
         assert isinstance(pid, int), repr(pid)
         self.root = weakref.ref(root)
         self.pid = pid
         # Children are pids.
         self.children = []
         self.parentid = parentid
         self.initial_cwd = initial_cwd
         self.cwd = None
         self.files = set()
+        self.only_touched = set()
         self.executable = None
         self.command = None
 
         if parentid:
           self.root().processes[parentid].children.append(pid)
 
       def to_results_process(self):
         """Resolves file case sensitivity and or late-bound strings."""
         children = [
           self.root().processes[c].to_results_process() for c in self.children
         ]
         # When resolving files, it's normal to get dupe because a file could be
         # opened multiple times with different case. Resolve the deduplication
         # here.
         def render_to_string_and_fix_case(x):
           """Returns the native file path case if the file exists.
 
           Converts late-bound strings.
           """
           if not x:
             return x
           # TODO(maruel): Do not upconvert to unicode here, on linux we don't
           # know the file path encoding so they must be treated as bytes.
           x = unicode(x)
           if not os.path.exists(x):
             return x
           return get_native_path_case(x)
 
+        # Filters out directories. Some may have passed through.
+        files = set(map(render_to_string_and_fix_case, self.files))
+        only_touched = set(
+            map(render_to_string_and_fix_case, self.only_touched))
+        only_touched -= files
+
         files = [
-          f for f in set(map(render_to_string_and_fix_case, self.files))
+          Results.File(None, f, False, None) for f in files
           if not os.path.isdir(f)
         ]
+        # Using 0 as size means the file's content is ignored since the file was
+        # never opened for I/O.
+        files.extend(
+          Results.File(None, f, False, 0) for f in only_touched
+          if not os.path.isdir(f)
+        )
         return Results.Process(
             self.pid,
             files,
             render_to_string_and_fix_case(self.executable),
             self.command,
             render_to_string_and_fix_case(self.initial_cwd),
             children)
 
       def add_file(self, filepath):
         if self.root().blacklist(unicode(filepath)):
@@ skipping 2229 matching lines @@
     return command(argv[1:])
   except TracingFailure, e:
     sys.stderr.write('\nError: ')
     sys.stderr.write(str(e))
     sys.stderr.write('\n')
     return 1
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv[1:]))