Load target build files in parallel using Python multiprocessing.

pylib/gyp/input.py

 # Copyright (c) 2012 Google Inc. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 from compiler.ast import Const
 from compiler.ast import Dict
 from compiler.ast import Discard
 from compiler.ast import List
 from compiler.ast import Module
 from compiler.ast import Node
 from compiler.ast import Stmt
 import compiler
 import copy
 import gyp.common
+import multiprocessing
 import optparse
 import os.path
 import re
 import shlex
 import subprocess
 import sys
+import time
 
 
 # A list of types that are treated as linkable.
 linkable_types = ['executable', 'shared_library', 'loadable_module']
 
 # A list of sections that contain links to other targets.
 dependency_sections = ['dependencies', 'export_dependent_settings']
 
 # base_path_sections is a list of sections defined by GYP that contain
 # pathnames.  The generators can provide more keys, the two lists are merged
@@ skipping 313 matching lines @@
 
   # If the generator needs absolue paths, then do so.
   if absolute_build_file_paths:
     build_file_path = os.path.abspath(build_file_path)
 
   if build_file_path in data['target_build_files']:
     # Already loaded.
     return
   data['target_build_files'].add(build_file_path)
 
+  data_keys = set(data.keys())

[M-A Ruel, 2012/09/05 19:45:39]

Why double copy?
data_keys = set(data)

[dmazzoni, 2012/09/06 16:40:59]

Done.

+  aux_data_keys = set(aux_data.keys())

[M-A Ruel, 2012/09/05 19:45:39]

same

[dmazzoni, 2012/09/06 16:40:59]

Done.

+
   gyp.DebugOutput(gyp.DEBUG_INCLUDES,
                   "Loading Target Build File '%s'" % build_file_path)
 
   build_file_data = LoadOneBuildFile(build_file_path, data, aux_data, variables,
                                      includes, True, check)
 
   # Store DEPTH for later use in generators.
   build_file_data['_DEPTH'] = depth
 
   # Set up the included_files key indicating which .gyp files contributed to
@@ skipping 47 matching lines @@
             "Unable to find targets in build file %s" % build_file_path
 
     # No longer needed.
     del build_file_data['target_defaults']
 
   # Look for dependencies.  This means that dependency resolution occurs
   # after "pre" conditionals and variable expansion, but before "post" -
   # in other words, you can't put a "dependencies" section inside a "post"
   # conditional within a target.
 
+  dependencies = []
   if 'targets' in build_file_data:
     for target_dict in build_file_data['targets']:
       if 'dependencies' not in target_dict:
         continue
       for dependency in target_dict['dependencies']:
-        other_build_file = \
-            gyp.common.ResolveTarget(build_file_path, dependency, None)[0]
-        try:
-          LoadTargetBuildFile(other_build_file, data, aux_data, variables,
-                              includes, depth, check)
-        except Exception, e:
-          gyp.common.ExceptionAppend(
-            e, 'while loading dependencies of %s' % build_file_path)
-          raise
+        dependencies.append(
+            gyp.common.ResolveTarget(build_file_path, dependency, None)[0])
 
-  return data
+  data_out = {}
+  for key in data:
+    if key not in data_keys:
+      data_out[key] = data[key]
+  aux_data_out = {}
+  for key in aux_data:
+    if key not in aux_data_keys:
+      aux_data_out[key] = aux_data[key]
 
+  return (build_file_path,
+          data_out,
+          aux_data_out,
+          dependencies)
+
+
+def CallLoadTargetBuildFile(global_flags,
+                            build_file_path, input_data,
+                            input_aux_data, input_variables,
+                            includes, depth, check):
+  for key, value in global_flags.iteritems():
+    globals()[key] = value
+  return LoadTargetBuildFile(build_file_path, input_data,
+                             input_aux_data, input_variables,
+                             includes, depth, check)
+
+
+def LoadTargetBuildFileCallback(result):

[M-A Ruel, 2012/09/06 18:02:36]

A bit of docstrings explaining the dataflows would help.

+  (build_file_path0, data0, aux_data0, dependencies0) = result
+  gyp.data['target_build_files'].add(build_file_path0)
+  for key in data0:

[M-A Ruel, 2012/09/06 18:02:36]

gyp.data.update(data0) ?

+    gyp.data[key] = data0[key]
+  for key in aux_data0:
+    gyp.aux_data[key] = aux_data0[key]
+  for new_dependency in dependencies0:
+    if new_dependency not in gyp.scheduled:
+      gyp.scheduled.add(new_dependency)
+      gyp.dependencies.append(new_dependency)
+  gyp.pending -= 1
+
+
+def LoadTargetBuildFileParallel(build_file_path, data, aux_data,
+                                variables, includes, depth, check):
+  gyp.dependencies = [build_file_path]
+  gyp.scheduled = set([build_file_path])
+  gyp.pending = 0
+  gyp.data = data
+  gyp.aux_data = aux_data
+  while gyp.dependencies or gyp.pending:
+    if not gyp.dependencies:
+      time.sleep(0.003)

[M-A Ruel, 2012/09/05 19:45:39]

it'll likely slow down the equivalent of time.sleep(0.01) on Windows, you would
have to confirm manually.

[dmazzoni, 2012/09/06 16:40:59]

On Linux and Mac, any delay is better than none.

I'm having trouble getting it to work on Windows at all - multiprocessing
complains that it can't find gyp_chromium after forking. Any ideas?

[M-A Ruel, 2012/09/06 18:02:36]

It's because fork doesn't exist on Windows. So it is 100% emulated by python.

As such, it is much better if the inner code to be processed can be separated
into a smaller process with a smaller script first. This is cumbersome, so you
may just want to not enable it on Windows.

+      continue
+
+    dependency = gyp.dependencies.pop()
+
+    gyp.pending += 1
+    data_in = {}
+    data_in['target_build_files'] = data['target_build_files']
+    aux_data_in = {}
+    global_flags = {
+      'path_sections': globals()['path_sections'],
+      'non_configuration_keys': globals()['non_configuration_keys'],
+      'absolute_build_file_paths': globals()['absolute_build_file_paths'],
+      'multiple_toolsets': globals()['multiple_toolsets']}
+
+    if 'pool' not in dir(gyp):
+      gyp.pool = multiprocessing.Pool(8)
+    gyp.pool.apply_async(
+        CallLoadTargetBuildFile,
+        args = (global_flags, dependency,
+                data_in, aux_data_in,
+                variables, includes, depth, check),
+        callback = LoadTargetBuildFileCallback)
+    time.sleep(0.003)

[M-A Ruel, 2012/09/06 18:02:36]

You could use a Queue.Queue instead, so you wouldn't have to poll&sleep at all

[M-A Ruel, 2012/09/18 19:42:18]

You didn't tell why you didn't want to use a single Queue for tasks completed in
gyp.dependencies.

In particular, the LoadTargetBuildFileCallback is called in a separate thread
and you are not using locking at all.

[dmazzoni, 2012/09/19 21:14:52]

You're right, I think there was a possible race condition. I fixed it with a
condition variable that gets notified each time a callback completes; this also
eliminates the need for polling or sleeping.

As for a Queue, I think it would be more code to manage subprocesses and results
explicitly and I'm not sure I see the benefit. multiprocessing.Pool already uses
a Queue under the hood, so I think this is giving us the same benefit with the
advantage of using a simpler high-level API.

 
 # Look for the bracket that matches the first bracket seen in a
 # string, and return the start and end as a tuple.  For example, if
 # the input is something like "<(foo <(bar)) blah", then it would
 # return (1, 13), indicating the entire string except for the leading
 # "<" and trailing " blah".
 def FindEnclosingBracketGroup(input):
   brackets = { '}': '{',
                ']': '[',
                ')': '(', }
@@ skipping 1919 matching lines @@
   # NOTE: data contains both "target" files (.gyp) and "includes" (.gypi), as
   # well as meta-data (e.g. 'included_files' key). 'target_build_files' keeps
   # track of the keys corresponding to "target" files.
   data = {'target_build_files': set()}
   aux_data = {}
   for build_file in build_files:
     # Normalize paths everywhere.  This is important because paths will be
     # used as keys to the data dict and for references between input files.
     build_file = os.path.normpath(build_file)
     try:
-      LoadTargetBuildFile(build_file, data, aux_data, variables, includes,
-                          depth, check)
+      LoadTargetBuildFileParallel(build_file, data, aux_data,
+                                  variables, includes, depth, check)
     except Exception, e:
       gyp.common.ExceptionAppend(e, 'while trying to load %s' % build_file)
       raise
 
   # Build a dict to access each target's subdict by qualified name.
   targets = BuildTargetsDict(data)
 
   # Fully qualify all dependency links.
   QualifyDependencies(targets)
 
@@ skipping 87 matching lines @@
     ValidateRunAsInTarget(target, target_dict, build_file)
     ValidateActionsInTarget(target, target_dict, build_file)
 
   # Generators might not expect ints.  Turn them into strs.
   TurnIntIntoStrInDict(data)
 
   # TODO(mark): Return |data| for now because the generator needs a list of
   # build files that came in.  In the future, maybe it should just accept
   # a list, and not the whole data dict.
   return [flat_list, targets, data]