Remove the hack to concat .d files into a single deps file.

pylib/gyp/generator/make.py

 # Copyright (c) 2012 Google Inc. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 # Notes:
 #
 # This is all roughly based on the Makefile system used by the Linux
 # kernel, but is a non-recursive make -- we put the entire dependency
 # graph in front of make and let it figure it out.
 #
@@ skipping 420 matching lines @@
     $(call do_postbuilds)
   )
 )
 endef
 
 # Declare the "%(default_target)s" target first so it is the default,
 # even though we don't have the deps yet.
 .PHONY: %(default_target)s
 %(default_target)s:
 
+# make looks for ways to re-generate included makefiles, but in our case, we
+# don't have a direct way. Explicitly telling make that it has nothing to do
+# for them makes it go faster.
+%%.d: ;
+
 # Use FORCE_DO_CMD to force a target to run.  Should be coupled with
 # do_cmd.
 .PHONY: FORCE_DO_CMD
 FORCE_DO_CMD:
 
 """)
 
 SHARED_HEADER_MAC_COMMANDS = """
 quiet_cmd_objc = CXX($(TOOLSET)) $@
 cmd_objc = $(CC.$(TOOLSET)) $(GYP_OBJCFLAGS) $(DEPFLAGS) -c -o $@ $<
@@ skipping 65 matching lines @@
 
 SHARED_FOOTER = """\
 # "all" is a concatenation of the "all" targets from all the included
 # sub-makefiles. This is just here to clarify.
 all:
 
 # Add in dependency-tracking rules.  $(all_deps) is the list of every single
 # target in our tree. Only consider the ones with .d (dependency) info:
 d_files := $(wildcard $(foreach f,$(all_deps),$(depsdir)/$(f).d))
 ifneq ($(d_files),)
-  # Rather than include each individual .d file, concatenate them into a
-  # single file which make is able to load faster.  We split this into
-  # commands that take 1000 files at a time to avoid overflowing the
-  # command line.
-  $(shell cat $(wordlist 1,1000,$(d_files)) > $(depsdir)/all.deps)
-%(generate_all_deps)s
-  # make looks for ways to re-generate included makefiles, but in our case, we
-  # don't have a direct way. Explicitly telling make that it has nothing to do
-  # for them makes it go faster.
-  $(depsdir)/all.deps: ;
-
-  include $(depsdir)/all.deps
+  include $(d_files)
 endif
 """
 
 header = """\
 # This file is generated by gyp; do not edit.
 
 """
 
 # Maps every compilable file extension to the do_cmd that compiles it.
 COMPILABLE_EXTENSIONS = {
@@ skipping 98 matching lines @@
 
 class MakefileWriter:
   """MakefileWriter packages up the writing of one target-specific foobar.mk.
 
   Its only real entry point is Write(), and is mostly used for namespacing.
   """
 
   def __init__(self, generator_flags, flavor):
     self.generator_flags = generator_flags
     self.flavor = flavor
-    # Keep track of the total number of outputs for this makefile.
-    self._num_outputs = 0
 
     self.suffix_rules_srcdir = {}
     self.suffix_rules_objdir1 = {}
     self.suffix_rules_objdir2 = {}
 
     # Generate suffix rules for all compilable extensions.
     for ext in COMPILABLE_EXTENSIONS.keys():
       # Suffix rules for source folder.
       self.suffix_rules_srcdir.update({ext: ("""\
 $(obj).$(TOOLSET)/$(TARGET)/%%.o: $(srcdir)/%%%s FORCE_DO_CMD
         @$(call do_cmd,%s,1)
 """ % (ext, COMPILABLE_EXTENSIONS[ext]))})
 
       # Suffix rules for generated source files.
       self.suffix_rules_objdir1.update({ext: ("""\
 $(obj).$(TOOLSET)/$(TARGET)/%%.o: $(obj).$(TOOLSET)/%%%s FORCE_DO_CMD
         @$(call do_cmd,%s,1)
 """ % (ext, COMPILABLE_EXTENSIONS[ext]))})
       self.suffix_rules_objdir2.update({ext: ("""\
 $(obj).$(TOOLSET)/$(TARGET)/%%.o: $(obj)/%%%s FORCE_DO_CMD
         @$(call do_cmd,%s,1)
 """ % (ext, COMPILABLE_EXTENSIONS[ext]))})
 
 
-  def NumOutputs(self):
-    return self._num_outputs
-
-
   def Write(self, qualified_target, base_path, output_filename, spec, configs,
             part_of_all):
     """The main entry point: writes a .mk file for a single target.
 
     Arguments:
       qualified_target: target we're generating
       base_path: path relative to source root we're building in, used to resolve
                  target-relative paths
       output_filename: output .mk file name to write
       spec, configs: gyp info
@@ skipping 292 matching lines @@
         # Only write the 'obj' and 'builddir' rules for the "primary" output
         # (:1); it's superfluous for the "extra outputs", and this avoids
         # accidentally writing duplicate dummy rules for those outputs.
         self.WriteLn('%s: obj := $(abs_obj)' % outputs[0])
         self.WriteLn('%s: builddir := $(abs_builddir)' % outputs[0])
         self.WriteMakeRule(outputs, inputs + ['FORCE_DO_CMD'], actions)
         for output in outputs:
           assert ' ' not in output, (
               "Spaces in rule filenames not yet supported (%s)"  % output)
         self.WriteLn('all_deps += %s' % ' '.join(outputs))
-        self._num_outputs += len(outputs)
 
         action = [self.ExpandInputRoot(ac, rule_source_root,
                                        rule_source_dirname)
                   for ac in rule['action']]
         mkdirs = ''
         if len(dirs) > 0:
           mkdirs = 'mkdir -p %s; ' % ' '.join(dirs)
         cd_action = 'cd %s; ' % Sourceify(self.path or '.')
 
         # action, cd_action, and mkdirs get written to a toplevel variable
@@ skipping 165 matching lines @@
     compilable = filter(Compilable, sources)
     objs = map(self.Objectify, map(self.Absolutify, map(Target, compilable)))
     self.WriteList(objs, 'OBJS')
 
     for obj in objs:
       assert ' ' not in obj, (
           "Spaces in object filenames not supported (%s)"  % obj)
     self.WriteLn('# Add to the list of files we specially track '
                  'dependencies for.')
     self.WriteLn('all_deps += $(OBJS)')
-    self._num_outputs += len(objs)
     self.WriteLn()
 
     # Make sure our dependencies are built first.
     if deps:
       self.WriteMakeRule(['$(OBJS)'], deps,
                          comment = 'Make sure our dependencies are built '
                                    'before any of us.',
                          order_only = True)
 
     # Make sure the actions and rules run first.
@@ skipping 436 matching lines @@
     self.WriteMakeRule(outputs, inputs,
                        actions = ['$(call do_cmd,%s%s)' % (command, suffix)],
                        comment = comment,
                        force = True)
     # Add our outputs to the list of targets we read depfiles from.
     # all_deps is only used for deps file reading, and for deps files we replace
     # spaces with ? because escaping doesn't work with make's $(sort) and
     # other functions.
     outputs = [QuoteSpaces(o, SPACE_REPLACEMENT) for o in outputs]
     self.WriteLn('all_deps += %s' % ' '.join(outputs))
-    self._num_outputs += len(outputs)
 
 
   def WriteMakeRule(self, outputs, inputs, actions=None, comment=None,
                     order_only=False, force=False, phony=False,
                     multiple_output_trick=True):
     """Write a Makefile rule, with some extra tricks.
 
     outputs: a list of outputs for the rule (note: this is not directly
              supported by make; see comments below)
     inputs: a list of inputs for the rule
@@ skipping 402 matching lines @@
   # Put build-time support tools next to the root Makefile.
   dest_path = os.path.dirname(makefile_path)
   gyp.common.CopyTool(flavor, dest_path)
 
   # Find the list of targets that derive from the gyp file(s) being built.
   needed_targets = set()
   for build_file in params['build_files']:
     for target in gyp.common.AllTargets(target_list, target_dicts, build_file):
       needed_targets.add(target)
 
-  num_outputs = 0
   build_files = set()
   include_list = set()
   for qualified_target in target_list:
     build_file, target, toolset = gyp.common.ParseQualifiedTarget(
         qualified_target)
 
     this_make_global_settings = data[build_file].get('make_global_settings', {})
     assert make_global_settings_dict == this_make_global_settings, (
         "make_global_settings needs to be the same for all targets.")
 
@@ skipping 20 matching lines @@
 
     spec = target_dicts[qualified_target]
     configs = spec['configurations']
 
     if flavor == 'mac':
       gyp.xcode_emulation.MergeGlobalXcodeSettingsToSpec(data[build_file], spec)
 
     writer = MakefileWriter(generator_flags, flavor)
     writer.Write(qualified_target, base_path, output_file, spec, configs,
                  part_of_all=qualified_target in needed_targets)
-    num_outputs += writer.NumOutputs()
 
     # Our root_makefile lives at the source root.  Compute the relative path
     # from there to the output_file for including.
     mkfile_rel_path = gyp.common.RelativePath(output_file,
                                               os.path.dirname(makefile_path))
     include_list.add(mkfile_rel_path)
 
   # Write out per-gyp (sub-project) Makefiles.
   depth_rel_path = gyp.common.RelativePath(options.depth, os.getcwd())
   for build_file in build_files:
@@ skipping 26 matching lines @@
         "ifeq ($(strip $(foreach prefix,$(NO_LOAD),\\\n"
         "    $(findstring $(join ^,$(prefix)),\\\n"
         "                 $(join ^," + include_file + ")))),)\n")
     root_makefile.write("  include " + include_file + "\n")
     root_makefile.write("endif\n")
   root_makefile.write('\n')
 
   if generator_flags.get('auto_regeneration', True):
     WriteAutoRegenerationRule(params, root_makefile, makefile_name, build_files)
 
-  # Write the rule to load dependencies.  We batch 1000 files at a time to
-  # avoid overflowing the command line.
-  all_deps = ""
-  for i in range(1001, num_outputs, 1000):
-    all_deps += ("""
-  ifneq ($(word %(start)d,$(d_files)),)
-    $(shell cat $(wordlist %(start)d,%(end)d,$(d_files)) >> $(depsdir)/all.deps)
-  endif""" % { 'start': i, 'end': i + 999 })
-
-  # Add a check to make sure we tried to process all the .d files.
-  all_deps += """
-  ifneq ($(word %(last)d,$(d_files)),)
-    $(error Found unprocessed dependency files (gyp didn't generate enough rules!))
-  endif
-""" % { 'last': ((num_outputs / 1000) + 1) * 1000 + 1 }
-
-  root_makefile.write(SHARED_FOOTER % { 'generate_all_deps': all_deps })
+  root_makefile.write(SHARED_FOOTER)
 
   root_makefile.close()