ninja: rules chained only by output of first one failing

pylib/gyp/generator/ninja.py

 # Copyright (c) 2012 Google Inc. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import copy
 import gyp
 import gyp.common
 import gyp.msvs_emulation
 import gyp.MSVSVersion
 import gyp.system_test
@@ skipping 286 matching lines @@
 
     assert targets == filter(None, targets), targets
     if len(targets) == 0:
       return None
     if len(targets) > 1:
       stamp = self.GypPathToUniqueOutput(name + '.stamp')
       targets = self.ninja.build(stamp, 'stamp', targets)
       self.ninja.newline()
     return targets[0]
 
+  def _SourcesHandledByRule(self, rule, sources):
+    """Get a list of files that from |sources| that |rule| handles."""
+    rule_ext = rule['extension']
+    result = []
+    for s in sources:

[Evan Martin, 2012/05/18 17:19:06]

s/s/source/

[scottmg, 2012/05/18 18:06:30]

Done.

+      if s.endswith('.' + rule_ext):
+        inputs, outputs = self._GetInputsAndOutputsForRule(rule, s)
+        if s in outputs:
+          # Don't the outputs of this rule with this rule, even if by

[Evan Martin, 2012/05/18 17:19:06]

This comment accidentally a word.

[scottmg, 2012/05/18 18:06:30]

Done.

+          # extension we ought to.
+          continue
+        result.append(s)
+    return result
+
+  def _GetInputsAndOutputsForRule(self, rule, source):
+    """Get the expanded inputs and outputs of the given |rule| for a
+    particular |source|."""
+    dirname, basename = os.path.split(source)
+    root, ext = os.path.splitext(basename)
+    outputs = [self.ExpandRuleVariables(o, root, dirname,
+                                        source, ext, basename)
+               for o in rule['outputs']]
+    inputs = [self.ExpandRuleVariables(i, root, dirname,
+                                       source, ext, basename)
+              for i in rule.get('inputs', [])]
+    return inputs, outputs
+
+  def _GetSources(self, spec):
+    """Get all sources for the given |spec|. In addition to the sources
+    directly specified in the gyp file, it includes sources generated by
+    actions and copies. Outputs of rules are not included because otherwise
+    rules that generate the same extension as they process cause much
+    confusion. Rule outputs are added when the rule is processed."""
+    sources = spec.get('sources', [])
+    for action in spec.get('actions', []):
+      #sources.extend(action['inputs'])

[Evan Martin, 2012/05/18 17:19:06]

Probably should delete dead code

[scottmg, 2012/05/18 18:06:30]

Done.

+      if int(action.get('process_outputs_as_sources', False)):
+        sources.extend(action.get('outputs', []))
+    for copy in spec.get('copies', []):
+      sources.extend(copy.get('files', []))
+    return set(sources)
+
+
   def WriteSpec(self, spec, config_name, generator_flags):
     """The main entry point for NinjaWriter: write the build rules for a spec.
 
     Returns a Target object, which represents the output paths for this spec.
     Returns None if there are no outputs (e.g. a settings-only 'none' type
     target)."""
 
     self.config_name = config_name
     self.name = spec['target_name']
     self.toolset = spec['toolset']
@@ skipping 25 matching lines @@
           compile_depends.append(target.PreCompileInput())
       actions_depends = filter(None, actions_depends)
       compile_depends = filter(None, compile_depends)
       actions_depends = self.WriteCollapsedDependencies('actions_depends',
                                                         actions_depends)
       compile_depends = self.WriteCollapsedDependencies('compile_depends',
                                                         compile_depends)
       self.target.preaction_stamp = actions_depends
       self.target.precompile_stamp = compile_depends
 
+    # Build a list of sources. We do this as a pre-process so that we have
+    # access to generated outputs for actions for other actions and rules.
+    sources = self._GetSources(spec)
+
     # Write out actions, rules, and copies.  These must happen before we
     # compile any sources, so compute a list of predependencies for sources
     # while we do it.
-    extra_sources = []
     mac_bundle_depends = []
     self.target.actions_stamp = self.WriteActionsRulesCopies(
-        spec, extra_sources, actions_depends, mac_bundle_depends)
+        spec, sources, actions_depends, mac_bundle_depends)
 
     # If we have actions/rules/copies, we depend directly on those, but
     # otherwise we depend on dependent target's actions/rules/copies etc.
     # We never need to explicitly depend on previous target's link steps,
     # because no compile ever depends on them.
     compile_depends_stamp = (self.target.actions_stamp or compile_depends)
 
     # Write out the compilation steps, if any.
     link_deps = []
-    sources = spec.get('sources', []) + extra_sources
     if sources:
       link_deps = self.WriteSources(
           config_name, config, sources, compile_depends_stamp,
           gyp.xcode_emulation.MacPrefixHeader(
               self.xcode_settings, self.GypPathToNinja,
               lambda path, lang: self.GypPathToUniqueOutput(path + '-' + lang)))
       # Some actions/rules output 'sources' that are already object files.
       link_deps += [self.GypPathToNinja(f)
           for f in sources if f.endswith(self.obj_ext)]
 
@@ skipping 51 matching lines @@
   def WriteWinIdlFiles(self, spec, prebuild):
     """Writes rules to match MSVS's implicit idl handling."""
     assert self.flavor == 'win'
     if self.msvs_settings.HasExplicitIdlRules(spec):
       return []
     outputs = []
     for source in filter(lambda x: x.endswith('.idl'), spec['sources']):
       self._WinIdlRule(source, prebuild, outputs)
     return outputs
 
-  def WriteActionsRulesCopies(self, spec, extra_sources, prebuild,
+  def WriteActionsRulesCopies(self, spec, sources, prebuild,
                               mac_bundle_depends):
     """Write out the Actions, Rules, and Copies steps.  Return a path
     representing the outputs of these steps."""
     outputs = []
     extra_mac_bundle_resources = []
 
     if 'actions' in spec:
-      outputs += self.WriteActions(spec['actions'], extra_sources, prebuild,
+      outputs += self.WriteActions(spec['actions'], sources, prebuild,
                                    extra_mac_bundle_resources)
     if 'rules' in spec:
-      outputs += self.WriteRules(spec['rules'], extra_sources, prebuild,
+      outputs += self.WriteRules(spec['rules'], sources, prebuild,
                                  extra_mac_bundle_resources)
     if 'copies' in spec:
       outputs += self.WriteCopies(spec['copies'], prebuild)
 
     if 'sources' in spec and self.flavor == 'win':
       outputs += self.WriteWinIdlFiles(spec, prebuild)
 
     stamp = self.WriteCollapsedDependencies('actions_rules_copies', outputs)
 
     if self.is_mac_bundle:
@@ skipping 11 matching lines @@
     |message| is a hand-written description, or None if not available.
     |fallback| is the gyp-level name of the step, usable as a fallback.
     """
     if self.toolset != 'target':
       verb += '(%s)' % self.toolset
     if message:
       return '%s %s' % (verb, self.ExpandSpecial(message))
     else:
       return '%s %s: %s' % (verb, self.name, fallback)
 
-  def WriteActions(self, actions, extra_sources, prebuild,
+  def WriteActions(self, actions, sources, prebuild,
                    extra_mac_bundle_resources):
     # Actions cd into the base directory.
     env = self.GetXcodeEnv()
     if self.flavor == 'win':
       env = self.msvs_settings.GetVSMacroEnv('$!PRODUCT_DIR')
     all_outputs = []
     for action in actions:
       # First write out a rule for the action.
       name = re.sub(r'[ {}$]', '_', action['action_name'])
       description = self.GenerateDescription('ACTION',
                                              action.get('message', None),
                                              name)
       is_cygwin = (self.msvs_settings.IsRuleRunUnderCygwin(action)
                    if self.flavor == 'win' else False)
       args = action['action']
       args = [self.msvs_settings.ConvertVSMacros(arg, self.base_to_build)
               for arg in args] if self.flavor == 'win' else args
       rule_name = self.WriteNewNinjaRule(name, args, description,
                                          is_cygwin, env=env)
 
       inputs = [self.GypPathToNinja(i, env) for i in action['inputs']]
-      if int(action.get('process_outputs_as_sources', False)):
-        extra_sources += action['outputs']
       if int(action.get('process_outputs_as_mac_bundle_resources', False)):
         extra_mac_bundle_resources += action['outputs']
       outputs = [self.GypPathToNinja(o, env) for o in action['outputs']]
 
       # Then write out an edge using the rule.
       self.ninja.build(outputs, rule_name, inputs,
                        order_only=prebuild)
       all_outputs += outputs
 
       self.ninja.newline()
 
     return all_outputs
 
-  def WriteRules(self, rules, extra_sources, prebuild,
-                 extra_mac_bundle_resources):

[Evan Martin, 2012/05/18 17:19:06]

It's so pleasing to see this param go, it always felt like a huge hack to me.

+  def WriteRules(self, rules, sources, prebuild, extra_mac_bundle_resources):
     all_outputs = []
     for rule in rules:
       # First write out a rule for the rule action.
       name = rule['rule_name']
       args = rule['action']
       description = self.GenerateDescription(
           'RULE',
           rule.get('message', None),
           ('%s ' + generator_default_variables['RULE_INPUT_PATH']) % name)
       is_cygwin = (self.msvs_settings.IsRuleRunUnderCygwin(rule)
@@ skipping 13 matching lines @@
       for argument in args:
         for var in special_locals:
           if ('${%s}' % var) in argument:
             needed_variables.add(var)
 
       def cygwin_munge(path):
         if is_cygwin:
           return path.replace('\\', '/')
         return path
 
+      rule_sources = self._SourcesHandledByRule(rule, sources)
       # For each source file, write an edge that generates all the outputs.
-      for source in rule.get('rule_sources', []):
+      for source in rule_sources:
         dirname, basename = os.path.split(source)
         root, ext = os.path.splitext(basename)
-
-        # Gather the list of inputs and outputs, expanding $vars if possible.
-        outputs = [self.ExpandRuleVariables(o, root, dirname,
-                                            source, ext, basename)
-                   for o in rule['outputs']]
-        inputs = [self.ExpandRuleVariables(i, root, dirname,
-                                           source, ext, basename)
-                  for i in rule.get('inputs', [])]
+        inputs, outputs = self._GetInputsAndOutputsForRule(rule, source)
 
         if int(rule.get('process_outputs_as_sources', False)):
-          extra_sources += outputs
+          sources.update(outputs)
         if int(rule.get('process_outputs_as_mac_bundle_resources', False)):
           extra_mac_bundle_resources += outputs
 
         extra_bindings = []
         for var in needed_variables:
           if var == 'root':
             extra_bindings.append(('root', cygwin_munge(root)))
           elif var == 'dirname':
             extra_bindings.append(('dirname', cygwin_munge(dirname)))
           elif var == 'source':
@@ skipping 907 matching lines @@
 
   user_config = params.get('generator_flags', {}).get('config', None)
   if user_config:
     GenerateOutputForConfig(target_list, target_dicts, data, params,
                             user_config)
   else:
     config_names = target_dicts[target_list[0]]['configurations'].keys()
     for config_name in config_names:
       GenerateOutputForConfig(target_list, target_dicts, data, params,
                               config_name)