Next level of changes to get more of Chrome building

pylib/gyp/generator/ninja.py

Index: pylib/gyp/generator/ninja.py
diff --git a/pylib/gyp/generator/ninja.py b/pylib/gyp/generator/ninja.py
index 7c66a1d0a7544eef40a0d46917fc5cb8ed274226..46c41f60d9a1180ae3d379f8b68ce615700b4cf9 100644
--- a/pylib/gyp/generator/ninja.py
+++ b/pylib/gyp/generator/ninja.py
@@ -8,6 +8,7 @@ import gyp.common
 import gyp.msvs_emulation
 import gyp.system_test
 import gyp.xcode_emulation
+import json
 import os.path
 import re
 import subprocess
@@ -216,7 +217,11 @@ class NinjaWriter:
 
     if self.flavor == 'win':
       # Don't use os.path.normpath here. Callers pass in './foo' and expect
-      # the result to be runnable, but normpath removes the prefix.
+      # the result to be runnable, but normpath removes the prefix. If the
+      # variable is $! prefixed on Windows, don't modify it (used for compiler
+      # flags, etc.)
+      if path.startswith('$!'):
+        return path[2:]
       return path.replace('/', '\\')
     return path
 
@@ -303,10 +308,11 @@ class NinjaWriter:
     self.target = Target(spec['type'])
 
     self.is_mac_bundle = gyp.xcode_emulation.IsMacBundle(self.flavor, spec)
+    self.xcode_settings = self.msvs_settings = None
     if self.flavor == 'mac':
       self.xcode_settings = gyp.xcode_emulation.XcodeSettings(spec)
-    else:
-      self.xcode_settings = None
+    if self.flavor == 'win':
+      self.msvs_settings = gyp.msvs_emulation.MsvsSettings(spec)
 
     # Compute predepends for all rules.
     # actions_depends is the dependencies this target depends on before running
@@ -351,7 +357,7 @@ class NinjaWriter:
     sources = spec.get('sources', []) + extra_sources
     if sources:
       link_deps = self.WriteSources(
-          config_name, config, sources, compile_depends_stamp,
+          config_name, config, sources, compile_depends_stamp, spec,
           gyp.xcode_emulation.MacPrefixHeader(
               self.xcode_settings, self.GypPathToNinja,
               lambda path, lang: self.GypPathToUniqueOutput(path + '-' + lang)))
@@ -435,14 +441,24 @@ class NinjaWriter:
                                              action.get('message', None),
                                              name)
       rule_name = self.WriteNewNinjaRule(name, action['action'], description,
-                                         env=env)
+                                         env=env, rule=action)
 
-      inputs = [self.GypPathToNinja(i, env) for i in action['inputs']]
+      if self.flavor == 'win':
+        inputs = [self.msvs_settings.ConvertVSMacros(i)
+                  for i in action['inputs']]
+      else:
+        inputs = actions['inputs']
+      inputs = [self.GypPathToNinja(i, env) for i in inputs]
       if int(action.get('process_outputs_as_sources', False)):
         extra_sources += action['outputs']
       if int(action.get('process_outputs_as_mac_bundle_resources', False)):
         extra_mac_bundle_resources += action['outputs']
-      outputs = [self.GypPathToNinja(o, env) for o in action['outputs']]
+      if self.flavor == 'win':
+        outputs = [self.msvs_settings.ConvertVSMacros(i)
+                   for i in action['outputs']]
+      else:
+        outputs = action['outputs']
+      outputs = [self.GypPathToNinja(o, env) for o in outputs]
 
       # Then write out an edge using the rule.
       self.ninja.build(outputs, rule_name, inputs,
@@ -464,7 +480,7 @@ class NinjaWriter:
           'RULE',
           rule.get('message', None),
           ('%s ' + generator_default_variables['RULE_INPUT_PATH']) % name)
-      rule_name = self.WriteNewNinjaRule(name, args, description)
+      rule_name = self.WriteNewNinjaRule(name, args, description, rule=rule)
 
       # TODO: if the command references the outputs directly, we should
       # simplify it to just use $out.
@@ -479,6 +495,12 @@ class NinjaWriter:
           if ('${%s}' % var) in argument:
             needed_variables.add(var)
 
+      is_cygwin = gyp.msvs_emulation.IsRuleRunUnderCygwin(rule)
+      def cygwin_munge(path):
+        if is_cygwin:
+          return path.replace('\\', '/')
+        return path
+
       # For each source file, write an edge that generates all the outputs.
       for source in rule.get('rule_sources', []):
         dirname, basename = os.path.split(source)
@@ -498,23 +520,27 @@ class NinjaWriter:
         extra_bindings = []
         for var in needed_variables:
           if var == 'root':
-            extra_bindings.append(('root', root))
+            extra_bindings.append(('root', cygwin_munge(root)))
           elif var == 'dirname':
-            extra_bindings.append(('dirname', dirname))
+            extra_bindings.append(('dirname', cygwin_munge(dirname)))
           elif var == 'source':
             # '$source' is a parameter to the rule action, which means
             # it shouldn't be converted to a Ninja path.  But we don't
             # want $!PRODUCT_DIR in there either.
             source_expanded = self.ExpandSpecial(source, self.base_to_build)
-            extra_bindings.append(('source', source_expanded))
+            extra_bindings.append(('source', cygwin_munge(source_expanded)))
           elif var == 'ext':
-            extra_bindings.append(('ext', ext))
+            extra_bindings.append(('ext', cygwin_munge(ext)))
           elif var == 'name':
             extra_bindings.append(('name', basename))
           else:
             assert var == None, repr(var)
 
-        inputs = map(self.GypPathToNinja, rule.get('inputs', []))
+        inputs = []
+        for input in rule.get('inputs', []):
+          inputs.append(self.ExpandRuleVariables(input, root, dirname, source,
+                                                 ext, basename))
+        inputs = map(self.GypPathToNinja, inputs)
         outputs = map(self.GypPathToNinja, outputs)
         self.ninja.build(outputs, rule_name, self.GypPathToNinja(source),
                          implicit=inputs,
@@ -574,7 +600,7 @@ class NinjaWriter:
                                 ('env', env)])
     bundle_depends.append(out)
 
-  def WriteSources(self, config_name, config, sources, predepends,
+  def WriteSources(self, config_name, config, sources, predepends, spec,
                    precompiled_header):
     """Write build rules to compile all of |sources|."""
     if self.toolset == 'target':
@@ -583,6 +609,7 @@ class NinjaWriter:
       self.ninja.variable('cxx', '$cxx_target')
       self.ninja.variable('ld', '$ld_target')
 
+    extra_defines = []
     if self.flavor == 'mac':
       cflags = self.xcode_settings.GetCflags(config_name)
       cflags_c = self.xcode_settings.GetCflagsC(config_name)
@@ -591,17 +618,32 @@ class NinjaWriter:
                     self.xcode_settings.GetCflagsObjC(config_name)
       cflags_objcc = ['$cflags_cc'] + \
                      self.xcode_settings.GetCflagsObjCC(config_name)
+    elif self.flavor == 'win':
+      cflags = self.msvs_settings.GetCflags(config_name)
+      cflags_c = self.msvs_settings.GetCflagsC(config_name)
+      cflags_cc = self.msvs_settings.GetCflagsCC(config_name)
+      extra_defines = self.msvs_settings.GetComputedDefines(config_name)
+      self.msvs_settings.HandleIdlFiles(
+          self.ninja, spec, sources, config_name,
+          self.GypPathToNinja,
+          self.GypPathToUniqueOutput,
+          self.ExpandRuleVariables)
     else:
       cflags = config.get('cflags', [])
       cflags_c = config.get('cflags_c', [])
       cflags_cc = config.get('cflags_cc', [])
 
+    defines = config.get('defines', []) + extra_defines
     self.WriteVariableList('defines',
         [QuoteShellArgument(ninja_syntax.escape('-D' + d), self.flavor)
-         for d in config.get('defines', [])])
+         for d in defines])
+    include_dirs = config.get('include_dirs', [])
+    if self.flavor == 'win':
+      include_dirs = self.msvs_settings.AdjustIncludeDirs(include_dirs,
+                                                          config_name)
     self.WriteVariableList('includes',
-                           ['-I' + self.GypPathToNinja(i)
-                            for i in config.get('include_dirs', [])])
+        [QuoteShellArgument('-I' + self.GypPathToNinja(i), self.flavor)
+         for i in include_dirs])
 
     pch_commands = precompiled_header.GetGchBuildCommands()
     if self.flavor == 'mac':
@@ -636,7 +678,6 @@ class NinjaWriter:
       elif self.flavor == 'mac' and ext == 'mm':
         command = 'objcxx'
       else:
-        # TODO: should we assert here on unexpected extensions?
         continue
       input = self.GypPathToNinja(source)
       output = self.GypPathToUniqueOutput(filename + self.obj_ext)
@@ -711,6 +752,14 @@ class NinjaWriter:
       ldflags = self.xcode_settings.GetLdflags(config_name,
           self.ExpandSpecial(generator_default_variables['PRODUCT_DIR']),
           self.GypPathToNinja)
+    elif self.flavor == 'win':
+      ldflags = self.msvs_settings.GetLdflags(config_name, spec,
+          self.ExpandSpecial(generator_default_variables['PRODUCT_DIR']),
+          self.GypPathToNinja)
+      libflags = self.msvs_settings.GetLibFlags(config_name, spec)
+      self.WriteVariableList('libflags',
+                             gyp.common.uniquer(map(self.ExpandSpecial,
+                                                    libflags)))
     else:
       ldflags = config.get('ldflags', [])
     self.WriteVariableList('ldflags',
@@ -721,6 +770,8 @@ class NinjaWriter:
                                        spec.get('libraries', [])))
     if self.flavor == 'mac':
       libraries = self.xcode_settings.AdjustLibraries(libraries)
+    elif self.flavor == 'win':
+      libraries = self.msvs_settings.AdjustLibraries(libraries)
     self.WriteVariableList('libs', libraries)
 
     self.target.binary = output
@@ -731,8 +782,8 @@ class NinjaWriter:
         import_lib = output + '.lib'
         extra_bindings.append(('dll', output))
         extra_bindings.append(('implib', import_lib))
-        self.target.binary = import_lib
         output = [output, import_lib]
+        self.target.binary = import_lib
 
     self.ninja.build(output, command, link_deps,
                      implicit=list(implicit_deps),
@@ -934,7 +985,7 @@ class NinjaWriter:
       values = []
     self.ninja.variable(var, ' '.join(values))
 
-  def WriteNewNinjaRule(self, name, args, description, env={}):
+  def WriteNewNinjaRule(self, name, args, description, env={}, rule=None):
     """Write out a new ninja "rule" statement for a given command.
 
     Returns the name of the new rule."""
@@ -950,19 +1001,26 @@ class NinjaWriter:
 
     args = args[:]
 
+    is_cygwin = rule and int(rule.get('msvs_cygwin_shell', 1))
+
     # gyp dictates that commands are run from the base directory.
     # cd into the directory before running, and adjust paths in
     # the arguments to point to the proper locations.
     if self.flavor == 'win':
-      cd = 'cmd /s /c "cd %s && ' % self.build_to_base
+      cd = 'cmd /s /c "'
+      if not is_cygwin:
+        cd += 'cd %s && ' % self.build_to_base
     else:
       cd = 'cd %s; ' % self.build_to_base
     args = [self.ExpandSpecial(arg, self.base_to_build) for arg in args]
     env = self.ComputeExportEnvString(env)
     if self.flavor == 'win':
-      # TODO(scottmg): Respect msvs_cygwin setting here.
-      # If there's no command, fake one to match the dangling |&&| above.
-      command = gyp.msvs_emulation.EncodeCmdExeList(args) or 'cmd /c'
+      if is_cygwin:
+        command = gyp.msvs_emulation.BuildCygwinBashCommandLine(
+            self.msvs_settings, rule, args, self.build_to_base, self.config_name)
+      else:
+        # If there's no command, fake one to match the dangling |&&| above.
+        command = self.msvs_settings.EncodeCmdExeList(args) or 'cmd /c'
     else:
       command = gyp.common.EncodePOSIXShellList(args)
     if env:
@@ -1011,6 +1069,21 @@ def CalculateVariables(default_variables, params):
     default_variables['STATIC_LIB_SUFFIX'] = '.lib'
     default_variables['SHARED_LIB_PREFIX'] = ''
     default_variables['SHARED_LIB_SUFFIX'] = '.dll'
+    generator_flags = params.get('generator_flags', {})
+    msvs_version = gyp.msvs_emulation.GetVSVersion(generator_flags)
+
+    # Set a variable so conditions can be based on msvs_version.
+    default_variables['MSVS_VERSION'] = msvs_version.ShortName()
+
+    # To determine processor word size on Windows, in addition to checking
+    # PROCESSOR_ARCHITECTURE (which reflects the word size of the current
+    # process), it is also necessary to check PROCESSOR_ARCHITEW6432 (which
+    # contains the actual word size of the system when running thru WOW64).
+    if ('64' in os.environ.get('PROCESSOR_ARCHITECTURE', '') or
+        '64' in os.environ.get('PROCESSOR_ARCHITEW6432', '')):
+      default_variables['MSVS_OS_BITS'] = 64
+    else:
+      default_variables['MSVS_OS_BITS'] = 32
   else:
     operating_system = flavor
     if flavor == 'android':
@@ -1033,6 +1106,7 @@ def OpenOutput(path):
 
 def GenerateOutputForConfig(target_list, target_dicts, data, params,
                             config_name):
+  print "Generating ninja build files for %s..." % config_name
   options = params['options']
   flavor = gyp.common.GetFlavor(params)
   generator_flags = params.get('generator_flags', {})
@@ -1042,16 +1116,17 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
   build_dir = os.path.join(generator_flags.get('output_dir', 'out'),
                            config_name)
 
+  top_build_dir = os.path.join(options.toplevel_dir, build_dir)
   master_ninja = ninja_syntax.Writer(
-      OpenOutput(os.path.join(options.toplevel_dir, build_dir, 'build.ninja')),
+      OpenOutput(os.path.join(top_build_dir, 'build.ninja')),
       width=120)
 
   # Put build-time support tools in out/{config_name}.
-  gyp.common.CopyTool(flavor, os.path.join(options.toplevel_dir, build_dir))
+  gyp.common.CopyTool(flavor, top_build_dir)
 
   # Grab make settings for CC/CXX.
   if flavor == 'win':
-    cc = cxx = 'cl'
+    cc = cxx = gyp.msvs_emulation.GetCLPath(generator_flags)
   else:
     cc, cxx = 'gcc', 'g++'
   build_file, _, _ = gyp.common.ParseQualifiedTarget(target_list[0])
@@ -1068,7 +1143,9 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
   master_ninja.variable('cc', os.environ.get('CC', cc))
   master_ninja.variable('cxx', os.environ.get('CXX', cxx))
   if flavor == 'win':
-    master_ninja.variable('ld', 'link')
+    master_ninja.variable('ld', gyp.msvs_emulation.GetLinkPath(generator_flags))
+    master_ninja.variable('idl',
+        gyp.msvs_emulation.GetMidlPath(generator_flags))
   else:
     master_ninja.variable('ld', flock + ' linker.lock $cxx')
 
@@ -1076,7 +1153,8 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
   master_ninja.variable('cc_target', os.environ.get('CC_target', '$cc'))
   master_ninja.variable('cxx_target', os.environ.get('CXX_target', '$cxx'))
   if flavor == 'win':
-    master_ninja.variable('ld_target', 'link')
+    master_ninja.variable('ld_target', gyp.msvs_emulation.GetLinkPath(
+        generator_flags))
   else:
     master_ninja.variable('ld_target', flock + ' linker.lock $cxx_target')
 
@@ -1103,19 +1181,29 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
     master_ninja.rule(
       'cc',
       description='CC $out',
-      command=('cmd /c $cc /nologo /showIncludes '
-               '$defines $includes $cflags $cflags_c '
-               '$cflags_pch_c /c $in /Fo$out '
-               '| ninja-deplist-helper -f cl -o $out.dl'),
-      deplist='$out.dl')
+      command=('cmd /s /c "$cc /nologo /showIncludes '
+               '@$out.rsp '
+               '$cflags_pch_c /c $in /Fo$out /Fd$out.pdb '
+               '| ninja-deplist-helper -d .ninja_depdb -q -f cl -o $in"'),
+      depdb='$in',
+      rspfile='$out.rsp',
+      rspfile_content='$defines $includes $cflags $cflags_c')
     master_ninja.rule(
       'cxx',
       description='CXX $out',
-      command=('cmd /c $cxx /nologo /showIncludes '
-               '$defines $includes $cflags $cflags_cc '
-               '$cflags_pch_cc /c $in /Fo$out '
-               '| ninja-deplist-helper -f cl -o $out.dl'),
-      deplist='$out.dl')
+      command=('cmd /s /c "$cxx /nologo /showIncludes '
+               '@$out.rsp '
+               '$cflags_pch_cc /c $in /Fo$out /Fd$out.pdb '
+               '| ninja-deplist-helper -d .ninja_depdb -q -f cl -o $in"'),
+      depdb='$in',
+      rspfile='$out.rsp',
+      rspfile_content='$defines $includes $cflags $cflags_cc')
+    master_ninja.rule(
+      'idl',
+      description='IDL $outdir',
+      command=('python gyp-win-tool quiet-midl $outdir '
+               '$tlb $h $dlldata $iid $proxy $in '
+               '$idlflags'))
 
   if flavor != 'mac' and flavor != 'win':
     master_ninja.rule(
@@ -1141,19 +1229,29 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
     master_ninja.rule(
       'alink',
       description='LIB $out',
-      command='lib /nologo /OUT:$out $in')
-    master_ninja.rule(
-      'solink',
-      description='LINK(DLL) $dll',
-      command=('$ld /nologo /IMPLIB:$implib /DLL $ldflags /OUT:$dll $in $libs'))
-    master_ninja.rule(
-      'solink_module',
-      description='LINK(DLL) $dll',
-      command=('$ld /nologo /IMPLIB:$implib /DLL $ldflags /OUT:$dll $in $libs'))
+      command='lib /nologo /ignore:4221 $libflags /OUT:$out @$out.rsp',
+      rspfile='$out.rsp',
+      rspfile_content='$in')
+    # TODO(scottmg): The lib is the main output, rather than the dll. link
+    # doesn't create the import lib unless there's exports from the dll,
+    # however. This means the dependencies are wrong and the dll will always
+    # relink if it has no exports. We could touch a dummy file, but we don't
+    # want to do && touch, because that would mean we'd have to prepend cmd,
+    # which would impose a short line length on the link command line. That
+    # can be resolved by adding support for @rsp files, but neither ninja or
+    # gyp takes care of that right now. So, currently, we force the export
+    # of the entry point, and then ignore the warning that we're doing so,
+    # which causes the lib to always get generated.
+    dlldesc = 'LINK(DLL) $dll and $implib'
+    dllcmd = ('$ld /nologo /IMPLIB:$implib /DLL $ldflags /OUT:$dll '
+               '/PDB:$dll.pdb $libs @$dll.rsp')
+    master_ninja.rule('solink', description=dlldesc, command=dllcmd,
+                      rspfile='$dll.rsp', rspfile_content='$in')
+    master_ninja.rule('solink_module', description=dlldesc, command=dllcmd)
     master_ninja.rule(
       'link',
       description='LINK $out',
-      command=('$ld /nologo $ldflags /OUT:$out $in $libs'))
+      command=('$ld /nologo $ldflags /OUT:$out /PDB:$out.pdb $in $libs'))
   else:
     master_ninja.rule(
       'objc',
@@ -1212,8 +1310,7 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
     master_ninja.rule(
       'copy',
       description='COPY $in $out',
-      command='cmd /c mklink /h $out $in >nul || mklink /h /j $out $in >nul || '
-              'python gyp-win-tool recursive-mirror $in $out')
+      command='python gyp-win-tool recursive-mirror $in $out')
   else:
     master_ninja.rule(
       'stamp',
@@ -1256,11 +1353,9 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
       obj += '.' + toolset
     output_file = os.path.join(obj, base_path, name + '.ninja')
 
-    abs_build_dir=os.path.abspath(os.path.join(options.toplevel_dir, build_dir))
+    abs_build_dir=os.path.abspath(top_build_dir)
     writer = NinjaWriter(target_outputs, base_path, build_dir,
-                         OpenOutput(os.path.join(options.toplevel_dir,
-                                                 build_dir,
-                                                 output_file)),
+                         OpenOutput(os.path.join(top_build_dir, output_file)),
                          flavor, abs_build_dir=abs_build_dir)
     master_ninja.subninja(output_file)