tools: backport patches from upstream gyp

this is a backport of 4 gyp commits since last bump, syncing us to
https://chromium.googlesource.com/external/gyp/+/a478c1ab51ea3e04e79791ac3d1dad01b3f57434
this is instead of a bump and rebase of floating patches

the goal is to fix the ninja generator on Windows

also includes:
* windows: use "mkdir" even when copying directory

Original commit messages:

a478c1ab51ea3e04e79791ac3d1dad01b3f57434:
  win: mkdir even when copying directory

  * also "fix" the paths in the message
  * un-skip test/copies/gyptest-all.py

  BUG=gyp:536

  Change-Id: Id8ff7941b995c25d68d454138cd8b04940fdd82b
  Reviewed-on: https://chromium-review.googlesource.com/487521
  Commit-Queue: Dirk Pranke <dpranke@chromium.org>
  Reviewed-by: Dirk Pranke <dpranke@chromium.org>

ffd524cefaad622e72995e852ffb0b18e83f8054
  win ninja/make: Always use a native compiler executable with MSVS 2017

  A host-native executable will always be used, and it will be a cross
  compiler if the target architecture differs from the host architecture.

  BUG=683729

  Change-Id: I02a09e1755dd2ab7eca5c9d1957d7aeb56db6af6
  Reviewed-on: https://chromium-review.googlesource.com/486400
  Commit-Queue: Mark Mentovai <mark@chromium.org>
  Reviewed-by: Dirk Pranke <dpranke@chromium.org>

e8850240a433259052705fb8c56e51795b7dc9c3
  Fix MSVC++ 32-on-32 builds after b62d04ff85e6

  BUG=683729

  Change-Id: Ic8c227960b859ddc3c19fce0e98144510f5e74bf
  Reviewed-on: https://chromium-review.googlesource.com/486380
  Reviewed-by: Dirk Pranke <dpranke@chromium.org>
  Commit-Queue: Mark Mentovai <mark@chromium.org>

b62d04ff85e6234e4fec7fff9377dd96c09d41a7
  win,ninja: ninja generator better on windows

  * add compatibility with VS2017
  * adjust `_TargetConfig` and `/FS` for VS2017 compat
  * find new place of `vcvarsall.bat` in VS2017
  * normalize "path like" arguments of actions
  * better check for `.lib` and `.def` file names

  BUG=683729

  Change-Id: I123bff7bd8a0011cf65d27a62b5267ba884e3b42
  Reviewed-on: https://chromium-review.googlesource.com/482580
  Reviewed-by: Dirk Pranke <dpranke@chromium.org>
  Reviewed-by: Mark Mentovai <mark@chromium.org>
  Commit-Queue: Dirk Pranke <dpranke@chromium.org>

Ref: nodejs#12281
PR-URL: nodejs#12632

tools/gyp/pylib/gyp/MSVSVersion.py

@@ -13,6 +13,10 @@
 import glob
 
 
+def JoinPath(*args):
+  return os.path.normpath(os.path.join(*args))
+
+
 class VisualStudioVersion(object):
   """Information regarding a version of Visual Studio."""
 
@@ -71,45 +75,59 @@ def DefaultToolset(self):
     of a user override."""
     return self.default_toolset
 
+
   def _SetupScriptInternal(self, target_arch):
     """Returns a command (with arguments) to be used to set up the
     environment."""
+    assert target_arch in ('x86', 'x64'), "target_arch not supported"
     # If WindowsSDKDir is set and SetEnv.Cmd exists then we are using the
     # depot_tools build tools and should run SetEnv.Cmd to set up the
     # environment. The check for WindowsSDKDir alone is not sufficient because
     # this is set by running vcvarsall.bat.
-    assert target_arch in ('x86', 'x64')
-    sdk_dir = os.environ.get('WindowsSDKDir')
-    if sdk_dir:
-      setup_path = os.path.normpath(os.path.join(sdk_dir, 'Bin/SetEnv.Cmd'))
+    sdk_dir = os.environ.get('WindowsSDKDir', '')
+    setup_path = JoinPath(sdk_dir, 'Bin', 'SetEnv.Cmd')
     if self.sdk_based and sdk_dir and os.path.exists(setup_path):
       return [setup_path, '/' + target_arch]
-    else:
-      # We don't use VC/vcvarsall.bat for x86 because vcvarsall calls
-      # vcvars32, which it can only find if VS??COMNTOOLS is set, which it
-      # isn't always.
-      if target_arch == 'x86':
-        if self.short_name >= '2013' and self.short_name[-1] != 'e' and (
-            os.environ.get('PROCESSOR_ARCHITECTURE') == 'AMD64' or
-            os.environ.get('PROCESSOR_ARCHITEW6432') == 'AMD64'):
-          # VS2013 and later, non-Express have a x64-x86 cross that we want
-          # to prefer.
-          return [os.path.normpath(
-             os.path.join(self.path, 'VC/vcvarsall.bat')), 'amd64_x86']
-        # Otherwise, the standard x86 compiler.
-        return [os.path.normpath(
-          os.path.join(self.path, 'Common7/Tools/vsvars32.bat'))]
+
+    is_host_arch_x64 = (
+      os.environ.get('PROCESSOR_ARCHITECTURE') == 'AMD64' or
+      os.environ.get('PROCESSOR_ARCHITEW6432') == 'AMD64'
+    )
+
+    # For VS2017 (and newer) it's fairly easy
+    if self.short_name >= '2017':
+      script_path = JoinPath(self.path,
+                             'VC', 'Auxiliary', 'Build', 'vcvarsall.bat')
+
+      # Always use a native executable, cross-compiling if necessary.
+      host_arch = 'amd64' if is_host_arch_x64 else 'x86'
+      msvc_target_arch = 'amd64' if target_arch == 'x64' else 'x86'
+      arg = host_arch
+      if host_arch != msvc_target_arch:
+        arg += '_' + msvc_target_arch
+
+      return [script_path, arg]
+
+    # We try to find the best version of the env setup batch.
+    vcvarsall = JoinPath(self.path, 'VC', 'vcvarsall.bat')
+    if target_arch == 'x86':
+      if self.short_name >= '2013' and self.short_name[-1] != 'e' and \
+         is_host_arch_x64:
+        # VS2013 and later, non-Express have a x64-x86 cross that we want
+        # to prefer.
+        return [vcvarsall, 'amd64_x86']
       else:
-        assert target_arch == 'x64'
-        arg = 'x86_amd64'
-        # Use the 64-on-64 compiler if we're not using an express
-        # edition and we're running on a 64bit OS.
-        if self.short_name[-1] != 'e' and (
-            os.environ.get('PROCESSOR_ARCHITECTURE') == 'AMD64' or
-            os.environ.get('PROCESSOR_ARCHITEW6432') == 'AMD64'):
-          arg = 'amd64'
-        return [os.path.normpath(
-            os.path.join(self.path, 'VC/vcvarsall.bat')), arg]
+        # Otherwise, the standard x86 compiler. We don't use VC/vcvarsall.bat
+        # for x86 because vcvarsall calls vcvars32, which it can only find if
+        # VS??COMNTOOLS is set, which isn't guaranteed.
+        return [JoinPath(self.path, 'Common7', 'Tools', 'vsvars32.bat')]
+    elif target_arch == 'x64':
+      arg = 'x86_amd64'
+      # Use the 64-on-64 compiler if we're not using an express edition and
+      # we're running on a 64bit OS.
+      if self.short_name[-1] != 'e' and is_host_arch_x64:
+        arg = 'amd64'
+      return [vcvarsall, arg]
 
   def SetupScript(self, target_arch):
     script_data = self._SetupScriptInternal(target_arch)

tools/gyp/pylib/gyp/generator/msvs.py

@@ -1717,14 +1717,17 @@ def _GetCopies(spec):
         src_bare = src[:-1]
         base_dir = posixpath.split(src_bare)[0]
         outer_dir = posixpath.split(src_bare)[1]
-        cmd = 'cd "%s" && xcopy /e /f /y "%s" "%s\\%s\\"' % (
-            _FixPath(base_dir), outer_dir, _FixPath(dst), outer_dir)
+        fixed_dst = _FixPath(dst)
+        full_dst = '"%s\\%s\\"' % (fixed_dst, outer_dir)
+        cmd = 'mkdir %s 2>nul & cd "%s" && xcopy /e /f /y "%s" %s' % (
+            full_dst, _FixPath(base_dir), outer_dir, full_dst)
         copies.append(([src], ['dummy_copies', dst], cmd,
-                       'Copying %s to %s' % (src, dst)))
+                       'Copying %s to %s' % (src, fixed_dst)))
       else:
+        fix_dst = _FixPath(cpy['destination'])
         cmd = 'mkdir "%s" 2>nul & set ERRORLEVEL=0 & copy /Y "%s" "%s"' % (
-            _FixPath(cpy['destination']), _FixPath(src), _FixPath(dst))
-        copies.append(([src], [dst], cmd, 'Copying %s to %s' % (src, dst)))
+            fix_dst, _FixPath(src), _FixPath(dst))
+        copies.append(([src], [dst], cmd, 'Copying %s to %s' % (src, fix_dst)))
   return copies
 
 

tools/gyp/pylib/gyp/msvs_emulation.py

@@ -30,6 +30,10 @@ def QuoteForRspFile(arg):
   # works more or less because most programs (including the compiler, etc.)
   # use that function to handle command line arguments.
 
+  # Use a heuristic to try to find args that are paths, and normalize them
+  if arg.find('/') > 0 or arg.count('/') > 1:
+    arg = os.path.normpath(arg)
+
   # For a literal quote, CommandLineToArgvW requires 2n+1 backslashes
   # preceding it, and results in n backslashes + the quote. So we substitute
   # in 2* what we match, +1 more, plus the quote.
@@ -269,7 +273,8 @@ def ConvertVSMacros(self, s, base_to_build=None, config=None):
   def AdjustLibraries(self, libraries):
     """Strip -l from library if it's specified with that."""
     libs = [lib[2:] if lib.startswith('-l') else lib for lib in libraries]
-    return [lib + '.lib' if not lib.endswith('.lib') else lib for lib in libs]
+    return [lib + '.lib' if not lib.lower().endswith('.lib') else lib
+            for lib in libs]
 
   def _GetAndMunge(self, field, path, default, prefix, append, map):
     """Retrieve a value from |field| at |path| or return |default|. If
@@ -306,7 +311,10 @@ def _TargetConfig(self, config):
     # There's two levels of architecture/platform specification in VS. The
     # first level is globally for the configuration (this is what we consider
     # "the" config at the gyp level, which will be something like 'Debug' or
-    # 'Release_x64'), and a second target-specific configuration, which is an
+    # 'Release'), VS2015 and later only use this level
+    if self.vs_version.short_name >= 2015:
+      return config
+    # and a second target-specific configuration, which is an
     # override for the global one. |config| is remapped here to take into
     # account the local target-specific overrides to the global configuration.
     arch = self.GetArch(config)
@@ -468,8 +476,10 @@ def GetCflags(self, config):
         prefix='/arch:')
     cflags.extend(['/FI' + f for f in self._Setting(
         ('VCCLCompilerTool', 'ForcedIncludeFiles'), config, default=[])])
-    if self.vs_version.short_name in ('2013', '2013e', '2015'):
-      # New flag required in 2013 to maintain previous PDB behavior.
+    if self.vs_version.project_version >= 12.0:
+      # New flag introduced in VS2013 (project version 12.0) Forces writes to
+      # the program database (PDB) to be serialized through MSPDBSRV.EXE.
+      # https://msdn.microsoft.com/en-us/library/dn502518.aspx
       cflags.append('/FS')
     # ninja handles parallelism by itself, don't have the compiler do it too.
     cflags = filter(lambda x: not x.startswith('/MP'), cflags)
@@ -529,7 +539,8 @@ def GetDefFile(self, gyp_to_build_path):
     """Returns the .def file from sources, if any.  Otherwise returns None."""
     spec = self.spec
     if spec['type'] in ('shared_library', 'loadable_module', 'executable'):
-      def_files = [s for s in spec.get('sources', []) if s.endswith('.def')]
+      def_files = [s for s in spec.get('sources', [])
+                   if s.lower().endswith('.def')]
       if len(def_files) == 1:
         return gyp_to_build_path(def_files[0])
       elif len(def_files) > 1: