Merge pull request #2033 from aaronfranke/arcom-command-length

Use a temp file for ARCOM to fix command line too long errors

Author: dsnopek

tools/android.py

@@ -1,6 +1,7 @@
 import os
 import sys
 
+import ar_tempfile
 import common_compiler_flags
 import my_spawn
 
@@ -112,6 +113,9 @@ def generate(env):
     env["RANLIB"] = toolchain + "/bin/llvm-ranlib"
     env["SHLIBSUFFIX"] = ".so"
 
+    # Configure ARCOM response file in case the command line call to AR is too long.
+    ar_tempfile.configure(env)
+
     env.Append(
         CCFLAGS=["--target=" + arch_info["target"] + env["android_api_level"], "-march=" + arch_info["march"], "-fPIC"]
     )

tools/ar_tempfile.py

@@ -0,0 +1,29 @@
+import os
+import re
+
+# Backslashes that aren't escaping a quote or another backslash.
+WINPATHSEP_RE = re.compile(r"\\([^\"'\\]|$)")
+
+
+def tempfile_arg_esc_func(arg):
+    from SCons.Subst import quote_spaces
+
+    arg = quote_spaces(arg)
+    # GCC requires double Windows slashes, let's use UNIX separator
+    return WINPATHSEP_RE.sub(r"/\1", arg)
+
+
+# The generated bindings can produce enough object files to exceed the command
+# line length limit when creating the static library. This works around that by
+# writing the command to a response file. Refer to the engine equivalent in
+# https://github.com/godotengine/godot/blob/master/platform/windows/detect.py
+def configure(env):
+    if "ARCOM_ORIG" in env:
+        return  # Already configured, don't wrap ARCOM in TEMPFILE twice.
+
+    env["ARCOM_ORIG"] = env["ARCOM"]
+    # This is SCons lazy evaluation syntax, only switching to a file when the command is actually too long.
+    env["ARCOM"] = "${TEMPFILE('$ARCOM_ORIG', '$ARCOMSTR')}"
+    env["TEMPFILESUFFIX"] = ".rsp"
+    if os.name == "nt":
+        env["TEMPFILEARGESCFUNC"] = tempfile_arg_esc_func

tools/linux.py

@@ -1,3 +1,4 @@
+import ar_tempfile
 import common_compiler_flags
 from SCons.Tool import clang, clangxx
 from SCons.Variables import BoolVariable
@@ -52,4 +53,7 @@ def generate(env):
     if env["lto"] == "auto":
         env["lto"] = "full"
 
+    # Configure ARCOM response file in case the command line call to AR is too long.
+    ar_tempfile.configure(env)
+
     common_compiler_flags.generate(env)

tools/my_spawn.py

@@ -33,20 +33,15 @@ def mySubProcess(cmdline, env):
         return rv
 
     def mySpawn(sh, escape, cmd, args, env):
+        # Used by TEMPFILE, which spawns a "del" command to clean up the response file.
+        # See the equivalent code in the engine in `methods.py`.
+        if cmd == "del":
+            os.remove(args[1])
+            return 0
+
         newargs = " ".join(args[1:])
         cmdline = cmd + " " + newargs
 
-        rv = 0
-        if len(cmdline) > 32000 and cmd.endswith("ar"):
-            cmdline = cmd + " " + args[1] + " " + args[2] + " "
-            for i in range(3, len(args)):
-                rv = mySubProcess(cmdline + args[i], env)
-                if rv:
-                    break
-        else:
-            rv = mySubProcess(cmdline, env)
-
-        return rv
+        return mySubProcess(cmdline, env)
 
     env["SPAWN"] = mySpawn
-    env.Replace(ARFLAGS=["q"])

tools/windows.py

@@ -1,6 +1,7 @@
 import os
 import sys
 
+import ar_tempfile
 import common_compiler_flags
 import my_spawn
 from SCons.Tool import mingw, msvc
@@ -150,9 +151,12 @@ def generate(env):
                 ]
             )
 
-        # Long line hack. Use custom spawn, quick AR append (to avoid files with the same names to override each other).
+        # Long line hack. Use custom spawn to work around the command line length limit.
         my_spawn.configure(env)
 
+        # Configure ARCOM response file in case the command line call to AR is too long.
+        ar_tempfile.configure(env)
+
     else:
         env["use_mingw"] = True
         # Cross-compilation using MinGW
@@ -201,6 +205,9 @@ def generate(env):
         if sys.platform == "win32" or sys.platform == "msys":
             my_spawn.configure(env)
 
+        # Configure ARCOM response file in case the command line call to AR is too long.
+        ar_tempfile.configure(env)
+
     env.Append(CPPDEFINES=["WINDOWS_ENABLED"])
 
     # Refer to https://github.com/godotengine/godot/blob/master/platform/windows/detect.py