merge bitcoin#24534: make gen-sdk deterministic

contrib/macdeploy/README.md

@@ -15,13 +15,16 @@ When complete, it will have produced `Dash-Qt.dmg`.
 A free Apple Developer Account is required to proceed.
 
 Our current macOS SDK
-(`Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz`) can be
-extracted from
+(`Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz`)
+can be extracted from
 [Xcode_12.2.xip](https://download.developer.apple.com/Developer_Tools/Xcode_12.2/Xcode_12.2.xip).
+
 Alternatively, after logging in to your account go to 'Downloads', then 'More'
 and search for [`Xcode_12.2`](https://developer.apple.com/download/all/?q=Xcode%2012.2).
+
 An Apple ID and cookies enabled for the hostname are needed to download this.
-The `sha256sum` of the archive should be `28d352f8c14a43d9b8a082ac6338dc173cb153f964c6e8fb6ba389e5be528bd0`.
+
+The `sha256sum` of the downloaded XIP archive should be `28d352f8c14a43d9b8a082ac6338dc173cb153f964c6e8fb6ba389e5be528bd0`.
 
 After Xcode version 7.x, Apple started shipping the `Xcode.app` in a `.xip`
 archive. This makes the SDK less-trivial to extract on non-macOS machines. One
@@ -55,7 +58,10 @@ previous stage) as the first argument.
 ./contrib/macdeploy/gen-sdk '/path/to/Xcode.app'
 ```
 
+The `sha256sum` of the generated TAR.GZ archive should be `e7ca56bc8804d16624fad68be2e71647747d6629cacaaa3de5fbfa7f444e9eae`.
+
 ## Deterministic macOS DMG Notes
+
 Working macOS DMGs are created in Linux by combining a recent `clang`, the Apple
 `binutils` (`ld`, `ar`, etc) and DMG authoring tools.
 

contrib/macdeploy/gen-sdk

@@ -8,6 +8,21 @@ import gzip
 import os
 import contextlib
 
+# monkey-patch Python 3.8 and older to fix wrong TAR header handling
+# see https://github.com/bitcoin/bitcoin/pull/24534
+# and https://github.com/python/cpython/pull/18080 for more info
+if sys.version_info < (3, 9):
+    _old_create_header = tarfile.TarInfo._create_header
+    def _create_header(info, format, encoding, errors):
+        buf = _old_create_header(info, format, encoding, errors)
+        # replace devmajor/devminor with binary zeroes
+        buf = buf[:329] + bytes(16) + buf[345:]
+        # recompute checksum
+        chksum = tarfile.calc_chksums(buf)[0]
+        buf = buf[:-364] + bytes("%06o\0" % chksum, "ascii") + buf[-357:]
+        return buf
+    tarfile.TarInfo._create_header = staticmethod(_create_header)
+
 @contextlib.contextmanager
 def cd(path):
     """Context manager that restores PWD even if an exception was raised."""
@@ -75,14 +90,21 @@ def run():
                 tarinfo.name = str(pathlib.Path(alt_base_dir, tarinfo.name))
             if tarinfo.linkname and tarinfo.linkname.startswith("./"):
                 tarinfo.linkname = str(pathlib.Path(alt_base_dir, tarinfo.linkname))
+            # make metadata deterministic
+            tarinfo.mtime = 0
+            tarinfo.uid, tarinfo.uname =  0, ''
+            tarinfo.gid, tarinfo.gname = 0, ''
+            # don't use isdir() as there are also executable files present
+            tarinfo.mode = 0o0755 if tarinfo.mode & 0o0100 else 0x0644
             return tarinfo
         with cd(dir_to_add):
+            # recursion already adds entries in sorted order
             tarfp.add(".", recursive=True, filter=change_tarinfo_base)
 
     print("Creating output .tar.gz file...")
     with out_sdktgz_path.open("wb") as fp:
         with gzip.GzipFile(fileobj=fp, mode='wb', compresslevel=9, mtime=0) as gzf:
-            with tarfile.open(mode="w", fileobj=gzf) as tarfp:
+            with tarfile.open(mode="w", fileobj=gzf, format=tarfile.GNU_FORMAT) as tarfp:
                 print("Adding MacOSX SDK {} files...".format(sdk_version))
                 tarfp_add_with_base_change(tarfp, sdk_dir, out_name)
                 print("Adding libc++ headers...")