backport: merge bitcoin#25643, #23583, #23618, #23603, #23817, #24520, #24508, #25639, #26018, #25437, #25484, #23585, #24549, #24733, #21991, #22526, #25633, #24597, #24955, #25099, #24552, #21851, #25389, #25357, partial #22318 (guix backports: part 3)

.gitlab-ci.yml

@@ -42,13 +42,13 @@ builder-image:
   image: $CI_REGISTRY_IMAGE:builder-$CI_COMMIT_REF_SLUG
   variables:
     SDK_URL: https://bitcoincore.org/depends-sources/sdks
-    XCODE_VERSION: "12.1"
-    XCODE_BUILD_ID: 12A7403
+    XCODE_VERSION: "12.2"
+    XCODE_BUILD_ID: 12B45b
     MAKEJOBS: -j4
   before_script:
     - echo HOST=$HOST
     - |
-      if [ "$HOST" = "x86_64-apple-darwin19" ]; then
+      if [ "$HOST" = "x86_64-apple-darwin" ]; then
         mkdir -p depends/SDKs
         mkdir -p depends/sdk-sources
         OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers.tar.gz"
@@ -193,12 +193,12 @@ x86_64-pc-linux-gnu-nowallet:
     HOST: x86_64-pc-linux-gnu
     DEP_OPTS: "NO_WALLET=1"
 
-x86_64-apple-darwin19:
+x86_64-apple-darwin:
   extends:
     - .build-depends-template
     - .skip-in-fast-mode-template
   variables:
-    HOST: x86_64-apple-darwin19
+    HOST: x86_64-apple-darwin
 
 ###
 
@@ -302,7 +302,7 @@ mac-build:
     - .build-template
     - .skip-in-fast-mode-template
   needs:
-    - x86_64-apple-darwin19
+    - x86_64-apple-darwin
   variables:
     BUILD_TARGET: mac
 

ci/test/00_setup_env_mac.sh

@@ -7,10 +7,10 @@
 export LC_ALL=C.UTF-8
 
 export CONTAINER_NAME=ci_macos_cross
-export HOST=x86_64-apple-darwin19
+export HOST=x86_64-apple-darwin
 export PACKAGES="cmake libcap-dev libz-dev libbz2-dev python3-dev python3-setuptools"
-export XCODE_VERSION=12.1
-export XCODE_BUILD_ID=12A7403
+export XCODE_VERSION=12.2
+export XCODE_BUILD_ID=12B45b
 export RUN_UNIT_TESTS=false
 export RUN_INTEGRATION_TESTS=false
 export GOAL="all deploy"

ci/test/00_setup_env_mac_host.sh

@@ -7,7 +7,7 @@
 export LC_ALL=C.UTF-8
 
 export CONTAINER_NAME=ci_macos
-export HOST=x86_64-apple-darwin19
+export HOST=x86_64-apple-darwin
 export PIP_PACKAGES="zmq lief"
 export RUN_UNIT_TESTS=true
 export RUN_INTEGRATION_TESTS=false

configure.ac

@@ -20,6 +20,7 @@ BITCOIN_TX_NAME=dash-tx
 BITCOIN_WALLET_TOOL_NAME=dash-wallet
 
 dnl Unless the user specified ARFLAGS, force it to be cr
+dnl This is also the default as-of libtool 2.4.7
 AC_ARG_VAR(ARFLAGS, [Flags for the archiver, defaults to <cr> if not set])
 if test "x${ARFLAGS+set}" != "xset"; then
   ARFLAGS="cr"

contrib/containers/ci/Dockerfile

@@ -37,7 +37,7 @@ RUN pip3 install \
     codespell==1.17.1 \
     flake8==3.8.3 \
     jinja2 \
-    lief==0.12.0 \
+    lief==0.12.1 \
     pyzmq \
     vulture==2.3 \
     yq \

contrib/containers/guix/scripts/guix-start

@@ -13,8 +13,8 @@ if [[ ! -d "$WORKSPACE_PATH" ]]; then
     exit 1
 fi
 
-XCODE_VERSION="12.1"
-XCODE_RELEASE="12A7403"
+XCODE_VERSION="12.2"
+XCODE_RELEASE="12B45b"
 XCODE_ARCHIVE="Xcode-${XCODE_VERSION}-${XCODE_RELEASE}-extracted-SDK-with-libcxx-headers"
 
 # Check if macOS SDK is present, if not, download it

contrib/devtools/security-check.py

@@ -206,12 +206,9 @@ def check_MACHO_control_flow(binary) -> bool:
 ]
 
 BASE_MACHO = [
-    ('PIE', check_PIE),
     ('NOUNDEFS', check_MACHO_NOUNDEFS),
-    ('NX', check_NX),
     ('LAZY_BINDINGS', check_MACHO_LAZY_BINDINGS),
     ('Canary', check_MACHO_Canary),
-    ('CONTROL_FLOW', check_MACHO_control_flow),
 ]
 
 CHECKS = {
@@ -226,7 +223,10 @@ def check_MACHO_control_flow(binary) -> bool:
         lief.ARCHITECTURES.X86: BASE_PE,
     },
     lief.EXE_FORMATS.MACHO: {
-        lief.ARCHITECTURES.X86: BASE_MACHO,
+        lief.ARCHITECTURES.X86: BASE_MACHO + [('PIE', check_PIE),
+                                              ('NX', check_NX),
+                                              ('CONTROL_FLOW', check_MACHO_control_flow)],
+        lief.ARCHITECTURES.ARM64: BASE_MACHO,
     }
 }
 

contrib/devtools/symbol-check.py

@@ -217,7 +217,7 @@ def check_MACHO_min_os(binary) -> bool:
     return False
 
 def check_MACHO_sdk(binary) -> bool:
-    if binary.build_version.sdk == [10, 15, 6]:
+    if binary.build_version.sdk == [11, 0, 0]:
         return True
     return False
 

contrib/devtools/test-security-check.py

@@ -116,21 +116,34 @@ def test_MACHO(self):
         executable = 'test1'
         cc = determine_wellknown_cmd('CC', 'clang')
         write_testcode(source)
+        arch = get_arch(cc, source, executable)
+
+        if arch == lief.ARCHITECTURES.X86:
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fno-stack-protector']),
+                (1, executable+': failed NOUNDEFS LAZY_BINDINGS Canary PIE NX CONTROL_FLOW'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fstack-protector-all']),
+                (1, executable+': failed NOUNDEFS LAZY_BINDINGS PIE NX CONTROL_FLOW'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-fstack-protector-all']),
+                (1, executable+': failed NOUNDEFS LAZY_BINDINGS PIE CONTROL_FLOW'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all']),
+                (1, executable+': failed LAZY_BINDINGS PIE CONTROL_FLOW'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all']),
+                (1, executable+': failed PIE CONTROL_FLOW'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']),
+                (1, executable+': failed PIE'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']),
+                (0, ''))
+        else:
+            # arm64 darwin doesn't support non-PIE binaries, control flow or executable stacks
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fno-stack-protector']),
+                (1, executable+': failed NOUNDEFS LAZY_BINDINGS Canary'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fstack-protector-all']),
+                (1, executable+': failed NOUNDEFS LAZY_BINDINGS'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-fstack-protector-all']),
+                (1, executable+': failed LAZY_BINDINGS'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-bind_at_load','-fstack-protector-all']),
+                (0, ''))
 
-        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fno-stack-protector']),
-            (1, executable+': failed PIE NOUNDEFS NX LAZY_BINDINGS Canary CONTROL_FLOW'))
-        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fstack-protector-all']),
-            (1, executable+': failed PIE NOUNDEFS NX LAZY_BINDINGS CONTROL_FLOW'))
-        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-fstack-protector-all']),
-            (1, executable+': failed PIE NOUNDEFS LAZY_BINDINGS CONTROL_FLOW'))
-        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all']),
-            (1, executable+': failed PIE LAZY_BINDINGS CONTROL_FLOW'))
-        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all']),
-            (1, executable+': failed PIE CONTROL_FLOW'))
-        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']),
-            (1, executable+': failed PIE'))
-        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']),
-            (0, ''))
 
         clean_files(source, executable)
 

contrib/gitian-build.py

@@ -74,8 +74,8 @@ def build():
 
     if args.macos:
         print('\nCompiling ' + args.version + ' MacOS')
-        subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/depends-sources/sdks/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz'])
-        subprocess.check_output(["echo 'be17f48fd0b08fb4dcd229f55a6ae48d9f781d210839b4ea313ef17dd12d6ea5 inputs/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz' | sha256sum -c"], shell=True)
+        subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/depends-sources/sdks/Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz'])
+        subprocess.check_output(["echo 'df75d30ecafc429e905134333aeae56ac65fac67cb4182622398fd717df77619 inputs/Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz' | sha256sum -c"], shell=True)
         subprocess.check_call(['bin/gbuild', '--fetch-tags',  '-j', args.jobs, '-m', args.memory, '--commit', 'dash='+args.commit, '--url', 'dash='+args.url, '../dash/contrib/gitian-descriptors/gitian-osx.yml'])
         subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx-unsigned', '--destination', '../gitian.sigs/', '../dash/contrib/gitian-descriptors/gitian-osx.yml'])
         subprocess.check_call('mv build/out/dashcore-*-osx-unsigned.tar.gz inputs/', shell=True)
@@ -218,7 +218,7 @@ def main():
     args.macos = 'm' in args.os
 
     # Disable for MacOS if no SDK found
-    if args.macos and not os.path.isfile('gitian-builder/inputs/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz'):
+    if args.macos and not os.path.isfile('gitian-builder/inputs/Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz'):
         print('Cannot build for MacOS, SDK does not exist. Will build for other OSes')
         args.macos = False
 

contrib/gitian-descriptors/gitian-linux.yml

@@ -110,7 +110,7 @@ script: |
   done
   }
 
-  pip3 install lief==0.12.0
+  pip3 install lief==0.12.1
 
   # Faketime for depends so intermediate results are comparable
   export PATH_orig=${PATH}

contrib/gitian-descriptors/gitian-osx.yml

@@ -33,12 +33,12 @@ remotes:
 - "url": "https://github.com/dashpay/dash.git"
   "dir": "dash"
 files:
-- "Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz"
+- "Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz"
 script: |
   set -e -o pipefail
 
   WRAP_DIR=$HOME/wrapped
-  HOSTS="x86_64-apple-darwin19"
+  HOSTS="x86_64-apple-darwin"
   CONFIGFLAGS="--enable-reduce-exports --disable-miner --disable-bench --disable-gui-tests --disable-fuzz-binary XORRISOFS=${WRAP_DIR}/xorrisofs DMG=${WRAP_DIR}/dmg --enable-crash-hooks"
   FAKETIME_HOST_PROGS=""
   FAKETIME_PROGS="ar ranlib date dmg xorrisofs"
@@ -96,7 +96,7 @@ script: |
   done
   }
 
-  pip3 install lief==0.12.0
+  pip3 install lief==0.12.1
 
   # Faketime for depends so intermediate results are comparable
   export PATH_orig=${PATH}
@@ -108,7 +108,7 @@ script: |
   BASEPREFIX="${PWD}/depends"
 
   mkdir -p ${BASEPREFIX}/SDKs
-  tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz
+  tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz
 
   # Build dependencies for each host
   for i in $HOSTS; do
@@ -172,8 +172,8 @@ script: |
     find ${DISTNAME} -path '*.dSYM*' | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}-debug.tar.gz
     cd ../../
   done
-  mv ${OUTDIR}/${DISTNAME}-x86_64-apple-darwin19.tar.gz ${OUTDIR}/${DISTNAME}-osx64.tar.gz
-  mv ${OUTDIR}/${DISTNAME}-x86_64-apple-darwin19-debug.tar.gz ${OUTDIR}/${DISTNAME}-osx64-debug.tar.gz
+  mv ${OUTDIR}/${DISTNAME}-x86_64-apple-darwin.tar.gz ${OUTDIR}/${DISTNAME}-osx64.tar.gz
+  mv ${OUTDIR}/${DISTNAME}-x86_64-apple-darwin-debug.tar.gz ${OUTDIR}/${DISTNAME}-osx64-debug.tar.gz
 
   # Compress ccache (otherwise the assert file will get too huge)
   if [ "$CCACHE_DIR" != "" ]; then

contrib/gitian-descriptors/gitian-win.yml

@@ -115,7 +115,7 @@ script: |
   done
   }
 
-  pip3 install lief==0.12.0
+  pip3 install lief==0.12.1
 
   # Faketime for depends so intermediate results are comparable
   export PATH_orig=${PATH}

contrib/guix/Dockerfile

@@ -59,5 +59,5 @@ RUN mkdir base_cache sources SDKs
 WORKDIR /dash
 
 RUN mkdir -p depends/SDKs && \
-    curl -L https://bitcoincore.org/depends-sources/sdks/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz | tar -xz -C depends/SDKs
+    curl -L https://bitcoincore.org/depends-sources/sdks/Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz | tar -xz -C depends/SDKs
 

contrib/guix/README.md

@@ -75,7 +75,7 @@ crucial differences:
 
 1. Since only Windows and macOS build outputs require codesigning, the `HOSTS`
    environment variable will have a sane default value of `x86_64-w64-mingw32
-   x86_64-apple-darwin18` instead of all the platforms.
+   x86_64-apple-darwin arm64-apple-darwin` instead of all the platforms.
 2. The `guix-codesign` command ***requires*** a `DETACHED_SIGS_REPO` flag.
     * _**DETACHED_SIGS_REPO**_
 
@@ -159,7 +159,7 @@ which case you can override the default list by setting the space-separated
 `HOSTS` environment variable:
 
 ```sh
-env HOSTS='x86_64-w64-mingw32 x86_64-apple-darwin18' ./contrib/guix/guix-build
+env HOSTS='x86_64-w64-mingw32 x86_64-apple-darwin' ./contrib/guix/guix-build
 ```
 
 See the [recognized environment variables][env-vars-list] section for more
@@ -224,7 +224,7 @@ details.
 
   _(defaults to "x86\_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu
   powerpc64-linux-gnu powerpc64le-linux-gnu
-  x86\_64-w64-mingw32 x86\_64-apple-darwin18")_
+  x86\_64-w64-mingw32 x86\_64-apple-darwin arm64-apple-darwin")_
 
 * _**SOURCES_PATH**_
 
@@ -249,7 +249,7 @@ details.
   Set the path where _extracted_ SDKs can be found. This is passed through to
   the depends tree. Note that this is should be set to the _parent_ directory of
   the actual SDK (e.g. `SDK_PATH=$HOME/Downloads/macOS-SDKs` instead of
-  `$HOME/Downloads/macOS-SDKs/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers`).
+  `$HOME/Downloads/macOS-SDKs/Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers`).
 
   The path that this environment variable points to **must be a directory**, and
   **NOT a symlink to a directory**.

contrib/guix/guix-attest

@@ -19,8 +19,16 @@ source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash"
 ################
 
 check_tools cat env basename mkdir diff sort
+
 if [ -z "$NO_SIGN" ]; then
-    check_tools gpg
+    # make it possible to override the gpg binary
+    GPG=${GPG:-gpg}
+
+    # $GPG can contain extra arguments passed to the binary
+    # so let's check only the existence of arg[0]
+    # shellcheck disable=SC2206
+    GPG_ARRAY=($GPG)
+    check_tools "${GPG_ARRAY[0]}"
 fi
 
 ################
@@ -90,7 +98,7 @@ if [ -z "${signer_name}" ]; then
     signer_name="$gpg_key_name"
 fi
 
-if [ -z "$NO_SIGN" ] && ! gpg --dry-run --list-secret-keys "${gpg_key_name}" >/dev/null 2>&1; then
+if [ -z "$NO_SIGN" ] && ! ${GPG} --dry-run --list-secret-keys "${gpg_key_name}" >/dev/null 2>&1; then
     echo "ERR: GPG can't seem to find any key named '${gpg_key_name}'"
     exit 1
 fi
@@ -240,11 +248,11 @@ mkdir -p "$outsigdir"
         echo "Signing SHA256SUMS to produce SHA256SUMS.asc"
         for i in *.SHA256SUMS; do
             if [ ! -e "$i".asc ]; then
-                gpg --detach-sign \
-                    --digest-algo sha256 \
-                    --local-user "$gpg_key_name" \
-                    --armor \
-                    --output "$i".asc "$i"
+                ${GPG} --detach-sign \
+                       --digest-algo sha256 \
+                       --local-user "$gpg_key_name" \
+                       --armor \
+                       --output "$i".asc "$i"
             else
                 echo "Signature already there"
             fi

contrib/guix/guix-build

@@ -76,7 +76,7 @@ mkdir -p "$VERSION_BASE"
 # Default to building for all supported HOSTs (overridable by environment)
 export HOSTS="${HOSTS:-x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu
                        x86_64-w64-mingw32
-                       x86_64-apple-darwin18}"
+                       x86_64-apple-darwin arm64-apple-darwin}"
 
 # Usage: distsrc_for_host HOST
 #
@@ -121,7 +121,7 @@ else
 fi
 
 ################
-# When building for darwin, the macOS SDK should exists
+# When building for darwin, the macOS SDK should exist
 ################
 
 for host in $HOSTS; do
@@ -130,8 +130,9 @@ for host in $HOSTS; do
             OSX_SDK="$(make -C "${PWD}/depends" --no-print-directory HOST="$host" print-OSX_SDK | sed 's@^[^=]\+=@@g')"
             if [ -e "$OSX_SDK" ]; then
                 echo "Found macOS SDK at '${OSX_SDK}', using..."
+                break
             else
-                echo "macOS SDK does not exist at '${OSX_SDK}', please place the extracted, untarred SDK there to perform darwin builds, exiting..."
+                echo "macOS SDK does not exist at '${OSX_SDK}', please place the extracted, untarred SDK there to perform darwin builds, or define SDK_PATH environment variable. Exiting..."
                 exit 1
             fi
             ;;

contrib/guix/guix-codesign

@@ -91,7 +91,7 @@ fi
 ################
 
 # Default to building for all supported HOSTs (overridable by environment)
-export HOSTS="${HOSTS:-x86_64-w64-mingw32 x86_64-apple-darwin18}"
+export HOSTS="${HOSTS:-x86_64-w64-mingw32 x86_64-apple-darwin arm64-apple-darwin}"
 
 # Usage: distsrc_for_host HOST
 #
@@ -152,10 +152,10 @@ outdir_for_host() {
 unsigned_tarball_for_host() {
     case "$1" in
         *mingw*)
-            echo "$(outdir_for_host "$1")/${DISTNAME}-win-unsigned.tar.gz"
+            echo "$(outdir_for_host "$1")/${DISTNAME}-win64-unsigned.tar.gz"
             ;;
         *darwin*)
-            echo "$(outdir_for_host "$1")/${DISTNAME}-osx-unsigned.tar.gz"
+            echo "$(outdir_for_host "$1")/${DISTNAME}-${1}-unsigned.tar.gz"
             ;;
         *)
             exit 1