From f24e122d13db7061b1086ddfd21d3a1c5294213b Mon Sep 17 00:00:00 2001 From: Tim Ruffing Date: Fri, 26 Feb 2021 15:52:40 +0100 Subject: [PATCH 1/5] ci: Switch all Linux builds to Debian The experiment of using Nix Shell was not really successful. Most notably, Nix uses a bunch of wrapper scripts around compilers, which make the build much less "pure". This may be useful but it's exactly not what we want for CI. In particular, this resulted in gcc being used for the "clang" builds because a wrapper script redefined the CC env variable. This now builds a single docker image (Debian) for all architectures that we test in CI on Linux. --- .cirrus.yml | 20 ++++++++++---------- ci/cirrus.sh | 4 ---- ci/linux-debian-s390-qemu.Dockerfile | 6 ------ ci/linux-debian.Dockerfile | 12 ++++++++++++ ci/linux-nixos.Dockerfile | 12 ------------ ci/mkshell.nix | 12 ------------ ci/shell-i686.nix | 4 ---- ci/shell.nix | 4 ---- 8 files changed, 22 insertions(+), 52 deletions(-) delete mode 100644 ci/linux-debian-s390-qemu.Dockerfile create mode 100644 ci/linux-debian.Dockerfile delete mode 100644 ci/linux-nixos.Dockerfile delete mode 100644 ci/mkshell.nix delete mode 100644 ci/shell-i686.nix delete mode 100644 ci/shell.nix diff --git a/.cirrus.yml b/.cirrus.yml index 3b7e5f2823f6d..ac33507733897 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -33,9 +33,9 @@ cat_logs_snippet: &CAT_LOGS - env task: - name: "x86_64: Linux (Alpine Linux, Nix Shell)" + name: "x86_64: Linux (Debian stable)" container: - dockerfile: ci/linux-nixos.Dockerfile + dockerfile: ci/linux-debian.Dockerfile # Reduce number of CPUs to be able to do more builds in parallel. cpu: 1 # More than enough for our scripts. @@ -82,13 +82,13 @@ task: - env: CC: clang test_script: - - nix-shell ci/shell.nix --run ./ci/cirrus.sh + - ./ci/cirrus.sh << : *CAT_LOGS task: - name: "i686: Linux (Alpine Linux, Nix Shell)" + name: "i686: Linux (Debian stable)" container: - dockerfile: ci/linux-nixos.Dockerfile + dockerfile: ci/linux-debian.Dockerfile cpu: 1 memory: 1G env: @@ -99,16 +99,16 @@ task: SCHNORRSIG: yes matrix: - env: - CC: gcc + CC: i686-linux-gnu-gcc - env: - CC: clang + CC: clang --target=i686-pc-linux-gnu -isystem /usr/i686-linux-gnu/include matrix: - env: BIGNUM: gmp - env: BIGNUM: no test_script: - - nix-shell ci/shell-i686.nix --run ./ci/cirrus.sh + - ./ci/cirrus.sh << : *CAT_LOGS task: @@ -168,9 +168,9 @@ task: << : *CAT_LOGS task: - name: "s390x (big-endian): Linux (Debian QEMU)" + name: "s390x (big-endian): Linux (Debian stable, QEMU)" container: - dockerfile: ci/linux-debian-s390-qemu.Dockerfile + dockerfile: ci/linux-debian.Dockerfile cpu: 1 memory: 1G env: diff --git a/ci/cirrus.sh b/ci/cirrus.sh index e10037a468474..afd72b493c999 100755 --- a/ci/cirrus.sh +++ b/ci/cirrus.sh @@ -12,10 +12,6 @@ valgrind --version || true ./autogen.sh -# Nix doesn't store GNU file in /usr/bin, see https://lists.gnu.org/archive/html/bug-libtool/2015-09/msg00000.html . -# The -i'' is necessary for macOS portability, see https://stackoverflow.com/a/4247319 . -sed -i'' -e 's@/usr/bin/file@$(which file)@g' configure - ./configure \ --enable-experimental="$EXPERIMENTAL" \ --with-test-override-wide-multiply="$WIDEMUL" --with-bignum="$BIGNUM" --with-asm="$ASM" \ diff --git a/ci/linux-debian-s390-qemu.Dockerfile b/ci/linux-debian-s390-qemu.Dockerfile deleted file mode 100644 index d527be55cb23f..0000000000000 --- a/ci/linux-debian-s390-qemu.Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -FROM debian - -RUN dpkg --add-architecture s390x -RUN apt-get update -RUN apt-get install --no-install-recommends --no-upgrade -y make automake libtool -RUN apt-get install --no-install-recommends --no-upgrade -y gcc-s390x-linux-gnu libc6-dev-s390x-cross qemu-user libc6:s390x diff --git a/ci/linux-debian.Dockerfile b/ci/linux-debian.Dockerfile new file mode 100644 index 0000000000000..8fe50f17e0651 --- /dev/null +++ b/ci/linux-debian.Dockerfile @@ -0,0 +1,12 @@ +FROM debian:stable + +RUN dpkg --add-architecture i386 +RUN dpkg --add-architecture s390x +RUN apt-get update + +# dkpg-dev: to make pkg-config work in cross-builds +RUN apt-get install --no-install-recommends --no-upgrade -y \ + make automake libtool pkg-config dpkg-dev valgrind qemu-user \ + gcc clang libc6-dbg libgmp-dev \ + gcc-i686-linux-gnu libc6-dev-i386-cross libc6-dbg:i386 libgmp-dev:i386 \ + gcc-s390x-linux-gnu libc6-dev-s390x-cross libc6-dbg:s390x diff --git a/ci/linux-nixos.Dockerfile b/ci/linux-nixos.Dockerfile deleted file mode 100644 index 0017073c9dc3d..0000000000000 --- a/ci/linux-nixos.Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM nixos/nix - -COPY ci/shell.nix /tmp -COPY ci/shell-i686.nix /tmp - -RUN nix-channel --remove nixpkgs -RUN nix-channel --add https://nixos.org/channels/nixos-20.09 nixpkgs -RUN nix-channel --update - -# Run dummy command "true" in the nix-shell just to get the packages prepared. -RUN nix-shell /tmp/shell.nix --command true -RUN nix-shell /tmp/shell-i686.nix --command true diff --git a/ci/mkshell.nix b/ci/mkshell.nix deleted file mode 100644 index 3886b556d0997..0000000000000 --- a/ci/mkshell.nix +++ /dev/null @@ -1,12 +0,0 @@ -{ pkgs }: - -with pkgs; - -mkShell { - buildInputs = [ - bash file pkgconfig autoconf automake libtool gmp valgrind clang gcc - ]; - shellHook = '' - echo Running nix-shell with nixpkgs version: $(nix eval --raw nixpkgs.lib.version) - ''; -} diff --git a/ci/shell-i686.nix b/ci/shell-i686.nix deleted file mode 100644 index 12528dd9f005f..0000000000000 --- a/ci/shell-i686.nix +++ /dev/null @@ -1,4 +0,0 @@ -let - pkgs = (import {}).pkgsi686Linux; -in -import ./mkshell.nix { inherit pkgs; } diff --git a/ci/shell.nix b/ci/shell.nix deleted file mode 100644 index e83dc870f0442..0000000000000 --- a/ci/shell.nix +++ /dev/null @@ -1,4 +0,0 @@ -let - pkgs = (import {}); -in -import ./mkshell.nix { inherit pkgs; } From b994a8be3cf8ab0fc6a622980a9845bb82cc17db Mon Sep 17 00:00:00 2001 From: Tim Ruffing Date: Fri, 26 Feb 2021 17:36:34 +0100 Subject: [PATCH 2/5] ci: Print information about binaries using "file" --- ci/cirrus.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ci/cirrus.sh b/ci/cirrus.sh index afd72b493c999..2175a04e0dbf1 100755 --- a/ci/cirrus.sh +++ b/ci/cirrus.sh @@ -21,6 +21,13 @@ valgrind --version || true --with-valgrind="$WITH_VALGRIND" \ --host="$HOST" $EXTRAFLAGS +make -j2 + +# Print information about binaries so that we can see that the architecture is correct +file *tests || true +file bench_* || true +file .libs/* || true + if [ -n "$BUILD" ] then make -j2 "$BUILD" @@ -28,7 +35,6 @@ fi if [ "$RUN_VALGRIND" = "yes" ] then - make -j2 # the `--error-exitcode` is required to make the test fail if valgrind found errors, otherwise it'll return 0 (https://www.valgrind.org/docs/manual/manual-core.html) valgrind --error-exitcode=42 ./tests 16 valgrind --error-exitcode=42 ./exhaustive_tests @@ -36,7 +42,6 @@ fi if [ -n "$QEMU_CMD" ] then - make -j2 $QEMU_CMD ./tests 16 $QEMU_CMD ./exhaustive_tests fi From c7f754fe4d5e032fd150c4b9b985855e9fcaa521 Mon Sep 17 00:00:00 2001 From: Tim Ruffing Date: Mon, 1 Mar 2021 23:50:54 +0100 Subject: [PATCH 3/5] ci: Run PRs on merge result instead of on the source branch This is taken from Bitcoin Core's .cirrus.yml --- .cirrus.yml | 14 ++++++++++++-- ci/linux-debian.Dockerfile | 1 + 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index ac33507733897..ce904d3e43e36 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -16,8 +16,6 @@ env: CTIMETEST: yes BENCH: yes ITERS: 2 - # We only need the top commit - CIRRUS_CLONE_DEPTH: 1 cat_logs_snippet: &CAT_LOGS always: @@ -32,6 +30,14 @@ cat_logs_snippet: &CAT_LOGS - cat test_env.log || true - env +merge_base_script_snippet: &MERGE_BASE + merge_base_script: + - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi + - git fetch $CIRRUS_REPO_CLONE_URL $CIRRUS_BASE_BRANCH + - git config --global user.email "ci@ci.ci" + - git config --global user.name "ci" + - git merge FETCH_HEAD # Merge base to detect silent merge conflicts + task: name: "x86_64: Linux (Debian stable)" container: @@ -81,6 +87,7 @@ task: CC: gcc - env: CC: clang + << : *MERGE_BASE test_script: - ./ci/cirrus.sh << : *CAT_LOGS @@ -107,6 +114,7 @@ task: BIGNUM: gmp - env: BIGNUM: no + << : *MERGE_BASE test_script: - ./ci/cirrus.sh << : *CAT_LOGS @@ -163,6 +171,7 @@ task: - brew link valgrind brew_script: - brew install automake libtool gmp gcc@9 + << : *MERGE_BASE test_script: - ./ci/cirrus.sh << : *CAT_LOGS @@ -184,6 +193,7 @@ task: EXPERIMENTAL: yes SCHNORRSIG: yes CTIMETEST: no + << : *MERGE_BASE test_script: # https://sourceware.org/bugzilla/show_bug.cgi?id=27008 - rm /etc/ld.so.cache diff --git a/ci/linux-debian.Dockerfile b/ci/linux-debian.Dockerfile index 8fe50f17e0651..201ace4f69a57 100644 --- a/ci/linux-debian.Dockerfile +++ b/ci/linux-debian.Dockerfile @@ -6,6 +6,7 @@ RUN apt-get update # dkpg-dev: to make pkg-config work in cross-builds RUN apt-get install --no-install-recommends --no-upgrade -y \ + git ca-certificates \ make automake libtool pkg-config dpkg-dev valgrind qemu-user \ gcc clang libc6-dbg libgmp-dev \ gcc-i686-linux-gnu libc6-dev-i386-cross libc6-dbg:i386 libgmp-dev:i386 \ From 28eccdf80641f71fada0ee4065c8127468162176 Mon Sep 17 00:00:00 2001 From: Tim Ruffing Date: Tue, 2 Mar 2021 00:09:05 +0100 Subject: [PATCH 4/5] ci: Split output of logs into multiple sections --- .cirrus.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index ce904d3e43e36..646518b7840ad 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -19,15 +19,20 @@ env: cat_logs_snippet: &CAT_LOGS always: - test_logs_script: + cat_tests_log_script: - cat tests.log || true + cat_exhaustive_tests_log_script: - cat exhaustive_tests.log || true + cat_valgrind_ctime_test_log_script: - cat valgrind_ctime_test.log || true + cat_bench_log_script: - cat bench.log || true on_failure: - debug_output_script: + cat_config_log_script: - cat config.log || true + cat_test_env_script: - cat test_env.log || true + cat_ci_env_script: - env merge_base_script_snippet: &MERGE_BASE From 9361f360bb04156c7a0fa8f2664680b74d463ed5 Mon Sep 17 00:00:00 2001 From: Tim Ruffing Date: Wed, 3 Mar 2021 01:21:18 +0100 Subject: [PATCH 5/5] ci: Select number of parallel make jobs depending on CI environment This should improve compilation times on macOS. Things can certainly be improved further, e.g., by running the benchmarks in parallel. --- .cirrus.yml | 3 +++ ci/cirrus.sh | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 646518b7840ad..9399fbda47e2f 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -16,6 +16,7 @@ env: CTIMETEST: yes BENCH: yes ITERS: 2 + MAKEFLAGS: -j2 cat_logs_snippet: &CAT_LOGS always: @@ -131,6 +132,8 @@ task: env: HOMEBREW_NO_AUTO_UPDATE: 1 HOMEBREW_NO_INSTALL_CLEANUP: 1 + # Cirrus gives us a fixed number of 12 virtual CPUs. Not that we even have that many jobs at the moment... + MAKEFLAGS: -j13 matrix: << : *ENV_MATRIX matrix: diff --git a/ci/cirrus.sh b/ci/cirrus.sh index 2175a04e0dbf1..f223a91ca0a5b 100755 --- a/ci/cirrus.sh +++ b/ci/cirrus.sh @@ -21,7 +21,8 @@ valgrind --version || true --with-valgrind="$WITH_VALGRIND" \ --host="$HOST" $EXTRAFLAGS -make -j2 +# We have set "-j" in MAKEFLAGS. +make # Print information about binaries so that we can see that the architecture is correct file *tests || true @@ -30,7 +31,7 @@ file .libs/* || true if [ -n "$BUILD" ] then - make -j2 "$BUILD" + make "$BUILD" fi if [ "$RUN_VALGRIND" = "yes" ]