From 98e971436b6763820aa4c37a6a86da6d0edd4f6e Mon Sep 17 00:00:00 2001
From: Mak <mordamax@gmail.com>
Date: Fri, 16 Aug 2024 13:53:19 +0100
Subject: [PATCH 1/8] Move tests-misc.yml to GHA

---
 .github/scripts/check-each-crate.py |  69 +++++
 .github/workflows/tests-misc.yml    | 379 ++++++++++++++++++++++++++++
 .github/workflows/tests.yml         |  16 +-
 .gitlab/pipeline/test.yml           | 300 +---------------------
 4 files changed, 462 insertions(+), 302 deletions(-)
 create mode 100755 .github/scripts/check-each-crate.py
 create mode 100644 .github/workflows/tests-misc.yml

diff --git a/.github/scripts/check-each-crate.py b/.github/scripts/check-each-crate.py
new file mode 100755
index 000000000000..7a53e812ddfc
--- /dev/null
+++ b/.github/scripts/check-each-crate.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+
+# A script that checks each workspace crate individually.
+# It's relevant to check workspace crates individually because otherwise their compilation problems
+# due to feature misconfigurations won't be caught, as exemplified by
+# https://github.com/paritytech/substrate/issues/12705
+#
+# `check-each-crate.py target_group groups_total`
+#
+# - `target_group`: Integer starting from 1, the group this script should execute.
+# - `groups_total`: Integer starting from 1, total number of groups.
+# - `disable_forklift`: Boolean, whether to disable forklift or not.
+
+import subprocess, sys
+
+# Get all crates
+output = subprocess.check_output(["cargo", "tree", "--locked", "--workspace", "--depth", "0", "--prefix", "none"])
+
+# Convert the output into a proper list
+crates = []
+for line in output.splitlines():
+	if line != b"":
+		line = line.decode('utf8').split(" ")
+		crate_name = line[0]
+		# The crate path is always the last element in the line.
+		crate_path = line[len(line) - 1].replace("(", "").replace(")", "")
+		crates.append((crate_name, crate_path))
+
+# Make the list unique and sorted
+crates = list(set(crates))
+crates.sort()
+
+target_group = int(sys.argv[1]) - 1
+groups_total = int(sys.argv[2])
+disable_forklift = bool(sys.argv[3] if len(sys.argv) > 3 else False)
+
+print(f"Target group: {target_group}, Total groups: {groups_total}, Disable forklift: {disable_forklift}", file=sys.stderr)
+
+if len(crates) == 0:
+	print("No crates detected!", file=sys.stderr)
+	sys.exit(1)
+
+print(f"Total crates: {len(crates)}", file=sys.stderr)
+
+crates_per_group = len(crates) // groups_total
+
+# If this is the last runner, we need to take care of crates
+# after the group that we lost because of the integer division.
+if target_group + 1 == groups_total:
+	overflow_crates = len(crates) % groups_total
+else:
+	overflow_crates = 0
+
+print(f"Crates per group: {crates_per_group}", file=sys.stderr)
+
+# Check each crate
+for i in range(0, crates_per_group + overflow_crates):
+	crate = crates_per_group * target_group + i
+
+	print(f"Checking {crates[crate][0]}", file=sys.stderr)
+
+	cmd = ["cargo", "check", "--locked"]
+
+	cmd.insert(0, 'forklift') if not disable_forklift else None
+
+	res = subprocess.run(cmd, cwd = crates[crate][1])
+
+	if res.returncode != 0:
+		sys.exit(1)
diff --git a/.github/workflows/tests-misc.yml b/.github/workflows/tests-misc.yml
new file mode 100644
index 000000000000..3a12f0dfdb65
--- /dev/null
+++ b/.github/workflows/tests-misc.yml
@@ -0,0 +1,379 @@
+name: tests misc
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    types: [ opened, synchronize, reopened, ready_for_review ]
+  merge_group:
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  changes:
+    permissions:
+      pull-requests: read
+    uses: ./.github/workflows/reusable-check-changed-files.yml
+
+  set-image:
+    needs: [ changes ]
+    if: needs.changes.outputs.rust || needs.changes.outputs.current-workflow
+    # GitHub Actions allows using 'env' in a container context.
+    # However, env variables don't work for forks: https://github.com/orgs/community/discussions/44322
+    # This workaround sets the container image for each job using 'set-image' job output.
+    runs-on: ubuntu-latest
+    outputs:
+      IMAGE: ${{ steps.set_image.outputs.IMAGE }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - id: set_image
+        run: cat .github/env >> $GITHUB_OUTPUT
+
+  # more information about this job can be found here:
+  # https://github.com/paritytech/substrate/pull/3778
+  test-full-crypto-feature:
+    needs: [ set-image ]
+    runs-on: arc-runners-polkadot-sdk-beefy
+    timeout-minutes: 60
+    container:
+      image: ${{ needs.set-image.outputs.IMAGE }}
+    env:
+      # Enable debug assertions since we are running optimized builds for testing
+      # but still want to have debug assertions.
+      RUSTFLAGS: "-C debug-assertions"
+      RUST_BACKTRACE: 1
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4.1.7
+
+      - name: script
+        run: |
+          cd substrate/primitives/core/
+          time forklift cargo build --locked --no-default-features --features full_crypto
+          cd ../application-crypto
+          time forklift cargo build --locked --no-default-features --features full_crypto
+
+  test-frame-examples-compile-to-wasm:
+    timeout-minutes: 20
+    # into one job
+    needs: [ set-image, test-full-crypto-feature ]
+    runs-on: arc-runners-polkadot-sdk-beefy
+    container:
+      image: ${{ needs.set-image.outputs.IMAGE }}
+    env:
+      # Enable debug assertions since we are running optimized builds for testing
+      # but still want to have debug assertions.
+      RUSTFLAGS: "-C debug-assertions"
+      RUST_BACKTRACE: 1
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: script
+        run: |
+          cd substrate/frame/examples/offchain-worker/
+          time forklift cargo build --locked --target=wasm32-unknown-unknown --no-default-features
+          cd ../basic
+          time forklift cargo build --locked --target=wasm32-unknown-unknown --no-default-features
+
+  test-frame-ui:
+    timeout-minutes: 60
+    needs: [ set-image, test-frame-examples-compile-to-wasm ]
+    runs-on: arc-runners-polkadot-sdk-beefy
+    container:
+      image: ${{ needs.set-image.outputs.IMAGE }}
+    env:
+      # Enable debug assertions since we are running optimized builds for testing
+      # but still want to have debug assertions.
+      RUSTFLAGS: "-C debug-assertions -D warnings"
+      RUST_BACKTRACE: 1
+      WASM_BUILD_NO_COLOR: 1
+      WASM_BUILD_RUSTFLAGS: "-C debug-assertions -D warnings"
+      # Ensure we run the UI tests.
+      RUN_UI_TESTS: 1
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: script
+        run: |
+          time forklift cargo test --locked -q --profile testnet -p frame-support-test --features=frame-feature-testing,no-metadata-docs,try-runtime,experimental
+          time forklift cargo test --locked -q --profile testnet -p frame-support-test --features=frame-feature-testing,frame-feature-testing-2,no-metadata-docs,try-runtime,experimental
+          time forklift cargo test --locked -q --profile testnet -p xcm-procedural
+          time forklift cargo test --locked -q --profile testnet -p frame-election-provider-solution-type
+          time forklift cargo test --locked -q --profile testnet -p sp-api-test
+          # There is multiple version of sp-runtime-interface in the repo. So we point to the manifest.
+          time forklift cargo test --locked -q --profile testnet --manifest-path substrate/primitives/runtime-interface/Cargo.toml
+
+  test-deterministic-wasm:
+    timeout-minutes: 20
+    needs: [ set-image, test-frame-ui ]
+    runs-on: arc-runners-polkadot-sdk-beefy
+    container:
+      image: ${{ needs.set-image.outputs.IMAGE }}
+    env:
+      WASM_BUILD_NO_COLOR: 1
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: script
+        run: |
+          # build runtime
+          time forklift cargo build -q --locked --release -p westend-runtime -p rococo-runtime
+          # make checksum
+          sha256sum target/release/wbuild/*-runtime/target/wasm32-unknown-unknown/release/*.wasm > checksum.sha256
+          cargo clean
+          # build again
+          time forklift cargo build -q --locked --release -p westend-runtime -p rococo-runtime
+          # confirm checksum
+          sha256sum -c checksum.sha256
+
+  cargo-check-benches-branches:
+    needs: [ set-image ]
+    if: ${{ github.event_name == 'pull_request' }}
+    timeout-minutes: 60
+    outputs:
+      branch: ${{ steps.branch.outputs.branch }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Branch
+        id: branch
+        run: |
+          echo "branch=['${{ github.base_ref }}', '${{ github.head_ref }}']" >> $GITHUB_OUTPUT
+
+  cargo-check-benches:
+    needs: [ set-image, cargo-check-benches-branches ]
+    timeout-minutes: 60
+    strategy:
+      matrix:
+        branch: ${{ fromJSON(needs.cargo-check-benches-branches.outputs.branch) }}
+    runs-on: arc-runners-polkadot-sdk-beefy
+    container:
+      image: ${{ needs.set-image.outputs.IMAGE }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ matrix.branch }}
+
+      - name: script
+        run: |
+          ARTIFACTS_DIR=./artifacts
+          BENCH_TRIE_READ=::trie::read::small
+          BENCH_NODE_IMPORT=::node::import::sr25519::transfer_keep_alive::paritydb::small
+          mkdir -p $ARTIFACTS_DIR
+
+          SKIP_WASM_BUILD=1 forklift time cargo check --locked --benches --all;
+          forklift cargo run --locked --release -p node-bench -- $BENCH_TRIE_READ --json | tee $ARTIFACTS_DIR/bench_trie_read_small.json;
+          forklift cargo run --locked --release -p node-bench -- $BENCH_NODE_IMPORT --json | tee $ARTIFACTS_DIR/bench_transfer_keep_alive.json
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4.3.6
+        with:
+          path: ./artifacts
+          name: cargo-check-benches-${{ matrix.branch }}-${{ github.sha }}
+          retention-days: 1
+
+  node-bench-regression-guard:
+    timeout-minutes: 20
+    runs-on: arc-runners-polkadot-sdk-default
+    needs: [ set-image, cargo-check-benches ]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4.1.7
+
+      - name: Extract branch name
+        shell: bash
+        run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT
+        id: extract_branch
+
+      - name: Download artifact (master run)
+        uses: actions/download-artifact@v4.1.8
+        with:
+          name: cargo-check-benches-${{ github.base_ref }}-${{ github.sha }}
+          path: ./artifacts/master
+
+      - name: Download artifact (current run)
+        uses: actions/download-artifact@v4.1.8
+        with:
+          name: cargo-check-benches-${{ github.head_ref }}-${{ github.sha }}
+          path: ./artifacts/current
+
+      - name: script
+        run: |
+          echo "------- IMPORTANT -------"
+          echo "node-bench-regression-guard depends on the results of a cargo-check-benches job"
+          echo "In case of this job failure, check your pipeline's cargo-check-benches"
+          docker run --rm \
+          -v $PWD/artifacts/master:/artifacts/master \
+          -v $PWD/artifacts/current:/artifacts/current \
+          paritytech/node-bench-regression-guard:latest \
+          node-bench-regression-guard --reference /artifacts/master --compare-with /artifacts/current
+
+  test-node-metrics:
+    needs: [ set-image ]
+    timeout-minutes: 30
+    runs-on: arc-runners-polkadot-sdk-beefy
+    container:
+      image: ${{ needs.set-image.outputs.IMAGE }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4.1.7
+
+      - name: Run tests
+        id: tests
+        env:
+          RUST_TOOLCHAIN: stable
+          # Enable debug assertions since we are running optimized builds for testing
+          # but still want to have debug assertions.
+          RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
+        run: |
+          forklift cargo build --bin polkadot-execute-worker --bin polkadot-prepare-worker --profile testnet --verbose --locked
+          mkdir -p ./artifacts
+          forklift cargo test --profile testnet --locked --features=runtime-metrics -p polkadot-node-metrics > ./artifacts/log.txt
+          echo "Metrics test passed"
+
+      - name: Upload artifacts if failed
+        if: ${{ steps.tests.outcome != 'success' }}
+        uses: actions/upload-artifact@v4.3.6
+        with:
+          name: node-metrics-failed
+          path: ./artifacts
+
+  # more information about this job can be found here:
+  # https://github.com/paritytech/substrate/pull/6916
+  check-tracing:
+    timeout-minutes: 20
+    needs: [ set-image, test-node-metrics ]
+    runs-on: arc-runners-polkadot-sdk-beefy
+    container:
+      image: ${{ needs.set-image.outputs.IMAGE }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4.1.7
+
+      - name: script
+        run: |
+          forklift cargo test --locked --manifest-path ./substrate/primitives/tracing/Cargo.toml --no-default-features
+          forklift cargo test --locked --manifest-path ./substrate/primitives/tracing/Cargo.toml --no-default-features --features=with-tracing
+
+  check-metadata-hash:
+    timeout-minutes: 20
+    needs: [ set-image, check-tracing ]
+    runs-on: arc-runners-polkadot-sdk-beefy
+    container:
+      image: ${{ needs.set-image.outputs.IMAGE }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4.1.7
+
+      - name: script
+        run: |
+          time forklift cargo build --locked -p westend-runtime --features metadata-hash
+
+  cargo-hfuzz:
+    timeout-minutes: 20
+    needs: [ set-image, check-metadata-hash ]
+    runs-on: arc-runners-polkadot-sdk-beefy
+    container:
+      image: ${{ needs.set-image.outputs.IMAGE }}
+    env:
+      # max 10s per iteration, 60s per file
+      HFUZZ_RUN_ARGS: |
+        --exit_upon_crash
+        --exit_code_upon_crash 1
+        --timeout 10
+        --run_time 60
+
+      # use git version of honggfuzz-rs until v0.5.56 is out, we need a few recent changes:
+      # https://github.com/rust-fuzz/honggfuzz-rs/pull/75 to avoid breakage on debian
+      # https://github.com/rust-fuzz/honggfuzz-rs/pull/81 fix to the above pr
+      # https://github.com/rust-fuzz/honggfuzz-rs/pull/82 fix for handling absolute CARGO_TARGET_DIR
+      HFUZZ_BUILD_ARGS: |
+        --config=patch.crates-io.honggfuzz.git="https://github.com/altaua/honggfuzz-rs"
+        --config=patch.crates-io.honggfuzz.rev="205f7c8c059a0d98fe1cb912cdac84f324cb6981"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4.1.7
+
+      - name: Run honggfuzz
+        run: |
+          cd substrate/primitives/arithmetic/fuzzer
+          forklift cargo hfuzz build
+          for target in $(cargo read-manifest | jq -r '.targets | .[] | .name');
+          do
+            forklift cargo hfuzz run "$target" || { printf "fuzzing failure for %s\n" "$target"; exit 1; };
+          done
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4.3.6
+        with:
+          path: substrate/primitives/arithmetic/fuzzer/hfuzz_workspace/
+          name: hfuzz-${{ github.sha }}
+
+  cargo-check-each-crate:
+    timeout-minutes: 140
+    needs: [ set-image ]
+    runs-on: arc-runners-polkadot-sdk-beefy
+    container:
+      image: ${{ needs.set-image.outputs.IMAGE }}
+    env:
+      RUSTFLAGS: "-D warnings"
+      CI_JOB_NAME: cargo-check-each-crate
+    strategy:
+      matrix:
+        index: [ 1,2,3,4,5,6,7 ] # 7 parallel jobs
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4.1.7
+
+      - name: Check Rust
+        run: |
+          rustup show
+          rustup +nightly show
+
+      - name: script
+        run: |
+          PYTHONUNBUFFERED=x time .github/scripts/check-each-crate.py ${{ matrix.index }} ${{ strategy.job-total }}
+
+# TODO: enable when we have a macos Self-Hosted runners
+#  cargo-check-each-crate-macos:
+#    timeout-minutes: 120
+#    needs: [ set-image ]
+#    runs-on: macos-latest
+#    env:
+#      RUSTFLAGS: "-D warnings"
+#      CI_JOB_NAME: cargo-check-each-crate
+#      IMAGE: ${{ needs.set-image.outputs.IMAGE }}
+#    strategy:
+#      fail-fast: false
+#      matrix:
+#        index: [ 1,2,3,4,5,6,7,8,9,10 ] # 10 parallel jobs
+#    steps:
+#      - name: Checkout
+#        uses: actions/checkout@v4.1.7
+#
+#      - run: |
+#          VERSION=$(echo $IMAGE | sed -E 's/.*:bullseye-([^-]+)-.*/\1/')
+#          echo $VERSION
+#          echo "VERSION=$VERSION" >> $GITHUB_ENV
+#
+#      - run: |
+#          rustup install $VERSION
+#          rustup default $VERSION
+#
+#      - name: Check Rust
+#        run: |
+#          rustup show
+#          rustup +nightly show
+#
+#      - name: MacOS Deps
+#        run: |
+#          brew install protobuf openssl pkg-config zlib xz zstd llvm jq curl gcc make cmake
+#          rustup target add wasm32-unknown-unknown --toolchain $VERSION
+#          rustup component add rust-src rustfmt clippy --toolchain $VERSION
+#
+#      - name: script
+#        run: |
+#          PYTHONUNBUFFERED=x time .github/scripts/check-each-crate.py ${{ matrix.index }} ${{ strategy.job-total }} True
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 1be2dd7921e0..78bd4fd2456f 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -68,11 +68,17 @@ jobs:
           cd polkadot/scripts/list-syscalls
           ./list-syscalls.rb ../../../target/x86_64-unknown-linux-musl/production/polkadot-execute-worker --only-used-syscalls | diff -u execute-worker-syscalls -
           ./list-syscalls.rb ../../../target/x86_64-unknown-linux-musl/production/polkadot-prepare-worker --only-used-syscalls | diff -u prepare-worker-syscalls -
-    # todo:
-    # after_script:
-    # - if [[ "$CI_JOB_STATUS" == "failed" ]]; then
-    #   printf "The x86_64 syscalls used by the worker binaries have changed. Please review if this is expected and update polkadot/scripts/list-syscalls/*-worker-syscalls as needed.\n";
-    #   fi
+
+  test-syscalls-on-failure:
+    needs: [ test-syscalls ]
+    if: needs.test-syscalls.result == 'failure'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: script
+        run: |
+          echo "The x86_64 syscalls used by the worker binaries have changed. Please review if this is expected and update polkadot/scripts/list-syscalls/*-worker-syscalls as needed.\n"
 
   cargo-check-all-benches:
     needs: [ set-image, changes ]
diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml
index 319c95ad6112..37ee025f61f7 100644
--- a/.gitlab/pipeline/test.yml
+++ b/.gitlab/pipeline/test.yml
@@ -35,7 +35,7 @@ codecov-start:
     - .pipeline-stopper-artifacts
     - .run-immediately
   script:
-    - !reference [.codecov-check, script]
+    - !reference [ .codecov-check, script ]
     - >
       if [ "$CI_COMMIT_REF_NAME" != "master" ]; then
         codecovcli -v create-commit -t ${CODECOV_TOKEN} -r paritytech/polkadot-sdk --commit-sha ${CI_COMMIT_SHA} --fail-on-error --pr ${CI_COMMIT_REF_NAME} --git-service github;
@@ -57,7 +57,7 @@ codecov-finish:
   needs:
     - test-linux-stable-codecov
   script:
-    - !reference [.codecov-check, script]
+    - !reference [ .codecov-check, script ]
     - codecovcli -v create-report-results -t ${CODECOV_TOKEN} -r paritytech/polkadot-sdk --commit-sha ${CI_COMMIT_SHA} --git-service github
     - codecovcli -v get-report-results -t ${CODECOV_TOKEN} -r paritytech/polkadot-sdk --commit-sha ${CI_COMMIT_SHA} --git-service github
     - codecovcli -v send-notifications -t ${CODECOV_TOKEN} -r paritytech/polkadot-sdk --commit-sha ${CI_COMMIT_SHA} --git-service github
@@ -83,7 +83,7 @@ test-linux-stable-codecov:
   parallel: 2
   script:
     # tools
-    - !reference [.codecov-check, script]
+    - !reference [ .codecov-check, script ]
     - rustup component add llvm-tools-preview
     - mkdir -p target/coverage/result/
     # Place real test call here
@@ -250,159 +250,6 @@ test-rustdoc:
   script:
     - time cargo doc --workspace --all-features --no-deps
 
-test-node-metrics:
-  stage: test
-  extends:
-    - .docker-env
-    - .common-refs
-    - .run-immediately
-    - .collect-artifacts-short
-  variables:
-    RUST_TOOLCHAIN: stable
-    # Enable debug assertions since we are running optimized builds for testing
-    # but still want to have debug assertions.
-    RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
-  script:
-    # Build the required workers.
-    - cargo build --bin polkadot-execute-worker --bin polkadot-prepare-worker --profile testnet --verbose --locked
-    - mkdir -p artifacts
-    - time cargo test --profile testnet
-      --locked
-      --features=runtime-metrics -p polkadot-node-metrics > artifacts/log.txt
-
-test-deterministic-wasm:
-  stage: test
-  extends:
-    - .docker-env
-    - .common-refs
-  # DAG
-  needs:
-    - job: test-frame-ui
-      artifacts: false
-  script:
-    # build runtime
-    - WASM_BUILD_NO_COLOR=1 cargo build -q --locked --release -p westend-runtime -p rococo-runtime
-    # make checksum
-    - sha256sum target/release/wbuild/*-runtime/target/wasm32-unknown-unknown/release/*.wasm > checksum.sha256
-    - cargo clean
-    # build again
-    - WASM_BUILD_NO_COLOR=1 cargo build -q --locked --release -p westend-runtime -p rococo-runtime
-    # confirm checksum
-    - sha256sum -c checksum.sha256
-
-cargo-check-benches:
-  stage: test
-  artifacts:
-    expire_in: 10 days
-  variables:
-    CI_JOB_NAME: "cargo-check-benches"
-  extends:
-    - .docker-env
-    - .common-refs
-    - .run-immediately
-    - .collect-artifacts
-    - .pipeline-stopper-artifacts
-  before_script:
-    # TODO: DON'T FORGET TO CHANGE FOR PROD VALUES!!!
-    # merges in the master branch on PRs. skip if base is not master
-    - 'if [ $CI_COMMIT_REF_NAME != "master" ]; then
-      BASE=$(curl -s -H "Authorization: Bearer ${GITHUB_PR_TOKEN}" https://api.github.com/repos/paritytech-stg/polkadot-sdk/pulls/${CI_COMMIT_REF_NAME} | jq -r .base.ref);
-      printf "Merging base branch %s\n" "${BASE:=master}";
-      if [ $BASE != "master" ]; then
-      echo "$BASE is not master, skipping merge";
-      else
-      git config user.email "ci@gitlab.parity.io";
-      git fetch origin "refs/heads/${BASE}";
-      git merge --verbose --no-edit FETCH_HEAD;
-      fi
-      fi'
-    - !reference [.forklift-cache, before_script]
-  parallel: 2
-  script:
-    - mkdir -p ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA
-    # this job is executed in parallel on two runners
-    - echo "___Running benchmarks___";
-    - case ${CI_NODE_INDEX} in
-      1)
-      SKIP_WASM_BUILD=1 time cargo check --locked --benches --all;
-      cargo run --locked --release -p node-bench -- ::trie::read::small --json
-      | tee ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::trie::read::small.json;
-      echo "___Cache could be uploaded___";
-      ;;
-      2)
-      cargo run --locked --release -p node-bench -- ::node::import::sr25519::transfer_keep_alive::paritydb::small --json
-      | tee ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::node::import::sr25519::transfer_keep_alive::paritydb::small.json
-      ;;
-      esac
-
-node-bench-regression-guard:
-  # it's not belong to `build` semantically, but dag jobs can't depend on each other
-  # within the single stage - https://gitlab.com/gitlab-org/gitlab/-/issues/30632
-  # more: https://github.com/paritytech/substrate/pull/8519#discussion_r608012402
-  stage: build
-  extends:
-    - .docker-env
-    - .common-refs
-  needs:
-    # this is a DAG
-    - job: cargo-check-benches
-      artifacts: true
-    # polls artifact from master to compare with current result
-    # need to specify both parallel jobs from master because of the bug
-    # https://gitlab.com/gitlab-org/gitlab/-/issues/39063
-    - project: $CI_PROJECT_PATH
-      job: "cargo-check-benches 1/2"
-      ref: master
-      artifacts: true
-    - project: $CI_PROJECT_PATH
-      job: "cargo-check-benches 2/2"
-      ref: master
-      artifacts: true
-  variables:
-    CI_IMAGE: "paritytech/node-bench-regression-guard:latest"
-  before_script: [""]
-  script:
-    - if [ $(ls -la artifacts/benches/ | grep master | wc -l) == 0 ]; then
-      echo "Couldn't find master artifacts";
-      exit 1;
-      fi
-    - echo "------- IMPORTANT -------"
-    - echo "node-bench-regression-guard depends on the results of a cargo-check-benches job"
-    - echo "In case of this job failure, check your pipeline's cargo-check-benches"
-    - "node-bench-regression-guard --reference artifacts/benches/master-*
-      --compare-with artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA"
-  after_script: [""]
-
-# if this fails run `bot update-ui` in the Pull Request or "./scripts/update-ui-tests.sh" locally
-# see ./docs/contributor/CONTRIBUTING.md#ui-tests
-test-frame-ui:
-  stage: test
-  extends:
-    - .docker-env
-    - .common-refs
-  # DAG
-  needs:
-    - job: test-frame-examples-compile-to-wasm
-      artifacts: false
-  variables:
-    # Enable debug assertions since we are running optimized builds for testing
-    # but still want to have debug assertions.
-    RUSTFLAGS: "-C debug-assertions -D warnings"
-    RUST_BACKTRACE: 1
-    WASM_BUILD_NO_COLOR: 1
-    WASM_BUILD_RUSTFLAGS: "-C debug-assertions -D warnings"
-    # Ensure we run the UI tests.
-    RUN_UI_TESTS: 1
-  script:
-    - time cargo test --locked -q --profile testnet -p frame-support-test --features=frame-feature-testing,no-metadata-docs,try-runtime,experimental
-    - time cargo test --locked -q --profile testnet -p frame-support-test --features=frame-feature-testing,frame-feature-testing-2,no-metadata-docs,try-runtime,experimental
-    - time cargo test --locked -q --profile testnet -p xcm-procedural
-    - time cargo test --locked -q --profile testnet -p frame-election-provider-solution-type
-    - time cargo test --locked -q --profile testnet -p sp-api-test
-    # There is multiple version of sp-runtime-interface in the repo. So we point to the manifest.
-    - time cargo test --locked -q --profile testnet --manifest-path substrate/primitives/runtime-interface/Cargo.toml
-    - cat /cargo_target_dir/debug/.fingerprint/memory_units-759eddf317490d2b/lib-memory_units.json || true
-
 # This job runs all benchmarks defined in the `/bin/node/runtime` once to check that there are no errors.
 quick-benchmarks:
   stage: test
@@ -437,29 +284,6 @@ quick-benchmarks-omni:
     - time cargo build --locked --quiet --release -p asset-hub-westend-runtime --features runtime-benchmarks
     - time cargo run --locked --release -p frame-omni-bencher --quiet -- v1 benchmark pallet --runtime target/release/wbuild/asset-hub-westend-runtime/asset_hub_westend_runtime.compact.compressed.wasm --all --steps 2 --repeat 1 --quiet
 
-test-frame-examples-compile-to-wasm:
-  # into one job
-  stage: test
-  extends:
-    - .docker-env
-    - .common-refs
-  # DAG
-  needs:
-    - job: test-full-crypto-feature
-      artifacts: false
-  variables:
-    # Enable debug assertions since we are running optimized builds for testing
-    # but still want to have debug assertions.
-    RUSTFLAGS: "-C debug-assertions"
-    RUST_BACKTRACE: 1
-  script:
-    - cd ./substrate/frame/examples/offchain-worker/
-    - cargo build --locked --target=wasm32-unknown-unknown --no-default-features
-    - cd ../basic
-    - cargo build --locked --target=wasm32-unknown-unknown --no-default-features
-  # FIXME
-  allow_failure: true
-
 test-linux-stable-int:
   stage: test
   extends:
@@ -480,124 +304,6 @@ test-linux-stable-int:
     - WASM_BUILD_NO_COLOR=1
       time cargo test -p staging-node-cli --release --locked -- --ignored
 
-# more information about this job can be found here:
-# https://github.com/paritytech/substrate/pull/6916
-check-tracing:
-  stage: test
-  extends:
-    - .docker-env
-    - .common-refs
-    - .run-immediately
-    - .pipeline-stopper-artifacts
-  script:
-    # with-tracing must be explicitly activated, we run a test to ensure this works as expected in both cases
-    - time cargo test --locked --manifest-path ./substrate/primitives/tracing/Cargo.toml --no-default-features
-    - time cargo test --locked --manifest-path ./substrate/primitives/tracing/Cargo.toml --no-default-features --features=with-tracing
-
-# Check that `westend-runtime` compiles with the `metadata-hash` feature enabled.
-check-metadata-hash:
-  stage: test
-  extends:
-    - .docker-env
-    - .common-refs
-    - .run-immediately
-    - .pipeline-stopper-artifacts
-  script:
-    - time cargo build --locked -p westend-runtime --features metadata-hash
-
-# more information about this job can be found here:
-# https://github.com/paritytech/substrate/pull/3778
-test-full-crypto-feature:
-  stage: test
-  extends:
-    - .docker-env
-    - .common-refs
-    - .run-immediately
-  variables:
-    # Enable debug assertions since we are running optimized builds for testing
-    # but still want to have debug assertions.
-    RUSTFLAGS: "-C debug-assertions"
-    RUST_BACKTRACE: 1
-  script:
-    - cd substrate/primitives/core/
-    - time cargo build --locked --no-default-features --features full_crypto
-    - cd ../application-crypto
-    - time cargo build --locked --no-default-features --features full_crypto
-
-cargo-check-each-crate:
-  stage: test
-  extends:
-    - .docker-env
-    - .common-refs
-    - .run-immediately
-    # - .collect-artifacts
-  variables:
-    RUSTFLAGS: "-D warnings"
-    # $CI_JOB_NAME is set manually so that cache could be shared for all jobs
-    # "cargo-check-each-crate I/N" jobs
-    CI_JOB_NAME: cargo-check-each-crate
-  timeout: 2h
-  script:
-    - PYTHONUNBUFFERED=x time .gitlab/check-each-crate.py "$CI_NODE_INDEX" "$CI_NODE_TOTAL"
-  parallel: 6
-
-cargo-check-each-crate-macos:
-  stage: test
-  extends:
-    - .docker-env
-    - .common-refs
-    - .run-immediately
-    # - .collect-artifacts
-  before_script:
-    # skip timestamp script, the osx bash doesn't support printf %()T
-    - !reference [.job-switcher, before_script]
-    - !reference [.rust-info-script, script]
-    - !reference [.pipeline-stopper-vars, script]
-  variables:
-    SKIP_WASM_BUILD: 1
-  script:
-    # TODO: use parallel jobs, as per cargo-check-each-crate, once more Mac runners are available
-    # - time ./scripts/ci/gitlab/check-each-crate.py 1 1
-    - time cargo check --workspace --locked
-  timeout: 2h
-  tags:
-    - osx
-
-cargo-hfuzz:
-  stage: test
-  extends:
-    - .docker-env
-    - .common-refs
-  # DAG
-  needs:
-    - job: check-tracing
-      artifacts: false
-  variables:
-    # max 10s per iteration, 60s per file
-    HFUZZ_RUN_ARGS: >
-      --exit_upon_crash
-      --exit_code_upon_crash 1
-      --timeout 10
-      --run_time 60
-    # use git version of honggfuzz-rs until v0.5.56 is out, we need a few recent changes:
-    # https://github.com/rust-fuzz/honggfuzz-rs/pull/75 to avoid breakage on debian
-    # https://github.com/rust-fuzz/honggfuzz-rs/pull/81 fix to the above pr
-    # https://github.com/rust-fuzz/honggfuzz-rs/pull/82 fix for handling absolute CARGO_TARGET_DIR
-    HFUZZ_BUILD_ARGS: >
-      --config=patch.crates-io.honggfuzz.git="https://github.com/altaua/honggfuzz-rs"
-      --config=patch.crates-io.honggfuzz.rev="205f7c8c059a0d98fe1cb912cdac84f324cb6981"
-  artifacts:
-    name: "hfuzz-$CI_COMMIT_SHORT_SHA"
-    expire_in: 7 days
-    when: on_failure
-    paths:
-      - substrate/primitives/arithmetic/fuzzer/hfuzz_workspace/
-  script:
-    - cd ./substrate/primitives/arithmetic/fuzzer
-    - cargo hfuzz build
-    - for target in $(cargo read-manifest | jq -r '.targets | .[] | .name'); do
-      cargo hfuzz run "$target" || { printf "fuzzing failure for %s\n" "$target"; exit 1; }; done
-
 .subsystem-benchmark-template:
   stage: test
   artifacts:

From 10a3ed529feb59d908c95f37fe6351ef7a4718a6 Mon Sep 17 00:00:00 2001
From: Maksym H <mordamax@gmail.com>
Date: Tue, 20 Aug 2024 10:32:07 +0100
Subject: [PATCH 2/8] followups

---
 .github/workflows/tests-misc.yml | 144 +++++++++++++++++--------------
 .gitlab/check-each-crate.py      |  61 -------------
 2 files changed, 79 insertions(+), 126 deletions(-)
 delete mode 100755 .gitlab/check-each-crate.py

diff --git a/.github/workflows/tests-misc.yml b/.github/workflows/tests-misc.yml
index 3a12f0dfdb65..d7647366d784 100644
--- a/.github/workflows/tests-misc.yml
+++ b/.github/workflows/tests-misc.yml
@@ -11,6 +11,8 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
+# Jobs in this workflow depend on each other, only for limiting peak amount of spawned workers
+
 jobs:
   changes:
     permissions:
@@ -52,9 +54,9 @@ jobs:
       - name: script
         run: |
           cd substrate/primitives/core/
-          time forklift cargo build --locked --no-default-features --features full_crypto
+          forklift cargo build --locked --no-default-features --features full_crypto
           cd ../application-crypto
-          time forklift cargo build --locked --no-default-features --features full_crypto
+          forklift cargo build --locked --no-default-features --features full_crypto
 
   test-frame-examples-compile-to-wasm:
     timeout-minutes: 20
@@ -74,9 +76,9 @@ jobs:
       - name: script
         run: |
           cd substrate/frame/examples/offchain-worker/
-          time forklift cargo build --locked --target=wasm32-unknown-unknown --no-default-features
+          forklift cargo build --locked --target=wasm32-unknown-unknown --no-default-features
           cd ../basic
-          time forklift cargo build --locked --target=wasm32-unknown-unknown --no-default-features
+          forklift cargo build --locked --target=wasm32-unknown-unknown --no-default-features
 
   test-frame-ui:
     timeout-minutes: 60
@@ -98,13 +100,13 @@ jobs:
         uses: actions/checkout@v4
       - name: script
         run: |
-          time forklift cargo test --locked -q --profile testnet -p frame-support-test --features=frame-feature-testing,no-metadata-docs,try-runtime,experimental
-          time forklift cargo test --locked -q --profile testnet -p frame-support-test --features=frame-feature-testing,frame-feature-testing-2,no-metadata-docs,try-runtime,experimental
-          time forklift cargo test --locked -q --profile testnet -p xcm-procedural
-          time forklift cargo test --locked -q --profile testnet -p frame-election-provider-solution-type
-          time forklift cargo test --locked -q --profile testnet -p sp-api-test
+          forklift cargo test --locked -q --profile testnet -p frame-support-test --features=frame-feature-testing,no-metadata-docs,try-runtime,experimental
+          forklift cargo test --locked -q --profile testnet -p frame-support-test --features=frame-feature-testing,frame-feature-testing-2,no-metadata-docs,try-runtime,experimental
+          forklift cargo test --locked -q --profile testnet -p xcm-procedural
+          forklift cargo test --locked -q --profile testnet -p frame-election-provider-solution-type
+          forklift cargo test --locked -q --profile testnet -p sp-api-test
           # There is multiple version of sp-runtime-interface in the repo. So we point to the manifest.
-          time forklift cargo test --locked -q --profile testnet --manifest-path substrate/primitives/runtime-interface/Cargo.toml
+          forklift cargo test --locked -q --profile testnet --manifest-path substrate/primitives/runtime-interface/Cargo.toml
 
   test-deterministic-wasm:
     timeout-minutes: 20
@@ -120,12 +122,12 @@ jobs:
       - name: script
         run: |
           # build runtime
-          time forklift cargo build -q --locked --release -p westend-runtime -p rococo-runtime
+          forklift cargo build -q --locked --release -p westend-runtime -p rococo-runtime
           # make checksum
           sha256sum target/release/wbuild/*-runtime/target/wasm32-unknown-unknown/release/*.wasm > checksum.sha256
           cargo clean
           # build again
-          time forklift cargo build -q --locked --release -p westend-runtime -p rococo-runtime
+          forklift cargo build -q --locked --release -p westend-runtime -p rococo-runtime
           # confirm checksum
           sha256sum -c checksum.sha256
 
@@ -164,7 +166,7 @@ jobs:
           BENCH_NODE_IMPORT=::node::import::sr25519::transfer_keep_alive::paritydb::small
           mkdir -p $ARTIFACTS_DIR
 
-          SKIP_WASM_BUILD=1 forklift time cargo check --locked --benches --all;
+          SKIP_WASM_BUILD=1 forklift cargo check --locked --benches --all;
           forklift cargo run --locked --release -p node-bench -- $BENCH_TRIE_READ --json | tee $ARTIFACTS_DIR/bench_trie_read_small.json;
           forklift cargo run --locked --release -p node-bench -- $BENCH_NODE_IMPORT --json | tee $ARTIFACTS_DIR/bench_transfer_keep_alive.json
 
@@ -183,11 +185,6 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4.1.7
 
-      - name: Extract branch name
-        shell: bash
-        run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT
-        id: extract_branch
-
       - name: Download artifact (master run)
         uses: actions/download-artifact@v4.1.8
         with:
@@ -201,15 +198,15 @@ jobs:
           path: ./artifacts/current
 
       - name: script
+        id: compare
         run: |
-          echo "------- IMPORTANT -------"
-          echo "node-bench-regression-guard depends on the results of a cargo-check-benches job"
-          echo "In case of this job failure, check your pipeline's cargo-check-benches"
+          FAILED_MSG='### node-bench-regression-guard failed ❌, check the regression in *cargo-check-benches* job'
           docker run --rm \
           -v $PWD/artifacts/master:/artifacts/master \
           -v $PWD/artifacts/current:/artifacts/current \
           paritytech/node-bench-regression-guard:latest \
-          node-bench-regression-guard --reference /artifacts/master --compare-with /artifacts/current
+          node-bench-regression-guard --reference /artifacts/master --compare-with /artifacts/current \
+          || echo $FAILED_MSG >> $GITHUB_STEP_SUMMARY && echo $FAILED_MSG && exit 1
 
   test-node-metrics:
     needs: [ set-image ]
@@ -270,7 +267,7 @@ jobs:
 
       - name: script
         run: |
-          time forklift cargo build --locked -p westend-runtime --features metadata-hash
+          forklift cargo build --locked -p westend-runtime --features metadata-hash
 
   cargo-hfuzz:
     timeout-minutes: 20
@@ -335,45 +332,62 @@ jobs:
 
       - name: script
         run: |
-          PYTHONUNBUFFERED=x time .github/scripts/check-each-crate.py ${{ matrix.index }} ${{ strategy.job-total }}
-
-# TODO: enable when we have a macos Self-Hosted runners
-#  cargo-check-each-crate-macos:
-#    timeout-minutes: 120
-#    needs: [ set-image ]
-#    runs-on: macos-latest
-#    env:
-#      RUSTFLAGS: "-D warnings"
-#      CI_JOB_NAME: cargo-check-each-crate
-#      IMAGE: ${{ needs.set-image.outputs.IMAGE }}
-#    strategy:
-#      fail-fast: false
-#      matrix:
-#        index: [ 1,2,3,4,5,6,7,8,9,10 ] # 10 parallel jobs
-#    steps:
-#      - name: Checkout
-#        uses: actions/checkout@v4.1.7
-#
-#      - run: |
-#          VERSION=$(echo $IMAGE | sed -E 's/.*:bullseye-([^-]+)-.*/\1/')
-#          echo $VERSION
-#          echo "VERSION=$VERSION" >> $GITHUB_ENV
-#
-#      - run: |
-#          rustup install $VERSION
-#          rustup default $VERSION
-#
-#      - name: Check Rust
-#        run: |
-#          rustup show
-#          rustup +nightly show
-#
-#      - name: MacOS Deps
-#        run: |
-#          brew install protobuf openssl pkg-config zlib xz zstd llvm jq curl gcc make cmake
-#          rustup target add wasm32-unknown-unknown --toolchain $VERSION
-#          rustup component add rust-src rustfmt clippy --toolchain $VERSION
-#
-#      - name: script
-#        run: |
-#          PYTHONUNBUFFERED=x time .github/scripts/check-each-crate.py ${{ matrix.index }} ${{ strategy.job-total }} True
+          PYTHONUNBUFFERED=x .github/scripts/check-each-crate.py ${{ matrix.index }} ${{ strategy.job-total }}
+
+  # TODO: enable when we have a macos Self-Hosted runners
+  #  cargo-check-each-crate-macos:
+  #    timeout-minutes: 120
+  #    needs: [ set-image ]
+  #    runs-on: macos-latest
+  #    env:
+  #      RUSTFLAGS: "-D warnings"
+  #      CI_JOB_NAME: cargo-check-each-crate
+  #      IMAGE: ${{ needs.set-image.outputs.IMAGE }}
+  #    strategy:
+  #      fail-fast: false
+  #      matrix:
+  #        index: [ 1,2,3,4,5,6,7,8,9,10 ] # 10 parallel jobs
+  #    steps:
+  #      - name: Checkout
+  #        uses: actions/checkout@v4.1.7
+  #
+  #      - run: |
+  #          VERSION=$(echo $IMAGE | sed -E 's/.*:bullseye-([^-]+)-.*/\1/')
+  #          echo $VERSION
+  #          echo "VERSION=$VERSION" >> $GITHUB_ENV
+  #
+  #      - run: |
+  #          rustup install $VERSION
+  #          rustup default $VERSION
+  #
+  #      - name: Check Rust
+  #        run: |
+  #          rustup show
+  #          rustup +nightly show
+  #
+  #      - name: MacOS Deps
+  #        run: |
+  #          brew install protobuf openssl pkg-config zlib xz zstd llvm jq curl gcc make cmake
+  #          rustup target add wasm32-unknown-unknown --toolchain $VERSION
+  #          rustup component add rust-src rustfmt clippy --toolchain $VERSION
+  #
+  #      - name: script
+  #        run: |
+  #          PYTHONUNBUFFERED=x .github/scripts/check-each-crate.py ${{ matrix.index }} ${{ strategy.job-total }} True
+
+  confirm-required-jobs-passed:
+    runs-on: ubuntu-latest
+    name: All test misc tests passed
+    # If any new job gets added, be sure to add it to this array
+    needs:
+      - test-full-crypto-feature
+      - test-frame-examples-compile-to-wasm
+      - test-frame-ui
+      - cargo-check-benches
+      - node-bench-regression-guard
+      - test-node-metrics
+      - check-tracing
+      - cargo-hfuzz
+      - cargo-check-each-crate
+    steps:
+      - run: echo '### Good job! All the required tests passed 🚀' >> $GITHUB_STEP_SUMMARY
\ No newline at end of file
diff --git a/.gitlab/check-each-crate.py b/.gitlab/check-each-crate.py
deleted file mode 100755
index 9b654f8071ac..000000000000
--- a/.gitlab/check-each-crate.py
+++ /dev/null
@@ -1,61 +0,0 @@
-#!/usr/bin/env python3
-
-# A script that checks each workspace crate individually.
-# It's relevant to check workspace crates individually because otherwise their compilation problems
-# due to feature misconfigurations won't be caught, as exemplified by
-# https://github.com/paritytech/substrate/issues/12705
-#
-# `check-each-crate.py target_group groups_total`
-#
-# - `target_group`: Integer starting from 1, the group this script should execute.
-# - `groups_total`: Integer starting from 1, total number of groups.
-
-import subprocess, sys
-
-# Get all crates
-output = subprocess.check_output(["cargo", "tree", "--locked", "--workspace", "--depth", "0", "--prefix", "none"])
-
-# Convert the output into a proper list
-crates = []
-for line in output.splitlines():
-	if line != b"":
-		line = line.decode('utf8').split(" ")
-		crate_name = line[0]
-		# The crate path is always the last element in the line.
-		crate_path = line[len(line) - 1].replace("(", "").replace(")", "")
-		crates.append((crate_name, crate_path))
-
-# Make the list unique and sorted
-crates = list(set(crates))
-crates.sort()
-
-target_group = int(sys.argv[1]) - 1
-groups_total = int(sys.argv[2])
-
-if len(crates) == 0:
-	print("No crates detected!", file=sys.stderr)
-	sys.exit(1)
-
-print(f"Total crates: {len(crates)}", file=sys.stderr)
-
-crates_per_group = len(crates) // groups_total
-
-# If this is the last runner, we need to take care of crates
-# after the group that we lost because of the integer division.
-if target_group + 1 == groups_total:
-	overflow_crates = len(crates) % groups_total
-else:
-	overflow_crates = 0
-
-print(f"Crates per group: {crates_per_group}", file=sys.stderr)
-
-# Check each crate
-for i in range(0, crates_per_group + overflow_crates):
-	crate = crates_per_group * target_group + i
-
-	print(f"Checking {crates[crate][0]}", file=sys.stderr)
-
-	res = subprocess.run(["forklift", "cargo", "check", "--locked"], cwd = crates[crate][1])
-
-	if res.returncode != 0:
-		sys.exit(1)

From 18d11b457a9bf071d2ce95d4f9da2640aa552f88 Mon Sep 17 00:00:00 2001
From: Maksym H <mordamax@gmail.com>
Date: Wed, 28 Aug 2024 13:31:09 +0100
Subject: [PATCH 3/8] remove cancel pipelines for cleaned up jobs

---
 .gitlab-ci.yml | 86 +++++++-------------------------------------------
 1 file changed, 11 insertions(+), 75 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7f2babc6bd47..663f319f1359 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -21,7 +21,7 @@ workflow:
     - if: $CI_COMMIT_BRANCH
 
 variables:
-  CI_IMAGE: !reference [.ci-unified, variables, CI_IMAGE]
+  CI_IMAGE: !reference [ .ci-unified, variables, CI_IMAGE ]
   # BUILDAH_IMAGE is defined in group variables
   BUILDAH_COMMAND: "buildah --storage-driver overlay2"
   RELENG_SCRIPTS_BRANCH: "master"
@@ -39,7 +39,7 @@ default:
       - runner_system_failure
       - unknown_failure
       - api_failure
-  cache: {}
+  cache: { }
   interruptible: true
 
 .collect-artifacts:
@@ -68,8 +68,8 @@ default:
 
 .common-before-script:
   before_script:
-    - !reference [.job-switcher, before_script]
-    - !reference [.pipeline-stopper-vars, script]
+    - !reference [ .job-switcher, before_script ]
+    - !reference [ .pipeline-stopper-vars, script ]
 
 .job-switcher:
   before_script:
@@ -78,8 +78,8 @@ default:
 .kubernetes-env:
   image: "${CI_IMAGE}"
   before_script:
-    - !reference [.common-before-script, before_script]
-    - !reference [.prepare-env, before_script]
+    - !reference [ .common-before-script, before_script ]
+    - !reference [ .prepare-env, before_script ]
   tags:
     - kubernetes-parity-build
 
@@ -107,12 +107,12 @@ default:
 .docker-env:
   image: "${CI_IMAGE}"
   variables:
-    FL_FORKLIFT_VERSION: !reference [.forklift, variables, FL_FORKLIFT_VERSION]
+    FL_FORKLIFT_VERSION: !reference [ .forklift, variables, FL_FORKLIFT_VERSION ]
   before_script:
-    - !reference [.common-before-script, before_script]
-    - !reference [.prepare-env, before_script]
-    - !reference [.rust-info-script, script]
-    - !reference [.forklift-cache, before_script]
+    - !reference [ .common-before-script, before_script ]
+    - !reference [ .prepare-env, before_script ]
+    - !reference [ .rust-info-script, script ]
+    - !reference [ .forklift-cache, before_script ]
   tags:
     - linux-docker
 
@@ -285,61 +285,11 @@ cancel-pipeline-test-linux-stable3:
   needs:
     - job: "test-linux-stable 3/3"
 
-cancel-pipeline-cargo-check-benches1:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: "cargo-check-benches 1/2"
-
-cancel-pipeline-cargo-check-benches2:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: "cargo-check-benches 2/2"
-
 cancel-pipeline-test-linux-stable-int:
   extends: .cancel-pipeline-template
   needs:
     - job: test-linux-stable-int
 
-cancel-pipeline-cargo-check-each-crate-1:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: "cargo-check-each-crate 1/6"
-
-cancel-pipeline-cargo-check-each-crate-2:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: "cargo-check-each-crate 2/6"
-
-cancel-pipeline-cargo-check-each-crate-3:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: "cargo-check-each-crate 3/6"
-
-cancel-pipeline-cargo-check-each-crate-4:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: "cargo-check-each-crate 4/6"
-
-cancel-pipeline-cargo-check-each-crate-5:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: "cargo-check-each-crate 5/6"
-
-cancel-pipeline-cargo-check-each-crate-6:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: "cargo-check-each-crate 6/6"
-
-cancel-pipeline-cargo-check-each-crate-macos:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: cargo-check-each-crate-macos
-
-cancel-pipeline-check-tracing:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: check-tracing
-
 cancel-pipeline-cargo-clippy:
   extends: .cancel-pipeline-template
   needs:
@@ -360,16 +310,6 @@ cancel-pipeline-build-linux-substrate:
   needs:
     - job: build-linux-substrate
 
-cancel-pipeline-test-node-metrics:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: test-node-metrics
-
-cancel-pipeline-test-frame-ui:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: test-frame-ui
-
 cancel-pipeline-quick-benchmarks:
   extends: .cancel-pipeline-template
   needs:
@@ -380,10 +320,6 @@ cancel-pipeline-check-try-runtime:
   needs:
     - job: check-try-runtime
 
-cancel-pipeline-test-frame-examples-compile-to-wasm:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: test-frame-examples-compile-to-wasm
 
 cancel-pipeline-build-short-benchmark:
   extends: .cancel-pipeline-template

From e9e760a3e8454f6b825a035ba81176a77730324b Mon Sep 17 00:00:00 2001
From: Maksym H <mordamax@gmail.com>
Date: Thu, 29 Aug 2024 16:29:14 +0100
Subject: [PATCH 4/8] add persistent runner

---
 .github/workflows/tests-misc.yml | 45 ++++++++++++++++++++------------
 1 file changed, 28 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/tests-misc.yml b/.github/workflows/tests-misc.yml
index d7647366d784..afa80b5e7a3a 100644
--- a/.github/workflows/tests-misc.yml
+++ b/.github/workflows/tests-misc.yml
@@ -14,31 +14,42 @@ concurrency:
 # Jobs in this workflow depend on each other, only for limiting peak amount of spawned workers
 
 jobs:
-  changes:
-    permissions:
-      pull-requests: read
-    uses: ./.github/workflows/reusable-check-changed-files.yml
+  #changes:
+  #  permissions:
+  #    pull-requests: read
+  #  uses: ./.github/workflows/reusable-check-changed-files.yml
 
   set-image:
-    needs: [ changes ]
-    if: needs.changes.outputs.rust || needs.changes.outputs.current-workflow
+    # needs: [ changes ]
+    # if: needs.changes.outputs.rust || needs.changes.outputs.current-workflow
     # GitHub Actions allows using 'env' in a container context.
     # However, env variables don't work for forks: https://github.com/orgs/community/discussions/44322
     # This workaround sets the container image for each job using 'set-image' job output.
     runs-on: ubuntu-latest
     outputs:
       IMAGE: ${{ steps.set_image.outputs.IMAGE }}
+      RUNNER: ${{ steps.set_runner.outputs.RUNNER }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
       - id: set_image
         run: cat .github/env >> $GITHUB_OUTPUT
-
+      # By default, we use spot machines that can be terminated at any time.
+      # Merge queues use persistent runners to avoid kicking off from queue when the runner is terminated.
+      - id: set_runner
+        run: |
+          # Run merge queues on persistent runners
+          if [[ $GITHUB_REF_NAME == *"gh-readonly-queue"* ]]; then
+            echo "RUNNER=arc-runners-polkadot-sdk-beefy-persistent" >> $GITHUB_OUTPUT
+          else
+            echo "RUNNER=arc-runners-polkadot-sdk-beefy" >> $GITHUB_OUTPUT
+          fi  
+  
   # more information about this job can be found here:
   # https://github.com/paritytech/substrate/pull/3778
   test-full-crypto-feature:
     needs: [ set-image ]
-    runs-on: arc-runners-polkadot-sdk-beefy
+    runs-on: ${{ needs.set-image.outputs.RUNNER }}
     timeout-minutes: 60
     container:
       image: ${{ needs.set-image.outputs.IMAGE }}
@@ -62,7 +73,7 @@ jobs:
     timeout-minutes: 20
     # into one job
     needs: [ set-image, test-full-crypto-feature ]
-    runs-on: arc-runners-polkadot-sdk-beefy
+    runs-on: ${{ needs.set-image.outputs.RUNNER }}
     container:
       image: ${{ needs.set-image.outputs.IMAGE }}
     env:
@@ -83,7 +94,7 @@ jobs:
   test-frame-ui:
     timeout-minutes: 60
     needs: [ set-image, test-frame-examples-compile-to-wasm ]
-    runs-on: arc-runners-polkadot-sdk-beefy
+    runs-on: ${{ needs.set-image.outputs.RUNNER }}
     container:
       image: ${{ needs.set-image.outputs.IMAGE }}
     env:
@@ -111,7 +122,7 @@ jobs:
   test-deterministic-wasm:
     timeout-minutes: 20
     needs: [ set-image, test-frame-ui ]
-    runs-on: arc-runners-polkadot-sdk-beefy
+    runs-on: ${{ needs.set-image.outputs.RUNNER }}
     container:
       image: ${{ needs.set-image.outputs.IMAGE }}
     env:
@@ -150,7 +161,7 @@ jobs:
     strategy:
       matrix:
         branch: ${{ fromJSON(needs.cargo-check-benches-branches.outputs.branch) }}
-    runs-on: arc-runners-polkadot-sdk-beefy
+    runs-on: ${{ needs.set-image.outputs.RUNNER }}
     container:
       image: ${{ needs.set-image.outputs.IMAGE }}
     steps:
@@ -211,7 +222,7 @@ jobs:
   test-node-metrics:
     needs: [ set-image ]
     timeout-minutes: 30
-    runs-on: arc-runners-polkadot-sdk-beefy
+    runs-on: ${{ needs.set-image.outputs.RUNNER }}
     container:
       image: ${{ needs.set-image.outputs.IMAGE }}
     steps:
@@ -243,7 +254,7 @@ jobs:
   check-tracing:
     timeout-minutes: 20
     needs: [ set-image, test-node-metrics ]
-    runs-on: arc-runners-polkadot-sdk-beefy
+    runs-on: ${{ needs.set-image.outputs.RUNNER }}
     container:
       image: ${{ needs.set-image.outputs.IMAGE }}
     steps:
@@ -258,7 +269,7 @@ jobs:
   check-metadata-hash:
     timeout-minutes: 20
     needs: [ set-image, check-tracing ]
-    runs-on: arc-runners-polkadot-sdk-beefy
+    runs-on: ${{ needs.set-image.outputs.RUNNER }}
     container:
       image: ${{ needs.set-image.outputs.IMAGE }}
     steps:
@@ -272,7 +283,7 @@ jobs:
   cargo-hfuzz:
     timeout-minutes: 20
     needs: [ set-image, check-metadata-hash ]
-    runs-on: arc-runners-polkadot-sdk-beefy
+    runs-on: ${{ needs.set-image.outputs.RUNNER }}
     container:
       image: ${{ needs.set-image.outputs.IMAGE }}
     env:
@@ -312,7 +323,7 @@ jobs:
   cargo-check-each-crate:
     timeout-minutes: 140
     needs: [ set-image ]
-    runs-on: arc-runners-polkadot-sdk-beefy
+    runs-on: ${{ needs.set-image.outputs.RUNNER }}
     container:
       image: ${{ needs.set-image.outputs.IMAGE }}
     env:

From 913dde9e45d9132b002c733612347726751da40f Mon Sep 17 00:00:00 2001
From: Maksym H <1177472+mordamax@users.noreply.github.com>
Date: Tue, 3 Sep 2024 16:35:10 +0100
Subject: [PATCH 5/8] Update .github/workflows/tests-misc.yml

---
 .github/workflows/tests-misc.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/tests-misc.yml b/.github/workflows/tests-misc.yml
index afa80b5e7a3a..4be312b13a71 100644
--- a/.github/workflows/tests-misc.yml
+++ b/.github/workflows/tests-misc.yml
@@ -190,7 +190,7 @@ jobs:
 
   node-bench-regression-guard:
     timeout-minutes: 20
-    runs-on: arc-runners-polkadot-sdk-default
+    runs-on: arc-runners-polkadot-sdk
     needs: [ set-image, cargo-check-benches ]
     steps:
       - name: Checkout

From 2a73604e9f24e3951a4b729559f1bac3d6db24d5 Mon Sep 17 00:00:00 2001
From: Maksym H <mordamax@gmail.com>
Date: Wed, 4 Sep 2024 17:16:37 +0100
Subject: [PATCH 6/8] Update tests-misc.yml

---
 .github/workflows/tests-misc.yml | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/tests-misc.yml b/.github/workflows/tests-misc.yml
index 4be312b13a71..c35b40d822ca 100644
--- a/.github/workflows/tests-misc.yml
+++ b/.github/workflows/tests-misc.yml
@@ -211,13 +211,20 @@ jobs:
       - name: script
         id: compare
         run: |
-          FAILED_MSG='### node-bench-regression-guard failed ❌, check the regression in *cargo-check-benches* job'
           docker run --rm \
           -v $PWD/artifacts/master:/artifacts/master \
           -v $PWD/artifacts/current:/artifacts/current \
           paritytech/node-bench-regression-guard:latest \
-          node-bench-regression-guard --reference /artifacts/master --compare-with /artifacts/current \
-          || echo $FAILED_MSG >> $GITHUB_STEP_SUMMARY && echo $FAILED_MSG && exit 1
+          node-bench-regression-guard --reference /artifacts/master --compare-with /artifacts/current
+
+          if [ $? -ne 0 ]; then
+            FAILED_MSG='### node-bench-regression-guard failed ❌, check the regression in *cargo-check-benches* job'
+            echo $FAILED_MSG
+            echo $FAILED_MSG >> $GITHUB_STEP_SUMMARY
+            exit 1
+          else
+            echo "### node-bench-regression-guard passed ✅" >> $GITHUB_STEP_SUMMARY
+          fi
 
   test-node-metrics:
     needs: [ set-image ]

From 6b1d7cef062ff8ff2426786be846d881a6635626 Mon Sep 17 00:00:00 2001
From: Maksym H <mordamax@gmail.com>
Date: Thu, 5 Sep 2024 12:48:13 +0100
Subject: [PATCH 7/8] hack for folklift

---
 .github/workflows/tests-misc.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/tests-misc.yml b/.github/workflows/tests-misc.yml
index c35b40d822ca..16bd07254600 100644
--- a/.github/workflows/tests-misc.yml
+++ b/.github/workflows/tests-misc.yml
@@ -350,6 +350,8 @@ jobs:
 
       - name: script
         run: |
+          mkdir -p /github/home/.forklift
+          cp .forklift/config.toml /github/home/.forklift/config.toml
           PYTHONUNBUFFERED=x .github/scripts/check-each-crate.py ${{ matrix.index }} ${{ strategy.job-total }}
 
   # TODO: enable when we have a macos Self-Hosted runners

From 0ed5b445c10d12899bbc278cb693ea7eb736019d Mon Sep 17 00:00:00 2001
From: Maksym H <mordamax@gmail.com>
Date: Thu, 5 Sep 2024 13:45:57 +0100
Subject: [PATCH 8/8] Update tests-misc.yml

---
 .github/workflows/tests-misc.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/tests-misc.yml b/.github/workflows/tests-misc.yml
index 16bd07254600..824e8c11c2a4 100644
--- a/.github/workflows/tests-misc.yml
+++ b/.github/workflows/tests-misc.yml
@@ -395,7 +395,7 @@ jobs:
   #        run: |
   #          PYTHONUNBUFFERED=x .github/scripts/check-each-crate.py ${{ matrix.index }} ${{ strategy.job-total }} True
 
-  confirm-required-jobs-passed:
+  confirm-required-test-misc-jobs-passed:
     runs-on: ubuntu-latest
     name: All test misc tests passed
     # If any new job gets added, be sure to add it to this array
@@ -407,7 +407,7 @@ jobs:
       - node-bench-regression-guard
       - test-node-metrics
       - check-tracing
-      - cargo-hfuzz
       - cargo-check-each-crate
+      # - cargo-hfuzz remove from required for now, as it's flaky
     steps:
       - run: echo '### Good job! All the required tests passed 🚀' >> $GITHUB_STEP_SUMMARY
\ No newline at end of file