From 6e394f84f882f5493529a3d3ae99f9f498b41ef5 Mon Sep 17 00:00:00 2001 From: Alexander Samusev <41779041+alvicsam@users.noreply.github.com> Date: Mon, 28 Aug 2023 20:33:16 +0200 Subject: [PATCH] [ci] Add missing components to CI (#1201) * [ci] Add missing components to CI * add timestamp * add pipeline-stopper-artifacts to jobs * break clippy * mv definition * fix clippy * comment timestamp, rm verbose * commen cancel * rm --verbose from build jobs * disable rusty-cachier before_script * enable timestamp back * rollback .gitignore * rollback .gitignore * rollback .gitignore --- .gitlab-ci.yml | 190 ++++++++++++++++++++++++++++--------- .gitlab/pipeline/build.yml | 14 +-- .gitlab/pipeline/check.yml | 4 +- .gitlab/pipeline/test.yml | 16 ++-- 4 files changed, 161 insertions(+), 63 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 13a11b9bb4f3..b9b3cd2d63fb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,32 +1,9 @@ -# .gitlab-ci.yml +# polkadot-sdk | CI definitions (via GitLab CI) # -# substrate +# FYI: Pipelines can be triggered manually through the web UI (if you have enough permissions) # -# pipelines can be triggered manually in the web -# -# Currently the file is divided into subfiles. Each stage has a different file which -# can be found here: gitlab/pipeline/.yml -# -# Instead of YAML anchors "extends" is used. -# Useful links: -# https://docs.gitlab.com/ee/ci/yaml/index.html#extends -# https://docs.gitlab.com/ee/ci/yaml/yaml_optimization.html#reference-tags -# -# SAMPLE JOB TEMPLATE - This is not a complete example but is enough to build a -# simple CI job. For full documentation, visit https://docs.gitlab.com/ee/ci/yaml/ -# -# my-example-job: -# stage: test # One of the stages listed below this job (required) -# image: paritytech/tools:latest # Any docker image (required) -# allow_failure: true # Allow the pipeline to continue if this job fails (default: false) -# needs: -# - job: test-linux # Any jobs that are required to run before this job (optional) -# variables: -# MY_ENVIRONMENT_VARIABLE: "some useful value" # Environment variables passed to the job (optional) -# script: -# - echo "List of shell commands to run in your job" -# - echo "You can also just specify a script here, like so:" -# - ./gitlab/my_amazing_script.sh +# Currently, entire CI instructions are split into different subfiles. Each CI stage has a corresponding +# file which can be found here: .gitlab/pipeline/.yml stages: - check @@ -84,15 +61,18 @@ default: .prepare-env: before_script: - # TODO: remove unset invocation when we'll be free from 'ENV RUSTC_WRAPPER=sccache' & sccache - # itself in all images - - unset RUSTC_WRAPPER # $WASM_BUILD_WORKSPACE_HINT enables wasm-builder to find the Cargo.lock from within generated # packages - export WASM_BUILD_WORKSPACE_HINT="$PWD" # ensure that RUSTFLAGS are set correctly - echo $RUSTFLAGS +.common-before-script: + before_script: + - !reference [.job-switcher, before_script] + - !reference [.timestamp, before_script] + - !reference [.pipeline-stopper-vars, script] + .job-switcher: before_script: - if echo "$CI_DISABLED_JOBS" | grep -xF "$CI_JOB_NAME"; then echo "The job has been cancelled in CI settings"; exit 0; fi @@ -100,7 +80,7 @@ default: .kubernetes-env: image: "${CI_IMAGE}" before_script: - # - !reference [.job-switcher, before_script] + - !reference [.common-before-script, before_script] - !reference [.prepare-env, before_script] tags: - kubernetes-parity-build @@ -112,9 +92,10 @@ default: - rustup +nightly show - cargo +nightly --version +# collecting vars for pipeline stopper +# they will be used if the job fails .pipeline-stopper-vars: script: - - !reference [.job-switcher, before_script] - echo "Collecting env variables for the cancel-pipeline job" - echo "FAILED_JOB_URL=${CI_JOB_URL}" > pipeline-stopper.env - echo "FAILED_JOB_NAME=${CI_JOB_NAME}" >> pipeline-stopper.env @@ -128,28 +109,26 @@ default: .docker-env: image: "${CI_IMAGE}" before_script: - - !reference [.job-switcher, before_script] + - !reference [.common-before-script, before_script] - !reference [.prepare-env, before_script] - !reference [.rust-info-script, script] - !reference [.rusty-cachier, before_script] - - !reference [.pipeline-stopper-vars, script] - after_script: - - !reference [.rusty-cachier, after_script] tags: - linux-docker-vm-c2 # rusty-cachier's hidden job. Parts of this job are used to instrument the pipeline's other real jobs with rusty-cachier -# Description of the commands is available here - https://gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client#description +# rusty-cachier's commands are described here: https://gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client#description .rusty-cachier: before_script: # - curl -s https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client/-/raw/release/util/install.sh | bash - # - rusty-cachier environment check --gracefully - # - $(rusty-cachier environment inject) - # - rusty-cachier project mtime + # - mkdir -p cargo_home cargo_target_dir + # - export CARGO_HOME=$CI_PROJECT_DIR/cargo_home + # - export CARGO_TARGET_DIR=$CI_PROJECT_DIR/cargo_target_dir + # - find . \( -path ./cargo_target_dir -o -path ./cargo_home \) -prune -o -type f -exec touch -t 202005260100 {} + + # - git restore-mtime + # - rusty-cachier --version + # - rusty-cachier project touch-changed - echo tbd - after_script: - - echo tbd - # - env RUSTY_CACHIER_SUPRESS_OUTPUT=true rusty-cachier snapshot destroy .common-refs: rules: @@ -233,6 +212,125 @@ include: # zombienet jobs - .gitlab/pipeline/zombienet.yml # # timestamp handler - # - project: parity/infrastructure/ci_cd/shared - # ref: v0.2 - # file: /common/timestamp.yml + - project: parity/infrastructure/ci_cd/shared + ref: v0.2 + file: /common/timestamp.yml + +# This job cancels the whole pipeline if any of provided jobs fail. +# In a DAG, every jobs chain is executed independently of others. The `fail_fast` principle suggests +# to fail the pipeline as soon as possible to shorten the feedback loop. +.cancel-pipeline-template: + stage: .post + rules: + - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs + when: on_failure + variables: + PROJECT_ID: "${CI_PROJECT_ID}" + PROJECT_NAME: "${CI_PROJECT_NAME}" + PIPELINE_ID: "${CI_PIPELINE_ID}" + FAILED_JOB_URL: "${FAILED_JOB_URL}" + FAILED_JOB_NAME: "${FAILED_JOB_NAME}" + PR_NUM: "${PR_NUM}" + trigger: + project: "parity/infrastructure/ci_cd/pipeline-stopper" + branch: "as-improve" + +remove-cancel-pipeline-message: + stage: .post + rules: + - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs + variables: + PROJECT_ID: "${CI_PROJECT_ID}" + PROJECT_NAME: "${CI_PROJECT_NAME}" + PIPELINE_ID: "${CI_PIPELINE_ID}" + FAILED_JOB_URL: "https://gitlab.com" + FAILED_JOB_NAME: "nope" + PR_NUM: "${CI_COMMIT_REF_NAME}" + trigger: + project: "parity/infrastructure/ci_cd/pipeline-stopper" +# need to copy jobs this way because otherwise gitlab will wait +# for all 3 jobs to finish instead of cancelling if one fails +cancel-pipeline-test-linux-stable1: + extends: .cancel-pipeline-template + needs: + - job: "test-linux-stable 1/3" + +cancel-pipeline-test-linux-stable2: + extends: .cancel-pipeline-template + needs: + - job: "test-linux-stable 2/3" + +cancel-pipeline-test-linux-stable3: + extends: .cancel-pipeline-template + needs: + - job: "test-linux-stable 3/3" + +cancel-pipeline-test-linux-stable-additional-tests: + extends: .cancel-pipeline-template + needs: + - job: "test-linux-stable-additional-tests" + +cancel-pipeline-test-linux-stable-slow: + extends: .cancel-pipeline-template + needs: + - job: "test-linux-stable-slow" + +cancel-pipeline-cargo-check-benches1: + extends: .cancel-pipeline-template + needs: + - job: "cargo-check-benches 1/2" + +cancel-pipeline-cargo-check-benches2: + extends: .cancel-pipeline-template + needs: + - job: "cargo-check-benches 2/2" + +cancel-pipeline-test-linux-stable-int: + extends: .cancel-pipeline-template + needs: + - job: test-linux-stable-int + +cancel-pipeline-cargo-check-each-crate-1: + extends: .cancel-pipeline-template + needs: + - job: "cargo-check-each-crate 1/6" + +cancel-pipeline-cargo-check-each-crate-2: + extends: .cancel-pipeline-template + needs: + - job: "cargo-check-each-crate 2/6" + +cancel-pipeline-cargo-check-each-crate-3: + extends: .cancel-pipeline-template + needs: + - job: "cargo-check-each-crate 3/6" + +cancel-pipeline-cargo-check-each-crate-4: + extends: .cancel-pipeline-template + needs: + - job: "cargo-check-each-crate 4/6" + +cancel-pipeline-cargo-check-each-crate-5: + extends: .cancel-pipeline-template + needs: + - job: "cargo-check-each-crate 5/6" + +cancel-pipeline-cargo-check-each-crate-6: + extends: .cancel-pipeline-template + needs: + - job: "cargo-check-each-crate 6/6" + +cancel-pipeline-cargo-check-each-crate-macos: + extends: .cancel-pipeline-template + needs: + - job: cargo-check-each-crate-macos + +cancel-pipeline-check-tracing: + extends: .cancel-pipeline-template + needs: + - job: check-tracing + +cancel-pipeline-cargo-clippy: + extends: .cancel-pipeline-template + needs: + - job: cargo-clippy diff --git a/.gitlab/pipeline/build.yml b/.gitlab/pipeline/build.yml index 5782b4745e70..684f6dd5c252 100644 --- a/.gitlab/pipeline/build.yml +++ b/.gitlab/pipeline/build.yml @@ -46,8 +46,8 @@ build-test-collators: - .run-immediately - .collect-artifacts script: - - time cargo build --locked --profile testnet --verbose -p test-parachain-adder-collator - - time cargo build --locked --profile testnet --verbose -p test-parachain-undying-collator + - time cargo build --locked --profile testnet -p test-parachain-adder-collator + - time cargo build --locked --profile testnet -p test-parachain-undying-collator # pack artifacts - mkdir -p ./artifacts - mv ./target/testnet/adder-collator ./artifacts/. @@ -66,7 +66,7 @@ build-malus: - .run-immediately - .collect-artifacts script: - - time cargo build --locked --profile testnet --verbose -p polkadot-test-malus --bin malus --bin polkadot-prepare-worker --bin polkadot-execute-worker + - time cargo build --locked --profile testnet -p polkadot-test-malus --bin malus --bin polkadot-prepare-worker --bin polkadot-execute-worker # pack artifacts - mkdir -p ./artifacts - mv ./target/testnet/malus ./artifacts/. @@ -111,7 +111,7 @@ build-rustdoc: script: # FIXME: it fails with `RUSTDOCFLAGS="-Dwarnings"` and `--all-features` # FIXME: return to stable when https://github.com/rust-lang/rust/issues/96937 gets into stable - - time cargo doc --workspace --verbose --no-deps + - time cargo doc --workspace --no-deps - rm -f ./target/doc/.lock - mv ./target/doc ./crate-docs # FIXME: remove me after CI image gets nonroot @@ -280,7 +280,7 @@ build-linux-substrate: # see https://github.com/paritytech/ci_cd/issues/682#issuecomment-1340953589 - git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_SHA" script: - - WASM_BUILD_NO_COLOR=1 time cargo build --locked --release --verbose + - WASM_BUILD_NO_COLOR=1 time cargo build --locked --release - mv $CARGO_TARGET_DIR/release/substrate-node ./artifacts/substrate/substrate - echo -n "Substrate version = " - if [ "${CI_COMMIT_TAG}" ]; then @@ -308,7 +308,7 @@ build-linux-substrate: - mkdir -p ./artifacts/subkey script: - cd ./substrate/bin/utils/subkey - - SKIP_WASM_BUILD=1 time cargo build --locked --release --verbose + - SKIP_WASM_BUILD=1 time cargo build --locked --release # - cd - # - mv $CARGO_TARGET_DIR/release/subkey ./artifacts/subkey/. # - echo -n "Subkey version = " @@ -331,7 +331,7 @@ build-subkey-linux: # - mkdir -p ./artifacts/subkey # script: # - cd ./bin/utils/subkey -# - SKIP_WASM_BUILD=1 time cargo build --locked --release --verbose +# - SKIP_WASM_BUILD=1 time cargo build --locked --release # - cd - # - mv ./target/release/subkey ./artifacts/subkey/. # - echo -n "Subkey version = " diff --git a/.gitlab/pipeline/check.yml b/.gitlab/pipeline/check.yml index 123e02a8cd1c..15f110279acd 100644 --- a/.gitlab/pipeline/check.yml +++ b/.gitlab/pipeline/check.yml @@ -3,10 +3,9 @@ cargo-clippy: extends: - .docker-env - .common-refs + - .pipeline-stopper-artifacts script: - SKIP_WASM_BUILD=1 env -u RUSTFLAGS cargo clippy --all-targets --locked --workspace - # fixme! - allow_failure: true check-try-runtime: stage: check @@ -43,7 +42,6 @@ cargo-deny-licenses: script: - $CARGO_DENY_CMD --hide-inclusion-graph after_script: - # - !reference [.rusty-cachier, after_script] - echo "___The complete log is in the artifacts___" - $CARGO_DENY_CMD 2> deny.log - if [ $CI_JOB_STATUS != 'success' ]; then diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index ca8da4ea7566..cbba5dfe422c 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -11,6 +11,7 @@ test-linux-stable: - .docker-env - .common-refs - .run-immediately + - .pipeline-stopper-artifacts variables: RUST_TOOLCHAIN: stable # Enable debug assertions since we are running optimized builds for testing @@ -32,7 +33,6 @@ test-linux-stable: --workspace \ --locked \ --release \ - --verbose \ --no-fail-fast \ --features runtime-benchmarks,try-runtime,experimental \ --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL} @@ -66,7 +66,6 @@ test-linux-oldkernel-stable: # --workspace \ # --locked \ # --release \ -# --verbose \ # --no-fail-fast \ # --features runtime-benchmarks,try-runtime \ # --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL} @@ -79,6 +78,7 @@ test-linux-stable-additional-tests: - .docker-env - .common-refs - .run-immediately + - .pipeline-stopper-artifacts variables: RUST_TOOLCHAIN: stable # Enable debug assertions since we are running optimized builds for testing @@ -91,7 +91,6 @@ test-linux-stable-additional-tests: --workspace \ --locked \ --release \ - --verbose \ --features runtime-benchmarks,try-runtime allow_failure: true @@ -104,6 +103,7 @@ test-linux-stable-slow: - .docker-env - .common-refs - .run-immediately + - .pipeline-stopper-artifacts variables: RUST_TOOLCHAIN: stable # Enable debug assertions since we are running optimized builds for testing @@ -116,7 +116,6 @@ test-linux-stable-slow: --workspace \ --locked \ --release \ - --verbose \ --features runtime-benchmarks,try-runtime allow_failure: true @@ -166,7 +165,7 @@ test-rustdoc: SKIP_WASM_BUILD: 1 RUSTDOCFLAGS: "-Dwarnings" script: - - time cargo doc --workspace --all-features --verbose --no-deps + - time cargo doc --workspace --all-features --no-deps allow_failure: true cargo-check-all-benches: @@ -216,6 +215,7 @@ cargo-check-benches: - .common-refs - .run-immediately - .collect-artifacts + - .pipeline-stopper-artifacts before_script: # TODO: DON'T FORGET TO CHANGE FOR PROD VALUES!!! # merges in the master branch on PRs. skip if base is not master @@ -348,6 +348,7 @@ test-linux-stable-int: - .docker-env - .common-refs - .run-immediately + - .pipeline-stopper-artifacts variables: # Enable debug assertions since we are running optimized builds for testing # but still want to have debug assertions. @@ -370,6 +371,7 @@ check-tracing: - .docker-env - .common-refs - .run-immediately + - .pipeline-stopper-artifacts script: # with-tracing must be explicitly activated, we run a test to ensure this works as expected in both cases - time cargo test --locked --manifest-path ./substrate/primitives/tracing/Cargo.toml --no-default-features @@ -390,9 +392,9 @@ test-full-crypto-feature: RUST_BACKTRACE: 1 script: - cd substrate/primitives/core/ - - time cargo build --locked --verbose --no-default-features --features full_crypto + - time cargo build --locked --no-default-features --features full_crypto - cd ../application-crypto - - time cargo build --locked --verbose --no-default-features --features full_crypto + - time cargo build --locked --no-default-features --features full_crypto cargo-check-each-crate: stage: test