diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 594c9d1dded4f..9c1e7025bdebf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -100,176 +100,6 @@ skip-if-draft: #### stage: check -check-runtime: - stage: check - image: paritytech/tools:latest - <<: *kubernetes-build - only: - - /^[0-9]+$/ - variables: - <<: *default-vars - GITLAB_API: "https://gitlab.parity.io/api/v4" - GITHUB_API_PROJECT: "parity%2Finfrastructure%2Fgithub-api" - script: - - ./.maintain/gitlab/check_runtime.sh - allow_failure: true - -check-signed-tag: - stage: check - image: paritytech/tools:latest - <<: *kubernetes-build - only: - - /^ci-release-.*$/ - - /^v[0-9]+\.[0-9]+\.[0-9]+.*$/ - script: - - ./.maintain/gitlab/check_signed.sh - -check-line-width: - stage: check - image: paritytech/tools:latest - <<: *kubernetes-build - only: - - /^[0-9]+$/ - script: - - ./.maintain/gitlab/check_line_width.sh - allow_failure: true - -test-dependency-rules: - stage: check - image: paritytech/tools:latest - <<: *kubernetes-build - except: - variables: - - $DEPLOY_TAG - script: - - .maintain/ensure-deps.sh - -#### stage: test - -cargo-audit: - stage: test - <<: *docker-env - except: - - /^[0-9]+$/ - script: - - cargo audit - allow_failure: true - -cargo-deny: - stage: test - <<: *docker-env - script: - - cargo deny check --hide-inclusion-graph -c .maintain/deny.toml - after_script: - - echo "___The complete log is in the artifacts___" - - cargo deny check -c .maintain/deny.toml 2> deny.log - artifacts: - name: $CI_COMMIT_SHORT_SHA - expire_in: 3 days - when: always - paths: - - deny.log - -cargo-check-benches: - stage: test - <<: *docker-env - script: - - BUILD_DUMMY_WASM_BINARY=1 time cargo +nightly check --benches --all - - cargo run --release -p node-bench -- ::node::import::native::sr25519::transfer_keep_alive::paritydb::small - - cargo run --release -p node-bench -- ::trie::read::small - - sccache -s - -cargo-check-subkey: - stage: test - <<: *docker-env - except: - - /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 - script: - - cd ./bin/utils/subkey - - BUILD_DUMMY_WASM_BINARY=1 time cargo check --release - - sccache -s - -test-linux-stable: &test-linux - stage: test - <<: *docker-env - variables: - <<: *default-vars - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" - RUST_BACKTRACE: 1 - except: - variables: - - $DEPLOY_TAG - script: - - WASM_BUILD_NO_COLOR=1 time cargo test --all --release --verbose --locked - - sccache -s - -unleash-check: - stage: test - <<: *docker-env - only: - - master - - tags - script: - - cargo install cargo-unleash ${CARGO_UNLEASH_INSTALL_PARAMS} - - cargo unleash check ${CARGO_UNLEASH_PKG_DEF} - -test-frame-staking: - # into one job - stage: test - <<: *docker-env - variables: - <<: *default-vars - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: -Cdebug-assertions=y - RUST_BACKTRACE: 1 - except: - variables: - - $DEPLOY_TAG - script: - - cd frame/staking/ - - WASM_BUILD_NO_COLOR=1 time cargo test --release --verbose --no-default-features --features "std" - - sccache -s - -test-frame-examples-compile-to-wasm: - # into one job - stage: test - <<: *docker-env - variables: - <<: *default-vars - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: -Cdebug-assertions=y - RUST_BACKTRACE: 1 - except: - variables: - - $DEPLOY_TAG - script: - - cd frame/example-offchain-worker/ - - cargo +nightly build --target=wasm32-unknown-unknown --no-default-features - - cd ../example - - cargo +nightly build --target=wasm32-unknown-unknown --no-default-features - - sccache -s - -test-wasmtime: - stage: test - <<: *docker-env - variables: - <<: *default-vars - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: -Cdebug-assertions=y - RUST_BACKTRACE: 1 - except: - variables: - - $DEPLOY_TAG - script: - - cd client/executor - - WASM_BUILD_NO_COLOR=1 time cargo test --release --verbose --features wasmtime - - sccache -s - test-runtime-benchmarks: # into one job stage: test @@ -287,457 +117,3 @@ test-runtime-benchmarks: - cd bin/node/cli - WASM_BUILD_NO_COLOR=1 time cargo test --release --verbose --features runtime-benchmarks - sccache -s - -test-linux-stable-int: - <<: *test-linux - except: - refs: - - /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 - variables: - - $DEPLOY_TAG - script: - - echo "___Logs will be partly shown at the end in case of failure.___" - - echo "___Full log will be saved to the job artifacts only in case of failure.___" - - WASM_BUILD_NO_COLOR=1 - RUST_LOG=sync=trace,consensus=trace,client=trace,state-db=trace,db=trace,forks=trace,state_db=trace,storage_cache=trace - time cargo test -p node-cli --release --verbose --locked -- --ignored - &> ${CI_COMMIT_SHORT_SHA}_int_failure.log - - sccache -s - after_script: - - awk '/FAILED|^error\[/,0' ${CI_COMMIT_SHORT_SHA}_int_failure.log - artifacts: - name: $CI_COMMIT_SHORT_SHA - when: on_failure - expire_in: 3 days - paths: - - ${CI_COMMIT_SHORT_SHA}_int_failure.log - -check-web-wasm: - stage: test - <<: *docker-env - except: - - /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 - script: - # WASM support is in progress. As more and more crates support WASM, we - # should add entries here. See https://github.com/paritytech/substrate/issues/2416 - - time cargo build --target=wasm32-unknown-unknown -p sp-io - - time cargo build --target=wasm32-unknown-unknown -p sp-runtime - - time cargo build --target=wasm32-unknown-unknown -p sp-std - - time cargo build --target=wasm32-unknown-unknown -p sc-consensus-aura - - time cargo build --target=wasm32-unknown-unknown -p sc-consensus-babe - - time cargo build --target=wasm32-unknown-unknown -p sp-consensus - - time cargo build --target=wasm32-unknown-unknown -p sc-telemetry - # Note: the command below is a bit weird because several Cargo issues prevent us from compiling the node in a more straight-forward way. - - time cargo +nightly build --manifest-path=bin/node/cli/Cargo.toml --no-default-features --features browser --target=wasm32-unknown-unknown -Z features=itarget - - sccache -s - -test-full-crypto-feature: - stage: test - <<: *docker-env - variables: - <<: *default-vars - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: -Cdebug-assertions=y - RUST_BACKTRACE: 1 - except: - variables: - - $DEPLOY_TAG - script: - - cd primitives/core/ - - time cargo +nightly build --verbose --no-default-features --features full_crypto - - cd ../application-crypto - - time cargo +nightly build --verbose --no-default-features --features full_crypto - - sccache -s - -cargo-check-macos: - stage: test - # shell runner on mac ignores the image set in *docker-env - <<: *docker-env - script: - - BUILD_DUMMY_WASM_BINARY=1 time cargo check --release - - sccache -s - tags: - - osx - -test-prometheus-alerting-rules: - stage: test - image: paritytech/tools:latest - <<: *kubernetes-build - script: - - promtool check rules .maintain/monitoring/alerting-rules/alerting-rules.yaml - - cat .maintain/monitoring/alerting-rules/alerting-rules.yaml | promtool test rules .maintain/monitoring/alerting-rules/alerting-rule-tests.yaml - -#### stage: build - -check-polkadot-companion-status: - stage: build - image: paritytech/tools:latest - <<: *kubernetes-build - only: - - /^[0-9]+$/ # PRs - script: - - ./.maintain/gitlab/check_polkadot_companion_status.sh - -check-polkadot-companion-build: - stage: build - <<: *docker-env - needs: - - job: test-linux-stable-int - artifacts: false - script: - - ./.maintain/gitlab/check_polkadot_companion_build.sh - allow_failure: true - -test-browser-node: - stage: build - <<: *docker-env - needs: - - job: check-web-wasm - artifacts: false - variables: - <<: *default-vars - CHROMEDRIVER_ARGS: "--log-level=INFO --whitelisted-ips=127.0.0.1" - CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER: "wasm-bindgen-test-runner" - WASM_BINDGEN_TEST_TIMEOUT: 120 - script: - - cargo +nightly test --target wasm32-unknown-unknown -p node-browser-testing -Z features=itarget - -build-linux-substrate: &build-binary - stage: build - <<: *collect-artifacts - <<: *docker-env - <<: *build-only - needs: - - job: test-linux-stable - artifacts: false - before_script: - - mkdir -p ./artifacts/substrate/ - except: - variables: - - $DEPLOY_TAG - script: - - WASM_BUILD_NO_COLOR=1 time cargo build --release --verbose - - mv ./target/release/substrate ./artifacts/substrate/. - - echo -n "Substrate version = " - - if [ "${CI_COMMIT_TAG}" ]; then - echo "${CI_COMMIT_TAG}" | tee ./artifacts/substrate/VERSION; - else - ./artifacts/substrate/substrate --version | - sed -n -E 's/^substrate ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p' | - tee ./artifacts/substrate/VERSION; - fi - - sha256sum ./artifacts/substrate/substrate | tee ./artifacts/substrate/substrate.sha256 - - printf '\n# building node-template\n\n' - - ./.maintain/node-template-release.sh ./artifacts/substrate/substrate-node-template.tar.gz - - cp -r .maintain/docker/substrate.Dockerfile ./artifacts/substrate/ - - sccache -s - - -build-linux-subkey: &build-subkey - <<: *build-binary - needs: - - job: cargo-check-subkey - artifacts: false - before_script: - - mkdir -p ./artifacts/subkey - script: - - cd ./bin/utils/subkey - - BUILD_DUMMY_WASM_BINARY=1 time cargo build --release --verbose - - cd - - - mv ./target/release/subkey ./artifacts/subkey/. - - echo -n "Subkey version = " - - ./artifacts/subkey/subkey --version | - sed -n -E 's/^subkey ([0-9.]+.*)/\1/p' | - tee ./artifacts/subkey/VERSION; - - sha256sum ./artifacts/subkey/subkey | tee ./artifacts/subkey/subkey.sha256 - - cp -r .maintain/docker/subkey.Dockerfile ./artifacts/subkey/ - - sccache -s - -build-macos-subkey: - <<: *build-subkey - only: - - master - - /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 - tags: - - osx - -build-rust-doc-release: - stage: build - <<: *docker-env - allow_failure: true - artifacts: - name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc" - when: on_success - expire_in: 7 days - paths: - - ./crate-docs - <<: *build-only - script: - - rm -f ./crate-docs/index.html # use it as an indicator if the job succeeds - - BUILD_DUMMY_WASM_BINARY=1 RUSTDOCFLAGS="--html-in-header $(pwd)/.maintain/rustdoc-header.html" - time cargo +nightly doc --release --all --verbose - - cp -R ./target/doc ./crate-docs - - echo "" > ./crate-docs/index.html - - sccache -s - -#### stage: post-build-test - -trigger-contracts-ci: - stage: post-build-test - needs: - - job: build-linux-substrate - artifacts: false - - job: test-linux-stable - artifacts: false - trigger: - project: parity/srml-contracts-waterfall - branch: master - strategy: depend - only: - - master - - schedules - -#### stage: publish - -.build-push-docker-image: &build-push-docker-image - <<: *build-only - <<: *kubernetes-build - image: docker:stable - services: - - docker:dind - variables: &docker-build-vars - <<: *default-vars - DOCKER_HOST: tcp://localhost:2375 - DOCKER_DRIVER: overlay2 - GIT_STRATEGY: none - DOCKERFILE: $PRODUCT.Dockerfile - CONTAINER_IMAGE: parity/$PRODUCT - before_script: - - test "$Docker_Hub_User_Parity" -a "$Docker_Hub_Pass_Parity" - || ( echo "no docker credentials provided"; exit 1 ) - - docker login -u "$Docker_Hub_User_Parity" -p "$Docker_Hub_Pass_Parity" - - docker info - script: - - cd ./artifacts/$PRODUCT/ - - VERSION="$(cat ./VERSION)" - - echo "${PRODUCT} version = ${VERSION}" - - test -z "${VERSION}" && exit 1 - - docker build - --build-arg VCS_REF="${CI_COMMIT_SHA}" - --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" - --tag $CONTAINER_IMAGE:$VERSION - --tag $CONTAINER_IMAGE:latest - --file $DOCKERFILE . - - docker push $CONTAINER_IMAGE:$VERSION - - docker push $CONTAINER_IMAGE:latest - -publish-docker-substrate: - stage: publish - <<: *build-push-docker-image - # collect VERSION artifact here to pass it on to kubernetes - <<: *collect-artifacts - needs: - - job: build-linux-substrate - artifacts: true - variables: - <<: *docker-build-vars - PRODUCT: substrate - after_script: - - docker logout - # only VERSION information is needed for the deployment - - find ./artifacts/ -depth -not -name VERSION -type f -delete - -publish-docker-subkey: - stage: publish - <<: *build-push-docker-image - needs: - - job: build-linux-subkey - artifacts: true - variables: - <<: *docker-build-vars - PRODUCT: subkey - after_script: - - docker logout - -publish-s3-release: - stage: publish - <<: *build-only - <<: *kubernetes-build - needs: - - job: build-linux-substrate - artifacts: true - - job: build-linux-subkey - artifacts: true - image: paritytech/awscli:latest - variables: - GIT_STRATEGY: none - BUCKET: "releases.parity.io" - PREFIX: "substrate/${ARCH}-${DOCKER_OS}" - script: - - aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/substrate/VERSION)/ - - echo "update objects in latest path" - - aws s3 sync s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/substrate/VERSION)/ s3://${BUCKET}/${PREFIX}/latest/ - after_script: - - aws s3 ls s3://${BUCKET}/${PREFIX}/latest/ - --recursive --human-readable --summarize - - -publish-s3-doc: - stage: publish - image: paritytech/awscli:latest - allow_failure: true - needs: - - job: build-rust-doc-release - artifacts: true - <<: *build-only - <<: *kubernetes-build - variables: - GIT_STRATEGY: none - BUCKET: "releases.parity.io" - PREFIX: "substrate-rustdoc" - script: - - test -r ./crate-docs/index.html || ( - echo "./crate-docs/index.html not present, build:rust:doc:release job not complete"; - exit 1 - ) - - aws s3 sync --delete --size-only --only-show-errors - ./crate-docs/ s3://${BUCKET}/${PREFIX}/ - after_script: - - aws s3 ls s3://${BUCKET}/${PREFIX}/ - --human-readable --summarize - -publish-draft-release: - stage: publish - image: paritytech/tools:latest - only: - - /^ci-release-.*$/ - - /^v[0-9]+\.[0-9]+\.[0-9]+.*$/ - script: - - ./.maintain/gitlab/publish_draft_release.sh - allow_failure: true - -publish-to-crates-io: - stage: publish - <<: *docker-env - only: - - /^ci-release-.*$/ - - /^v[0-9]+\.[0-9]+\.[0-9]+.*$/ - script: - - cargo install cargo-unleash ${CARGO_UNLEASH_INSTALL_PARAMS} - - cargo unleash em-dragons --no-check ${CARGO_UNLEASH_PKG_DEF} - allow_failure: true - -.deploy-template: &deploy - stage: kubernetes - when: manual - retry: 1 - image: paritytech/kubetools:latest - <<: *build-only - tags: - # this is the runner that is used to deploy it - - kubernetes-parity-build - before_script: - - test -z "${DEPLOY_TAG}" && - test -f ./artifacts/substrate/VERSION && - DEPLOY_TAG="$(cat ./artifacts/substrate/VERSION)" - - test "${DEPLOY_TAG}" || ( echo "Neither DEPLOY_TAG nor VERSION information available"; exit 1 ) - script: - - echo "Substrate version = ${DEPLOY_TAG}" - # or use helm to render the template - - helm template - --values ./.maintain/kubernetes/values.yaml - --set image.tag=${DEPLOY_TAG} - --set validator.keys=${VALIDATOR_KEYS} - ./.maintain/kubernetes | kubectl apply -f - --dry-run=false - - echo "# substrate namespace ${KUBE_NAMESPACE}" - - kubectl -n ${KUBE_NAMESPACE} get all - - echo "# substrate's nodes' external ip addresses:" - - kubectl get nodes -l node=substrate - -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{range @.status.addresses[?(@.type=="ExternalIP")]}{.address}{"\n"}{end}' - - echo "# substrate' nodes" - - kubectl -n ${KUBE_NAMESPACE} get pods - -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.nodeName}{"\n"}{end}' - - echo "# wait for the rollout to complete" - - kubectl -n ${KUBE_NAMESPACE} rollout status statefulset/substrate - -# have environment:url eventually point to the logs - -.deploy-cibuild: &deploy-cibuild - <<: *deploy - needs: - - job: publish-docker-substrate - artifacts: false - -.deploy-tag: &deploy-tag - <<: *deploy - only: - variables: - - $DEPLOY_TAG - -# have environment:url eventually point to the logs - -deploy-ew3: - <<: *deploy-cibuild - environment: - name: parity-prod-ew3 - -deploy-ue1: - <<: *deploy-cibuild - environment: - name: parity-prod-ue1 - -deploy-ew3-tag: - <<: *deploy-tag - environment: - name: parity-prod-ew3 - -deploy-ue1-tag: - <<: *deploy-tag - environment: - name: parity-prod-ue1 - -.validator-deploy: &validator-deploy - <<: *build-only - stage: flaming-fir - needs: - # script will fail if there is no artifacts/substrate/VERSION - - job: publish-docker-substrate - artifacts: true - image: parity/azure-ansible:v1 - allow_failure: true - when: manual - interruptible: true - tags: - - linux-docker - -validator 1 4: - <<: *validator-deploy - script: - - ./.maintain/flamingfir-deploy.sh flamingfir-validator1 - -validator 2 4: - <<: *validator-deploy - script: - - ./.maintain/flamingfir-deploy.sh flamingfir-validator2 - -validator 3 4: - <<: *validator-deploy - script: - - ./.maintain/flamingfir-deploy.sh flamingfir-validator3 - -validator 4 4: - <<: *validator-deploy - script: - - ./.maintain/flamingfir-deploy.sh flamingfir-validator4 - -#### stage: .post - -check-labels: - stage: .post - image: paritytech/tools:latest - <<: *kubernetes-build - only: - - /^[0-9]+$/ - script: - - ./.maintain/gitlab/check_labels.sh