diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ae6a81611f2dd..d36fe2b57add7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -46,6 +46,7 @@ variables: &default-vars VAULT_AUTH_PATH: "gitlab-parity-io-jwt" VAULT_AUTH_ROLE: "cicd_gitlab_parity_${CI_PROJECT_NAME}" SIMNET_FEATURES_PATH: "simnet_tests/tests" + PIPELINE_SCRIPTS_TAG: "v0.1" default: cache: {} @@ -538,29 +539,37 @@ cargo-check-macos: #### stage: build -check-polkadot-companion-status: - stage: build - image: paritytech/tools:latest - <<: *kubernetes-env - <<: *vault-secrets - rules: - - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - script: - - ./.maintain/gitlab/check_polkadot_companion_status.sh - -check-polkadot-companion-build: +.check-dependent-project: &check-dependent-project stage: build <<: *docker-env <<: *test-refs-no-trigger <<: *vault-secrets - needs: - - job: test-linux-stable-int - artifacts: false script: - - ./.maintain/gitlab/check_polkadot_companion_build.sh - after_script: - - cd polkadot && git rev-parse --abbrev-ref HEAD - allow_failure: true + - git clone + --depth=1 + "--branch=$PIPELINE_SCRIPTS_TAG" + https://github.com/paritytech/pipeline-scripts + - ./pipeline-scripts/check_dependent_project.sh + paritytech + substrate + --substrate + "$DEPENDENT_REPO" + "$GITHUB_PR_TOKEN" + - cd "$DEPENDENT_REPO" && git rev-parse --abbrev-ref HEAD + +# Individual jobs are set up for each dependent project so that they can be ran in parallel. +# Arguably we could generate a job for each companion in the PR's description using Gitlab's +# parent-child pipelines but that's more complicated. + +check-dependent-polkadot: + <<: *check-dependent-project + variables: + DEPENDENT_REPO: polkadot + +check-dependent-cumulus: + <<: *check-dependent-project + variables: + DEPENDENT_REPO: cumulus build-linux-substrate: &build-binary stage: build @@ -884,7 +893,7 @@ deploy-prometheus-alerting-rules: # Runs "quick" and "long" tests on nightly schedule and on commit / merge to master # A "quick" test is a smoke test where basic check-expect tests run by # checking values from metrics exposed by the app. -# A "long" test is the load testing where we send 50K transactions into the +# A "long" test is the load testing where we send 50K transactions into the # network and check if all completed successfully simnet-tests: stage: deploy diff --git a/.maintain/gitlab/check_polkadot_companion_build.sh b/.maintain/gitlab/check_polkadot_companion_build.sh deleted file mode 100755 index 72bfaf7151522..0000000000000 --- a/.maintain/gitlab/check_polkadot_companion_build.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env sh -# -# check if a pr is compatible with polkadot companion pr or master if not -# available -# -# to override one that was just mentioned mark companion pr in the body of the -# polkadot pr like -# -# polkadot companion: paritytech/polkadot#567 -# - -set -e - -github_api_substrate_pull_url="https://api.github.com/repos/paritytech/substrate/pulls" -# use github api v3 in order to access the data without authentication -github_header="Authorization: token ${GITHUB_PR_TOKEN}" - -boldprint () { printf "|\n| \033[1m${@}\033[0m\n|\n" ; } -boldcat () { printf "|\n"; while read l; do printf "| \033[1m${l}\033[0m\n"; done; printf "|\n" ; } - - - -boldcat <<-EOT - - -check_polkadot_companion_build -============================== - -this job checks if there is a string in the description of the pr like - -polkadot companion: paritytech/polkadot#567 - - -it will then run cargo check from this polkadot's branch with substrate code -from this pull request. otherwise, it will uses master instead - - -EOT - -# Set the user name and email to make merging work -git config --global user.name 'CI system' -git config --global user.email '<>' - -# Merge master into our branch before building Polkadot to make sure we don't miss -# any commits that are required by Polkadot. -git fetch --depth 100 origin -git merge origin/master - -# Clone the current Polkadot master branch into ./polkadot. -# NOTE: we need to pull enough commits to be able to find a common -# ancestor for successfully performing merges below. -git clone --depth 20 https://github.com/paritytech/polkadot.git - -cd polkadot - -# either it's a pull request then check for a companion otherwise use -# polkadot:master -if expr match "${CI_COMMIT_REF_NAME}" '^[0-9]\+$' >/dev/null -then - boldprint "this is pull request no ${CI_COMMIT_REF_NAME}" - - pr_data_file="$(mktemp)" - # get the last reference to a pr in polkadot - curl -sSL -H "${github_header}" -o "${pr_data_file}" \ - "${github_api_substrate_pull_url}/${CI_COMMIT_REF_NAME}" - - pr_body="$(sed -n -r 's/^[[:space:]]+"body": (".*")[^"]+$/\1/p' "${pr_data_file}")" - - pr_companion="$(echo "${pr_body}" | sed -n -r \ - -e 's;^.*[Cc]ompanion.*paritytech/polkadot#([0-9]+).*$;\1;p' \ - -e 's;^.*[Cc]ompanion.*https://github.com/paritytech/polkadot/pull/([0-9]+).*$;\1;p' \ - | tail -n 1)" - - if [ "${pr_companion}" ] - then - boldprint "companion pr specified/detected: #${pr_companion}" - git fetch origin refs/pull/${pr_companion}/head:pr/${pr_companion} - git checkout pr/${pr_companion} - git merge origin/master - else - boldprint "no companion branch found - building polkadot:master" - fi - rm -f "${pr_data_file}" -else - boldprint "this is not a pull request - building polkadot:master" -fi - -# Patch all Substrate crates in Polkadot -diener patch --crates-to-patch ../ --substrate --path Cargo.toml - -# We need to update specifically our patched Substrate crates so that other -# crates that depend on them (e.g. Polkadot, BEEFY) use this unified version -# NOTE: There's no way to only update patched crates, so we use a heuristic -# of updating a crucial Substrate crate (`sp-core`) to minimize the impact of -# updating unrelated dependencies -cargo update -p sp-core - -# Test Polkadot pr or master branch with this Substrate commit. -time cargo test --workspace --release --verbose --features=runtime-benchmarks diff --git a/.maintain/gitlab/check_polkadot_companion_status.sh b/.maintain/gitlab/check_polkadot_companion_status.sh deleted file mode 100755 index e0412c7b7bec7..0000000000000 --- a/.maintain/gitlab/check_polkadot_companion_status.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/sh -# -# check for a polkadot companion pr and ensure it has approvals and is -# mergeable -# - -github_api_substrate_pull_url="https://api.github.com/repos/paritytech/substrate/pulls" -github_api_polkadot_pull_url="https://api.github.com/repos/paritytech/polkadot/pulls" -# use github api v3 in order to access the data without authentication -github_header="Authorization: token ${GITHUB_PR_TOKEN}" - -boldprint () { printf "|\n| \033[1m${@}\033[0m\n|\n" ; } -boldcat () { printf "|\n"; while read l; do printf "| \033[1m${l}\033[0m\n"; done; printf "|\n" ; } - - - -boldcat <<-EOT - - -check_polkadot_companion_status -=============================== - -this job checks if there is a string in the description of the pr like - -polkadot companion: paritytech/polkadot#567 - -and checks its status. - - -EOT - - -if ! [ "${CI_COMMIT_REF_NAME}" -gt 0 2>/dev/null ] -then - boldprint "this doesn't seem to be a pull request" - exit 1 -fi - -boldprint "this is pull request no ${CI_COMMIT_REF_NAME}" - -pr_body="$(curl -H "${github_header}" -s ${github_api_substrate_pull_url}/${CI_COMMIT_REF_NAME} \ - | sed -n -r 's/^[[:space:]]+"body": (".*")[^"]+$/\1/p')" - -# get companion if explicitly specified -pr_companion="$(echo "${pr_body}" | sed -n -r \ - -e 's;^.*[Cc]ompanion.*paritytech/polkadot#([0-9]+).*$;\1;p' \ - -e 's;^.*[Cc]ompanion.*https://github.com/paritytech/polkadot/pull/([0-9]+).*$;\1;p' \ - | tail -n 1)" - -if [ -z "${pr_companion}" ] -then - boldprint "no companion pr found" - exit 0 -fi - -boldprint "companion pr: #${pr_companion}" - -# check the status of that pull request - needs to be -# approved and mergable - -curl -H "${github_header}" -sS -o companion_pr.json \ - ${github_api_polkadot_pull_url}/${pr_companion} - -pr_head_sha=$(jq -r -e '.head.sha' < companion_pr.json) -boldprint "Polkadot PR's HEAD SHA: $pr_head_sha" - -curl -H "${github_header}" -sS -o companion_pr_reviews.json \ - ${github_api_polkadot_pull_url}/${pr_companion}/reviews - -# If there are any 'CHANGES_REQUESTED' reviews for the *current* review -jq -r -e '.[] | select(.state == "CHANGES_REQUESTED").commit_id' \ - < companion_pr_reviews.json > companion_pr_reviews_current.json -while IFS= read -r line; do - if [ "$line" = "$pr_head_sha" ]; then - boldprint "polkadot pr #${pr_companion} has CHANGES_REQUESTED for the latest commit" - exit 1 - fi -done < companion_pr_reviews_current.json - -# Then we check for at least 1 APPROVED -if [ -z "$(jq -r -e '.[].state | select(. == "APPROVED")' < companion_pr_reviews.json)" ]; then - boldprint "polkadot pr #${pr_companion} not APPROVED" - exit 1 -fi - -boldprint "polkadot pr #${pr_companion} state APPROVED" - -if jq -e .merged < companion_pr.json >/dev/null -then - boldprint "polkadot pr #${pr_companion} already merged" - exit 0 -fi - -if jq -e '.mergeable' < companion_pr.json >/dev/null -then - boldprint "polkadot pr #${pr_companion} mergeable" -else - boldprint "polkadot pr #${pr_companion} not mergeable" - exit 1 -fi - -exit 0