diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index f203fb286c7..aec9b5d8b63 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -6,6 +6,7 @@ export MOJAVE_ANKA_TAG_BASE=${MOJAVE_ANKA_TAG_BASE:-'clean::cicd::git-ssh::nas:: export MOJAVE_ANKA_TEMPLATE_NAME=${MOJAVE_ANKA_TEMPLATE_NAME:-'10.14.6_6C_14G_40G'} export PLATFORMS_JSON_ARRAY='[]' [[ -z "$ROUNDS" ]] && export ROUNDS='1' +DISABLE_CONCURRENCY=${DISABLE_CONCURRENCY:-false} LINUX_CONCURRENCY='8' MAC_CONCURRENCY='2' LINUX_CONCURRENCY_GROUP='eos-scheduled-build' @@ -134,7 +135,9 @@ EOF - 'registry_2' pre-execute-sleep: 10 pre-commands: - - "git clone git@github.com:EOSIO/mac-anka-fleet.git && cd mac-anka-fleet && . ./ensure-tag.bash -u 12 -r 25G -a '-n'" + - "rm -rf mac-anka-fleet; git clone git@github.com:EOSIO/mac-anka-fleet.git && cd mac-anka-fleet && . ./ensure-tag.bash -u 12 -r 25G -a '-n'" + - thedyrt/skip-checkout#v0.1.1: + cd: ~ env: REPO: ${BUILDKITE_PULL_REQUEST_REPO:-$BUILDKITE_REPO} REPO_COMMIT: $BUILDKITE_COMMIT @@ -149,7 +152,7 @@ EOF skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}${SKIP_BUILD} EOF fi - if [ "$BUILDKITE_SOURCE" = "schedule" ]; then + if [ "$BUILDKITE_SOURCE" = "schedule" ] && [[ $DISABLE_CONCURRENCY != true ]]; then cat <> ~/.ssh/known_hosts" + - "git clone \$BUILDKITE_REPO ." + - "$GIT_FETCH git checkout -f \$BUILDKITE_COMMIT" - "buildkite-agent artifact download build.tar.gz . --step '$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - Build' && tar -xzf build.tar.gz" - "./.cicd/test.sh scripts/serial-test.sh $TEST_NAME" + plugins: + - thedyrt/skip-checkout#v0.1.1: + cd: ~ env: IMAGE_TAG: $(echo "$PLATFORM_JSON" | jq -r .FILE_NAME) PLATFORM_TYPE: $PLATFORM_TYPE @@ -321,12 +334,14 @@ EOF - 'registry_1' - 'registry_2' pre-execute-sleep: 10 + - thedyrt/skip-checkout#v0.1.1: + cd: ~ timeout: ${TIMEOUT:-60} agents: "queue=mac-anka-node-fleet" skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}${SKIP_SERIAL_TESTS} EOF fi - if [ "$BUILDKITE_SOURCE" = "schedule" ]; then + if [ "$BUILDKITE_SOURCE" = "schedule" ] && [[ $DISABLE_CONCURRENCY != true ]]; then cat <> ~/.ssh/known_hosts" + - "git clone \$BUILDKITE_REPO ." + - "$GIT_FETCH git checkout -f \$BUILDKITE_COMMIT" - "buildkite-agent artifact download build.tar.gz . --step '$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - Build' ${BUILD_SOURCE} && tar -xzf build.tar.gz" - "./.cicd/test.sh scripts/long-running-test.sh $TEST_NAME" + plugins: + - thedyrt/skip-checkout#v0.1.1: + cd: ~ env: IMAGE_TAG: $(echo "$PLATFORM_JSON" | jq -r .FILE_NAME) PLATFORM_TYPE: $PLATFORM_TYPE @@ -381,12 +402,14 @@ EOF - 'registry_1' - 'registry_2' pre-execute-sleep: 10 + - thedyrt/skip-checkout#v0.1.1: + cd: ~ timeout: ${TIMEOUT:-180} agents: "queue=mac-anka-node-fleet" skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}${SKIP_LONG_RUNNING_TESTS:-true} EOF fi - if [ "$BUILDKITE_SOURCE" = "schedule" ]; then + if [ "$BUILDKITE_SOURCE" = "schedule" ] && [[ $DISABLE_CONCURRENCY != true ]]; then cat <> ~/.ssh/known_hosts" + - "git clone \$BUILDKITE_REPO ." + - "$GIT_FETCH git checkout -f \$BUILDKITE_COMMIT" + - "echo '+++ :compression: Extracting Test Metrics Code'" + - "tar -zxf .cicd/metrics/test-metrics.tar.gz" + - "echo '+++ :javascript: Running test-metrics.js'" + - "node --max-old-space-size=32768 test-metrics.js" + plugins: + - thedyrt/skip-checkout#v0.1.1: + cd: ~ agents: queue: "$BUILDKITE_TEST_AGENT_QUEUE" timeout: ${TIMEOUT:-10} @@ -464,8 +493,14 @@ cat <> ~/.ssh/known_hosts" + - "git clone \$BUILDKITE_REPO ." + - "$GIT_FETCH git checkout -f \$BUILDKITE_COMMIT" - "buildkite-agent artifact download build.tar.gz . --step ':centos: CentOS 7.6 - Build' && tar -xzf build.tar.gz" - "./.cicd/package.sh" + plugins: + - thedyrt/skip-checkout#v0.1.1: + cd: ~ env: IMAGE_TAG: "centos-7.6-$PLATFORM_TYPE" PLATFORM_TYPE: $PLATFORM_TYPE @@ -478,8 +513,14 @@ cat <> ~/.ssh/known_hosts" + - "git clone \$BUILDKITE_REPO ." + - "$GIT_FETCH git checkout -f \$BUILDKITE_COMMIT" - "buildkite-agent artifact download build.tar.gz . --step ':ubuntu: Ubuntu 16.04 - Build' && tar -xzf build.tar.gz" - "./.cicd/package.sh" + plugins: + - thedyrt/skip-checkout#v0.1.1: + cd: ~ env: IMAGE_TAG: "ubuntu-16.04-$PLATFORM_TYPE" PLATFORM_TYPE: $PLATFORM_TYPE @@ -492,8 +533,14 @@ cat <> ~/.ssh/known_hosts" + - "git clone \$BUILDKITE_REPO ." + - "$GIT_FETCH git checkout -f \$BUILDKITE_COMMIT" - "buildkite-agent artifact download build.tar.gz . --step ':ubuntu: Ubuntu 18.04 - Build' && tar -xzf build.tar.gz" - "./.cicd/package.sh" + plugins: + - thedyrt/skip-checkout#v0.1.1: + cd: ~ env: IMAGE_TAG: "ubuntu-18.04-$PLATFORM_TYPE" PLATFORM_TYPE: $PLATFORM_TYPE @@ -522,6 +569,8 @@ cat <> ~/.ssh/known_hosts" + - "git clone \$BUILDKITE_REPO ." + - "$GIT_FETCH git checkout -f \$BUILDKITE_COMMIT" + - "buildkite-agent artifact download eosio.rb . --step ':darwin: macOS 10.14 - Package Builder'" + - "buildkite-agent artifact upload eosio.rb" + plugins: + - thedyrt/skip-checkout#v0.1.1: + cd: ~ agents: queue: "automation-basic-builder-fleet" timeout: "${TIMEOUT:-5}" diff --git a/.cicd/pipeline.yml b/.cicd/pipeline.yml index 900aea633e0..e1460f9c7de 100644 --- a/.cicd/pipeline.yml +++ b/.cicd/pipeline.yml @@ -3,9 +3,16 @@ steps: - label: ":pipeline: Generate Pipeline Steps" command: + - "ssh-keyscan -H github.com >> ~/.ssh/known_hosts" + - "git clone $BUILDKITE_REPO ." + - "[[ $BUILDKITE_BRANCH =~ ^pull/[0-9]+/head: ]] && git fetch -v --prune origin refs/pull/$(echo $BUILDKITE_BRANCH | cut -d/ -f2)/head" + - "git checkout -f $BUILDKITE_COMMIT" - "./.cicd/generate-pipeline.sh > generated-pipeline.yml" - "buildkite-agent pipeline upload < generated-pipeline.yml" - "buildkite-agent artifact upload generated-pipeline.yml" + plugins: + - thedyrt/skip-checkout#v0.1.1: + cd: ~ agents: queue: "automation-basic-builder-fleet" timeout: ${TIMEOUT:-10} \ No newline at end of file