From cbcd448ea3169a20f2f245f6ef51e5a1f741fde8 Mon Sep 17 00:00:00 2001 From: Zak Fein Date: Sun, 21 Jul 2024 20:31:19 +0300 Subject: [PATCH] Clean old files, clean Zombie tests Makefile, change version numeric --- .github/workflows/zombie-tests.yaml | 4 +- housekeeping/docker/kagome-dev/Makefile | 6 +- zombienet/docker/Makefile | 177 ++++++------------- zombienet/docker/Makefile.old | 129 -------------- zombienet/docker/builder.Dockerfile | 74 -------- zombienet/docker/polkadot_binary.Dockerfile | 34 ---- zombienet/docker/polkadot_builder.Dockerfile | 2 + zombienet/docker/tester.Dockerfile.old | 24 --- 8 files changed, 64 insertions(+), 386 deletions(-) delete mode 100644 zombienet/docker/Makefile.old delete mode 100644 zombienet/docker/builder.Dockerfile delete mode 100644 zombienet/docker/polkadot_binary.Dockerfile delete mode 100644 zombienet/docker/tester.Dockerfile.old diff --git a/.github/workflows/zombie-tests.yaml b/.github/workflows/zombie-tests.yaml index c0f338a6f7..18a8da8763 100644 --- a/.github/workflows/zombie-tests.yaml +++ b/.github/workflows/zombie-tests.yaml @@ -60,7 +60,7 @@ jobs: ARTIFACTS_REPO=$(grep '^ARTIFACTS_REPO ?=' Makefile | sed 's/ARTIFACTS_REPO ?= //') REGION=$(grep '^REGION ?=' Makefile | sed 's/REGION ?= //') SHORT_COMMIT_HASH=$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2) - PACKAGE_VERSION="$(date +'%d.%m.%y')-${SHORT_COMMIT_HASH}-${{ github.event.inputs.build_type }}" + PACKAGE_VERSION="$(date +'%y.%m.%d')-${SHORT_COMMIT_HASH}-${{ github.event.inputs.build_type }}" echo "PACKAGE_VERSION=${PACKAGE_VERSION}" echo "package_version=${PACKAGE_VERSION}" >> $GITHUB_ENV echo "short_commit_hash=${SHORT_COMMIT_HASH}" >> $GITHUB_ENV @@ -174,7 +174,7 @@ jobs: ARTIFACTS_REPO=$(grep '^ARTIFACTS_REPO ?=' Makefile | sed 's/ARTIFACTS_REPO ?= //') REGION=$(grep '^REGION ?=' Makefile | sed 's/REGION ?= //') SHORT_COMMIT_HASH=$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2) - PACKAGE_VERSION="$(date +'%d.%m.%y')-${SHORT_COMMIT_HASH}-${{ github.event.inputs.build_type }}" + PACKAGE_VERSION="$(date +'%y.%m.%d')-${SHORT_COMMIT_HASH}-${{ github.event.inputs.build_type }}" echo "PACKAGE_VERSION=${PACKAGE_VERSION}" echo "package_version=${PACKAGE_VERSION}" >> $GITHUB_ENV diff --git a/housekeeping/docker/kagome-dev/Makefile b/housekeeping/docker/kagome-dev/Makefile index 299d050d83..130721aff4 100644 --- a/housekeeping/docker/kagome-dev/Makefile +++ b/housekeeping/docker/kagome-dev/Makefile @@ -90,7 +90,7 @@ kagome_dev_docker_build: cp /opt/kagome/$(BUILD_DIR)/node/kagome /tmp/kagome/kagome && \ cd /opt/kagome/housekeeping/docker/kagome-dev && \ ./build_apt_package.sh \ - \"$$(date +'%d.%m.%y')-$${SHORT_COMMIT_HASH}-$(BUILD_TYPE)\" \ + \"$$(date +'%y.%m.%d')-$${SHORT_COMMIT_HASH}-$(BUILD_TYPE)\" \ $(PACKAGE_ARCHITECTURE) \ kagome-dev \ /tmp/kagome \ @@ -108,12 +108,12 @@ upload_apt_package: SHORT_COMMIT_HASH=$$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2); \ gcloud config set artifacts/repository $(ARTIFACTS_REPO); \ gcloud config set artifacts/location $(REGION); \ - gcloud artifacts apt upload $(ARTIFACTS_REPO) --source=./pkg/kagome-dev_$$(date +'%d.%m.%y')-$${SHORT_COMMIT_HASH}-$(BUILD_TYPE)_$(PACKAGE_ARCHITECTURE).deb + gcloud artifacts apt upload $(ARTIFACTS_REPO) --source=./pkg/kagome-dev_$$(date +'%y.%m.%d')-$${SHORT_COMMIT_HASH}-$(BUILD_TYPE)_$(PACKAGE_ARCHITECTURE).deb runtime_cache: CONTAINER_NAME=kagome_dev_runtime_cache_$$(openssl rand -hex 6); \ - RUNTIME_VERSION=$$(python3 get_wasmedge_version.py); \ + RUNTIME_VERSION=$$(python3 get_wasmedge_version.py)-$$(date +'%y.%m.%d'); \ DOCKER_EXEC_RESULT=0 ; \ echo "Runtime version: $$RUNTIME_VERSION"; \ docker run -d --name $$CONTAINER_NAME \ diff --git a/zombienet/docker/Makefile b/zombienet/docker/Makefile index cc58600191..34073bf3a2 100644 --- a/zombienet/docker/Makefile +++ b/zombienet/docker/Makefile @@ -1,61 +1,59 @@ -RUST_IMAGE=rust:1.79-slim-bookworm +# Shared Environment Variables # bookworm (Debian 12) MINIDEB_IMAGE=bitnami/minideb@sha256:6cc3baf349947d587a9cd4971e81ff3ffc0d17382f2b5b6de63d6542bff10c16 -SCCACHE_VERSION ?= 0.7.4 - -# empty for local usage +## DOCKER_REGISTRY_PATH empty for local usage +GOOGLE_APPLICATION_CREDENTIALS ?= DOCKER_REGISTRY_PATH ?= - PLATFORM ?= linux/amd64 ARCHITECTURE ?= amd64 -ARTIFACTS_REPO ?= kagome-apt -REGION ?= europe-north1 +# polkadot_builder Variables +RUST_IMAGE=rust:1.79-slim-bookworm +SCCACHE_VERSION ?= 0.7.4 -#USE_GCP_SCCACHE ?= true +# polkadot_binary Variables SCCACHE_GCS_BUCKET ?= -GOOGLE_APPLICATION_CREDENTIALS ?= -PROJECT_ID ?= - CARGO_PACKETS=-p test-parachain-adder-collator -p polkadot-test-malus -p test-parachain-undying-collator RESULT_BINARIES=./target/release/malus ./target/release/undying-collator ./target/release/adder-collator RESULT_BIN_NAMES=malus undying-collator adder-collator +POLKADOT_BINARY_DEPENDENCIES=libstdc++6, zlib1g, libgcc-s1, libc6 + +# upload_apt_package Variables +ARTIFACTS_REPO ?= kagome-apt +REGION ?= europe-north1 +# zombie_tester Variables +PROJECT_ID ?= POLKADOT_BINARY_PACKAGE_VERSION ?= -KAGOME_PACKAGE_VERSION ?= -export DOCKER_BUILDKIT=1 -# BUILDKIT_PROGRESS - auto, plain, tty, rawjson -export BUILDKIT_PROGRESS=auto +# copy_logs_to_host Variables +# COPY_LOGS_TO_HOST: boolean flag to determine whether to copy logs to host +COPY_LOGS_TO_HOST ?= true +HOST_LOGS_PATH ?= /tmp/test_logs +CONTAINER_NAME ?= zombienet-test +# tests Variables +ZOMBIE_TESTER_IMAGE ?= zombie_tester:latest +KAGOME_PACKAGE_VERSION ?= WORKING_DIR := $(shell pwd)/../../../kagome +DELETE_IMAGE_AFTER_TEST ?= true -BRANCH_NAME ?= master -KAGOME_IMAGE ?= qdrvm/kagome:latest -ZOMBIE_TESTER_IMAGE ?= zombie_tester:latest -ZOOMBIE_BUILDER_IMAGE ?= qdrvm/zombie-builder:testing -CONTAINER_NAME ?= zombienet-test - -COPY_LOGS_TO_HOST ?= true -HOST_LOGS_PATH ?= /tmp/test_logs +export DOCKER_BUILDKIT=1 +# BUILDKIT_PROGRESS - auto, plain, tty, rawjson +export BUILDKIT_PROGRESS=auto -CUMULUS_RELEASE ?= v0.9.420 -POLKADOT_RELEASE ?= v0.9.42 -POLKADOT_RELEASE_GLOBAL ?= v0.9.42 # run_test takes two arguments: # (1) docker image name # (2) test file path -# COPY_LOGS_TO_HOST: boolean flag to determine whether to copy logs to host - define run_test @CONTAINER_NAME=$$(openssl rand -hex 6); \ START_TIME=$$(date +%s); \ echo "Running test in container $$CONTAINER_NAME with params \n image:$(1), test:$(2) and \n Kagome package version: $(KAGOME_PACKAGE_VERSION)"; \ TEST_PATH=$$(echo $(2) | xargs); \ docker run --name $$CONTAINER_NAME \ - --platform linux/amd64 \ + --platform $(PLATFORM) \ --entrypoint "/bin/bash" \ -v $(WORKING_DIR):/home/nonroot/bin/kagome \ -v $(GOOGLE_APPLICATION_CREDENTIALS):/root/.gcp/google_creds.json \ @@ -77,6 +75,10 @@ define run_test echo "Test duration: $$MINUTES min $$SECONDS sec"; \ echo "Test finished with exit code $$TEST_EXIT_CODE"; \ echo "Runtime cache directory content:"; \ + if [ "$(DELETE_IMAGE_AFTER_TEST)" = "true" ]; then \ + docker rm -f $$CONTAINER_NAME; \ + echo "Container $$CONTAINER_NAME removed"; \ + fi; \ exit $$TEST_EXIT_CODE endef @@ -87,6 +89,7 @@ get_versions: echo "short_commit_hash: `git rev-parse HEAD | head -c 7`" >> commit_hash.txt polkadot_builder: + $(MAKE) get_versions if [ -f polkadot-sdk-versions.txt ]; then \ POLKADOT_SDK_RELEASE=$$(grep 'polkadot_format_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \ docker build --platform $(PLATFORM) \ @@ -109,36 +112,6 @@ polkadot_builder_push: echo "One or more files are missing."; \ fi -zombie_tester_push: - if [ -f polkadot-sdk-versions.txt ]; then \ - ZOMBIENET_RELEASE=$$(grep 'short_version:' zombienet-versions.txt | cut -d ' ' -f 2); \ - POLKADOT_SDK_RELEASE=$$(grep 'polkadot_format_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \ - docker push $(DOCKER_REGISTRY_PATH)zombie_tester:$${POLKADOT_SDK_RELEASE}_$${ZOMBIENET_RELEASE} ; \ - docker push $(DOCKER_REGISTRY_PATH)zombie_tester:latest ; \ - else \ - echo "One or more files are missing."; \ - fi - -zombie_tester: - if [ -f polkadot-sdk-versions.txt ]; then \ - ZOMBIENET_RELEASE=$$(grep 'short_version:' zombienet-versions.txt | cut -d ' ' -f 2); \ - POLKADOT_SDK_RELEASE=$$(grep 'polkadot_format_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \ - PROJECT_ID=$(PROJECT_ID) \ - docker build \ - --platform $(PLATFORM) \ - --no-cache \ - -t $(DOCKER_REGISTRY_PATH)zombie_tester:latest \ - -t $(DOCKER_REGISTRY_PATH)zombie_tester:$${POLKADOT_SDK_RELEASE}_$${ZOMBIENET_RELEASE} \ - --secret id=google_creds,src=$(GOOGLE_APPLICATION_CREDENTIALS) \ - -f zombie_tester.Dockerfile \ - --build-arg PROJECT_ID=$$PROJECT_ID \ - --build-arg POLKADOT_BINARY_PACKAGE_VERSION="$(POLKADOT_BINARY_PACKAGE_VERSION)" \ - --build-arg ZOMBIENET_RELEASE=$$ZOMBIENET_RELEASE \ - --build-arg POLKADOT_SDK_RELEASE=$$POLKADOT_SDK_RELEASE . ; \ - else \ - echo "One or more files are missing."; \ - fi - polkadot_binary: if [ -f polkadot-sdk-versions.txt ]; then \ CONTAINER_NAME=polkadot_build_$$(openssl rand -hex 6); \ @@ -177,7 +150,7 @@ polkadot_binary: polkadot-binary \ /tmp/polkadot_binary \ 'Polkadot binaries: $(RESULT_BIN_NAMES)' \ - 'libstdc++6, zlib1g, libgcc-s1, libc6' ; \ + '$(POLKADOT_BINARY_DEPENDENCIES)' ; \ " || DOCKER_EXEC_RESULT=$$? ; \ if [ $$DOCKER_EXEC_RESULT -ne 0 ]; then \ echo "Error: Docker exec failed with return code $$DOCKER_EXEC_RESULT"; \ @@ -197,23 +170,35 @@ upload_apt_package: gcloud artifacts versions delete $$POLKADOT_RELEASE_GLOBAL_NUMERIC-$$SHORT_COMMIT_HASH --package=polkadot-binary --quiet || true ; \ gcloud artifacts apt upload $(ARTIFACTS_REPO) --source=./pkg/polkadot-binary_$$POLKADOT_RELEASE_GLOBAL_NUMERIC-$${SHORT_COMMIT_HASH}_$(ARCHITECTURE).deb - -polkadot_binary_temp: +zombie_tester: if [ -f polkadot-sdk-versions.txt ]; then \ + ZOMBIENET_RELEASE=$$(grep 'short_version:' zombienet-versions.txt | cut -d ' ' -f 2); \ POLKADOT_SDK_RELEASE=$$(grep 'polkadot_format_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \ - POLKADOT_RELEASE_GLOBAL=$$(grep 'short_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \ - POLKADOT_RELEASE_GLOBAL_NUMERIC=$$(grep 'numeric_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \ - CARGO_PACKETS="-p test-parachain-adder-collator -p polkadot-test-malus -p test-parachain-undying-collator" \ - DOCKER_BUILDKIT=0 docker build -t $(POLKADOT_BINARY_REPO)/polkadot_binary:$$POLKADOT_SDK_RELEASE -t $(POLKADOT_BINARY_REPO)/polkadot_binary:latest \ - -f polkadot_builder.Dockerfile \ - --build-arg ZOMBIENET_RELEASE=$(ZOMBIENET_RELEASE) \ - --build-arg POLKADOT_SDK_RELEASE=$$POLKADOT_SDK_RELEASE \ - --build-arg POLKADOT_RELEASE_GLOBAL=$$POLKADOT_RELEASE_GLOBAL \ - --build-arg POLKADOT_RELEASE_GLOBAL_NUMERIC=$$POLKADOT_RELEASE_GLOBAL_NUMERIC . ; \ + PROJECT_ID=$(PROJECT_ID) \ + docker build \ + --platform $(PLATFORM) \ + --no-cache \ + -t $(DOCKER_REGISTRY_PATH)zombie_tester:latest \ + -t $(DOCKER_REGISTRY_PATH)zombie_tester:$${POLKADOT_SDK_RELEASE}_$${ZOMBIENET_RELEASE} \ + --secret id=google_creds,src=$(GOOGLE_APPLICATION_CREDENTIALS) \ + -f zombie_tester.Dockerfile \ + --build-arg PROJECT_ID=$$PROJECT_ID \ + --build-arg POLKADOT_BINARY_PACKAGE_VERSION="$(POLKADOT_BINARY_PACKAGE_VERSION)" \ + --build-arg ZOMBIENET_RELEASE=$$ZOMBIENET_RELEASE \ + --build-arg POLKADOT_SDK_RELEASE=$$POLKADOT_SDK_RELEASE . ; \ else \ echo "One or more files are missing."; \ fi +zombie_tester_push: + if [ -f polkadot-sdk-versions.txt ]; then \ + ZOMBIENET_RELEASE=$$(grep 'short_version:' zombienet-versions.txt | cut -d ' ' -f 2); \ + POLKADOT_SDK_RELEASE=$$(grep 'polkadot_format_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \ + docker push $(DOCKER_REGISTRY_PATH)zombie_tester:$${POLKADOT_SDK_RELEASE}_$${ZOMBIENET_RELEASE} ; \ + docker push $(DOCKER_REGISTRY_PATH)zombie_tester:latest ; \ + else \ + echo "One or more files are missing."; \ + fi copy_logs_to_host: @CONTAINER_NAME=$(CONTAINER_NAME); \ @@ -222,7 +207,7 @@ copy_logs_to_host: echo "Copying logs from container $$CONTAINER_NAME to host path $(HOST_LOGS_PATH)"; \ docker commit $$CONTAINER_NAME $$FINISHED_IMAGE_NAME; \ echo "Starting temporary container $$FINISHED_CONTAINER_NAME to copy logs"; \ - docker run -d --name $$FINISHED_CONTAINER_NAME --platform linux/amd64 --entrypoint "/bin/bash" $$FINISHED_IMAGE_NAME -c "tail -f /dev/null"; \ + docker run -d --name $$FINISHED_CONTAINER_NAME --platform $(PLATFORM) --entrypoint "/bin/bash" $$FINISHED_IMAGE_NAME -c "tail -f /dev/null"; \ mkdir -p $(HOST_LOGS_PATH); \ DIRS_TO_COPY=$$(docker exec $$FINISHED_CONTAINER_NAME "/bin/bash" -c "find /tmp/ -type d -name 'zombie-*'"); \ for DIR in $$DIRS_TO_COPY; do \ @@ -236,54 +221,6 @@ copy_logs_to_host: docker rm -f $$FINISHED_CONTAINER_NAME; \ docker rmi $$FINISHED_IMAGE_NAME - - - -fetch_and_run: - if [ -f polkadot-sdk-versions.txt ]; then \ - POLKADOT_SDK_RELEASE=$$(grep 'polkadot_format_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \ - POLKADOT_RELEASE_GLOBAL=$$(grep 'short_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \ - POLKADOT_RELEASE_GLOBAL_NUMERIC=$$(grep 'numeric_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \ - ZOMBIENET_RELEASE=$$(grep 'short_version:' zombienet-versions.txt | cut -d ' ' -f 2); \ - DOCKER_BUILDKIT=0 docker build -t $(POLKADOT_BINARY_REPO)/polkadot_builder:$$POLKADOT_SDK_RELEASE -t $(POLKADOT_BINARY_REPO)/polkadot_builder:latest \ - -f polkadot_builder.Dockerfile \ - --build-arg ZOMBIENET_RELEASE=$$ZOMBIENET_RELEASE \ - --build-arg POLKADOT_SDK_RELEASE=$$POLKADOT_SDK_RELEASE \ - --build-arg POLKADOT_RELEASE_GLOBAL=$$POLKADOT_RELEASE_GLOBAL \ - --build-arg POLKADOT_RELEASE_GLOBAL_NUMERIC=$$POLKADOT_RELEASE_GLOBAL_NUMERIC . ; \ - else \ - echo "One or more files are missing."; \ - fi - - - -build_polkadot_binary: - IMAGE=$(POLKADOT_BINARY_REPO)/polkadot_builder:latest; \ - CARGO_PACKETS="-p test-parachain-adder-collator -p polkadot-test-malus -p test-parachain-undying-collator" \ - docker run --rm --platform linux/amd64 -v $(PWD):/workspace -w /workspace $$IMAGE cargo build --release $$CARGO_PACKETS - - - -builder: - DOCKER_BUILDKIT=0 docker build -t qdrvm/zombie-builder:latest -f builder.Dockerfile --build-arg CUMULUS_RELEASE=$(CUMULUS_RELEASE) --build-arg POLKADOT_RELEASE=$(POLKADOT_RELEASE) --build-arg POLKADOT_RELEASE_GLOBAL=$(POLKADOT_RELEASE_GLOBAL) . - -tester: - python3 version.py https://github.com/paritytech/polkadot-sdk && \ - if [ -f polkadot-sdk-versions.txt ]; then \ - POLKADOT_SDK_RELEASE=$$(grep 'polkadot_format_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \ - BRANCH_NAME="$(BRANCH_NAME)" \ - KAGOME_IMAGE="$(KAGOME_IMAGE)" \ - DOCKER_BUILDKIT=0 docker build --platform linux/amd64 --no-cache -t qdrvm/zombie-tester:latest -t zombie-tester:latest -f tester.Dockerfile \ - --build-arg BRANCH_NAME=$$BRANCH_NAME \ - --build-arg KAGOME_IMAGE=$$KAGOME_IMAGE \ - --build-arg POLKADOT_SDK_RELEASE=$$POLKADOT_SDK_RELEASE . ; \ - else \ - echo "One or more files are missing."; \ - fi - -tests: - docker run --platform linux/amd64 -it qdrvm/zombie-tester:latest bash - test-polkadot-functional-0001-parachains-pvf: $(call run_test, $(ZOMBIE_TESTER_IMAGE), "kagome/zombienet/polkadot/functional/0001-parachains-pvf.zndsl") diff --git a/zombienet/docker/Makefile.old b/zombienet/docker/Makefile.old deleted file mode 100644 index 0f3312c9bc..0000000000 --- a/zombienet/docker/Makefile.old +++ /dev/null @@ -1,129 +0,0 @@ -BRANCH_NAME ?= master -KAGOME_IMAGE ?= qdrvm/kagome:latest -ZOMBIE_TESTER_IMAGE ?= qdrvm/zombie-tester:latest -ZOOMBIE_BUILDER_IMAGE ?= qdrvm/zombie-builder:testing -CONTAINER_NAME ?= zombienet-test - -COPY_LOGS_TO_HOST ?= true -HOST_LOGS_PATH ?= /tmp/test_logs - -CUMULUS_RELEASE ?= v0.9.420 -POLKADOT_RELEASE ?= v0.9.42 -POLKADOT_RELEASE_GLOBAL ?= v0.9.42 - -# run_test takes two arguments: -# (1) docker image name -# (2) test file path -# COPY_LOGS_TO_HOST: boolean flag to determine whether to copy logs to host - -define run_test - @CONTAINER_NAME=$$(openssl rand -hex 6); \ - echo "Running test in container $$CONTAINER_NAME with params image:$(1), test:$(2)"; \ - docker run --name $$CONTAINER_NAME --platform linux/amd64 $(1) zombienet-linux-x64 test -p native $(2); \ - TEST_EXIT_CODE=$$(docker inspect $$CONTAINER_NAME --format='{{.State.ExitCode}}'); \ - if [ "$(COPY_LOGS_TO_HOST)" = "true" ]; then \ - $(MAKE) copy_logs_to_host CONTAINER_NAME=$$CONTAINER_NAME; \ - fi; \ - echo "Test finished with exit code $$TEST_EXIT_CODE"; \ - exit $$TEST_EXIT_CODE -endef - -copy_logs_to_host: - @CONTAINER_NAME=$(CONTAINER_NAME); \ - FINISHED_CONTAINER_NAME=$$CONTAINER_NAME-finished; \ - FINISHED_IMAGE_NAME=$$CONTAINER_NAME-finished-image; \ - echo "Copying logs from container $$CONTAINER_NAME to host path $(HOST_LOGS_PATH)"; \ - docker commit $$CONTAINER_NAME $$FINISHED_IMAGE_NAME; \ - echo "Starting temporary container $$FINISHED_CONTAINER_NAME to copy logs"; \ - docker run -d --name $$FINISHED_CONTAINER_NAME --platform linux/amd64 --entrypoint "/bin/bash" $$FINISHED_IMAGE_NAME -c "tail -f /dev/null"; \ - mkdir -p $(HOST_LOGS_PATH); \ - DIRS_TO_COPY=$$(docker exec $$FINISHED_CONTAINER_NAME "/bin/bash" -c "find /tmp/ -type d -name 'zombie-*'"); \ - for DIR in $$DIRS_TO_COPY; do \ - docker cp "$$FINISHED_CONTAINER_NAME:$$DIR/logs" "$(HOST_LOGS_PATH)/$$(basename $$DIR)"; \ - done; \ - echo "Stop and removing container $$FINISHED_CONTAINER_NAME and image $$FINISHED_IMAGE_NAME"; \ - docker stop $$FINISHED_CONTAINER_NAME; \ - docker rm -f $$FINISHED_CONTAINER_NAME; \ - docker rmi $$FINISHED_IMAGE_NAME - -builder_latest: - python version.py https://github.com/paritytech/polkadot-sdk && \ - python version.py https://github.com/paritytech/zombienet && \ - -fetch_and_run: - if [ -f polkadot-sdk-versions.txt ]; then \ - POLKADOT_SDK_RELEASE=$$(grep 'polkadot_format_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \ - POLKADOT_RELEASE_GLOBAL=$$(grep 'short_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \ - POLKADOT_RELEASE_GLOBAL_NUMERIC=$$(grep 'numeric_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \ - ZOMBIENET_RELEASE=$$(grep 'short_version:' zombienet-versions.txt | cut -d ' ' -f 2); \ - DOCKER_BUILDKIT=0 docker build -t qdrvm/zombie-builder:$$POLKADOT_SDK_RELEASE-testing -t qdrvm/zombie-builder:testing \ - -f zombie_builder.Dockerfile \ - --build-arg ZOMBIENET_RELEASE=$$ZOMBIENET_RELEASE \ - --build-arg POLKADOT_SDK_RELEASE=$$POLKADOT_SDK_RELEASE \ - --build-arg POLKADOT_RELEASE_GLOBAL=$$POLKADOT_RELEASE_GLOBAL \ - --build-arg POLKADOT_RELEASE_GLOBAL_NUMERIC=$$POLKADOT_RELEASE_GLOBAL_NUMERIC . ; \ - else \ - echo "One or more files are missing."; \ - fi - -builder_push: - if [ -f polkadot-sdk-versions.txt ]; then \ - POLKADOT_SDK_RELEASE=$$(grep 'polkadot_format_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \ - docker push qdrvm/zombie-builder:$$POLKADOT_SDK_RELEASE-testing ; \ - docker push qdrvm/zombie-builder:testing ; \ - else \ - echo "One or more files are missing."; \ - fi - -builder: - DOCKER_BUILDKIT=0 docker build -t qdrvm/zombie-builder:latest -f builder.Dockerfile --build-arg CUMULUS_RELEASE=$(CUMULUS_RELEASE) --build-arg POLKADOT_RELEASE=$(POLKADOT_RELEASE) --build-arg POLKADOT_RELEASE_GLOBAL=$(POLKADOT_RELEASE_GLOBAL) . - -tester: - python3 version.py https://github.com/paritytech/polkadot-sdk && \ - if [ -f polkadot-sdk-versions.txt ]; then \ - POLKADOT_SDK_RELEASE=$$(grep 'polkadot_format_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \ - BRANCH_NAME="$(BRANCH_NAME)" \ - KAGOME_IMAGE="$(KAGOME_IMAGE)" \ - DOCKER_BUILDKIT=0 docker build --platform linux/amd64 --no-cache -t qdrvm/zombie-tester:latest -t zombie-tester:latest -f tester.Dockerfile \ - --build-arg BRANCH_NAME=$$BRANCH_NAME \ - --build-arg KAGOME_IMAGE=$$KAGOME_IMAGE \ - --build-arg POLKADOT_SDK_RELEASE=$$POLKADOT_SDK_RELEASE . ; \ - else \ - echo "One or more files are missing."; \ - fi - -tests: - docker run --platform linux/amd64 -it qdrvm/zombie-tester:latest bash - -test-polkadot-functional-0001-parachains-pvf: - $(call run_test, $(ZOMBIE_TESTER_IMAGE), "kagome/zombienet/polkadot/functional/0001-parachains-pvf.zndsl") - -test-polkadot-functional-0002-parachains-disputes: - $(call run_test, $(ZOMBIE_TESTER_IMAGE), "kagome/zombienet/polkadot/functional/0002-parachains-disputes.zndsl") - -test-polkadot-functional-0003-beefy-and-mmr: - $(call run_test, $(ZOMBIE_TESTER_IMAGE), "kagome/zombienet/polkadot/functional/0003-beefy-and-mmr.zndsl") - -test-polkadot-functional-0004-parachains-garbage-candidate: - $(call run_test, $(ZOMBIE_TESTER_IMAGE), "kagome/zombienet/polkadot/functional/0004-parachains-garbage-candidate.zndsl") - -test-polkadot-functional-0005-parachains-disputes-past-session: - $(call run_test, $(ZOMBIE_TESTER_IMAGE), "kagome/zombienet/polkadot/functional/0005-parachains-disputes-past-session.zndsl") - -test-polkadot-functional-0006-parachains-max-tranche0: - $(call run_test, $(ZOMBIE_TESTER_IMAGE), "kagome/zombienet/polkadot/functional/0006-parachains-max-tranche0.zndsl") - -test-polkadot-functional-0007-dispute-freshly-finalized: - $(call run_test, $(ZOMBIE_TESTER_IMAGE), "kagome/zombienet/polkadot/functional/0007-dispute-freshly-finalized.zndsl") - -test-polkadot-functional-0008-dispute-old-finalized: - $(call run_test, $(ZOMBIE_TESTER_IMAGE), "kagome/zombienet/polkadot/functional/0008-dispute-old-finalized.zndsl") - -test-polkadot-functional-0009-approval-voting-coalescing: - $(call run_test, $(ZOMBIE_TESTER_IMAGE), "kagome/zombienet/polkadot/functional/0009-approval-voting-coalescing.zndsl") - -test-polkadot-functional-0010-validator-disabling: - $(call run_test, $(ZOMBIE_TESTER_IMAGE), "kagome/zombienet/polkadot/functional/0010-validator-disabling.zndsl") - -test-polkadot-functional-0011-async-backing-6-seconds-rate: - $(call run_test, $(ZOMBIE_TESTER_IMAGE), "kagome/zombienet/polkadot/functional/0011-async-backing-6-seconds-rate.zndsl") diff --git a/zombienet/docker/builder.Dockerfile b/zombienet/docker/builder.Dockerfile deleted file mode 100644 index c0fd659b63..0000000000 --- a/zombienet/docker/builder.Dockerfile +++ /dev/null @@ -1,74 +0,0 @@ -ARG POLKADOT_RELEASE_GLOBAL -ARG POLKADOT_RELEASE_GLOBAL_NUMERIC -ARG ZOMBIENET_RELEASE - -FROM node:bookworm-slim as zombie-builder -USER root - -RUN apt-get update && apt update && \ - apt-get install -y --no-install-recommends git ssh curl nano build-essential protobuf-compiler libprotobuf-dev time && \ - apt install -y --no-install-recommends clang ca-certificates openssl && \ - rm -rf /var/lib/apt/lists/* && \ - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y -ENV RUSTUP_HOME=/opt/rust CARGO_HOME=/opt/cargo PATH=/root/.cargo/bin:$PATH -RUN rustup default stable && rustup target add wasm32-unknown-unknown && rustup component add rust-src -RUN groupadd --gid 10001 nonroot && \ - useradd --home-dir /home/nonroot \ - --create-home \ - --shell /bin/bash \ - --gid nonroot \ - --groups nonroot \ - --uid 10000 nonroot -WORKDIR /home/nonroot/ - -FROM zombie-builder AS zombie-builder-polkadot-sdk -ARG POLKADOT_SDK_RELEASE -WORKDIR /home/nonroot/ -RUN git clone --depth 1 --branch $POLKADOT_SDK_RELEASE https://github.com/paritytech/polkadot-sdk.git - -FROM zombie-builder-polkadot-sdk AS zombie-builder-polkadot-sdk-bin -WORKDIR /home/nonroot/polkadot-sdk/ -RUN cargo update -p test-parachain-adder-collator \ - -p polkadot-test-malus \ - -p test-parachain-undying-collator -RUN cargo build --profile testnet -p test-parachain-adder-collator \ - -p polkadot-test-malus \ - -p test-parachain-undying-collator -RUN find /home/nonroot/polkadot-sdk/target/ -maxdepth 2 -print - -FROM docker.io/parity/polkadot:$POLKADOT_RELEASE_GLOBAL AS polkadot - -FROM docker.io/parity/polkadot-parachain:$POLKADOT_RELEASE_GLOBAL_NUMERIC AS polkadot-parachain - -FROM zombie-builder AS final - -RUN mkdir -p /home/nonroot/bin -COPY --from=zombie-builder-polkadot-sdk-bin /home/nonroot/polkadot-sdk/target/testnet/malus /home/nonroot/bin -COPY --from=zombie-builder-polkadot-sdk-bin /home/nonroot/polkadot-sdk/target/testnet/adder-collator /home/nonroot/bin -COPY --from=zombie-builder-polkadot-sdk-bin /home/nonroot/polkadot-sdk/target/testnet/undying-collator /home/nonroot/bin -COPY --from=polkadot /usr/bin/polkadot /home/nonroot/bin -COPY --from=polkadot /usr/lib/polkadot/polkadot-execute-worker /home/nonroot/bin -COPY --from=polkadot /usr/lib/polkadot/polkadot-prepare-worker /home/nonroot/bin -COPY --from=polkadot-parachain /usr/local/bin/polkadot-parachain /home/nonroot/bin -COPY polkadot-sdk-versions.txt /home/nonroot/polkadot-sdk-versions.txt -COPY zombienet-versions.txt /home/nonroot/zombienet-versions.txt -ENV PATH=/home/nonroot/bin:$PATH - -RUN apt-get update && \ - apt-get install -y --no-install-recommends software-properties-common curl gpg gpg-agent wget nano usrmerge && \ - curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg && \ - echo \ - "deb http://deb.debian.org/debian/ experimental main" | tee -a /etc/apt/sources.list.d/docker.list > /dev/null && \ - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ - add-apt-repository -y "deb http://deb.debian.org/debian/ testing main" && \ - apt-get update && \ - apt-get install --no-install-recommends -y libstdc++6 libc6 libnsl2 && \ - rm -rf /var/lib/apt/lists/* - -ARG ZOMBIENET_RELEASE -RUN wget https://github.com/paritytech/zombienet/releases/download/$ZOMBIENET_RELEASE/zombienet-linux-x64 && \ - chmod +x zombienet-linux-x64 && \ - cp zombienet-linux-x64 /home/nonroot/bin && \ - rm -rf zombienet-linux-x64 - -RUN chown -R root. /home/nonroot diff --git a/zombienet/docker/polkadot_binary.Dockerfile b/zombienet/docker/polkadot_binary.Dockerfile deleted file mode 100644 index 36681f4680..0000000000 --- a/zombienet/docker/polkadot_binary.Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -ARG AUTHOR="k.azovtsev@qdrvm.io " - -ARG POLKADOT_RELEASE_GLOBAL -ARG POLKADOT_RELEASE_GLOBAL_NUMERIC -ARG POLKADOT_SDK_RELEASE -ARG ZOMBIENET_RELEASE - -ARG POLKADOT_BUILDER_IMAGE -ARG MINIDEB_IMAGE - - - - -FROM ${POLKADOT_BUILDER_IMAGE} AS polkadot-sdk-builder - -LABEL org.opencontainers.image.authors="${AUTHOR}" -LABEL org.opencontainers.image.description="Polkadot SDK binary builder image" - -RUN cargo update \ - -p test-parachain-adder-collator \ - -p polkadot-test-malus \ - -p test-parachain-undying-collator - -RUN cargo build --profile testnet \ - -p test-parachain-adder-collator \ - -p polkadot-test-malus \ - -p test-parachain-undying-collator - -RUN echo "Listing files in /home/nonroot/polkadot-sdk/target: " && \ - find /home/nonroot/polkadot-sdk/target/ -maxdepth 2 -print - - - - diff --git a/zombienet/docker/polkadot_builder.Dockerfile b/zombienet/docker/polkadot_builder.Dockerfile index 19a6d0b0ce..6f2dec11e0 100644 --- a/zombienet/docker/polkadot_builder.Dockerfile +++ b/zombienet/docker/polkadot_builder.Dockerfile @@ -8,6 +8,8 @@ ARG RUST_IMAGE FROM ${RUST_IMAGE} AS polkadot-sdk-builder +ARG AUTHOR +ENV AUTHOR=${AUTHOR} LABEL org.opencontainers.image.authors="${AUTHOR}" LABEL org.opencontainers.image.description="Polkadot SDK builder image" diff --git a/zombienet/docker/tester.Dockerfile.old b/zombienet/docker/tester.Dockerfile.old deleted file mode 100644 index 5afbb24732..0000000000 --- a/zombienet/docker/tester.Dockerfile.old +++ /dev/null @@ -1,24 +0,0 @@ -ARG POLKADOT_SDK_RELEASE -ARG BRANCH_NAME -ARG KAGOME_IMAGE - -FROM $KAGOME_IMAGE as kagome - -FROM qdrvm/zombie-builder:$POLKADOT_SDK_RELEASE-testing AS tester -ARG BRANCH_NAME -ENV BRANCH_NAME=${BRANCH_NAME} - -COPY --from=kagome /usr/local/bin/kagome /home/nonroot/bin - -WORKDIR /home/nonroot - -RUN echo "Cloning branch: ${BRANCH_NAME}" -RUN git clone https://github.com/qdrvm/kagome.git -b ${BRANCH_NAME} - -RUN mkdir -p /tmp/kagome -RUN cd kagome/zombienet && ./precompile.sh - -RUN mkdir /home/nonroot/.local && \ - chown nonroot:nonroot /home/nonroot/.local && \ - chown -R nonroot:nonroot /tmp -USER nonroot