From 9a433152206285c108da55ac7bb9d807e39f7d2e Mon Sep 17 00:00:00 2001 From: LexLuthr <88259624+LexLuthr@users.noreply.github.com> Date: Thu, 21 Sep 2023 14:06:06 +0400 Subject: [PATCH] feat: update docker devnet build (#1702) * update makefile, remove lotus container build * multiplatform build * cleanup * fix lotus image name * update readme * pull boost images * clarify build_boost --- .github/workflows/container-build.yml | 32 +++++---------- Makefile | 57 ++++++++++++++++++++++++--- README.md | 8 +--- 3 files changed, 64 insertions(+), 33 deletions(-) diff --git a/.github/workflows/container-build.yml b/.github/workflows/container-build.yml index 48cc268bf..fa9fbc559 100644 --- a/.github/workflows/container-build.yml +++ b/.github/workflows/container-build.yml @@ -16,7 +16,9 @@ jobs: contents: read packages: write env: - LOTUS_TEST_IMAGE: 'filecoin/lotus-all-in-one:v1.23.2' + LOTUS_VERSION: 'v1.23.3' + LOTUS_SOURCE_IMAGE: 'ghcr.io/filecoin-shipyard/lotus-containers:lotus' + NETWORK_NAME: 'devnet' FFI_BUILD_FROM_SOURCE: '0' DOCKER_BUILDKIT: '1' steps: @@ -24,8 +26,12 @@ jobs: uses: actions/checkout@v3 - name: Update Boost modules run: make build/.update-modules + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 + with: + platforms: linux/amd64,linux/arm64 - name: Log in to the Container registry uses: docker/login-action@v2 if: ${{ github.event_name != 'pull_request' }} @@ -41,7 +47,7 @@ jobs: target: boost-dev push: ${{ github.event_name != 'pull_request' }} build-args: | - LOTUS_TEST_IMAGE=${{ env.LOTUS_TEST_IMAGE }} + LOTUS_TEST_IMAGE=${{ env.LOTUS_SOURCE_IMAGE }}-${{ env.LOTUS_VERSION }}-${{ env.NETWORK_NAME }} FFI_BUILD_FROM_SOURCE=${{ env.FFI_BUILD_FROM_SOURCE }} - name: Build booster-http-dev uses: ./.github/actions/container-builder @@ -51,7 +57,7 @@ jobs: target: booster-http-dev push: ${{ github.event_name != 'pull_request' }} build-args: | - LOTUS_TEST_IMAGE=${{ env.LOTUS_TEST_IMAGE }} + LOTUS_TEST_IMAGE=${{ env.LOTUS_SOURCE_IMAGE }}-${{ env.LOTUS_VERSION }}-${{ env.NETWORK_NAME }} FFI_BUILD_FROM_SOURCE=${{ env.FFI_BUILD_FROM_SOURCE }} - name: Build booster-bitswap-dev uses: ./.github/actions/container-builder @@ -61,23 +67,5 @@ jobs: target: booster-bitswap-dev push: ${{ github.event_name != 'pull_request' }} build-args: | - LOTUS_TEST_IMAGE=${{ env.LOTUS_TEST_IMAGE }} - FFI_BUILD_FROM_SOURCE=${{ env.FFI_BUILD_FROM_SOURCE }} - - name: Build lotus-dev - uses: ./.github/actions/container-builder - with: - name: lotus-dev - context: ./docker/devnet/lotus - push: ${{ github.event_name != 'pull_request' }} - build-args: | - LOTUS_TEST_IMAGE=${{ env.LOTUS_TEST_IMAGE }} + LOTUS_TEST_IMAGE=${{ env.LOTUS_SOURCE_IMAGE }}-${{ env.LOTUS_VERSION }}-${{ env.NETWORK_NAME }} FFI_BUILD_FROM_SOURCE=${{ env.FFI_BUILD_FROM_SOURCE }} - - name: Build lotus-miner-dev - uses: ./.github/actions/container-builder - with: - name: lotus-miner-dev - context: ./docker/devnet/lotus-miner - push: ${{ github.event_name != 'pull_request' }} - build-args: | - LOTUS_TEST_IMAGE=${{ env.LOTUS_TEST_IMAGE }} - FFI_BUILD_FROM_SOURCE=${{ env.FFI_BUILD_FROM_SOURCE }} \ No newline at end of file diff --git a/Makefile b/Makefile index 8798b88f7..70abe6097 100644 --- a/Makefile +++ b/Makefile @@ -217,9 +217,22 @@ docsgen-openrpc-boost: docsgen-openrpc-bin ## DOCKER IMAGES docker_user?=filecoin -lotus_version?=v1.23.2 +lotus_version?=v1.23.3 ffi_from_source?=0 build_lotus?=0 +build_boost?=1 +boost_version?=v2.1.0-rc1 +ifeq ($(build_boost),1) +#v1: build boost images currently checked out branch + boost_build_cmd=docker/boost + booster_http_build_cmd=docker/booster-http + booster_bitswap_build_cmd=docker/booster-bitswap +else +# v2: pull images from the github repo + boost_build_cmd=pull/boost retag/boost + booster_http_build_cmd=pull/booster-http retag/booster-http + booster_bitswap_build_cmd=pull/booster-bitswap retag/booster-bitswap +endif ifeq ($(build_lotus),1) # v1: building lotus image with provided lotus version lotus_info_msg=!!! building lotus base image from github: branch/tag $(lotus_version) !!! @@ -227,9 +240,9 @@ ifeq ($(build_lotus),1) lotus_build_cmd=update/lotus docker/lotus-all-in-one lotus_base_image=$(docker_user)/lotus-all-in-one:$(lotus_version)-debug else -# v2 (default): using lotus image - lotus_base_image?=$(docker_user)/lotus-all-in-one:$(lotus_version)-debug - lotus_info_msg=using lotus image from dockerhub: $(lotus_base_image) +# v2 (default): using prebuilt lotus image + lotus_base_image?=ghcr.io/filecoin-shipyard/lotus-containers:lotus-$(lotus_version)-devnet + lotus_info_msg=using lotus image from github: $(lotus_base_image) lotus_build_cmd=info/lotus-all-in-one endif docker_build_cmd=docker build --build-arg LOTUS_TEST_IMAGE=$(lotus_base_image) \ @@ -250,6 +263,30 @@ docker/lotus-all-in-one: info/lotus-all-in-one | $(lotus_src_dir) -t $(lotus_base_image) --build-arg GOFLAGS=-tags=debug . .PHONY: docker/lotus-all-in-one +pull/boost: + docker pull ghcr.io/filecoin-project/boost:boost-dev-$(boost_version) +.PHONY: pull/boost + +pull/booster-http: + docker pull ghcr.io/filecoin-project/boost:boost-http-dev-$(boost_version) +.PHONY: pull/boost + +pull/booster-bitswap: + docker pull ghcr.io/filecoin-project/boost:boost-bitswap-dev-$(boost_version) +.PHONY: pull/boost + +retag/boost: + docker tag ghcr.io/filecoin-project/boost:boost-dev-$(boost_version) $(docker_user)/boost-dev:dev +.PHONY: retag/boost + +retag/booster-http: + docker tag ghcr.io/filecoin-project/boost:boost-http-dev-$(boost_version) $(docker_user)/booster-http-dev:dev +.PHONY: retag/booster-http + +retag/booster-bitswap: + docker tag ghcr.io/filecoin-project/boost:boost-bitswap-dev-$(boost_version) $(docker_user)/booster-bitswap-dev:dev +.PHONY: retag/booster-bitswap + # boost-client main docker/mainnet/boost-client: build/.update-modules DOCKER_BUILDKIT=1 $(docker_build_cmd) \ @@ -276,10 +313,20 @@ docker/booster-bitswap: -t $(docker_user)/booster-bitswap-dev:dev --build-arg BUILD_VERSION=dev \ -f docker/devnet/Dockerfile.source --target booster-bitswap-dev . .PHONY: docker/booster-bitswap -docker/all: $(lotus_build_cmd) docker/boost docker/booster-http docker/booster-bitswap \ +docker/all: $(lotus_build_cmd) $(boost_build_cmd) $(booster_http_build_cmd) $(booster_bitswap_build_cmd) \ docker/lotus docker/lotus-miner .PHONY: docker/all +### To allow devs to pull individual images. Require build_boost=0 and boost_version to be supplied +docker/get-boost: $(boost_build_cmd) +.PHONY: docker/get-boost + +docker/get-booster-http: $(booster_http_build_cmd) +.PHONY: docker/get-booster-http + +docker/get-booster-bitswap: $(booster_bitswap_build_cmd) +.PHONY: docker/get-booster-http + test-lid: cd ./extern/boostd-data && ARCH=$(ARCH) docker-compose up --build --exit-code-from go-tests diff --git a/README.md b/README.md index 02f2f556c..d7ec9546b 100644 --- a/README.md +++ b/README.md @@ -28,12 +28,8 @@ Compile and install using the instructions at the `Building and installing` sect make clean docker/all ``` -On ARM-based systems (*Apple M1/M2*) you need to force building Filecoin's Rust libraries from the source -``` -make clean docker/all ffi_from_source=1 build_lotus=1 -``` - -If you need to build containers using a specific version of lotus then provide the version as a parameter, e.g. `make clean docker/all lotus_version=v1.20.0-rc2 build_lotus=1`. The version must be a tag or a remote branch name of [Lotus git repo](https://github.com/filecoin-project/lotus). +If you need to build containers using a specific version of lotus then provide the version as a parameter, e.g. `make clean docker/all lotus_version=v1.23.3`. The version must be a tag or a remote branch name of [Lotus git repo](https://github.com/filecoin-project/lotus). +If the branch or tag you requested does not exist in our [Github image repository](https://github.com/filecoin-shipyard/lotus-containers/pkgs/container/lotus-containers) then you can build the lotus image manually with `make clean docker/all lotus_version=test/branch1 build_lotus=1`. We are shipping images all releases from Lotus in our [Github image repo](https://github.com/filecoin-shipyard/lotus-containers/pkgs/container/lotus-containers). ### Start devnet Docker stack