From 060fa5629a0befc1d59f897762f76b23f8ef6176 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Fri, 15 Sep 2023 16:07:02 +0400 Subject: [PATCH 1/7] update makefile, remove lotus container build --- .github/workflows/container-build.yml | 32 +++++++++------------------ Makefile | 8 +++---- 2 files changed, 14 insertions(+), 26 deletions(-) diff --git a/.github/workflows/container-build.yml b/.github/workflows/container-build.yml index 48cc268bf..2027f8948 100644 --- a/.github/workflows/container-build.yml +++ b/.github/workflows/container-build.yml @@ -16,9 +16,13 @@ jobs: contents: read packages: write env: - LOTUS_TEST_IMAGE: 'filecoin/lotus-all-in-one:v1.23.2' + LOTUS_VERSION: 'v1.23.3' + LOTUS_SOURCE_IMAGE: 'ghcr.io/filecoin-shipyard/lotus-containers:lotus' + NETWORK_NAME: 'debug' FFI_BUILD_FROM_SOURCE: '0' DOCKER_BUILDKIT: '1' + platform: + - linux/amd64,linux/arm64 steps: - name: Checkout uses: actions/checkout@v3 @@ -26,6 +30,8 @@ jobs: run: make build/.update-modules - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 + with: + platforms: ${{ matrix.platform }} - name: Log in to the Container registry uses: docker/login-action@v2 if: ${{ github.event_name != 'pull_request' }} @@ -41,7 +47,7 @@ jobs: target: boost-dev push: ${{ github.event_name != 'pull_request' }} build-args: | - LOTUS_TEST_IMAGE=${{ env.LOTUS_TEST_IMAGE }} + LOTUS_TEST_IMAGE=${{ env.LOTUS_SOURCE_IMAGE }}-${{ env.LOTUS_VERSION }}-${{ env.NETWORK_NAME }} FFI_BUILD_FROM_SOURCE=${{ env.FFI_BUILD_FROM_SOURCE }} - name: Build booster-http-dev uses: ./.github/actions/container-builder @@ -51,7 +57,7 @@ jobs: target: booster-http-dev push: ${{ github.event_name != 'pull_request' }} build-args: | - LOTUS_TEST_IMAGE=${{ env.LOTUS_TEST_IMAGE }} + LOTUS_TEST_IMAGE=${{ env.LOTUS_SOURCE_IMAGE }}-${{ env.LOTUS_VERSION }}-${{ env.NETWORK_NAME }} FFI_BUILD_FROM_SOURCE=${{ env.FFI_BUILD_FROM_SOURCE }} - name: Build booster-bitswap-dev uses: ./.github/actions/container-builder @@ -61,23 +67,5 @@ jobs: target: booster-bitswap-dev push: ${{ github.event_name != 'pull_request' }} build-args: | - LOTUS_TEST_IMAGE=${{ env.LOTUS_TEST_IMAGE }} - FFI_BUILD_FROM_SOURCE=${{ env.FFI_BUILD_FROM_SOURCE }} - - name: Build lotus-dev - uses: ./.github/actions/container-builder - with: - name: lotus-dev - context: ./docker/devnet/lotus - push: ${{ github.event_name != 'pull_request' }} - build-args: | - LOTUS_TEST_IMAGE=${{ env.LOTUS_TEST_IMAGE }} + LOTUS_TEST_IMAGE=${{ env.LOTUS_SOURCE_IMAGE }}-${{ env.LOTUS_VERSION }}-${{ env.NETWORK_NAME }} FFI_BUILD_FROM_SOURCE=${{ env.FFI_BUILD_FROM_SOURCE }} - - name: Build lotus-miner-dev - uses: ./.github/actions/container-builder - with: - name: lotus-miner-dev - context: ./docker/devnet/lotus-miner - push: ${{ github.event_name != 'pull_request' }} - build-args: | - LOTUS_TEST_IMAGE=${{ env.LOTUS_TEST_IMAGE }} - FFI_BUILD_FROM_SOURCE=${{ env.FFI_BUILD_FROM_SOURCE }} \ No newline at end of file diff --git a/Makefile b/Makefile index 8798b88f7..781eebd26 100644 --- a/Makefile +++ b/Makefile @@ -217,7 +217,7 @@ docsgen-openrpc-boost: docsgen-openrpc-bin ## DOCKER IMAGES docker_user?=filecoin -lotus_version?=v1.23.2 +lotus_version?=v1.23.3 ffi_from_source?=0 build_lotus?=0 ifeq ($(build_lotus),1) @@ -227,9 +227,9 @@ ifeq ($(build_lotus),1) lotus_build_cmd=update/lotus docker/lotus-all-in-one lotus_base_image=$(docker_user)/lotus-all-in-one:$(lotus_version)-debug else -# v2 (default): using lotus image - lotus_base_image?=$(docker_user)/lotus-all-in-one:$(lotus_version)-debug - lotus_info_msg=using lotus image from dockerhub: $(lotus_base_image) +# v2 (default): using prebuilt lotus image + lotus_base_image?=ghcr.io/filecoin-shipyard/lotus-containers:lotus-$(lotus_version)-devnet + lotus_info_msg=using lotus image from github: $(lotus_base_image) lotus_build_cmd=info/lotus-all-in-one endif docker_build_cmd=docker build --build-arg LOTUS_TEST_IMAGE=$(lotus_base_image) \ From 79f92a58842bb50d491e433d67d4bbe7fa64cd42 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Fri, 15 Sep 2023 16:17:08 +0400 Subject: [PATCH 2/7] multiplatform build --- .github/workflows/container-build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/container-build.yml b/.github/workflows/container-build.yml index 2027f8948..3a28433cf 100644 --- a/.github/workflows/container-build.yml +++ b/.github/workflows/container-build.yml @@ -28,10 +28,12 @@ jobs: uses: actions/checkout@v3 - name: Update Boost modules run: make build/.update-modules + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 with: - platforms: ${{ matrix.platform }} + platforms: linux/amd64,linux/arm64 - name: Log in to the Container registry uses: docker/login-action@v2 if: ${{ github.event_name != 'pull_request' }} From 0b7ef3fd28290abdaa3d3cc967bd8a2b5435e17c Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Fri, 15 Sep 2023 16:18:02 +0400 Subject: [PATCH 3/7] cleanup --- .github/workflows/container-build.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/container-build.yml b/.github/workflows/container-build.yml index 3a28433cf..d3bf21c37 100644 --- a/.github/workflows/container-build.yml +++ b/.github/workflows/container-build.yml @@ -21,8 +21,6 @@ jobs: NETWORK_NAME: 'debug' FFI_BUILD_FROM_SOURCE: '0' DOCKER_BUILDKIT: '1' - platform: - - linux/amd64,linux/arm64 steps: - name: Checkout uses: actions/checkout@v3 From fb3c7a28dd908136c7c052bbf27dd877fe8af2d5 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Fri, 15 Sep 2023 16:21:46 +0400 Subject: [PATCH 4/7] fix lotus image name --- .github/workflows/container-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container-build.yml b/.github/workflows/container-build.yml index d3bf21c37..fa9fbc559 100644 --- a/.github/workflows/container-build.yml +++ b/.github/workflows/container-build.yml @@ -18,7 +18,7 @@ jobs: env: LOTUS_VERSION: 'v1.23.3' LOTUS_SOURCE_IMAGE: 'ghcr.io/filecoin-shipyard/lotus-containers:lotus' - NETWORK_NAME: 'debug' + NETWORK_NAME: 'devnet' FFI_BUILD_FROM_SOURCE: '0' DOCKER_BUILDKIT: '1' steps: From 2c259b20c7686d6849bd916040bde4836bdbe2db Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Mon, 18 Sep 2023 15:12:07 +0400 Subject: [PATCH 5/7] update readme --- README.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 02f2f556c..d7ec9546b 100644 --- a/README.md +++ b/README.md @@ -28,12 +28,8 @@ Compile and install using the instructions at the `Building and installing` sect make clean docker/all ``` -On ARM-based systems (*Apple M1/M2*) you need to force building Filecoin's Rust libraries from the source -``` -make clean docker/all ffi_from_source=1 build_lotus=1 -``` - -If you need to build containers using a specific version of lotus then provide the version as a parameter, e.g. `make clean docker/all lotus_version=v1.20.0-rc2 build_lotus=1`. The version must be a tag or a remote branch name of [Lotus git repo](https://github.com/filecoin-project/lotus). +If you need to build containers using a specific version of lotus then provide the version as a parameter, e.g. `make clean docker/all lotus_version=v1.23.3`. The version must be a tag or a remote branch name of [Lotus git repo](https://github.com/filecoin-project/lotus). +If the branch or tag you requested does not exist in our [Github image repository](https://github.com/filecoin-shipyard/lotus-containers/pkgs/container/lotus-containers) then you can build the lotus image manually with `make clean docker/all lotus_version=test/branch1 build_lotus=1`. We are shipping images all releases from Lotus in our [Github image repo](https://github.com/filecoin-shipyard/lotus-containers/pkgs/container/lotus-containers). ### Start devnet Docker stack From eae26101e4cd84fc3c0844864a95273bf1e6de0a Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Mon, 18 Sep 2023 17:14:14 +0400 Subject: [PATCH 6/7] pull boost images --- Makefile | 49 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 781eebd26..e48191bef 100644 --- a/Makefile +++ b/Makefile @@ -220,6 +220,19 @@ docker_user?=filecoin lotus_version?=v1.23.3 ffi_from_source?=0 build_lotus?=0 +build_boost?=1 +boost_version?=v2.1.0-rc1 +ifeq ($(build_boost),1) +#v1: build boost images currently checked out branch + boost_build_cmd=docker/boost + booster_http_build_cmd=docker/booster-http + booster_bitswap_build_cmd=docker/booster-bitswap +else +# v2: pull images from the github repo + boost_build_cmd=pull/boost retag/boost + booster_http_build_cmd=pull/booster-http retag/booster-http + booster_bitswap_build_cmd=pull/booster-bitswap retag/booster-bitswap +endif ifeq ($(build_lotus),1) # v1: building lotus image with provided lotus version lotus_info_msg=!!! building lotus base image from github: branch/tag $(lotus_version) !!! @@ -250,6 +263,30 @@ docker/lotus-all-in-one: info/lotus-all-in-one | $(lotus_src_dir) -t $(lotus_base_image) --build-arg GOFLAGS=-tags=debug . .PHONY: docker/lotus-all-in-one +pull/boost: + docker pull ghcr.io/filecoin-project/boost:boost-dev-$(boost_version) +.PHONY: pull/boost + +pull/booster-http: + docker pull ghcr.io/filecoin-project/boost:boost-http-dev-$(boost_version) +.PHONY: pull/boost + +pull/booster-bitswap: + docker pull ghcr.io/filecoin-project/boost:boost-bitswap-dev-$(boost_version) +.PHONY: pull/boost + +retag/boost: + docker tag ghcr.io/filecoin-project/boost:boost-dev-$(boost_version) $(docker_user)/boost-dev:dev +.PHONY: retag/boost + +retag/booster-http: + docker tag ghcr.io/filecoin-project/boost:boost-http-dev-$(boost_version) $(docker_user)/booster-http-dev:dev +.PHONY: retag/booster-http + +retag/booster-bitswap: + docker tag ghcr.io/filecoin-project/boost:boost-bitswap-dev-$(boost_version) $(docker_user)/booster-bitswap-dev:dev +.PHONY: retag/booster-bitswap + # boost-client main docker/mainnet/boost-client: build/.update-modules DOCKER_BUILDKIT=1 $(docker_build_cmd) \ @@ -276,10 +313,20 @@ docker/booster-bitswap: -t $(docker_user)/booster-bitswap-dev:dev --build-arg BUILD_VERSION=dev \ -f docker/devnet/Dockerfile.source --target booster-bitswap-dev . .PHONY: docker/booster-bitswap -docker/all: $(lotus_build_cmd) docker/boost docker/booster-http docker/booster-bitswap \ +docker/all: $(lotus_build_cmd) $(boost_build_cmd) $(booster_http_build_cmd) $(booster_bitswap_build_cmd) \ docker/lotus docker/lotus-miner .PHONY: docker/all +### To allow devs to pull individual images +docker/get-boost: $(boost_build_cmd) +.PHONY: docker/get-boost + +docker/get-booster-http: $(booster_http_build_cmd) +.PHONY: docker/get-booster-http + +docker/get-booster-bitswap: $(booster_bitswap_build_cmd) +.PHONY: docker/get-booster-http + test-lid: cd ./extern/boostd-data && ARCH=$(ARCH) docker-compose up --build --exit-code-from go-tests From 589f56c878e1a27e2e37859a831bccac2cd68212 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Mon, 18 Sep 2023 17:32:15 +0400 Subject: [PATCH 7/7] clarify build_boost --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e48191bef..70abe6097 100644 --- a/Makefile +++ b/Makefile @@ -317,7 +317,7 @@ docker/all: $(lotus_build_cmd) $(boost_build_cmd) $(booster_http_build_cmd) $(bo docker/lotus docker/lotus-miner .PHONY: docker/all -### To allow devs to pull individual images +### To allow devs to pull individual images. Require build_boost=0 and boost_version to be supplied docker/get-boost: $(boost_build_cmd) .PHONY: docker/get-boost