feat: update docker devnet build (#1702)

* update makefile, remove lotus container build * multiplatform build * cleanup * fix lotus image name * update readme * pull boost images * clarify build_boost
filecoin-project · Sep 21, 2023 · 9a43315 · 9a43315
1 parent 0278a9b
commit 9a43315
Show file tree

Hide file tree

Showing 3 changed files with 64 additions and 33 deletions.
diff --git a/.github/workflows/container-build.yml b/.github/workflows/container-build.yml
@@ -16,16 +16,22 @@ jobs:
       contents: read
       packages: write
     env:
-      LOTUS_TEST_IMAGE: 'filecoin/lotus-all-in-one:v1.23.2'
+      LOTUS_VERSION: 'v1.23.3'
+      LOTUS_SOURCE_IMAGE: 'ghcr.io/filecoin-shipyard/lotus-containers:lotus'
+      NETWORK_NAME: 'devnet'
       FFI_BUILD_FROM_SOURCE: '0'
       DOCKER_BUILDKIT: '1'
     steps:
       - name: Checkout
         uses: actions/checkout@v3
       - name: Update Boost modules
         run: make build/.update-modules
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
+        with:
+          platforms: linux/amd64,linux/arm64
       - name: Log in to the Container registry
         uses: docker/login-action@v2
         if: ${{ github.event_name != 'pull_request' }}
@@ -41,7 +47,7 @@ jobs:
           target: boost-dev
           push: ${{ github.event_name != 'pull_request' }}
           build-args: |
-            LOTUS_TEST_IMAGE=${{ env.LOTUS_TEST_IMAGE }}
+            LOTUS_TEST_IMAGE=${{ env.LOTUS_SOURCE_IMAGE }}-${{ env.LOTUS_VERSION }}-${{ env.NETWORK_NAME }}
             FFI_BUILD_FROM_SOURCE=${{ env.FFI_BUILD_FROM_SOURCE }}
       - name: Build booster-http-dev
         uses: ./.github/actions/container-builder
@@ -51,7 +57,7 @@ jobs:
           target: booster-http-dev
           push: ${{ github.event_name != 'pull_request' }}
           build-args: |
-            LOTUS_TEST_IMAGE=${{ env.LOTUS_TEST_IMAGE }}
+            LOTUS_TEST_IMAGE=${{ env.LOTUS_SOURCE_IMAGE }}-${{ env.LOTUS_VERSION }}-${{ env.NETWORK_NAME }}
             FFI_BUILD_FROM_SOURCE=${{ env.FFI_BUILD_FROM_SOURCE }}
       - name: Build booster-bitswap-dev
         uses: ./.github/actions/container-builder
@@ -61,23 +67,5 @@ jobs:
           target: booster-bitswap-dev
           push: ${{ github.event_name != 'pull_request' }}
           build-args: |
-            LOTUS_TEST_IMAGE=${{ env.LOTUS_TEST_IMAGE }}
-            FFI_BUILD_FROM_SOURCE=${{ env.FFI_BUILD_FROM_SOURCE }}
-      - name: Build lotus-dev
-        uses: ./.github/actions/container-builder
-        with:
-          name: lotus-dev
-          context: ./docker/devnet/lotus
-          push: ${{ github.event_name != 'pull_request' }}
-          build-args: |
-            LOTUS_TEST_IMAGE=${{ env.LOTUS_TEST_IMAGE }}
+            LOTUS_TEST_IMAGE=${{ env.LOTUS_SOURCE_IMAGE }}-${{ env.LOTUS_VERSION }}-${{ env.NETWORK_NAME }}
             FFI_BUILD_FROM_SOURCE=${{ env.FFI_BUILD_FROM_SOURCE }}
-      - name: Build lotus-miner-dev
-        uses: ./.github/actions/container-builder
-        with:
-          name: lotus-miner-dev
-          context: ./docker/devnet/lotus-miner
-          push: ${{ github.event_name != 'pull_request' }}
-          build-args: |
-            LOTUS_TEST_IMAGE=${{ env.LOTUS_TEST_IMAGE }}
-            FFI_BUILD_FROM_SOURCE=${{ env.FFI_BUILD_FROM_SOURCE }}
diff --git a/Makefile b/Makefile
@@ -217,19 +217,32 @@ docsgen-openrpc-boost: docsgen-openrpc-bin
 
 ## DOCKER IMAGES
 docker_user?=filecoin
-lotus_version?=v1.23.2
+lotus_version?=v1.23.3
 ffi_from_source?=0
 build_lotus?=0
+build_boost?=1
+boost_version?=v2.1.0-rc1
+ifeq ($(build_boost),1)
+#v1: build boost images currently checked out branch
+	boost_build_cmd=docker/boost
+	booster_http_build_cmd=docker/booster-http
+	booster_bitswap_build_cmd=docker/booster-bitswap
+else
+# v2: pull images from the github repo
+	boost_build_cmd=pull/boost retag/boost
+	booster_http_build_cmd=pull/booster-http retag/booster-http
+	booster_bitswap_build_cmd=pull/booster-bitswap retag/booster-bitswap
+endif
 ifeq ($(build_lotus),1)
 # v1: building lotus image with provided lotus version
 	lotus_info_msg=!!! building lotus base image from github: branch/tag $(lotus_version) !!!
 	override lotus_src_dir=/tmp/lotus-$(lotus_version)
 	lotus_build_cmd=update/lotus docker/lotus-all-in-one
 	lotus_base_image=$(docker_user)/lotus-all-in-one:$(lotus_version)-debug
 else
-# v2 (default): using lotus image
-	lotus_base_image?=$(docker_user)/lotus-all-in-one:$(lotus_version)-debug
-	lotus_info_msg=using lotus image from dockerhub: $(lotus_base_image)
+# v2 (default): using prebuilt lotus image
+	lotus_base_image?=ghcr.io/filecoin-shipyard/lotus-containers:lotus-$(lotus_version)-devnet
+	lotus_info_msg=using lotus image from github: $(lotus_base_image)
 	lotus_build_cmd=info/lotus-all-in-one
 endif
 docker_build_cmd=docker build --build-arg LOTUS_TEST_IMAGE=$(lotus_base_image) \
@@ -250,6 +263,30 @@ docker/lotus-all-in-one: info/lotus-all-in-one | $(lotus_src_dir)
 		-t $(lotus_base_image) --build-arg GOFLAGS=-tags=debug .
 .PHONY: docker/lotus-all-in-one
 
+pull/boost:
+	docker pull ghcr.io/filecoin-project/boost:boost-dev-$(boost_version)
+.PHONY: pull/boost
+
+pull/booster-http:
+	docker pull ghcr.io/filecoin-project/boost:boost-http-dev-$(boost_version)
+.PHONY: pull/boost
+
+pull/booster-bitswap:
+	docker pull ghcr.io/filecoin-project/boost:boost-bitswap-dev-$(boost_version)
+.PHONY: pull/boost
+
+retag/boost:
+	docker tag ghcr.io/filecoin-project/boost:boost-dev-$(boost_version) $(docker_user)/boost-dev:dev
+.PHONY: retag/boost
+
+retag/booster-http:
+	docker tag ghcr.io/filecoin-project/boost:boost-http-dev-$(boost_version) $(docker_user)/booster-http-dev:dev
+.PHONY: retag/booster-http
+
+retag/booster-bitswap:
+	docker tag ghcr.io/filecoin-project/boost:boost-bitswap-dev-$(boost_version) $(docker_user)/booster-bitswap-dev:dev
+.PHONY: retag/booster-bitswap
+
 # boost-client main
 docker/mainnet/boost-client: build/.update-modules
 	DOCKER_BUILDKIT=1 $(docker_build_cmd) \
@@ -276,10 +313,20 @@ docker/booster-bitswap:
 		-t $(docker_user)/booster-bitswap-dev:dev --build-arg BUILD_VERSION=dev \
 		-f docker/devnet/Dockerfile.source --target booster-bitswap-dev .
 .PHONY: docker/booster-bitswap
-docker/all: $(lotus_build_cmd) docker/boost docker/booster-http docker/booster-bitswap \
+docker/all: $(lotus_build_cmd) $(boost_build_cmd) $(booster_http_build_cmd) $(booster_bitswap_build_cmd) \
 	docker/lotus docker/lotus-miner
 .PHONY: docker/all
 
+### To allow devs to pull individual images. Require build_boost=0 and boost_version to be supplied
+docker/get-boost: $(boost_build_cmd)
+.PHONY: docker/get-boost
+
+docker/get-booster-http: $(booster_http_build_cmd)
+.PHONY: docker/get-booster-http
+
+docker/get-booster-bitswap: $(booster_bitswap_build_cmd)
+.PHONY: docker/get-booster-http
+
 test-lid:
 	cd ./extern/boostd-data && ARCH=$(ARCH) docker-compose up --build --exit-code-from go-tests
 

diff --git a/README.md b/README.md
@@ -28,12 +28,8 @@ Compile and install using the instructions at the `Building and installing` sect
 make clean docker/all
 ```
 
-On ARM-based systems (*Apple M1/M2*) you need to force building Filecoin's Rust libraries from the source
-```
-make clean docker/all ffi_from_source=1 build_lotus=1
-```
-
-If you need to build containers using a specific version of lotus then provide the version as a parameter, e.g. `make clean docker/all lotus_version=v1.20.0-rc2 build_lotus=1`. The version must be a tag or a remote branch name of [Lotus git repo](https://github.com/filecoin-project/lotus).
+If you need to build containers using a specific version of lotus then provide the version as a parameter, e.g. `make clean docker/all lotus_version=v1.23.3`. The version must be a tag or a remote branch name of [Lotus git repo](https://github.com/filecoin-project/lotus).
+If the branch or tag you requested does not exist in our [Github image repository](https://github.com/filecoin-shipyard/lotus-containers/pkgs/container/lotus-containers) then you can build the lotus image manually with  `make clean docker/all lotus_version=test/branch1 build_lotus=1`. We are shipping images all releases from Lotus in our [Github image repo](https://github.com/filecoin-shipyard/lotus-containers/pkgs/container/lotus-containers).
 
 ### Start devnet Docker stack