Selectively download nimbus-eth2 submodules

status-im · Sep 18, 2023 · 2c2aa16 · 2c2aa16
1 parent a65b1c9
commit 2c2aa16
Show file tree

Hide file tree

Showing 6 changed files with 30 additions and 14 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -241,7 +241,7 @@ jobs:
 
       - name: Build Nim and Nimbus-eth1 dependencies
         run: |
-          make -j${ncpu} ARCH_OVERRIDE=${PLATFORM} CI_CACHE=NimBinCache update
+          make -j${ncpu} ARCH_OVERRIDE=${PLATFORM} CI_CACHE=NimBinCache update-from-ci
 
       - name: Run nimbus-eth1 tests (Windows)
         if: runner.os == 'Windows'

diff --git a/.github/workflows/fluffy.yml b/.github/workflows/fluffy.yml
@@ -63,7 +63,7 @@ jobs:
 
       - name: Build Nim and Nimbus-eth1 dependencies
         run: |
-          make -j${ncpu} ARCH_OVERRIDE=${PLATFORM} CI_CACHE=NimBinaries update
+          make -j${ncpu} ARCH_OVERRIDE=${PLATFORM} CI_CACHE=NimBinaries update-from-ci
 
       - name: build uTP test app container
         run: |
@@ -253,7 +253,7 @@ jobs:
         run: |
           # use CC to make sure Nim compiler and subsequent test
           # using the same glibc version.
-          env CC=gcc make -j${ncpu} ARCH_OVERRIDE=${PLATFORM} CI_CACHE=NimBinaries update
+          env CC=gcc make -j${ncpu} ARCH_OVERRIDE=${PLATFORM} CI_CACHE=NimBinaries update-from-ci
 
       - name: Run fluffy tests (Windows)
         if: runner.os == 'Windows'

diff --git a/.github/workflows/nimbus_verified_proxy.yml b/.github/workflows/nimbus_verified_proxy.yml
@@ -181,7 +181,7 @@ jobs:
 
       - name: Build Nim and Nimbus-eth1 dependencies
         run: |
-          make -j${ncpu} ARCH_OVERRIDE=${PLATFORM} CI_CACHE=NimBinaries update
+          make -j${ncpu} ARCH_OVERRIDE=${PLATFORM} CI_CACHE=NimBinaries update-from-ci
 
       - name: Run verified proxy tests (Windows)
         if: runner.os == 'Windows'

diff --git a/.github/workflows/simulators.yml b/.github/workflows/simulators.yml
@@ -33,7 +33,7 @@ jobs:
       - name: Build Nim and deps
         run: |
           ncpu=$(nproc)
-          make -j${ncpu} ARCH_OVERRIDE=x64 CI_CACHE=NimBinaries update
+          make -j${ncpu} ARCH_OVERRIDE=x64 CI_CACHE=NimBinaries update-from-ci
           make -j${ncpu} deps
 
       - name: Run Simulators
@@ -74,7 +74,7 @@ jobs:
       - name: Build Nim and deps
         run: |
           ncpu=$(sysctl -n hw.ncpu)
-          make -j${ncpu} ARCH_OVERRIDE=x64 CI_CACHE=NimBinaries update
+          make -j${ncpu} ARCH_OVERRIDE=x64 CI_CACHE=NimBinaries update-from-ci
           make -j${ncpu} deps
 
       - name: Run Simulators
@@ -144,7 +144,7 @@ jobs:
       - name: Build Nim and deps
         run: |
           ncpu=${NUMBER_OF_PROCESSORS}
-          mingw32-make -j${ncpu} ARCH_OVERRIDE=x64 CI_CACHE=NimBinaries update
+          mingw32-make -j${ncpu} ARCH_OVERRIDE=x64 CI_CACHE=NimBinaries update-from-ci
           mingw32-make -j${ncpu} deps
 
       - name: Run Simulators

diff --git a/Makefile b/Makefile
@@ -104,7 +104,18 @@ FLUFFY_TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(FLUFFY_TOOLS))
 
 ifeq ($(NIM_PARAMS),)
 # "variables.mk" was not included, so we update the submodules.
-GIT_SUBMODULE_UPDATE := git submodule update --init --recursive
+# selectively download nimbus-eth2 submodules because we don't need all of it's modules
+# also holesky already exceeds github LFS quota
+GIT_SUBMODULE_UPDATE := git -c submodule."vendor/nimbus-eth2".update=none submodule update --init --recursive; \
+  git submodule update vendor/nimbus-eth2; \
+  cd vendor/nimbus-eth2; \
+  git submodule update --init vendor/eth2-networks; \
+  git submodule update --init vendor/holesky; \
+  git submodule update --init vendor/sepolia; \
+  git submodule update --init vendor/gnosis-chain-configs; \
+  git submodule update --init --recursive vendor/nim-kzg4844; \
+  cd ../..
+
 .DEFAULT:
 	+@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \
 		$(GIT_SUBMODULE_UPDATE); \
@@ -186,6 +197,11 @@ update: | update-common
 	rm -rf nimbus.nims && \
 		$(MAKE) nimbus.nims $(HANDLE_OUTPUT)
 
+update-from-ci: | sanity-checks update-test
+	rm -rf nimbus.nims && \
+		$(MAKE) nimbus.nims $(HANDLE_OUTPUT)
+	+ "$(MAKE)" --no-print-directory deps-common
+
 # builds the tools, wherever they are
 $(TOOLS): | build deps
 	for D in $(TOOLS_DIRS); do [ -e "$${D}/$@.nim" ] && TOOL_DIR="$${D}" && break; done && \

diff --git a/docker/dist/entry_point.sh b/docker/dist/entry_point.sh
@@ -85,7 +85,7 @@ if [[ "${PLATFORM}" == "Windows_amd64" ]]; then
 
   build_rocksdb TARGET_OS=MINGW CXX="${CXX}"
 
-  make -j$(nproc) update
+  make -j$(nproc) update-from-ci
 
   make \
     -j$(nproc) \
@@ -133,7 +133,7 @@ elif [[ "${PLATFORM}" == "Linux_arm32v7" ]]; then
 
   build_rocksdb TARGET_ARCHITECTURE=arm CXX="${CXX}"
 
-  make -j$(nproc) update
+  make -j$(nproc) update-from-ci
 
   env CFLAGS="" make \
     -j$(nproc) \
@@ -154,7 +154,7 @@ elif [[ "${PLATFORM}" == "Linux_arm64v8" ]]; then
 
   build_rocksdb TARGET_ARCHITECTURE=arm64 CXX="${CXX}"
 
-  make -j$(nproc) update
+  make -j$(nproc) update-from-ci
 
   make \
     -j$(nproc) \
@@ -183,7 +183,7 @@ elif [[ "${PLATFORM}" == "macOS_amd64" ]]; then
 
   build_rocksdb TARGET_OS=Darwin CXX="${CXX}" AR="${AR}"
 
-  make -j$(nproc) update
+  make -j$(nproc) update-from-ci
 
   make \
     -j$(nproc) \
@@ -225,7 +225,7 @@ elif [[ "${PLATFORM}" == "macOS_arm64" ]]; then
 
   build_rocksdb TARGET_OS=Darwin TARGET_ARCHITECTURE=arm64 CXX="${CXX}" AR="${AR}"
 
-  make -j$(nproc) update
+  make -j$(nproc) update-from-ci
 
   make \
     -j$(nproc) \
@@ -259,7 +259,7 @@ else
 
   build_rocksdb
 
-  make -j$(nproc) update
+  make -j$(nproc) update-from-ci
 
   make \
     -j$(nproc) \