[bsc] fix init-from-gcs when using pruneancient

dysnix · Jul 19, 2024 · 4a40ec3 · 4a40ec3
1 parent 488b2da
commit 4a40ec3
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 5 deletions.
diff --git a/dysnix/bsc/Chart.yaml b/dysnix/bsc/Chart.yaml
@@ -1,7 +1,7 @@
 apiVersion: v2
 name: bsc
 description: Binance Smart Chain chart for Kubernetes
-version: 0.6.45
+version: 0.6.46
 appVersion: 1.4.11
 
 keywords:

diff --git a/dysnix/bsc/templates/scripts/_init_from_gcs.tpl b/dysnix/bsc/templates/scripts/_init_from_gcs.tpl
@@ -24,8 +24,10 @@ S_STATE_URL="/state_url"
 S_ANCIENT_URL="/ancient_url"
 S_STATS="/stats"
 MAX_USED_SPACE_PERCENT={{ .Values.bsc.initFromGCS.maxUsedSpacePercent }}
+S5CMD_SYNC_STATE_OPTS=""
+S5CMD_SYNC_ANCIENT_OPTS="--part-size 200 --concurrency 2"
 S5CMD_STATE_OPTS=""
-S5CMD_ANCIENT_OPTS="--part-size 200 --concurrency 2"
+S5CMD_ANCIENT_OPTS=""
 {{- if .Values.bsc.pruneancient }}
 # we expect the source snapshot to be pruned, thus we may increase workers from default 256 to speed things up
 # as ancient dir has less than 100 files with the size less than 10GB, we want to run full speed on state dir instead
@@ -131,9 +133,9 @@ while [ "${SYNC}" -gt 0 ] ; do
     # sync from cloud to local disk, with removing existing [missing in the cloud] files
     # run multiple syncs in background
 
-    time ${S5CMD} sync --delete ${S5CMD_STATE_OPTS} s3://${STATE_SRC}/* ${STATE_TMP_DIR}/ > cplist_state.txt &
+    time ${S5CMD} ${S5CMD_STATE_OPTS} sync --delete ${S5CMD_SYNC_STATE_OPTS} s3://${STATE_SRC}/* ${STATE_TMP_DIR}/ > cplist_state.txt &
     STATE_CP_PID=$!
-    time nice ${S5CMD} sync --delete ${S5CMD_ANCIENT_OPTS} s3://${ANCIENT_SRC}/* ${ANCIENT_TMP_DIR}/ > cplist_ancient.txt &
+    time nice ${S5CMD} ${S5CMD_ANCIENT_OPTS} sync --delete ${S5CMD_SYNC_ANCIENT_OPTS} s3://${ANCIENT_SRC}/* ${ANCIENT_TMP_DIR}/ > cplist_ancient.txt &
     ANCIENT_CP_PID=$!
 
     # wait for all syncs to complete

diff --git a/dysnix/bsc/values.yaml b/dysnix/bsc/values.yaml
@@ -153,7 +153,7 @@ bsc:
     baseUrlOverride: ""                   # "bucket/path/to/dir"
     fullResyncOnSrcUpdate: false
     maxUsedSpacePercent: 93               # percents
-    boostStateCopyWorkers: 800
+    boostStateCopyWorkers: 800            # 800 is ok for 16vCPU and 1500MB/s disk
   syncToGCS:
     enabled: false
     image: peakcom/s5cmd:v2.2.2