From 0bdf31246dba33320a9c5d0783420ba59fae1904 Mon Sep 17 00:00:00 2001
From: Vladyslav Deryhin <vladstarrr@icloud.com>
Date: Thu, 26 Oct 2023 19:11:23 +0300
Subject: [PATCH 1/5] [geth] add upload/download datadir snapshot from S3

---
 dysnix/geth/Chart.yaml                        |   2 +-
 dysnix/geth/templates/configmap-scripts.yaml  |   8 ++
 dysnix/geth/templates/s3-configmap.yaml       |  10 ++
 dysnix/geth/templates/s3-cronjob-rbac.yaml    |  43 ++++++++
 dysnix/geth/templates/s3-cronjob.yaml         |  71 +++++++++++++
 dysnix/geth/templates/s3-secret.yaml          |  10 ++
 .../geth/templates/scripts/_init-from-s3.tpl  | 100 ++++++++++++++++++
 dysnix/geth/templates/scripts/_s3-cron.tpl    |  84 +++++++++++++++
 dysnix/geth/templates/scripts/_s3-env.tpl     |  31 ++++++
 dysnix/geth/templates/scripts/_sync-to-s3.tpl |  75 +++++++++++++
 dysnix/geth/templates/statefulset.yaml        |  65 +++++++++++-
 dysnix/geth/values.yaml                       |  41 ++++++-
 12 files changed, 537 insertions(+), 3 deletions(-)
 create mode 100644 dysnix/geth/templates/s3-configmap.yaml
 create mode 100644 dysnix/geth/templates/s3-cronjob-rbac.yaml
 create mode 100644 dysnix/geth/templates/s3-cronjob.yaml
 create mode 100644 dysnix/geth/templates/s3-secret.yaml
 create mode 100644 dysnix/geth/templates/scripts/_init-from-s3.tpl
 create mode 100644 dysnix/geth/templates/scripts/_s3-cron.tpl
 create mode 100644 dysnix/geth/templates/scripts/_s3-env.tpl
 create mode 100644 dysnix/geth/templates/scripts/_sync-to-s3.tpl

diff --git a/dysnix/geth/Chart.yaml b/dysnix/geth/Chart.yaml
index 950baa38..63682ac2 100644
--- a/dysnix/geth/Chart.yaml
+++ b/dysnix/geth/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: geth
 description: Go-ethereum blockchain node Helm Chart
 
-version: 1.0.10
+version: 1.0.11
 appVersion: v1.13.2
 
 keywords:
diff --git a/dysnix/geth/templates/configmap-scripts.yaml b/dysnix/geth/templates/configmap-scripts.yaml
index c94717a4..8700dc57 100644
--- a/dysnix/geth/templates/configmap-scripts.yaml
+++ b/dysnix/geth/templates/configmap-scripts.yaml
@@ -7,3 +7,11 @@ metadata:
 data:
   check-readiness.sh: |-
     {{- include (print $.Template.BasePath "/scripts/_check-readiness.tpl") . | nindent 4 }}
+  init-from-s3.sh: |-
+    {{- include (print $.Template.BasePath "/scripts/_init-from-s3.tpl") . | nindent 4 }}
+  sync-to-s3.sh: |-
+    {{- include (print $.Template.BasePath "/scripts/_sync-to-s3.tpl") . | nindent 4 }}
+  s3-env.sh: |-
+    {{- include (print $.Template.BasePath "/scripts/_s3-env.tpl") . | nindent 4 }}
+  s3-cron.sh: |-
+    {{- include (print $.Template.BasePath "/scripts/_s3-cron.tpl") . | nindent 4 }}
diff --git a/dysnix/geth/templates/s3-configmap.yaml b/dysnix/geth/templates/s3-configmap.yaml
new file mode 100644
index 00000000..86768c24
--- /dev/null
+++ b/dysnix/geth/templates/s3-configmap.yaml
@@ -0,0 +1,10 @@
+{{- if or .Values.initFromS3.enabled .Values.syncToS3.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "geth.fullname" . }}-s3-config
+data:
+  DATA_DIR: /root/.ethereum
+  S3_BASE_URL: {{ tpl .Values.s3config.baseUrl . }}
+  FORCE_INIT: {{ ternary "True" "False" .Values.initFromS3.force | quote }}
+{{- end }}
diff --git a/dysnix/geth/templates/s3-cronjob-rbac.yaml b/dysnix/geth/templates/s3-cronjob-rbac.yaml
new file mode 100644
index 00000000..5e8afcd5
--- /dev/null
+++ b/dysnix/geth/templates/s3-cronjob-rbac.yaml
@@ -0,0 +1,43 @@
+{{- if .Values.syncToS3.cronjob.enabled -}}
+{{- $fullName := print (include "geth.fullname" .) "-s3-cronjob" }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ $fullName }}
+  labels: {{ include "geth.labels" . | nindent 4 }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ $fullName }}
+  labels: {{ include "geth.labels" . | nindent 4 }}
+rules:
+  - apiGroups: [""]
+    resources:
+      - pods
+    verbs:
+      - get
+      - list
+      - watch
+      - delete
+  - apiGroups: [""]
+    resources:
+      - configmaps
+    verbs:
+      - get
+      - patch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ $fullName }}
+  labels: {{ include "geth.labels" . | nindent 4 }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $fullName }}
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: Role
+  name: {{ $fullName }}
+  apiGroup: rbac.authorization.k8s.io
+{{- end }}
diff --git a/dysnix/geth/templates/s3-cronjob.yaml b/dysnix/geth/templates/s3-cronjob.yaml
new file mode 100644
index 00000000..fce74121
--- /dev/null
+++ b/dysnix/geth/templates/s3-cronjob.yaml
@@ -0,0 +1,71 @@
+{{- if .Values.syncToS3.cronjob.enabled }}
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ include "geth.fullname" . }}-sync-to-s3
+  labels:
+    {{- include "geth.labels" . | nindent 4 }}
+spec:
+  {{- with .Values.syncToS3.cronjob }}
+  schedule: "{{ .schedule }}"
+  concurrencyPolicy: Forbid
+  startingDeadlineSeconds: 300
+  jobTemplate:
+    metadata:
+      name: {{ include "geth.fullname" $ }}-sync-to-s3
+    spec:
+      activeDeadlineSeconds: 60
+      backoffLimit: 0
+      template:
+        metadata:
+          labels:
+            {{- include "geth.labels" $ | nindent 12 }}
+        spec:
+          restartPolicy: OnFailure
+          {{- with .imagePullSecrets }}
+          imagePullSecrets:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          serviceAccountName: {{ include "geth.fullname" $ }}-s3-cronjob
+          {{- with .podSecurityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .nodeSelector }}
+          nodeSelector:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .affinity }}
+          affinity:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .tolerations }}
+          tolerations:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          containers:
+          - name: enable-sync-to-s3
+            image: "{{ .image.repository }}:{{ .image.tag }}"
+            imagePullPolicy: {{ .image.pullPolicy | quote }}
+            {{- with .securityContext }}
+            securityContext:
+              {{- toYaml . | nindent 14 }}
+            {{- end }}
+            command:
+            - /bin/sh
+            - /scripts/s3-cron.sh
+            - enable_sync
+            - 5s
+            volumeMounts:
+            - name: scripts
+              mountPath: /scripts
+            {{- with .resources }}
+            resources:
+              {{- toYaml . | nindent 14 }}
+            {{- end }}
+          volumes:
+          - name: scripts
+            configMap:
+              name: {{ template "geth.fullname" $ }}-scripts
+  {{- end }}
+{{- end }}
diff --git a/dysnix/geth/templates/s3-secret.yaml b/dysnix/geth/templates/s3-secret.yaml
new file mode 100644
index 00000000..a94b046b
--- /dev/null
+++ b/dysnix/geth/templates/s3-secret.yaml
@@ -0,0 +1,10 @@
+{{- if or .Values.initFromS3.enabled .Values.syncToS3.enabled }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "geth.fullname" . }}-s3-secret
+data:
+  S3_ENDPOINT_URL: {{ .Values.s3config.endpointUrl | toString | b64enc }}
+  AWS_ACCESS_KEY_ID: {{ .Values.s3config.accessKeyId | toString | b64enc }}
+  AWS_SECRET_ACCESS_KEY: {{ .Values.s3config.secretAccessKey | toString | b64enc }}
+{{- end }}
diff --git a/dysnix/geth/templates/scripts/_init-from-s3.tpl b/dysnix/geth/templates/scripts/_init-from-s3.tpl
new file mode 100644
index 00000000..5b7db22e
--- /dev/null
+++ b/dysnix/geth/templates/scripts/_init-from-s3.tpl
@@ -0,0 +1,100 @@
+#!/usr/bin/env sh
+# shellcheck disable=SC2086,SC3037
+
+set -e
+
+. /scripts/s3-env.sh
+
+process_inputs() {
+  # download even if already initialized
+  if [ "$FORCE_INIT" = "True" ]; then
+    echo "Force init enabled, existing data will be deleted."
+    rm -f "$INITIALIZED_FILE"
+  fi
+  # check if we are already initialized
+  if [ -f "$INITIALIZED_FILE" ]; then
+    echo "Blockchain already initialized. Exiting..."; exit 0
+  fi
+  # check for S3 credentials
+  if [ -z "$S3_ENDPOINT_URL" ] || [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ]; then
+    echo "S3 credentials are not provided, exiting"; exit 1
+  fi
+}
+
+get_src_urls() {
+  chaindata_src=$("$S5CMD" cat "s3://${CHAINDATA_URL}")
+  ancient_src=$("$S5CMD" cat "s3://${ANCIENT_URL}")
+  remote_stats=$("$S5CMD" cat "s3://${STATS_URL}")
+}
+
+progress() {
+  case $1 in
+  "start")
+    while true; do
+      inodes=$(df -Phi "$DATA_DIR" | tail -n 1 | awk '{print $3}')
+      size=$(df -P -BG "$DATA_DIR" | tail -n 1 | awk '{print $3}')G
+      echo -e "$(date -Iseconds) | SOURCE TOTAL ${remote_stats} | DST USED Inodes:\t${inodes} Size:\t${size}"
+      sleep 2
+    done &
+    progress_pid=$! ;;
+  "stop")
+    kill "$progress_pid"
+    progress_pid=0 ;;
+  "*")
+    echo "Unknown arg" ;;
+  esac
+}
+
+check_lockfile() {
+  if "$S5CMD" cat "s3://${LOCKFILE_URL}" >/dev/null 2>&1; then
+    echo "Found existing lockfile, snapshot might be corrupted. Aborting download.."
+    exit 1
+  fi
+}
+
+# stop all background tasks
+interrupt() {
+  echo "Got interrupt signal, stopping..."
+  for i in "$@"; do kill $i; done
+}
+
+sync() {
+  # cleanup data always, s5cmd does not support "true" sync, it does not save object's timestamps
+  # https://github.com/peak/s5cmd/issues/532
+  echo "Cleaning up local data..."
+  rm -rf "$ANCIENT_DIR"
+  rm -rf "$CHAINDATA_DIR"
+  # recreate data directories
+  mkdir -p "$CHAINDATA_DIR"
+  mkdir -p "$ANCIENT_DIR"
+
+  echo "Starting download data from S3..."
+  progress start
+
+  # perform remote snapshot download and remove local objects which don't exist in snapshot
+  # run two jobs in parallel, one for chaindata, second for ancient data
+  time "$S5CMD" --stat sync $EXCLUDE_ANCIENT "s3://${chaindata_src}/*" "${CHAINDATA_DIR}/" >/dev/null &
+  download_chaindata=$!
+  time nice "$S5CMD" --stat sync --part-size 200 --concurrency 2 $EXCLUDE_CHAINDATA "s3://${ancient_src}/*" "${ANCIENT_DIR}/" >/dev/null &
+  download_ancient=$!
+
+  # handle interruption / termination
+  trap 'interrupt ${download_chaindata} ${download_ancient} ${progress_pid}' INT TERM
+  # wait for all syncs to complete
+  wait $download_chaindata $download_ancient
+
+  progress stop
+
+  # all done, mark as initialized
+  touch "$INITIALIZED_FILE"
+}
+
+
+main() {
+  process_inputs
+  get_src_urls
+  check_lockfile
+  sync
+}
+
+main
\ No newline at end of file
diff --git a/dysnix/geth/templates/scripts/_s3-cron.tpl b/dysnix/geth/templates/scripts/_s3-cron.tpl
new file mode 100644
index 00000000..c24e68d7
--- /dev/null
+++ b/dysnix/geth/templates/scripts/_s3-cron.tpl
@@ -0,0 +1,84 @@
+#!/usr/bin/env sh
+# shellcheck disable=SC1083
+
+MODE="$1"
+WAIT_TIMEOUT="$2"
+CONFIGMAP_NAME={{ include "geth.fullname" . }}-s3-config
+KUBECTL=$(which kubectl)
+PATCH_DATA=""
+POD_NAME={{ include "geth.fullname" . }}-0
+
+check_ret(){
+        ret="$1"
+        msg="$2"
+        # allow to override exit code, default value is ret
+        exit_code=${3:-${ret}}
+        if [ ! "$ret" -eq 0 ]; then
+                echo "$msg"
+                echo "return code ${ret}, exit code ${exit_code}"
+                exit "$exit_code"
+        fi
+}
+
+check_pod_readiness() {
+  # wait for pod to become ready
+  echo "$(date -Iseconds) Waiting ${WAIT_TIMEOUT} for pod ${1} to become ready ..."
+  "$KUBECTL" wait --timeout="$WAIT_TIMEOUT" --for=condition=Ready pod "$1"
+  check_ret $? "$(date -Iseconds) Pod ${1} is not ready, nothing to do, exiting" 0
+
+  # ensuring pod is not terminating now
+  # https://github.com/kubernetes/kubernetes/issues/22839
+  echo "$(date -Iseconds) Checking for pod ${1} to not terminate ..."
+  deletion_timestamp=$("$KUBECTL" get -o jsonpath='{.metadata.deletionTimestamp}' pod "$1")
+  check_ret $? "$(date -Iseconds) Cannot get pod ${1}, abort"
+
+  if [ -z "$deletion_timestamp" ]; then
+    echo "$(date -Iseconds) Pod ${1} is ready, continuing"
+  else
+    echo "$(date -Iseconds) Pod ${1} is terminating, try another time"
+  fi
+}
+
+enable_sync() {
+  echo "$(date -Iseconds) Patching configmap ${CONFIGMAP_NAME} to enable sync"
+  PATCH_DATA='{"data":{"SYNC_TO_S3":"True"}}'
+}
+
+disable_sync() {
+  echo "$(date -Iseconds) Patching configmap ${CONFIGMAP_NAME} to disable sync"
+  PATCH_DATA='{"data":{"SYNC_TO_S3":"False"}}'
+}
+
+patch_configmap() {
+  "$KUBECTL" patch configmap "$CONFIGMAP_NAME" --type merge --patch "$PATCH_DATA"
+  check_ret $? "$(date -Iseconds) Fatal: cannot patch configmap ${CONFIGMAP_NAME}, abort"
+}
+
+delete_pod() {
+  echo "$(date -Iseconds) Deleting pod ${1} to trigger action inside init container ..."
+  # delete the pod to trigger action inside init container
+  "$KUBECTL" delete pod "$1" --wait=false
+  check_ret $? "$(date -Iseconds) Fatal: cannot delete pod ${1}, abort"
+  echo "$(date -Iseconds) Pod ${1} deleted successfully, exiting. Check pod logs after restart."
+}
+
+main() {
+  case "$MODE" in
+  "enable_sync")
+    check_pod_readiness "$POD_NAME"
+    enable_sync
+    patch_configmap
+    delete_pod "$POD_NAME"
+    ;;
+  # intended to be run in sidecar after successful sync, don't interact with pod
+  "disable_sync")
+    disable_sync
+    patch_configmap
+    ;;
+  "*")
+    check_ret 1 "$(date -Iseconds) Mode value \"$MODE\" is incorrect, abort"
+    ;;
+  esac
+}
+
+main
\ No newline at end of file
diff --git a/dysnix/geth/templates/scripts/_s3-env.tpl b/dysnix/geth/templates/scripts/_s3-env.tpl
new file mode 100644
index 00000000..f43f74e7
--- /dev/null
+++ b/dysnix/geth/templates/scripts/_s3-env.tpl
@@ -0,0 +1,31 @@
+#!/usr/bin/env sh
+
+export S5CMD=/s5cmd
+
+# chaindata options
+export EXCLUDE_ANCIENT="--exclude *.cidx --exclude *.ridx --exclude *.cdat --exclude *.rdat"
+export EXCLUDE_CHAINDATA="--exclude *.ldb --exclude *.sst"
+
+# local directory structure config
+export DATA_DIR="${DATA_DIR:-/root/.ethereum}"
+export CHAINDATA_DIR="${CHAINDATA_DIR:-${DATA_DIR}/geth/chaindata}"
+export ANCIENT_DIR="${ANCIENT_DIR:-${CHAINDATA_DIR}/ancient}"
+export INITIALIZED_FILE="${DATA_DIR}/.initialized"
+
+# s3 config
+export S3_BASE_URL="${S3_BASE_URL?S3_BASE_URL not provided.}"
+
+# s3 directory structure config
+export S_COMPLETED="/completed"
+export S_CHAINDATA_URL="/chaindata_url"
+export S_ANCIENT_URL="/ancient_url"
+export S_STATS="/stats"
+export S_LOCKFILE="/lockfile"
+export COMPLETED_URL="${S3_BASE_URL}${S_COMPLETED}"
+export LOCKFILE_URL="${S3_BASE_URL}${S_LOCKFILE}"
+export STATS_URL="${S3_BASE_URL}${S_STATS}"
+export CHAINDATA_URL="${S3_BASE_URL}${S_CHAINDATA_URL}"
+export ANCIENT_URL="${S3_BASE_URL}${S_ANCIENT_URL}"
+
+# download/upload options
+export FORCE_INIT="${FORCE_INIT:-False}"
\ No newline at end of file
diff --git a/dysnix/geth/templates/scripts/_sync-to-s3.tpl b/dysnix/geth/templates/scripts/_sync-to-s3.tpl
new file mode 100644
index 00000000..add55cb7
--- /dev/null
+++ b/dysnix/geth/templates/scripts/_sync-to-s3.tpl
@@ -0,0 +1,75 @@
+#!/usr/bin/env sh
+# shellcheck disable=SC2086,SC3037
+
+set -e
+
+. /scripts/s3-env.sh
+
+process_inputs() {
+  # enable sync via env variable
+  if [ "$SYNC_TO_S3" != "True" ]; then
+    echo "Sync is not enabled in config, exiting"
+    exit 0
+  fi
+  # check for S3 credentials
+  if [ -z "$S3_ENDPOINT_URL" ] || [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ]; then
+    echo "S3 credentials are not provided, exiting"
+    exit 1
+  fi
+}
+
+check_recent_init() {
+  # if node has been initialized from snapshot <30 mins ago, skip upload
+  is_recent=$(find "$INITIALIZED_FILE" -type f -mmin +30 | wc -l | tr -d '[:blank:]')
+
+  if [ -f "$INITIALIZED_FILE" ] && [ "$is_recent" -eq 0 ]; then
+    echo "Node has been initialized recently, skipping the upload. Exiting..."; exit 0
+  fi
+}
+
+# stop all background processes
+interrupt() {
+  echo "Got interrupt signal, stopping..."
+  for i in "$@"; do kill $i; done
+}
+
+sync() {
+  # get destinations
+  chaindata_dst=$("$S5CMD" cat "s3://${CHAINDATA_URL}")
+  ancient_dst=$("$S5CMD" cat "s3://${ANCIENT_URL}")
+
+  # add lockfile while uploading
+  # shellcheck disable=SC3028
+  echo "${HOSTNAME} $(date +%s)" | "$S5CMD" pipe "s3://${LOCKFILE_URL}"
+
+  # perform upload of local data and remove destination objects which don't exist locally
+  # run two jobs in parallel, one for chaindata, second for ancient data
+  time "$S5CMD" --stat sync --delete $EXCLUDE_ANCIENT "${CHAINDATA_DIR}/" "s3://${chaindata_dst}/" >/dev/null &
+  upload_chaindata=$!
+  time nice "$S5CMD" --stat sync --delete --part-size 200 --concurrency 2 $EXCLUDE_CHAINDATA "${ANCIENT_DIR}/" "s3://${ancient_dst}/" >/dev/null &
+  upload_ancient=$!
+
+  # handle interruption / termination
+  trap 'interrupt ${upload_chaindata} ${upload_ancient}' INT TERM
+  # wait for parallel upload to complete
+  wait $upload_chaindata $upload_ancient
+
+  # mark upload as completed
+  date +%s | "$S5CMD" pipe "s3://${COMPLETED_URL}"
+  "$S5CMD" rm "s3://${LOCKFILE_URL}"
+}
+
+update_stats() {
+  inodes=$(df -Phi "${DATA_DIR}" | tail -n 1 | awk '{print $3}')
+  size=$(df -P -BG "${DATA_DIR}" | tail -n 1 | awk '{print $3}')G
+  echo -ne "Inodes:\t${inodes} Size:\t${size}" | "$S5CMD" pipe "s3://${STATS_URL}"
+}
+
+main() {
+  process_inputs
+  check_recent_init
+  sync
+  update_stats
+}
+
+main
\ No newline at end of file
diff --git a/dysnix/geth/templates/statefulset.yaml b/dysnix/geth/templates/statefulset.yaml
index 301715e8..8af61c7a 100644
--- a/dysnix/geth/templates/statefulset.yaml
+++ b/dysnix/geth/templates/statefulset.yaml
@@ -15,6 +15,10 @@ spec:
     metadata:
       annotations:
         checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+        {{- if or .Values.syncToS3.enabled .Values.initFromS3.enabled }}
+        checksum/s3-secret: {{ include (print $.Template.BasePath "/s3-secret.yaml") . | sha256sum }}
+        checksum/s3-configmap: {{ include (print $.Template.BasePath "/s3-configmap.yaml") . | sha256sum }}
+        {{- end }}
       labels:
         {{- include "geth.selectorLabels" . | nindent 8 }}
         {{- with .Values.podStatusLabels }}
@@ -26,7 +30,11 @@ spec:
       imagePullSecrets:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- if .Values.syncToS3.cronjob.enabled }}
+      serviceAccountName: {{ include "geth.fullname" . }}-s3-cronjob
+      {{- else }}
       serviceAccountName: {{ include "geth.serviceAccountName" . }}
+      {{- end }}
       {{- with .Values.podSecurityContext }}
       securityContext:
         {{- toYaml . | nindent 10 }}
@@ -43,8 +51,63 @@ spec:
       tolerations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      {{- with .Values.initContainers }}
       initContainers:
+      {{- if .Values.initFromS3.enabled }}
+      {{- with .Values.s3config }}
+      - name: init-from-s3
+        image: "{{ .image.repository }}:{{ .image.tag }}"
+        imagePullPolicy: {{ .image.pullPolicy | quote }}
+        command:
+        - sh
+        - /scripts/init-from-s3.sh
+        envFrom:
+        - configMapRef:
+            name: {{ include "geth.fullname" $ }}-s3-config
+        - secretRef:
+            name: {{ include "geth.fullname" $ }}-s3-secret
+        volumeMounts:
+        - name: scripts
+          mountPath: /scripts
+        - name: data
+          mountPath: /root/.ethereum
+      {{- end }}
+      {{- end }}
+      {{- if .Values.syncToS3.enabled }}
+      {{- with .Values.s3config }}
+      - name: sync-to-s3
+        image: "{{ .image.repository }}:{{ .image.tag }}"
+        imagePullPolicy: {{ .image.pullPolicy | quote }}
+        command:
+        - /bin/sh
+        - /scripts/sync-to-s3.sh
+        envFrom:
+        - configMapRef:
+            name: {{ include "geth.fullname" $ }}-s3-config
+        - secretRef:
+            name: {{ include "geth.fullname" $ }}-s3-secret
+        volumeMounts:
+        - name: scripts
+          mountPath: /scripts
+        - name: data
+          mountPath: /root/.ethereum
+      {{- end }}
+      {{- with .Values.syncToS3.cronjob }}
+      {{- if .enabled }}
+      - name: disable-sync-to-s3
+        image: "{{ .image.repository }}:{{ .image.tag }}"
+        imagePullPolicy: {{ .image.pullPolicy | quote }}
+        command:
+        - /bin/sh
+        - /scripts/s3-cron.sh
+        - disable_sync
+        - 5s
+        volumeMounts:
+        - name: scripts
+          mountPath: /scripts
+      {{- end }}
+      {{- end }}
+      {{- end }}
+      {{- with .Values.extraInitContainers }}
         {{- tpl (toYaml . | nindent 6) $ }}
       {{- end }}
       containers:
diff --git a/dysnix/geth/values.yaml b/dysnix/geth/values.yaml
index 69f2cb44..73f9f697 100644
--- a/dysnix/geth/values.yaml
+++ b/dysnix/geth/values.yaml
@@ -50,7 +50,7 @@ command: []
 extraArgs: []
 
 ## Extra init containers, can be templated
-initContainers: []
+extraInitContainers: []
   # - name: dumpconfig
   #   image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
   #   imagePullPolicy: "{{ .Values.image.pullPolicy }}"
@@ -265,3 +265,42 @@ config:
   pprof:
     enabled: false
     port: 6061
+
+### S3 directory structure
+### Currenly tested only 1 baseUrl per 1 node deployment
+###
+###   baseUrl
+###   ├── ancient_url
+###   └── chaindata_url
+###
+###   cat ancient_url   ->  baseUrl/<your_ancient_destination>
+###   cat chaindata_url ->  baseUrl/<your_chaindata_destination>
+###
+
+s3config:
+  image:
+    repository: peakcom/s5cmd
+    tag: v2.2.2
+    pullPolicy: IfNotPresent
+  baseUrl: S3-BUCKET-NAME/{{ .Release.Name }}
+  endpointUrl: https://storage.googleapis.com
+  accessKeyId: "REPLACEME"
+  secretAccessKey: "REPLACEME"
+
+initFromS3:
+  # enable initContainer
+  enabled: false
+  # download snapshot from S3 on every pod start
+  force: false
+
+syncToS3:
+  # enable initContainer
+  enabled: false
+  # restart pod and trigger sync to S3 inside initContainer by schedule
+  cronjob:
+    enabled: false
+    image:
+      repository: dysnix/kubectl
+      tag: v1.27
+      pullPolicy: IfNotPresent
+    schedule: "0 2 * * *"

From 94b12d083c0db2b0024236f84b435b4fd7a6424a Mon Sep 17 00:00:00 2001
From: Vladyslav Deryhin <vladstarrr@icloud.com>
Date: Thu, 26 Oct 2023 19:37:18 +0300
Subject: [PATCH 2/5] [geth] fix pod deletion timestamp check in s3-cron

---
 dysnix/geth/templates/scripts/_s3-cron.tpl | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/dysnix/geth/templates/scripts/_s3-cron.tpl b/dysnix/geth/templates/scripts/_s3-cron.tpl
index c24e68d7..aef49851 100644
--- a/dysnix/geth/templates/scripts/_s3-cron.tpl
+++ b/dysnix/geth/templates/scripts/_s3-cron.tpl
@@ -32,11 +32,8 @@ check_pod_readiness() {
   deletion_timestamp=$("$KUBECTL" get -o jsonpath='{.metadata.deletionTimestamp}' pod "$1")
   check_ret $? "$(date -Iseconds) Cannot get pod ${1}, abort"
 
-  if [ -z "$deletion_timestamp" ]; then
-    echo "$(date -Iseconds) Pod ${1} is ready, continuing"
-  else
-    echo "$(date -Iseconds) Pod ${1} is terminating, try another time"
-  fi
+  [ -n "$deletion_timestamp" ]
+  check_ret $? "$(date -Iseconds) Pod ${1} is terminating now, try another time" 1
 }
 
 enable_sync() {

From bac14ea38148ff4648f3742d255df499d275e653 Mon Sep 17 00:00:00 2001
From: Vladyslav Deryhin <vladstarrr@icloud.com>
Date: Thu, 26 Oct 2023 19:39:19 +0300
Subject: [PATCH 3/5] [geth] fix pod deletion timestamp check in s3-cron

---
 dysnix/geth/templates/scripts/_s3-cron.tpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dysnix/geth/templates/scripts/_s3-cron.tpl b/dysnix/geth/templates/scripts/_s3-cron.tpl
index aef49851..3cff7620 100644
--- a/dysnix/geth/templates/scripts/_s3-cron.tpl
+++ b/dysnix/geth/templates/scripts/_s3-cron.tpl
@@ -32,7 +32,7 @@ check_pod_readiness() {
   deletion_timestamp=$("$KUBECTL" get -o jsonpath='{.metadata.deletionTimestamp}' pod "$1")
   check_ret $? "$(date -Iseconds) Cannot get pod ${1}, abort"
 
-  [ -n "$deletion_timestamp" ]
+  [ -z "$deletion_timestamp" ]
   check_ret $? "$(date -Iseconds) Pod ${1} is terminating now, try another time" 1
 }
 

From 2270989226ed201ee92d48aa6c6bcbdb705ff802 Mon Sep 17 00:00:00 2001
From: Vladyslav Deryhin <vladstarrr@icloud.com>
Date: Fri, 27 Oct 2023 12:41:08 +0300
Subject: [PATCH 4/5] [geth] don't require s3 bucket to be filled with
 ancient/chaindata_url files, add more comments

---
 dysnix/geth/templates/s3-configmap.yaml       |  3 ++
 .../geth/templates/scripts/_init-from-s3.tpl  | 12 ++------
 dysnix/geth/templates/scripts/_s3-cron.tpl    |  2 +-
 dysnix/geth/templates/scripts/_s3-env.tpl     | 14 ++++-----
 dysnix/geth/templates/scripts/_sync-to-s3.tpl |  8 ++---
 dysnix/geth/values.yaml                       | 29 +++++++++----------
 6 files changed, 29 insertions(+), 39 deletions(-)

diff --git a/dysnix/geth/templates/s3-configmap.yaml b/dysnix/geth/templates/s3-configmap.yaml
index 86768c24..128300ea 100644
--- a/dysnix/geth/templates/s3-configmap.yaml
+++ b/dysnix/geth/templates/s3-configmap.yaml
@@ -5,6 +5,9 @@ metadata:
   name: {{ include "geth.fullname" . }}-s3-config
 data:
   DATA_DIR: /root/.ethereum
+  SYNC_TO_S3: "False"
   S3_BASE_URL: {{ tpl .Values.s3config.baseUrl . }}
+  S3_CHAINDATA_URL: {{ tpl .Values.s3config.chaindataUrl . }}
+  S3_ANCIENT_URL: {{ tpl .Values.s3config.ancientUrl . }}
   FORCE_INIT: {{ ternary "True" "False" .Values.initFromS3.force | quote }}
 {{- end }}
diff --git a/dysnix/geth/templates/scripts/_init-from-s3.tpl b/dysnix/geth/templates/scripts/_init-from-s3.tpl
index 5b7db22e..3c4ffad1 100644
--- a/dysnix/geth/templates/scripts/_init-from-s3.tpl
+++ b/dysnix/geth/templates/scripts/_init-from-s3.tpl
@@ -21,13 +21,8 @@ process_inputs() {
   fi
 }
 
-get_src_urls() {
-  chaindata_src=$("$S5CMD" cat "s3://${CHAINDATA_URL}")
-  ancient_src=$("$S5CMD" cat "s3://${ANCIENT_URL}")
-  remote_stats=$("$S5CMD" cat "s3://${STATS_URL}")
-}
-
 progress() {
+  remote_stats=$("$S5CMD" cat "s3://${STATS_URL}")
   case $1 in
   "start")
     while true; do
@@ -73,9 +68,9 @@ sync() {
 
   # perform remote snapshot download and remove local objects which don't exist in snapshot
   # run two jobs in parallel, one for chaindata, second for ancient data
-  time "$S5CMD" --stat sync $EXCLUDE_ANCIENT "s3://${chaindata_src}/*" "${CHAINDATA_DIR}/" >/dev/null &
+  time "$S5CMD" --stat sync $EXCLUDE_ANCIENT "s3://${CHAINDATA_URL}/*" "${CHAINDATA_DIR}/" >/dev/null &
   download_chaindata=$!
-  time nice "$S5CMD" --stat sync --part-size 200 --concurrency 2 $EXCLUDE_CHAINDATA "s3://${ancient_src}/*" "${ANCIENT_DIR}/" >/dev/null &
+  time nice "$S5CMD" --stat sync --part-size 200 --concurrency 2 $EXCLUDE_CHAINDATA "s3://${ANCIENT_URL}/*" "${ANCIENT_DIR}/" >/dev/null &
   download_ancient=$!
 
   # handle interruption / termination
@@ -92,7 +87,6 @@ sync() {
 
 main() {
   process_inputs
-  get_src_urls
   check_lockfile
   sync
 }
diff --git a/dysnix/geth/templates/scripts/_s3-cron.tpl b/dysnix/geth/templates/scripts/_s3-cron.tpl
index 3cff7620..7ad08782 100644
--- a/dysnix/geth/templates/scripts/_s3-cron.tpl
+++ b/dysnix/geth/templates/scripts/_s3-cron.tpl
@@ -67,7 +67,7 @@ main() {
     patch_configmap
     delete_pod "$POD_NAME"
     ;;
-  # intended to be run in sidecar after successful sync, don't interact with pod
+  # intended to be run inside initContainer after successful sync
   "disable_sync")
     disable_sync
     patch_configmap
diff --git a/dysnix/geth/templates/scripts/_s3-env.tpl b/dysnix/geth/templates/scripts/_s3-env.tpl
index f43f74e7..e39756c5 100644
--- a/dysnix/geth/templates/scripts/_s3-env.tpl
+++ b/dysnix/geth/templates/scripts/_s3-env.tpl
@@ -12,20 +12,18 @@ export CHAINDATA_DIR="${CHAINDATA_DIR:-${DATA_DIR}/geth/chaindata}"
 export ANCIENT_DIR="${ANCIENT_DIR:-${CHAINDATA_DIR}/ancient}"
 export INITIALIZED_FILE="${DATA_DIR}/.initialized"
 
-# s3 config
-export S3_BASE_URL="${S3_BASE_URL?S3_BASE_URL not provided.}"
-
 # s3 directory structure config
+export S3_BASE_URL="${S3_BASE_URL?S3_BASE_URL not provided.}"
+export S3_CHAINDATA_URL="${S3_CHAINDATA_URL?S3_CHAINDATA_URL not provided.}"
+export S3_ANCIENT_URL="${S3_ANCIENT_URL?S3_ANCIENT_URL not provided.}"
 export S_COMPLETED="/completed"
-export S_CHAINDATA_URL="/chaindata_url"
-export S_ANCIENT_URL="/ancient_url"
 export S_STATS="/stats"
 export S_LOCKFILE="/lockfile"
+export CHAINDATA_URL="${S3_BASE_URL}${S3_CHAINDATA_URL}"
+export ANCIENT_URL="${S3_BASE_URL}${S3_ANCIENT_URL}"
 export COMPLETED_URL="${S3_BASE_URL}${S_COMPLETED}"
 export LOCKFILE_URL="${S3_BASE_URL}${S_LOCKFILE}"
 export STATS_URL="${S3_BASE_URL}${S_STATS}"
-export CHAINDATA_URL="${S3_BASE_URL}${S_CHAINDATA_URL}"
-export ANCIENT_URL="${S3_BASE_URL}${S_ANCIENT_URL}"
 
 # download/upload options
-export FORCE_INIT="${FORCE_INIT:-False}"
\ No newline at end of file
+export FORCE_INIT="${FORCE_INIT:-False}"
diff --git a/dysnix/geth/templates/scripts/_sync-to-s3.tpl b/dysnix/geth/templates/scripts/_sync-to-s3.tpl
index add55cb7..b07d6365 100644
--- a/dysnix/geth/templates/scripts/_sync-to-s3.tpl
+++ b/dysnix/geth/templates/scripts/_sync-to-s3.tpl
@@ -34,19 +34,15 @@ interrupt() {
 }
 
 sync() {
-  # get destinations
-  chaindata_dst=$("$S5CMD" cat "s3://${CHAINDATA_URL}")
-  ancient_dst=$("$S5CMD" cat "s3://${ANCIENT_URL}")
-
   # add lockfile while uploading
   # shellcheck disable=SC3028
   echo "${HOSTNAME} $(date +%s)" | "$S5CMD" pipe "s3://${LOCKFILE_URL}"
 
   # perform upload of local data and remove destination objects which don't exist locally
   # run two jobs in parallel, one for chaindata, second for ancient data
-  time "$S5CMD" --stat sync --delete $EXCLUDE_ANCIENT "${CHAINDATA_DIR}/" "s3://${chaindata_dst}/" >/dev/null &
+  time "$S5CMD" --stat sync --delete $EXCLUDE_ANCIENT "${CHAINDATA_DIR}/" "s3://${CHAINDATA_URL}/" &
   upload_chaindata=$!
-  time nice "$S5CMD" --stat sync --delete --part-size 200 --concurrency 2 $EXCLUDE_CHAINDATA "${ANCIENT_DIR}/" "s3://${ancient_dst}/" >/dev/null &
+  time nice "$S5CMD" --stat sync --delete --part-size 200 --concurrency 2 $EXCLUDE_CHAINDATA "${ANCIENT_DIR}/" "s3://${ANCIENT_URL}/" &
   upload_ancient=$!
 
   # handle interruption / termination
diff --git a/dysnix/geth/values.yaml b/dysnix/geth/values.yaml
index 73f9f697..e1711797 100644
--- a/dysnix/geth/values.yaml
+++ b/dysnix/geth/values.yaml
@@ -266,26 +266,25 @@ config:
     enabled: false
     port: 6061
 
-### S3 directory structure
-### Currenly tested only 1 baseUrl per 1 node deployment
-###
-###   baseUrl
-###   ├── ancient_url
-###   └── chaindata_url
-###
-###   cat ancient_url   ->  baseUrl/<your_ancient_destination>
-###   cat chaindata_url ->  baseUrl/<your_chaindata_destination>
-###
-
 s3config:
   image:
     repository: peakcom/s5cmd
     tag: v2.2.2
     pullPolicy: IfNotPresent
-  baseUrl: S3-BUCKET-NAME/{{ .Release.Name }}
+  # Any S3-compatible object storage service should be supported, but has only been tested with GCS.
+  # I.e. Amazon S3, MinIO, DigitalOcean Spaces, CloudFlare R2.
+  # endpointUrl: https://s3.amazonaws.com
   endpointUrl: https://storage.googleapis.com
-  accessKeyId: "REPLACEME"
-  secretAccessKey: "REPLACEME"
+  # Assuming your S3 bucket name is `my-snapshot-bucket` and base directory name is Helm release name
+  baseUrl: my-snapshot-bucket/{{ .Release.Name }}
+  # These are relative to baseUrl
+  chaindataUrl: /chaindata
+  ancientUrl: /ancient
+  # How to create access key
+  # AWS S3 https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html
+  # GCS    https://cloud.google.com/storage/docs/authentication/managing-hmackeys#create
+  accessKeyId: REPLACEME
+  secretAccessKey: REPLACEME
 
 initFromS3:
   # enable initContainer
@@ -294,7 +293,7 @@ initFromS3:
   force: false
 
 syncToS3:
-  # enable initContainer
+  # enable initContainer (won't enable actual sync)
   enabled: false
   # restart pod and trigger sync to S3 inside initContainer by schedule
   cronjob:

From 1e583e07624a661cdcaa58cece2e3b05a7dc106d Mon Sep 17 00:00:00 2001
From: Vladyslav Deryhin <vladstarrr@icloud.com>
Date: Fri, 27 Oct 2023 14:24:28 +0300
Subject: [PATCH 5/5] [geth] separate rbac for disable-sync-to-s3

---
 dysnix/geth/templates/configmap-scripts.yaml |  2 ++
 dysnix/geth/templates/rbac.yaml              | 31 ++++++++++++++++++++
 dysnix/geth/templates/s3-cronjob-rbac.yaml   |  2 ++
 dysnix/geth/templates/statefulset.yaml       |  4 ---
 4 files changed, 35 insertions(+), 4 deletions(-)
 create mode 100644 dysnix/geth/templates/rbac.yaml

diff --git a/dysnix/geth/templates/configmap-scripts.yaml b/dysnix/geth/templates/configmap-scripts.yaml
index 8700dc57..afc9302e 100644
--- a/dysnix/geth/templates/configmap-scripts.yaml
+++ b/dysnix/geth/templates/configmap-scripts.yaml
@@ -7,6 +7,7 @@ metadata:
 data:
   check-readiness.sh: |-
     {{- include (print $.Template.BasePath "/scripts/_check-readiness.tpl") . | nindent 4 }}
+  {{- if or .Values.syncToS3.enabled .Values.initFromS3.eanbled }}
   init-from-s3.sh: |-
     {{- include (print $.Template.BasePath "/scripts/_init-from-s3.tpl") . | nindent 4 }}
   sync-to-s3.sh: |-
@@ -15,3 +16,4 @@ data:
     {{- include (print $.Template.BasePath "/scripts/_s3-env.tpl") . | nindent 4 }}
   s3-cron.sh: |-
     {{- include (print $.Template.BasePath "/scripts/_s3-cron.tpl") . | nindent 4 }}
+  {{- end }}
diff --git a/dysnix/geth/templates/rbac.yaml b/dysnix/geth/templates/rbac.yaml
new file mode 100644
index 00000000..975d4827
--- /dev/null
+++ b/dysnix/geth/templates/rbac.yaml
@@ -0,0 +1,31 @@
+{{- if .Values.syncToS3.enabled }}
+{{- $fullName := include "geth.fullname" . }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ $fullName }}
+  labels: {{ include "geth.labels" . | nindent 4 }}
+rules:
+  - apiGroups: [""]
+    resources:
+      - configmaps
+    resourceNames:
+      - {{ $fullName }}-s3-config
+    verbs:
+      - get
+      - patch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ $fullName }}
+  labels: {{ include "geth.labels" . | nindent 4 }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $fullName }}
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: Role
+  name: {{ $fullName }}
+  apiGroup: rbac.authorization.k8s.io
+{{- end }}
diff --git a/dysnix/geth/templates/s3-cronjob-rbac.yaml b/dysnix/geth/templates/s3-cronjob-rbac.yaml
index 5e8afcd5..4f5545a8 100644
--- a/dysnix/geth/templates/s3-cronjob-rbac.yaml
+++ b/dysnix/geth/templates/s3-cronjob-rbac.yaml
@@ -23,6 +23,8 @@ rules:
   - apiGroups: [""]
     resources:
       - configmaps
+    resourceNames:
+      - {{ include "geth.fullname" . }}-s3-config
     verbs:
       - get
       - patch
diff --git a/dysnix/geth/templates/statefulset.yaml b/dysnix/geth/templates/statefulset.yaml
index 8af61c7a..acfd5098 100644
--- a/dysnix/geth/templates/statefulset.yaml
+++ b/dysnix/geth/templates/statefulset.yaml
@@ -30,11 +30,7 @@ spec:
       imagePullSecrets:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      {{- if .Values.syncToS3.cronjob.enabled }}
-      serviceAccountName: {{ include "geth.fullname" . }}-s3-cronjob
-      {{- else }}
       serviceAccountName: {{ include "geth.serviceAccountName" . }}
-      {{- end }}
       {{- with .Values.podSecurityContext }}
       securityContext:
         {{- toYaml . | nindent 10 }}