From e7d3c7a554f2812a36e9488917b4a6bf2cfebeb4 Mon Sep 17 00:00:00 2001 From: Vladyslav Deryhin Date: Wed, 17 Apr 2024 15:51:04 +0300 Subject: [PATCH] [op-geth] v0.2.0 (#301) --- dysnix/op-geth/Chart.yaml | 2 +- .../op-geth/templates/configmap-scripts.yaml | 14 +- dysnix/op-geth/templates/rbac.yaml | 31 ++++ dysnix/op-geth/templates/s3-configmap.yaml | 12 ++ dysnix/op-geth/templates/s3-cronjob-rbac.yaml | 45 ++++++ dysnix/op-geth/templates/s3-cronjob.yaml | 71 +++++++++ dysnix/op-geth/templates/s3-secret.yaml | 14 ++ .../templates/scripts/_init-download.tpl | 23 --- .../templates/scripts/_init-from-s3.tpl | 86 +++++++++++ .../op-geth/templates/scripts/_liveness.tpl | 2 +- dysnix/op-geth/templates/scripts/_s3-cron.tpl | 81 ++++++++++ dysnix/op-geth/templates/scripts/_s3-env.tpl | 20 +++ .../op-geth/templates/scripts/_sync-to-s3.tpl | 56 +++++++ .../templates/scripts/_wait-for-sync.tpl | 36 +++++ dysnix/op-geth/templates/statefulset.yaml | 109 +++++++++++-- dysnix/op-geth/values.yaml | 146 +++++++++++++----- 16 files changed, 670 insertions(+), 78 deletions(-) create mode 100644 dysnix/op-geth/templates/rbac.yaml create mode 100644 dysnix/op-geth/templates/s3-configmap.yaml create mode 100644 dysnix/op-geth/templates/s3-cronjob-rbac.yaml create mode 100644 dysnix/op-geth/templates/s3-cronjob.yaml create mode 100644 dysnix/op-geth/templates/s3-secret.yaml delete mode 100644 dysnix/op-geth/templates/scripts/_init-download.tpl create mode 100644 dysnix/op-geth/templates/scripts/_init-from-s3.tpl create mode 100644 dysnix/op-geth/templates/scripts/_s3-cron.tpl create mode 100644 dysnix/op-geth/templates/scripts/_s3-env.tpl create mode 100644 dysnix/op-geth/templates/scripts/_sync-to-s3.tpl create mode 100644 dysnix/op-geth/templates/scripts/_wait-for-sync.tpl diff --git a/dysnix/op-geth/Chart.yaml b/dysnix/op-geth/Chart.yaml index 370c78e0..197a52d7 100644 --- a/dysnix/op-geth/Chart.yaml +++ b/dysnix/op-geth/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: op-geth description: Optimism op-geth execution-layer client Helm chart -version: 0.1.5 +version: 0.2.0 appVersion: v1.101308.2 keywords: diff --git a/dysnix/op-geth/templates/configmap-scripts.yaml b/dysnix/op-geth/templates/configmap-scripts.yaml index 94646914..629386e7 100644 --- a/dysnix/op-geth/templates/configmap-scripts.yaml +++ b/dysnix/op-geth/templates/configmap-scripts.yaml @@ -9,7 +9,17 @@ data: {{- include (print $.Template.BasePath "/scripts/_readiness.tpl") . | nindent 4 }} liveness.sh: |- {{- include (print $.Template.BasePath "/scripts/_liveness.tpl") . | nindent 4 }} - init-download.sh: |- - {{- include (print $.Template.BasePath "/scripts/_init-download.tpl") . | nindent 4 }} + wait-for-sync.sh: |- + {{- include (print $.Template.BasePath "/scripts/_wait-for-sync.tpl") . | nindent 4 }} init-genesis.sh: |- {{- include (print $.Template.BasePath "/scripts/_init-genesis.tpl") . | nindent 4 }} + {{- if or .Values.syncToS3.enabled .Values.initFromS3.enabled }} + init-from-s3.sh: |- + {{- include (print $.Template.BasePath "/scripts/_init-from-s3.tpl") . | nindent 4 }} + sync-to-s3.sh: |- + {{- include (print $.Template.BasePath "/scripts/_sync-to-s3.tpl") . | nindent 4 }} + s3-env.sh: |- + {{- include (print $.Template.BasePath "/scripts/_s3-env.tpl") . | nindent 4 }} + s3-cron.sh: |- + {{- include (print $.Template.BasePath "/scripts/_s3-cron.tpl") . | nindent 4 }} + {{- end }} diff --git a/dysnix/op-geth/templates/rbac.yaml b/dysnix/op-geth/templates/rbac.yaml new file mode 100644 index 00000000..753cacab --- /dev/null +++ b/dysnix/op-geth/templates/rbac.yaml @@ -0,0 +1,31 @@ +{{- if and .Values.syncToS3.enabled .Values.serviceAccount.create }} +{{- $fullName := include "op-geth.fullname" . }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $fullName }} + labels: {{ include "op-geth.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: + - configmaps + resourceNames: + - {{ $fullName }}-s3-config + verbs: + - get + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $fullName }} + labels: {{ include "op-geth.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ $fullName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ $fullName }} + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/dysnix/op-geth/templates/s3-configmap.yaml b/dysnix/op-geth/templates/s3-configmap.yaml new file mode 100644 index 00000000..f710da08 --- /dev/null +++ b/dysnix/op-geth/templates/s3-configmap.yaml @@ -0,0 +1,12 @@ +{{- if or .Values.initFromS3.enabled .Values.syncToS3.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "op-geth.fullname" . }}-s3-config +data: + DATA_DIR: {{ tpl .Values.s3config.local.datadir . | quote }} + INITIALIZED_FILE: {{ tpl .Values.s3config.local.initializedFile . | quote }} + SYNC_TO_S3: "False" + S3_BASE_URL: {{ tpl .Values.s3config.remote.baseUrl . }} + FORCE_INIT: {{ ternary "True" "False" .Values.initFromS3.force | quote }} +{{- end }} diff --git a/dysnix/op-geth/templates/s3-cronjob-rbac.yaml b/dysnix/op-geth/templates/s3-cronjob-rbac.yaml new file mode 100644 index 00000000..cecedd32 --- /dev/null +++ b/dysnix/op-geth/templates/s3-cronjob-rbac.yaml @@ -0,0 +1,45 @@ +{{- if .Values.syncToS3.cronjob.enabled -}} +{{- $fullName := print (include "op-geth.fullname" .) "-s3-cronjob" }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $fullName }} + labels: {{ include "op-geth.labels" . | nindent 4 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $fullName }} + labels: {{ include "op-geth.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: + - pods + verbs: + - get + - list + - watch + - delete + - apiGroups: [""] + resources: + - configmaps + resourceNames: + - {{ include "op-geth.fullname" . }}-s3-config + verbs: + - get + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $fullName }} + labels: {{ include "op-geth.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ $fullName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ $fullName }} + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/dysnix/op-geth/templates/s3-cronjob.yaml b/dysnix/op-geth/templates/s3-cronjob.yaml new file mode 100644 index 00000000..7050ec27 --- /dev/null +++ b/dysnix/op-geth/templates/s3-cronjob.yaml @@ -0,0 +1,71 @@ +{{- if .Values.syncToS3.cronjob.enabled }} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "op-geth.fullname" . }}-sync-to-s3 + labels: + {{- include "op-geth.labels" . | nindent 4 }} +spec: + {{- with .Values.syncToS3.cronjob }} + schedule: "{{ .schedule }}" + concurrencyPolicy: Forbid + startingDeadlineSeconds: 300 + jobTemplate: + metadata: + name: {{ include "op-geth.fullname" $ }}-sync-to-s3 + spec: + activeDeadlineSeconds: 60 + backoffLimit: 0 + template: + metadata: + labels: + {{- include "op-geth.labels" $ | nindent 12 }} + spec: + restartPolicy: OnFailure + {{- with .imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 12 }} + {{- end }} + serviceAccountName: {{ include "op-geth.fullname" $ }}-s3-cronjob + {{- with .podSecurityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .affinity }} + affinity: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .tolerations }} + tolerations: + {{- toYaml . | nindent 12 }} + {{- end }} + containers: + - name: enable-sync-to-s3 + image: "{{ .image.repository }}:{{ .image.tag }}" + imagePullPolicy: {{ .image.pullPolicy | quote }} + {{- with .securityContext }} + securityContext: + {{- toYaml . | nindent 14 }} + {{- end }} + command: + - /bin/sh + - /scripts/s3-cron.sh + - enable_sync + - 5s + volumeMounts: + - name: scripts + mountPath: /scripts + {{- with .resources }} + resources: + {{- toYaml . | nindent 14 }} + {{- end }} + volumes: + - name: scripts + configMap: + name: {{ include "op-geth.fullname" $ }}-scripts + {{- end }} +{{- end }} diff --git a/dysnix/op-geth/templates/s3-secret.yaml b/dysnix/op-geth/templates/s3-secret.yaml new file mode 100644 index 00000000..b97aee05 --- /dev/null +++ b/dysnix/op-geth/templates/s3-secret.yaml @@ -0,0 +1,14 @@ +{{- if or .Values.initFromS3.enabled .Values.syncToS3.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "op-geth.fullname" . }}-s3-secret +data: + {{- with .Values.s3config.remote }} + {{- if .endpointUrl }} + S3_ENDPOINT_URL: {{ .endpointUrl | toString | b64enc }} + {{- end }} + AWS_ACCESS_KEY_ID: {{ .accessKeyId | toString | b64enc }} + AWS_SECRET_ACCESS_KEY: {{ .secretAccessKey | toString | b64enc }} + {{- end }} +{{- end }} diff --git a/dysnix/op-geth/templates/scripts/_init-download.tpl b/dysnix/op-geth/templates/scripts/_init-download.tpl deleted file mode 100644 index 2aa23ed8..00000000 --- a/dysnix/op-geth/templates/scripts/_init-download.tpl +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env sh -set -e - -# TODO: improve zstd extraction with mbuffer - -mkdir -p /root/.ethereum/{{ .Values.config.network }} -cd /root/.ethereum/{{ .Values.config.network }} - -if [ ! -f /root/.ethereum/.downloaded ]; then - echo "Not initialized, proceeding with download..." - {{- if contains "lz4" .Values.init.download.url }} - apk add lz4 - wget -qO- {{ .Values.init.download.url }} | lz4 -cd | tar -xvf - - {{- else if contains "zstd" .Values.init.download.url }} - apk add zstd - wget -qO- {{ .Values.init.download.url }} | zstd -cd | tar -xvf - - {{- else }} - wget -qO- {{ .Values.init.download.url }} | tar -xvf - - {{- end }} - touch /root/.ethereum/.downloaded -else - echo "Already initialized, skipping." -fi \ No newline at end of file diff --git a/dysnix/op-geth/templates/scripts/_init-from-s3.tpl b/dysnix/op-geth/templates/scripts/_init-from-s3.tpl new file mode 100644 index 00000000..0318496f --- /dev/null +++ b/dysnix/op-geth/templates/scripts/_init-from-s3.tpl @@ -0,0 +1,86 @@ +#!/usr/bin/env sh +# shellcheck disable=SC2086,SC3037 + +set -e + +. /scripts/s3-env.sh + +process_inputs() { + # download even if we are already initialized + if [ "$FORCE_INIT" = "True" ]; then + echo "Force init enabled, existing data will be deleted." + rm -f "$INITIALIZED_FILE" + fi + # check for S3 credentials + if [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ]; then + echo "S3 credentials are not provided, exiting"; exit 1 + fi +} + +progress() { + remote_stats=$("$S5CMD" cat "s3://${STATS_URL}") + case $1 in + "start") + while true; do + inodes=$(df -Phi "$DATA_DIR" | tail -n 1 | awk '{print $3}') + size=$(df -P -BG "$DATA_DIR" | tail -n 1 | awk '{print $3}')G + echo -e "$(date -Iseconds) | SOURCE TOTAL ${remote_stats} | DST USED Inodes:\t${inodes} Size:\t${size}" + sleep 2 + done & + progress_pid=$! ;; + "stop") + kill "$progress_pid" + progress_pid=0 ;; + "*") + echo "Unknown arg" ;; + esac +} + +check_lockfile() { + if "$S5CMD" cat "s3://${LOCKFILE_URL}" >/dev/null 2>&1; then + echo "Found existing lockfile, snapshot might be corrupted. Aborting download.." + exit 1 + fi +} + +# stop all background tasks +interrupt() { + echo "Got interrupt signal, stopping..." + for i in "$@"; do kill $i; done +} + +sync() { + # check if we are already initialized + if [ -f "$INITIALIZED_FILE" ]; then + echo "Blockchain already initialized. Exiting..." + exit 0 + fi + # s5cmd does not support "true" sync, it does not save object's timestamps + # so we cleanup local datadir on our own + # https://github.com/peak/s5cmd/issues/532 + echo "Cleaning up local data..." + rm -rf "${DATA_DIR}" + mkdir -p "${DATA_DIR}" + + echo "Starting download data from S3..." + + # handle interruption / termination + trap 'interrupt ${progress_pid}' INT TERM + progress start + + # download remote snapshot to an empty datadir + time "$S5CMD" --stat --log error sync "s3://${S3_DATA_DIR}/*" "${DATA_DIR}/" + progress stop + + # all done, mark as initialized + touch "$INITIALIZED_FILE" +} + + +main() { + process_inputs + check_lockfile + sync +} + +main \ No newline at end of file diff --git a/dysnix/op-geth/templates/scripts/_liveness.tpl b/dysnix/op-geth/templates/scripts/_liveness.tpl index f3a07ec9..8a8339e7 100644 --- a/dysnix/op-geth/templates/scripts/_liveness.tpl +++ b/dysnix/op-geth/templates/scripts/_liveness.tpl @@ -7,7 +7,7 @@ set -e AGE_THRESHOLD=$1 -STATE_FILE=${2:-"/root/.ethereum/saved_block_number.txt"} +STATE_FILE=${2:-"{{ .Values.config.datadir }}/saved_block_number.txt"} HTTP_PORT="{{ .Values.config.http.port }}" if [ -z "${AGE_THRESHOLD}" ] || [ -z "${STATE_FILE}" ]; then diff --git a/dysnix/op-geth/templates/scripts/_s3-cron.tpl b/dysnix/op-geth/templates/scripts/_s3-cron.tpl new file mode 100644 index 00000000..9720f0d8 --- /dev/null +++ b/dysnix/op-geth/templates/scripts/_s3-cron.tpl @@ -0,0 +1,81 @@ +#!/usr/bin/env sh +# shellcheck disable=SC1083 + +MODE="$1" +WAIT_TIMEOUT="$2" +CONFIGMAP_NAME={{ include "op-geth.fullname" . }}-s3-config +KUBECTL=$(which kubectl) +PATCH_DATA="" +POD_NAME={{ include "op-geth.fullname" . }}-0 + +check_ret(){ + ret="$1" + msg="$2" + # allow to override exit code, default value is ret + exit_code=${3:-${ret}} + if [ ! "$ret" -eq 0 ]; then + echo "$msg" + echo "return code ${ret}, exit code ${exit_code}" + exit "$exit_code" + fi +} + +check_pod_readiness() { + # wait for pod to become ready + echo "$(date -Iseconds) Waiting ${WAIT_TIMEOUT} for pod ${1} to become ready ..." + "$KUBECTL" wait --timeout="$WAIT_TIMEOUT" --for=condition=Ready pod "$1" + check_ret $? "$(date -Iseconds) Pod ${1} is not ready, nothing to do, exiting" 0 + + # ensuring pod is not terminating now + # https://github.com/kubernetes/kubernetes/issues/22839 + echo "$(date -Iseconds) Checking for pod ${1} to not terminate ..." + deletion_timestamp=$("$KUBECTL" get -o jsonpath='{.metadata.deletionTimestamp}' pod "$1") + check_ret $? "$(date -Iseconds) Cannot get pod ${1}, abort" + + [ -z "$deletion_timestamp" ] + check_ret $? "$(date -Iseconds) Pod ${1} is terminating now, try another time" 1 +} + +enable_sync() { + echo "$(date -Iseconds) Patching configmap ${CONFIGMAP_NAME} to enable sync" + PATCH_DATA='{"data":{"SYNC_TO_S3":"True"}}' +} + +disable_sync() { + echo "$(date -Iseconds) Patching configmap ${CONFIGMAP_NAME} to disable sync" + PATCH_DATA='{"data":{"SYNC_TO_S3":"False"}}' +} + +patch_configmap() { + "$KUBECTL" patch configmap "$CONFIGMAP_NAME" --type merge --patch "$PATCH_DATA" + check_ret $? "$(date -Iseconds) Fatal: cannot patch configmap ${CONFIGMAP_NAME}, abort" +} + +delete_pod() { + echo "$(date -Iseconds) Deleting pod ${1} to trigger action inside init container ..." + # delete the pod to trigger action inside init container + "$KUBECTL" delete pod "$1" --wait=false + check_ret $? "$(date -Iseconds) Fatal: cannot delete pod ${1}, abort" + echo "$(date -Iseconds) Pod ${1} deleted successfully, exiting. Check pod logs after restart." +} + +main() { + case "$MODE" in + "enable_sync") + check_pod_readiness "$POD_NAME" + enable_sync + patch_configmap + delete_pod "$POD_NAME" + ;; + # intended to be run inside initContainer after successful sync + "disable_sync") + disable_sync + patch_configmap + ;; + "*") + check_ret 1 "$(date -Iseconds) Mode value \"$MODE\" is incorrect, abort" + ;; + esac +} + +main \ No newline at end of file diff --git a/dysnix/op-geth/templates/scripts/_s3-env.tpl b/dysnix/op-geth/templates/scripts/_s3-env.tpl new file mode 100644 index 00000000..0bcda9c8 --- /dev/null +++ b/dysnix/op-geth/templates/scripts/_s3-env.tpl @@ -0,0 +1,20 @@ +#!/usr/bin/env sh + +export S5CMD=/s5cmd + +# local directory structure config +export DATA_DIR="${DATA_DIR?DATA_DIR not provided.}" +export INITIALIZED_FILE="${INITIALIZED_FILE?INITIALIZED_FILE not provided.}" + +# s3 directory structure config +export S3_BASE_URL="${S3_BASE_URL?S3_BASE_URL not provided.}" +export S3_DATA_DIR="${S3_BASE_URL}/upload" +export S_COMPLETED="/completed" +export S_STATS="/stats" +export S_LOCKFILE="/lockfile" +export COMPLETED_URL="${S3_BASE_URL}${S_COMPLETED}" +export LOCKFILE_URL="${S3_BASE_URL}${S_LOCKFILE}" +export STATS_URL="${S3_BASE_URL}${S_STATS}" + +# download/upload options +export FORCE_INIT="${FORCE_INIT:-False}" diff --git a/dysnix/op-geth/templates/scripts/_sync-to-s3.tpl b/dysnix/op-geth/templates/scripts/_sync-to-s3.tpl new file mode 100644 index 00000000..a158616e --- /dev/null +++ b/dysnix/op-geth/templates/scripts/_sync-to-s3.tpl @@ -0,0 +1,56 @@ +#!/usr/bin/env sh +# shellcheck disable=SC2086,SC3037 + +set -e + +. /scripts/s3-env.sh + +process_inputs() { + # enable sync via env variable + if [ "$SYNC_TO_S3" != "True" ]; then + echo "Sync is not enabled in config, exiting" + exit 0 + fi + # check for S3 credentials + if [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ]; then + echo "S3 credentials are not provided, exiting" + exit 1 + fi +} + +check_recent_init() { + # if node has been initialized from snapshot <30 mins ago, skip upload + is_recent=$(find "$INITIALIZED_FILE" -type f -mmin +30 | wc -l | tr -d '[:blank:]') + + if [ -f "$INITIALIZED_FILE" ] && [ "$is_recent" -eq 0 ]; then + echo "Node has been initialized recently, skipping the upload. Exiting..."; exit 0 + fi +} + +sync() { + # add lockfile while uploading + # shellcheck disable=SC3028 + echo "${HOSTNAME} $(date +%s)" | "$S5CMD" pipe "s3://${LOCKFILE_URL}" + + # perform upload of local data and remove destination objects which don't exist locally + time "$S5CMD" --stat sync --delete "${DATA_DIR}/" "s3://${S3_DATA_DIR}/" + + # mark upload as completed + date +%s | "$S5CMD" pipe "s3://${COMPLETED_URL}" + "$S5CMD" rm "s3://${LOCKFILE_URL}" +} + +update_stats() { + inodes=$(df -Phi "${DATA_DIR}" | tail -n 1 | awk '{print $3}') + size=$(df -P -BG "${DATA_DIR}" | tail -n 1 | awk '{print $3}')G + echo -ne "Inodes:\t${inodes} Size:\t${size}" | "$S5CMD" pipe "s3://${STATS_URL}" +} + +main() { + process_inputs + check_recent_init + sync + update_stats +} + +main \ No newline at end of file diff --git a/dysnix/op-geth/templates/scripts/_wait-for-sync.tpl b/dysnix/op-geth/templates/scripts/_wait-for-sync.tpl new file mode 100644 index 00000000..e0990588 --- /dev/null +++ b/dysnix/op-geth/templates/scripts/_wait-for-sync.tpl @@ -0,0 +1,36 @@ +#!/usr/bin/env sh +# shellcheck disable=SC3040 + +# We assume that node is syncing from initial snapshot when: +# (get_block_number == 0x0) OR (is_syncing == true) + +set -e + +HTTP_PORT="{{ .Values.config.http.port }}" + +# expected output format: 0x50938d +get_block_number() { + wget "http://localhost:$HTTP_PORT" -qO- \ + --header 'Content-Type: application/json' \ + --post-data '{"jsonrpc":"2.0","method":"eth_blockNumber","id":1}' \ + | sed -r 's/.*"result":"([^"]+)".*/\1/g' +} + +# exit codes: 1 = sync completed, 0 = sync in progress +is_syncing() { + wget "http://localhost:$HTTP_PORT" -qO- \ + --header 'Content-Type: application/json' \ + --post-data '{"jsonrpc":"2.0","method":"eth_syncing","id":1}' \ + | grep -qv "false" +} + +if ! get_block_number | grep -qE '^0x[a-z0-9]+'; then + echo "Error reading block number"; exit 1 +fi + +if is_syncing || [ "$(get_block_number)" = "0x0" ]; then + echo "Initial sync is in progress" + exit 1 +else + exit 0 +fi \ No newline at end of file diff --git a/dysnix/op-geth/templates/statefulset.yaml b/dysnix/op-geth/templates/statefulset.yaml index 9943ee55..093c0b4d 100644 --- a/dysnix/op-geth/templates/statefulset.yaml +++ b/dysnix/op-geth/templates/statefulset.yaml @@ -14,6 +14,11 @@ spec: template: metadata: annotations: + checksum/configmap-scripts: {{ include (print $.Template.BasePath "/configmap-scripts.yaml") . | sha256sum }} + {{- if or .Values.syncToS3.enabled .Values.initFromS3.enabled }} + checksum/s3-secret: {{ include (print $.Template.BasePath "/s3-secret.yaml") . | sha256sum }} + checksum/s3-configmap: {{ include (print $.Template.BasePath "/s3-configmap.yaml") . | sha256sum }} + {{- end }} labels: {{- include "op-geth.selectorLabels" . | nindent 8 }} {{- with .Values.podStatusLabels }} @@ -47,16 +52,17 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} initContainers: - {{- if .Values.init.download.enabled }} - - name: init-download - image: "{{ .Values.init.download.image.repository }}:{{ .Values.init.download.image.tag }}" - imagePullPolicy: {{ .Values.init.download.image.pullPolicy | quote }} - command: ["sh", "/scripts/init-download.sh"] + {{- if .Values.init.chownData.enabled }} + - name: chown-data + image: "{{ .Values.init.chownData.image.repository }}:{{ .Values.init.chownData.image.tag }}" + imagePullPolicy: {{ .Values.init.chownData.image.pullPolicy | quote }} + securityContext: + runAsNonRoot: false + runAsUser: 0 + command: ["chown", "-R", "{{ .Values.securityContext.runAsUser }}:{{ .Values.securityContext.runAsGroup }}", "/data"] volumeMounts: - - name: scripts - mountPath: /scripts - name: data - mountPath: /root/.ethereum + mountPath: /data {{- end }} {{- if .Values.init.genesis.enabled }} - name: init-genesis @@ -71,7 +77,74 @@ spec: - name: scripts mountPath: /scripts - name: data - mountPath: /root/.ethereum + mountPath: {{ .Values.persistence.mountPath | default .Values.config.datadir }} + {{- end }} + {{- if .Values.initFromS3.enabled }} + {{- with .Values.s3config }} + - name: init-from-s3 + image: "{{ .image.repository }}:{{ .image.tag }}" + imagePullPolicy: {{ .image.pullPolicy | quote }} + {{- with $.Values.securityContext }} + securityContext: + {{- toYaml . | nindent 10 }} + {{- end }} + command: + - sh + - /scripts/init-from-s3.sh + envFrom: + - configMapRef: + name: {{ include "op-geth.fullname" $ }}-s3-config + - secretRef: + name: {{ include "op-geth.fullname" $ }}-s3-secret + volumeMounts: + - name: scripts + mountPath: /scripts + - name: data + mountPath: {{ $.Values.persistence.mountPath | default $.Values.config.datadir }} + {{- end }} + {{- end }} + {{- if .Values.syncToS3.enabled }} + {{- with .Values.s3config }} + - name: sync-to-s3 + image: "{{ .image.repository }}:{{ .image.tag }}" + imagePullPolicy: {{ .image.pullPolicy | quote }} + {{- with $.Values.securityContext }} + securityContext: + {{- toYaml . | nindent 10 }} + {{- end }} + command: + - /bin/sh + - /scripts/sync-to-s3.sh + envFrom: + - configMapRef: + name: {{ include "op-geth.fullname" $ }}-s3-config + - secretRef: + name: {{ include "op-geth.fullname" $ }}-s3-secret + volumeMounts: + - name: scripts + mountPath: /scripts + - name: data + mountPath: {{ $.Values.persistence.mountPath | default $.Values.config.datadir }} + {{- end }} + {{- with .Values.syncToS3.cronjob }} + {{- if .enabled }} + - name: disable-sync-to-s3 + image: "{{ .image.repository }}:{{ .image.tag }}" + imagePullPolicy: {{ .image.pullPolicy | quote }} + {{- with $.Values.securityContext }} + securityContext: + {{- toYaml . | nindent 10 }} + {{- end }} + command: + - /bin/sh + - /scripts/s3-cron.sh + - disable_sync + - 5s + volumeMounts: + - name: scripts + mountPath: /scripts + {{- end }} + {{- end }} {{- end }} {{- with .Values.extraInitContainers }} {{- tpl (toYaml . | nindent 6) $ }} @@ -89,7 +162,8 @@ spec: {{- tpl (toYaml .) $ | nindent 10 }} {{- end }} args: - {{- with .Values.config.network }} + - --datadir={{ .Values.config.datadir }} + {{- with (get .Values.config "op-network") }} - --op-network={{ . }} {{- end }} - --syncmode={{ .Values.config.syncmode }} @@ -136,7 +210,7 @@ spec: {{- end }} {{- end }} {{- with .Values.config.vmodule }} - - --vmodule={{ join "," . }} + - --log.vmodule={{ join "," . }} {{- end }} {{- with .Values.config.bootnodes }} - --bootnodes={{ join "," . }} @@ -146,6 +220,7 @@ spec: - --discovery.port={{ .Values.config.discovery.port }} {{- end }} - --nat={{ .Values.config.nat }} + - --state.scheme={{ .Values.config.state.scheme }} {{- with .Values.extraArgs }} {{- tpl (toYaml .) $ | nindent 8 }} {{- end }} @@ -184,13 +259,20 @@ spec: readinessProbe: {{- include "op-geth.healthcheck" (list $ .Values.readinessProbe) | nindent 10 }} {{- end }} + {{- if .Values.startupProbe.enabled }} + startupProbe: + {{- include "op-geth.healthcheck" (list $ .Values.startupProbe) | nindent 10 }} + {{- end }} volumeMounts: - name: data - mountPath: /root/.ethereum + mountPath: {{ .Values.persistence.mountPath | default .Values.config.datadir }} - name: secrets mountPath: /secrets - name: scripts mountPath: /scripts + {{- with .Values.extraVolumeMounts }} + {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} resources: {{- toYaml .Values.resources | nindent 10 }} {{- with .Values.sidecarContainers }} @@ -209,6 +291,9 @@ spec: path: {{ .Values.persistence.hostPath.path }} type: {{ .Values.persistence.hostPath.type }} {{- end }} + {{- with .Values.extraVolumes }} + {{- tpl (toYaml .) $ | nindent 6 }} + {{- end }} {{- if eq .Values.persistence.type "pvc" }} volumeClaimTemplates: - metadata: diff --git a/dysnix/op-geth/values.yaml b/dysnix/op-geth/values.yaml index 25ba22b2..166f81d3 100644 --- a/dysnix/op-geth/values.yaml +++ b/dysnix/op-geth/values.yaml @@ -31,8 +31,8 @@ podLabels: {} podStatusLabels: {} # manualstatus: in-service -podSecurityContext: {} - # fsGroup: 2000 +podSecurityContext: + fsGroup: 10001 securityContext: capabilities: @@ -40,10 +40,10 @@ securityContext: - ALL allowPrivilegeEscalation: false privileged: false + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - # runAsGroup: 1000 ## By disabling we fix "Unknown config environment variable envvar=GETH_" ## Enable if your workload depends on this functionality @@ -98,6 +98,17 @@ sidecarContainers: [] # path: /healthz # port: replica-mon +# Extra volumeMounts for op-geth container, can be templated +extraVolumeMounts: [] + # - name: testvolume + # mountPath: /test + +# Extra volumes, can be templated +extraVolumes: [] + # - name: testvolume + # persistentVolumeClaim: + # claimName: test-pvc + ## Services config services: rpc: @@ -116,21 +127,19 @@ services: port: 6060 publishNotReadyAddresses: true p2p: - enabled: false # enable if you want to use "snap" syncmode + enabled: true # disable if you are not using "snap" syncmode type: NodePort loadBalancerIP: "" port: 30303 - # it's better to set nodePort equal to .Values.config.port when the svc type is "NodePort" - # nodePort: 30303 + # nodePort: 30303 # exposed port of the node, must be unique across multiple node deployments annotations: {} publishNotReadyAddresses: true p2pDiscovery: - enabled: false # enable if you want to use "snap" syncmode + enabled: true # disable if you are not using "snap" syncmode type: NodePort loadBalancerIP: "" port: 30301 - # it's better to set nodePort equal to .Values.config.discovery.port when the svc type is "NodePort" - # nodePort: 30301 + # nodePort: 30301 # exposed port of the node, must be unique across multiple node deployments annotations: {} publishNotReadyAddresses: true @@ -178,16 +187,16 @@ serviceMonitor: # metricRelabelings: [] persistence: - type: pvc - # type: hostPath + type: pvc # possible values are: "pvc", "hostPath" pvc: - size: 2048Gi # starting point for full node + size: 2Ti # recommended disk size for op-mainnet full node accessMode: ReadWriteOnce storageClass: "" # set to "-" if you want to manually create persistent volume annotations: {} hostPath: - path: /data/geth - type: Directory # by default you need to create directory yourself + path: /blockchain/optimism # assume /blockchain is your host volume mount point + type: DirectoryOrCreate # automaticaly create "optimism" directory if it doesn't exist + mountPath: "" # mount path for container fs, leave blank to use value from .Values.config.datadir affinity: {} @@ -207,6 +216,18 @@ resources: {} # cpu: 100m # memory: 128Mi +# .startupProbe.exec.command can also be in a templated string format +startupProbe: + enabled: false + initialDelaySeconds: 10 + periodSeconds: 5 + failureThreshold: 120960 # periodSeconds * failureThreshold = 7 days + timeoutSeconds: 10 + exec: + command: + - sh + - /scripts/wait-for-sync.sh + # .livenessProbe.exec.command can also be in a templated string format livenessProbe: enabled: false @@ -238,11 +259,12 @@ readinessProbe: ## Main op-geth config config: jwt: "" # REQUIRED for authentication with op-node - network: op-mainnet # equivalent to --op-network arg, can be an empty string if you are using custom genesis + op-network: op-mainnet # must be an empty string if you are using custom genesis (f.e. when network is not in superchain-registry) + datadir: /data # data directory rollup: - halt: "" # stall sync on incompatible protocol (i.e. hardforks), possible values: major, minor, patch, none + halt: major # halt node on version mismatch, possible values: "major", "minor", "patch", "none" disabletxpoolgossip: true # tx pool gossip not supported currently, so disable it - sequencerhttp: https://mainnet-sequencer.optimism.io/ # url of sequencer, depends on .Values.config.network + sequencerhttp: https://mainnet-sequencer.optimism.io/ # url of sequencer, depends on chosen network http: port: 8545 vhosts: ["*"] @@ -256,35 +278,81 @@ config: authrpc: port: 8551 vhosts: ["*"] - cache: 0 # disable by default, let geth automatically set proper value https://blog.ethereum.org/2023/09/12/geth-v1-13-0 - syncmode: full # can be "full" or "snap" - gcmode: full # possible values: full, archive - snapshot: true # enable snapshot generation - verbosity: 3 - maxpeers: 0 # increase if you want to use "snap" syncmode - nodiscover: true # disable if you want to use "snap" syncmode - port: 30303 + state: + scheme: path # possible values are: "path", "hash". "hash" is going to be deprecated soon + cache: 0 # disable by default, let geth automatically set proper value https://blog.ethereum.org/2023/09/12/geth-v1-13-0 + syncmode: snap # possible values are: "snap", "full" + gcmode: full # possible values are: "full", "archive" + snapshot: true # enable state snapshot generation + maxpeers: 50 # set to 0 if you don't use "snap" syncmode + nodiscover: false # enable if you don't use "snap" syncmode + port: 30303 # TCP port for P2P communication discovery: - port: 30301 - useHostPort: false # effectively disables p2p k8s services - nat: "any" - bootnodes: [] - vmodule: [] + port: 30301 # UDP port for P2P discovery + useHostPort: false # allocate hostPorts for P2P communication instead of K8S service + nat: "any" # try to auto-detect node IP, refer to `geth --help` for other options + bootnodes: [] # built-in bootnodes are used when empty + verbosity: 3 # global log verbosity + vmodule: [] # per-module log verbosity # - rpc=5 metrics: enabled: false expensive: false -## initContainer for datadir snapshot download -## archive will be extracted using tarpipe +## initContainers configuration init: + chownData: + enabled: false + image: + repository: alpine + tag: 3.18 + pullPolicy: IfNotPresent genesis: enabled: false url: "" - download: + +## S3 snapshot sync config +s3config: + image: + repository: peakcom/s5cmd + tag: v2.2.2 + pullPolicy: IfNotPresent + # local storage config + local: + # datadir containing the state you want to upload (can be templated) + datadir: "{{ .Values.config.datadir }}/geth/chaindata" + # this file marks node as already initialized from snapshot + # should be placed outside of the datadir you are uploading + initializedFile: "{{ .Values.config.datadir }}/.initialized" + # remote storage config + remote: + # Assuming your S3 bucket name is `my-snapshot-bucket` and base directory name is Helm release name + # snapshot will be uploaded to {{ .baseUrl }}/upload directory + baseUrl: my-snapshot-bucket/{{ .Release.Name }} + # Any S3-compatible object storage service should be supported, but has only been tested with GCS. + # I.e. Amazon S3, MinIO, DigitalOcean Spaces, CloudFlare R2. + # endpointUrl: https://storage.googleapis.com + endpointUrl: "" + # How to create access key + # AWS S3 https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html + # GCS https://cloud.google.com/storage/docs/authentication/managing-hmackeys#create + accessKeyId: REPLACEME + secretAccessKey: REPLACEME + +initFromS3: + # enable initContainer + enabled: false + # re-download snapshot from S3 on every pod start + force: false + +syncToS3: + # enable initContainer (won't enable actual sync) + enabled: false + # restart pod and trigger sync to S3 inside initContainer by schedule + cronjob: enabled: false - url: https://r2-snapshots.fastnode.io/op/op-10-02-2024-full.tar.lz4 image: - repository: alpine - tag: 3.18 - pullPolicy: Always + repository: dysnix/kubectl + tag: v1.27 + pullPolicy: IfNotPresent + schedule: "0 2 * * *"