From d2597301f1b6873dbf5a0d04b58eae632de3c7dc Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Tue, 12 Mar 2024 11:37:58 -0400 Subject: [PATCH 01/22] Ensure that binary logs for PITR are restored to a shared location Signed-off-by: Matt Lord --- pkg/operator/vttablet/mysqlctld.go | 3 ++- pkg/operator/vttablet/pod.go | 5 +++++ pkg/operator/vttablet/vtbackup_pod.go | 9 +++++++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/pkg/operator/vttablet/mysqlctld.go b/pkg/operator/vttablet/mysqlctld.go index 442aaf6d..4860aa20 100644 --- a/pkg/operator/vttablet/mysqlctld.go +++ b/pkg/operator/vttablet/mysqlctld.go @@ -29,6 +29,7 @@ const ( vtRootInitScript = `set -ex mkdir -p /mnt/vt/bin cp --no-clobber /vt/bin/mysqlctld /mnt/vt/bin/ +cp --no-clobber $(command -v mysqlbinlog) /mnt/vt/bin/ || true mkdir -p /mnt/vt/config if [[ -d /vt/config/mycnf ]]; then cp --no-clobber -R /vt/config/mycnf /mnt/vt/config/ @@ -68,7 +69,7 @@ func init() { securityContext := &corev1.SecurityContext{} if planetscalev2.DefaultVitessRunAsUser >= 0 { - securityContext.RunAsUser = pointer.Int64Ptr(planetscalev2.DefaultVitessRunAsUser) + securityContext.RunAsUser = pointer.Int64(planetscalev2.DefaultVitessRunAsUser) } // Use an init container to copy only the files we need from the Vitess image. diff --git a/pkg/operator/vttablet/pod.go b/pkg/operator/vttablet/pod.go index 9dcc25f1..37b5f555 100644 --- a/pkg/operator/vttablet/pod.go +++ b/pkg/operator/vttablet/pod.go @@ -168,6 +168,11 @@ func UpdatePod(obj *corev1.Pod, spec *Spec) { var mysqldExporterContainer *corev1.Container mysqlctldAllFlags := mysqlctldFlags.Get(spec) + // Ensure that binary logs are restored to/from a location that all containers + // in the pod can access if no location was explicitly provided. + if _, ok := mysqlctldAllFlags["builtinbackup-incremental-restore-path"]; !ok { + mysqlctldAllFlags["builtinbackup-incremental-restore-path"] = vtDataRootPath + } mysql.UpdateMySQLServerVersion(mysqlctldAllFlags, spec.Images.Mysqld.Image()) if spec.Mysqld != nil { diff --git a/pkg/operator/vttablet/vtbackup_pod.go b/pkg/operator/vttablet/vtbackup_pod.go index 5823cdb1..1f5a5ec9 100644 --- a/pkg/operator/vttablet/vtbackup_pod.go +++ b/pkg/operator/vttablet/vtbackup_pod.go @@ -122,11 +122,11 @@ func NewBackupPod(key client.ObjectKey, backupSpec *BackupSpec, mysqldImage stri podSecurityContext := &corev1.PodSecurityContext{} if planetscalev2.DefaultVitessFSGroup >= 0 { - podSecurityContext.FSGroup = pointer.Int64Ptr(planetscalev2.DefaultVitessFSGroup) + podSecurityContext.FSGroup = pointer.Int64(planetscalev2.DefaultVitessFSGroup) } securityContext := &corev1.SecurityContext{} if planetscalev2.DefaultVitessRunAsUser >= 0 { - securityContext.RunAsUser = pointer.Int64Ptr(planetscalev2.DefaultVitessRunAsUser) + securityContext.RunAsUser = pointer.Int64(planetscalev2.DefaultVitessRunAsUser) } var containerResources corev1.ResourceRequirements @@ -134,6 +134,11 @@ func NewBackupPod(key client.ObjectKey, backupSpec *BackupSpec, mysqldImage stri update.ResourceRequirements(&containerResources, &tabletSpec.Mysqld.Resources) vtbackupAllFlags := vtbackupFlags.Get(backupSpec) + // Ensure that binary logs are restored to/from a location that all containers + // in the pod can access if no location was explicitly provided. + if _, ok := vtbackupAllFlags["builtinbackup-incremental-restore-path"]; !ok { + vtbackupAllFlags["builtinbackup-incremental-restore-path"] = vtDataRootPath + } mysql.UpdateMySQLServerVersion(vtbackupAllFlags, mysqldImage) pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ From e2e5e8b1e749b6f550ee72e77dce2f8ef326408d Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Fri, 15 Mar 2024 23:59:36 -0400 Subject: [PATCH 02/22] Add ability to specify flags for mysqlctld Signed-off-by: Matt Lord --- docs/api/index.html | 43 +++++++++++++++++++ pkg/apis/planetscale/v2/vitessshard_types.go | 13 ++++++ .../planetscale/v2/zz_generated.deepcopy.go | 23 ++++++++++ 3 files changed, 79 insertions(+) diff --git a/docs/api/index.html b/docs/api/index.html index cbf2b048..2163c9b9 100644 --- a/docs/api/index.html +++ b/docs/api/index.html @@ -1410,6 +1410,49 @@

LockserverStatus +

MysqlctldSpec +

+

+

MysqlctldSpec configures the local mysqlctld gRPC server within a tablet.

+

+ + + + + + + + + + + + + + + + + +
FieldDescription
+resources
+ + +Kubernetes core/v1.ResourceRequirements + + +		 +

Resources specify the compute resources to allocate for just the MySQL Control Daemon.

+
+extraFlags
+ +map[string]string + +		 +

ExtraFlags can optionally be used to override default flags set by the +operator, or pass additional flags to mysqlctld. All entries must be +key-value string pairs of the form “flag”: “value”. The flag name should +not have any prefix (just “flag”, not “-flag”). To set a boolean flag, +set the string value to either “true” or “false”.

+

MysqldExporterSpec

diff --git a/pkg/apis/planetscale/v2/vitessshard_types.go b/pkg/apis/planetscale/v2/vitessshard_types.go index 615e423f..c243484c 100644 --- a/pkg/apis/planetscale/v2/vitessshard_types.go +++ b/pkg/apis/planetscale/v2/vitessshard_types.go @@ -315,6 +315,19 @@ type MysqldSpec struct { ConfigOverrides string `json:"configOverrides,omitempty"` } +// MysqlctldSpec configures the local mysqlctld gRPC server within a tablet. +type MysqlctldSpec struct { + // Resources specify the compute resources to allocate for just the MySQL Control Daemon. + Resources corev1.ResourceRequirements `json:"resources"` + + // ExtraFlags can optionally be used to override default flags set by the + // operator, or pass additional flags to mysqlctld. All entries must be + // key-value string pairs of the form "flag": "value". The flag name should + // not have any prefix (just "flag", not "-flag"). To set a boolean flag, + // set the string value to either "true" or "false". + ExtraFlags map[string]string `json:"extraFlags,omitempty"` +} + // MysqldExporterSpec configures the local MySQL exporter within a tablet. type MysqldExporterSpec struct { // Resources specify the compute resources to allocate for just the MySQL Exporter. diff --git a/pkg/apis/planetscale/v2/zz_generated.deepcopy.go b/pkg/apis/planetscale/v2/zz_generated.deepcopy.go index 4243ff17..e30236a6 100644 --- a/pkg/apis/planetscale/v2/zz_generated.deepcopy.go +++ b/pkg/apis/planetscale/v2/zz_generated.deepcopy.go @@ -390,6 +390,29 @@ func (in *LockserverStatus) DeepCopy() *LockserverStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MysqlctldSpec) DeepCopyInto(out *MysqlctldSpec) { + *out = *in + in.Resources.DeepCopyInto(&out.Resources) + if in.ExtraFlags != nil { + in, out := &in.ExtraFlags, &out.ExtraFlags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MysqlctldSpec. +func (in *MysqlctldSpec) DeepCopy() *MysqlctldSpec { + if in == nil { + return nil + } + out := new(MysqlctldSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MysqldExporterSpec) DeepCopyInto(out *MysqldExporterSpec) { *out = *in From 10a95247108c9a5a2d065642f38cccb45651d4a9 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Tue, 26 Mar 2024 13:35:11 -0400 Subject: [PATCH 03/22] It's a tablet flag not a mysqlctld one Signed-off-by: Matt Lord --- pkg/operator/vttablet/pod.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/operator/vttablet/pod.go b/pkg/operator/vttablet/pod.go index 37b5f555..de4de832 100644 --- a/pkg/operator/vttablet/pod.go +++ b/pkg/operator/vttablet/pod.go @@ -170,8 +170,8 @@ func UpdatePod(obj *corev1.Pod, spec *Spec) { mysqlctldAllFlags := mysqlctldFlags.Get(spec) // Ensure that binary logs are restored to/from a location that all containers // in the pod can access if no location was explicitly provided. - if _, ok := mysqlctldAllFlags["builtinbackup-incremental-restore-path"]; !ok { - mysqlctldAllFlags["builtinbackup-incremental-restore-path"] = vtDataRootPath + if _, ok := vttabletAllFlags["builtinbackup-incremental-restore-path"]; !ok { + vttabletAllFlags["builtinbackup-incremental-restore-path"] = vtDataRootPath } mysql.UpdateMySQLServerVersion(mysqlctldAllFlags, spec.Images.Mysqld.Image()) From 48ba49f9ba1a86e56166a1f8c55c516d1e929c80 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Tue, 26 Mar 2024 20:25:42 -0400 Subject: [PATCH 04/22] Move the flag handling code Signed-off-by: Matt Lord --- pkg/operator/vttablet/pod.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/operator/vttablet/pod.go b/pkg/operator/vttablet/pod.go index de4de832..4f41f9e9 100644 --- a/pkg/operator/vttablet/pod.go +++ b/pkg/operator/vttablet/pod.go @@ -87,6 +87,11 @@ func UpdatePod(obj *corev1.Pod, spec *Spec) { key = strings.TrimLeft(key, "-") vttabletAllFlags[key] = value } + // Ensure that binary logs are restored to/from a location that all containers + // in the pod can access if no location was explicitly provided. + if _, ok := vttabletAllFlags["builtinbackup-incremental-restore-path"]; !ok { + vttabletAllFlags["builtinbackup-incremental-restore-path"] = vtDataRootPath + } mysql.UpdateMySQLServerVersion(vttabletAllFlags, spec.Images.Mysqld.Image()) // Compute all operator-generated env vars first. @@ -168,11 +173,6 @@ func UpdatePod(obj *corev1.Pod, spec *Spec) { var mysqldExporterContainer *corev1.Container mysqlctldAllFlags := mysqlctldFlags.Get(spec) - // Ensure that binary logs are restored to/from a location that all containers - // in the pod can access if no location was explicitly provided. - if _, ok := vttabletAllFlags["builtinbackup-incremental-restore-path"]; !ok { - vttabletAllFlags["builtinbackup-incremental-restore-path"] = vtDataRootPath - } mysql.UpdateMySQLServerVersion(mysqlctldAllFlags, spec.Images.Mysqld.Image()) if spec.Mysqld != nil { From 411993c0eed5e72587ad7975749e74d50141c40c Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Wed, 27 Mar 2024 11:30:36 -0400 Subject: [PATCH 05/22] Only set flag on 2.12+ Signed-off-by: Matt Lord --- pkg/operator/vttablet/pod.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/pkg/operator/vttablet/pod.go b/pkg/operator/vttablet/pod.go index 4f41f9e9..17ea992e 100644 --- a/pkg/operator/vttablet/pod.go +++ b/pkg/operator/vttablet/pod.go @@ -26,6 +26,7 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/pointer" "planetscale.dev/vitess-operator/pkg/operator/mysql" + "planetscale.dev/vitess-operator/version" "sigs.k8s.io/controller-runtime/pkg/client" topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/topo/topoproto" @@ -90,7 +91,14 @@ func UpdatePod(obj *corev1.Pod, spec *Spec) { // Ensure that binary logs are restored to/from a location that all containers // in the pod can access if no location was explicitly provided. if _, ok := vttabletAllFlags["builtinbackup-incremental-restore-path"]; !ok { - vttabletAllFlags["builtinbackup-incremental-restore-path"] = vtDataRootPath + parts := strings.Split(version.Version, ".") + if len(parts) > 1 { + major, _ := strconv.Atoi(parts[1]) + minor, _ := strconv.Atoi(parts[2]) + if major >= 2 && minor >= 12 { + vttabletAllFlags["builtinbackup-incremental-restore-path"] = vtDataRootPath + } + } } mysql.UpdateMySQLServerVersion(vttabletAllFlags, spec.Images.Mysqld.Image()) @@ -113,7 +121,7 @@ func UpdatePod(obj *corev1.Pod, spec *Spec) { securityContext := &corev1.SecurityContext{} if planetscalev2.DefaultVitessRunAsUser >= 0 { - securityContext.RunAsUser = pointer.Int64Ptr(planetscalev2.DefaultVitessRunAsUser) + securityContext.RunAsUser = pointer.Int64(planetscalev2.DefaultVitessRunAsUser) } vttabletLifecycle := &spec.Vttablet.Lifecycle @@ -320,13 +328,13 @@ func UpdatePod(obj *corev1.Pod, spec *Spec) { obj.Spec.SecurityContext = &corev1.PodSecurityContext{} } if planetscalev2.DefaultVitessFSGroup >= 0 { - obj.Spec.SecurityContext.FSGroup = pointer.Int64Ptr(planetscalev2.DefaultVitessFSGroup) + obj.Spec.SecurityContext.FSGroup = pointer.Int64(planetscalev2.DefaultVitessFSGroup) } if spec.Vttablet.TerminationGracePeriodSeconds != nil { obj.Spec.TerminationGracePeriodSeconds = spec.Vttablet.TerminationGracePeriodSeconds } else { - obj.Spec.TerminationGracePeriodSeconds = pointer.Int64Ptr(defaultTerminationGracePeriodSeconds) + obj.Spec.TerminationGracePeriodSeconds = pointer.Int64(defaultTerminationGracePeriodSeconds) } // In both the case of the user injecting their own affinity and the default, we From 79bf28de627fbe62655a334cfd82643a39cb7954 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Wed, 27 Mar 2024 12:39:15 -0400 Subject: [PATCH 06/22] Add PITR restore to backup test Signed-off-by: Matt Lord --- test/endtoend/backup_restore_test.sh | 1 + test/endtoend/utils.sh | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/test/endtoend/backup_restore_test.sh b/test/endtoend/backup_restore_test.sh index 1654c35b..dfc0c8b0 100755 --- a/test/endtoend/backup_restore_test.sh +++ b/test/endtoend/backup_restore_test.sh @@ -107,6 +107,7 @@ verifyVtGateVersion "20.0.0" checkSemiSyncSetup takeBackup "commerce/-" verifyListBackupsOutput +restoreBackup "$(vtctldclient GetTablets --keyspace commerce --tablet-type replica --shard '-' | head -1 | awk '{print $1}')" takedownShard resurrectShard checkSemiSyncSetup diff --git a/test/endtoend/utils.sh b/test/endtoend/utils.sh index 259e0abd..51c93189 100644 --- a/test/endtoend/utils.sh +++ b/test/endtoend/utils.sh @@ -104,6 +104,20 @@ function takeBackup() { exit 1 } +# restoreBackup: +# $1: tablet alias for which the backup needs to be restored +function restoreBackup() { + tabletAlias=$1 + + # Issue the PITR restore command to vtctldclient. + vtctldclient RestoreFromBackup --restore-to-timestamp $(date --rfc-3339=seconds) "${tabletAlias}" + + if [[ $? -ne 0 ]]; then + echo "Restore failed" + exit 1 + fi +} + function verifyListBackupsOutput() { backupCount=$(kubectl get vtb --no-headers | wc -l) for i in {1..600} ; do From 8277844ec355699619fa6805898af3b7574d3a64 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Wed, 27 Mar 2024 13:00:48 -0400 Subject: [PATCH 07/22] Use recent vtctldclient Signed-off-by: Matt Lord --- tools/get-e2e-test-deps.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/get-e2e-test-deps.sh b/tools/get-e2e-test-deps.sh index 2501650a..528b0911 100755 --- a/tools/get-e2e-test-deps.sh +++ b/tools/get-e2e-test-deps.sh @@ -40,8 +40,9 @@ fi if ! command -v vtctldclient &> /dev/null then echo "Downloading vtctldclient..." - version=16.0.1 - file=vitess-${version}-d1ba625.tar.gz + version="19.0.1" + hash="3a43ab8" + file="vitess-${version}-${hash}.tar.gz" wget https://github.com/vitessio/vitess/releases/download/v${version}/${file} tar -xzf ${file} cd ${file/.tar.gz/} From 9d2b978eaf61864f788d1134efccb908b02d8fa0 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Wed, 27 Mar 2024 13:15:26 -0400 Subject: [PATCH 08/22] Use different date command Signed-off-by: Matt Lord --- test/endtoend/utils.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/endtoend/utils.sh b/test/endtoend/utils.sh index 51c93189..f60cff27 100644 --- a/test/endtoend/utils.sh +++ b/test/endtoend/utils.sh @@ -108,9 +108,13 @@ function takeBackup() { # $1: tablet alias for which the backup needs to be restored function restoreBackup() { tabletAlias=$1 + if [[ -z "$tabletAlias" ]]; then + echo "Tablet alias not provided as restore target" + exit 1 + fi # Issue the PITR restore command to vtctldclient. - vtctldclient RestoreFromBackup --restore-to-timestamp $(date --rfc-3339=seconds) "${tabletAlias}" + vtctldclient RestoreFromBackup --restore-to-timestamp $(date -u "+%Y-%m-%dT%H:%M:%S") "${tabletAlias}" if [[ $? -ne 0 ]]; then echo "Restore failed" From db311ed6ba015b82f27ca9697ab1f3411b1f8af7 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Wed, 27 Mar 2024 13:44:58 -0400 Subject: [PATCH 09/22] Use different date format Signed-off-by: Matt Lord --- test/endtoend/utils.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/endtoend/utils.sh b/test/endtoend/utils.sh index f60cff27..c1b1bb9a 100644 --- a/test/endtoend/utils.sh +++ b/test/endtoend/utils.sh @@ -114,7 +114,7 @@ function restoreBackup() { fi # Issue the PITR restore command to vtctldclient. - vtctldclient RestoreFromBackup --restore-to-timestamp $(date -u "+%Y-%m-%dT%H:%M:%S") "${tabletAlias}" + vtctldclient RestoreFromBackup --restore-to-timestamp $(date -u "+%Y-%m-%d.%H%M%S") "${tabletAlias}" if [[ $? -ne 0 ]]; then echo "Restore failed" From 63abf801f23cdbfb749d683a66202fd980ba46e5 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Wed, 27 Mar 2024 16:48:06 -0400 Subject: [PATCH 10/22] Test incremental backup and restore Signed-off-by: Matt Lord --- pkg/operator/vttablet/pod.go | 8 +++-- test/endtoend/utils.sh | 60 +++++++++++++++++++++++++++++------- 2 files changed, 54 insertions(+), 14 deletions(-) diff --git a/pkg/operator/vttablet/pod.go b/pkg/operator/vttablet/pod.go index 17ea992e..932b1b08 100644 --- a/pkg/operator/vttablet/pod.go +++ b/pkg/operator/vttablet/pod.go @@ -91,10 +91,12 @@ func UpdatePod(obj *corev1.Pod, spec *Spec) { // Ensure that binary logs are restored to/from a location that all containers // in the pod can access if no location was explicitly provided. if _, ok := vttabletAllFlags["builtinbackup-incremental-restore-path"]; !ok { + // This flag only exists in 2.12.0 and later as it includes vitess + // 20.0 as a dependency. parts := strings.Split(version.Version, ".") - if len(parts) > 1 { - major, _ := strconv.Atoi(parts[1]) - minor, _ := strconv.Atoi(parts[2]) + if len(parts) > 0 { + major, _ := strconv.Atoi(parts[0]) + minor, _ := strconv.Atoi(parts[1]) if major >= 2 && minor >= 12 { vttabletAllFlags["builtinbackup-incremental-restore-path"] = vtDataRootPath } diff --git a/test/endtoend/utils.sh b/test/endtoend/utils.sh index c1b1bb9a..5883efc5 100644 --- a/test/endtoend/utils.sh +++ b/test/endtoend/utils.sh @@ -83,24 +83,47 @@ function removeBackupFiles() { # takeBackup: # $1: keyspace-shard for which the backup needs to be taken +declare INCREMENTAL_RESTORE_TIMESTAMP="" function takeBackup() { keyspaceShard=$1 initialBackupCount=$(kubectl get vtb --no-headers | wc -l) finalBackupCount=$((initialBackupCount+1)) - # issue the backupShard command to vtctldclient - vtctldclient BackupShard "$keyspaceShard" + # Issue the BackupShard command to vtctldclient. + vtctldclient BackupShard "${keyspaceShard}" for i in {1..600} ; do out=$(kubectl get vtb --no-headers | wc -l) - echo "$out" | grep "$finalBackupCount" > /dev/null 2>&1 - if [ $? -eq 0 ]; then - echo "Backup created" + if echo "${out}" | grep -c "${finalBackupCount}" >/dev/null; then + echo "Full backup created" + break + fi + sleep 3 + done + + sleep 10 + + # Now perform an incremental backup. + insertWithRetry + INCREMENTAL_RESTORE_TIMESTAMP=$(date -u "+%Y-%m-%dT%H:%M:%SZ") + sleep 1 + insertWithRetry + + sleep 10 + + vtctldclient BackupShard --incremental-from-pos=auto "${keyspaceShard}" + let finalBackupCount=${finalBackupCount}+1 + + for i in {1..600} ; do + out=$(kubectl get vtb --no-headers | wc -l) + if echo "${out}" | grep -c "${finalBackupCount}" >/dev/null; then + echo "Incremental backup created" return 0 fi sleep 3 done - echo -e "ERROR: Backup not created - $out. $backupCount backups expected." + + echo -e "ERROR: Backups not created - ${out}. ${backupCount} backups expected." exit 1 } @@ -108,18 +131,33 @@ function takeBackup() { # $1: tablet alias for which the backup needs to be restored function restoreBackup() { tabletAlias=$1 - if [[ -z "$tabletAlias" ]]; then + if [[ -z "${tabletAlias}" ]]; then echo "Tablet alias not provided as restore target" exit 1 fi # Issue the PITR restore command to vtctldclient. - vtctldclient RestoreFromBackup --restore-to-timestamp $(date -u "+%Y-%m-%d.%H%M%S") "${tabletAlias}" - - if [[ $? -ne 0 ]]; then - echo "Restore failed" + # This should restore the last full backup, followed by applying the + # binary logs to reach the desired timestamp. + if ! vtctldclient RestoreFromBackup --restore-to-timestamp "${INCREMENTAL_RESTORE_TIMESTAMP}" "${tabletAlias}"; then + echo "ERROR: failed to perform incremental restore" exit 1 fi + + cell="${tabletAlias%-*}" + uid="${tabletAlias##*-}" + + for i in {1..600} ; do + out=$(kubectl get pods --no-headers -l "planetscale.com/cell=${cell},planetscale.com/tablet-uid=${uid}" | grep "Running" | wc -l) + if echo "$out" | grep -c "1" >/dev/null; then + echo "Tablet ${tabletAlias} restore complete" + return 0 + fi + sleep 3 + done + + echo -e "ERROR: restored tablet ${tabletAlias} did not become healthy after the restore." + exit 1 } function verifyListBackupsOutput() { From 425b7255de2d035286e8dd7bd94518561f25b543 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Wed, 27 Mar 2024 23:26:48 -0400 Subject: [PATCH 11/22] WiP Signed-off-by: Matt Lord --- pkg/operator/vttablet/pod.go | 18 ++++++++++-------- test/endtoend/utils.sh | 7 ++++++- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/pkg/operator/vttablet/pod.go b/pkg/operator/vttablet/pod.go index 932b1b08..f3e2fa01 100644 --- a/pkg/operator/vttablet/pod.go +++ b/pkg/operator/vttablet/pod.go @@ -26,7 +26,6 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/pointer" "planetscale.dev/vitess-operator/pkg/operator/mysql" - "planetscale.dev/vitess-operator/version" "sigs.k8s.io/controller-runtime/pkg/client" topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/topo/topoproto" @@ -93,14 +92,17 @@ func UpdatePod(obj *corev1.Pod, spec *Spec) { if _, ok := vttabletAllFlags["builtinbackup-incremental-restore-path"]; !ok { // This flag only exists in 2.12.0 and later as it includes vitess // 20.0 as a dependency. - parts := strings.Split(version.Version, ".") - if len(parts) > 0 { - major, _ := strconv.Atoi(parts[0]) - minor, _ := strconv.Atoi(parts[1]) - if major >= 2 && minor >= 12 { - vttabletAllFlags["builtinbackup-incremental-restore-path"] = vtDataRootPath + /* + parts := strings.Split(version.Version, ".") + if len(parts) > 0 { + major, _ := strconv.Atoi(parts[0]) + minor, _ := strconv.Atoi(parts[1]) + if major >= 2 && minor >= 12 { + vttabletAllFlags["builtinbackup-incremental-restore-path"] = vtDataRootPath + } } - } + */ + vttabletAllFlags["builtinbackup-incremental-restore-path"] = vtDataRootPath } mysql.UpdateMySQLServerVersion(vttabletAllFlags, spec.Images.Mysqld.Image()) diff --git a/test/endtoend/utils.sh b/test/endtoend/utils.sh index 5883efc5..a70100e7 100644 --- a/test/endtoend/utils.sh +++ b/test/endtoend/utils.sh @@ -103,17 +103,22 @@ function takeBackup() { sleep 10 + mysql 'commerce:-' -e "flush binary logs" || echo "flushing binlogs failed" + # Now perform an incremental backup. insertWithRetry INCREMENTAL_RESTORE_TIMESTAMP=$(date -u "+%Y-%m-%dT%H:%M:%SZ") sleep 1 - insertWithRetry + + mysql 'commerce:-' -e "flush binary logs" || echo "flushing binlogs failed" sleep 10 vtctldclient BackupShard --incremental-from-pos=auto "${keyspaceShard}" let finalBackupCount=${finalBackupCount}+1 + mysql 'commerce:-' -e "flush binary logs" || echo "flushing binlogs failed" + for i in {1..600} ; do out=$(kubectl get vtb --no-headers | wc -l) if echo "${out}" | grep -c "${finalBackupCount}" >/dev/null; then From 75c701d8da4670b53fce7c5b9c27a01d3f0f3240 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 28 Mar 2024 08:06:58 +0200 Subject: [PATCH 12/22] attempt to fix tests Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- test/endtoend/utils.sh | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/test/endtoend/utils.sh b/test/endtoend/utils.sh index a70100e7..7ec9913b 100644 --- a/test/endtoend/utils.sh +++ b/test/endtoend/utils.sh @@ -101,24 +101,16 @@ function takeBackup() { sleep 3 done - sleep 10 - - mysql 'commerce:-' -e "flush binary logs" || echo "flushing binlogs failed" - # Now perform an incremental backup. insertWithRetry + sleep 2 INCREMENTAL_RESTORE_TIMESTAMP=$(date -u "+%Y-%m-%dT%H:%M:%SZ") - sleep 1 - - mysql 'commerce:-' -e "flush binary logs" || echo "flushing binlogs failed" - - sleep 10 + sleep 2 + insertWithRetry vtctldclient BackupShard --incremental-from-pos=auto "${keyspaceShard}" let finalBackupCount=${finalBackupCount}+1 - mysql 'commerce:-' -e "flush binary logs" || echo "flushing binlogs failed" - for i in {1..600} ; do out=$(kubectl get vtb --no-headers | wc -l) if echo "${out}" | grep -c "${finalBackupCount}" >/dev/null; then From f09d7be7a0d4a6ae67c5a6b96c3b15fbeec01ace Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 28 Mar 2024 08:26:21 +0200 Subject: [PATCH 13/22] more debug output Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- test/endtoend/utils.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/endtoend/utils.sh b/test/endtoend/utils.sh index 7ec9913b..f9dbf27e 100644 --- a/test/endtoend/utils.sh +++ b/test/endtoend/utils.sh @@ -102,11 +102,18 @@ function takeBackup() { done # Now perform an incremental backup. + echo "Checking @@gtid_exeuted just after full backup:" + mysql -e "select @@gtid_executed;" insertWithRetry + echo "Checking @@gtid_exeuted after some writes and before taking timestamp:" + mysql -e "select @@gtid_executed;" sleep 2 INCREMENTAL_RESTORE_TIMESTAMP=$(date -u "+%Y-%m-%dT%H:%M:%SZ") + echo "Timestamp is $INCREMENTAL_RESTORE_TIMESTAMP" sleep 2 insertWithRetry + echo "Checking @@gtid_exeuted jbefore taking incremental backup:" + mysql -e "select @@gtid_executed;" vtctldclient BackupShard --incremental-from-pos=auto "${keyspaceShard}" let finalBackupCount=${finalBackupCount}+1 @@ -136,6 +143,7 @@ function restoreBackup() { # Issue the PITR restore command to vtctldclient. # This should restore the last full backup, followed by applying the # binary logs to reach the desired timestamp. + echo "Restoring tablet ${tabletAlias} to timestamp ${INCREMENTAL_RESTORE_TIMESTAMP}" if ! vtctldclient RestoreFromBackup --restore-to-timestamp "${INCREMENTAL_RESTORE_TIMESTAMP}" "${tabletAlias}"; then echo "ERROR: failed to perform incremental restore" exit 1 From 77161354e7c35e700153c537e49e4af9f3f5b732 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 28 Mar 2024 08:41:35 +0200 Subject: [PATCH 14/22] more debug info Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- test/endtoend/utils.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/test/endtoend/utils.sh b/test/endtoend/utils.sh index f9dbf27e..6af58a26 100644 --- a/test/endtoend/utils.sh +++ b/test/endtoend/utils.sh @@ -143,6 +143,7 @@ function restoreBackup() { # Issue the PITR restore command to vtctldclient. # This should restore the last full backup, followed by applying the # binary logs to reach the desired timestamp. + find "$VTDATAROOT" -name MANIFEST | xargs cat echo "Restoring tablet ${tabletAlias} to timestamp ${INCREMENTAL_RESTORE_TIMESTAMP}" if ! vtctldclient RestoreFromBackup --restore-to-timestamp "${INCREMENTAL_RESTORE_TIMESTAMP}" "${tabletAlias}"; then echo "ERROR: failed to perform incremental restore" From 34523599f1222718c0f6d4478640c10c9574934f Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 28 Mar 2024 12:36:02 +0200 Subject: [PATCH 15/22] list backups as debug step Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- test/endtoend/utils.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/endtoend/utils.sh b/test/endtoend/utils.sh index 6af58a26..386eec43 100644 --- a/test/endtoend/utils.sh +++ b/test/endtoend/utils.sh @@ -143,7 +143,9 @@ function restoreBackup() { # Issue the PITR restore command to vtctldclient. # This should restore the last full backup, followed by applying the # binary logs to reach the desired timestamp. - find "$VTDATAROOT" -name MANIFEST | xargs cat + echo "Listing backups" + vtctldclient ListBackups "$keyspaceShard" + echo "End listing backups" echo "Restoring tablet ${tabletAlias} to timestamp ${INCREMENTAL_RESTORE_TIMESTAMP}" if ! vtctldclient RestoreFromBackup --restore-to-timestamp "${INCREMENTAL_RESTORE_TIMESTAMP}" "${tabletAlias}"; then echo "ERROR: failed to perform incremental restore" From b1a0cb35be505258ed24c6444cad4b1b701bdec0 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 28 Mar 2024 12:47:16 +0200 Subject: [PATCH 16/22] GetBackups Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- test/endtoend/utils.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/endtoend/utils.sh b/test/endtoend/utils.sh index 386eec43..5c74d0f2 100644 --- a/test/endtoend/utils.sh +++ b/test/endtoend/utils.sh @@ -144,7 +144,7 @@ function restoreBackup() { # This should restore the last full backup, followed by applying the # binary logs to reach the desired timestamp. echo "Listing backups" - vtctldclient ListBackups "$keyspaceShard" + vtctldclient GetBackups "$keyspaceShard" echo "End listing backups" echo "Restoring tablet ${tabletAlias} to timestamp ${INCREMENTAL_RESTORE_TIMESTAMP}" if ! vtctldclient RestoreFromBackup --restore-to-timestamp "${INCREMENTAL_RESTORE_TIMESTAMP}" "${tabletAlias}"; then From 919119b286ec18429fe3113c83fde2bf969b49ab Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Thu, 28 Mar 2024 13:04:48 -0400 Subject: [PATCH 17/22] Get test working locally Signed-off-by: Matt Lord --- test/endtoend/utils.sh | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/test/endtoend/utils.sh b/test/endtoend/utils.sh index 5c74d0f2..9888cd9c 100644 --- a/test/endtoend/utils.sh +++ b/test/endtoend/utils.sh @@ -83,7 +83,7 @@ function removeBackupFiles() { # takeBackup: # $1: keyspace-shard for which the backup needs to be taken -declare INCREMENTAL_RESTORE_TIMESTAMP="" +declare INCREMENTAL_RESTORE_POS="" function takeBackup() { keyspaceShard=$1 initialBackupCount=$(kubectl get vtb --no-headers | wc -l) @@ -102,18 +102,13 @@ function takeBackup() { done # Now perform an incremental backup. - echo "Checking @@gtid_exeuted just after full backup:" - mysql -e "select @@gtid_executed;" insertWithRetry - echo "Checking @@gtid_exeuted after some writes and before taking timestamp:" - mysql -e "select @@gtid_executed;" + pos=$(mysql -sN -e "select @@global.gtid_executed") + INCREMENTAL_RESTORE_POS="MySQL56/${pos//\\n/}" sleep 2 - INCREMENTAL_RESTORE_TIMESTAMP=$(date -u "+%Y-%m-%dT%H:%M:%SZ") - echo "Timestamp is $INCREMENTAL_RESTORE_TIMESTAMP" + echo "Backup position is $INCREMENTAL_RESTORE_POS" sleep 2 insertWithRetry - echo "Checking @@gtid_exeuted jbefore taking incremental backup:" - mysql -e "select @@gtid_executed;" vtctldclient BackupShard --incremental-from-pos=auto "${keyspaceShard}" let finalBackupCount=${finalBackupCount}+1 @@ -146,8 +141,8 @@ function restoreBackup() { echo "Listing backups" vtctldclient GetBackups "$keyspaceShard" echo "End listing backups" - echo "Restoring tablet ${tabletAlias} to timestamp ${INCREMENTAL_RESTORE_TIMESTAMP}" - if ! vtctldclient RestoreFromBackup --restore-to-timestamp "${INCREMENTAL_RESTORE_TIMESTAMP}" "${tabletAlias}"; then + echo "Restoring tablet ${tabletAlias} to position ${INCREMENTAL_RESTORE_POS}" + if ! vtctldclient RestoreFromBackup --restore-to-pos "${INCREMENTAL_RESTORE_POS}" "${tabletAlias}"; then echo "ERROR: failed to perform incremental restore" exit 1 fi From 49b933298063bfc5974389338cae79eb19ec73e7 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Wed, 10 Apr 2024 12:17:31 -0400 Subject: [PATCH 18/22] Ignore vtdataroot path created when running local tests Signed-off-by: Matt Lord --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 76cbd661..f77e5e70 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ build/_output build/_test /tools/_bin/ +vtdataroot # Created by https://www.gitignore.io/api/go,vim,emacs,visualstudiocode ### Emacs ### From 2aeb36b8bcdade520950f8dd2ecf31004c351ac6 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Wed, 10 Apr 2024 14:11:53 -0400 Subject: [PATCH 19/22] Fix backup test; use percona image This includes mysqld (of course) and mysqlbinlog But it does NOT include xtrabackup Signed-off-by: Matt Lord --- test/endtoend/operator/101_initial_cluster_backup.yaml | 2 +- tools/get-e2e-test-deps.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/endtoend/operator/101_initial_cluster_backup.yaml b/test/endtoend/operator/101_initial_cluster_backup.yaml index cb9f3c80..28deded4 100644 --- a/test/endtoend/operator/101_initial_cluster_backup.yaml +++ b/test/endtoend/operator/101_initial_cluster_backup.yaml @@ -21,7 +21,7 @@ spec: vtorc: vitess/lite:latest vtbackup: vitess/lite:latest mysqld: - mysql56Compatible: vitess/lite:latest + mysql80Compatible: percona/percona-server:8.0.34 mysqldExporter: prom/mysqld-exporter:v0.11.0 cells: - name: zone1 diff --git a/tools/get-e2e-test-deps.sh b/tools/get-e2e-test-deps.sh index 528b0911..5805c7a5 100755 --- a/tools/get-e2e-test-deps.sh +++ b/tools/get-e2e-test-deps.sh @@ -40,7 +40,7 @@ fi if ! command -v vtctldclient &> /dev/null then echo "Downloading vtctldclient..." - version="19.0.1" + version="19.0.3" hash="3a43ab8" file="vitess-${version}-${hash}.tar.gz" wget https://github.com/vitessio/vitess/releases/download/v${version}/${file} From 2f7768ab255717645fb9b629147d55f64d63bc74 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Tue, 30 Apr 2024 22:04:15 -0400 Subject: [PATCH 20/22] Use /vt/bin for mysqlbinlog and add it to PATH Signed-off-by: Matt Lord --- pkg/operator/vttablet/env_vars.go | 7 ++++++- pkg/operator/vttablet/mysqlctld.go | 2 +- test/endtoend/utils.sh | 2 ++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/pkg/operator/vttablet/env_vars.go b/pkg/operator/vttablet/env_vars.go index 3e3189c5..34c1b6da 100644 --- a/pkg/operator/vttablet/env_vars.go +++ b/pkg/operator/vttablet/env_vars.go @@ -18,10 +18,11 @@ package vttablet import ( "fmt" + "os" "strings" - "planetscale.dev/vitess-operator/pkg/operator/lazy" corev1 "k8s.io/api/core/v1" + "planetscale.dev/vitess-operator/pkg/operator/lazy" ) func init() { @@ -40,6 +41,10 @@ func init() { Name: "EXTRA_MY_CNF", Value: strings.Join(extraMyCnf.Get(spec), ":"), }, + // The mysqlbinlog binary is NOT in the vendor mysql container images + // so we copy it to /mnt/vt/bin when initializing the mysqld container + // so that mysqlctl can find it. + {Name: "PATH", Value: fmt.Sprintf("%s:%s", os.Getenv("PATH"), vtBinPath)}, } }) diff --git a/pkg/operator/vttablet/mysqlctld.go b/pkg/operator/vttablet/mysqlctld.go index 4860aa20..d204ca06 100644 --- a/pkg/operator/vttablet/mysqlctld.go +++ b/pkg/operator/vttablet/mysqlctld.go @@ -29,7 +29,7 @@ const ( vtRootInitScript = `set -ex mkdir -p /mnt/vt/bin cp --no-clobber /vt/bin/mysqlctld /mnt/vt/bin/ -cp --no-clobber $(command -v mysqlbinlog) /mnt/vt/bin/ || true +cp --no-clobber /usr/bin/mysqlbinlog /mnt/vt/bin/ mkdir -p /mnt/vt/config if [[ -d /vt/config/mycnf ]]; then cp --no-clobber -R /vt/config/mycnf /mnt/vt/config/ diff --git a/test/endtoend/utils.sh b/test/endtoend/utils.sh index 731bb9d9..88710b58 100644 --- a/test/endtoend/utils.sh +++ b/test/endtoend/utils.sh @@ -279,7 +279,9 @@ function waitForKeyspaceToBeServing() { nb_of_replica=$3 for i in {1..600} ; do out=$(mysql --table --execute="show vitess_tablets") + echo "Serving output: ${out}" numtablets=$(echo "$out" | grep -E "$ks(.*)$shard(.*)PRIMARY(.*)SERVING|$ks(.*)$shard(.*)REPLICA(.*)SERVING" | wc -l) + echo "Number of serving tablets: ${numtablets}" if [[ $numtablets -ge $((nb_of_replica+1)) ]]; then echo "Shard $ks/$shard is serving" return From ce28c33207d95a0e3f0843108eb155dd64f3bd3d Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Fri, 12 Jul 2024 14:17:48 -0400 Subject: [PATCH 21/22] Add mysqlbinlog to $VTROOT/bin in backup pods Signed-off-by: Matt Lord --- pkg/operator/vttablet/vtbackup_pod.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/operator/vttablet/vtbackup_pod.go b/pkg/operator/vttablet/vtbackup_pod.go index 1f5a5ec9..4bcb3007 100644 --- a/pkg/operator/vttablet/vtbackup_pod.go +++ b/pkg/operator/vttablet/vtbackup_pod.go @@ -35,6 +35,7 @@ const ( vtbackupInitScript = `set -ex mkdir -p /mnt/vt/bin cp --no-clobber /vt/bin/vtbackup /mnt/vt/bin/ +cp --no-clobber /usr/bin/mysqlbinlog /mnt/vt/bin/ mkdir -p /mnt/vt/config if [[ -d /vt/config/mycnf ]]; then cp --no-clobber -R /vt/config/mycnf /mnt/vt/config/ From 5e7df2b58f45d9097a64706ac50d498353c29937 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Fri, 12 Jul 2024 14:54:50 -0400 Subject: [PATCH 22/22] Testing... Signed-off-by: Matt Lord --- pkg/operator/vttablet/vtbackup_pod.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/operator/vttablet/vtbackup_pod.go b/pkg/operator/vttablet/vtbackup_pod.go index 4bcb3007..8e8b3a1e 100644 --- a/pkg/operator/vttablet/vtbackup_pod.go +++ b/pkg/operator/vttablet/vtbackup_pod.go @@ -116,6 +116,12 @@ func NewBackupPod(key client.ObjectKey, backupSpec *BackupSpec, mysqldImage stri MountPath: sslCertsPath, SubPath: "certs", }, + { + Name: vtRootVolumeName, + ReadOnly: true, + MountPath: vtBinPath, + SubPath: "bin", + }, } volumeMounts = append(volumeMounts, mysqldVolumeMounts.Get(tabletSpec)...) volumeMounts = append(volumeMounts, tabletVolumeMounts.Get(tabletSpec)...)