Test incremental backup and restore

Signed-off-by: Matt Lord <mattalord@gmail.com>

pkg/operator/vttablet/pod.go

@@ -91,10 +91,12 @@ func UpdatePod(obj *corev1.Pod, spec *Spec) {
 	// Ensure that binary logs are restored to/from a location that all containers
 	// in the pod can access if no location was explicitly provided.
 	if _, ok := vttabletAllFlags["builtinbackup-incremental-restore-path"]; !ok {
+		// This flag only exists in 2.12.0 and later as it includes vitess
+		// 20.0 as a dependency.
 		parts := strings.Split(version.Version, ".")
-		if len(parts) > 1 {
-			major, _ := strconv.Atoi(parts[1])
-			minor, _ := strconv.Atoi(parts[2])
+		if len(parts) > 0 {
+			major, _ := strconv.Atoi(parts[0])
+			minor, _ := strconv.Atoi(parts[1])
 			if major >= 2 && minor >= 12 {
 				vttabletAllFlags["builtinbackup-incremental-restore-path"] = vtDataRootPath
 			}

test/endtoend/utils.sh

@@ -83,43 +83,81 @@ function removeBackupFiles() {
 
 # takeBackup:
 # $1: keyspace-shard for which the backup needs to be taken
+declare INCREMENTAL_RESTORE_TIMESTAMP=""
 function takeBackup() {
   keyspaceShard=$1
   initialBackupCount=$(kubectl get vtb --no-headers | wc -l)
   finalBackupCount=$((initialBackupCount+1))
 
-  # issue the backupShard command to vtctldclient
-  vtctldclient BackupShard "$keyspaceShard"
+  # Issue the BackupShard command to vtctldclient.
+  vtctldclient BackupShard "${keyspaceShard}"
 
   for i in {1..600} ; do
     out=$(kubectl get vtb --no-headers | wc -l)
-    echo "$out" | grep "$finalBackupCount" > /dev/null 2>&1
-    if [ $? -eq 0 ]; then
-      echo "Backup created"
+    if echo "${out}" | grep -c "${finalBackupCount}" >/dev/null; then
+      echo "Full backup created"
+      break
+    fi
+    sleep 3
+  done
+
+  sleep 10
+
+  # Now perform an incremental backup.
+  insertWithRetry
+  INCREMENTAL_RESTORE_TIMESTAMP=$(date -u "+%Y-%m-%dT%H:%M:%SZ")
+  sleep 1
+  insertWithRetry
+
+  sleep 10
+
+  vtctldclient BackupShard --incremental-from-pos=auto "${keyspaceShard}"
+  let finalBackupCount=${finalBackupCount}+1
+
+  for i in {1..600} ; do
+    out=$(kubectl get vtb --no-headers | wc -l)
+    if echo "${out}" | grep -c "${finalBackupCount}" >/dev/null; then
+      echo "Incremental backup created"
       return 0
     fi
     sleep 3
   done
-  echo -e "ERROR: Backup not created - $out. $backupCount backups expected."
+
+  echo -e "ERROR: Backups not created - ${out}. ${backupCount} backups expected."
   exit 1
 }
 
 # restoreBackup:
 # $1: tablet alias for which the backup needs to be restored
 function restoreBackup() {
   tabletAlias=$1
-  if [[ -z "$tabletAlias" ]]; then
+  if [[ -z "${tabletAlias}" ]]; then
     echo "Tablet alias not provided as restore target"
     exit 1
   fi
 
   # Issue the PITR restore command to vtctldclient.
-  vtctldclient RestoreFromBackup --restore-to-timestamp $(date -u "+%Y-%m-%d.%H%M%S") "${tabletAlias}"
-
-  if [[ $? -ne 0 ]]; then
-    echo "Restore failed"
+  # This should restore the last full backup, followed by applying the
+  # binary logs to reach the desired timestamp.
+  if ! vtctldclient RestoreFromBackup --restore-to-timestamp "${INCREMENTAL_RESTORE_TIMESTAMP}" "${tabletAlias}"; then
+    echo "ERROR: failed to perform incremental restore"
     exit 1
   fi
+
+  cell="${tabletAlias%-*}"
+  uid="${tabletAlias##*-}"
+
+  for i in {1..600} ; do
+    out=$(kubectl get pods --no-headers -l "planetscale.com/cell=${cell},planetscale.com/tablet-uid=${uid}" | grep "Running" | wc -l)
+    if echo "$out" | grep -c "1" >/dev/null; then
+      echo "Tablet ${tabletAlias} restore complete"
+      return 0
+    fi
+    sleep 3
+  done
+
+  echo -e "ERROR: restored tablet ${tabletAlias} did not become healthy after the restore."
+  exit 1
 }
 
 function verifyListBackupsOutput() {