Merge pull request #968 from lightninglabs/backup-sqlite-db-before-mi…

…gration

tapdb: backup sqlite db before running database migration

sample-tapd.conf

@@ -218,6 +218,9 @@
 ; Skip applying migrations on startup
 ; sqlite.skipmigrations=false
 
+; Skip database backup before schema migration
+; sqlite.skipmigrationdbbackup=false
+
 ; The full path to the database
 ; sqlite.dbfile=~/.tapd/data/testnet/tapd.db
 

tapdb/migrations.go

@@ -26,20 +26,24 @@ const (
 )
 
 // MigrationTarget is a functional option that can be passed to applyMigrations
-// to specify a target version to migrate to.
-type MigrationTarget func(mig *migrate.Migrate) error
+// to specify a target version to migrate to. `currentDbVersion` is the current
+// (migration) version of the database, or None if unknown.
+// `maxMigrationVersion` is the maximum migration version known to the driver,
+// or None if unknown.
+type MigrationTarget func(mig *migrate.Migrate,
+	currentDbVersion int, maxMigrationVersion uint) error
 
 var (
 	// TargetLatest is a MigrationTarget that migrates to the latest
 	// version available.
-	TargetLatest = func(mig *migrate.Migrate) error {
+	TargetLatest = func(mig *migrate.Migrate, _ int, _ uint) error {
 		return mig.Up()
 	}
 
 	// TargetVersion is a MigrationTarget that migrates to the given
 	// version.
 	TargetVersion = func(version uint) MigrationTarget {
-		return func(mig *migrate.Migrate) error {
+		return func(mig *migrate.Migrate, _ int, _ uint) error {
 			return mig.Migrate(version)
 		}
 	}
@@ -146,17 +150,31 @@ func applyMigrations(fs fs.FS, driver database.Driver, path, dbName string,
 			ErrMigrationDowngrade, migrationVersion, latestVersion)
 	}
 
-	log.Infof("Applying migrations from version=%v", migrationVersion)
+	// Report the current version of the database before the migration.
+	currentDbVersion, _, err := driver.Version()
+	if err != nil {
+		return fmt.Errorf("unable to get current db version: %w", err)
+	}
+	log.Infof("Attempting to apply migration(s) "+
+		"(current_db_version=%v, latest_migration_version=%v)",
+		currentDbVersion, latestVersion)
 
 	// Apply our local logger to the migration instance.
 	sqlMigrate.Log = &migrationLogger{log}
 
 	// Execute the migration based on the target given.
-	err = targetVersion(sqlMigrate)
+	err = targetVersion(sqlMigrate, currentDbVersion, latestVersion)
 	if err != nil && !errors.Is(err, migrate.ErrNoChange) {
 		return err
 	}
 
+	// Report the current version of the database after the migration.
+	currentDbVersion, _, err = driver.Version()
+	if err != nil {
+		return fmt.Errorf("unable to get current db version: %w", err)
+	}
+	log.Infof("Database version after migration: %v", currentDbVersion)
+
 	return nil
 }
 

tapdb/migrations_test.go

@@ -2,6 +2,9 @@ package tapdb
 
 import (
 	"context"
+	"os"
+	"path/filepath"
+	"strings"
 	"testing"
 
 	"github.com/btcsuite/btcd/wire"
@@ -130,3 +133,90 @@ func TestMigrationDowngrade(t *testing.T) {
 	err := db.ExecuteMigrations(TargetLatest, WithLatestVersion(1))
 	require.ErrorIs(t, err, ErrMigrationDowngrade)
 }
+
+// findDbBackupFilePath walks the directory of the given database file path and
+// returns the path to the backup file.
+func findDbBackupFilePath(t *testing.T, dbFilePath string) string {
+	var dbBackupFilePath string
+	dir := filepath.Dir(dbFilePath)
+
+	err := filepath.Walk(
+		dir, func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+
+			hasSuffix := strings.HasSuffix(info.Name(), ".backup")
+			if !info.IsDir() && hasSuffix {
+				dbBackupFilePath = path
+			}
+			return nil
+		},
+	)
+	require.NoError(t, err)
+
+	return dbBackupFilePath
+}
+
+// TestSqliteMigrationBackup tests that the sqlite database backup and migration
+// functionality works.
+//
+// In this test we will load from file a database that is at version 14. The
+// on file database is already populated with asset data. We will create a
+// database backup, migrate the source db, and then check the following:
+//
+// 1. The asset data is present in the migrated database.
+// 2. The asset data is present in the backup database.
+func TestSqliteMigrationBackup(t *testing.T) {
+	ctx := context.Background()
+
+	db := NewTestSqliteDBWithVersion(t, 14)
+
+	// We need to insert some test data that will be affected by the
+	// migration number 15.
+	InsertTestdata(t, db.BaseDB, "migrations_test_00015_dummy_data.sql")
+
+	// And now that we have test data inserted, we can create a backup and
+	// migrate to the latest version.
+	err := db.ExecuteMigrations(db.backupAndMigrate)
+	require.NoError(t, err)
+
+	// Inspect the migrated database. Make sure the single asset that was
+	// inserted actually has two witnesses with the correct order.
+	_, assetStore := newAssetStoreFromDB(db.BaseDB)
+	assets, err := assetStore.FetchAllAssets(ctx, false, false, nil)
+	require.NoError(t, err)
+
+	require.Len(t, assets, 1)
+	require.Len(t, assets[0].PrevWitnesses, 2)
+	require.Equal(
+		t, wire.TxWitness{{0xaa}}, assets[0].PrevWitnesses[0].TxWitness,
+	)
+	require.Equal(
+		t, wire.TxWitness{{0xbb}}, assets[0].PrevWitnesses[1].TxWitness,
+	)
+
+	// Now we will inspect the backup database (which should not have been
+	// modified by the migration).
+	//
+	// Find the backup database file.
+	dbBackupFilePath := findDbBackupFilePath(t, db.cfg.DatabaseFileName)
+
+	// Construct a new database handle for the backup database.
+	backupDb := NewTestSqliteDbHandleFromPath(t, dbBackupFilePath)
+	require.NoError(t, err)
+
+	// Inspect the backup database.
+	_, assetStore = newAssetStoreFromDB(backupDb.BaseDB)
+	assets, err = assetStore.FetchAllAssets(ctx, false, false, nil)
+	require.NoError(t, err)
+
+	require.Len(t, assets, 1)
+	require.Len(t, assets[0].PrevWitnesses, 2)
+	require.Equal(
+		t, wire.TxWitness{{0xaa}}, assets[0].PrevWitnesses[0].TxWitness,
+	)
+	require.Equal(
+		t, wire.TxWitness{{0xbb}}, assets[0].PrevWitnesses[1].TxWitness,
+	)
+}

tapdb/sqlite.go

@@ -8,6 +8,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/golang-migrate/migrate/v4"
 	sqlite_migrate "github.com/golang-migrate/migrate/v4/database/sqlite"
 	"github.com/lightninglabs/taproot-assets/tapdb/sqlc"
 	"github.com/stretchr/testify/require"
@@ -48,11 +49,17 @@ var (
 
 // SqliteConfig holds all the config arguments needed to interact with our
 // sqlite DB.
+//
+// nolint: lll
 type SqliteConfig struct {
 	// SkipMigrations if true, then all the tables will be created on start
 	// up if they don't already exist.
 	SkipMigrations bool `long:"skipmigrations" description:"Skip applying migrations on startup."`
 
+	// SkipMigrationDbBackup if true, then a backup of the database will not
+	// be created before applying migrations.
+	SkipMigrationDbBackup bool `long:"skipmigrationdbbackup" description:"Skip creating a backup of the database before applying migrations."`
+
 	// DatabaseFileName is the full file path where the database file can be
 	// found.
 	DatabaseFileName string `long:"dbfile" description:"The full path to the database."`
@@ -140,7 +147,7 @@ func NewSqliteStore(cfg *SqliteConfig) (*SqliteStore, error) {
 	// Now that the database is open, populate the database with our set of
 	// schemas based on our embedded in-memory file system.
 	if !cfg.SkipMigrations {
-		if err := s.ExecuteMigrations(TargetLatest); err != nil {
+		if err := s.ExecuteMigrations(s.backupAndMigrate); err != nil {
 			return nil, fmt.Errorf("error executing migrations: "+
 				"%w", err)
 		}
@@ -149,6 +156,79 @@ func NewSqliteStore(cfg *SqliteConfig) (*SqliteStore, error) {
 	return s, nil
 }
 
+// backupSqliteDatabase creates a backup of the given SQLite database.
+func backupSqliteDatabase(srcDB *sql.DB, dbFullFilePath string) error {
+	if srcDB == nil {
+		return fmt.Errorf("backup source database is nil")
+	}
+
+	// Create a database backup file full path from the given source
+	// database full file path.
+	//
+	// Get the current time and format it as a Unix timestamp in
+	// nanoseconds.
+	timestamp := time.Now().UnixNano()
+
+	// Add the timestamp to the backup name.
+	backupFullFilePath := fmt.Sprintf(
+		"%s.%d.backup", dbFullFilePath, timestamp,
+	)
+
+	log.Infof("Creating backup of database file: %v -> %v",
+		dbFullFilePath, backupFullFilePath)
+
+	// Create the database backup.
+	vacuumIntoQuery := "VACUUM INTO ?;"
+	stmt, err := srcDB.Prepare(vacuumIntoQuery)
+	if err != nil {
+		return err
+	}
+	defer stmt.Close()
+
+	_, err = stmt.Exec(backupFullFilePath)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// backupAndMigrate is a helper function that creates a database backup before
+// initiating the migration, and then migrates the database to the latest
+// version.
+func (s *SqliteStore) backupAndMigrate(mig *migrate.Migrate,
+	currentDbVersion int, maxMigrationVersion uint) error {
+
+	// Determine if a database migration is necessary given the current
+	// database version and the maximum migration version.
+	versionUpgradePending := currentDbVersion < int(maxMigrationVersion)
+	if !versionUpgradePending {
+		log.Infof("Current database version is up-to-date, skipping "+
+			"migration attempt and backup creation "+
+			"(current_db_version=%v, max_migration_version=%v)",
+			currentDbVersion, maxMigrationVersion)
+		return nil
+	}
+
+	// At this point, we know that a database migration is necessary.
+	// Create a backup of the database before starting the migration.
+	if !s.cfg.SkipMigrationDbBackup {
+		log.Infof("Creating database backup (before applying " +
+			"migration(s))")
+
+		err := backupSqliteDatabase(s.DB, s.cfg.DatabaseFileName)
+		if err != nil {
+			return err
+		}
+	} else {
+		log.Infof("Skipping database backup creation before applying " +
+			"migration(s)")
+	}
+
+	log.Infof("Applying migrations to database")
+	return mig.Up()
+}
+
 // ExecuteMigrations runs migrations for the sqlite database, depending on the
 // target given, either all migrations or up to a given version.
 func (s *SqliteStore) ExecuteMigrations(target MigrationTarget,