Migrate to Xormigrate

cmd/server/setup.go

@@ -52,8 +52,6 @@ import (
 )
 
 func setupStore(c *cli.Context) (store.Store, error) {
-	// TODO: find a better way than global var to pass down to allow long migrations
-	server.Config.Server.Migrations.AllowLong = c.Bool("migrations-allow-long")
 	datasource := c.String("datasource")
 	driver := c.String("driver")
 	xorm := store.XORM{
@@ -90,7 +88,7 @@ func setupStore(c *cli.Context) (store.Store, error) {
 		log.Fatal().Err(err).Msg("could not open datastore")
 	}
 
-	if err := store.Migrate(); err != nil {
+	if err := store.Migrate(c.Bool("migrations-allow-long")); err != nil {
 		log.Fatal().Err(err).Msg("could not migrate datastore")
 	}
 

go.mod

@@ -61,6 +61,7 @@ require (
 	k8s.io/api v0.28.4
 	k8s.io/apimachinery v0.28.4
 	k8s.io/client-go v0.28.4
+	src.techknowlogick.com/xormigrate v1.7.1
 	xorm.io/builder v0.3.13
 	xorm.io/xorm v1.3.4
 )

go.sum

@@ -62,6 +62,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0=
 github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE=
+github.com/denisenkom/go-mssqldb v0.12.3 h1:pBSGx9Tq67pBOTLmxNuirNTeB8Vjmf886Kx+8Y+8shw=
 github.com/denisenkom/go-mssqldb v0.12.3/go.mod h1:k0mtMFOnU+AihqFxPMiF05rtiDrorD1Vrm1KEz5hxDo=
 github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0=
 github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
@@ -145,7 +146,9 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang-jwt/jwt/v5 v5.1.0 h1:UGKbA/IPjtS6zLcdB7i5TyACMgSbOTiR8qzXgw8HWQU=
 github.com/golang-jwt/jwt/v5 v5.1.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
+github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY=
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
+github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A=
 github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
@@ -236,6 +239,7 @@ github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dv
 github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jellydator/ttlcache/v3 v3.1.0 h1:0gPFG0IHHP6xyUyXq+JaD8fwkDCqgqwohXNJBcYE71g=
 github.com/jellydator/ttlcache/v3 v3.1.0/go.mod h1:hi7MGFdMAwZna5n2tuvh63DvFLzVKySzCVW6+0gA2n4=
+github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
 github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
 github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
@@ -671,8 +675,11 @@ sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kF
 sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E=
 sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
 sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
+src.techknowlogick.com/xormigrate v1.7.1 h1:RKGLLUAqJ+zO8iZ7eOc7oLH7f0cs2gfXSZSvBRBHnlY=
+src.techknowlogick.com/xormigrate v1.7.1/go.mod h1:YGNBdj8prENlySwIKmfoEXp7ILGjAltyKFXD0qLgD7U=
 xorm.io/builder v0.3.11-0.20220531020008-1bd24a7dc978/go.mod h1:aUW0S9eb9VCaPohFCH3j7czOx1PMW3i1HrSzbLYGBSE=
 xorm.io/builder v0.3.13 h1:a3jmiVVL19psGeXx8GIurTp7p0IIgqeDmwhcR6BAOAo=
 xorm.io/builder v0.3.13/go.mod h1:aUW0S9eb9VCaPohFCH3j7czOx1PMW3i1HrSzbLYGBSE=
+xorm.io/xorm v1.3.3/go.mod h1:qFJGFoVYbbIdnz2vaL5OxSQ2raleMpyRRalnq3n9OJo=
 xorm.io/xorm v1.3.4 h1:vWFKzR3DhGUDl5b4srhUjhDwjxkZAc4C7BFszpu0swI=
 xorm.io/xorm v1.3.4/go.mod h1:qFJGFoVYbbIdnz2vaL5OxSQ2raleMpyRRalnq3n9OJo=

server/config.go

@@ -70,10 +70,7 @@ var Config = struct {
 		RootPath            string
 		CustomCSSFile       string
 		CustomJsFile        string
-		Migrations          struct {
-			AllowLong bool
-		}
-		EnableSwagger bool
+		EnableSwagger       bool
 		// Open bool
 		// Orgs map[string]struct{}
 		// Admins map[string]struct{}

server/store/datastore/engine.go

@@ -16,6 +16,7 @@
 
 import (
 	"github.com/rs/zerolog"
+
 	"go.woodpecker-ci.org/woodpecker/server/store"
 	"go.woodpecker-ci.org/woodpecker/server/store/datastore/migration"
 
@@ -54,8 +55,8 @@
 }
 
 // Migrate old storage or init new one
-func (s storage) Migrate() error {
-	return migration.Migrate(s.engine)
+func (s storage) Migrate(allowLong bool) error {
+	return migration.Migrate(s.engine, allowLong)
 }
 
 func (s storage) Close() error {

server/store/datastore/migration/000_legacy_to_xormigrate.go

@@ -0,0 +1,33 @@
+package migration
+
+import (
+	"src.techknowlogick.com/xormigrate"
+	"xorm.io/xorm"
+)
+
+type v000Migrations struct {
+	Name string `xorm:"UNIQUE"`
+}
+
+func (m *v000Migrations) TableName() string {
+	return "migrations"
+}
+
+var legacyToXormigrate = xormigrate.Migration{
+	ID: "legacy-to-xormigrate",
+	MigrateSession: func(sess *xorm.Session) error {
+		var mig []*v000Migrations
+		if err := sess.Find(&mig); err != nil {
+			return err
+		}
+		for _, m := range mig {
+			if _, err := sess.Insert(&xormigrate.Migration{
+				ID: m.Name,
+			}); err != nil {
+				return err
+			}
+		}
+
+		return sess.DropTable("migrations")
+	},
+}

server/store/datastore/migration/000_legacy_to_xorm.go → server/store/datastore/migration/001_legacy_to_xorm.go

@@ -18,14 +18,14 @@ import (
 	"fmt"
 
 	"github.com/rs/zerolog/log"
+	"src.techknowlogick.com/xormigrate"
 	"xorm.io/xorm"
 	"xorm.io/xorm/schemas"
 )
 
-var legacy2Xorm = task{
-	name:     "xorm",
-	required: true,
-	fn: func(sess *xorm.Session) error {
+var legacy2Xorm = xormigrate.Migration{
+	ID: "xorm",
+	MigrateSession: func(sess *xorm.Session) error {
 		// make sure we have required migrations - else fail and point to last major version
 		for _, mig := range []string{
 			// users
@@ -74,7 +74,7 @@ var legacy2Xorm = task{
 			"create-table-build-config",
 			"populate-build-config",
 		} {
-			exist, err := sess.Exist(&migrations{mig})
+			exist, err := sess.Exist(&xormigrate.Migration{ID: mig})
 			if err != nil {
 				return fmt.Errorf("test migration existence: %w", err)
 			}

server/store/datastore/migration/001_repos_drop_fallback.go → server/store/datastore/migration/002_repos_drop_fallback.go

@@ -15,12 +15,13 @@
 package migration
 
 import (
+	"src.techknowlogick.com/xormigrate"
 	"xorm.io/xorm"
 )
 
-var alterTableReposDropFallback = task{
-	name: "alter-table-drop-repo-fallback",
-	fn: func(sess *xorm.Session) error {
+var alterTableReposDropFallback = xormigrate.Migration{
+	ID: "alter-table-drop-repo-fallback",
+	MigrateSession: func(sess *xorm.Session) error {
 		return dropTableColumns(sess, "repos", "repo_fallback")
 	},
 }

server/store/datastore/migration/002_repos_drop_allow_deploys_allow_tags.go → server/store/datastore/migration/003_repos_drop_allow_deploys_allow_tags.go

@@ -15,12 +15,13 @@
 package migration
 
 import (
+	"src.techknowlogick.com/xormigrate"
 	"xorm.io/xorm"
 )
 
-var alterTableReposDropAllowDeploysAllowTags = task{
-	name: "drop-allow-push-tags-deploys-columns",
-	fn: func(sess *xorm.Session) error {
+var alterTableReposDropAllowDeploysAllowTags = xormigrate.Migration{
+	ID: "drop-allow-push-tags-deploys-columns",
+	MigrateSession: func(sess *xorm.Session) error {
 		return dropTableColumns(sess, "repos",
 			"repo_allow_deploys",
 			"repo_allow_tags",

server/store/datastore/migration/003_fix_pr_secret_event_name.go → server/store/datastore/migration/004_fix_pr_secret_event_name.go

@@ -15,14 +15,15 @@
 package migration
 
 import (
+	"src.techknowlogick.com/xormigrate"
 	"xorm.io/xorm"
 
 	"go.woodpecker-ci.org/woodpecker/server/model"
 )
 
-var fixPRSecretEventName = task{
-	name: "fix-pr-secret-event-name",
-	fn: func(sess *xorm.Session) error {
+var fixPRSecretEventName = xormigrate.Migration{
+	ID: "fix-pr-secret-event-name",
+	MigrateSession: func(sess *xorm.Session) error {
 		const batchSize = 100
 		for start := 0; ; start += batchSize {
 			secrets := make([]*model.Secret, 0, batchSize)

server/store/datastore/migration/004_repos_drop_repo_counter.go → server/store/datastore/migration/005_repos_drop_repo_counter.go

@@ -15,12 +15,13 @@
 package migration
 
 import (
+	"src.techknowlogick.com/xormigrate"
 	"xorm.io/xorm"
 )
 
-var alterTableReposDropCounter = task{
-	name: "alter-table-drop-counter",
-	fn: func(sess *xorm.Session) error {
+var alterTableReposDropCounter = xormigrate.Migration{
+	ID: "alter-table-drop-counter",
+	MigrateSession: func(sess *xorm.Session) error {
 		return dropTableColumns(sess, "repos", "repo_counter")
 	},
 }

server/store/datastore/migration/005_drop_senders.go → server/store/datastore/migration/006_drop_senders.go

@@ -15,12 +15,13 @@
 package migration
 
 import (
+	"src.techknowlogick.com/xormigrate"
 	"xorm.io/xorm"
 )
 
-var dropSenders = task{
-	name: "drop-senders",
-	fn: func(sess *xorm.Session) error {
+var dropSenders = xormigrate.Migration{
+	ID: "drop-senders",
+	MigrateSession: func(sess *xorm.Session) error {
 		return sess.DropTable("senders")
 	},
 }

server/store/datastore/migration/006_log_data_type.go → server/store/datastore/migration/007_log_data_type.go

@@ -15,13 +15,14 @@
 package migration
 
 import (
+	"src.techknowlogick.com/xormigrate"
 	"xorm.io/xorm"
 	"xorm.io/xorm/schemas"
 )
 
-var alterTableLogUpdateColumnLogDataType = task{
-	name: "alter-table-logs-update-type-of-data",
-	fn: func(sess *xorm.Session) (err error) {
+var alterTableLogUpdateColumnLogDataType = xormigrate.Migration{
+	ID: "alter-table-logs-update-type-of-data",
+	MigrateSession: func(sess *xorm.Session) (err error) {
 		dialect := sess.Engine().Dialect().URI().DBType
 
 		switch dialect {

server/store/datastore/migration/007_secrets_add_user.go → server/store/datastore/migration/008_secrets_add_user.go

@@ -15,24 +15,25 @@
 package migration
 
 import (
+	"src.techknowlogick.com/xormigrate"
 	"xorm.io/xorm"
 )
 
-type SecretV007 struct {
+type SecretV008 struct {
 	Owner  string `json:"-"    xorm:"NOT NULL DEFAULT '' UNIQUE(s) INDEX 'secret_owner'"`
 	RepoID int64  `json:"-"    xorm:"NOT NULL DEFAULT 0 UNIQUE(s) INDEX 'secret_repo_id'"`
 	Name   string `json:"name" xorm:"NOT NULL UNIQUE(s) INDEX 'secret_name'"`
 }
 
 // TableName return database table name for xorm
-func (SecretV007) TableName() string {
+func (SecretV008) TableName() string {
 	return "secrets"
 }
 
-var alterTableSecretsAddUserCol = task{
-	name: "alter-table-add-secrets-user-id",
-	fn: func(sess *xorm.Session) error {
-		if err := sess.Sync(new(SecretV007)); err != nil {
+var alterTableSecretsAddUserCol = xormigrate.Migration{
+	ID: "alter-table-add-secrets-user-id",
+	MigrateSession: func(sess *xorm.Session) error {
+		if err := sess.Sync(new(SecretV008)); err != nil {
 			return err
 		}
 		if err := alterColumnDefault(sess, "secrets", "secret_repo_id", "0"); err != nil {

server/store/datastore/migration/008_lowercase_secret_names.go → server/store/datastore/migration/009_lowercase_secret_names.go

@@ -15,12 +15,13 @@
 package migration
 
 import (
+	"src.techknowlogick.com/xormigrate"
 	"xorm.io/xorm"
 )
 
-var lowercaseSecretNames = task{
-	name: "lowercase-secret-names",
-	fn: func(sess *xorm.Session) (err error) {
+var lowercaseSecretNames = xormigrate.Migration{
+	ID: "lowercase-secret-names",
+	MigrateSession: func(sess *xorm.Session) (err error) {
 		_, err = sess.Exec("UPDATE secrets SET secret_name = LOWER(secret_name);")
 		return err
 	},

server/store/datastore/migration/008_recreate_agents_table.go → server/store/datastore/migration/009_recreate_agents_table.go

@@ -15,14 +15,15 @@
 package migration
 
 import (
+	"src.techknowlogick.com/xormigrate"
 	"xorm.io/xorm"
 
 	"go.woodpecker-ci.org/woodpecker/server/model"
 )
 
-var recreateAgentsTable = task{
-	name: "recreate-agents-table",
-	fn: func(sess *xorm.Session) error {
+var recreateAgentsTable = xormigrate.Migration{
+	ID: "recreate-agents-table",
+	MigrateSession: func(sess *xorm.Session) error {
 		if err := sess.DropTable("agents"); err != nil {
 			return err
 		}

server/store/datastore/migration/009_rename_builds_to_pipeline.go → server/store/datastore/migration/010_rename_builds_to_pipeline.go

@@ -15,13 +15,13 @@
 package migration
 
 import (
+	"src.techknowlogick.com/xormigrate"
 	"xorm.io/xorm"
 )
 
-var renameBuildsToPipeline = task{
-	name:     "rename-builds-to-pipeline",
-	required: true,
-	fn: func(sess *xorm.Session) error {
+var renameBuildsToPipeline = xormigrate.Migration{
+	ID: "rename-builds-to-pipeline",
+	MigrateSession: func(sess *xorm.Session) error {
 		err := renameTable(sess, "builds", "pipelines")
 		if err != nil {
 			return err

server/store/datastore/migration/010_columns_rename_builds_to_pipeline.go → server/store/datastore/migration/011_columns_rename_builds_to_pipeline.go

@@ -17,6 +17,7 @@ package migration
 import (
 	"strings"
 
+	"src.techknowlogick.com/xormigrate"
 	"xorm.io/xorm"
 )
 
@@ -25,10 +26,9 @@ type oldTable struct {
 	columns []string
 }
 
-var renameColumnsBuildsToPipeline = task{
-	name:     "rename-columns-builds-to-pipeline",
-	required: true,
-	fn: func(sess *xorm.Session) error {
+var renameColumnsBuildsToPipeline = xormigrate.Migration{
+	ID: "rename-columns-builds-to-pipeline",
+	MigrateSession: func(sess *xorm.Session) error {
 		var oldColumns []*oldTable
 
 		oldColumns = append(oldColumns, &oldTable{

server/store/datastore/migration/011_columns_rename_procs_to_steps.go → server/store/datastore/migration/012_columns_rename_procs_to_steps.go

@@ -17,13 +17,13 @@ package migration
 import (
 	"strings"
 
+	"src.techknowlogick.com/xormigrate"
 	"xorm.io/xorm"
 )
 
-var renameTableProcsToSteps = task{
-	name:     "rename-procs-to-steps",
-	required: true,
-	fn: func(sess *xorm.Session) error {
+var renameTableProcsToSteps = xormigrate.Migration{
+	ID: "rename-procs-to-steps",
+	MigrateSession: func(sess *xorm.Session) error {
 		err := renameTable(sess, "procs", "steps")
 		if err != nil {
 			return err