woodpecker-ci · anbraten · Nov 28, 2023 · Oct 26, 2023 · Oct 27, 2023 · Nov 3, 2023
diff --git a/cmd/server/setup.go b/cmd/server/setup.go
@@ -52,7 +52,7 @@ import (
 )
 
 func setupStore(c *cli.Context) (store.Store, error) {
-	// TODO: find a better way than global var to pass down to allow long migrations
+	// TODO: find a better way than global var to pass down to allow long migrations -> remove for 3.x
 	server.Config.Server.Migrations.AllowLong = c.Bool("migrations-allow-long")
 	datasource := c.String("datasource")
 	driver := c.String("driver")
@@ -90,7 +90,7 @@ func setupStore(c *cli.Context) (store.Store, error) {
 		log.Fatal().Err(err).Msg("could not open datastore")
 	}
 
-	if err := store.Migrate(); err != nil {
+	if err := store.Migrate(c.Bool("migrations-allow-long")); err != nil {
 		log.Fatal().Err(err).Msg("could not migrate datastore")
 	}
 

diff --git a/go.mod b/go.mod
@@ -61,6 +61,7 @@ require (
 	k8s.io/api v0.28.4
 	k8s.io/apimachinery v0.28.4
 	k8s.io/client-go v0.28.4
+	src.techknowlogick.com/xormigrate v1.7.1
 	xorm.io/builder v0.3.13
 	xorm.io/xorm v1.3.4
 )

diff --git a/go.sum b/go.sum
@@ -62,6 +62,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0=
 github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE=
+github.com/denisenkom/go-mssqldb v0.12.3 h1:pBSGx9Tq67pBOTLmxNuirNTeB8Vjmf886Kx+8Y+8shw=
 github.com/denisenkom/go-mssqldb v0.12.3/go.mod h1:k0mtMFOnU+AihqFxPMiF05rtiDrorD1Vrm1KEz5hxDo=
 github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0=
 github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
@@ -145,7 +146,9 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang-jwt/jwt/v5 v5.1.0 h1:UGKbA/IPjtS6zLcdB7i5TyACMgSbOTiR8qzXgw8HWQU=
 github.com/golang-jwt/jwt/v5 v5.1.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
+github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY=
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
+github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A=
 github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
@@ -236,6 +239,7 @@ github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dv
 github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jellydator/ttlcache/v3 v3.1.0 h1:0gPFG0IHHP6xyUyXq+JaD8fwkDCqgqwohXNJBcYE71g=
 github.com/jellydator/ttlcache/v3 v3.1.0/go.mod h1:hi7MGFdMAwZna5n2tuvh63DvFLzVKySzCVW6+0gA2n4=
+github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
 github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
 github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
@@ -671,8 +675,11 @@ sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kF
 sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E=
 sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
 sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
+src.techknowlogick.com/xormigrate v1.7.1 h1:RKGLLUAqJ+zO8iZ7eOc7oLH7f0cs2gfXSZSvBRBHnlY=
+src.techknowlogick.com/xormigrate v1.7.1/go.mod h1:YGNBdj8prENlySwIKmfoEXp7ILGjAltyKFXD0qLgD7U=
 xorm.io/builder v0.3.11-0.20220531020008-1bd24a7dc978/go.mod h1:aUW0S9eb9VCaPohFCH3j7czOx1PMW3i1HrSzbLYGBSE=
 xorm.io/builder v0.3.13 h1:a3jmiVVL19psGeXx8GIurTp7p0IIgqeDmwhcR6BAOAo=
 xorm.io/builder v0.3.13/go.mod h1:aUW0S9eb9VCaPohFCH3j7czOx1PMW3i1HrSzbLYGBSE=
+xorm.io/xorm v1.3.3/go.mod h1:qFJGFoVYbbIdnz2vaL5OxSQ2raleMpyRRalnq3n9OJo=
 xorm.io/xorm v1.3.4 h1:vWFKzR3DhGUDl5b4srhUjhDwjxkZAc4C7BFszpu0swI=
 xorm.io/xorm v1.3.4/go.mod h1:qFJGFoVYbbIdnz2vaL5OxSQ2raleMpyRRalnq3n9OJo=
diff --git a/server/store/datastore/engine.go b/server/store/datastore/engine.go
@@ -16,6 +16,7 @@
 
 import (
 	"github.com/rs/zerolog"
+
 	"go.woodpecker-ci.org/woodpecker/server/store"
 	"go.woodpecker-ci.org/woodpecker/server/store/datastore/migration"
 
@@ -54,8 +55,8 @@
 }
 
 // Migrate old storage or init new one
-func (s storage) Migrate() error {
-	return migration.Migrate(s.engine)
+func (s storage) Migrate(allowLong bool) error {
+	return migration.Migrate(s.engine, allowLong)
 }
 
 func (s storage) Close() error {

diff --git a/server/store/datastore/migration/000_legacy_to_xormigrate.go b/server/store/datastore/migration/000_legacy_to_xormigrate.go
@@ -0,0 +1,74 @@
+package migration
+
+import (
+	"fmt"
+
+	"github.com/rs/zerolog/log"
+	"src.techknowlogick.com/xormigrate"
+	"xorm.io/xorm"
+
+	"go.woodpecker-ci.org/woodpecker/server/store/datastore/migration/legacy"
+)
+
+type v000Migrations struct {
+	Name string `xorm:"UNIQUE"`
+}
+
+func (m *v000Migrations) TableName() string {
+	return "migrations"
+}
+
+var legacyToXormigrate = xormigrate.Migration{
+	ID: "legacy-to-xormigrate",
+	Migrate: func(engine *xorm.Engine) error {
+		legacyMigrations := []string{
+			"xorm",
+			"alter-table-drop-repo-fallback",
+			"drop-allow-push-tags-deploys-columns",
+			"fix-pr-secret-event-name",
+			"alter-table-drop-counter",
+			"drop-senders",
+			"alter-table-logs-update-type-of-data",
+			"alter-table-add-secrets-user-id",
+			"recreate-agents-table",
+			"lowercase-secret-names",
+			"rename-builds-to-pipeline",
+			"rename-columns-builds-to-pipeline",
+			"rename-procs-to-steps",
+			"rename-remote-to-forge",
+			"rename-forge-id-to-forge-remote-id",
+			"remove-active-from-users",
+			"remove-inactive-repos",
+			"drop-files",
+			"remove-machine-col",
+			"parent-steps-to-workflows",
+			"drop-old-col",
+			"init-log_entries",
+			//"migrate-logs-to-log_entries", -> not required so we just skip it
+			"add-orgs",
+			"add-org-id",
+			"alter-table-tasks-update-type-of-task-data",
+			"alter-table-config-update-type-of-config-data",
+			"remove-plugin-only-option-from-secrets-table",
+			"convert-to-new-pipeline-error-format",
+		}
+
+		// TODO remove in 3.x and move to MigrateSession
+		if err := legacy.Migrate(engine); err != nil {
+			return err
+		}
+
+		for _, mig := range legacyMigrations {
+			exist, err := engine.Exist(&v000Migrations{mig})
+			if err != nil {
+				return fmt.Errorf("test migration existence: %w", err)
+			}
+			if !exist {
+				log.Error().Msgf("migration step '%s' missing, please upgrade to last stable v2.x version first", mig)
+				return fmt.Errorf("legacy migration step missing")
+			}
+		}
+
+		return engine.DropTables("migrations")
+	},
+}
diff --git a/...datastore/migration/000_legacy_to_xorm.go → ...re/migration/legacy/000_legacy_to_xorm.go b/...datastore/migration/000_legacy_to_xorm.go → ...re/migration/legacy/000_legacy_to_xorm.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package migration
+package legacy
 
 import (
 	"fmt"

diff --git a/...tore/migration/001_repos_drop_fallback.go → ...gration/legacy/001_repos_drop_fallback.go b/...tore/migration/001_repos_drop_fallback.go → ...gration/legacy/001_repos_drop_fallback.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package migration
+package legacy
 
 import (
 	"xorm.io/xorm"

diff --git a/...02_repos_drop_allow_deploys_allow_tags.go → ...02_repos_drop_allow_deploys_allow_tags.go b/...02_repos_drop_allow_deploys_allow_tags.go → ...02_repos_drop_allow_deploys_allow_tags.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package migration
+package legacy
 
 import (
 	"xorm.io/xorm"

diff --git a/...migration/003_fix_pr_secret_event_name.go → ...on/legacy/003_fix_pr_secret_event_name.go b/...migration/003_fix_pr_secret_event_name.go → ...on/legacy/003_fix_pr_secret_event_name.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package migration
+package legacy
 
 import (
 	"xorm.io/xorm"

diff --git a/.../migration/004_repos_drop_repo_counter.go → ...ion/legacy/004_repos_drop_repo_counter.go b/.../migration/004_repos_drop_repo_counter.go → ...ion/legacy/004_repos_drop_repo_counter.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package migration
+package legacy
 
 import (
 	"xorm.io/xorm"

diff --git a/...e/datastore/migration/005_drop_senders.go → ...tore/migration/legacy/005_drop_senders.go b/...e/datastore/migration/005_drop_senders.go → ...tore/migration/legacy/005_drop_senders.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package migration
+package legacy
 
 import (
 	"xorm.io/xorm"

diff --git a/.../datastore/migration/006_log_data_type.go → ...ore/migration/legacy/006_log_data_type.go b/.../datastore/migration/006_log_data_type.go → ...ore/migration/legacy/006_log_data_type.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package migration
+package legacy
 
 import (
 	"xorm.io/xorm"

diff --git a/...tastore/migration/007_secrets_add_user.go → .../migration/legacy/007_secrets_add_user.go b/...tastore/migration/007_secrets_add_user.go → .../migration/legacy/007_secrets_add_user.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package migration
+package legacy
 
 import (
 	"xorm.io/xorm"

diff --git a/...e/migration/008_lowercase_secret_names.go → ...tion/legacy/008_lowercase_secret_names.go b/...e/migration/008_lowercase_secret_names.go → ...tion/legacy/008_lowercase_secret_names.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package migration
+package legacy
 
 import (
 	"xorm.io/xorm"

diff --git a/...re/migration/008_recreate_agents_table.go → ...ation/legacy/008_recreate_agents_table.go b/...re/migration/008_recreate_agents_table.go → ...ation/legacy/008_recreate_agents_table.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package migration
+package legacy
 
 import (
 	"xorm.io/xorm"

diff --git a/...igration/009_rename_builds_to_pipeline.go → ...n/legacy/009_rename_builds_to_pipeline.go b/...igration/009_rename_builds_to_pipeline.go → ...n/legacy/009_rename_builds_to_pipeline.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package migration
+package legacy
 
 import (
 	"xorm.io/xorm"

diff --git a/.../010_columns_rename_builds_to_pipeline.go → .../010_columns_rename_builds_to_pipeline.go b/.../010_columns_rename_builds_to_pipeline.go → .../010_columns_rename_builds_to_pipeline.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package migration
+package legacy
 
 import (
 	"strings"

diff --git a/...tion/011_columns_rename_procs_to_steps.go → ...gacy/011_columns_rename_procs_to_steps.go b/...tion/011_columns_rename_procs_to_steps.go → ...gacy/011_columns_rename_procs_to_steps.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package migration
+package legacy
 
 import (
 	"strings"

diff --git a/...e/migration/012_rename_remote_to_forge.go → ...tion/legacy/012_rename_remote_to_forge.go b/...e/migration/012_rename_remote_to_forge.go → ...tion/legacy/012_rename_remote_to_forge.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package migration
+package legacy
 
 import (
 	"xorm.io/xorm"

diff --git a/...013_rename_forge_id_to_forge_remote_id.go → ...013_rename_forge_id_to_forge_remote_id.go b/...013_rename_forge_id_to_forge_remote_id.go → ...013_rename_forge_id_to_forge_remote_id.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package migration
+package legacy
 
 import (
 	"xorm.io/xorm"

diff --git a/...migration/014_remove_active_from_users.go → ...on/legacy/014_remove_active_from_users.go b/...migration/014_remove_active_from_users.go → ...on/legacy/014_remove_active_from_users.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package migration
+package legacy
 
 import (
 	"xorm.io/xorm"

diff --git a/...re/migration/015_remove_inactive_repos.go → ...ation/legacy/015_remove_inactive_repos.go b/...re/migration/015_remove_inactive_repos.go → ...ation/legacy/015_remove_inactive_repos.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package migration
+package legacy
 
 import (
 	"xorm.io/xorm"

diff --git a/...store/migration/016_remove_files_table.go → ...igration/legacy/016_remove_files_table.go b/...store/migration/016_remove_files_table.go → ...igration/legacy/016_remove_files_table.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package migration
+package legacy
 
 import (
 	"xorm.io/xorm"

diff --git a/...store/migration/017_remove_machine_col.go → ...igration/legacy/017_remove_machine_col.go b/...store/migration/017_remove_machine_col.go → ...igration/legacy/017_remove_machine_col.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package migration
+package legacy
 
 import (
 	"xorm.io/xorm"

diff --git a/.../datastore/migration/018_drop_old_cols.go → ...ore/migration/legacy/018_drop_old_cols.go b/.../datastore/migration/018_drop_old_cols.go → ...ore/migration/legacy/018_drop_old_cols.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package migration
+package legacy
 
 import (
 	"xorm.io/xorm"

diff --git a/...tastore/migration/019_alter_logs_table.go → .../migration/legacy/019_alter_logs_table.go b/...tastore/migration/019_alter_logs_table.go → .../migration/legacy/019_alter_logs_table.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package migration
+package legacy
 
 import (
 	"context"

diff --git a/...igration/020_parent_steps_to_workflows.go → ...n/legacy/020_parent_steps_to_workflows.go b/...igration/020_parent_steps_to_workflows.go → ...n/legacy/020_parent_steps_to_workflows.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package migration
+package legacy
 
 import (
 	"xorm.io/xorm"

diff --git a/...store/datastore/migration/021_add_orgs.go → ...atastore/migration/legacy/021_add_orgs.go b/...store/datastore/migration/021_add_orgs.go → ...atastore/migration/legacy/021_add_orgs.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package migration
+package legacy
 
 import (
 	"fmt"

diff --git a/...ore/datastore/migration/022_add_org_id.go → ...astore/migration/legacy/022_add_org_id.go b/...ore/datastore/migration/022_add_org_id.go → ...astore/migration/legacy/022_add_org_id.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package migration
+package legacy
 
 import (
 	"fmt"

diff --git a/...datastore/migration/023_task_data_type.go → ...re/migration/legacy/023_task_data_type.go b/...datastore/migration/023_task_data_type.go → ...re/migration/legacy/023_task_data_type.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package migration
+package legacy
 
 import (
 	"xorm.io/xorm"

diff --git a/...tastore/migration/024_config_data_type.go → .../migration/legacy/024_config_data_type.go b/...tastore/migration/024_config_data_type.go → .../migration/legacy/024_config_data_type.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package migration
+package legacy
 
 import (
 	"xorm.io/xorm"