From df6faa9bfca4387190ff7fec06154a28b6ddc7a8 Mon Sep 17 00:00:00 2001 From: Harshil Sharma Date: Wed, 29 Mar 2023 16:40:23 +0530 Subject: [PATCH] Cherrypicking 22606 to old focalboard --- .../store/sqlstore/data_migrations.go | 6 ++-- .../migrationstests/boards_migrator_test.go | 3 ++ ...uplicate_category_boards_migration_test.go | 28 +++++++++++++++++++ .../fixtures/test33_with_deleted_data.sql | 6 ++-- .../fixtures/test33_with_no_deleted_data.sql | 6 ++-- .../fixtures/test37_valid_data.sql | 6 ++-- .../test37_valid_data_no_hidden_boards.sql | 6 ++-- ...id_data_preference_but_no_hidden_board.sql | 6 ++-- .../fixtures/test37_valid_data_sqlite.sql | 6 ++-- ..._sqlite_preference_but_no_hidden_board.sql | 6 ++-- ...testDeDuplicateCategoryBoardsMigration.sql | 9 ++++++ 11 files changed, 70 insertions(+), 18 deletions(-) create mode 100644 server/services/store/sqlstore/migrationstests/de_duplicate_category_boards_migration_test.go create mode 100644 server/services/store/sqlstore/migrationstests/fixtures/testDeDuplicateCategoryBoardsMigration.sql diff --git a/server/services/store/sqlstore/data_migrations.go b/server/services/store/sqlstore/data_migrations.go index bbc7e221885..1794965cae9 100644 --- a/server/services/store/sqlstore/data_migrations.go +++ b/server/services/store/sqlstore/data_migrations.go @@ -868,10 +868,8 @@ func (s *SQLStore) doesDuplicateCategoryBoardsExist() (bool, error) { } func (s *SQLStore) runMySQLDeDuplicateCategoryBoardsMigration() error { - query := "WITH duplicates AS (SELECT id, ROW_NUMBER() OVER(PARTITION BY user_id, board_id) AS rownum " + - "FROM " + s.tablePrefix + "category_boards) " + - "DELETE " + s.tablePrefix + "category_boards FROM " + s.tablePrefix + "category_boards " + - "JOIN duplicates USING(id) WHERE duplicates.rownum > 1;" + query := "DELETE FROM " + s.tablePrefix + "category_boards WHERE id NOT IN " + + "(SELECT * FROM ( SELECT min(id) FROM " + s.tablePrefix + "category_boards GROUP BY user_id, board_id ) as data)" if _, err := s.db.Exec(query); err != nil { s.logger.Error("Failed to de-duplicate data in category_boards table", mlog.Err(err)) } diff --git a/server/services/store/sqlstore/migrationstests/boards_migrator_test.go b/server/services/store/sqlstore/migrationstests/boards_migrator_test.go index 9ec8ea3b23f..2808d96faec 100644 --- a/server/services/store/sqlstore/migrationstests/boards_migrator_test.go +++ b/server/services/store/sqlstore/migrationstests/boards_migrator_test.go @@ -246,6 +246,9 @@ func (bm *BoardsMigrator) MigrateToStep(step int) error { func (bm *BoardsMigrator) Interceptors() map[int]foundation.Interceptor { return map[int]foundation.Interceptor{ 18: bm.store.RunDeletedMembershipBoardsMigration, + 35: func() error { + return bm.store.RunDeDuplicateCategoryBoardsMigration(35) + }, } } diff --git a/server/services/store/sqlstore/migrationstests/de_duplicate_category_boards_migration_test.go b/server/services/store/sqlstore/migrationstests/de_duplicate_category_boards_migration_test.go new file mode 100644 index 00000000000..863cb43ffe3 --- /dev/null +++ b/server/services/store/sqlstore/migrationstests/de_duplicate_category_boards_migration_test.go @@ -0,0 +1,28 @@ +package migrationstests + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestRunDeDuplicateCategoryBoardsMigration(t *testing.T) { + th, tearDown := SetupTestHelper(t) + defer tearDown() + + if th.IsSQLite() { + t.Skip("SQLite is not supported for this") + } + + th.f.MigrateToStepSkippingLastInterceptor(35). + ExecFile("./fixtures/testDeDuplicateCategoryBoardsMigration.sql") + + th.f.RunInterceptor(35) + + // verifying count of rows + var count int + countQuery := "SELECT COUNT(*) FROM focalboard_category_boards" + row := th.f.DB().QueryRow(countQuery) + err := row.Scan(&count) + assert.NoError(t, err) + assert.Equal(t, 4, count) +} diff --git a/server/services/store/sqlstore/migrationstests/fixtures/test33_with_deleted_data.sql b/server/services/store/sqlstore/migrationstests/fixtures/test33_with_deleted_data.sql index c5593660bcb..f9e0b1a73e6 100644 --- a/server/services/store/sqlstore/migrationstests/fixtures/test33_with_deleted_data.sql +++ b/server/services/store/sqlstore/migrationstests/fixtures/test33_with_deleted_data.sql @@ -1,6 +1,8 @@ -INSERT INTO focalboard_category_boards values +INSERT INTO focalboard_category_boards +(id, user_id, category_id, board_id, create_at, update_at, delete_at, sort_order) +values ('id-1', 'user_id-1', 'category-id-1', 'board-id-1', 1672988834402, 1672988834402, 0, 0), ('id-2', 'user_id-1', 'category-id-2', 'board-id-1', 1672988834402, 1672988834402, 0, 0), ('id-3', 'user_id-2', 'category-id-3', 'board-id-2', 1672988834402, 1672988834402, 1672988834402, 0), ('id-4', 'user_id-2', 'category-id-3', 'board-id-4', 1672988834402, 1672988834402, 0, 0), -('id-5', 'user_id-3', 'category-id-4', 'board-id-3', 1672988834402, 1672988834402, 1672988834402, 0); \ No newline at end of file +('id-5', 'user_id-3', 'category-id-4', 'board-id-3', 1672988834402, 1672988834402, 1672988834402, 0); diff --git a/server/services/store/sqlstore/migrationstests/fixtures/test33_with_no_deleted_data.sql b/server/services/store/sqlstore/migrationstests/fixtures/test33_with_no_deleted_data.sql index 5f4337bb2cb..5bc32a714f7 100644 --- a/server/services/store/sqlstore/migrationstests/fixtures/test33_with_no_deleted_data.sql +++ b/server/services/store/sqlstore/migrationstests/fixtures/test33_with_no_deleted_data.sql @@ -1,6 +1,8 @@ -INSERT INTO focalboard_category_boards values +INSERT INTO focalboard_category_boards +(id, user_id, category_id, board_id, create_at, update_at, delete_at, sort_order) +values ('id-1', 'user_id-1', 'category-id-1', 'board-id-1', 1672988834402, 1672988834402, 0, 0), ('id-2', 'user_id-1', 'category-id-2', 'board-id-1', 1672988834402, 1672988834402, 0, 0), ('id-3', 'user_id-2', 'category-id-3', 'board-id-2', 1672988834402, 1672988834402, 0, 0), ('id-4', 'user_id-2', 'category-id-3', 'board-id-4', 1672988834402, 1672988834402, 0, 0), -('id-5', 'user_id-3', 'category-id-4', 'board-id-3', 1672988834402, 1672988834402, 0, 0); \ No newline at end of file +('id-5', 'user_id-3', 'category-id-4', 'board-id-3', 1672988834402, 1672988834402, 0, 0); diff --git a/server/services/store/sqlstore/migrationstests/fixtures/test37_valid_data.sql b/server/services/store/sqlstore/migrationstests/fixtures/test37_valid_data.sql index 8f75da3157c..552703ec62e 100644 --- a/server/services/store/sqlstore/migrationstests/fixtures/test37_valid_data.sql +++ b/server/services/store/sqlstore/migrationstests/fixtures/test37_valid_data.sql @@ -1,4 +1,6 @@ -INSERT INTO focalboard_category_boards VALUES +INSERT INTO focalboard_category_boards +(id, user_id, category_id, board_id, create_at, update_at, sort_order, hidden) +VALUES ('id-1', 'user-id-1', 'category-id-1', 'board-id-1', 1672889246832, 1672889246832, 0, false), ('id-2', 'user-id-1', 'category-id-2', 'board-id-2', 1672889246832, 1672889246832, 0, false), ('id-3', 'user-id-2', 'category-id-3', 'board-id-3', 1672889246832, 1672889246832, 0, false), @@ -7,4 +9,4 @@ INSERT INTO focalboard_category_boards VALUES INSERT INTO Preferences VALUES ('user-id-1', 'focalboard', 'hiddenBoardIDs', '["board-id-1"]'), -('user-id-2', 'focalboard', 'hiddenBoardIDs', '["board-id-3", "board-id-4"]'); \ No newline at end of file +('user-id-2', 'focalboard', 'hiddenBoardIDs', '["board-id-3", "board-id-4"]'); diff --git a/server/services/store/sqlstore/migrationstests/fixtures/test37_valid_data_no_hidden_boards.sql b/server/services/store/sqlstore/migrationstests/fixtures/test37_valid_data_no_hidden_boards.sql index 2986bad282c..3b6508f5c47 100644 --- a/server/services/store/sqlstore/migrationstests/fixtures/test37_valid_data_no_hidden_boards.sql +++ b/server/services/store/sqlstore/migrationstests/fixtures/test37_valid_data_no_hidden_boards.sql @@ -1,6 +1,8 @@ -INSERT INTO focalboard_category_boards VALUES +INSERT INTO focalboard_category_boards +(id, user_id, category_id, board_id, create_at, update_at, sort_order, hidden) +VALUES ('id-1', 'user-id-1', 'category-id-1', 'board-id-1', 1672889246832, 1672889246832, 0, false), ('id-2', 'user-id-1', 'category-id-2', 'board-id-2', 1672889246832, 1672889246832, 0, false), ('id-3', 'user-id-2', 'category-id-3', 'board-id-3', 1672889246832, 1672889246832, 0, false), ('id-4', 'user-id-2', 'category-id-3', 'board-id-4', 1672889246832, 1672889246832, 0, false), -('id-5', 'user-id-3', 'category-id-4', 'board-id-5', 1672889246832, 1672889246832, 0, false); \ No newline at end of file +('id-5', 'user-id-3', 'category-id-4', 'board-id-5', 1672889246832, 1672889246832, 0, false); diff --git a/server/services/store/sqlstore/migrationstests/fixtures/test37_valid_data_preference_but_no_hidden_board.sql b/server/services/store/sqlstore/migrationstests/fixtures/test37_valid_data_preference_but_no_hidden_board.sql index 943932e543c..75e12733296 100644 --- a/server/services/store/sqlstore/migrationstests/fixtures/test37_valid_data_preference_but_no_hidden_board.sql +++ b/server/services/store/sqlstore/migrationstests/fixtures/test37_valid_data_preference_but_no_hidden_board.sql @@ -1,4 +1,6 @@ -INSERT INTO focalboard_category_boards VALUES +INSERT INTO focalboard_category_boards +(id, user_id, category_id, board_id, create_at, update_at, sort_order, hidden) +VALUES ('id-1', 'user-id-1', 'category-id-1', 'board-id-1', 1672889246832, 1672889246832, 0, false), ('id-2', 'user-id-1', 'category-id-2', 'board-id-2', 1672889246832, 1672889246832, 0, false), ('id-3', 'user-id-2', 'category-id-3', 'board-id-3', 1672889246832, 1672889246832, 0, false), @@ -7,4 +9,4 @@ INSERT INTO focalboard_category_boards VALUES INSERT INTO Preferences VALUES ('user-id-1', 'focalboard', 'hiddenBoardIDs', ''), -('user-id-2', 'focalboard', 'hiddenBoardIDs', ''); \ No newline at end of file +('user-id-2', 'focalboard', 'hiddenBoardIDs', ''); diff --git a/server/services/store/sqlstore/migrationstests/fixtures/test37_valid_data_sqlite.sql b/server/services/store/sqlstore/migrationstests/fixtures/test37_valid_data_sqlite.sql index 114273c3098..deb74ccaf39 100644 --- a/server/services/store/sqlstore/migrationstests/fixtures/test37_valid_data_sqlite.sql +++ b/server/services/store/sqlstore/migrationstests/fixtures/test37_valid_data_sqlite.sql @@ -1,4 +1,6 @@ -INSERT INTO focalboard_category_boards VALUES +INSERT INTO focalboard_category_boards +(id, user_id, category_id, board_id, create_at, update_at, sort_order, hidden) +VALUES ('id-1', 'user-id-1', 'category-id-1', 'board-id-1', 1672889246832, 1672889246832, 0, false), ('id-2', 'user-id-1', 'category-id-2', 'board-id-2', 1672889246832, 1672889246832, 0, false), ('id-3', 'user-id-2', 'category-id-3', 'board-id-3', 1672889246832, 1672889246832, 0, false), @@ -7,4 +9,4 @@ INSERT INTO focalboard_category_boards VALUES INSERT INTO focalboard_preferences VALUES ('user-id-1', 'focalboard', 'hiddenBoardIDs', '["board-id-1"]'), -('user-id-2', 'focalboard', 'hiddenBoardIDs', '["board-id-3", "board-id-4"]'); \ No newline at end of file +('user-id-2', 'focalboard', 'hiddenBoardIDs', '["board-id-3", "board-id-4"]'); diff --git a/server/services/store/sqlstore/migrationstests/fixtures/test37_valid_data_sqlite_preference_but_no_hidden_board.sql b/server/services/store/sqlstore/migrationstests/fixtures/test37_valid_data_sqlite_preference_but_no_hidden_board.sql index 4b45cacdf63..62dc2b708e6 100644 --- a/server/services/store/sqlstore/migrationstests/fixtures/test37_valid_data_sqlite_preference_but_no_hidden_board.sql +++ b/server/services/store/sqlstore/migrationstests/fixtures/test37_valid_data_sqlite_preference_but_no_hidden_board.sql @@ -1,4 +1,6 @@ -INSERT INTO focalboard_category_boards VALUES +INSERT INTO focalboard_category_boards +(id, user_id, category_id, board_id, create_at, update_at, sort_order, hidden) +VALUES ('id-1', 'user-id-1', 'category-id-1', 'board-id-1', 1672889246832, 1672889246832, 0, false), ('id-2', 'user-id-1', 'category-id-2', 'board-id-2', 1672889246832, 1672889246832, 0, false), ('id-3', 'user-id-2', 'category-id-3', 'board-id-3', 1672889246832, 1672889246832, 0, false), @@ -7,4 +9,4 @@ INSERT INTO focalboard_category_boards VALUES INSERT INTO focalboard_preferences VALUES ('user-id-1', 'focalboard', 'hiddenBoardIDs', ''), -('user-id-2', 'focalboard', 'hiddenBoardIDs', ''); \ No newline at end of file +('user-id-2', 'focalboard', 'hiddenBoardIDs', ''); diff --git a/server/services/store/sqlstore/migrationstests/fixtures/testDeDuplicateCategoryBoardsMigration.sql b/server/services/store/sqlstore/migrationstests/fixtures/testDeDuplicateCategoryBoardsMigration.sql new file mode 100644 index 00000000000..69a7dc9bdea --- /dev/null +++ b/server/services/store/sqlstore/migrationstests/fixtures/testDeDuplicateCategoryBoardsMigration.sql @@ -0,0 +1,9 @@ +INSERT INTO focalboard_category_boards(id, user_id, category_id, board_id, create_at, update_at, sort_order) +VALUES + ('id_1', 'user_id_1', 'category_id_1', 'board_id_1', 0, 0, 0), + ('id_2', 'user_id_1', 'category_id_2', 'board_id_1', 0, 0, 0), + ('id_3', 'user_id_1', 'category_id_3', 'board_id_1', 0, 0, 0), + ('id_4', 'user_id_2', 'category_id_4', 'board_id_2', 0, 0, 0), + ('id_5', 'user_id_2', 'category_id_5', 'board_id_2', 0, 0, 0), + ('id_6', 'user_id_3', 'category_id_6', 'board_id_3', 0, 0, 0), + ('id_7', 'user_id_4', 'category_id_6', 'board_id_4', 0, 0, 0);