From 77f7bc4ad44ef194061f30ca4c09caecc2cb4805 Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Tue, 22 Nov 2022 15:01:21 -0800 Subject: [PATCH 1/6] Migrate stats columns from show to show_stats Frequently changing columns on the SHOW table were causing the show indexes to bloat faster than desired and impacting the system performance, specially in the Whiteboard queries for jobs. --- .../spcue/dao/postgres/JobDaoJdbc.java | 2 +- .../spcue/dao/postgres/ShowDaoJdbc.java | 2 +- .../spcue/dao/postgres/WhiteboardDaoJdbc.java | 15 ++++++++- .../postgres/migrations/V21__AddShowStats.sql | 33 +++++++++++++++++++ .../resources/conf/ddl/postgres/seed_data.sql | 4 ++- .../spcue/test/dao/postgres/ShowDaoTests.java | 8 ++--- .../resources/conf/ddl/postgres/test_data.sql | 8 +++-- 7 files changed, 62 insertions(+), 10 deletions(-) create mode 100644 cuebot/src/main/resources/conf/ddl/postgres/migrations/V21__AddShowStats.sql diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java index 890f5ea81..b2a26ebad 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java @@ -574,7 +574,7 @@ public void activateJob(JobInterface job, JobState jobState) { jobTotals[0] + jobTotals[1], layers.size(), job.getJobId()); getJdbcTemplate().update( - "UPDATE show SET int_frame_insert_count=int_frame_insert_count+?, int_job_insert_count=int_job_insert_count+1 WHERE pk_show=?", + "UPDATE show_stats SET int_frame_insert_count=int_frame_insert_count+?, int_job_insert_count=int_job_insert_count+1 WHERE pk_show=?", jobTotals[0] + jobTotals[1], job.getShowId()); updateState(job, jobState); diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java index 5d674ed82..e3d114243 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java @@ -262,7 +262,7 @@ public void updateFrameCounters(ShowInterface s, int exitStatus) { col = "int_frame_fail_count = int_frame_fail_count + 1"; } getJdbcTemplate().update( - "UPDATE show SET " + col + " WHERE pk_show=?", s.getShowId()); + "UPDATE show_stats SET " + col + " WHERE pk_show=?", s.getShowId()); } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java index 9ffe56beb..d95d51b4a 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java @@ -2059,7 +2059,19 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { private static final String GET_SHOW = "SELECT " + - "show.*," + + "show.pk_show," + + "show.str_name," + + "show.b_paused," + + "show.int_default_min_cores," + + "show.int_default_max_cores," + + "show.b_booking_enabled," + + "show.b_dispatch_enabled," + + "show.b_active," + + "show.str_comment_email," + + "show_stats.int_frame_insert_count," + + "show_stats.int_job_insert_count," + + "show_stats.int_frame_success_count," + + "show_stats.int_frame_fail_count," + "COALESCE(vs_show_stat.int_pending_count,0) AS int_pending_count," + "COALESCE(vs_show_stat.int_running_count,0) AS int_running_count," + "COALESCE(vs_show_stat.int_dead_count,0) AS int_dead_count," + @@ -2068,6 +2080,7 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "COALESCE(vs_show_stat.int_job_count,0) AS int_job_count " + "FROM " + "show " + + "JOIN show_stats ON (show.pk_show = show_stats.pk_show) " + "LEFT JOIN vs_show_stat ON (vs_show_stat.pk_show = show.pk_show) " + "LEFT JOIN vs_show_resource ON (vs_show_resource.pk_show=show.pk_show) " + "WHERE " + diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V21__AddShowStats.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V21__AddShowStats.sql new file mode 100644 index 000000000..4f7c86a7c --- /dev/null +++ b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V21__AddShowStats.sql @@ -0,0 +1,33 @@ +CREATE TABLE show_stats ( + pk_show VARCHAR(36) NOT NULL, + int_frame_insert_count BIGINT DEFAULT 0 NOT NULL, + int_job_insert_count BIGINT DEFAULT 0 NOT NULL, + int_frame_success_count BIGINT DEFAULT 0 NOT NULL, + int_frame_fail_count BIGINT DEFAULT 0 NOT NULL +); + +INSERT INTO show_stats ( + pk_show, + int_frame_insert_count, + int_job_insert_count, + int_frame_success_count, + int_frame_fail_count +) SELECT + pk_show, + int_frame_insert_count, + int_job_insert_count, + int_frame_success_count, + int_frame_fail_count + FROM show; + +CREATE UNIQUE INDEX c_show_stats_pk ON show_stats (pk_show); +ALTER TABLE show_stats ADD CONSTRAINT c_show_stats_pk PRIMARY KEY + USING INDEX c_show_stats_pk; + + +-- Destructive changes. Please test changes above prior to executing this. +ALTER TABLE show + DROP COLUMN int_frame_insert_count, + DROP COLUMN int_job_insert_count, + DROP COLUMN int_frame_success_count, + DROP COLUMN int_frame_fail_count; \ No newline at end of file diff --git a/cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql b/cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql index ebebaa75d..f5707b62c 100644 --- a/cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql +++ b/cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql @@ -1,4 +1,6 @@ -Insert into SHOW (PK_SHOW,STR_NAME,INT_DEFAULT_MAX_CORES,INT_DEFAULT_MIN_CORES,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT,B_BOOKING_ENABLED,B_DISPATCH_ENABLED,B_ACTIVE) values ('00000000-0000-0000-0000-000000000000', 'testing', 200000, 100, 0, 0, 0, 0, true, true, true); +Insert into SHOW (PK_SHOW,STR_NAME,INT_DEFAULT_MAX_CORES,INT_DEFAULT_MIN_CORES,B_BOOKING_ENABLED,B_DISPATCH_ENABLED,B_ACTIVE) values ('00000000-0000-0000-0000-000000000000', 'testing', 200000, 100, true, true, true); + +Insert into SHOW_STATS (PK_SHOW,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT) values ('00000000-0000-0000-0000-000000000000',0,0,0,0) Insert into SHOW_ALIAS (PK_SHOW_ALIAS,PK_SHOW,STR_NAME) values ('00000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000000', 'test'); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ShowDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ShowDaoTests.java index d430ab3b0..e44393ab6 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ShowDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ShowDaoTests.java @@ -219,20 +219,20 @@ public void testUpdateActive() { public void testUpdateFrameCounters() { ShowEntity show = showDao.findShowDetail(SHOW_NAME); int frameSuccess = jdbcTemplate.queryForObject( - "SELECT int_frame_success_count FROM show WHERE pk_show=?", + "SELECT int_frame_success_count FROM show_stats WHERE pk_show=?", Integer.class, show.id); showDao.updateFrameCounters(show, 0); int frameSucces2 = jdbcTemplate.queryForObject( - "SELECT int_frame_success_count FROM show WHERE pk_show=?", + "SELECT int_frame_success_count FROM show_stats WHERE pk_show=?", Integer.class, show.id); assertEquals(frameSuccess + 1,frameSucces2); int frameFail= jdbcTemplate.queryForObject( - "SELECT int_frame_fail_count FROM show WHERE pk_show=?", + "SELECT int_frame_fail_count FROM show_stats WHERE pk_show=?", Integer.class, show.id); showDao.updateFrameCounters(show, 1); int frameFail2 = jdbcTemplate.queryForObject( - "SELECT int_frame_fail_count FROM show WHERE pk_show=?", + "SELECT int_frame_fail_count FROM show_stats WHERE pk_show=?", Integer.class, show.id); assertEquals(frameFail+ 1,frameFail2); } diff --git a/cuebot/src/test/resources/conf/ddl/postgres/test_data.sql b/cuebot/src/test/resources/conf/ddl/postgres/test_data.sql index ae7b77354..4f9c2a0a0 100644 --- a/cuebot/src/test/resources/conf/ddl/postgres/test_data.sql +++ b/cuebot/src/test/resources/conf/ddl/postgres/test_data.sql @@ -1,6 +1,10 @@ -Insert into SHOW (PK_SHOW,STR_NAME,INT_DEFAULT_MAX_CORES,INT_DEFAULT_MIN_CORES,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT,B_BOOKING_ENABLED,B_DISPATCH_ENABLED,B_ACTIVE) values ('00000000-0000-0000-0000-000000000000','pipe',20000,100,0,0,0,0,true,true,true) +Insert into SHOW (PK_SHOW,STR_NAME,INT_DEFAULT_MAX_CORES,INT_DEFAULT_MIN_CORES,B_BOOKING_ENABLED,B_DISPATCH_ENABLED,B_ACTIVE) values ('00000000-0000-0000-0000-000000000000','pipe',20000,100,true,true,true) -Insert into SHOW (PK_SHOW,STR_NAME,INT_DEFAULT_MAX_CORES,INT_DEFAULT_MIN_CORES,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT,B_BOOKING_ENABLED,B_DISPATCH_ENABLED,B_ACTIVE) values ('00000000-0000-0000-0000-000000000001','edu',20000,100,0,0,0,0,true,true,true) +Insert into SHOW (PK_SHOW,STR_NAME,INT_DEFAULT_MAX_CORES,INT_DEFAULT_MIN_CORES,B_BOOKING_ENABLED,B_DISPATCH_ENABLED,B_ACTIVE) values ('00000000-0000-0000-0000-000000000001','edu',20000,100,true,true,true) + +Insert into SHOW_STATS (PK_SHOW,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT) values ('00000000-0000-0000-0000-000000000000',0,0,0,0) + +Insert into SHOW_STATS (PK_SHOW,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT) values ('00000000-0000-0000-0000-000000000001',0,0,0,0) Insert into SHOW_ALIAS (PK_SHOW_ALIAS,PK_SHOW,STR_NAME) values ('00000000-0000-0000-0000-000000000001','00000000-0000-0000-0000-000000000000','fx') From b294b2ffe010a3a82b47042d7e0fbac22639d5f2 Mon Sep 17 00:00:00 2001 From: dtavares Date: Fri, 20 Jan 2023 10:41:45 -0800 Subject: [PATCH 2/6] Fix merge issues Add gpu columns --- .../com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java index d95d51b4a..0937f30d4 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java @@ -2064,6 +2064,8 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "show.b_paused," + "show.int_default_min_cores," + "show.int_default_max_cores," + + "show.int_default_min_gpus," + + "show.int_default_max_gpus," + "show.b_booking_enabled," + "show.b_dispatch_enabled," + "show.b_active," + From e668e78fc95f2408076e8a9504db7562bd1da493 Mon Sep 17 00:00:00 2001 From: dtavares Date: Fri, 20 Jan 2023 10:49:59 -0800 Subject: [PATCH 3/6] Version bump --- VERSION.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.in b/VERSION.in index 5320adc1c..e34629406 100644 --- a/VERSION.in +++ b/VERSION.in @@ -1 +1 @@ -0.21 +0.22 From c0215877b8ea8475c157df953624217ef0074464 Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Thu, 26 Jan 2023 14:07:52 -0800 Subject: [PATCH 4/6] Update cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java Signed-off-by: Diego Tavares da Silva --- .../com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java index 0937f30d4..09adfa546 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java @@ -2082,7 +2082,7 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "COALESCE(vs_show_stat.int_job_count,0) AS int_job_count " + "FROM " + "show " + - "JOIN show_stats ON (show.pk_show = show_stats.pk_show) " + + "LEFT JOIN show_stats ON (show.pk_show = show_stats.pk_show) " + "LEFT JOIN vs_show_stat ON (vs_show_stat.pk_show = show.pk_show) " + "LEFT JOIN vs_show_resource ON (vs_show_resource.pk_show=show.pk_show) " + "WHERE " + From 8090c49adad35a603a5d1e6a3e32ba1662e5af34 Mon Sep 17 00:00:00 2001 From: dtavares Date: Thu, 26 Jan 2023 14:28:40 -0800 Subject: [PATCH 5/6] Creating a new show should also create a show_stats entry A show_stats row needs to be created and deleted together with a show row. This bug was introduced on https://gitlab.spimageworks.com/spi/dev/infrastructure/api/opencue/-/merge_requests/427 --- .../com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java | 8 ++++++++ .../imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java index e3d114243..548520a6f 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java @@ -138,9 +138,15 @@ public ShowEntity getShowDetail(HostInterface host) { private static final String INSERT_SHOW = "INSERT INTO show (pk_show,str_name) VALUES (?,?)"; + private static final String INSERT_SHOW_STATS = + "INSERT INTO show_stats " + + "(pk_show, int_frame_insert_count, int_job_insert_count, int_frame_success_count, int_frame_fail_count) " + + "VALUES (?, 0, 0, 0, 0)"; + public void insertShow(ShowEntity show) { show.id = SqlUtil.genKeyRandom(); getJdbcTemplate().update(INSERT_SHOW, show.id, show.name); + getJdbcTemplate().update(INSERT_SHOW_STATS, show.id); } private static final String SHOW_EXISTS = @@ -169,6 +175,8 @@ public void delete(ShowInterface s) { s.getShowId()); getJdbcTemplate().update("DELETE FROM show_alias WHERE pk_show=?", s.getShowId()); + getJdbcTemplate().update("DELETE FROM show_stats WHERE pk_show=?", + s.getShowId()); getJdbcTemplate().update("DELETE FROM show WHERE pk_show=?", s.getShowId()); } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java index 09adfa546..0937f30d4 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java @@ -2082,7 +2082,7 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "COALESCE(vs_show_stat.int_job_count,0) AS int_job_count " + "FROM " + "show " + - "LEFT JOIN show_stats ON (show.pk_show = show_stats.pk_show) " + + "JOIN show_stats ON (show.pk_show = show_stats.pk_show) " + "LEFT JOIN vs_show_stat ON (vs_show_stat.pk_show = show.pk_show) " + "LEFT JOIN vs_show_resource ON (vs_show_resource.pk_show=show.pk_show) " + "WHERE " + From 89e8a37c852925a506d9166db10d6b0b4196192e Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Wed, 1 Mar 2023 14:39:54 -0800 Subject: [PATCH 6/6] Rename V21__AddShowStats.sql to V16__AddShowStats.sql --- .../migrations/{V21__AddShowStats.sql => V16__AddShowStats.sql} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename cuebot/src/main/resources/conf/ddl/postgres/migrations/{V21__AddShowStats.sql => V16__AddShowStats.sql} (96%) diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V21__AddShowStats.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V16__AddShowStats.sql similarity index 96% rename from cuebot/src/main/resources/conf/ddl/postgres/migrations/V21__AddShowStats.sql rename to cuebot/src/main/resources/conf/ddl/postgres/migrations/V16__AddShowStats.sql index 4f7c86a7c..f97c14f7c 100644 --- a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V21__AddShowStats.sql +++ b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V16__AddShowStats.sql @@ -30,4 +30,4 @@ ALTER TABLE show DROP COLUMN int_frame_insert_count, DROP COLUMN int_job_insert_count, DROP COLUMN int_frame_success_count, - DROP COLUMN int_frame_fail_count; \ No newline at end of file + DROP COLUMN int_frame_fail_count;