Merge branch 'master' into DHIS2-18417

dhis2 · Nov 19, 2024 · 049dc43 · 049dc43
2 parents 10a0bd2 + 0a65033
commit 049dc43
Show file tree

Hide file tree

Showing 17 changed files with 117 additions and 40 deletions.
diff --git a/...rvice-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java b/...rvice-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java
@@ -358,14 +358,15 @@ protected String getTableName() {
   protected AnalyticsTable getRegularAnalyticsTable(
       AnalyticsTableUpdateParams params,
       List<Integer> dataYears,
-      List<AnalyticsTableColumn> columns) {
+      List<AnalyticsTableColumn> columns,
+      List<String> sortKey) {
     Calendar calendar = PeriodType.getCalendar();
     List<Integer> years = ListUtils.mutableCopy(dataYears);
     Logged logged = analyticsTableSettings.getTableLogged();
 
     Collections.sort(years);
 
-    AnalyticsTable table = new AnalyticsTable(getAnalyticsTableType(), columns, logged);
+    AnalyticsTable table = new AnalyticsTable(getAnalyticsTableType(), columns, sortKey, logged);
 
     for (Integer year : years) {
       List<String> checks = getPartitionChecks(year, getEndDate(calendar, year));
@@ -400,7 +401,7 @@ protected AnalyticsTable getLatestAnalyticsTable(
     Date endDate = params.getStartTime();
     boolean hasUpdatedData = hasUpdatedLatestData(lastAnyTableUpdate, endDate);
 
-    AnalyticsTable table = new AnalyticsTable(getAnalyticsTableType(), columns, logged);
+    AnalyticsTable table = new AnalyticsTable(getAnalyticsTableType(), columns, List.of(), logged);
 
     if (hasUpdatedData) {
       table.addTablePartition(

diff --git a/...vice-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java b/...vice-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java
@@ -161,6 +161,8 @@ public class JdbcAnalyticsTableManager extends AbstractJdbcTableManager {
               .selectExpression("ous.level as oulevel")
               .build());
 
+  private static final List<String> SORT_KEY = List.of("dx");
+
   public JdbcAnalyticsTableManager(
       IdentifiableObjectManager idObjectManager,
       OrganisationUnitService organisationUnitService,
@@ -206,7 +208,7 @@ public List<AnalyticsTable> getAnalyticsTables(AnalyticsTableUpdateParams params
     AnalyticsTable table =
         params.isLatestUpdate()
             ? getLatestAnalyticsTable(params, getColumns(params))
-            : getRegularAnalyticsTable(params, getDataYears(params), getColumns(params));
+            : getRegularAnalyticsTable(params, getDataYears(params), getColumns(params), SORT_KEY);
 
     return table.hasTablePartitions() ? List.of(table) : List.of();
   }

diff --git a/...e-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java b/...e-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java
@@ -91,6 +91,8 @@ public class JdbcCompletenessTableManager extends AbstractJdbcTableManager {
               .selectExpression("ps.year")
               .build());
 
+  private static final List<String> SORT_KEY = List.of("dx");
+
   public JdbcCompletenessTableManager(
       IdentifiableObjectManager idObjectManager,
       OrganisationUnitService organisationUnitService,
@@ -132,7 +134,7 @@ public List<AnalyticsTable> getAnalyticsTables(AnalyticsTableUpdateParams params
     AnalyticsTable table =
         params.isLatestUpdate()
             ? getLatestAnalyticsTable(params, getColumns())
-            : getRegularAnalyticsTable(params, getDataYears(params), getColumns());
+            : getRegularAnalyticsTable(params, getDataYears(params), getColumns(), SORT_KEY);
 
     return table.hasTablePartitions() ? List.of(table) : List.of();
   }

diff --git a/...ytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java b/...ytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java
@@ -102,6 +102,8 @@ public class JdbcCompletenessTargetTableManager extends AbstractJdbcTableManager
               .selectExpression("doc.coenddate")
               .build());
 
+  private static final List<String> SORT_KEY = List.of("dx");
+
   public JdbcCompletenessTargetTableManager(
       IdentifiableObjectManager idObjectManager,
       OrganisationUnitService organisationUnitService,
@@ -143,7 +145,7 @@ public List<AnalyticsTable> getAnalyticsTables(AnalyticsTableUpdateParams params
     Logged logged = analyticsTableSettings.getTableLogged();
     return params.isLatestUpdate()
         ? List.of()
-        : List.of(new AnalyticsTable(getAnalyticsTableType(), getColumns(), logged));
+        : List.of(new AnalyticsTable(getAnalyticsTableType(), getColumns(), SORT_KEY, logged));
   }
 
   @Override

diff --git a/...-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java b/...-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java
@@ -77,6 +77,8 @@ public class JdbcOrgUnitTargetTableManager extends AbstractJdbcTableManager {
               .selectExpression("oug.uid")
               .build());
 
+  private static final List<String> SORT_KEY = List.of("oug");
+
   public JdbcOrgUnitTargetTableManager(
       IdentifiableObjectManager idObjectManager,
       OrganisationUnitService organisationUnitService,
@@ -118,7 +120,7 @@ public List<AnalyticsTable> getAnalyticsTables(AnalyticsTableUpdateParams params
     Logged logged = analyticsTableSettings.getTableLogged();
     return params.isLatestUpdate()
         ? List.of()
-        : List.of(new AnalyticsTable(getAnalyticsTableType(), getColumns(), logged));
+        : List.of(new AnalyticsTable(getAnalyticsTableType(), getColumns(), SORT_KEY, logged));
   }
 
   @Override

diff --git a/...alytics/src/main/java/org/hisp/dhis/analytics/table/JdbcValidationResultTableManager.java b/...alytics/src/main/java/org/hisp/dhis/analytics/table/JdbcValidationResultTableManager.java
@@ -99,6 +99,8 @@ public class JdbcValidationResultTableManager extends AbstractJdbcTableManager {
               .selectExpression("ps.year")
               .build());
 
+  private static final List<String> SORT_KEY = List.of("dx");
+
   public JdbcValidationResultTableManager(
       IdentifiableObjectManager idObjectManager,
       OrganisationUnitService organisationUnitService,
@@ -138,8 +140,9 @@ public AnalyticsTableType getAnalyticsTableType() {
   public List<AnalyticsTable> getAnalyticsTables(AnalyticsTableUpdateParams params) {
     AnalyticsTable table =
         params.isLatestUpdate()
-            ? new AnalyticsTable(AnalyticsTableType.VALIDATION_RESULT, List.of(), Logged.LOGGED)
-            : getRegularAnalyticsTable(params, getDataYears(params), getColumns());
+            ? new AnalyticsTable(
+                AnalyticsTableType.VALIDATION_RESULT, List.of(), List.of(), Logged.LOGGED)
+            : getRegularAnalyticsTable(params, getDataYears(params), getColumns(), SORT_KEY);
 
     return table.hasTablePartitions() ? List.of(table) : List.of();
   }

diff --git a/...s-service-analytics/src/main/java/org/hisp/dhis/analytics/table/model/AnalyticsTable.java b/...s-service-analytics/src/main/java/org/hisp/dhis/analytics/table/model/AnalyticsTable.java
@@ -74,11 +74,21 @@ public class AnalyticsTable extends Table {
    *
    * @param tableType the {@link AnalyticsTableType}.
    * @param columns the list of {@link Column}.
+   * @param sortKey the sort key.
    * @param logged the {@link Logged} property.
    */
   public AnalyticsTable(
-      AnalyticsTableType tableType, List<AnalyticsTableColumn> columns, Logged logged) {
-    super(toStaging(tableType.getTableName()), toColumns(columns), List.of(), logged);
+      AnalyticsTableType tableType,
+      List<AnalyticsTableColumn> columns,
+      List<String> sortKey,
+      Logged logged) {
+    super(
+        toStaging(tableType.getTableName()),
+        toColumns(columns),
+        List.of(),
+        sortKey,
+        List.of(),
+        logged);
     this.tableType = tableType;
     this.analyticsTableColumns = columns;
   }

diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/model/Table.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/model/Table.java
@@ -61,10 +61,13 @@ public class Table {
   /** Table primary key column name(s). Optional. */
   private final List<String> primaryKey;
 
-  /** Table checks. PostgreSQL-only feature. Optional. */
+  /** Table sort key. Applies to column-oriented databases including Doris and ClickHouse only. */
+  private final List<String> sortKey;
+
+  /** Table checks. Applies to PostgreSQL only. Optional. */
   private final List<String> checks;
 
-  /** Whether table is logged or unlogged. PostgreSQL-only feature. */
+  /** Whether table is logged or unlogged. Applies to PostgreSQL only. */
   private final Logged logged;
 
   /** Parent table. This table inherits from the parent if specified. Optional. */
@@ -84,6 +87,7 @@ public Table(String name, List<Column> columns, List<String> primaryKey) {
     this.name = name;
     this.columns = columns;
     this.primaryKey = primaryKey;
+    this.sortKey = List.of();
     this.checks = List.of();
     this.logged = Logged.UNLOGGED;
     this.parent = null;
@@ -102,6 +106,7 @@ public Table(String name, List<Column> columns, List<String> primaryKey, Logged
     this.name = name;
     this.columns = columns;
     this.primaryKey = primaryKey;
+    this.sortKey = List.of();
     this.checks = List.of();
     this.logged = logged;
     this.parent = null;
@@ -114,18 +119,21 @@ public Table(String name, List<Column> columns, List<String> primaryKey, Logged
    * @param name the table name.
    * @param columns the list of {@link Column}.
    * @param primaryKey the primary key.
+   * @param sortKey the sort key.
    * @param checks the list of checks.
    * @param logged the {@link Logged} parameter.
    */
   public Table(
       String name,
       List<Column> columns,
       List<String> primaryKey,
+      List<String> sortKey,
       List<String> checks,
       Logged logged) {
     this.name = name;
     this.columns = columns;
     this.primaryKey = primaryKey;
+    this.sortKey = sortKey;
     this.checks = checks;
     this.logged = logged;
     this.parent = null;
@@ -152,6 +160,7 @@ public Table(
     this.name = name;
     this.columns = columns;
     this.primaryKey = primaryKey;
+    this.sortKey = List.of();
     this.checks = checks;
     this.logged = logged;
     this.parent = parent;
@@ -200,6 +209,15 @@ public String getFirstPrimaryKey() {
     return hasPrimaryKey() ? primaryKey.get(0) : null;
   }
 
+  /**
+   * Indicates whether the table has a sort key.
+   *
+   * @return true if the table has a sort key.
+   */
+  public boolean hasSortKey() {
+    return isNotEmpty(sortKey);
+  }
+
   /**
    * Indicates whether the table has at least one check.
    *

diff --git a/...vice-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java b/...vice-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java
@@ -81,7 +81,10 @@ void testGetTablePartitions() {
                 .selectExpression("value")
                 .build());
 
-    AnalyticsTable tA = new AnalyticsTable(AnalyticsTableType.DATA_VALUE, columns, Logged.UNLOGGED);
+    List<String> sortKey = List.of("dx");
+
+    AnalyticsTable tA =
+        new AnalyticsTable(AnalyticsTableType.DATA_VALUE, columns, sortKey, Logged.UNLOGGED);
     tA.addTablePartition(
         List.of(),
         2010,
@@ -93,7 +96,7 @@ void testGetTablePartitions() {
         new DateTime(2011, 1, 1, 0, 0).toDate(),
         new DateTime(2011, 12, 31, 0, 0).toDate());
     AnalyticsTable tB =
-        new AnalyticsTable(AnalyticsTableType.ORG_UNIT_TARGET, columns, Logged.UNLOGGED);
+        new AnalyticsTable(AnalyticsTableType.ORG_UNIT_TARGET, columns, sortKey, Logged.UNLOGGED);
     List<AnalyticsTablePartition> partitions = tableService.getTablePartitions(List.of(tA, tB));
 
     assertEquals(3, partitions.size());

diff --git a/...-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerTest.java b/...-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerTest.java
@@ -300,16 +300,16 @@ void testGetLatestAnalyticsTableNoFullTableUpdate() {
       "Verify if the method swapParentTable is called with the swapped table name not the staging table name")
   void testSwapTable() {
     Date startTime = new DateTime(2019, 3, 1, 10, 0).toDate();
+    List<AnalyticsTableColumn> columns =
+        List.of(
+            AnalyticsTableColumn.builder()
+                .name("year")
+                .dataType(INTEGER)
+                .selectExpression("")
+                .build());
+    List<String> sortKey = List.of("dx");
     AnalyticsTable table =
-        new AnalyticsTable(
-            AnalyticsTableType.DATA_VALUE,
-            List.of(
-                AnalyticsTableColumn.builder()
-                    .name("year")
-                    .dataType(INTEGER)
-                    .selectExpression("")
-                    .build()),
-            LOGGED);
+        new AnalyticsTable(AnalyticsTableType.DATA_VALUE, columns, sortKey, LOGGED);
     table.addTablePartition(List.of(), 2023, new DateTime(2023, 1, 1, 0, 0).toDate(), null);
     AnalyticsTableUpdateParams params =
         AnalyticsTableUpdateParams.newBuilder().startTime(startTime).build().withLatestPartition();

diff --git a/...lytics/src/test/java/org/hisp/dhis/analytics/table/model/AnalyticsTablePartitionTest.java b/...lytics/src/test/java/org/hisp/dhis/analytics/table/model/AnalyticsTablePartitionTest.java
@@ -59,10 +59,12 @@ class AnalyticsTablePartitionTest {
               .selectExpression("value")
               .build());
 
+  private final List<String> sortKeyA = List.of("data");
+
   @Test
   void testGetName() {
     AnalyticsTable table =
-        new AnalyticsTable(AnalyticsTableType.DATA_VALUE, columnsA, Logged.UNLOGGED);
+        new AnalyticsTable(AnalyticsTableType.DATA_VALUE, columnsA, sortKeyA, Logged.UNLOGGED);
 
     List<String> checks = List.of("value = 2023");
 
@@ -83,7 +85,7 @@ void testGetName() {
   @Test
   void testIsLatestPartition() {
     AnalyticsTable table =
-        new AnalyticsTable(AnalyticsTableType.COMPLETENESS, columnsA, Logged.UNLOGGED);
+        new AnalyticsTable(AnalyticsTableType.COMPLETENESS, columnsA, sortKeyA, Logged.UNLOGGED);
 
     AnalyticsTablePartition partition =
         new AnalyticsTablePartition(

diff --git a/...rvice-analytics/src/test/java/org/hisp/dhis/analytics/table/model/AnalyticsTableTest.java b/...rvice-analytics/src/test/java/org/hisp/dhis/analytics/table/model/AnalyticsTableTest.java
@@ -37,6 +37,7 @@
 import static org.hisp.dhis.db.model.constraint.Nullable.NULL;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.util.List;
 import org.hisp.dhis.analytics.AnalyticsTableType;
@@ -67,26 +68,41 @@ class AnalyticsTableTest {
               .selectExpression("value")
               .build());
 
+  private final List<String> sortKeyA = List.of("data");
+
+  @Test
+  void testConstructor() {
+    AnalyticsTable table =
+        new AnalyticsTable(AnalyticsTableType.DATA_VALUE, columnsA, sortKeyA, Logged.UNLOGGED);
+
+    assertEquals(AnalyticsTableType.DATA_VALUE, table.getTableType());
+    assertTrue(table.getPrimaryKey().isEmpty());
+    assertEquals(sortKeyA, table.getSortKey());
+    assertTrue(table.getChecks().isEmpty());
+    assertEquals(Logged.UNLOGGED, table.getLogged());
+  }
+
   @Test
   void testGetTableNameDataValue() {
     AnalyticsTable table =
-        new AnalyticsTable(AnalyticsTableType.DATA_VALUE, columnsA, Logged.UNLOGGED);
+        new AnalyticsTable(AnalyticsTableType.DATA_VALUE, columnsA, sortKeyA, Logged.UNLOGGED);
     assertEquals("analytics", table.getMainName());
     assertEquals("analytics_temp", table.getName());
   }
 
   @Test
   void testGetTableNameCompleteness() {
     AnalyticsTable table =
-        new AnalyticsTable(AnalyticsTableType.COMPLETENESS, columnsA, Logged.UNLOGGED);
+        new AnalyticsTable(AnalyticsTableType.COMPLETENESS, columnsA, sortKeyA, Logged.UNLOGGED);
     assertEquals("analytics_completeness", table.getMainName());
     assertEquals("analytics_completeness_temp", table.getName());
   }
 
   @Test
   void testGetTableNameValidationResult() {
     AnalyticsTable table =
-        new AnalyticsTable(AnalyticsTableType.VALIDATION_RESULT, columnsA, Logged.UNLOGGED);
+        new AnalyticsTable(
+            AnalyticsTableType.VALIDATION_RESULT, columnsA, sortKeyA, Logged.UNLOGGED);
     assertEquals("analytics_validationresult", table.getMainName());
     assertEquals("analytics_validationresult_temp", table.getName());
   }
@@ -151,7 +167,7 @@ void testGetDimensionAndFactColumns() {
                 .build());
 
     AnalyticsTable table =
-        new AnalyticsTable(AnalyticsTableType.DATA_VALUE, columns, Logged.UNLOGGED);
+        new AnalyticsTable(AnalyticsTableType.DATA_VALUE, columns, List.of(), Logged.UNLOGGED);
 
     assertEquals(3, table.getDimensionColumns().size());
     assertEquals("dx", table.getDimensionColumns().get(0).getName());
@@ -189,9 +205,9 @@ void testGetTablePartitionName() {
   @Test
   void testEquals() {
     AnalyticsTable tableA =
-        new AnalyticsTable(AnalyticsTableType.DATA_VALUE, columnsA, Logged.UNLOGGED);
+        new AnalyticsTable(AnalyticsTableType.DATA_VALUE, columnsA, sortKeyA, Logged.UNLOGGED);
     AnalyticsTable tableB =
-        new AnalyticsTable(AnalyticsTableType.DATA_VALUE, columnsA, Logged.UNLOGGED);
+        new AnalyticsTable(AnalyticsTableType.DATA_VALUE, columnsA, sortKeyA, Logged.UNLOGGED);
     List<AnalyticsTable> uniqueList = new UniqueArrayList<>();
     uniqueList.add(tableA);
     uniqueList.add(tableB);

diff --git a/...ervice-analytics/src/test/java/org/hisp/dhis/analytics/util/AnalyticsIndexHelperTest.java b/...ervice-analytics/src/test/java/org/hisp/dhis/analytics/util/AnalyticsIndexHelperTest.java
@@ -103,6 +103,6 @@ private AnalyticsTable stubAnalyticsTable() {
                 .indexType(BTREE)
                 .build());
 
-    return new AnalyticsTable(EVENT, columns, Logged.UNLOGGED);
+    return new AnalyticsTable(EVENT, columns, List.of(), Logged.UNLOGGED);
   }
 }