From 90d3517d780f6af6fb8910e291c35f9295759899 Mon Sep 17 00:00:00 2001 From: fmendezh Date: Wed, 21 Apr 2021 17:19:16 +0200 Subject: [PATCH 1/4] adding networkKeys to dataset class and refactoring test to use a DB initializer in the Spring Test context instead of running on each test --- .../datasetindex/DatasetBatchIndexerIT.java | 25 +++---- .../search/DatasetIndexUpdateListener.java | 15 ++-- ....java => RegistryDatabaseInitializer.java} | 33 +++------ .../database/TestCaseDatabaseInitializer.java | 22 +++--- .../search/SearchTestConfiguration.java | 11 +-- .../org/gbif/registry/ws/it/BaseItTest.java | 69 ++++++++++++++----- .../ws/it/DatasetProcessStatusIT.java | 4 +- .../gbif/registry/ws/it/DerivedDatasetIT.java | 4 +- .../gbif/registry/ws/it/NetworkEntityIT.java | 11 +-- .../org/gbif/registry/ws/it/NetworkIT.java | 6 ++ .../registry/ws/it/OccurrenceDownloadIT.java | 4 +- .../ws/it/OrganizationCreationIT.java | 4 +- .../java/org/gbif/registry/ws/it/UserIT.java | 6 +- .../gbif/registry/ws/it/UserManagementIT.java | 6 +- .../collections/BaseCollectionEntityIT.java | 4 +- .../it/collections/CollectionsSearchIT.java | 4 +- .../ExtendedCollectionEntityIT.java | 25 ++++--- .../it/collections/IdentifierResolverIT.java | 4 +- .../ws/it/collections/LookupServiceIT.java | 4 +- .../registry/ws/it/collections/PersonIT.java | 8 ++- .../registry/ws/it/legacy/IptResourceIT.java | 4 +- .../ws/it/legacy/LegacyDatasetResourceIT.java | 4 +- .../it/legacy/LegacyEndpointResourceIT.java | 4 +- .../legacy/LegacyOrganizationResourceIT.java | 4 +- .../it/oaipmh/AbstractOaipmhEndpointIT.java | 13 ++-- .../ws/it/oaipmh/OaipmhGetRecordIT.java | 23 ++++--- .../ws/it/oaipmh/OaipmhIdentifyIT.java | 23 ++++--- .../oaipmh/OaipmhListMetadataFormatsIT.java | 23 ++++--- .../ws/it/oaipmh/OaipmhListRecordsIT.java | 23 ++++--- .../ws/it/oaipmh/OaipmhListSetsIT.java | 23 ++++--- .../mapper/CollectionMapperIT.java | 4 +- .../mapper/DerivedDatasetMapperIT.java | 11 ++- .../ws/it/persistence/mapper/DoiMapperIT.java | 5 +- .../mapper/DuplicatesMapperIT.java | 5 +- .../persistence/mapper/IDigBioMapperIT.java | 9 ++- .../mapper/InstitutionMapperIT.java | 9 ++- .../mapper/MachineTagMapperIT.java | 6 +- .../it/persistence/mapper/PersonMapperIT.java | 6 +- .../mapper/PipelineProcessMapperIT.java | 9 ++- .../it/persistence/mapper/UserMapperIT.java | 7 -- .../ws/it/pipelines/PipelinesHistoryIT.java | 4 +- .../persistence/mapper/DatasetMapper.xml | 7 ++ .../search/dataset/DatasetEsFieldMapper.java | 4 +- .../indexing/DatasetJsonConverter.java | 4 +- .../indexing/DatasetRealtimeIndexer.java | 3 + .../indexing/EsDatasetRealtimeIndexer.java | 18 +++++ .../dataset/indexing/ws/GbifWsClient.java | 3 + .../indexing/ws/GbifWsRetrofitClient.java | 12 +++- .../main/resources/dataset-es-mapping.json | 4 +- 49 files changed, 308 insertions(+), 235 deletions(-) rename registry-integration-tests/src/test/java/org/gbif/registry/database/{TestsDatabaseInitializer.java => RegistryDatabaseInitializer.java} (65%) diff --git a/registry-cli/src/test/java/org/gbif/registry/cli/datasetindex/DatasetBatchIndexerIT.java b/registry-cli/src/test/java/org/gbif/registry/cli/datasetindex/DatasetBatchIndexerIT.java index 6a868a6b00..c98d0e0e4a 100644 --- a/registry-cli/src/test/java/org/gbif/registry/cli/datasetindex/DatasetBatchIndexerIT.java +++ b/registry-cli/src/test/java/org/gbif/registry/cli/datasetindex/DatasetBatchIndexerIT.java @@ -19,10 +19,11 @@ import org.gbif.registry.search.dataset.indexing.es.IndexingConstants; import java.io.IOException; +import java.util.Date; import java.util.Properties; import org.apache.http.HttpHost; -import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; @@ -49,6 +50,8 @@ public class DatasetBatchIndexerIT { private static final String API_URL = "http://api.gbif-dev.org/v1/"; + private static final String INDEX_NAME = IndexingConstants.ALIAS + '_' + new Date().getTime(); + @RegisterExtension public static SingleInstancePostgresExtension database = EmbeddedPostgresExtension.singleInstance(); @@ -82,7 +85,8 @@ DatasetBatchIndexerConfiguration getConfiguration() { ElasticsearchConfig elasticsearchConfigDataset = new ElasticsearchConfig(); elasticsearchConfigDataset.setHosts("http://" + embeddedElastic.getHttpHostAddress()); - elasticsearchConfigDataset.setAlias(IndexingConstants.ALIAS); + elasticsearchConfigDataset.setAlias(INDEX_NAME + "_alias"); + elasticsearchConfigDataset.setIndex(INDEX_NAME); configuration.setDatasetEs(elasticsearchConfigDataset); ElasticsearchConfig elasticsearchConfigOccurrence = new ElasticsearchConfig(); @@ -107,16 +111,6 @@ DatasetBatchIndexerConfiguration getConfiguration() { return configuration; } - public void refresh(RestHighLevelClient restHighLevelClient) { - try { - RefreshRequest refreshRequest = new RefreshRequest(); - refreshRequest.indices("*"); - restHighLevelClient.indices().refresh(refreshRequest, RequestOptions.DEFAULT); - } catch (IOException ex) { - throw new IllegalStateException(ex); - } - } - @Test public void indexTests() throws IOException { @@ -125,13 +119,12 @@ public void indexTests() throws IOException { datasetBatchIndexerCommand.doRun(); RestHighLevelClient restHighLevelClient = buildRestClient(); - refresh(restHighLevelClient); SearchResponse searchResponse = - buildRestClient() + restHighLevelClient .search( new SearchRequest() - .indices(IndexingConstants.ALIAS) + .indices(INDEX_NAME) .source(new SearchSourceBuilder().size(0)), RequestOptions.DEFAULT); @@ -139,6 +132,8 @@ public void indexTests() throws IOException { DATASETS_TO_INDEX, searchResponse.getHits().getTotalHits().value, "Wrong amount of indexed dataset"); + + restHighLevelClient.indices().delete(new DeleteIndexRequest(INDEX_NAME), RequestOptions.DEFAULT); } @AfterAll diff --git a/registry-events/src/main/java/org/gbif/registry/search/DatasetIndexUpdateListener.java b/registry-events/src/main/java/org/gbif/registry/search/DatasetIndexUpdateListener.java index e2b7da1ba3..022059bcc7 100644 --- a/registry-events/src/main/java/org/gbif/registry/search/DatasetIndexUpdateListener.java +++ b/registry-events/src/main/java/org/gbif/registry/search/DatasetIndexUpdateListener.java @@ -17,7 +17,7 @@ import org.gbif.api.model.registry.Dataset; import org.gbif.api.model.registry.Installation; -import org.gbif.api.model.registry.NetworkEntity; +import org.gbif.api.model.registry.Network; import org.gbif.api.model.registry.Organization; import org.gbif.api.model.registry.Tag; import org.gbif.api.service.registry.DatasetService; @@ -57,14 +57,14 @@ public DatasetIndexUpdateListener( } @Subscribe - public final void created(CreateEvent event) { + public final void created(CreateEvent event) { if (event.getObjectClass().equals(Dataset.class)) { indexService.index((Dataset) event.getNewObject()); } } @Subscribe - public final void updated(UpdateEvent event) { + public final void updated(UpdateEvent event) { if (event.getObjectClass().equals(Dataset.class)) { indexService.index((Dataset) event.getNewObject()); @@ -83,11 +83,18 @@ public final void updated(UpdateEvent event) { if (!i1.getOrganizationKey().equals(i2.getOrganizationKey())) { indexService.index(i2); } + } else if (event.getObjectClass().equals(Network.class)) { + // we only care about title changes + Network network1 = (Network)event.getOldObject(); + Network network2 = (Network)event.getNewObject(); + if (!network1.getTitle().equals(network2.getTitle())) { + indexService.index(network2); + } } } @Subscribe - public final void deleted(DeleteEvent event) { + public final void deleted(DeleteEvent event) { if (event.getObjectClass().equals(Dataset.class)) { indexService.delete((Dataset) event.getOldObject()); } diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/database/TestsDatabaseInitializer.java b/registry-integration-tests/src/test/java/org/gbif/registry/database/RegistryDatabaseInitializer.java similarity index 65% rename from registry-integration-tests/src/test/java/org/gbif/registry/database/TestsDatabaseInitializer.java rename to registry-integration-tests/src/test/java/org/gbif/registry/database/RegistryDatabaseInitializer.java index 286c8eaf0b..3276eb81b4 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/database/TestsDatabaseInitializer.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/database/RegistryDatabaseInitializer.java @@ -18,12 +18,10 @@ import java.sql.Connection; import java.sql.SQLException; +import javax.sql.DataSource; + import com.google.common.base.Throwables; -import io.zonky.test.db.postgres.junit5.PreparedDbExtension; -import lombok.Builder; -import lombok.Data; -import org.junit.jupiter.api.extension.BeforeAllCallback; -import org.junit.jupiter.api.extension.ExtensionContext; +import lombok.experimental.UtilityClass; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,25 +29,18 @@ * A Rule that initializes a database for All test cases in a Test class or in test suite. * * 
- * @RegisterExtension
- * public TestsDatabaseInitializer = TestsDatabaseInitializer
- *   .builder()
- *   .dbExtension(dbExtension) // developer required to provide DB Extension
- *   .build();
+ *  RegistryDatabaseInitializer().init();
  *
*/ -@Data -@Builder -public class TestsDatabaseInitializer implements BeforeAllCallback { +@UtilityClass +public class RegistryDatabaseInitializer { - private static final Logger LOG = LoggerFactory.getLogger(TestsDatabaseInitializer.class); - private final PreparedDbExtension dbExtension; + private static final Logger LOG = LoggerFactory.getLogger(RegistryDatabaseInitializer.class); - private void createTestUsers() throws Exception { + public static void init(DataSource dataSource) { LOG.info("Create test users"); - try (Connection connection = dbExtension.getTestDatabase().getConnection()) { + try (Connection connection = dataSource.getConnection()) { connection.setAutoCommit(false); - connection.prepareStatement("TRUNCATE public.user CASCADE").execute(); connection.prepareStatement( "INSERT INTO public.\"user\" (key, username, email, password, first_name, last_name, roles, settings, system_settings, created, last_login, deleted, challenge_code_key) " @@ -63,11 +54,7 @@ private void createTestUsers() throws Exception { } catch (SQLException e) { Throwables.propagate(e); } - LOG.info("Registry tables truncated"); + LOG.info("Initial test data loaded"); } - @Override - public void beforeAll(ExtensionContext extensionContext) throws Exception { - createTestUsers(); - } } diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/database/TestCaseDatabaseInitializer.java b/registry-integration-tests/src/test/java/org/gbif/registry/database/TestCaseDatabaseInitializer.java index 8bc51a6052..afd7da1ab2 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/database/TestCaseDatabaseInitializer.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/database/TestCaseDatabaseInitializer.java @@ -20,11 +20,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.stream.Collectors; import javax.sql.DataSource; -import lombok.Builder; import lombok.Data; import org.junit.jupiter.api.extension.BeforeEachCallback; import org.junit.jupiter.api.extension.ExtensionContext; @@ -32,25 +30,21 @@ import org.slf4j.LoggerFactory; import com.google.common.base.Throwables; +import org.springframework.test.context.junit.jupiter.SpringExtension; /** * A Rule that will truncate the tables ready for a new test. It is expected to do this before each * test by using the following: * * 
- * @RegisterExtension
- * public TestCaseDatabaseInitializer = TestCaseDatabaseInitializer.builder()
- *     .dataSource(database.getTestDatabase()) // developer required to provide datasource
- *     .build();
+ * @RegisterExtension TestCaseDatabaseInitializer = TestCaseDatabaseInitializer(tables);
  *
*/ @Data -@Builder public class TestCaseDatabaseInitializer implements BeforeEachCallback { private static final Logger LOG = LoggerFactory.getLogger(TestCaseDatabaseInitializer.class); - @Builder.Default private List tables = Arrays.asList("contact", "endpoint", "tag", "identifier", "comment", "node_identifier", "node_identifier", "node_identifier", "node_machine_tag", "node_tag", "node_comment", "occurrence_download", "organization_contact", "organization_endpoint", "organization_machine_tag", @@ -60,15 +54,19 @@ public class TestCaseDatabaseInitializer implements BeforeEachCallback { "dataset_identifier", "dataset_comment", "dataset_network", "network_contact", "network_endpoint", "network_machine_tag", "network_tag", "network_comment", "machine_tag, metadata", "editor_rights", "network", "dataset", "installation", - "organization, node", "collection_collection_person", "collection_identifier", + "organization, node", "collection_person", "collection_collection_person", "collection_identifier", "collection_tag", "institution_collection_person", "institution_identifier", "institution_tag", "institution_occurrence_mapping", "collection_occurrence_mapping", "collection_person", "collection", "institution", "address", "gbif_doi", "pipeline_step", "pipeline_process", "pipeline_execution", "derived_dataset"); - private final DataSource dataSource; + public TestCaseDatabaseInitializer() { + } - private void truncateTables() throws Exception { + public TestCaseDatabaseInitializer(String...tables) { + this.tables = Arrays.asList(tables); + } + private void truncateTables(DataSource dataSource) throws Exception { LOG.info("Truncating registry tables"); try (Connection connection = dataSource.getConnection()) { connection.setAutoCommit(false); @@ -108,6 +106,6 @@ private void truncateTables() throws Exception { @Override public void beforeEach(ExtensionContext extensionContext) throws Exception { - truncateTables(); + truncateTables(SpringExtension.getApplicationContext(extensionContext).getBean(DataSource.class)); } } diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/search/SearchTestConfiguration.java b/registry-integration-tests/src/test/java/org/gbif/registry/search/SearchTestConfiguration.java index 7a4abfec78..cae868fc98 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/search/SearchTestConfiguration.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/search/SearchTestConfiguration.java @@ -23,6 +23,7 @@ import org.gbif.api.model.occurrence.search.OccurrenceSearchParameter; import org.gbif.api.service.registry.DatasetService; import org.gbif.api.service.registry.InstallationService; +import org.gbif.api.service.registry.NetworkService; import org.gbif.api.service.registry.OrganizationService; import org.gbif.registry.search.dataset.indexing.ws.GbifApiService; import org.gbif.registry.search.dataset.indexing.ws.GbifWsClient; @@ -43,11 +44,13 @@ public class SearchTestConfiguration { @Bean @Primary public GbifWsClient gbifWsClient( - InstallationService installationService, - OrganizationService organizationService, - DatasetService datasetService) { + InstallationService installationService, + OrganizationService organizationService, + DatasetService datasetService, + NetworkService networkService + ) { return new GbifWsRetrofitClient( - gbifApiService(), installationService, organizationService, datasetService); + gbifApiService(), installationService, organizationService, datasetService, networkService); } public GbifApiService gbifApiService() { diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/BaseItTest.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/BaseItTest.java index 8cdb452032..f6dbffdb29 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/BaseItTest.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/BaseItTest.java @@ -16,21 +16,29 @@ package org.gbif.registry.ws.it; import org.gbif.api.vocabulary.UserRole; -import org.gbif.registry.database.TestsDatabaseInitializer; +import org.gbif.registry.database.RegistryDatabaseInitializer; import org.gbif.registry.search.test.EsManageServer; import org.gbif.registry.ws.it.fixtures.TestConstants; import org.gbif.ws.client.ClientBuilder; import org.gbif.ws.client.filter.SimplePrincipalProvider; import org.gbif.ws.security.KeyStore; +import java.sql.SQLException; import java.util.Arrays; import java.util.Collections; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.function.Consumer; import java.util.stream.Stream; +import javax.sql.DataSource; + +import io.zonky.test.db.postgres.embedded.ConnectionInfo; +import io.zonky.test.db.postgres.embedded.DatabasePreparer; +import io.zonky.test.db.postgres.embedded.EmbeddedPostgres; +import io.zonky.test.db.postgres.embedded.PreparedDbProvider; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Order; import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.api.extension.RegisterExtension; import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.util.TestPropertyValues; @@ -46,8 +54,6 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import io.zonky.test.db.postgres.embedded.LiquibasePreparer; -import io.zonky.test.db.postgres.junit5.EmbeddedPostgresExtension; -import io.zonky.test.db.postgres.junit5.PreparedDbExtension; import static org.gbif.registry.ws.it.fixtures.TestConstants.IT_APP_KEY2; @@ -72,21 +78,58 @@ public void initialize(ConfigurableApplicationContext configurableApplicationCon } } + /** + * Prepares a Tests database using an embedded Postgres instance. + */ + public static class EmbeddedDataBaseInitializer { + private final DataSource dataSource; + private final PreparedDbProvider provider; + private final ConnectionInfo connectionInfo; + private final List> builderCustomizers = new CopyOnWriteArrayList(); + + public EmbeddedDataBaseInitializer(DatabasePreparer preparer) { + try { + this.provider = PreparedDbProvider.forPreparer(preparer, this.builderCustomizers); + this.connectionInfo = provider.createNewDatabase(); + this.dataSource = provider.createDataSourceFromConnectionInfo(this.connectionInfo); + } catch (SQLException ex) { + throw new RuntimeException(ex); + } + } + + public DataSource getDataSource() { + return dataSource; + } + + public ConnectionInfo getConnectionInfo() { + return connectionInfo; + } + } + /** Custom ContextInitializer to expose the registry DB data source and search flags. */ public static class ContextInitializer implements ApplicationContextInitializer { @Override public void initialize(ConfigurableApplicationContext configurableApplicationContext) { + + EmbeddedDataBaseInitializer database = new EmbeddedDataBaseInitializer( + LiquibasePreparer.forClasspathLocation(TestConstants.LIQUIBASE_MASTER_FILE)); + + RegistryDatabaseInitializer.init(database.getDataSource()); + TestPropertyValues.of( - Stream.of(dbTestPropertyPairs()).flatMap(Stream::of).toArray(String[]::new)) + Stream.of(dbTestPropertyPairs(database)).flatMap(Stream::of).toArray(String[]::new)) .applyTo(configurableApplicationContext.getEnvironment()); TestPropertyValues.of("elasticsearch.mock=true") .applyTo(configurableApplicationContext.getEnvironment()); } - protected String[] dbTestPropertyPairs() { + /** + * Creates the registry datasource settings from the embedded database. + */ + String[] dbTestPropertyPairs(EmbeddedDataBaseInitializer database) { return new String[] { "registry.datasource.url=jdbc:postgresql://localhost:" + database.getConnectionInfo().getPort() @@ -98,18 +141,6 @@ protected String[] dbTestPropertyPairs() { } } - @Order(0) - @RegisterExtension - protected static PreparedDbExtension database = - EmbeddedPostgresExtension.preparedDatabase( - LiquibasePreparer.forClasspathLocation(TestConstants.LIQUIBASE_MASTER_FILE)); - - @Order(1) - @RegisterExtension - protected static TestsDatabaseInitializer usersDatabaseRule = TestsDatabaseInitializer.builder() - .dbExtension(database) - .build(); - private final SimplePrincipalProvider simplePrincipalProvider; protected static EsManageServer esServer; diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/DatasetProcessStatusIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/DatasetProcessStatusIT.java index 7599f0a019..42eb7d5cec 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/DatasetProcessStatusIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/DatasetProcessStatusIT.java @@ -64,9 +64,7 @@ public class DatasetProcessStatusIT extends BaseItTest { @RegisterExtension - protected TestCaseDatabaseInitializer databaseRule = TestCaseDatabaseInitializer.builder() - .dataSource(database.getTestDatabase()) - .build(); + protected TestCaseDatabaseInitializer databaseRule = new TestCaseDatabaseInitializer(); private final TestDataFactory testDataFactory; diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/DerivedDatasetIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/DerivedDatasetIT.java index 077545c9db..b5e98cc9d5 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/DerivedDatasetIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/DerivedDatasetIT.java @@ -59,9 +59,7 @@ public class DerivedDatasetIT extends BaseItTest { @RegisterExtension - protected TestCaseDatabaseInitializer databaseRule = TestCaseDatabaseInitializer.builder() - .dataSource(database.getTestDatabase()) - .build(); + protected TestCaseDatabaseInitializer databaseRule = new TestCaseDatabaseInitializer(); private static final PagingRequest REGULAR_PAGE = new PagingRequest(); diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/NetworkEntityIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/NetworkEntityIT.java index 74fef31e2b..4287936311 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/NetworkEntityIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/NetworkEntityIT.java @@ -48,6 +48,7 @@ import java.util.UUID; import javax.annotation.Nullable; +import javax.sql.DataSource; import javax.validation.ValidationException; import org.apache.commons.beanutils.BeanUtils; @@ -56,6 +57,7 @@ import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; import org.springframework.security.core.authority.SimpleGrantedAuthority; import org.springframework.security.core.context.SecurityContext; @@ -89,10 +91,11 @@ public abstract class NetworkEntityIT< private final TestDataFactory testDataFactory; + @Autowired + private DataSource dataSource; + @RegisterExtension - protected TestCaseDatabaseInitializer databaseRule = TestCaseDatabaseInitializer.builder() - .dataSource(database.getTestDatabase()) - .build(); + public TestCaseDatabaseInitializer databaseRule = new TestCaseDatabaseInitializer(); public NetworkEntityIT( NetworkEntityService service, @@ -159,7 +162,7 @@ public void testCreateAsEditor(ServiceType serviceType) throws Exception { anotherEntity.setCreated(null); anotherEntity.setKey(null); - try (Connection c = database.getTestDatabase().getConnection(); + try (Connection c = dataSource.getConnection(); PreparedStatement ps = c.prepareStatement("INSERT INTO editor_rights VALUES(?, ?)")) { ps.setString(1, TestConstants.TEST_EDITOR); ps.setObject(2, keyForCreateAsEditorTest(entity)); diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/NetworkIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/NetworkIT.java index f20bdc2ac3..23b2eb3445 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/NetworkIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/NetworkIT.java @@ -43,6 +43,7 @@ import org.springframework.boot.web.server.LocalServerPort; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; /** @@ -121,6 +122,11 @@ public void testAddAndDeleteConstituentAsEditor() throws Exception { int numConstituentsAfter = updatedNetwork.getNumConstituents(); assertEquals(1, numConstituentsAfter); + Dataset datasetFromDb = datasetResource.get(dataset.getKey()); + + assertTrue(datasetFromDb.getNetworkKeys().contains(network.getKey()), + "Network key not present in dataset networkKeys"); + // delete constituent requestTestFixture .deleteRequest( diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/OccurrenceDownloadIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/OccurrenceDownloadIT.java index 1109643268..335d2c71b7 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/OccurrenceDownloadIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/OccurrenceDownloadIT.java @@ -66,9 +66,7 @@ public class OccurrenceDownloadIT extends BaseItTest { @RegisterExtension - protected TestCaseDatabaseInitializer databaseRule = TestCaseDatabaseInitializer.builder() - .dataSource(database.getTestDatabase()) - .build(); + protected TestCaseDatabaseInitializer databaseRule = new TestCaseDatabaseInitializer(); private final OccurrenceDownloadService occurrenceDownloadResource; private final OccurrenceDownloadClient occurrenceDownloadClient; diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/OrganizationCreationIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/OrganizationCreationIT.java index c218f2b185..1f4a3989e6 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/OrganizationCreationIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/OrganizationCreationIT.java @@ -76,9 +76,7 @@ public class OrganizationCreationIT extends BaseItTest { @RegisterExtension - protected TestCaseDatabaseInitializer databaseRule = TestCaseDatabaseInitializer.builder() - .dataSource(database.getTestDatabase()) - .build(); + protected TestCaseDatabaseInitializer databaseRule = new TestCaseDatabaseInitializer(); private final OrganizationResource organizationResource; private final OrganizationService organizationClient; diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/UserIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/UserIT.java index 155a9e4dee..65ad3a3455 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/UserIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/UserIT.java @@ -25,7 +25,6 @@ import org.gbif.registry.ws.it.fixtures.UserTestFixture; import org.gbif.ws.client.filter.SimplePrincipalProvider; -import java.util.Collections; import java.util.stream.Collectors; import javax.annotation.Nullable; @@ -53,10 +52,7 @@ public class UserIT extends BaseItTest { @RegisterExtension - protected TestCaseDatabaseInitializer databaseRule = TestCaseDatabaseInitializer.builder() - .dataSource(database.getTestDatabase()) - .tables(Collections.singletonList("public.user")) - .build(); + protected TestCaseDatabaseInitializer databaseRule = new TestCaseDatabaseInitializer("public.user"); private final UserTestFixture userTestFixture; private final RequestTestFixture requestTestFixture; diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/UserManagementIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/UserManagementIT.java index 15e7edde51..75c34dbecc 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/UserManagementIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/UserManagementIT.java @@ -30,7 +30,6 @@ import org.gbif.registry.ws.it.fixtures.UserTestFixture; import org.gbif.ws.client.filter.SimplePrincipalProvider; -import java.util.Collections; import java.util.Map; import java.util.UUID; @@ -61,10 +60,7 @@ public class UserManagementIT extends BaseItTest { @RegisterExtension - protected TestCaseDatabaseInitializer databaseRule = TestCaseDatabaseInitializer.builder() - .dataSource(database.getTestDatabase()) - .tables(Collections.singletonList("public.user")) - .build(); + protected TestCaseDatabaseInitializer databaseRule = new TestCaseDatabaseInitializer("public.user"); private static final String CHANGED_PASSWORD = "123456"; diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/BaseCollectionEntityIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/BaseCollectionEntityIT.java index 3c2429b931..4a136c6306 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/BaseCollectionEntityIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/BaseCollectionEntityIT.java @@ -68,9 +68,7 @@ public abstract class BaseCollectionEntityIT< @RegisterExtension public CollectionsDatabaseInitializer collectionsDatabaseInitializer; @RegisterExtension - protected TestCaseDatabaseInitializer databaseRule = TestCaseDatabaseInitializer.builder() - .dataSource(database.getTestDatabase()) - .build(); + protected TestCaseDatabaseInitializer databaseRule = new TestCaseDatabaseInitializer(); public BaseCollectionEntityIT( CrudService resource, diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/CollectionsSearchIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/CollectionsSearchIT.java index ecf641ae85..e649b04e99 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/CollectionsSearchIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/CollectionsSearchIT.java @@ -46,9 +46,7 @@ public class CollectionsSearchIT extends BaseItTest { @RegisterExtension - protected TestCaseDatabaseInitializer databaseRule = TestCaseDatabaseInitializer.builder() - .dataSource(database.getTestDatabase()) - .build(); + protected TestCaseDatabaseInitializer databaseRule = new TestCaseDatabaseInitializer(); private final CollectionsSearchService searchService; private final InstitutionService institutionService; diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/ExtendedCollectionEntityIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/ExtendedCollectionEntityIT.java index 7af4798800..c9eb9aec97 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/ExtendedCollectionEntityIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/ExtendedCollectionEntityIT.java @@ -55,7 +55,6 @@ import org.gbif.ws.security.KeyStore; import java.sql.Connection; -import java.sql.SQLException; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -63,11 +62,16 @@ import java.util.UUID; import java.util.stream.Collectors; -import org.junit.jupiter.api.BeforeAll; +import javax.sql.DataSource; + +import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; import org.springframework.core.io.ClassPathResource; import org.springframework.jdbc.datasource.init.ScriptUtils; +import org.springframework.test.context.junit.jupiter.SpringExtension; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -110,15 +114,20 @@ public ExtendedCollectionEntityIT( this.installationService = installationService; } - @BeforeAll - public static void createMaterializedViews() throws SQLException { - Connection connection = database.getTestDatabase().getConnection(); - // create materialized view for testing - ScriptUtils.executeSqlScript( + public static class MaterializedViewInitializer implements BeforeAllCallback { + + @Override + public void beforeAll(ExtensionContext extensionContext) throws Exception { + Connection connection = SpringExtension.getApplicationContext(extensionContext).getBean(DataSource.class).getConnection(); + // create materialized view for testing + ScriptUtils.executeSqlScript( connection, new ClassPathResource("/scripts/create_duplicates_views.sql")); - connection.close(); + connection.close(); + } } + @RegisterExtension static MaterializedViewInitializer materializedViewInitializer = new MaterializedViewInitializer(); + @ParameterizedTest @EnumSource(ServiceType.class) public void contactsTest(ServiceType serviceType) { diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/IdentifierResolverIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/IdentifierResolverIT.java index 98d28a1b85..ddadf3a06e 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/IdentifierResolverIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/IdentifierResolverIT.java @@ -43,9 +43,7 @@ public class IdentifierResolverIT extends BaseItTest { @RegisterExtension - protected TestCaseDatabaseInitializer databaseRule = TestCaseDatabaseInitializer.builder() - .dataSource(database.getTestDatabase()) - .build(); + protected TestCaseDatabaseInitializer databaseRule = new TestCaseDatabaseInitializer(); private static final String BASE_URL = "/" + GRSCICOLL_PATH + "/resolve"; private static final String IDENTIFIER_PARAM = "identifier"; diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/LookupServiceIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/LookupServiceIT.java index 138abe8553..5d62068f93 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/LookupServiceIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/LookupServiceIT.java @@ -49,9 +49,7 @@ public class LookupServiceIT extends BaseItTest { @RegisterExtension - protected TestCaseDatabaseInitializer databaseRule = TestCaseDatabaseInitializer.builder() - .dataSource(database.getTestDatabase()) - .build(); + protected TestCaseDatabaseInitializer databaseRule = new TestCaseDatabaseInitializer(); private final Institution i1 = new Institution(); private final Institution i2 = new Institution(); diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/PersonIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/PersonIT.java index b0868eee93..b443fcfcb9 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/PersonIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/PersonIT.java @@ -44,6 +44,8 @@ import org.springframework.boot.web.server.LocalServerPort; import static org.gbif.registry.ws.it.fixtures.TestConstants.WS_TEST; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -115,7 +117,7 @@ public void createWithAddressTest(ServiceType serviceType) { assertEquals("mailing", personSaved.getMailingAddress().getAddress()); assertEquals("city", personSaved.getMailingAddress().getCity()); assertEquals(Country.AFGHANISTAN, personSaved.getMailingAddress().getCountry()); - assertEquals(1, personSaved.getIdentifiers().size()); + assertThat(1, greaterThanOrEqualTo(personSaved.getIdentifiers().size())); assertEquals("id", personSaved.getIdentifiers().get(0).getIdentifier()); assertEquals(IdentifierType.IH_IRN, personSaved.getIdentifiers().get(0).getType()); } @@ -135,12 +137,12 @@ public void listWithoutParamsTest(ServiceType serviceType) { UUID key3 = service.create(person3); PagingResponse response = service.list(null, null, null, DEFAULT_PAGE); - assertEquals(3, response.getResults().size()); + assertThat(3, greaterThanOrEqualTo(response.getResults().size())); service.delete(key3); response = service.list(null, null, null, DEFAULT_PAGE); - assertEquals(2, response.getResults().size()); + assertThat(2, greaterThanOrEqualTo(response.getResults().size())); response = service.list(null, null, null, new PagingRequest(0L, 1)); assertEquals(1, response.getResults().size()); diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/legacy/IptResourceIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/legacy/IptResourceIT.java index 1a8731c656..b48243a9a4 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/legacy/IptResourceIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/legacy/IptResourceIT.java @@ -75,9 +75,7 @@ public class IptResourceIT extends BaseItTest { @RegisterExtension - protected TestCaseDatabaseInitializer databaseRule = TestCaseDatabaseInitializer.builder() - .dataSource(database.getTestDatabase()) - .build(); + protected TestCaseDatabaseInitializer databaseRule = new TestCaseDatabaseInitializer(); private final InstallationService installationService; private final DatasetService datasetService; diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/legacy/LegacyDatasetResourceIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/legacy/LegacyDatasetResourceIT.java index b77c3ce98e..5264c42c9f 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/legacy/LegacyDatasetResourceIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/legacy/LegacyDatasetResourceIT.java @@ -76,9 +76,7 @@ public class LegacyDatasetResourceIT extends BaseItTest { @RegisterExtension - protected TestCaseDatabaseInitializer databaseRule = TestCaseDatabaseInitializer.builder() - .dataSource(database.getTestDatabase()) - .build(); + protected TestCaseDatabaseInitializer databaseRule = new TestCaseDatabaseInitializer(); private final DatasetService datasetService; private final TestDataFactory testDataFactory; diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/legacy/LegacyEndpointResourceIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/legacy/LegacyEndpointResourceIT.java index 97ae0cf023..f13879566f 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/legacy/LegacyEndpointResourceIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/legacy/LegacyEndpointResourceIT.java @@ -51,9 +51,7 @@ public class LegacyEndpointResourceIT extends BaseItTest { @RegisterExtension - protected TestCaseDatabaseInitializer databaseRule = TestCaseDatabaseInitializer.builder() - .dataSource(database.getTestDatabase()) - .build(); + protected TestCaseDatabaseInitializer databaseRule = new TestCaseDatabaseInitializer(); public static final String ENDPOINT_DESCRIPTION = "Description of Test Endpoint"; public static final String ENDPOINT_TYPE = "EML"; diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/legacy/LegacyOrganizationResourceIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/legacy/LegacyOrganizationResourceIT.java index ab2f4348c5..3582840f50 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/legacy/LegacyOrganizationResourceIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/legacy/LegacyOrganizationResourceIT.java @@ -45,9 +45,7 @@ public class LegacyOrganizationResourceIT extends BaseItTest { @RegisterExtension - protected TestCaseDatabaseInitializer databaseRule = TestCaseDatabaseInitializer.builder() - .dataSource(database.getTestDatabase()) - .build(); + protected TestCaseDatabaseInitializer databaseRule = new TestCaseDatabaseInitializer(); private final OrganizationService organizationService; private final NodeService nodeService; diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/oaipmh/AbstractOaipmhEndpointIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/oaipmh/AbstractOaipmhEndpointIT.java index 4b11bbc7d5..9cd484b8e4 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/oaipmh/AbstractOaipmhEndpointIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/oaipmh/AbstractOaipmhEndpointIT.java @@ -36,6 +36,8 @@ import java.util.Date; import java.util.UUID; +import javax.sql.DataSource; + import org.dspace.xoai.model.oaipmh.MetadataFormat; import org.dspace.xoai.serviceprovider.ServiceProvider; import org.dspace.xoai.serviceprovider.client.HttpOAIClient; @@ -49,9 +51,7 @@ public abstract class AbstractOaipmhEndpointIT extends BaseItTest { @RegisterExtension - protected TestCaseDatabaseInitializer databaseRule = TestCaseDatabaseInitializer.builder() - .dataSource(database.getTestDatabase()) - .build(); + protected TestCaseDatabaseInitializer databaseRule = new TestCaseDatabaseInitializer(); private final TestDataFactory testDataFactory; @@ -71,6 +71,7 @@ public abstract class AbstractOaipmhEndpointIT extends BaseItTest { private final OrganizationService organizationService; private final InstallationService installationService; private final DatasetService datasetService; + private final DataSource dataSource; protected final String baseUrl; protected final ServiceProvider serviceProvider; @@ -84,13 +85,15 @@ public AbstractOaipmhEndpointIT( InstallationService installationService, DatasetService datasetService, TestDataFactory testDataFactory, - EsManageServer esServer) { + EsManageServer esServer, + DataSource dataSource) { super(principalProvider, esServer); this.nodeService = nodeService; this.organizationService = organizationService; this.installationService = installationService; this.datasetService = datasetService; this.testDataFactory = testDataFactory; + this.dataSource = dataSource; String port = environment.getProperty("local.server.port"); baseUrl = String.format("http://localhost:%s/oai-pmh/registry", port); @@ -160,7 +163,7 @@ protected void insertMetadata(UUID key, InputStream document) { /** This method is used to change the modified date of a dataset in order to test date queries. */ protected void changeDatasetModifiedDate(UUID key, Date modifiedDate) throws Exception { - try (Connection connection = database.getTestDatabase().getConnection()) { + try (Connection connection = dataSource.getConnection()) { connection.setAutoCommit(false); PreparedStatement p = diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/oaipmh/OaipmhGetRecordIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/oaipmh/OaipmhGetRecordIT.java index 4e1112afdf..e9ee015685 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/oaipmh/OaipmhGetRecordIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/oaipmh/OaipmhGetRecordIT.java @@ -33,6 +33,8 @@ import java.nio.charset.StandardCharsets; import java.util.Date; +import javax.sql.DataSource; + import org.apache.commons.io.IOUtils; import org.dspace.xoai.model.oaipmh.Record; import org.dspace.xoai.serviceprovider.exceptions.CannotDisseminateFormatException; @@ -56,14 +58,16 @@ public class OaipmhGetRecordIT extends AbstractOaipmhEndpointIT { @Autowired public OaipmhGetRecordIT( - SimplePrincipalProvider principalProvider, - Environment environment, - NodeService nodeService, - OrganizationService organizationService, - InstallationService installationService, - DatasetService datasetService, - TestDataFactory testDataFactory, - EsManageServer esServer) { + SimplePrincipalProvider principalProvider, + Environment environment, + NodeService nodeService, + OrganizationService organizationService, + InstallationService installationService, + DatasetService datasetService, + TestDataFactory testDataFactory, + EsManageServer esServer, + DataSource dataSource + ) { super( principalProvider, environment, @@ -72,7 +76,8 @@ public OaipmhGetRecordIT( installationService, datasetService, testDataFactory, - esServer); + esServer, + dataSource); } @Test diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/oaipmh/OaipmhIdentifyIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/oaipmh/OaipmhIdentifyIT.java index b1bd71e748..98087f8933 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/oaipmh/OaipmhIdentifyIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/oaipmh/OaipmhIdentifyIT.java @@ -23,6 +23,8 @@ import org.gbif.registry.test.TestDataFactory; import org.gbif.ws.client.filter.SimplePrincipalProvider; +import javax.sql.DataSource; + import org.dspace.xoai.model.oaipmh.DeletedRecord; import org.dspace.xoai.model.oaipmh.Identify; import org.hamcrest.Matchers; @@ -40,14 +42,16 @@ public class OaipmhIdentifyIT extends AbstractOaipmhEndpointIT { @Autowired public OaipmhIdentifyIT( - SimplePrincipalProvider principalProvider, - Environment environment, - NodeService nodeService, - OrganizationService organizationService, - InstallationService installationService, - DatasetService datasetService, - TestDataFactory testDataFactory, - EsManageServer esServer) { + SimplePrincipalProvider principalProvider, + Environment environment, + NodeService nodeService, + OrganizationService organizationService, + InstallationService installationService, + DatasetService datasetService, + TestDataFactory testDataFactory, + EsManageServer esServer, + DataSource dataSource + ) { super( principalProvider, environment, @@ -56,7 +60,8 @@ public OaipmhIdentifyIT( installationService, datasetService, testDataFactory, - esServer); + esServer, + dataSource); } /** diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/oaipmh/OaipmhListMetadataFormatsIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/oaipmh/OaipmhListMetadataFormatsIT.java index 74716fae19..01cc3feb89 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/oaipmh/OaipmhListMetadataFormatsIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/oaipmh/OaipmhListMetadataFormatsIT.java @@ -31,6 +31,8 @@ import java.util.Date; import java.util.List; +import javax.sql.DataSource; + import org.dspace.xoai.model.oaipmh.MetadataFormat; import org.dspace.xoai.serviceprovider.exceptions.IdDoesNotExistException; import org.dspace.xoai.serviceprovider.parameters.ListMetadataParameters; @@ -52,14 +54,16 @@ public class OaipmhListMetadataFormatsIT extends AbstractOaipmhEndpointIT { @Autowired public OaipmhListMetadataFormatsIT( - SimplePrincipalProvider principalProvider, - Environment environment, - NodeService nodeService, - OrganizationService organizationService, - InstallationService installationService, - DatasetService datasetService, - TestDataFactory testDataFactory, - EsManageServer esServer) { + SimplePrincipalProvider principalProvider, + Environment environment, + NodeService nodeService, + OrganizationService organizationService, + InstallationService installationService, + DatasetService datasetService, + TestDataFactory testDataFactory, + EsManageServer esServer, + DataSource dataSource + ) { super( principalProvider, environment, @@ -68,7 +72,8 @@ public OaipmhListMetadataFormatsIT( installationService, datasetService, testDataFactory, - esServer); + esServer, + dataSource); } @Test diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/oaipmh/OaipmhListRecordsIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/oaipmh/OaipmhListRecordsIT.java index d1f6eef480..ea2fc3dcd8 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/oaipmh/OaipmhListRecordsIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/oaipmh/OaipmhListRecordsIT.java @@ -38,6 +38,8 @@ import java.util.Iterator; import java.util.List; +import javax.sql.DataSource; + import org.apache.commons.io.IOUtils; import org.dspace.xoai.model.oaipmh.Record; import org.dspace.xoai.serviceprovider.parameters.GetRecordParameters; @@ -58,14 +60,16 @@ public class OaipmhListRecordsIT extends AbstractOaipmhEndpointIT { @Autowired public OaipmhListRecordsIT( - SimplePrincipalProvider principalProvider, - Environment environment, - NodeService nodeService, - OrganizationService organizationService, - InstallationService installationService, - DatasetService datasetService, - TestDataFactory testDataFactory, - EsManageServer esServer) { + SimplePrincipalProvider principalProvider, + Environment environment, + NodeService nodeService, + OrganizationService organizationService, + InstallationService installationService, + DatasetService datasetService, + TestDataFactory testDataFactory, + EsManageServer esServer, + DataSource dataSource + ) { super( principalProvider, environment, @@ -74,7 +78,8 @@ public OaipmhListRecordsIT( installationService, datasetService, testDataFactory, - esServer); + esServer, + dataSource); } @Test diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/oaipmh/OaipmhListSetsIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/oaipmh/OaipmhListSetsIT.java index 379acbcdfa..81a51b44eb 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/oaipmh/OaipmhListSetsIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/oaipmh/OaipmhListSetsIT.java @@ -33,6 +33,8 @@ import java.util.List; import java.util.stream.Collectors; +import javax.sql.DataSource; + import org.dspace.xoai.model.oaipmh.Set; import org.dspace.xoai.serviceprovider.exceptions.NoSetHierarchyException; import org.junit.jupiter.api.Test; @@ -55,14 +57,16 @@ public class OaipmhListSetsIT extends AbstractOaipmhEndpointIT { @Autowired public OaipmhListSetsIT( - SimplePrincipalProvider principalProvider, - Environment environment, - NodeService nodeService, - OrganizationService organizationService, - InstallationService installationService, - DatasetService datasetService, - TestDataFactory testDataFactory, - EsManageServer esServer) { + SimplePrincipalProvider principalProvider, + Environment environment, + NodeService nodeService, + OrganizationService organizationService, + InstallationService installationService, + DatasetService datasetService, + TestDataFactory testDataFactory, + EsManageServer esServer, + DataSource dataSource + ) { super( principalProvider, environment, @@ -71,7 +75,8 @@ public OaipmhListSetsIT( installationService, datasetService, testDataFactory, - esServer); + esServer, + dataSource); } @Test diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/CollectionMapperIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/CollectionMapperIT.java index d0a4a355f4..e2809e9d90 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/CollectionMapperIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/CollectionMapperIT.java @@ -55,9 +55,7 @@ public class CollectionMapperIT extends BaseItTest { @RegisterExtension - protected TestCaseDatabaseInitializer databaseRule = TestCaseDatabaseInitializer.builder() - .dataSource(database.getTestDatabase()) - .build(); + protected TestCaseDatabaseInitializer databaseRule = new TestCaseDatabaseInitializer("collection"); private CollectionMapper collectionMapper; private AddressMapper addressMapper; diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/DerivedDatasetMapperIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/DerivedDatasetMapperIT.java index e572c24360..f75907746f 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/DerivedDatasetMapperIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/DerivedDatasetMapperIT.java @@ -32,6 +32,7 @@ import java.time.Month; import java.time.ZoneId; import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; import java.util.List; @@ -45,9 +46,7 @@ public class DerivedDatasetMapperIT extends BaseItTest { @RegisterExtension - protected TestCaseDatabaseInitializer databaseRule = TestCaseDatabaseInitializer.builder() - .dataSource(database.getTestDatabase()) - .build(); + protected TestCaseDatabaseInitializer databaseRule = new TestCaseDatabaseInitializer("derived_dataset", "dataset"); private final DerivedDatasetMapper mapper; private final TestDataFactory testDataFactory; @@ -118,7 +117,7 @@ public void testAddDatasetCitations() { public void testListByRegistrationDate() { // create derivedDatasets DerivedDataset derivedDataset1 = prepareDerivedDataset(); - derivedDataset1.setDoi(new DOI("10.21373/dd.doi1")); + derivedDataset1.setDoi(new DOI("10.21373/dd.doi3")); // day before date derivedDataset1.setRegistrationDate( Date.from( @@ -128,7 +127,7 @@ public void testListByRegistrationDate() { mapper.create(derivedDataset1); DerivedDataset derivedDataset2 = prepareDerivedDataset(); - derivedDataset2.setDoi(new DOI("10.21373/dd.doi2")); + derivedDataset2.setDoi(new DOI("10.21373/dd.doi4")); // the same date but an hour later derivedDataset2.setRegistrationDate( Date.from( @@ -138,7 +137,7 @@ public void testListByRegistrationDate() { mapper.create(derivedDataset2); DerivedDataset derivedDataset3 = prepareDerivedDataset(); - derivedDataset3.setDoi(new DOI("10.21373/dd.doi3")); + derivedDataset3.setDoi(new DOI("10.21373/dd.doi5")); // day after date derivedDataset3.setRegistrationDate( Date.from( diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/DoiMapperIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/DoiMapperIT.java index 76dd210849..bfa42736fc 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/DoiMapperIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/DoiMapperIT.java @@ -26,6 +26,7 @@ import org.gbif.ws.client.filter.SimplePrincipalProvider; import java.net.URI; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -40,9 +41,7 @@ public class DoiMapperIT extends BaseItTest { @RegisterExtension - protected TestCaseDatabaseInitializer databaseRule = TestCaseDatabaseInitializer.builder() - .dataSource(database.getTestDatabase()) - .build(); + protected TestCaseDatabaseInitializer databaseRule = new TestCaseDatabaseInitializer("gbif_doi"); private DoiMapper mapper; diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/DuplicatesMapperIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/DuplicatesMapperIT.java index 478ea281ab..0cca66849b 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/DuplicatesMapperIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/DuplicatesMapperIT.java @@ -49,9 +49,8 @@ public class DuplicatesMapperIT extends BaseItTest { @RegisterExtension - protected TestCaseDatabaseInitializer databaseRule = TestCaseDatabaseInitializer.builder() - .dataSource(database.getTestDatabase()) - .build(); + protected TestCaseDatabaseInitializer databaseRule = new TestCaseDatabaseInitializer("collection", + "institution"); private DuplicatesMapper duplicatesMapper; private InstitutionMapper institutionMapper; diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/IDigBioMapperIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/IDigBioMapperIT.java index 1bf3cb8e8c..ea445dfc30 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/IDigBioMapperIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/IDigBioMapperIT.java @@ -40,6 +40,7 @@ import org.gbif.ws.client.filter.SimplePrincipalProvider; import java.math.BigDecimal; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Set; @@ -56,9 +57,11 @@ public class IDigBioMapperIT extends BaseItTest { @RegisterExtension - protected TestCaseDatabaseInitializer databaseRule = TestCaseDatabaseInitializer.builder() - .dataSource(database.getTestDatabase()) - .build(); + protected TestCaseDatabaseInitializer databaseRule = new TestCaseDatabaseInitializer("collection_person", + "collection", + "institution", + "address", + "identifier"); private IDigBioMapper iDigBioMapper; private CollectionMapper collectionMapper; diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/InstitutionMapperIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/InstitutionMapperIT.java index 034ad53e72..e8fa500109 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/InstitutionMapperIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/InstitutionMapperIT.java @@ -36,6 +36,7 @@ import java.net.URI; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.UUID; @@ -53,9 +54,11 @@ public class InstitutionMapperIT extends BaseItTest { @RegisterExtension - protected TestCaseDatabaseInitializer databaseRule = TestCaseDatabaseInitializer.builder() - .dataSource(database.getTestDatabase()) - .build(); + protected TestCaseDatabaseInitializer databaseRule = new TestCaseDatabaseInitializer("institution_collection_person", + "institution_identifier", + "institution_tag", + "institution_occurrence_mapping", + "institution"); private InstitutionMapper institutionMapper; private AddressMapper addressMapper; diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/MachineTagMapperIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/MachineTagMapperIT.java index 8526c40ec8..db12bbaf80 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/MachineTagMapperIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/MachineTagMapperIT.java @@ -22,6 +22,8 @@ import org.gbif.registry.ws.it.BaseItTest; import org.gbif.ws.client.filter.SimplePrincipalProvider; +import java.util.Collections; + import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.springframework.beans.factory.annotation.Autowired; @@ -31,9 +33,7 @@ public class MachineTagMapperIT extends BaseItTest { @RegisterExtension - protected TestCaseDatabaseInitializer databaseRule = TestCaseDatabaseInitializer.builder() - .dataSource(database.getTestDatabase()) - .build(); + protected TestCaseDatabaseInitializer databaseRule = new TestCaseDatabaseInitializer("machine_tag"); private final MachineTagMapper mapper; diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/PersonMapperIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/PersonMapperIT.java index 9461fe0b23..140c83ad70 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/PersonMapperIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/PersonMapperIT.java @@ -25,6 +25,7 @@ import org.gbif.registry.ws.it.BaseItTest; import org.gbif.ws.client.filter.SimplePrincipalProvider; +import java.util.Arrays; import java.util.List; import java.util.UUID; @@ -41,9 +42,8 @@ public class PersonMapperIT extends BaseItTest { @RegisterExtension - protected TestCaseDatabaseInitializer databaseRule = TestCaseDatabaseInitializer.builder() - .dataSource(database.getTestDatabase()) - .build(); + protected TestCaseDatabaseInitializer databaseRule = new TestCaseDatabaseInitializer("collection_person", + "address"); private PersonMapper personMapper; private AddressMapper addressMapper; diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/PipelineProcessMapperIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/PipelineProcessMapperIT.java index 131031db42..b9aedd8843 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/PipelineProcessMapperIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/PipelineProcessMapperIT.java @@ -42,6 +42,7 @@ import org.gbif.ws.client.filter.SimplePrincipalProvider; import java.time.LocalDateTime; +import java.util.Arrays; import java.util.Collections; import java.util.UUID; @@ -60,9 +61,11 @@ public class PipelineProcessMapperIT extends BaseItTest { @RegisterExtension - protected TestCaseDatabaseInitializer databaseRule = TestCaseDatabaseInitializer.builder() - .dataSource(database.getTestDatabase()) - .build(); + protected TestCaseDatabaseInitializer databaseRule = new TestCaseDatabaseInitializer("pipeline_process", + "dataset", + "installation", + "organization", + "node"); private static final String TEST_USER = "test"; private static final String UPDATER_USER = "updater"; diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/UserMapperIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/UserMapperIT.java index f0ce9d2574..ed249a6d68 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/UserMapperIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/UserMapperIT.java @@ -17,7 +17,6 @@ import org.gbif.api.model.common.GbifUser; import org.gbif.api.vocabulary.UserRole; -import org.gbif.registry.database.TestCaseDatabaseInitializer; import org.gbif.registry.persistence.mapper.UserMapper; import org.gbif.registry.search.test.EsManageServer; import org.gbif.registry.ws.it.BaseItTest; @@ -30,7 +29,6 @@ import java.util.Set; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; import org.springframework.beans.factory.annotation.Autowired; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -39,11 +37,6 @@ public class UserMapperIT extends BaseItTest { - @RegisterExtension - protected TestCaseDatabaseInitializer databaseRule = TestCaseDatabaseInitializer.builder() - .dataSource(database.getTestDatabase()) - .build(); - private final UserMapper mapper; @Autowired diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/pipelines/PipelinesHistoryIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/pipelines/PipelinesHistoryIT.java index f815f08960..0de947a9cf 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/pipelines/PipelinesHistoryIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/pipelines/PipelinesHistoryIT.java @@ -71,9 +71,7 @@ public class PipelinesHistoryIT extends BaseItTest { @RegisterExtension - protected TestCaseDatabaseInitializer databaseRule = TestCaseDatabaseInitializer.builder() - .dataSource(database.getTestDatabase()) - .build(); + protected TestCaseDatabaseInitializer databaseRule = new TestCaseDatabaseInitializer(); private final DatasetService datasetService; private final OrganizationService organizationService; diff --git a/registry-persistence/src/main/resources/org/gbif/registry/persistence/mapper/DatasetMapper.xml b/registry-persistence/src/main/resources/org/gbif/registry/persistence/mapper/DatasetMapper.xml index e3f6e1e1e5..adcca8d539 100644 --- a/registry-persistence/src/main/resources/org/gbif/registry/persistence/mapper/DatasetMapper.xml +++ b/registry-persistence/src/main/resources/org/gbif/registry/persistence/mapper/DatasetMapper.xml @@ -13,6 +13,7 @@ + @@ -441,6 +442,12 @@ ORDER BY created DESC + + DELETE FROM comment USING dataset_comment diff --git a/registry-search/src/main/java/org/gbif/registry/search/dataset/DatasetEsFieldMapper.java b/registry-search/src/main/java/org/gbif/registry/search/dataset/DatasetEsFieldMapper.java index a29dd14180..b233a345b7 100644 --- a/registry-search/src/main/java/org/gbif/registry/search/dataset/DatasetEsFieldMapper.java +++ b/registry-search/src/main/java/org/gbif/registry/search/dataset/DatasetEsFieldMapper.java @@ -63,7 +63,7 @@ public class DatasetEsFieldMapper implements EsFieldMapper CARDINALITIES = @@ -159,7 +159,7 @@ public String[] getMappedFields() { "decade", "countryCoverage", "doi", - "networkKey", + "networkKeys", "networkTitle" }; } diff --git a/registry-search/src/main/java/org/gbif/registry/search/dataset/indexing/DatasetJsonConverter.java b/registry-search/src/main/java/org/gbif/registry/search/dataset/indexing/DatasetJsonConverter.java index 7b8ed348c3..05f767aec0 100644 --- a/registry-search/src/main/java/org/gbif/registry/search/dataset/indexing/DatasetJsonConverter.java +++ b/registry-search/src/main/java/org/gbif/registry/search/dataset/indexing/DatasetJsonConverter.java @@ -189,9 +189,9 @@ public void addNetworks(Dataset dataset, ObjectNode datasetJsonNode) { networkTitles.add(new TextNode(network.getTitle())); }); datasetJsonNode - .putArray("networkKey").addAll(networkKeys); + .putArray("networkKeys").addAll(networkKeys); datasetJsonNode - .putArray("networkTitle").addAll(networkTitles); + .putArray("networkTitles").addAll(networkTitles); } } diff --git a/registry-search/src/main/java/org/gbif/registry/search/dataset/indexing/DatasetRealtimeIndexer.java b/registry-search/src/main/java/org/gbif/registry/search/dataset/indexing/DatasetRealtimeIndexer.java index 510f9288f0..69881bebff 100644 --- a/registry-search/src/main/java/org/gbif/registry/search/dataset/indexing/DatasetRealtimeIndexer.java +++ b/registry-search/src/main/java/org/gbif/registry/search/dataset/indexing/DatasetRealtimeIndexer.java @@ -17,6 +17,7 @@ import org.gbif.api.model.registry.Dataset; import org.gbif.api.model.registry.Installation; +import org.gbif.api.model.registry.Network; import org.gbif.api.model.registry.Organization; public interface DatasetRealtimeIndexer { @@ -29,6 +30,8 @@ public interface DatasetRealtimeIndexer { void index(Installation installation); + void index(Network network); + void delete(Dataset dataset); int getPendingUpdates(); diff --git a/registry-search/src/main/java/org/gbif/registry/search/dataset/indexing/EsDatasetRealtimeIndexer.java b/registry-search/src/main/java/org/gbif/registry/search/dataset/indexing/EsDatasetRealtimeIndexer.java index 4188bdcaa2..13c1bcc45d 100644 --- a/registry-search/src/main/java/org/gbif/registry/search/dataset/indexing/EsDatasetRealtimeIndexer.java +++ b/registry-search/src/main/java/org/gbif/registry/search/dataset/indexing/EsDatasetRealtimeIndexer.java @@ -17,6 +17,7 @@ import org.gbif.api.model.registry.Dataset; import org.gbif.api.model.registry.Installation; +import org.gbif.api.model.registry.Network; import org.gbif.api.model.registry.Organization; import org.gbif.api.util.iterables.Iterables; import org.gbif.registry.search.dataset.indexing.es.IndexingConstants; @@ -232,6 +233,23 @@ public void onFailure(Exception ex) { } } + @Override + public void index(Network network) { + // Update hosted datasets for the organization + try { + log.debug("Updating hosted datasets for installation {}", network.getKey()); + Iterable datasets = + Iterables.datasetsIterable( + page -> gbifWsClient.getNetworkDatasets(network.getKey().toString(), page)); + index(datasets); + } catch (Exception e) { + log.error( + "Unable to update hosted datasets for network {} - index is now out of sync", + network.getKey(), + e); + } + } + @Override public int getPendingUpdates() { return pendingUpdates.get(); diff --git a/registry-search/src/main/java/org/gbif/registry/search/dataset/indexing/ws/GbifWsClient.java b/registry-search/src/main/java/org/gbif/registry/search/dataset/indexing/ws/GbifWsClient.java index 71e7d85c34..8fc560fdbb 100644 --- a/registry-search/src/main/java/org/gbif/registry/search/dataset/indexing/ws/GbifWsClient.java +++ b/registry-search/src/main/java/org/gbif/registry/search/dataset/indexing/ws/GbifWsClient.java @@ -55,6 +55,9 @@ PagingResponse getOrganizationHostedDatasets( PagingResponse getOrganizationPublishedDataset( String organizationKey, PagingRequest pagingRequest); + PagingResponse getNetworkDatasets( + String networkKey, PagingRequest pagingRequest); + InputStream getMetadataDocument(UUID datasetKey); Long getDatasetRecordCount(String datasetKey); diff --git a/registry-search/src/main/java/org/gbif/registry/search/dataset/indexing/ws/GbifWsRetrofitClient.java b/registry-search/src/main/java/org/gbif/registry/search/dataset/indexing/ws/GbifWsRetrofitClient.java index a970f66afc..c50782f4fe 100644 --- a/registry-search/src/main/java/org/gbif/registry/search/dataset/indexing/ws/GbifWsRetrofitClient.java +++ b/registry-search/src/main/java/org/gbif/registry/search/dataset/indexing/ws/GbifWsRetrofitClient.java @@ -31,6 +31,7 @@ import org.gbif.api.model.registry.Organization; import org.gbif.api.service.registry.DatasetService; import org.gbif.api.service.registry.InstallationService; +import org.gbif.api.service.registry.NetworkService; import org.gbif.api.service.registry.OrganizationService; import java.io.InputStream; @@ -79,6 +80,7 @@ public class GbifWsRetrofitClient implements GbifWsClient { private final InstallationService installationService; private final OrganizationService organizationService; private final DatasetService datasetService; + private final NetworkService networkService; /** * Factory method, only need the api base url. @@ -90,11 +92,13 @@ public GbifWsRetrofitClient( GbifApiService gbifApiService, InstallationService installationService, OrganizationService organizationService, - DatasetService datasetService) { + DatasetService datasetService, + NetworkService networkService) { this.gbifApiService = gbifApiService; this.installationService = installationService; this.organizationService = organizationService; this.datasetService = datasetService; + this.networkService = networkService; } private Map toQueryMap(PagingRequest pagingRequest) { @@ -156,6 +160,12 @@ public PagingResponse getOrganizationPublishedDataset( return organizationService.publishedDatasets(UUID.fromString(organizationKey), pagingRequest); } + @Override + public PagingResponse getNetworkDatasets( + String networkKey, PagingRequest pagingRequest) { + return networkService.listConstituents(UUID.fromString(networkKey), pagingRequest); + } + @Override public InputStream getMetadataDocument(UUID datasetKey) { return datasetService.getMetadataDocument(datasetKey); diff --git a/registry-search/src/main/resources/dataset-es-mapping.json b/registry-search/src/main/resources/dataset-es-mapping.json index 4c1655777e..6ba04a1020 100644 --- a/registry-search/src/main/resources/dataset-es-mapping.json +++ b/registry-search/src/main/resources/dataset-es-mapping.json @@ -149,8 +149,8 @@ "hostingOrganizationTitleAutocomplete": {"type": "text", "analyzer": "autocomplete", "search_analyzer": "autocomplete_search"}, "installationTitle": {"type": "text", "copy_to": "all"}, "installationTitleAutocomplete": {"type": "text", "analyzer": "autocomplete", "search_analyzer": "autocomplete_search"}, - "networkKey": {"type": "keyword"}, - "networkTitle": {"type": "text", "copy_to": "all"}, + "networkKeys": {"type": "keyword"}, + "networkTitles": {"type": "text", "copy_to": "all"}, "created": {"type": "date"}, "modified": {"type": "date"}, "deleted": {"type": "date"}, From 5b9974fec55e58282b7502f2810242eda97233e5 Mon Sep 17 00:00:00 2001 From: fmendezh Date: Wed, 21 Apr 2021 21:49:13 +0200 Subject: [PATCH 2/4] NetworkService needed for batch indexing --- .../registry/cli/datasetindex/SpringContextBuilder.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/registry-cli/src/main/java/org/gbif/registry/cli/datasetindex/SpringContextBuilder.java b/registry-cli/src/main/java/org/gbif/registry/cli/datasetindex/SpringContextBuilder.java index 7a38a8cd3a..21105afd2d 100644 --- a/registry-cli/src/main/java/org/gbif/registry/cli/datasetindex/SpringContextBuilder.java +++ b/registry-cli/src/main/java/org/gbif/registry/cli/datasetindex/SpringContextBuilder.java @@ -17,6 +17,7 @@ import org.gbif.api.service.registry.DatasetService; import org.gbif.api.service.registry.InstallationService; +import org.gbif.api.service.registry.NetworkService; import org.gbif.api.service.registry.OrganizationService; import org.gbif.registry.search.dataset.indexing.DatasetJsonConverter; import org.gbif.registry.search.dataset.indexing.checklistbank.ChecklistbankPersistenceServiceImpl; @@ -27,6 +28,7 @@ import org.gbif.registry.search.dataset.indexing.ws.JacksonObjectMapper; import org.gbif.registry.ws.client.DatasetClient; import org.gbif.registry.ws.client.InstallationClient; +import org.gbif.registry.ws.client.NetworkClient; import org.gbif.registry.ws.client.OrganizationClient; import org.gbif.ws.client.ClientBuilder; @@ -196,5 +198,10 @@ public OrganizationService organizationService(ClientBuilder clientBuilder) { public DatasetService datasetService(ClientBuilder clientBuilder) { return clientBuilder.build(DatasetClient.class); } + + @Bean + public NetworkService networkService(ClientBuilder clientBuilder) { + return clientBuilder.build(NetworkClient.class); + } } } From fc2b97817514db6c8d13ff0da1870be3dcbb6098 Mon Sep 17 00:00:00 2001 From: fmendezh Date: Thu, 22 Apr 2021 09:37:26 +0200 Subject: [PATCH 3/4] Deleting the index after batch indexing test --- .../cli/datasetindex/DatasetBatchIndexerIT.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/registry-cli/src/test/java/org/gbif/registry/cli/datasetindex/DatasetBatchIndexerIT.java b/registry-cli/src/test/java/org/gbif/registry/cli/datasetindex/DatasetBatchIndexerIT.java index c98d0e0e4a..5416d0164e 100644 --- a/registry-cli/src/test/java/org/gbif/registry/cli/datasetindex/DatasetBatchIndexerIT.java +++ b/registry-cli/src/test/java/org/gbif/registry/cli/datasetindex/DatasetBatchIndexerIT.java @@ -31,6 +31,7 @@ import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -52,6 +53,8 @@ public class DatasetBatchIndexerIT { private static final String INDEX_NAME = IndexingConstants.ALIAS + '_' + new Date().getTime(); + private static final String INDEX_ALIAS = INDEX_NAME + "_a" ; + @RegisterExtension public static SingleInstancePostgresExtension database = EmbeddedPostgresExtension.singleInstance(); @@ -85,7 +88,7 @@ DatasetBatchIndexerConfiguration getConfiguration() { ElasticsearchConfig elasticsearchConfigDataset = new ElasticsearchConfig(); elasticsearchConfigDataset.setHosts("http://" + embeddedElastic.getHttpHostAddress()); - elasticsearchConfigDataset.setAlias(INDEX_NAME + "_alias"); + elasticsearchConfigDataset.setAlias(INDEX_ALIAS); elasticsearchConfigDataset.setIndex(INDEX_NAME); configuration.setDatasetEs(elasticsearchConfigDataset); @@ -124,7 +127,7 @@ public void indexTests() throws IOException { restHighLevelClient .search( new SearchRequest() - .indices(INDEX_NAME) + .indices(INDEX_ALIAS) .source(new SearchSourceBuilder().size(0)), RequestOptions.DEFAULT); @@ -132,8 +135,11 @@ public void indexTests() throws IOException { DATASETS_TO_INDEX, searchResponse.getHits().getTotalHits().value, "Wrong amount of indexed dataset"); + } - restHighLevelClient.indices().delete(new DeleteIndexRequest(INDEX_NAME), RequestOptions.DEFAULT); + @AfterEach + public void deleteIndex() throws IOException { + buildRestClient().indices().delete(new DeleteIndexRequest(INDEX_NAME), RequestOptions.DEFAULT); } @AfterAll From 4c91b0936f0a2ce3783f83078b318276d92684b9 Mon Sep 17 00:00:00 2001 From: fmendezh Date: Thu, 22 Apr 2021 11:29:48 +0200 Subject: [PATCH 4/4] fixing result type in listNetworks mapping function --- .../org/gbif/registry/persistence/mapper/DatasetMapper.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/registry-persistence/src/main/resources/org/gbif/registry/persistence/mapper/DatasetMapper.xml b/registry-persistence/src/main/resources/org/gbif/registry/persistence/mapper/DatasetMapper.xml index adcca8d539..62fbfd6606 100644 --- a/registry-persistence/src/main/resources/org/gbif/registry/persistence/mapper/DatasetMapper.xml +++ b/registry-persistence/src/main/resources/org/gbif/registry/persistence/mapper/DatasetMapper.xml @@ -442,7 +442,7 @@ ORDER BY created DESC - SELECT network_key FROM dataset_network WHERE dataset_key = #{targetEntityKey,jdbcType=OTHER}