diff --git a/README.md b/README.md index 4fe61865..23256916 100644 --- a/README.md +++ b/README.md @@ -192,6 +192,7 @@ Elasticsearch-Evolution can be configured to your needs: - **placeholderSuffix** (default=}): Suffix of placeholders in migration scripts. - **historyIndex** (default=es_evolution): Name of the history index that will be used by Elasticsearch-Evolution. In this index Elasticsearch-Evolution will persist his internal state and tracks which migration script has already been executed. - **historyMaxQuerySize** (default=1000): The maximum query size while validating already executed scripts. This query size have to be higher than the total count of your migration scripts. +- **validateOnMigrate** (default=true): Whether to fail when a previously applied migration script has been modified after it was applied. ### 5.1 Spring Boot @@ -288,12 +289,16 @@ ElasticsearchEvolution.configure() ### v0.4.1-SNAPSHOT + +- Previously applied migration scripts are now checked for modifications and rejected if they've been modified after they were applied. The old behaviour can be restored by setting the new configuration parameter `validateOnMigrate` to false (default: true) ([#155](https://github.com/senacor/elasticsearch-evolution/isse/155)) + - version updates (spring-boot 2.7.4) - added java 19 compatibility tests - added spring boot 2.7 compatibility tests -- added Elasticsearch 8.4, 8.3, and 8,2 compatibility tests +- added Elasticsearch 8.4, 8.3, and 8,2 compatibility test - added Opensearch 2.3, 2.2, 2.1 and 2.0 compatibility tests + ### v0.4.0 - **breaking change**: drop `org.elasticsearch.client.RestHighLevelClient` and replace with `org.elasticsearch.client.RestClient` (LowLevelClient). This will drop the big transitive dependency `org.elasticsearch:elasticsearch` and opens compatibility to Elasticsearch 8 and OpenSearch. diff --git a/elasticsearch-evolution-core/src/main/java/com/senacor/elasticsearch/evolution/core/ElasticsearchEvolution.java b/elasticsearch-evolution-core/src/main/java/com/senacor/elasticsearch/evolution/core/ElasticsearchEvolution.java index ef80d791..bc90a4ba 100644 --- a/elasticsearch-evolution-core/src/main/java/com/senacor/elasticsearch/evolution/core/ElasticsearchEvolution.java +++ b/elasticsearch-evolution-core/src/main/java/com/senacor/elasticsearch/evolution/core/ElasticsearchEvolution.java @@ -168,6 +168,7 @@ protected MigrationService createMigrationService() { 10_000, getRestClient(), ContentType.parse(getConfig().getDefaultContentType()), - getConfig().getEncoding()); + getConfig().getEncoding(), + getConfig().getValidateOnMigrate()); } } diff --git a/elasticsearch-evolution-core/src/main/java/com/senacor/elasticsearch/evolution/core/api/config/ElasticsearchEvolutionConfig.java b/elasticsearch-evolution-core/src/main/java/com/senacor/elasticsearch/evolution/core/api/config/ElasticsearchEvolutionConfig.java index 03e410db..33facad4 100644 --- a/elasticsearch-evolution-core/src/main/java/com/senacor/elasticsearch/evolution/core/api/config/ElasticsearchEvolutionConfig.java +++ b/elasticsearch-evolution-core/src/main/java/com/senacor/elasticsearch/evolution/core/api/config/ElasticsearchEvolutionConfig.java @@ -83,6 +83,11 @@ public class ElasticsearchEvolutionConfig { */ private int historyMaxQuerySize = 1_000; + /** + * Whether to fail when a previously applied migration script has been modified after it was applied. + */ + private boolean validateOnMigrate = true; + /** * Loads this configuration into a new ElasticsearchEvolution instance. * @@ -236,6 +241,15 @@ public ElasticsearchEvolutionConfig setHistoryMaxQuerySize(int historyMaxQuerySi return this; } + public boolean getValidateOnMigrate() { + return validateOnMigrate; + } + + public ElasticsearchEvolutionConfig setValidateOnMigrate(boolean validateOnMigrate) { + this.validateOnMigrate = validateOnMigrate; + return this; + } + @Override public String toString() { return "ElasticsearchEvolutionConfig{" + @@ -251,6 +265,7 @@ public String toString() { ", placeholderReplacement=" + placeholderReplacement + ", historyIndex='" + historyIndex + '\'' + ", historyMaxQuerySize=" + historyMaxQuerySize + + ", validateOnMigrate=" + validateOnMigrate + '}'; } } diff --git a/elasticsearch-evolution-core/src/main/java/com/senacor/elasticsearch/evolution/core/internal/migration/execution/MigrationServiceImpl.java b/elasticsearch-evolution-core/src/main/java/com/senacor/elasticsearch/evolution/core/internal/migration/execution/MigrationServiceImpl.java index 5dbfd737..599e8261 100644 --- a/elasticsearch-evolution-core/src/main/java/com/senacor/elasticsearch/evolution/core/internal/migration/execution/MigrationServiceImpl.java +++ b/elasticsearch-evolution-core/src/main/java/com/senacor/elasticsearch/evolution/core/internal/migration/execution/MigrationServiceImpl.java @@ -37,17 +37,20 @@ public class MigrationServiceImpl implements MigrationService { private final RestClient restClient; private final ContentType defaultContentType; private final Charset encoding; + private final boolean validateOnMigrate; public MigrationServiceImpl(HistoryRepository historyRepository, int waitUntilUnlockedMinTimeInMillis, int waitUntilUnlockedMaxTimeInMillis, RestClient restClient, ContentType defaultContentType, - Charset encoding) { + Charset encoding, + boolean validateOnMigrate) { this.historyRepository = requireNonNull(historyRepository, "historyRepository must not be null"); this.restClient = requireNonNull(restClient, "restClient must not be null"); this.defaultContentType = requireNonNull(defaultContentType); this.encoding = requireNonNull(encoding); + this.validateOnMigrate = validateOnMigrate; this.waitUntilUnlockedMinTimeInMillis = requireCondition(waitUntilUnlockedMinTimeInMillis, min -> min >= 0 && min <= waitUntilUnlockedMaxTimeInMillis, "waitUntilUnlockedMinTimeInMillis (%s) must not be negative and must not be greater than waitUntilUnlockedMaxTimeInMillis (%s)", @@ -185,9 +188,20 @@ List getPendingScriptsToBeExecuted(Collection()).when(historyRepository).findAll(); MigrationServiceImpl underTest = new MigrationServiceImpl(historyRepository, - 0, 0, restClient, defaultContentType, encoding); + 0, 0, restClient, defaultContentType, encoding, true); ParsedMigrationScript parsedMigrationScript1_1 = createParsedMigrationScript("1.1"); ParsedMigrationScript parsedMigrationScript1_0 = createParsedMigrationScript("1.0"); List parsedMigrationScripts = asList( @@ -121,7 +121,7 @@ void scriptsAndHistoryInSync_noScriptsWillBeReturned() { createMigrationScriptProtocol("1.1", true) ))).when(historyRepository).findAll(); MigrationServiceImpl underTest = new MigrationServiceImpl(historyRepository, - 0, 0, restClient, defaultContentType, encoding); + 0, 0, restClient, defaultContentType, encoding, true); List parsedMigrationScripts = asList( createParsedMigrationScript("1.1"), @@ -142,7 +142,7 @@ void lastHistoryVersionWasFailing_AllScriptsInclFailedWillBeReturned() { createMigrationScriptProtocol("1.1", false) ))).when(historyRepository).findAll(); MigrationServiceImpl underTest = new MigrationServiceImpl(historyRepository, - 0, 0, restClient, defaultContentType, encoding); + 0, 0, restClient, defaultContentType, encoding, true); ParsedMigrationScript parsedMigrationScript1_0 = createParsedMigrationScript("1.0"); ParsedMigrationScript parsedMigrationScript1_1 = createParsedMigrationScript("1.1"); @@ -167,7 +167,7 @@ void moreHistoryVersionsThanScripts_warningIsShownAnNoScriptsWillBeReturned() { createMigrationScriptProtocol("1.2", false) ))).when(historyRepository).findAll(); MigrationServiceImpl underTest = new MigrationServiceImpl(historyRepository, - 0, 0, restClient, defaultContentType, encoding); + 0, 0, restClient, defaultContentType, encoding, true); ParsedMigrationScript parsedMigrationScript1_0 = createParsedMigrationScript("1.0"); ParsedMigrationScript parsedMigrationScript1_1 = createParsedMigrationScript("1.1"); @@ -190,7 +190,7 @@ void outOfOrderExecutionIsNotSupported() { createMigrationScriptProtocol("1.1", true) ))).when(historyRepository).findAll(); MigrationServiceImpl underTest = new MigrationServiceImpl(historyRepository, - 0, 0, restClient, defaultContentType, encoding); + 0, 0, restClient, defaultContentType, encoding, true); ParsedMigrationScript parsedMigrationScript1_0 = createParsedMigrationScript("1.0"); ParsedMigrationScript parsedMigrationScript1_0_1 = createParsedMigrationScript("1.0.1"); @@ -204,6 +204,79 @@ void outOfOrderExecutionIsNotSupported() { .isInstanceOf(MigrationException.class) .hasMessage("The logged execution in the Elasticsearch-Evolution history index at position 1 is version 1.1 and in the same position in the given migration scripts is version 1.0.1! Out of order execution is not supported. Or maybe you have added new migration scripts in between or have to cleanup the Elasticsearch-Evolution history index manually"); } + + @Test + void failingScriptWasEdited_shouldReturnAllScriptsInclFailing() { + doReturn(new TreeSet<>(asList( + createMigrationScriptProtocol("1.0", true, 1), + createMigrationScriptProtocol("1.1", false, 2) + ))).when(historyRepository).findAll(); + MigrationServiceImpl underTest = new MigrationServiceImpl(historyRepository, + 0, 0, restClient, defaultContentType, encoding, true); + + ParsedMigrationScript parsedMigrationScript1_0 = createParsedMigrationScript("1.0", 1); + ParsedMigrationScript parsedMigrationScript1_1 = createParsedMigrationScript("1.1", 3); + ParsedMigrationScript parsedMigrationScript1_2 = createParsedMigrationScript("1.2", 4); + List parsedMigrationScripts = asList( + parsedMigrationScript1_1, + parsedMigrationScript1_0, + parsedMigrationScript1_2); + + List res = underTest.getPendingScriptsToBeExecuted(parsedMigrationScripts); + + assertThat(res).hasSize(2); + assertThat(res.get(0)).isSameAs(parsedMigrationScript1_1); + assertThat(res.get(1)).isSameAs(parsedMigrationScript1_2); + InOrder order = inOrder(historyRepository); + order.verify(historyRepository).findAll(); + order.verifyNoMoreInteractions(); + } + + @Test + void successfulScriptWasEdited_shouldThrowChecksumMismatchException() { + doReturn(new TreeSet<>(asList( + createMigrationScriptProtocol("1.0", true, 1), + createMigrationScriptProtocol("1.1", true, 2) + ))).when(historyRepository).findAll(); + MigrationServiceImpl underTest = new MigrationServiceImpl(historyRepository, + 0, 0, restClient, defaultContentType, encoding, true); + + ParsedMigrationScript parsedMigrationScript1_0 = createParsedMigrationScript("1.0", 1); + ParsedMigrationScript parsedMigrationScript1_1 = createParsedMigrationScript("1.1", 3); + List parsedMigrationScripts = asList( + parsedMigrationScript1_1, + parsedMigrationScript1_0); + + assertThatThrownBy(() -> underTest.getPendingScriptsToBeExecuted(parsedMigrationScripts)) + .isInstanceOf(MigrationException.class) + .hasMessage("The logged execution for the migration script at position 1 (V1.1__1.1.http) " + + "has a different checksum from the given migration script! " + + "Modifying already-executed scripts is not supported."); + } + + @Test + void successfulScriptWasEdited_shouldContinueIfValidateOnMigrateIsDisabled() { + doReturn(new TreeSet<>(asList( + createMigrationScriptProtocol("1.0", true, 1), + createMigrationScriptProtocol("1.1", true, 2) + ))).when(historyRepository).findAll(); + MigrationServiceImpl underTest = new MigrationServiceImpl(historyRepository, + 0, 0, restClient, defaultContentType, encoding, false); + + ParsedMigrationScript parsedMigrationScript1_0 = createParsedMigrationScript("1.0", 1); + ParsedMigrationScript parsedMigrationScript1_1 = createParsedMigrationScript("1.1", 3); + List parsedMigrationScripts = asList( + parsedMigrationScript1_1, + parsedMigrationScript1_0); + + List res = underTest.getPendingScriptsToBeExecuted(parsedMigrationScripts); + + assertThat(res).hasSize(0); + assertThat(res).isEmpty(); + InOrder order = inOrder(historyRepository); + order.verify(historyRepository).findAll(); + order.verifyNoMoreInteractions(); + } } @Nested @@ -215,7 +288,7 @@ void OK_resultIsSetCorrect() throws IOException { doReturn(responseMock).when(restClient).performRequest(any()); MigrationServiceImpl underTest = new MigrationServiceImpl(historyRepository, - 0, 0, restClient, defaultContentType, encoding); + 0, 0, restClient, defaultContentType, encoding, true); MigrationScriptProtocol res = underTest.executeScript(script).getProtocol(); @@ -271,7 +344,7 @@ void OK_requestWithBody() throws IOException { doReturn(responseMock).when(restClient).performRequest(any()); MigrationServiceImpl underTest = new MigrationServiceImpl(historyRepository, - 0, 0, restClient, defaultContentType, encoding); + 0, 0, restClient, defaultContentType, encoding, true); MigrationScriptProtocol res = underTest.executeScript(script).getProtocol(); @@ -311,7 +384,7 @@ void OK_requestWithCustomContentTypeAndDefaultCharset() throws IOException { doReturn(responseMock).when(restClient).performRequest(any()); MigrationServiceImpl underTest = new MigrationServiceImpl(historyRepository, - 0, 0, restClient, defaultContentType, encoding); + 0, 0, restClient, defaultContentType, encoding, true); MigrationScriptProtocol res = underTest.executeScript(script).getProtocol(); @@ -350,7 +423,7 @@ void OK_requestWithCustomContentTypeAndCustomCharset() throws IOException { doReturn(responseMock).when(restClient).performRequest(any()); MigrationServiceImpl underTest = new MigrationServiceImpl(historyRepository, - 0, 0, restClient, defaultContentType, encoding); + 0, 0, restClient, defaultContentType, encoding, true); MigrationScriptProtocol res = underTest.executeScript(script).getProtocol(); @@ -390,7 +463,7 @@ void OK_requestWithCustomHeader() throws IOException { doReturn(responseMock).when(restClient).performRequest(any()); MigrationServiceImpl underTest = new MigrationServiceImpl(historyRepository, - 0, 0, restClient, defaultContentType, encoding); + 0, 0, restClient, defaultContentType, encoding, true); MigrationScriptProtocol res = underTest.executeScript(script).getProtocol(); @@ -426,7 +499,7 @@ void executeScript_failed_status(Exception handledError) throws IOException { doThrow(handledError).when(restClient).performRequest(any()); MigrationServiceImpl underTest = new MigrationServiceImpl(historyRepository, - 0, 0, restClient, defaultContentType, encoding); + 0, 0, restClient, defaultContentType, encoding, true); ExecutionResult res = underTest.executeScript(script); @@ -445,7 +518,7 @@ void executeScript_failed_status(int httpStatusCode) throws IOException { doReturn(responseMock).when(restClient).performRequest(any()); MigrationServiceImpl underTest = new MigrationServiceImpl(historyRepository, - 0, 0, restClient, defaultContentType, encoding); + 0, 0, restClient, defaultContentType, encoding, true); ExecutionResult res = underTest.executeScript(script); @@ -467,7 +540,7 @@ void executeScript_OK_status(int httpStatusCode) throws IOException { doReturn(responseMock).when(restClient).performRequest(any()); MigrationServiceImpl underTest = new MigrationServiceImpl(historyRepository, - 0, 0, restClient, defaultContentType, encoding); + 0, 0, restClient, defaultContentType, encoding, true); MigrationScriptProtocol res = underTest.executeScript(script).getProtocol(); @@ -490,7 +563,7 @@ void allOK() throws IOException { Response responseMock = createResponseMock(200); doReturn(responseMock).when(restClient).performRequest(any()); MigrationServiceImpl underTest = new MigrationServiceImpl(historyRepository, - 0, 0, restClient, defaultContentType, encoding); + 0, 0, restClient, defaultContentType, encoding, true); List res = underTest.executePendingScripts(scripts); @@ -523,7 +596,7 @@ void firstExecutionFailed() throws IOException { Response responseMock = createResponseMock(statusCode); doReturn(responseMock).when(restClient).performRequest(any()); MigrationServiceImpl underTest = new MigrationServiceImpl(historyRepository, - 0, 0, restClient, defaultContentType, encoding); + 0, 0, restClient, defaultContentType, encoding, true); assertThatThrownBy(() -> underTest.executePendingScripts(scripts)) .isInstanceOf(MigrationException.class) @@ -556,7 +629,7 @@ void error_unlockWasNotSuccessful() throws IOException { Response responseMock = createResponseMock(200); doReturn(responseMock).when(restClient).performRequest(any()); MigrationServiceImpl underTest = new MigrationServiceImpl(historyRepository, - 0, 0, restClient, defaultContentType, encoding); + 0, 0, restClient, defaultContentType, encoding, true); assertThatThrownBy(() -> underTest.executePendingScripts(scripts)) .isInstanceOf(MigrationException.class) @@ -585,7 +658,7 @@ void error_lockWasNotSuccessful() { doReturn(true).when(historyRepository).unlock(); MigrationServiceImpl underTest = new MigrationServiceImpl(historyRepository, - 0, 0, restClient, defaultContentType, encoding); + 0, 0, restClient, defaultContentType, encoding, true); assertThatThrownBy(() -> underTest.executePendingScripts(scripts)) .isInstanceOf(MigrationException.class) @@ -603,7 +676,7 @@ void error_lockWasNotSuccessful() { @Test void emptyScriptsCollection() { MigrationServiceImpl underTest = new MigrationServiceImpl(historyRepository, - 0, 0, restClient, defaultContentType, encoding); + 0, 0, restClient, defaultContentType, encoding, true); List res = underTest.executePendingScripts(emptyList()); @@ -622,9 +695,13 @@ private Response createResponseMock(int statusCode) { } private MigrationScriptProtocol createMigrationScriptProtocol(String version, boolean success) { + return createMigrationScriptProtocol(version, success, 1); + } + + private MigrationScriptProtocol createMigrationScriptProtocol(String version, boolean success, int checksum) { return new MigrationScriptProtocol() .setVersion(version) - .setChecksum(1) + .setChecksum(checksum) .setSuccess(success) .setLocked(true) .setDescription(version) @@ -632,13 +709,17 @@ private MigrationScriptProtocol createMigrationScriptProtocol(String version, bo } private ParsedMigrationScript createParsedMigrationScript(String version) { + return createParsedMigrationScript(version, 1); + } + + private ParsedMigrationScript createParsedMigrationScript(String version, int checksum) { return new ParsedMigrationScript() .setFileNameInfo( new FileNameInfoImpl(fromVersion(version), version, createDefaultScriptName(version))) - .setChecksum(1) + .setChecksum(checksum) .setMigrationScriptRequest(new MigrationScriptRequest() - .setHttpMethod(DELETE) - .setPath("/")); + .setHttpMethod(DELETE) + .setPath("/")); } private String createDefaultScriptName(String version) {