From 5d5e8ad4f7fa8bfb15fedf7e4f3abb78f18e032c Mon Sep 17 00:00:00 2001 From: Ashish Date: Fri, 2 Jun 2023 12:30:24 +0530 Subject: [PATCH] [Remote Store] Fix flaky test - closes #7703 (#7879) Signed-off-by: Ashish Singh --- ...bstractRemoteStoreMockRepositoryIntegTestCase.java | 4 ++-- .../remotestore/RemoteStoreBackpressureIT.java | 2 +- .../remotestore/RemoteStoreRefreshListenerIT.java | 11 +++++++++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java index dc312ffa6676d..2bcbf3f5b614d 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java @@ -70,7 +70,7 @@ protected void deleteRepo() { assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_NAME)); } - protected void setup(Path repoLocation, double ioFailureRate, String skipExceptionBlobList) { + protected void setup(Path repoLocation, double ioFailureRate, String skipExceptionBlobList, long maxFailure) { logger.info("--> Creating repository={} at the path={}", REPOSITORY_NAME, repoLocation); // The random_control_io_exception_rate setting ensures that 10-25% of all operations to remote store results in /// IOException. skip_exception_on_verification_file & skip_exception_on_list_blobs settings ensures that the @@ -85,7 +85,7 @@ protected void setup(Path repoLocation, double ioFailureRate, String skipExcepti .put("skip_exception_on_list_blobs", true) // Skipping is required for metadata as it is part of recovery .put("skip_exception_on_blobs", skipExceptionBlobList) - .put("max_failure_number", Long.MAX_VALUE) + .put("max_failure_number", maxFailure) ); internalCluster().startDataOnlyNodes(1); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java index 10169c969015f..c46eab6468c6b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java @@ -29,7 +29,7 @@ public class RemoteStoreBackpressureIT extends AbstractRemoteStoreMockRepository public void testWritesRejected() { Path location = randomRepoPath().toAbsolutePath(); - setup(location, 1d, "metadata"); + setup(location, 1d, "metadata", Long.MAX_VALUE); Settings request = Settings.builder().put(REMOTE_REFRESH_SEGMENT_PRESSURE_ENABLED.getKey(), true).build(); ClusterUpdateSettingsResponse clusterUpdateResponse = client().admin() diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRefreshListenerIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRefreshListenerIT.java index eb95c2a270d1a..4005e6359a2f7 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRefreshListenerIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRefreshListenerIT.java @@ -15,6 +15,9 @@ import org.opensearch.test.OpenSearchIntegTestCase; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Locale; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -27,7 +30,7 @@ public class RemoteStoreRefreshListenerIT extends AbstractRemoteStoreMockReposit public void testRemoteRefreshRetryOnFailure() throws Exception { Path location = randomRepoPath().toAbsolutePath(); - setup(location, randomDoubleBetween(0.1, 0.25, true), "metadata"); + setup(location, randomDoubleBetween(0.1, 0.15, true), "metadata", 10L); // Here we are having flush/refresh after each iteration of indexing. However, the refresh will not always succeed // due to IOExceptions that are thrown while doing uploadBlobs. @@ -47,8 +50,12 @@ public void testRemoteRefreshRetryOnFailure() throws Exception { assertBusy(() -> { Set filesInLocal = getSegmentFiles(location.getRoot().resolve(segmentDataLocalPath)); Set filesInRepo = getSegmentFiles(segmentDataRepoPath); + List sortedFilesInLocal = new ArrayList<>(filesInLocal), sortedFilesInRepo = new ArrayList<>(filesInRepo); + Collections.sort(sortedFilesInLocal); + Collections.sort(sortedFilesInRepo); + logger.info("Local files = {}, Repo files = {}", sortedFilesInLocal, sortedFilesInRepo); assertTrue(filesInRepo.containsAll(filesInLocal)); - }, 60, TimeUnit.SECONDS); + }, 90, TimeUnit.SECONDS); deleteRepo(); }