Skip to content

Commit

Permalink
[Remote Store] Fix flaky test - closes opensearch-project#7703 (opens…
Browse files Browse the repository at this point in the history
…earch-project#7879)

Signed-off-by: Ashish Singh <ssashish@amazon.com>
  • Loading branch information
ashking94 authored Jun 2, 2023
1 parent ecd0879 commit 5d5e8ad
Show file tree
Hide file tree
Showing 3 changed files with 12 additions and 5 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ protected void deleteRepo() {
assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_NAME));
}

protected void setup(Path repoLocation, double ioFailureRate, String skipExceptionBlobList) {
protected void setup(Path repoLocation, double ioFailureRate, String skipExceptionBlobList, long maxFailure) {
logger.info("--> Creating repository={} at the path={}", REPOSITORY_NAME, repoLocation);
// The random_control_io_exception_rate setting ensures that 10-25% of all operations to remote store results in
/// IOException. skip_exception_on_verification_file & skip_exception_on_list_blobs settings ensures that the
Expand All @@ -85,7 +85,7 @@ protected void setup(Path repoLocation, double ioFailureRate, String skipExcepti
.put("skip_exception_on_list_blobs", true)
// Skipping is required for metadata as it is part of recovery
.put("skip_exception_on_blobs", skipExceptionBlobList)
.put("max_failure_number", Long.MAX_VALUE)
.put("max_failure_number", maxFailure)
);

internalCluster().startDataOnlyNodes(1);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ public class RemoteStoreBackpressureIT extends AbstractRemoteStoreMockRepository

public void testWritesRejected() {
Path location = randomRepoPath().toAbsolutePath();
setup(location, 1d, "metadata");
setup(location, 1d, "metadata", Long.MAX_VALUE);

Settings request = Settings.builder().put(REMOTE_REFRESH_SEGMENT_PRESSURE_ENABLED.getKey(), true).build();
ClusterUpdateSettingsResponse clusterUpdateResponse = client().admin()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@
import org.opensearch.test.OpenSearchIntegTestCase;

import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.concurrent.TimeUnit;
Expand All @@ -27,7 +30,7 @@ public class RemoteStoreRefreshListenerIT extends AbstractRemoteStoreMockReposit
public void testRemoteRefreshRetryOnFailure() throws Exception {

Path location = randomRepoPath().toAbsolutePath();
setup(location, randomDoubleBetween(0.1, 0.25, true), "metadata");
setup(location, randomDoubleBetween(0.1, 0.15, true), "metadata", 10L);

// Here we are having flush/refresh after each iteration of indexing. However, the refresh will not always succeed
// due to IOExceptions that are thrown while doing uploadBlobs.
Expand All @@ -47,8 +50,12 @@ public void testRemoteRefreshRetryOnFailure() throws Exception {
assertBusy(() -> {
Set<String> filesInLocal = getSegmentFiles(location.getRoot().resolve(segmentDataLocalPath));
Set<String> filesInRepo = getSegmentFiles(segmentDataRepoPath);
List<String> sortedFilesInLocal = new ArrayList<>(filesInLocal), sortedFilesInRepo = new ArrayList<>(filesInRepo);
Collections.sort(sortedFilesInLocal);
Collections.sort(sortedFilesInRepo);
logger.info("Local files = {}, Repo files = {}", sortedFilesInLocal, sortedFilesInRepo);
assertTrue(filesInRepo.containsAll(filesInLocal));
}, 60, TimeUnit.SECONDS);
}, 90, TimeUnit.SECONDS);
deleteRepo();
}

Expand Down

0 comments on commit 5d5e8ad

Please sign in to comment.