Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rewrite SnapshotLifecycleIT as as ESIntegTestCase #46356

Merged
merged 2 commits into from
Sep 5, 2019
Merged

Rewrite SnapshotLifecycleIT as as ESIntegTestCase #46356

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5701c2c
Rewrite SnapshotLifecycleIT as as ESIntegTestCase
dakrone Sep 3, 2019
6f56d7c
Add error logging when exceptions are thrown
dakrone Sep 5, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions ...ugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyItem.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,22 @@ public static SnapshotInProgress fromEntry(SnapshotsInProgress.Entry entry) {
entry.state(), entry.startTime(), entry.failure());
}

public SnapshotId getSnapshotId() {
return snapshotId;
}

public SnapshotsInProgress.State getState() {
return state;
}

public long getStartTime() {
return startTime;
}

public String getFailure() {
return failure;
}

@Override
public void writeTo(StreamOutput out) throws IOException {
this.snapshotId.writeTo(out);
Expand Down
4 changes: 4 additions & 0 deletions ...ore/src/main/java/org/elasticsearch/xpack/core/slm/action/GetSnapshotLifecycleAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,10 @@ public Response(StreamInput in) throws IOException {
this.lifecycles = in.readList(SnapshotLifecyclePolicyItem::new);
}

public List<SnapshotLifecyclePolicyItem> getPolicies() {
return this.lifecycles;
}

@Override
public String toString() {
return Strings.toString(this);
Expand Down
1 change: 0 additions & 1 deletion x-pack/plugin/ilm/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,3 @@ gradle.projectsEvaluated {
}

integTest.enabled = false

4 changes: 0 additions & 4 deletions x-pack/plugin/ilm/qa/multi-node/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,4 @@ testClusters.integTest {
setting 'xpack.ml.enabled', 'false'
setting 'xpack.license.self_generated.type', 'trial'
setting 'indices.lifecycle.poll_interval', '1000ms'
// TODO: Find a way to run these tests with more than one snapshot pool thread. Currently we need to limit to one thread so that the
// rate limiting settings in SnapshotLifecycleIT doesn't result in blocked snapshot threads because multiple threads overshoot
// the limit simultaneously and the rate limiter then moves to wait minutes to make up for this.
setting 'thread_pool.snapshot.max', '1'
}
183 changes: 1 addition & 182 deletions ...search/xpack/slm/SnapshotLifecycleIT.java → ...ch/xpack/slm/SnapshotLifecycleRestIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
import org.apache.http.util.EntityUtils;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
Expand Down Expand Up @@ -41,23 +40,20 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.TimeUnit;

import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.xpack.core.slm.history.SnapshotHistoryItem.CREATE_OPERATION;
import static org.elasticsearch.xpack.core.slm.history.SnapshotHistoryItem.DELETE_OPERATION;
import static org.elasticsearch.xpack.core.slm.history.SnapshotHistoryStore.SLM_HISTORY_INDEX_PREFIX;
import static org.elasticsearch.xpack.ilm.TimeSeriesLifecycleActionsIT.getStepKeyForIndex;
import static org.hamcrest.Matchers.anyOf;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.startsWith;

@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/46205")
public class SnapshotLifecycleIT extends ESRestTestCase {
public class SnapshotLifecycleRestIT extends ESRestTestCase {

@Override
protected boolean waitForAllSnapshotsWiped() {
Expand Down Expand Up @@ -335,183 +331,6 @@ public void testBasicTimeBasedRetenion() throws Exception {
}
}

@SuppressWarnings("unchecked")
public void testSnapshotInProgress() throws Exception {
final String indexName = "test";
final String policyName = "test-policy";
final String repoId = "my-repo";
int docCount = 20;
for (int i = 0; i < docCount; i++) {
index(client(), indexName, "" + i, "foo", "bar");
}

// Create a snapshot repo
initializeRepo(repoId, "1b");

createSnapshotPolicy(policyName, "snap", "1 2 3 4 5 ?", repoId, indexName, true);

Response executeRepsonse = client().performRequest(new Request("PUT", "/_slm/policy/" + policyName + "/_execute"));

try (XContentParser parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, EntityUtils.toByteArray(executeRepsonse.getEntity()))) {
final String snapshotName = parser.mapStrings().get("snapshot_name");

// Check that the executed snapshot shows up in the SLM output
assertBusy(() -> {
try {
Response response = client().performRequest(new Request("GET", "/_slm/policy" + (randomBoolean() ? "" : "?human")));
Map<String, Object> policyResponseMap;
try (InputStream content = response.getEntity().getContent()) {
policyResponseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), content, true);
}
assertThat(policyResponseMap.size(), greaterThan(0));
Optional<Map<String, Object>> inProgress = Optional.ofNullable((Map<String, Object>) policyResponseMap.get(policyName))
.map(policy -> (Map<String, Object>) policy.get("in_progress"));

if (inProgress.isPresent()) {
Map<String, Object> inProgressMap = inProgress.get();
assertThat(inProgressMap.get("name"), equalTo(snapshotName));
assertNotNull(inProgressMap.get("uuid"));
assertThat(inProgressMap.get("state"), equalTo("STARTED"));
assertThat((long) inProgressMap.get("start_time_millis"), greaterThan(0L));
assertNull(inProgressMap.get("failure"));
} else {
fail("expected in_progress to contain a running snapshot, but the response was " + policyResponseMap);
}
} catch (ResponseException e) {
fail("expected policy to exist but it does not: " + EntityUtils.toString(e.getResponse().getEntity()));
}
});

// Cancel the snapshot since it is not going to complete quickly
try {
client().performRequest(new Request("DELETE", "/_snapshot/" + repoId + "/" + snapshotName));
} catch (Exception e) {
// ignore
}
}
}

@SuppressWarnings("unchecked")
public void testRetentionWhileSnapshotInProgress() throws Exception {
final String indexName = "test";
final String slowPolicy = "slow";
final String fastPolicy = "fast";
final String slowRepo = "slow-repo";
final String fastRepo = "fast-repo";
int docCount = 20;
for (int i = 0; i < docCount; i++) {
index(client(), indexName, "" + i, "foo", "bar");
}

// Create snapshot repos, one fast and one slow
initializeRepo(slowRepo, "1b");
initializeRepo(fastRepo, "10mb");

createSnapshotPolicy(slowPolicy, "slow-snap", "1 2 3 4 5 ?", slowRepo, indexName, true,
new SnapshotRetentionConfiguration(TimeValue.timeValueSeconds(0), null, null));
createSnapshotPolicy(fastPolicy, "fast-snap", "1 2 3 4 5 ?", fastRepo, indexName, true,
new SnapshotRetentionConfiguration(TimeValue.timeValueSeconds(0), null, null));

// Create a snapshot and wait for it to be complete (need something that can be deleted)
final String completedSnapshotName = executePolicy(fastPolicy);
assertBusy(() -> {
try {
Response getResp = client().performRequest(new Request("GET",
"/_snapshot/" + fastRepo + "/" + completedSnapshotName + "/_status"));
try (InputStream content = getResp.getEntity().getContent()) {
Map<String, Object> snaps = XContentHelper.convertToMap(XContentType.JSON.xContent(), content, true);
logger.info("--> waiting for snapshot {} to be successful, got: {}", completedSnapshotName, snaps);
List<Map<String, Object>> snaps2 = (List<Map<String, Object>>) snaps.get("snapshots");
assertThat(snaps2.get(0).get("state"), equalTo("SUCCESS"));

// Check that no in_progress snapshots show up
Response response = client().performRequest(new Request("GET", "/_slm/policy"));
Map<String, Object> policyResponseMap;
try (InputStream content2 = response.getEntity().getContent()) {
policyResponseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), content2, true);
}
assertThat(policyResponseMap.size(), greaterThan(0));
Optional<Map<String, Object>> inProgress = Optional.ofNullable((Map<String, Object>) policyResponseMap.get(slowPolicy))
.map(policy -> (Map<String, Object>) policy.get("in_progress"));

// Ensure no snapshots are running
assertFalse("expected no in progress snapshots but got " + inProgress.orElse(null), inProgress.isPresent());
}
} catch (NullPointerException | ResponseException e) {
fail("unable to retrieve completed snapshot: " + e);
}
}, 60, TimeUnit.SECONDS);

// Take another snapshot
final String slowSnapshotName = executePolicy(slowPolicy);

// Check that the executed snapshot shows up in the SLM output as in_progress
assertBusy(() -> {
try {
Response response = client().performRequest(new Request("GET", "/_slm/policy"));
Map<String, Object> policyResponseMap;
try (InputStream content = response.getEntity().getContent()) {
policyResponseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), content, true);
}
logger.info("--> checking for 'slow-*' snapshot to show up in policy response, got: " + policyResponseMap);
assertThat(policyResponseMap.size(), greaterThan(0));
Optional<Map<String, Object>> inProgress = Optional.ofNullable((Map<String, Object>) policyResponseMap.get(slowPolicy))
.map(policy -> (Map<String, Object>) policy.get("in_progress"));

if (inProgress.isPresent()) {
Map<String, Object> inProgressMap = inProgress.get();
assertThat(inProgressMap.get("name"), equalTo(slowSnapshotName));
assertNotNull(inProgressMap.get("uuid"));
assertThat(inProgressMap.get("state"), anyOf(equalTo("STARTED"), equalTo("INIT")));
assertThat((long) inProgressMap.get("start_time_millis"), greaterThan(0L));
assertNull(inProgressMap.get("failure"));
} else {
fail("expected in_progress to contain a running snapshot, but the response was " + policyResponseMap);
}
} catch (ResponseException e) {
fail("expected policy to exist but it does not: " + EntityUtils.toString(e.getResponse().getEntity()));
}
}, 60, TimeUnit.SECONDS);

// Run retention every second
ClusterUpdateSettingsRequest req = new ClusterUpdateSettingsRequest();
req.transientSettings(Settings.builder().put(LifecycleSettings.SLM_RETENTION_SCHEDULE, "*/1 * * * * ?"));
try (XContentBuilder builder = jsonBuilder()) {
req.toXContent(builder, ToXContent.EMPTY_PARAMS);
Request r = new Request("PUT", "/_cluster/settings");
r.setJsonEntity(Strings.toString(builder));
client().performRequest(r);
}

// Cancel the snapshot since it is not going to complete quickly, do it in a thread because
// cancelling the snapshot can take a long time and we might as well check retention while
// its deleting
Thread t = new Thread(() -> {
try {
assertOK(client().performRequest(new Request("DELETE", "/_snapshot/" + slowRepo + "/" + slowSnapshotName)));
} catch (IOException e) {
fail("should not have thrown " + e);
}
});
t.start();

// Check that the snapshot created by the policy has been removed by retention
assertBusy(() -> {
// We expect a failed response because the snapshot should not exist
try {
Response response = client().performRequest(new Request("GET", "/_snapshot/" + slowRepo + "/" + completedSnapshotName));
String resp = EntityUtils.toString(response.getEntity());
logger.info("--> checking to see if snapshot has been deleted, got: " + resp);
assertThat(resp, containsString("snapshot_missing_exception"));
} catch (ResponseException e) {
assertThat(EntityUtils.toString(e.getResponse().getEntity()), containsString("snapshot_missing_exception"));
}
}, 60, TimeUnit.SECONDS);

t.join(5000);
}

/**
* Execute the given policy and return the generated snapshot name
*/
Expand Down
Loading