From 3c1f33124259c56e78cad4b7e916fdd3ac615f04 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Mon, 15 Jul 2019 13:40:01 -0600 Subject: [PATCH] Add SnapshotRetentionConfiguration for retention configuration (#43777) * Add SnapshotRetentionConfiguration for retention configuration This commit adds the `SnapshotRetentionConfiguration` class and its HLRC counterpart to encapsulate the configuration for SLM retention. Currently only a single parameter is supported as an example (we still need to discuss the different options we want to support and their names) to keep the size of the PR down. It also does not yet include version serialization checks since the original SLM branch has not yet been merged. Relates to #43663 * Fix REST tests * Fix more documentation * Use Objects.equals to avoid NPE * Put `randomSnapshotLifecyclePolicy` in only one place * Occasionally return retention with no configuration --- .../SnapshotLifecyclePolicy.java | 20 +++- .../SnapshotRetentionConfiguration.java | 95 +++++++++++++++++++ .../documentation/ILMDocumentationIT.java | 6 +- docs/reference/ilm/apis/slm-api.asciidoc | 14 ++- .../ilm/getting-started-slm.asciidoc | 6 +- .../elasticsearch/common/unit/TimeValue.java | 8 ++ .../SnapshotLifecyclePolicy.java | 22 ++++- .../SnapshotRetentionConfiguration.java | 95 +++++++++++++++++++ .../SnapshotLifecyclePolicyItemTests.java | 4 +- .../SnapshotLifecyclePolicyMetadataTests.java | 17 +++- .../history/SnapshotHistoryStoreTests.java | 20 +--- .../SnapshotLifecycleIT.java | 6 +- .../xpack/security/PermissionsIT.java | 3 +- .../SnapshotLifecyclePolicyTests.java | 74 ++++++++------- .../SnapshotLifecycleServiceTests.java | 4 +- 15 files changed, 312 insertions(+), 82 deletions(-) create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/snapshotlifecycle/SnapshotRetentionConfiguration.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotRetentionConfiguration.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshotlifecycle/SnapshotLifecyclePolicy.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshotlifecycle/SnapshotLifecyclePolicy.java index 8d8e78184ff59..12351121323ea 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshotlifecycle/SnapshotLifecyclePolicy.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshotlifecycle/SnapshotLifecyclePolicy.java @@ -38,11 +38,13 @@ public class SnapshotLifecyclePolicy implements ToXContentObject { private final String schedule; private final String repository; private final Map configuration; + private final SnapshotRetentionConfiguration retentionPolicy; private static final ParseField NAME = new ParseField("name"); private static final ParseField SCHEDULE = new ParseField("schedule"); private static final ParseField REPOSITORY = new ParseField("repository"); private static final ParseField CONFIG = new ParseField("config"); + private static final ParseField RETENTION = new ParseField("retention"); private static final IndexNameExpressionResolver.DateMathExpressionResolver DATE_MATH_RESOLVER = new IndexNameExpressionResolver.DateMathExpressionResolver(); @@ -54,7 +56,8 @@ public class SnapshotLifecyclePolicy implements ToXContentObject { String schedule = (String) a[1]; String repo = (String) a[2]; Map config = (Map) a[3]; - return new SnapshotLifecyclePolicy(id, name, schedule, repo, config); + SnapshotRetentionConfiguration retention = (SnapshotRetentionConfiguration) a[4]; + return new SnapshotLifecyclePolicy(id, name, schedule, repo, config, retention); }); static { @@ -62,15 +65,18 @@ public class SnapshotLifecyclePolicy implements ToXContentObject { PARSER.declareString(ConstructingObjectParser.constructorArg(), SCHEDULE); PARSER.declareString(ConstructingObjectParser.constructorArg(), REPOSITORY); PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> p.map(), CONFIG); + PARSER.declareObject(ConstructingObjectParser.constructorArg(), SnapshotRetentionConfiguration::parse, RETENTION); } public SnapshotLifecyclePolicy(final String id, final String name, final String schedule, - final String repository, Map configuration) { + final String repository, Map configuration, + SnapshotRetentionConfiguration retentionPolicy) { this.id = Objects.requireNonNull(id); this.name = name; this.schedule = schedule; this.repository = repository; this.configuration = configuration; + this.retentionPolicy = retentionPolicy; } public String getId() { @@ -93,6 +99,10 @@ public Map getConfig() { return this.configuration; } + public SnapshotRetentionConfiguration getRetentionPolicy() { + return this.retentionPolicy; + } + public static SnapshotLifecyclePolicy parse(XContentParser parser, String id) { return PARSER.apply(parser, id); } @@ -104,13 +114,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(SCHEDULE.getPreferredName(), this.schedule); builder.field(REPOSITORY.getPreferredName(), this.repository); builder.field(CONFIG.getPreferredName(), this.configuration); + builder.field(RETENTION.getPreferredName(), this.retentionPolicy); builder.endObject(); return builder; } @Override public int hashCode() { - return Objects.hash(id, name, schedule, repository, configuration); + return Objects.hash(id, name, schedule, repository, configuration, retentionPolicy); } @Override @@ -127,7 +138,8 @@ public boolean equals(Object obj) { Objects.equals(name, other.name) && Objects.equals(schedule, other.schedule) && Objects.equals(repository, other.repository) && - Objects.equals(configuration, other.configuration); + Objects.equals(configuration, other.configuration) && + Objects.equals(retentionPolicy, other.retentionPolicy); } @Override diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshotlifecycle/SnapshotRetentionConfiguration.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshotlifecycle/SnapshotRetentionConfiguration.java new file mode 100644 index 0000000000000..f31a3c8af0db3 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshotlifecycle/SnapshotRetentionConfiguration.java @@ -0,0 +1,95 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.snapshotlifecycle; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Objects; + +public class SnapshotRetentionConfiguration implements ToXContentObject { + + public static final SnapshotRetentionConfiguration EMPTY = new SnapshotRetentionConfiguration((TimeValue) null); + + private static final ParseField EXPIRE_AFTER = new ParseField("expire_after"); + + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("snapshot_retention", true, a -> { + TimeValue expireAfter = a[0] == null ? null : TimeValue.parseTimeValue((String) a[0], EXPIRE_AFTER.getPreferredName()); + return new SnapshotRetentionConfiguration(expireAfter); + }); + + static { + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), EXPIRE_AFTER); + } + + // TODO: add the rest of the configuration values + private final TimeValue expireAfter; + + public SnapshotRetentionConfiguration(TimeValue expireAfter) { + this.expireAfter = expireAfter; + } + + public static SnapshotRetentionConfiguration parse(XContentParser parser, String name) { + return PARSER.apply(parser, null); + } + + public TimeValue getExpireAfter() { + return this.expireAfter; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + if (expireAfter != null) { + builder.field(EXPIRE_AFTER.getPreferredName(), expireAfter.getStringRep()); + } + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(expireAfter); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (obj.getClass() != getClass()) { + return false; + } + SnapshotRetentionConfiguration other = (SnapshotRetentionConfiguration) obj; + return Objects.equals(this.expireAfter, other.expireAfter); + } + + @Override + public String toString() { + return Strings.toString(this); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/ILMDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/ILMDocumentationIT.java index c7ac357a31ef0..12e5b166d1e52 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/ILMDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/ILMDocumentationIT.java @@ -63,6 +63,7 @@ import org.elasticsearch.client.snapshotlifecycle.SnapshotInvocationRecord; import org.elasticsearch.client.snapshotlifecycle.SnapshotLifecyclePolicy; import org.elasticsearch.client.snapshotlifecycle.SnapshotLifecyclePolicyMetadata; +import org.elasticsearch.client.snapshotlifecycle.SnapshotRetentionConfiguration; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; @@ -773,8 +774,11 @@ public void testAddSnapshotLifecyclePolicy() throws Exception { // tag::slm-put-snapshot-lifecycle-policy Map config = new HashMap<>(); config.put("indices", Collections.singletonList("idx")); + SnapshotRetentionConfiguration retention = + new SnapshotRetentionConfiguration(TimeValue.timeValueDays(30)); SnapshotLifecyclePolicy policy = new SnapshotLifecyclePolicy( - "policy_id", "name", "1 2 3 * * ?", "my_repository", config); + "policy_id", "name", "1 2 3 * * ?", + "my_repository", config, retention); PutSnapshotLifecyclePolicyRequest request = new PutSnapshotLifecyclePolicyRequest(policy); // end::slm-put-snapshot-lifecycle-policy diff --git a/docs/reference/ilm/apis/slm-api.asciidoc b/docs/reference/ilm/apis/slm-api.asciidoc index a27297593e9f5..85c7b46f6279e 100644 --- a/docs/reference/ilm/apis/slm-api.asciidoc +++ b/docs/reference/ilm/apis/slm-api.asciidoc @@ -59,7 +59,8 @@ PUT /_slm/policy/daily-snapshots "indices": ["data-*", "important"], <5> "ignore_unavailable": false, "include_global_state": false - } + }, + "retention": {} } -------------------------------------------------- // CONSOLE @@ -136,7 +137,8 @@ The output looks similar to the following: "indices": ["data-*", "important"], "ignore_unavailable": false, "include_global_state": false - } + }, + "retention": {} }, "next_execution": "2019-04-24T01:30:00.000Z", <3> "next_execution_millis": 1556048160000 @@ -221,8 +223,9 @@ Which, in this case shows an error because the index did not exist: "indices": ["data-*", "important"], "ignore_unavailable": false, "include_global_state": false - } - }, + }, + "retention": {}, + } "last_failure": { <1> "snapshot_name": "daily-snap-2019.04.02-lohisb5ith2n8hxacaq3mw", "time_string": "2019-04-02T01:30:00.000Z", @@ -304,7 +307,8 @@ Which now includes the successful snapshot information: "indices": ["data-*", "important"], "ignore_unavailable": true, "include_global_state": false - } + }, + "retention": {} }, "last_success": { <2> "snapshot_name": "daily-snap-2019.04.24-tmtnyjtrsxkhbrrdcgg18a", diff --git a/docs/reference/ilm/getting-started-slm.asciidoc b/docs/reference/ilm/getting-started-slm.asciidoc index 5849101ffe6c3..f9cf619eb1fc9 100644 --- a/docs/reference/ilm/getting-started-slm.asciidoc +++ b/docs/reference/ilm/getting-started-slm.asciidoc @@ -95,7 +95,8 @@ PUT /_slm/policy/nightly-snapshots "repository": "my_repository", <3> "config": { <4> "indices": ["*"] <5> - } + }, + "retention": {} } -------------------------------------------------- // CONSOLE @@ -171,7 +172,8 @@ next time the policy will be executed. "repository": "my_repository", "config": { "indices": ["*"], - } + }, + "retention": {} }, "last_success": { <1> "snapshot_name": "nightly-snap-2019.04.24-tmtnyjtrsxkhbrrdcgg18a", <2> diff --git a/libs/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java b/libs/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java index c208e7d795391..edca86637e1b5 100644 --- a/libs/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java +++ b/libs/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java @@ -71,6 +71,14 @@ public static TimeValue timeValueHours(long hours) { return new TimeValue(hours, TimeUnit.HOURS); } + public static TimeValue timeValueDays(long days) { + // 106751.9 days is Long.MAX_VALUE nanoseconds, so we cannot store 106752 days + if (days > 106751) { + throw new IllegalArgumentException("time value cannot store values greater than 106751 days"); + } + return new TimeValue(days, TimeUnit.DAYS); + } + /** * @return the unit used for the this time value, see {@link #duration()} */ diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecyclePolicy.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecyclePolicy.java index 5db1996a45982..f3e11cb385787 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecyclePolicy.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecyclePolicy.java @@ -51,11 +51,13 @@ public class SnapshotLifecyclePolicy extends AbstractDiffable configuration; + private final SnapshotRetentionConfiguration retentionPolicy; private static final ParseField NAME = new ParseField("name"); private static final ParseField SCHEDULE = new ParseField("schedule"); private static final ParseField REPOSITORY = new ParseField("repository"); private static final ParseField CONFIG = new ParseField("config"); + private static final ParseField RETENTION = new ParseField("retention"); private static final IndexNameExpressionResolver.DateMathExpressionResolver DATE_MATH_RESOLVER = new IndexNameExpressionResolver.DateMathExpressionResolver(); private static final String POLICY_ID_METADATA_FIELD = "policy"; @@ -69,7 +71,8 @@ public class SnapshotLifecyclePolicy extends AbstractDiffable config = (Map) a[3]; - return new SnapshotLifecyclePolicy(id, name, schedule, repo, config); + SnapshotRetentionConfiguration retention = (SnapshotRetentionConfiguration) a[4]; + return new SnapshotLifecyclePolicy(id, name, schedule, repo, config, retention); }); static { @@ -77,15 +80,18 @@ public class SnapshotLifecyclePolicy extends AbstractDiffable p.map(), CONFIG); + PARSER.declareObject(ConstructingObjectParser.constructorArg(), SnapshotRetentionConfiguration::parse, RETENTION); } public SnapshotLifecyclePolicy(final String id, final String name, final String schedule, - final String repository, Map configuration) { + final String repository, Map configuration, + SnapshotRetentionConfiguration retentionPolicy) { this.id = Objects.requireNonNull(id); this.name = name; this.schedule = schedule; this.repository = repository; this.configuration = configuration; + this.retentionPolicy = retentionPolicy; } public SnapshotLifecyclePolicy(StreamInput in) throws IOException { @@ -94,6 +100,7 @@ public SnapshotLifecyclePolicy(StreamInput in) throws IOException { this.schedule = in.readString(); this.repository = in.readString(); this.configuration = in.readMap(); + this.retentionPolicy = new SnapshotRetentionConfiguration(in); } public String getId() { @@ -116,6 +123,10 @@ public Map getConfig() { return this.configuration; } + public SnapshotRetentionConfiguration getRetentionPolicy() { + return this.retentionPolicy; + } + public long calculateNextExecution() { final Cron schedule = new Cron(this.schedule); return schedule.getNextValidTimeAfter(System.currentTimeMillis()); @@ -257,6 +268,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(this.schedule); out.writeString(this.repository); out.writeMap(this.configuration); + this.retentionPolicy.writeTo(out); } @Override @@ -266,13 +278,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(SCHEDULE.getPreferredName(), this.schedule); builder.field(REPOSITORY.getPreferredName(), this.repository); builder.field(CONFIG.getPreferredName(), this.configuration); + builder.field(RETENTION.getPreferredName(), this.retentionPolicy); builder.endObject(); return builder; } @Override public int hashCode() { - return Objects.hash(id, name, schedule, repository, configuration); + return Objects.hash(id, name, schedule, repository, configuration, retentionPolicy); } @Override @@ -289,7 +302,8 @@ public boolean equals(Object obj) { Objects.equals(name, other.name) && Objects.equals(schedule, other.schedule) && Objects.equals(repository, other.repository) && - Objects.equals(configuration, other.configuration); + Objects.equals(configuration, other.configuration) && + Objects.equals(retentionPolicy, other.retentionPolicy); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotRetentionConfiguration.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotRetentionConfiguration.java new file mode 100644 index 0000000000000..707f89d173f9f --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotRetentionConfiguration.java @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.snapshotlifecycle; + +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Objects; + +public class SnapshotRetentionConfiguration implements ToXContentObject, Writeable { + + public static final SnapshotRetentionConfiguration EMPTY = new SnapshotRetentionConfiguration((TimeValue) null); + + private static final ParseField EXPIRE_AFTER = new ParseField("expire_after"); + + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("snapshot_retention", true, a -> { + TimeValue expireAfter = a[0] == null ? null : TimeValue.parseTimeValue((String) a[0], EXPIRE_AFTER.getPreferredName()); + return new SnapshotRetentionConfiguration(expireAfter); + }); + + static { + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), EXPIRE_AFTER); + } + + // TODO: add the rest of the configuration values + private final TimeValue expireAfter; + + public SnapshotRetentionConfiguration(@Nullable TimeValue expireAfter) { + this.expireAfter = expireAfter; + } + + SnapshotRetentionConfiguration(StreamInput in) throws IOException { + this.expireAfter = in.readOptionalTimeValue(); + } + + public static SnapshotRetentionConfiguration parse(XContentParser parser, String name) { + return PARSER.apply(parser, null); + } + + public TimeValue getExpireAfter() { + return this.expireAfter; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + if (expireAfter != null) { + builder.field(EXPIRE_AFTER.getPreferredName(), expireAfter.getStringRep()); + } + builder.endObject(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalTimeValue(this.expireAfter); + } + + @Override + public int hashCode() { + return Objects.hash(expireAfter); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (obj.getClass() != getClass()) { + return false; + } + SnapshotRetentionConfiguration other = (SnapshotRetentionConfiguration) obj; + return Objects.equals(this.expireAfter, other.expireAfter); + } + + @Override + public String toString() { + return Strings.toString(this); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecyclePolicyItemTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecyclePolicyItemTests.java index e243a4bd3a1b9..84014f6b5641e 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecyclePolicyItemTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecyclePolicyItemTests.java @@ -10,7 +10,7 @@ import org.elasticsearch.test.AbstractWireSerializingTestCase; import org.elasticsearch.test.ESTestCase; -import static org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicyMetadataTests.createRandomPolicy; +import static org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicyMetadataTests.randomSnapshotLifecyclePolicy; import static org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicyMetadataTests.createRandomPolicyMetadata; public class SnapshotLifecyclePolicyItemTests extends AbstractWireSerializingTestCase { @@ -25,7 +25,7 @@ protected SnapshotLifecyclePolicyItem mutateInstance(SnapshotLifecyclePolicyItem switch (between(0, 4)) { case 0: String newPolicyId = randomValueOtherThan(instance.getPolicy().getId(), () -> randomAlphaOfLengthBetween(5, 10)); - return new SnapshotLifecyclePolicyItem(createRandomPolicy(newPolicyId), + return new SnapshotLifecyclePolicyItem(randomSnapshotLifecyclePolicy(newPolicyId), instance.getVersion(), instance.getModifiedDate(), instance.getLastSuccess(), diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecyclePolicyMetadataTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecyclePolicyMetadataTests.java index 39fc692bfc905..7d97989a0ee59 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecyclePolicyMetadataTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecyclePolicyMetadataTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.core.snapshotlifecycle; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.AbstractSerializingTestCase; import org.elasticsearch.test.ESTestCase; @@ -50,7 +51,7 @@ protected SnapshotLifecyclePolicyMetadata mutateInstance(SnapshotLifecyclePolicy switch (between(0, 5)) { case 0: return SnapshotLifecyclePolicyMetadata.builder(instance) - .setPolicy(randomValueOtherThan(instance.getPolicy(), () -> createRandomPolicy(randomAlphaOfLength(10)))) + .setPolicy(randomValueOtherThan(instance.getPolicy(), () -> randomSnapshotLifecyclePolicy(randomAlphaOfLength(10)))) .build(); case 1: return SnapshotLifecyclePolicyMetadata.builder(instance) @@ -81,7 +82,7 @@ protected SnapshotLifecyclePolicyMetadata mutateInstance(SnapshotLifecyclePolicy public static SnapshotLifecyclePolicyMetadata createRandomPolicyMetadata(String policyId) { SnapshotLifecyclePolicyMetadata.Builder builder = SnapshotLifecyclePolicyMetadata.builder() - .setPolicy(createRandomPolicy(policyId)) + .setPolicy(randomSnapshotLifecyclePolicy(policyId)) .setVersion(randomNonNegativeLong()) .setModifiedDate(randomNonNegativeLong()); if (randomBoolean()) { @@ -96,7 +97,7 @@ public static SnapshotLifecyclePolicyMetadata createRandomPolicyMetadata(String return builder.build(); } - public static SnapshotLifecyclePolicy createRandomPolicy(String policyId) { + public static SnapshotLifecyclePolicy randomSnapshotLifecyclePolicy(String policyId) { Map config = new HashMap<>(); for (int i = 0; i < randomIntBetween(2, 5); i++) { config.put(randomAlphaOfLength(4), randomAlphaOfLength(4)); @@ -105,10 +106,16 @@ public static SnapshotLifecyclePolicy createRandomPolicy(String policyId) { randomAlphaOfLength(4), randomSchedule(), randomAlphaOfLength(4), - config); + config, + randomRetention()); } - private static String randomSchedule() { + public static SnapshotRetentionConfiguration randomRetention() { + return new SnapshotRetentionConfiguration(rarely() ? null : + TimeValue.parseTimeValue(randomTimeValue(), "random retention generation")); + } + + public static String randomSchedule() { return randomIntBetween(0, 59) + " " + randomIntBetween(0, 59) + " " + randomIntBetween(0, 12) + " * * ?"; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/snapshotlifecycle/history/SnapshotHistoryStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/snapshotlifecycle/history/SnapshotHistoryStoreTests.java index 146af810fdd79..8324741198159 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/snapshotlifecycle/history/SnapshotHistoryStoreTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/snapshotlifecycle/history/SnapshotHistoryStoreTests.java @@ -20,11 +20,11 @@ import java.time.Instant; import java.time.ZoneOffset; -import java.util.HashMap; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings.SLM_HISTORY_INDEX_ENABLED_SETTING; +import static org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicyMetadataTests.randomSnapshotLifecyclePolicy; import static org.elasticsearch.xpack.core.snapshotlifecycle.history.SnapshotHistoryStore.getHistoryIndexNameForTime; import static org.elasticsearch.xpack.core.snapshotlifecycle.history.SnapshotLifecycleTemplateRegistry.INDEX_TEMPLATE_VERSION; import static org.hamcrest.Matchers.containsString; @@ -174,22 +174,4 @@ public void testIndexNameGeneration() { assertThat(getHistoryIndexNameForTime(Instant.ofEpochMilli(2833165811000L).atZone(ZoneOffset.UTC)), equalTo(".slm-history-" + indexTemplateVersion + "-2059.10")); } - - public static SnapshotLifecyclePolicy randomSnapshotLifecyclePolicy(String id) { - Map config = new HashMap<>(); - for (int i = 0; i < randomIntBetween(2, 5); i++) { - config.put(randomAlphaOfLength(4), randomAlphaOfLength(4)); - } - return new SnapshotLifecyclePolicy(id, - randomAlphaOfLength(4), - randomSchedule(), - randomAlphaOfLength(4), - config); - } - - private static String randomSchedule() { - return randomIntBetween(0, 59) + " " + - randomIntBetween(0, 59) + " " + - randomIntBetween(0, 12) + " * * ?"; - } } diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleIT.java index f0996d7a0df1a..56acbf91232c6 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleIT.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicy; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotRetentionConfiguration; import java.io.IOException; import java.io.InputStream; @@ -43,7 +44,7 @@ public class SnapshotLifecycleIT extends ESRestTestCase { public void testMissingRepo() throws Exception { SnapshotLifecyclePolicy policy = new SnapshotLifecyclePolicy("test-policy", "snap", - "*/1 * * * * ?", "missing-repo", Collections.emptyMap()); + "*/1 * * * * ?", "missing-repo", Collections.emptyMap(), SnapshotRetentionConfiguration.EMPTY); Request putLifecycle = new Request("PUT", "/_slm/policy/test-policy"); XContentBuilder lifecycleBuilder = JsonXContent.contentBuilder(); @@ -294,7 +295,8 @@ private void createSnapshotPolicy(String policyName, String snapshotNamePattern, () -> randomAlphaOfLength(5)), randomAlphaOfLength(4)); } } - SnapshotLifecyclePolicy policy = new SnapshotLifecyclePolicy(policyName, snapshotNamePattern, schedule, repoId, snapConfig); + SnapshotLifecyclePolicy policy = new SnapshotLifecyclePolicy(policyName, snapshotNamePattern, schedule, repoId, snapConfig, + SnapshotRetentionConfiguration.EMPTY); Request putLifecycle = new Request("PUT", "/_slm/policy/" + policyName); XContentBuilder lifecycleBuilder = JsonXContent.contentBuilder(); diff --git a/x-pack/plugin/ilm/qa/with-security/src/test/java/org/elasticsearch/xpack/security/PermissionsIT.java b/x-pack/plugin/ilm/qa/with-security/src/test/java/org/elasticsearch/xpack/security/PermissionsIT.java index 8450aafc47036..156ee195cda55 100644 --- a/x-pack/plugin/ilm/qa/with-security/src/test/java/org/elasticsearch/xpack/security/PermissionsIT.java +++ b/x-pack/plugin/ilm/qa/with-security/src/test/java/org/elasticsearch/xpack/security/PermissionsIT.java @@ -24,6 +24,7 @@ import org.elasticsearch.client.snapshotlifecycle.GetSnapshotLifecyclePolicyRequest; import org.elasticsearch.client.snapshotlifecycle.PutSnapshotLifecyclePolicyRequest; import org.elasticsearch.client.snapshotlifecycle.SnapshotLifecyclePolicy; +import org.elasticsearch.client.snapshotlifecycle.SnapshotRetentionConfiguration; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; @@ -188,7 +189,7 @@ public void testSLMWithPermissions() throws Exception { Map config = new HashMap<>(); config.put("indices", Collections.singletonList("index")); SnapshotLifecyclePolicy policy = new SnapshotLifecyclePolicy( - "policy_id", "name", "1 2 3 * * ?", "my_repository", config); + "policy_id", "name", "1 2 3 * * ?", "my_repository", config, SnapshotRetentionConfiguration.EMPTY); PutSnapshotLifecyclePolicyRequest request = new PutSnapshotLifecyclePolicyRequest(policy); expectThrows(ElasticsearchStatusException.class, diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecyclePolicyTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecyclePolicyTests.java index c2aac7120ab8a..b0406c8209a98 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecyclePolicyTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecyclePolicyTests.java @@ -11,12 +11,15 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.AbstractSerializingTestCase; import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicy; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicyMetadataTests; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotRetentionConfiguration; import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import static org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicyMetadataTests.randomSnapshotLifecyclePolicy; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; @@ -30,29 +33,34 @@ public class SnapshotLifecyclePolicyTests extends AbstractSerializingTestCase", "1 * * * * ?", "repo", Collections.emptyMap()); + p = new SnapshotLifecyclePolicy("id", "", "1 * * * * ?", "repo", Collections.emptyMap(), + SnapshotRetentionConfiguration.EMPTY); assertThat(p.generateSnapshotName(context), startsWith("name-2019.03.15-")); assertThat(p.generateSnapshotName(context).length(), greaterThan("name-2019.03.15-".length())); - p = new SnapshotLifecyclePolicy("id", "", "1 * * * * ?", "repo", Collections.emptyMap()); + p = new SnapshotLifecyclePolicy("id", "", "1 * * * * ?", "repo", Collections.emptyMap(), + SnapshotRetentionConfiguration.EMPTY); assertThat(p.generateSnapshotName(context), startsWith("name-2019.03.01-")); - p = new SnapshotLifecyclePolicy("id", "", "1 * * * * ?", "repo", Collections.emptyMap()); + p = new SnapshotLifecyclePolicy("id", "", "1 * * * * ?", "repo", Collections.emptyMap(), + SnapshotRetentionConfiguration.EMPTY); assertThat(p.generateSnapshotName(context), startsWith("name-2019-03-15.21:09:00-")); } public void testNextExecutionTime() { - SnapshotLifecyclePolicy p = new SnapshotLifecyclePolicy("id", "name", "0 1 2 3 4 ? 2099", "repo", Collections.emptyMap()); + SnapshotLifecyclePolicy p = new SnapshotLifecyclePolicy("id", "name", "0 1 2 3 4 ? 2099", "repo", Collections.emptyMap(), + SnapshotRetentionConfiguration.EMPTY); assertThat(p.calculateNextExecution(), equalTo(4078864860000L)); } public void testValidation() { SnapshotLifecyclePolicy policy = new SnapshotLifecyclePolicy("a,b", "", - "* * * * * L", " ", Collections.emptyMap()); + "* * * * * L", " ", Collections.emptyMap(), SnapshotRetentionConfiguration.EMPTY); ValidationException e = policy.validate(); assertThat(e.validationErrors(), @@ -63,7 +71,7 @@ public void testValidation() { "invalid schedule: invalid cron expression [* * * * * L]")); policy = new SnapshotLifecyclePolicy("_my_policy", "mySnap", - " ", "repo", Collections.emptyMap()); + " ", "repo", Collections.emptyMap(), SnapshotRetentionConfiguration.EMPTY); e = policy.validate(); assertThat(e.validationErrors(), @@ -79,7 +87,7 @@ public void testMetadataValidation() { configuration.put("metadata", metadataString); SnapshotLifecyclePolicy policy = new SnapshotLifecyclePolicy("mypolicy", "", - "1 * * * * ?", "myrepo", configuration); + "1 * * * * ?", "myrepo", configuration, SnapshotRetentionConfiguration.EMPTY); ValidationException e = policy.validate(); assertThat(e.validationErrors(), contains("invalid configuration.metadata [" + metadataString + "]: must be an object if present")); @@ -92,7 +100,7 @@ public void testMetadataValidation() { configuration.put("metadata", metadata); SnapshotLifecyclePolicy policy = new SnapshotLifecyclePolicy("mypolicy", "", - "1 * * * * ?", "myrepo", configuration); + "1 * * * * ?", "myrepo", configuration, SnapshotRetentionConfiguration.EMPTY); ValidationException e = policy.validate(); assertThat(e.validationErrors(), contains("invalid configuration.metadata: field name [policy] is reserved and " + "will be added automatically")); @@ -112,7 +120,7 @@ public void testMetadataValidation() { configuration.put("metadata", metadata); SnapshotLifecyclePolicy policy = new SnapshotLifecyclePolicy("mypolicy", "", - "1 * * * * ?", "myrepo", configuration); + "1 * * * * ?", "myrepo", configuration, SnapshotRetentionConfiguration.EMPTY); ValidationException e = policy.validate(); assertThat(e.validationErrors(), contains("invalid configuration.metadata: must be smaller than [1004] bytes, but is [" + totalBytes + "] bytes")); @@ -130,51 +138,37 @@ protected SnapshotLifecyclePolicy createTestInstance() { return randomSnapshotLifecyclePolicy(id); } - public static SnapshotLifecyclePolicy randomSnapshotLifecyclePolicy(String id) { - Map config = new HashMap<>(); - for (int i = 0; i < randomIntBetween(2, 5); i++) { - config.put(randomAlphaOfLength(4), randomAlphaOfLength(4)); - } - return new SnapshotLifecyclePolicy(id, - randomAlphaOfLength(4), - randomSchedule(), - randomAlphaOfLength(4), - config); - } - - private static String randomSchedule() { - return randomIntBetween(0, 59) + " " + - randomIntBetween(0, 59) + " " + - randomIntBetween(0, 12) + " * * ?"; - } - @Override - protected SnapshotLifecyclePolicy mutateInstance(SnapshotLifecyclePolicy instance) throws IOException { - switch (between(0, 4)) { + protected SnapshotLifecyclePolicy mutateInstance(SnapshotLifecyclePolicy instance) { + switch (between(0, 5)) { case 0: return new SnapshotLifecyclePolicy(instance.getId() + randomAlphaOfLength(2), instance.getName(), instance.getSchedule(), instance.getRepository(), - instance.getConfig()); + instance.getConfig(), + instance.getRetentionPolicy()); case 1: return new SnapshotLifecyclePolicy(instance.getId(), instance.getName() + randomAlphaOfLength(2), instance.getSchedule(), instance.getRepository(), - instance.getConfig()); + instance.getConfig(), + instance.getRetentionPolicy()); case 2: return new SnapshotLifecyclePolicy(instance.getId(), instance.getName(), - randomValueOtherThan(instance.getSchedule(), SnapshotLifecyclePolicyTests::randomSchedule), + randomValueOtherThan(instance.getSchedule(), SnapshotLifecyclePolicyMetadataTests::randomSchedule), instance.getRepository(), - instance.getConfig()); + instance.getConfig(), + instance.getRetentionPolicy()); case 3: return new SnapshotLifecyclePolicy(instance.getId(), instance.getName(), instance.getSchedule(), instance.getRepository() + randomAlphaOfLength(2), - instance.getConfig()); + instance.getConfig(), + instance.getRetentionPolicy()); case 4: Map newConfig = new HashMap<>(); for (int i = 0; i < randomIntBetween(2, 5); i++) { @@ -184,7 +178,15 @@ protected SnapshotLifecyclePolicy mutateInstance(SnapshotLifecyclePolicy instanc instance.getName() + randomAlphaOfLength(2), instance.getSchedule(), instance.getRepository(), - newConfig); + newConfig, + instance.getRetentionPolicy()); + case 5: + return new SnapshotLifecyclePolicy(instance.getId(), + instance.getName(), + instance.getSchedule(), + instance.getRepository(), + instance.getConfig(), + randomValueOtherThan(instance.getRetentionPolicy(), SnapshotLifecyclePolicyMetadataTests::randomRetention)); default: throw new AssertionError("failure, got illegal switch case"); } diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleServiceTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleServiceTests.java index 395aef7ee761e..be4878993abe8 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleServiceTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleServiceTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecycleMetadata; import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicy; import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicyMetadata; +import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotRetentionConfiguration; import org.elasticsearch.xpack.core.watcher.watch.ClockMock; import java.util.ArrayList; @@ -328,7 +329,8 @@ public static SnapshotLifecyclePolicy createPolicy(String id, String schedule) { indices.add("foo-*"); indices.add(randomAlphaOfLength(4)); config.put("indices", indices); - return new SnapshotLifecyclePolicy(id, randomAlphaOfLength(4), schedule, randomAlphaOfLength(4), config); + return new SnapshotLifecyclePolicy(id, randomAlphaOfLength(4), schedule, randomAlphaOfLength(4), config, + SnapshotRetentionConfiguration.EMPTY); } public static String randomSchedule() {