Skip to content

Commit

Permalink
Add SnapshotRetentionConfiguration for retention configuration (#43777)
Browse files Browse the repository at this point in the history
* Add SnapshotRetentionConfiguration for retention configuration

This commit adds the `SnapshotRetentionConfiguration` class and its HLRC
counterpart to encapsulate the configuration for SLM retention.
Currently only a single parameter is supported as an example (we still
need to discuss the different options we want to support and their
names) to keep the size of the PR down. It also does not yet include version serialization checks
since the original SLM branch has not yet been merged.

Relates to #43663

* Fix REST tests

* Fix more documentation

* Use Objects.equals to avoid NPE

* Put `randomSnapshotLifecyclePolicy` in only one place

* Occasionally return retention with no configuration
  • Loading branch information
dakrone committed Jul 17, 2019
1 parent 23cfe40 commit 3c1f331
Show file tree
Hide file tree
Showing 15 changed files with 312 additions and 82 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,13 @@ public class SnapshotLifecyclePolicy implements ToXContentObject {
private final String schedule;
private final String repository;
private final Map<String, Object> configuration;
private final SnapshotRetentionConfiguration retentionPolicy;

private static final ParseField NAME = new ParseField("name");
private static final ParseField SCHEDULE = new ParseField("schedule");
private static final ParseField REPOSITORY = new ParseField("repository");
private static final ParseField CONFIG = new ParseField("config");
private static final ParseField RETENTION = new ParseField("retention");
private static final IndexNameExpressionResolver.DateMathExpressionResolver DATE_MATH_RESOLVER =
new IndexNameExpressionResolver.DateMathExpressionResolver();

Expand All @@ -54,23 +56,27 @@ public class SnapshotLifecyclePolicy implements ToXContentObject {
String schedule = (String) a[1];
String repo = (String) a[2];
Map<String, Object> config = (Map<String, Object>) a[3];
return new SnapshotLifecyclePolicy(id, name, schedule, repo, config);
SnapshotRetentionConfiguration retention = (SnapshotRetentionConfiguration) a[4];
return new SnapshotLifecyclePolicy(id, name, schedule, repo, config, retention);
});

static {
PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME);
PARSER.declareString(ConstructingObjectParser.constructorArg(), SCHEDULE);
PARSER.declareString(ConstructingObjectParser.constructorArg(), REPOSITORY);
PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> p.map(), CONFIG);
PARSER.declareObject(ConstructingObjectParser.constructorArg(), SnapshotRetentionConfiguration::parse, RETENTION);
}

public SnapshotLifecyclePolicy(final String id, final String name, final String schedule,
final String repository, Map<String, Object> configuration) {
final String repository, Map<String, Object> configuration,
SnapshotRetentionConfiguration retentionPolicy) {
this.id = Objects.requireNonNull(id);
this.name = name;
this.schedule = schedule;
this.repository = repository;
this.configuration = configuration;
this.retentionPolicy = retentionPolicy;
}

public String getId() {
Expand All @@ -93,6 +99,10 @@ public Map<String, Object> getConfig() {
return this.configuration;
}

public SnapshotRetentionConfiguration getRetentionPolicy() {
return this.retentionPolicy;
}

public static SnapshotLifecyclePolicy parse(XContentParser parser, String id) {
return PARSER.apply(parser, id);
}
Expand All @@ -104,13 +114,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.field(SCHEDULE.getPreferredName(), this.schedule);
builder.field(REPOSITORY.getPreferredName(), this.repository);
builder.field(CONFIG.getPreferredName(), this.configuration);
builder.field(RETENTION.getPreferredName(), this.retentionPolicy);
builder.endObject();
return builder;
}

@Override
public int hashCode() {
return Objects.hash(id, name, schedule, repository, configuration);
return Objects.hash(id, name, schedule, repository, configuration, retentionPolicy);
}

@Override
Expand All @@ -127,7 +138,8 @@ public boolean equals(Object obj) {
Objects.equals(name, other.name) &&
Objects.equals(schedule, other.schedule) &&
Objects.equals(repository, other.repository) &&
Objects.equals(configuration, other.configuration);
Objects.equals(configuration, other.configuration) &&
Objects.equals(retentionPolicy, other.retentionPolicy);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.client.snapshotlifecycle;

import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;

import java.io.IOException;
import java.util.Objects;

public class SnapshotRetentionConfiguration implements ToXContentObject {

public static final SnapshotRetentionConfiguration EMPTY = new SnapshotRetentionConfiguration((TimeValue) null);

private static final ParseField EXPIRE_AFTER = new ParseField("expire_after");

private static final ConstructingObjectParser<SnapshotRetentionConfiguration, Void> PARSER =
new ConstructingObjectParser<>("snapshot_retention", true, a -> {
TimeValue expireAfter = a[0] == null ? null : TimeValue.parseTimeValue((String) a[0], EXPIRE_AFTER.getPreferredName());
return new SnapshotRetentionConfiguration(expireAfter);
});

static {
PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), EXPIRE_AFTER);
}

// TODO: add the rest of the configuration values
private final TimeValue expireAfter;

public SnapshotRetentionConfiguration(TimeValue expireAfter) {
this.expireAfter = expireAfter;
}

public static SnapshotRetentionConfiguration parse(XContentParser parser, String name) {
return PARSER.apply(parser, null);
}

public TimeValue getExpireAfter() {
return this.expireAfter;
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
if (expireAfter != null) {
builder.field(EXPIRE_AFTER.getPreferredName(), expireAfter.getStringRep());
}
builder.endObject();
return builder;
}

@Override
public int hashCode() {
return Objects.hash(expireAfter);
}

@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (obj.getClass() != getClass()) {
return false;
}
SnapshotRetentionConfiguration other = (SnapshotRetentionConfiguration) obj;
return Objects.equals(this.expireAfter, other.expireAfter);
}

@Override
public String toString() {
return Strings.toString(this);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@
import org.elasticsearch.client.snapshotlifecycle.SnapshotInvocationRecord;
import org.elasticsearch.client.snapshotlifecycle.SnapshotLifecyclePolicy;
import org.elasticsearch.client.snapshotlifecycle.SnapshotLifecyclePolicyMetadata;
import org.elasticsearch.client.snapshotlifecycle.SnapshotRetentionConfiguration;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.ImmutableOpenMap;
Expand Down Expand Up @@ -773,8 +774,11 @@ public void testAddSnapshotLifecyclePolicy() throws Exception {
// tag::slm-put-snapshot-lifecycle-policy
Map<String, Object> config = new HashMap<>();
config.put("indices", Collections.singletonList("idx"));
SnapshotRetentionConfiguration retention =
new SnapshotRetentionConfiguration(TimeValue.timeValueDays(30));
SnapshotLifecyclePolicy policy = new SnapshotLifecyclePolicy(
"policy_id", "name", "1 2 3 * * ?", "my_repository", config);
"policy_id", "name", "1 2 3 * * ?",
"my_repository", config, retention);
PutSnapshotLifecyclePolicyRequest request =
new PutSnapshotLifecyclePolicyRequest(policy);
// end::slm-put-snapshot-lifecycle-policy
Expand Down
14 changes: 9 additions & 5 deletions docs/reference/ilm/apis/slm-api.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,8 @@ PUT /_slm/policy/daily-snapshots
"indices": ["data-*", "important"], <5>
"ignore_unavailable": false,
"include_global_state": false
}
},
"retention": {}
}
--------------------------------------------------
// CONSOLE
Expand Down Expand Up @@ -136,7 +137,8 @@ The output looks similar to the following:
"indices": ["data-*", "important"],
"ignore_unavailable": false,
"include_global_state": false
}
},
"retention": {}
},
"next_execution": "2019-04-24T01:30:00.000Z", <3>
"next_execution_millis": 1556048160000
Expand Down Expand Up @@ -221,8 +223,9 @@ Which, in this case shows an error because the index did not exist:
"indices": ["data-*", "important"],
"ignore_unavailable": false,
"include_global_state": false
}
},
},
"retention": {},
}
"last_failure": { <1>
"snapshot_name": "daily-snap-2019.04.02-lohisb5ith2n8hxacaq3mw",
"time_string": "2019-04-02T01:30:00.000Z",
Expand Down Expand Up @@ -304,7 +307,8 @@ Which now includes the successful snapshot information:
"indices": ["data-*", "important"],
"ignore_unavailable": true,
"include_global_state": false
}
},
"retention": {}
},
"last_success": { <2>
"snapshot_name": "daily-snap-2019.04.24-tmtnyjtrsxkhbrrdcgg18a",
Expand Down
6 changes: 4 additions & 2 deletions docs/reference/ilm/getting-started-slm.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,8 @@ PUT /_slm/policy/nightly-snapshots
"repository": "my_repository", <3>
"config": { <4>
"indices": ["*"] <5>
}
},
"retention": {}
}
--------------------------------------------------
// CONSOLE
Expand Down Expand Up @@ -171,7 +172,8 @@ next time the policy will be executed.
"repository": "my_repository",
"config": {
"indices": ["*"],
}
},
"retention": {}
},
"last_success": { <1>
"snapshot_name": "nightly-snap-2019.04.24-tmtnyjtrsxkhbrrdcgg18a", <2>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,14 @@ public static TimeValue timeValueHours(long hours) {
return new TimeValue(hours, TimeUnit.HOURS);
}

public static TimeValue timeValueDays(long days) {
// 106751.9 days is Long.MAX_VALUE nanoseconds, so we cannot store 106752 days
if (days > 106751) {
throw new IllegalArgumentException("time value cannot store values greater than 106751 days");
}
return new TimeValue(days, TimeUnit.DAYS);
}

/**
* @return the unit used for the this time value, see {@link #duration()}
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,11 +51,13 @@ public class SnapshotLifecyclePolicy extends AbstractDiffable<SnapshotLifecycleP
private final String schedule;
private final String repository;
private final Map<String, Object> configuration;
private final SnapshotRetentionConfiguration retentionPolicy;

private static final ParseField NAME = new ParseField("name");
private static final ParseField SCHEDULE = new ParseField("schedule");
private static final ParseField REPOSITORY = new ParseField("repository");
private static final ParseField CONFIG = new ParseField("config");
private static final ParseField RETENTION = new ParseField("retention");
private static final IndexNameExpressionResolver.DateMathExpressionResolver DATE_MATH_RESOLVER =
new IndexNameExpressionResolver.DateMathExpressionResolver();
private static final String POLICY_ID_METADATA_FIELD = "policy";
Expand All @@ -69,23 +71,27 @@ public class SnapshotLifecyclePolicy extends AbstractDiffable<SnapshotLifecycleP
String schedule = (String) a[1];
String repo = (String) a[2];
Map<String, Object> config = (Map<String, Object>) a[3];
return new SnapshotLifecyclePolicy(id, name, schedule, repo, config);
SnapshotRetentionConfiguration retention = (SnapshotRetentionConfiguration) a[4];
return new SnapshotLifecyclePolicy(id, name, schedule, repo, config, retention);
});

static {
PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME);
PARSER.declareString(ConstructingObjectParser.constructorArg(), SCHEDULE);
PARSER.declareString(ConstructingObjectParser.constructorArg(), REPOSITORY);
PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> p.map(), CONFIG);
PARSER.declareObject(ConstructingObjectParser.constructorArg(), SnapshotRetentionConfiguration::parse, RETENTION);
}

public SnapshotLifecyclePolicy(final String id, final String name, final String schedule,
final String repository, Map<String, Object> configuration) {
final String repository, Map<String, Object> configuration,
SnapshotRetentionConfiguration retentionPolicy) {
this.id = Objects.requireNonNull(id);
this.name = name;
this.schedule = schedule;
this.repository = repository;
this.configuration = configuration;
this.retentionPolicy = retentionPolicy;
}

public SnapshotLifecyclePolicy(StreamInput in) throws IOException {
Expand All @@ -94,6 +100,7 @@ public SnapshotLifecyclePolicy(StreamInput in) throws IOException {
this.schedule = in.readString();
this.repository = in.readString();
this.configuration = in.readMap();
this.retentionPolicy = new SnapshotRetentionConfiguration(in);
}

public String getId() {
Expand All @@ -116,6 +123,10 @@ public Map<String, Object> getConfig() {
return this.configuration;
}

public SnapshotRetentionConfiguration getRetentionPolicy() {
return this.retentionPolicy;
}

public long calculateNextExecution() {
final Cron schedule = new Cron(this.schedule);
return schedule.getNextValidTimeAfter(System.currentTimeMillis());
Expand Down Expand Up @@ -257,6 +268,7 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeString(this.schedule);
out.writeString(this.repository);
out.writeMap(this.configuration);
this.retentionPolicy.writeTo(out);
}

@Override
Expand All @@ -266,13 +278,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.field(SCHEDULE.getPreferredName(), this.schedule);
builder.field(REPOSITORY.getPreferredName(), this.repository);
builder.field(CONFIG.getPreferredName(), this.configuration);
builder.field(RETENTION.getPreferredName(), this.retentionPolicy);
builder.endObject();
return builder;
}

@Override
public int hashCode() {
return Objects.hash(id, name, schedule, repository, configuration);
return Objects.hash(id, name, schedule, repository, configuration, retentionPolicy);
}

@Override
Expand All @@ -289,7 +302,8 @@ public boolean equals(Object obj) {
Objects.equals(name, other.name) &&
Objects.equals(schedule, other.schedule) &&
Objects.equals(repository, other.repository) &&
Objects.equals(configuration, other.configuration);
Objects.equals(configuration, other.configuration) &&
Objects.equals(retentionPolicy, other.retentionPolicy);
}

@Override
Expand Down
Loading

0 comments on commit 3c1f331

Please sign in to comment.