Skip to content

Commit

Permalink
[SNAPSHOT] Add repository validation
Browse files Browse the repository at this point in the history
  • Loading branch information
imotov committed Oct 7, 2014
1 parent 708f274 commit ccfcce8
Show file tree
Hide file tree
Showing 40 changed files with 1,398 additions and 43 deletions.
16 changes: 16 additions & 0 deletions docs/reference/modules/snapshots.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ on all data and master nodes. The following settings are supported:
`compress`:: Turns on compression of the snapshot files. Compression is applied only to metadata files (index mapping and settings). Data files are not compressed. Defaults to `true`.
`chunk_size`:: Big files can be broken down into chunks during snapshotting if needed. The chunk size can be specified in bytes or by
using size value notation, i.e. 1g, 10m, 5k. Defaults to `null` (unlimited chunk size).
`verify`:: Verify repository upon creation. Defaults to `true`.

[float]
===== Read-only URL Repository
Expand Down Expand Up @@ -99,6 +100,21 @@ Other repository backends are available in these official plugins:
* https://github.com/elasticsearch/elasticsearch-hadoop/tree/master/repository-hdfs[HDFS Plugin] for Hadoop environments
* https://github.com/elasticsearch/elasticsearch-cloud-azure#azure-repository[Azure Cloud Plugin] for Azure storage repositories


===== Repository Verification
added[1.4.0]

When repository is registered, it's immediately verified on all master and data nodes to make sure that it's functional
on all nodes currently present in the cluster. The verification process can also be executed manually by running the
following command:

[source,js]
-----------------------------------
$ curl -XPOST 'http://localhost:9200/_snapshot/my_backup/_verify'
-----------------------------------

It returns a list of nodes

[float]
=== Snapshot

Expand Down
4 changes: 4 additions & 0 deletions rest-api-spec/api/snapshot.create_repository.json
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,10 @@
"timeout": {
"type" : "time",
"description" : "Explicit operation timeout"
},
"verify": {
"type" : "boolean",
"description" : "Whether to verify the repository after creation"
}
}
},
Expand Down
28 changes: 28 additions & 0 deletions rest-api-spec/api/snapshot.verify_repository.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
{
"snapshot.verify_repository": {
"documentation": "http://www.elasticsearch.org/guide/en/elasticsearch/reference/master/modules-snapshots.html",
"methods": ["POST"],
"url": {
"path": "/_snapshot/{repository}/_verify",
"paths": ["/_snapshot/{repository}/_verify"],
"parts": {
"repository": {
"type": "string",
"required" : true,
"description": "A repository name"
}
},
"params": {
"master_timeout": {
"type" : "time",
"description" : "Explicit operation timeout for connection to master node"
},
"timeout": {
"type" : "time",
"description" : "Explicit operation timeout"
}
}
},
"body": null
}
}
8 changes: 8 additions & 0 deletions rest-api-spec/test/snapshot.get_repository/10_basic.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -48,3 +48,11 @@ setup:

- is_true: test_repo1
- is_true: test_repo2

---
"Verify created repository":
- do:
snapshot.verify_repository:
repository: test_repo2

- is_true: nodes
3 changes: 3 additions & 0 deletions src/main/java/org/elasticsearch/action/ActionModule.java
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@
import org.elasticsearch.action.admin.cluster.repositories.get.TransportGetRepositoriesAction;
import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryAction;
import org.elasticsearch.action.admin.cluster.repositories.put.TransportPutRepositoryAction;
import org.elasticsearch.action.admin.cluster.repositories.verify.TransportVerifyRepositoryAction;
import org.elasticsearch.action.admin.cluster.repositories.verify.VerifyRepositoryAction;
import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteAction;
import org.elasticsearch.action.admin.cluster.reroute.TransportClusterRerouteAction;
import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsAction;
Expand Down Expand Up @@ -239,6 +241,7 @@ protected void configure() {
registerAction(PutRepositoryAction.INSTANCE, TransportPutRepositoryAction.class);
registerAction(GetRepositoriesAction.INSTANCE, TransportGetRepositoriesAction.class);
registerAction(DeleteRepositoryAction.INSTANCE, TransportDeleteRepositoryAction.class);
registerAction(VerifyRepositoryAction.INSTANCE, TransportVerifyRepositoryAction.class);
registerAction(GetSnapshotsAction.INSTANCE, TransportGetSnapshotsAction.class);
registerAction(DeleteSnapshotAction.INSTANCE, TransportDeleteSnapshotAction.class);
registerAction(CreateSnapshotAction.INSTANCE, TransportCreateSnapshotAction.class);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

import org.elasticsearch.ElasticsearchGenerationException;
import org.elasticsearch.ElasticsearchIllegalArgumentException;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.support.master.AcknowledgedRequest;
import org.elasticsearch.common.bytes.BytesReference;
Expand Down Expand Up @@ -52,6 +53,8 @@ public class PutRepositoryRequest extends AcknowledgedRequest<PutRepositoryReque

private String type;

private boolean verify = true;

private Settings settings = EMPTY_SETTINGS;

PutRepositoryRequest() {
Expand Down Expand Up @@ -178,6 +181,20 @@ public Settings settings() {
return this.settings;
}

/**
* Sets whether or not the repository should be verified after creation
*/
public PutRepositoryRequest verify(boolean verify) {
this.verify = verify;
return this;
}

/**
* Returns true if repository should be verified after creation
*/
public boolean verify() {
return this.verify;
}

/**
* Parses repository definition.
Expand Down Expand Up @@ -268,6 +285,12 @@ public void readFrom(StreamInput in) throws IOException {
type = in.readString();
settings = readSettingsFromStream(in);
readTimeout(in);
if (in.getVersion().onOrAfter(Version.V_1_4_0)) {
verify = in.readBoolean();
} else {
// we received this request from an older client that doesn't expect us to validate the request
verify = false;
}
}

@Override
Expand All @@ -277,5 +300,8 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeString(type);
writeSettingsToStream(settings, out);
writeTimeout(out);
if (out.getVersion().onOrAfter(Version.V_1_4_0)) {
out.writeBoolean(verify);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,17 @@ public PutRepositoryRequestBuilder setSettings(Map<String, Object> source) {
return this;
}

/**
* Sets whether or not repository should be verified after creation
*
* @param verify true if repository should be verified after registration, false otherwise
* @return this builder
*/
public PutRepositoryRequestBuilder setVerify(boolean verify) {
request.verify(verify);
return this;
}

@Override
protected void doExecute(ActionListener<PutRepositoryResponse> listener) {
client.putRepository(request, listener);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,9 @@ protected ClusterBlockException checkBlock(PutRepositoryRequest request, Cluster
@Override
protected void masterOperation(final PutRepositoryRequest request, ClusterState state, final ActionListener<PutRepositoryResponse> listener) throws ElasticsearchException {

repositoriesService.registerRepository(new RepositoriesService.RegisterRepositoryRequest("put_repository [" + request.name() + "]", request.name(), request.type())
repositoriesService.registerRepository(
new RepositoriesService.RegisterRepositoryRequest("put_repository [" + request.name() + "]",
request.name(), request.type(), request.verify())
.settings(request.settings())
.masterNodeTimeout(request.masterNodeTimeout())
.ackTimeout(request.timeout()), new ActionListener<ClusterStateUpdateResponse>() {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.action.admin.cluster.repositories.verify;

import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.master.TransportMasterNodeOperationAction;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.block.ClusterBlockException;
import org.elasticsearch.cluster.block.ClusterBlockLevel;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.repositories.RepositoriesService;
import org.elasticsearch.repositories.RepositoryVerificationException;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;

/**
* Transport action for verifying repository operation
*/
public class TransportVerifyRepositoryAction extends TransportMasterNodeOperationAction<VerifyRepositoryRequest, VerifyRepositoryResponse> {

private final RepositoriesService repositoriesService;

protected final ClusterName clusterName;

@Inject
public TransportVerifyRepositoryAction(Settings settings, ClusterName clusterName, TransportService transportService, ClusterService clusterService,
RepositoriesService repositoriesService, ThreadPool threadPool, ActionFilters actionFilters) {
super(settings, VerifyRepositoryAction.NAME, transportService, clusterService, threadPool, actionFilters);
this.repositoriesService = repositoriesService;
this.clusterName = clusterName;
}

@Override
protected String executor() {
return ThreadPool.Names.MANAGEMENT;
}

@Override
protected VerifyRepositoryRequest newRequest() {
return new VerifyRepositoryRequest();
}

@Override
protected VerifyRepositoryResponse newResponse() {
return new VerifyRepositoryResponse();
}

@Override
protected ClusterBlockException checkBlock(VerifyRepositoryRequest request, ClusterState state) {
return state.blocks().indexBlockedException(ClusterBlockLevel.METADATA, "");
}

@Override
protected void masterOperation(final VerifyRepositoryRequest request, ClusterState state, final ActionListener<VerifyRepositoryResponse> listener) throws ElasticsearchException {
repositoriesService.verifyRepository(request.name(), new ActionListener<RepositoriesService.VerifyResponse>() {
@Override
public void onResponse(RepositoriesService.VerifyResponse verifyResponse) {
if (verifyResponse.failed()) {
listener.onFailure(new RepositoryVerificationException(request.name(), verifyResponse.failureDescription()));
} else {
listener.onResponse(new VerifyRepositoryResponse(clusterName, verifyResponse.nodes()));
}
}

@Override
public void onFailure(Throwable e) {
listener.onFailure(e);
}
});
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.action.admin.cluster.repositories.verify;

import org.elasticsearch.action.admin.cluster.ClusterAction;
import org.elasticsearch.client.ClusterAdminClient;

/**
* Unregister repository action
*/
public class VerifyRepositoryAction extends ClusterAction<VerifyRepositoryRequest, VerifyRepositoryResponse, VerifyRepositoryRequestBuilder> {

public static final VerifyRepositoryAction INSTANCE = new VerifyRepositoryAction();
public static final String NAME = "cluster:admin/repository/verify";

private VerifyRepositoryAction() {
super(NAME);
}

@Override
public VerifyRepositoryResponse newResponse() {
return new VerifyRepositoryResponse();
}

@Override
public VerifyRepositoryRequestBuilder newRequestBuilder(ClusterAdminClient client) {
return new VerifyRepositoryRequestBuilder(client);
}
}

Loading

0 comments on commit ccfcce8

Please sign in to comment.