Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Don't load global state when only restoring indices #29239

Merged
merged 5 commits into from
Mar 28, 2018
Merged

Don't load global state when only restoring indices #29239

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
c3e3c83
Don't load global state when only restoring indices
tlrx Mar 24, 2018
ddae0d7
Apply feedback
tlrx Mar 26, 2018
063dec5
Add another test
tlrx Mar 26, 2018
3099553
Add another test with corrupted index metadata
tlrx Mar 26, 2018
496c6d8
Reduce number of indices/shards/docs
tlrx Mar 27, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 14 additions & 7 deletions server/src/main/java/org/elasticsearch/repositories/Repository.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

import org.apache.lucene.index.IndexCommit;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.metadata.RepositoryMetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
Expand Down Expand Up @@ -78,15 +79,21 @@ interface Factory {
SnapshotInfo getSnapshotInfo(SnapshotId snapshotId);

/**
* Returns global metadata associate with the snapshot.
* <p>
* The returned meta data contains global metadata as well as metadata for all indices listed in the indices parameter.
* Returns global metadata associated with the snapshot.
*
* @param snapshot snapshot
* @param indices list of indices
* @return information about snapshot
* @param snapshotId the snapshot id to load the global metadata from
* @return the global metadata about the snapshot
*/
MetaData getSnapshotGlobalMetaData(SnapshotId snapshotId);

/**
* Returns the index metadata associated with the snapshot.
*
* @param snapshotId the snapshot id to load the index metadata from
* @param index the {@link IndexId} to load the metadata from
* @return the index metadata about the given index for the given snapshot
*/
MetaData getSnapshotMetaData(SnapshotInfo snapshot, List<IndexId> indices) throws IOException;
IndexMetaData getSnapshotIndexMetaData(SnapshotId snapshotId, IndexId index) throws IOException;

/**
* Returns a {@link RepositoryData} to describe the data in the repository, including the snapshots
Expand Down
70 changes: 43 additions & 27 deletions server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -480,11 +480,6 @@ public SnapshotInfo finalizeSnapshot(final SnapshotId snapshotId,
return blobStoreSnapshot;
}

@Override
public MetaData getSnapshotMetaData(SnapshotInfo snapshot, List<IndexId> indices) throws IOException {
return readSnapshotMetaData(snapshot.snapshotId(), snapshot.version(), indices, false);
}

@Override
public SnapshotInfo getSnapshotInfo(final SnapshotId snapshotId) {
try {
Expand All @@ -496,38 +491,59 @@ public SnapshotInfo getSnapshotInfo(final SnapshotId snapshotId) {
}
}

private MetaData readSnapshotMetaData(SnapshotId snapshotId, Version snapshotVersion, List<IndexId> indices, boolean ignoreIndexErrors) throws IOException {
MetaData metaData;
@Override
public MetaData getSnapshotGlobalMetaData(final SnapshotId snapshotId) {
try {
return globalMetaDataFormat.read(snapshotsBlobContainer, snapshotId.getUUID());
} catch (NoSuchFileException ex) {
throw new SnapshotMissingException(metadata.name(), snapshotId, ex);
} catch (IOException ex) {
throw new SnapshotException(metadata.name(), snapshotId, "failed to read global metadata", ex);
}
}

@Override
public IndexMetaData getSnapshotIndexMetaData(final SnapshotId snapshotId, final IndexId index) throws IOException {
final BlobPath indexPath = basePath().add("indices").add(index.getId());
return indexMetaDataFormat.read(blobStore().blobContainer(indexPath), snapshotId.getUUID());
}

/**
* Returns the global metadata associated with the snapshot.
* <p>
* The returned meta data contains global metadata as well as metadata
* for all indices listed in the indices parameter.
*/
private MetaData readSnapshotMetaData(final SnapshotId snapshotId,
final Version snapshotVersion,
final List<IndexId> indices,
final boolean ignoreErrors) throws IOException {
if (snapshotVersion == null) {
// When we delete corrupted snapshots we might not know which version we are dealing with
// We can try detecting the version based on the metadata file format
assert ignoreIndexErrors;
assert ignoreErrors;
if (globalMetaDataFormat.exists(snapshotsBlobContainer, snapshotId.getUUID()) == false) {
throw new SnapshotMissingException(metadata.name(), snapshotId);
}
}
try {
metaData = globalMetaDataFormat.read(snapshotsBlobContainer, snapshotId.getUUID());
} catch (NoSuchFileException ex) {
throw new SnapshotMissingException(metadata.name(), snapshotId, ex);
} catch (IOException ex) {
throw new SnapshotException(metadata.name(), snapshotId, "failed to get snapshots", ex);
}
MetaData.Builder metaDataBuilder = MetaData.builder(metaData);
for (IndexId index : indices) {
BlobPath indexPath = basePath().add("indices").add(index.getId());
BlobContainer indexMetaDataBlobContainer = blobStore().blobContainer(indexPath);
try {
metaDataBuilder.put(indexMetaDataFormat.read(indexMetaDataBlobContainer, snapshotId.getUUID()), false);
} catch (ElasticsearchParseException | IOException ex) {
if (ignoreIndexErrors) {
logger.warn(() -> new ParameterizedMessage("[{}] [{}] failed to read metadata for index", snapshotId, index.getName()), ex);
} else {
throw ex;

final MetaData.Builder metaData = MetaData.builder(getSnapshotGlobalMetaData(snapshotId));
if (indices != null) {
for (IndexId index : indices) {
try {
metaData.put(getSnapshotIndexMetaData(snapshotId, index), false);
} catch (ElasticsearchParseException | IOException ex) {
if (ignoreErrors == false) {
throw new SnapshotException(metadata.name(), snapshotId,
"[" + index.getName() + "] failed to read metadata for index", ex);
} else {
logger.warn(() ->
new ParameterizedMessage("[{}] [{}] failed to read metadata for index", snapshotId, index.getName()), ex);
}
}
}
}
return metaDataBuilder.build();
return metaData.build();
}

/**
Expand Down
89 changes: 52 additions & 37 deletions server/src/main/java/org/elasticsearch/snapshots/RestoreService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@
import org.elasticsearch.index.Index;
import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.repositories.IndexId;
import org.elasticsearch.repositories.RepositoriesService;
import org.elasticsearch.repositories.Repository;
import org.elasticsearch.repositories.RepositoryData;
Expand All @@ -91,6 +92,7 @@
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_VERSION_CREATED;
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_VERSION_UPGRADED;
import static org.elasticsearch.common.util.set.Sets.newHashSet;
import static org.elasticsearch.snapshots.SnapshotUtils.filterIndices;

/**
* Service responsible for restoring snapshots
Expand Down Expand Up @@ -182,17 +184,34 @@ public void restoreSnapshot(final RestoreRequest request, final ActionListener<R
if (matchingSnapshotId.isPresent() == false) {
throw new SnapshotRestoreException(request.repositoryName, request.snapshotName, "snapshot does not exist");
}

final SnapshotId snapshotId = matchingSnapshotId.get();
final SnapshotInfo snapshotInfo = repository.getSnapshotInfo(snapshotId);
final Snapshot snapshot = new Snapshot(request.repositoryName, snapshotId);
List<String> filteredIndices = SnapshotUtils.filterIndices(snapshotInfo.indices(), request.indices(), request.indicesOptions());
final MetaData metaData = repository.getSnapshotMetaData(snapshotInfo, repositoryData.resolveIndices(filteredIndices));

// Make sure that we can restore from this snapshot
validateSnapshotRestorable(request.repositoryName, snapshotInfo);

// Find list of indices that we need to restore
final Map<String, String> renamedIndices = renamedIndices(request, filteredIndices);
// Resolve the indices from the snapshot that need to be restored
final List<String> indicesInSnapshot = filterIndices(snapshotInfo.indices(), request.indices(), request.indicesOptions());

final MetaData.Builder metaDataBuilder;
if (request.includeGlobalState()) {
metaDataBuilder = MetaData.builder(repository.getSnapshotGlobalMetaData(snapshotId));
} else {
metaDataBuilder = MetaData.builder();
}

final List<IndexId> indexIdsInSnapshot = repositoryData.resolveIndices(indicesInSnapshot);
for (IndexId indexId : indexIdsInSnapshot) {
metaDataBuilder.put(repository.getSnapshotIndexMetaData(snapshotId, indexId), false);
}

final MetaData metaData = metaDataBuilder.build();

// Apply renaming on index names, returning a map of names where
// the key is the renamed index and the value is the original name
final Map<String, String> indices = renamedIndices(request, indicesInSnapshot);

// Now we can start the actual restore process by adding shards to be recovered in the cluster state
// and updating cluster metadata (global and index) as needed
Expand Down Expand Up @@ -222,12 +241,13 @@ public ClusterState execute(ClusterState currentState) {
RoutingTable.Builder rtBuilder = RoutingTable.builder(currentState.routingTable());
ImmutableOpenMap<ShardId, RestoreInProgress.ShardRestoreStatus> shards;
Set<String> aliases = new HashSet<>();
if (!renamedIndices.isEmpty()) {

if (indices.isEmpty() == false) {
// We have some indices to restore
ImmutableOpenMap.Builder<ShardId, RestoreInProgress.ShardRestoreStatus> shardsBuilder = ImmutableOpenMap.builder();
final Version minIndexCompatibilityVersion = currentState.getNodes().getMaxNodeVersion()
.minimumIndexCompatibilityVersion();
for (Map.Entry<String, String> indexEntry : renamedIndices.entrySet()) {
for (Map.Entry<String, String> indexEntry : indices.entrySet()) {
String index = indexEntry.getValue();
boolean partial = checkPartial(index);
SnapshotRecoverySource recoverySource = new SnapshotRecoverySource(snapshot, snapshotInfo.version(), index);
Expand Down Expand Up @@ -304,21 +324,42 @@ public ClusterState execute(ClusterState currentState) {
}

shards = shardsBuilder.build();
RestoreInProgress.Entry restoreEntry = new RestoreInProgress.Entry(snapshot, overallState(RestoreInProgress.State.INIT, shards), Collections.unmodifiableList(new ArrayList<>(renamedIndices.keySet())), shards);
RestoreInProgress.Entry restoreEntry = new RestoreInProgress.Entry(snapshot, overallState(RestoreInProgress.State.INIT, shards), Collections.unmodifiableList(new ArrayList<>(indices.keySet())), shards);
builder.putCustom(RestoreInProgress.TYPE, new RestoreInProgress(restoreEntry));
} else {
shards = ImmutableOpenMap.of();
}

checkAliasNameConflicts(renamedIndices, aliases);
checkAliasNameConflicts(indices, aliases);

// Restore global state if needed
restoreGlobalStateIfRequested(mdBuilder);
if (request.includeGlobalState()) {
if (metaData.persistentSettings() != null) {
Settings settings = metaData.persistentSettings();
clusterSettings.validateUpdate(settings);
mdBuilder.persistentSettings(settings);
}
if (metaData.templates() != null) {
// TODO: Should all existing templates be deleted first?
for (ObjectCursor<IndexTemplateMetaData> cursor : metaData.templates().values()) {
mdBuilder.put(cursor.value);
}
}
if (metaData.customs() != null) {
for (ObjectObjectCursor<String, MetaData.Custom> cursor : metaData.customs()) {
if (!RepositoriesMetaData.TYPE.equals(cursor.key)) {
// Don't restore repositories while we are working with them
// TODO: Should we restore them at the end?
mdBuilder.putCustom(cursor.key, cursor.value);
}
}
}
}

if (completed(shards)) {
// We don't have any indices to restore - we are done
restoreInfo = new RestoreInfo(snapshotId.getName(),
Collections.unmodifiableList(new ArrayList<>(renamedIndices.keySet())),
Collections.unmodifiableList(new ArrayList<>(indices.keySet())),
shards.size(),
shards.size() - failedShards(shards));
}
Expand Down Expand Up @@ -426,32 +467,6 @@ private IndexMetaData updateIndexSettings(IndexMetaData indexMetaData, Settings
return builder.settings(settingsBuilder).build();
}

private void restoreGlobalStateIfRequested(MetaData.Builder mdBuilder) {
if (request.includeGlobalState()) {
if (metaData.persistentSettings() != null) {
Settings settings = metaData.persistentSettings();
clusterSettings.validateUpdate(settings);
mdBuilder.persistentSettings(settings);
}
if (metaData.templates() != null) {
// TODO: Should all existing templates be deleted first?
for (ObjectCursor<IndexTemplateMetaData> cursor : metaData.templates().values()) {
mdBuilder.put(cursor.value);
}
}
if (metaData.customs() != null) {
for (ObjectObjectCursor<String, MetaData.Custom> cursor : metaData.customs()) {
if (!RepositoriesMetaData.TYPE.equals(cursor.key)) {
// Don't restore repositories while we are working with them
// TODO: Should we restore them at the end?
mdBuilder.putCustom(cursor.key, cursor.value);
}
}
}
}
}


@Override
public void onFailure(String source, Exception e) {
logger.warn(() -> new ParameterizedMessage("[{}] failed to restore snapshot", snapshotId), e);
Expand Down Expand Up @@ -757,7 +772,7 @@ private Map<String, String> renamedIndices(RestoreRequest request, List<String>
"indices [" + index + "] and [" + previousIndex + "] are renamed into the same index [" + renamedIndex + "]");
}
}
return renamedIndices;
return Collections.unmodifiableMap(renamedIndices);
}

/**
Expand Down
13 changes: 6 additions & 7 deletions server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ public RepositoryData getRepositoryData(final String repositoryName) {
* @throws SnapshotMissingException if snapshot is not found
*/
public SnapshotInfo snapshot(final String repositoryName, final SnapshotId snapshotId) {
List<SnapshotsInProgress.Entry> entries = currentSnapshots(repositoryName, Arrays.asList(snapshotId.getName()));
List<SnapshotsInProgress.Entry> entries = currentSnapshots(repositoryName, Collections.singletonList(snapshotId.getName()));
if (!entries.isEmpty()) {
return inProgressSnapshot(entries.iterator().next());
}
Expand Down Expand Up @@ -593,13 +593,13 @@ public List<SnapshotsInProgress.Entry> currentSnapshots(final String repository,
*/
public Map<ShardId, IndexShardSnapshotStatus> snapshotShards(final String repositoryName,
final SnapshotInfo snapshotInfo) throws IOException {
Map<ShardId, IndexShardSnapshotStatus> shardStatus = new HashMap<>();
Repository repository = repositoriesService.repository(repositoryName);
RepositoryData repositoryData = repository.getRepositoryData();
MetaData metaData = repository.getSnapshotMetaData(snapshotInfo, repositoryData.resolveIndices(snapshotInfo.indices()));
final Repository repository = repositoriesService.repository(repositoryName);
final RepositoryData repositoryData = repository.getRepositoryData();

final Map<ShardId, IndexShardSnapshotStatus> shardStatus = new HashMap<>();
for (String index : snapshotInfo.indices()) {
IndexId indexId = repositoryData.resolveIndexId(index);
IndexMetaData indexMetaData = metaData.indices().get(index);
IndexMetaData indexMetaData = repository.getSnapshotIndexMetaData(snapshotInfo.snapshotId(), indexId);
if (indexMetaData != null) {
int numberOfShards = indexMetaData.getNumberOfShards();
for (int i = 0; i < numberOfShards; i++) {
Expand Down Expand Up @@ -633,7 +633,6 @@ public Map<ShardId, IndexShardSnapshotStatus> snapshotShards(final String reposi
return unmodifiableMap(shardStatus);
}


private SnapshotShardFailure findShardFailure(List<SnapshotShardFailure> shardFailures, ShardId shardId) {
for (SnapshotShardFailure shardFailure : shardFailures) {
if (shardId.getIndexName().equals(shardFailure.index()) && shardId.getId() == shardFailure.shardId()) {
Expand Down
7 changes: 6 additions & 1 deletion server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2605,7 +2605,12 @@ public SnapshotInfo getSnapshotInfo(SnapshotId snapshotId) {
}

@Override
public MetaData getSnapshotMetaData(SnapshotInfo snapshot, List<IndexId> indices) throws IOException {
public MetaData getSnapshotGlobalMetaData(SnapshotId snapshotId) {
return null;
}

@Override
public IndexMetaData getSnapshotIndexMetaData(SnapshotId snapshotId, IndexId index) throws IOException {
return null;
}

Expand Down
Loading