Skip to content

Commit

Permalink
Don't load global state when only restoring indices (#29239)
Browse files Browse the repository at this point in the history 
Restoring a snapshot, or getting the status of finished
snapshots, currently always load the global state metadata
 file from the repository even if it not required. This
slows down the restore process (or listing statuses process)
 and can also be an issue if the global state cannot be
deserialized (because it has unknown customs for example).

This commit splits the Repository.getSnapshotMetadata()
method into two distincts methods: getGlobalMetadata()
and getIndexMetadata() that are now called only when needed.
  • Loading branch information
@tlrx
tlrx authored Mar 28, 2018
1 parent 1f6a3c1 commit 36f8531
Show file tree
Hide file tree
Showing 7 changed files with 480 additions and 85 deletions.
21 changes: 14 additions & 7 deletions server/src/main/java/org/elasticsearch/repositories/Repository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

import org.apache.lucene.index.IndexCommit;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.metadata.RepositoryMetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
Expand Down Expand Up @@ -78,15 +79,21 @@ interface Factory {
SnapshotInfo getSnapshotInfo(SnapshotId snapshotId);

/**
* Returns global metadata associate with the snapshot.
* <p>
* The returned meta data contains global metadata as well as metadata for all indices listed in the indices parameter.
* Returns global metadata associated with the snapshot.
*
* @param snapshot snapshot
* @param indices list of indices
* @return information about snapshot
* @param snapshotId the snapshot id to load the global metadata from
* @return the global metadata about the snapshot
*/
MetaData getSnapshotGlobalMetaData(SnapshotId snapshotId);

/**
* Returns the index metadata associated with the snapshot.
*
* @param snapshotId the snapshot id to load the index metadata from
* @param index the {@link IndexId} to load the metadata from
* @return the index metadata about the given index for the given snapshot
*/
MetaData getSnapshotMetaData(SnapshotInfo snapshot, List<IndexId> indices) throws IOException;
IndexMetaData getSnapshotIndexMetaData(SnapshotId snapshotId, IndexId index) throws IOException;

/**
* Returns a {@link RepositoryData} to describe the data in the repository, including the snapshots
Expand Down
70 changes: 43 additions & 27 deletions server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -480,11 +480,6 @@ public SnapshotInfo finalizeSnapshot(final SnapshotId snapshotId,
return blobStoreSnapshot;
}

@Override
public MetaData getSnapshotMetaData(SnapshotInfo snapshot, List<IndexId> indices) throws IOException {
return readSnapshotMetaData(snapshot.snapshotId(), snapshot.version(), indices, false);
}

@Override
public SnapshotInfo getSnapshotInfo(final SnapshotId snapshotId) {
try {
Expand All @@ -496,38 +491,59 @@ public SnapshotInfo getSnapshotInfo(final SnapshotId snapshotId) {
}
}

private MetaData readSnapshotMetaData(SnapshotId snapshotId, Version snapshotVersion, List<IndexId> indices, boolean ignoreIndexErrors) throws IOException {
MetaData metaData;
@Override
public MetaData getSnapshotGlobalMetaData(final SnapshotId snapshotId) {
try {
return globalMetaDataFormat.read(snapshotsBlobContainer, snapshotId.getUUID());
} catch (NoSuchFileException ex) {
throw new SnapshotMissingException(metadata.name(), snapshotId, ex);
} catch (IOException ex) {
throw new SnapshotException(metadata.name(), snapshotId, "failed to read global metadata", ex);
}
}

@Override
public IndexMetaData getSnapshotIndexMetaData(final SnapshotId snapshotId, final IndexId index) throws IOException {
final BlobPath indexPath = basePath().add("indices").add(index.getId());
return indexMetaDataFormat.read(blobStore().blobContainer(indexPath), snapshotId.getUUID());
}

/**
* Returns the global metadata associated with the snapshot.
* <p>
* The returned meta data contains global metadata as well as metadata
* for all indices listed in the indices parameter.
*/
private MetaData readSnapshotMetaData(final SnapshotId snapshotId,
final Version snapshotVersion,
final List<IndexId> indices,
final boolean ignoreErrors) throws IOException {
if (snapshotVersion == null) {
// When we delete corrupted snapshots we might not know which version we are dealing with
// We can try detecting the version based on the metadata file format
assert ignoreIndexErrors;
assert ignoreErrors;
if (globalMetaDataFormat.exists(snapshotsBlobContainer, snapshotId.getUUID()) == false) {
throw new SnapshotMissingException(metadata.name(), snapshotId);
}
}
try {
metaData = globalMetaDataFormat.read(snapshotsBlobContainer, snapshotId.getUUID());
} catch (NoSuchFileException ex) {
throw new SnapshotMissingException(metadata.name(), snapshotId, ex);
} catch (IOException ex) {
throw new SnapshotException(metadata.name(), snapshotId, "failed to get snapshots", ex);
}
MetaData.Builder metaDataBuilder = MetaData.builder(metaData);
for (IndexId index : indices) {
BlobPath indexPath = basePath().add("indices").add(index.getId());
BlobContainer indexMetaDataBlobContainer = blobStore().blobContainer(indexPath);
try {
metaDataBuilder.put(indexMetaDataFormat.read(indexMetaDataBlobContainer, snapshotId.getUUID()), false);
} catch (ElasticsearchParseException | IOException ex) {
if (ignoreIndexErrors) {
logger.warn(() -> new ParameterizedMessage("[{}] [{}] failed to read metadata for index", snapshotId, index.getName()), ex);
} else {
throw ex;

final MetaData.Builder metaData = MetaData.builder(getSnapshotGlobalMetaData(snapshotId));
if (indices != null) {
for (IndexId index : indices) {
try {
metaData.put(getSnapshotIndexMetaData(snapshotId, index), false);
} catch (ElasticsearchParseException | IOException ex) {
if (ignoreErrors == false) {
throw new SnapshotException(metadata.name(), snapshotId,
"[" + index.getName() + "] failed to read metadata for index", ex);
} else {
logger.warn(() ->
new ParameterizedMessage("[{}] [{}] failed to read metadata for index", snapshotId, index.getName()), ex);
}
}
}
}
return metaDataBuilder.build();
return metaData.build();
}

/**
Expand Down
89 changes: 52 additions & 37 deletions server/src/main/java/org/elasticsearch/snapshots/RestoreService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@
import org.elasticsearch.index.Index;
import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.repositories.IndexId;
import org.elasticsearch.repositories.RepositoriesService;
import org.elasticsearch.repositories.Repository;
import org.elasticsearch.repositories.RepositoryData;
Expand All @@ -91,6 +92,7 @@
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_VERSION_CREATED;
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_VERSION_UPGRADED;
import static org.elasticsearch.common.util.set.Sets.newHashSet;
import static org.elasticsearch.snapshots.SnapshotUtils.filterIndices;

/**
* Service responsible for restoring snapshots
Expand Down Expand Up @@ -182,17 +184,34 @@ public void restoreSnapshot(final RestoreRequest request, final ActionListener<R
if (matchingSnapshotId.isPresent() == false) {
throw new SnapshotRestoreException(request.repositoryName, request.snapshotName, "snapshot does not exist");
}

final SnapshotId snapshotId = matchingSnapshotId.get();
final SnapshotInfo snapshotInfo = repository.getSnapshotInfo(snapshotId);
final Snapshot snapshot = new Snapshot(request.repositoryName, snapshotId);
List<String> filteredIndices = SnapshotUtils.filterIndices(snapshotInfo.indices(), request.indices(), request.indicesOptions());
final MetaData metaData = repository.getSnapshotMetaData(snapshotInfo, repositoryData.resolveIndices(filteredIndices));

// Make sure that we can restore from this snapshot
validateSnapshotRestorable(request.repositoryName, snapshotInfo);

// Find list of indices that we need to restore
final Map<String, String> renamedIndices = renamedIndices(request, filteredIndices);
// Resolve the indices from the snapshot that need to be restored
final List<String> indicesInSnapshot = filterIndices(snapshotInfo.indices(), request.indices(), request.indicesOptions());

final MetaData.Builder metaDataBuilder;
if (request.includeGlobalState()) {
metaDataBuilder = MetaData.builder(repository.getSnapshotGlobalMetaData(snapshotId));
} else {
metaDataBuilder = MetaData.builder();
}

final List<IndexId> indexIdsInSnapshot = repositoryData.resolveIndices(indicesInSnapshot);
for (IndexId indexId : indexIdsInSnapshot) {
metaDataBuilder.put(repository.getSnapshotIndexMetaData(snapshotId, indexId), false);
}

final MetaData metaData = metaDataBuilder.build();

// Apply renaming on index names, returning a map of names where
// the key is the renamed index and the value is the original name
final Map<String, String> indices = renamedIndices(request, indicesInSnapshot);

// Now we can start the actual restore process by adding shards to be recovered in the cluster state
// and updating cluster metadata (global and index) as needed
Expand Down Expand Up @@ -222,12 +241,13 @@ public ClusterState execute(ClusterState currentState) {
RoutingTable.Builder rtBuilder = RoutingTable.builder(currentState.routingTable());
ImmutableOpenMap<ShardId, RestoreInProgress.ShardRestoreStatus> shards;
Set<String> aliases = new HashSet<>();
if (!renamedIndices.isEmpty()) {

if (indices.isEmpty() == false) {
// We have some indices to restore
ImmutableOpenMap.Builder<ShardId, RestoreInProgress.ShardRestoreStatus> shardsBuilder = ImmutableOpenMap.builder();
final Version minIndexCompatibilityVersion = currentState.getNodes().getMaxNodeVersion()
.minimumIndexCompatibilityVersion();
for (Map.Entry<String, String> indexEntry : renamedIndices.entrySet()) {
for (Map.Entry<String, String> indexEntry : indices.entrySet()) {
String index = indexEntry.getValue();
boolean partial = checkPartial(index);
SnapshotRecoverySource recoverySource = new SnapshotRecoverySource(snapshot, snapshotInfo.version(), index);
Expand Down Expand Up @@ -304,21 +324,42 @@ public ClusterState execute(ClusterState currentState) {
}

shards = shardsBuilder.build();
RestoreInProgress.Entry restoreEntry = new RestoreInProgress.Entry(snapshot, overallState(RestoreInProgress.State.INIT, shards), Collections.unmodifiableList(new ArrayList<>(renamedIndices.keySet())), shards);
RestoreInProgress.Entry restoreEntry = new RestoreInProgress.Entry(snapshot, overallState(RestoreInProgress.State.INIT, shards), Collections.unmodifiableList(new ArrayList<>(indices.keySet())), shards);
builder.putCustom(RestoreInProgress.TYPE, new RestoreInProgress(restoreEntry));
} else {
shards = ImmutableOpenMap.of();
}

checkAliasNameConflicts(renamedIndices, aliases);
checkAliasNameConflicts(indices, aliases);

// Restore global state if needed
restoreGlobalStateIfRequested(mdBuilder);
if (request.includeGlobalState()) {
if (metaData.persistentSettings() != null) {
Settings settings = metaData.persistentSettings();
clusterSettings.validateUpdate(settings);
mdBuilder.persistentSettings(settings);
}
if (metaData.templates() != null) {
// TODO: Should all existing templates be deleted first?
for (ObjectCursor<IndexTemplateMetaData> cursor : metaData.templates().values()) {
mdBuilder.put(cursor.value);
}
}
if (metaData.customs() != null) {
for (ObjectObjectCursor<String, MetaData.Custom> cursor : metaData.customs()) {
if (!RepositoriesMetaData.TYPE.equals(cursor.key)) {
// Don't restore repositories while we are working with them
// TODO: Should we restore them at the end?
mdBuilder.putCustom(cursor.key, cursor.value);
}
}
}
}

if (completed(shards)) {
// We don't have any indices to restore - we are done
restoreInfo = new RestoreInfo(snapshotId.getName(),
Collections.unmodifiableList(new ArrayList<>(renamedIndices.keySet())),
Collections.unmodifiableList(new ArrayList<>(indices.keySet())),
shards.size(),
shards.size() - failedShards(shards));
}
Expand Down Expand Up @@ -426,32 +467,6 @@ private IndexMetaData updateIndexSettings(IndexMetaData indexMetaData, Settings
return builder.settings(settingsBuilder).build();
}

private void restoreGlobalStateIfRequested(MetaData.Builder mdBuilder) {
if (request.includeGlobalState()) {
if (metaData.persistentSettings() != null) {
Settings settings = metaData.persistentSettings();
clusterSettings.validateUpdate(settings);
mdBuilder.persistentSettings(settings);
}
if (metaData.templates() != null) {
// TODO: Should all existing templates be deleted first?
for (ObjectCursor<IndexTemplateMetaData> cursor : metaData.templates().values()) {
mdBuilder.put(cursor.value);
}
}
if (metaData.customs() != null) {
for (ObjectObjectCursor<String, MetaData.Custom> cursor : metaData.customs()) {
if (!RepositoriesMetaData.TYPE.equals(cursor.key)) {
// Don't restore repositories while we are working with them
// TODO: Should we restore them at the end?
mdBuilder.putCustom(cursor.key, cursor.value);
}
}
}
}
}


@Override
public void onFailure(String source, Exception e) {
logger.warn(() -> new ParameterizedMessage("[{}] failed to restore snapshot", snapshotId), e);
Expand Down Expand Up @@ -757,7 +772,7 @@ private Map<String, String> renamedIndices(RestoreRequest request, List<String>
"indices [" + index + "] and [" + previousIndex + "] are renamed into the same index [" + renamedIndex + "]");
}
}
return renamedIndices;
return Collections.unmodifiableMap(renamedIndices);
}

/**
Expand Down
13 changes: 6 additions & 7 deletions server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ public RepositoryData getRepositoryData(final String repositoryName) {
* @throws SnapshotMissingException if snapshot is not found
*/
public SnapshotInfo snapshot(final String repositoryName, final SnapshotId snapshotId) {
List<SnapshotsInProgress.Entry> entries = currentSnapshots(repositoryName, Arrays.asList(snapshotId.getName()));
List<SnapshotsInProgress.Entry> entries = currentSnapshots(repositoryName, Collections.singletonList(snapshotId.getName()));
if (!entries.isEmpty()) {
return inProgressSnapshot(entries.iterator().next());
}
Expand Down Expand Up @@ -593,13 +593,13 @@ public List<SnapshotsInProgress.Entry> currentSnapshots(final String repository,
*/
public Map<ShardId, IndexShardSnapshotStatus> snapshotShards(final String repositoryName,
final SnapshotInfo snapshotInfo) throws IOException {
Map<ShardId, IndexShardSnapshotStatus> shardStatus = new HashMap<>();
Repository repository = repositoriesService.repository(repositoryName);
RepositoryData repositoryData = repository.getRepositoryData();
MetaData metaData = repository.getSnapshotMetaData(snapshotInfo, repositoryData.resolveIndices(snapshotInfo.indices()));
final Repository repository = repositoriesService.repository(repositoryName);
final RepositoryData repositoryData = repository.getRepositoryData();

final Map<ShardId, IndexShardSnapshotStatus> shardStatus = new HashMap<>();
for (String index : snapshotInfo.indices()) {
IndexId indexId = repositoryData.resolveIndexId(index);
IndexMetaData indexMetaData = metaData.indices().get(index);
IndexMetaData indexMetaData = repository.getSnapshotIndexMetaData(snapshotInfo.snapshotId(), indexId);
if (indexMetaData != null) {
int numberOfShards = indexMetaData.getNumberOfShards();
for (int i = 0; i < numberOfShards; i++) {
Expand Down Expand Up @@ -633,7 +633,6 @@ public Map<ShardId, IndexShardSnapshotStatus> snapshotShards(final String reposi
return unmodifiableMap(shardStatus);
}


private SnapshotShardFailure findShardFailure(List<SnapshotShardFailure> shardFailures, ShardId shardId) {
for (SnapshotShardFailure shardFailure : shardFailures) {
if (shardId.getIndexName().equals(shardFailure.index()) && shardId.getId() == shardFailure.shardId()) {
Expand Down
7 changes: 6 additions & 1 deletion server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2605,7 +2605,12 @@ public SnapshotInfo getSnapshotInfo(SnapshotId snapshotId) {
}

@Override
public MetaData getSnapshotMetaData(SnapshotInfo snapshot, List<IndexId> indices) throws IOException {
public MetaData getSnapshotGlobalMetaData(SnapshotId snapshotId) {
return null;
}

@Override
public IndexMetaData getSnapshotIndexMetaData(SnapshotId snapshotId, IndexId index) throws IOException {
return null;
}

Expand Down
Loading

0 comments on commit 36f8531

Please sign in to comment.