Allow searchable snapshots to work on source-only repositories (#67975)

Provides the ability to mount a snapshot from a source-only repository as a searchable snapshot. This allows combining the space benefits of source-only snapshots with the query capabilities of searchable snapshots and runtime fields.
elastic · Jan 26, 2021 · 48d7902 · 48d7902
1 parent 4f3749c
commit 48d7902
Show file tree

Hide file tree

Showing 6 changed files with 61 additions and 12 deletions.
diff --git a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java
@@ -51,6 +51,10 @@ public FilterRepository(Repository in) {
         this.in = in;
     }
 
+    public Repository getDelegate() {
+        return in;
+    }
+
     @Override
     public RepositoryMetadata getMetadata() {
         return in.getMetadata();

diff --git a/...k/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java b/...k/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.env.ShardLock;
+import org.elasticsearch.index.engine.EngineConfig;
 import org.elasticsearch.index.engine.EngineFactory;
 import org.elasticsearch.index.engine.ReadOnlyEngine;
 import org.elasticsearch.index.mapper.MapperService;
@@ -172,13 +173,17 @@ protected void closeInternal() {
      */
     public static EngineFactory getEngineFactory() {
         return config -> new ReadOnlyEngine(config, null, new TranslogStats(0, 0, 0, 0, 0), true,
-            reader -> {
-                try {
-                    return SeqIdGeneratingFilterReader.wrap(reader, config.getPrimaryTermSupplier().getAsLong());
-                } catch (IOException e) {
-                    throw new UncheckedIOException(e);
-                }
-            }, true);
+            readerWrapper(config), true);
+    }
+
+    public static Function<DirectoryReader, DirectoryReader> readerWrapper(EngineConfig engineConfig) {
+        return reader -> {
+            try {
+                return SeqIdGeneratingFilterReader.wrap(reader, engineConfig.getPrimaryTermSupplier().getAsLong());
+            } catch (IOException e) {
+                throw new UncheckedIOException(e);
+            }
+        };
     }
 
     /**

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java
@@ -86,6 +86,7 @@
 import org.elasticsearch.xpack.core.ssl.SSLConfigurationReloader;
 import org.elasticsearch.xpack.core.ssl.SSLService;
 import org.elasticsearch.xpack.core.watcher.WatcherMetadata;
+import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants;
 
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -363,7 +364,8 @@ public Map<String, Repository.Factory> getRepositories(Environment env, NamedXCo
 
     @Override
     public Optional<EngineFactory> getEngineFactory(IndexSettings indexSettings) {
-        if (indexSettings.getValue(SourceOnlySnapshotRepository.SOURCE_ONLY)) {
+        if (indexSettings.getValue(SourceOnlySnapshotRepository.SOURCE_ONLY) &&
+            SearchableSnapshotsConstants.isSearchableSnapshotStore(indexSettings.getSettings()) == false) {
             return Optional.of(SourceOnlySnapshotRepository.getEngineFactory());
         }
 

diff --git a/...le-snapshots/src/main/java/org/elasticsearch/index/store/SearchableSnapshotDirectory.java b/...le-snapshots/src/main/java/org/elasticsearch/index/store/SearchableSnapshotDirectory.java
@@ -53,6 +53,7 @@
 import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
 import org.elasticsearch.snapshots.SnapshotId;
+import org.elasticsearch.snapshots.SourceOnlySnapshotRepository;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants;
 import org.elasticsearch.xpack.searchablesnapshots.cache.CacheService;
@@ -563,7 +564,10 @@ public static Directory create(
         }
 
         final String repositoryName = SNAPSHOT_REPOSITORY_SETTING.get(indexSettings.getSettings());
-        final Repository repository = repositories.repository(repositoryName);
+        Repository repository = repositories.repository(repositoryName);
+        if (repository instanceof SourceOnlySnapshotRepository) {
+            repository = ((SourceOnlySnapshotRepository) repository).getDelegate();
+        }
         if (repository instanceof BlobStoreRepository == false) {
             throw new IllegalArgumentException("Repository [" + repository + "] is not searchable");
         }

diff --git a/...pshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java b/...pshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java
@@ -55,6 +55,7 @@
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
 import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.snapshots.SourceOnlySnapshotRepository;
 import org.elasticsearch.threadpool.ExecutorBuilder;
 import org.elasticsearch.threadpool.ScalingExecutorBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -301,7 +302,16 @@ public Optional<EngineFactory> getEngineFactory(IndexSettings indexSettings) {
         if (SearchableSnapshotsConstants.isSearchableSnapshotStore(indexSettings.getSettings())
             && indexSettings.getSettings().getAsBoolean("index.frozen", false) == false) {
             return Optional.of(
-                engineConfig -> new ReadOnlyEngine(engineConfig, null, new TranslogStats(), false, Function.identity(), false)
+                engineConfig -> new ReadOnlyEngine(
+                    engineConfig,
+                    null,
+                    new TranslogStats(),
+                    false,
+                    indexSettings.getValue(SourceOnlySnapshotRepository.SOURCE_ONLY)
+                        ? SourceOnlySnapshotRepository.readerWrapper(engineConfig)
+                        : Function.identity(),
+                    false
+                )
             );
         }
         return Optional.empty();

diff --git a/.../org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java b/.../org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java
@@ -47,11 +47,18 @@ public abstract class AbstractSearchableSnapshotsRestTestCase extends ESRestTest
     protected abstract Settings repositorySettings();
 
     private void runSearchableSnapshotsTest(SearchableSnapshotsTestCaseBody testCaseBody) throws Exception {
+        runSearchableSnapshotsTest(testCaseBody, false);
+    }
+
+    private void runSearchableSnapshotsTest(SearchableSnapshotsTestCaseBody testCaseBody, boolean sourceOnly) throws Exception {
         final String repositoryType = repositoryType();
-        final Settings repositorySettings = repositorySettings();
+        Settings repositorySettings = repositorySettings();
+        if (sourceOnly) {
+            repositorySettings = Settings.builder().put("delegate_type", repositoryType).put(repositorySettings).build();
+        }
 
         logger.info("creating repository [{}] of type [{}]", REPOSITORY_NAME, repositoryType);
-        registerRepository(REPOSITORY_NAME, repositoryType, true, repositorySettings);
+        registerRepository(REPOSITORY_NAME, sourceOnly ? "source" : repositoryType, true, repositorySettings);
 
         final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
         final int numberOfShards = randomIntBetween(1, 5);
@@ -140,6 +147,23 @@ public void testSearchResultsWhenFrozen() throws Exception {
         });
     }
 
+    public void testSourceOnlyRepository() throws Exception {
+        runSearchableSnapshotsTest((indexName, numDocs) -> {
+            for (int i = 0; i < 10; i++) {
+                if (randomBoolean()) {
+                    logger.info("clearing searchable snapshots cache for [{}] before search", indexName);
+                    clearCache(indexName);
+                }
+                Map<String, Object> searchResults = search(
+                    indexName,
+                    QueryBuilders.matchAllQuery(),
+                    randomFrom(Boolean.TRUE, Boolean.FALSE, null)
+                );
+                assertThat(extractValue(searchResults, "hits.total.value"), equalTo(numDocs));
+            }
+        }, true);
+    }
+
     public void testCloseAndReopen() throws Exception {
         runSearchableSnapshotsTest((restoredIndexName, numDocs) -> {
             closeIndex(restoredIndexName);