From eded7c9cf721e5b98e7da864ac37a18dd34e591f Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Sat, 15 Dec 2018 11:11:31 +0100 Subject: [PATCH 1/3] Add hybridfs store type With this commit we introduce a new store type `hybridfs` that is a hybrid between `mmapfs` and `niofs`. This store type chooses different strategies to read Lucene files based on the read access pattern (random or linear) in order to optimize performance. This store type has been available in earlier versions of Elasticsearch as `default_fs`. We have chosen a different name now in order to convey the intent of the store type instead of tying it to the fact whether it is the default choice. --- docs/reference/index-modules/store.asciidoc | 22 ++++++-- .../org/elasticsearch/index/IndexModule.java | 5 +- .../index/store/FsDirectoryService.java | 56 ++++++++++++------- 3 files changed, 56 insertions(+), 27 deletions(-) diff --git a/docs/reference/index-modules/store.asciidoc b/docs/reference/index-modules/store.asciidoc index c2b3d700e9b7c..a5478a5bc120a 100644 --- a/docs/reference/index-modules/store.asciidoc +++ b/docs/reference/index-modules/store.asciidoc @@ -40,7 +40,7 @@ The following sections lists all the different storage types supported. `fs`:: Default file system implementation. This will pick the best implementation -depending on the operating environment, which is currently `mmapfs` on all +depending on the operating environment, which is currently `hybridfs` on all supported systems but is subject to change. [[simplefs]]`simplefs`:: @@ -67,12 +67,22 @@ process equal to the size of the file being mapped. Before using this class, be sure you have allowed plenty of <>. +[[hybridfs]]`hybridfs`:: + +The `hybridfs` type is a hybrid of `niofs` and `mmapfs`, which chooses the best +file system type for each type of file based on the read access pattern. +Currently only the Lucene term dictionary, norms and doc values files are +memory mapped. All other files are opened using Lucene `NIOFSDirectory`. +Similarly to `mmapfs` be sure you have allowed plenty of +<>. + [[allow-mmapfs]] -You can restrict the use of the `mmapfs` store type via the setting -`node.store.allow_mmapfs`. This is a boolean setting indicating whether or not -`mmapfs` is allowed. The default is to allow `mmapfs`. This setting is useful, -for example, if you are in an environment where you can not control the ability -to create a lot of memory maps so you need disable the ability to use `mmapfs`. +You can restrict the use of the `mmapfs` and the related `hybridfs` store type +via the setting `node.store.allow_mmapfs`. This is a boolean setting indicating +whether or not memory-mapping is allowed. The default is to allow it. This +setting is useful, for example, if you are in an environment where you can not +control the ability to create a lot of memory maps so you need disable the +ability to use memory-mapping. === Pre-loading data into the file system cache diff --git a/server/src/main/java/org/elasticsearch/index/IndexModule.java b/server/src/main/java/org/elasticsearch/index/IndexModule.java index 7f2eae492fd56..288fed89d2ce9 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexModule.java +++ b/server/src/main/java/org/elasticsearch/index/IndexModule.java @@ -302,6 +302,7 @@ public static boolean isBuiltinType(String storeType) { public enum Type { + HYBRIDFS("hybridfs"), NIOFS("niofs"), MMAPFS("mmapfs"), SIMPLEFS("simplefs"), @@ -330,7 +331,7 @@ public String getSettingsKey() { public static Type fromSettingsKey(final String key) { final Type type = TYPES.get(key); if (type == null) { - throw new IllegalArgumentException("no matching type for [" + key + "]"); + throw new IllegalArgumentException("no matching store type for [" + key + "]"); } return type; } @@ -356,7 +357,7 @@ public interface IndexSearcherWrapperFactory { public static Type defaultStoreType(final boolean allowMmapfs) { if (allowMmapfs && Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) { - return Type.MMAPFS; + return Type.HYBRIDFS; } else if (Constants.WINDOWS) { return Type.SIMPLEFS; } else { diff --git a/server/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java b/server/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java index f95cdb3a9f692..e391679407fc4 100644 --- a/server/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java +++ b/server/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.store; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.FileSwitchDirectory; import org.apache.lucene.store.LockFactory; import org.apache.lucene.store.MMapDirectory; @@ -30,6 +31,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.ShardPath; @@ -37,10 +39,17 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Collections; import java.util.HashSet; import java.util.Set; public class FsDirectoryService extends DirectoryService { + /* + * We are mmapping norms, docvalues as well as term dictionaries, all other files are served through NIOFS + * this provides good random access performance while not creating unnecessary mmaps for files like stored + * fields etc. + */ + private static final Set PRIMARY_EXTENSIONS = Collections.unmodifiableSet(Sets.newHashSet("nvd", "dvd", "tim")); protected final IndexStore indexStore; public static final Setting INDEX_LOCK_FACTOR_SETTING = new Setting<>("index.store.fs.fs_lock", "native", (s) -> { @@ -78,27 +87,36 @@ public Directory newDirectory() throws IOException { protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException { final String storeType = indexSettings.getSettings().get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.FS.getSettingsKey()); + IndexModule.Type type; if (IndexModule.Type.FS.match(storeType)) { - final IndexModule.Type type = - IndexModule.defaultStoreType(IndexModule.NODE_STORE_ALLOW_MMAPFS.get(indexSettings.getNodeSettings())); - switch (type) { - case MMAPFS: - return new MMapDirectory(location, lockFactory); - case SIMPLEFS: - return new SimpleFSDirectory(location, lockFactory); - case NIOFS: - return new NIOFSDirectory(location, lockFactory); - default: - throw new AssertionError("unexpected built-in store type [" + type + "]"); - } - } else if (IndexModule.Type.SIMPLEFS.match(storeType)) { - return new SimpleFSDirectory(location, lockFactory); - } else if (IndexModule.Type.NIOFS.match(storeType)) { - return new NIOFSDirectory(location, lockFactory); - } else if (IndexModule.Type.MMAPFS.match(storeType)) { - return new MMapDirectory(location, lockFactory); + type = IndexModule.defaultStoreType(IndexModule.NODE_STORE_ALLOW_MMAPFS.get(indexSettings.getNodeSettings())); + } else { + type = IndexModule.Type.fromSettingsKey(storeType); + } + switch (type) { + case HYBRIDFS: + // Use Lucene defaults + final FSDirectory primaryDirectory = FSDirectory.open(location, lockFactory); + if (primaryDirectory instanceof MMapDirectory) { + return new FileSwitchDirectory(PRIMARY_EXTENSIONS, primaryDirectory, new NIOFSDirectory(location, lockFactory), true) { + @Override + public String[] listAll() throws IOException { + // Avoid doing listAll twice: + return primaryDirectory.listAll(); + } + }; + } else { + return primaryDirectory; + } + case MMAPFS: + return new MMapDirectory(location, lockFactory); + case SIMPLEFS: + return new SimpleFSDirectory(location, lockFactory); + case NIOFS: + return new NIOFSDirectory(location, lockFactory); + default: + throw new AssertionError("unexpected built-in store type [" + type + "]"); } - throw new IllegalArgumentException("No directory found for type [" + storeType + "]"); } private static Directory setPreload(Directory directory, Path location, LockFactory lockFactory, From 3b9d1b623f5b467655a46cf77c634be5de7b7768 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Sat, 15 Dec 2018 11:36:32 +0100 Subject: [PATCH 2/3] Add hybridfs to IndexStoreTests --- .../java/org/elasticsearch/index/store/IndexStoreTests.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java b/server/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java index 155473c83cf30..8d377703cc898 100644 --- a/server/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java +++ b/server/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.store; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FileSwitchDirectory; import org.apache.lucene.store.MMapDirectory; import org.apache.lucene.store.NIOFSDirectory; import org.apache.lucene.store.NoLockFactory; @@ -64,6 +65,9 @@ private void doTestStoreDirectory(Index index, Path tempDir, String typeSettingV new ShardPath(false, tempDir, tempDir, new ShardId(index, 0))); try (Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) { switch (type) { + case HYBRIDFS: + assertTrue(type + " " + directory.toString(), directory instanceof FileSwitchDirectory); + break; case NIOFS: assertTrue(type + " " + directory.toString(), directory instanceof NIOFSDirectory); break; @@ -75,7 +79,7 @@ private void doTestStoreDirectory(Index index, Path tempDir, String typeSettingV break; case FS: if (Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) { - assertTrue(directory.toString(), directory instanceof MMapDirectory); + assertTrue(directory.toString(), directory instanceof FileSwitchDirectory); } else if (Constants.WINDOWS) { assertTrue(directory.toString(), directory instanceof SimpleFSDirectory); } else { From 870b0060563a1a8964d9e060fb4f4cff1264fcdf Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Mon, 17 Dec 2018 19:09:11 +0100 Subject: [PATCH 3/3] Address review comments --- .../elasticsearch/index/store/FsDirectoryService.java | 3 +-- .../org/elasticsearch/index/store/IndexStoreTests.java | 9 +++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java b/server/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java index e391679407fc4..345ef1eef41f6 100644 --- a/server/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java +++ b/server/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java @@ -46,8 +46,7 @@ public class FsDirectoryService extends DirectoryService { /* * We are mmapping norms, docvalues as well as term dictionaries, all other files are served through NIOFS - * this provides good random access performance while not creating unnecessary mmaps for files like stored - * fields etc. + * this provides good random access performance and does not lead to page cache thrashing. */ private static final Set PRIMARY_EXTENSIONS = Collections.unmodifiableSet(Sets.newHashSet("nvd", "dvd", "tim")); diff --git a/server/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java b/server/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java index 8d377703cc898..eb85219bcc2f0 100644 --- a/server/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java +++ b/server/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java @@ -66,7 +66,7 @@ private void doTestStoreDirectory(Index index, Path tempDir, String typeSettingV try (Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) { switch (type) { case HYBRIDFS: - assertTrue(type + " " + directory.toString(), directory instanceof FileSwitchDirectory); + assertHybridDirectory(directory); break; case NIOFS: assertTrue(type + " " + directory.toString(), directory instanceof NIOFSDirectory); @@ -79,7 +79,7 @@ private void doTestStoreDirectory(Index index, Path tempDir, String typeSettingV break; case FS: if (Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) { - assertTrue(directory.toString(), directory instanceof FileSwitchDirectory); + assertHybridDirectory(directory); } else if (Constants.WINDOWS) { assertTrue(directory.toString(), directory instanceof SimpleFSDirectory); } else { @@ -92,4 +92,9 @@ private void doTestStoreDirectory(Index index, Path tempDir, String typeSettingV } } + private void assertHybridDirectory(Directory directory) { + assertTrue(directory.toString(), directory instanceof FileSwitchDirectory); + Directory primaryDirectory = ((FileSwitchDirectory) directory).getPrimaryDir(); + assertTrue("primary directory " + primaryDirectory.toString(), primaryDirectory instanceof MMapDirectory); + } }