Introducing ZStd compression codec plugin (opensearch-project#9658) (o…

…pensearch-project#9832) (opensearch-project#9844)

* introducing zstd compression codec plugin

* Moving zstd compression codec as a plugin

* introducing zstd compression codec plugin

* Adding checks to EngineConfig and fixing tests

* incorporating review comments

* fixing tests

* introducing zstd compression codec plugin

* addressing review comments

* nit fixes

* implementing codec aliases

* addressing review comments

* review comments

* moving codec aliases to custom codec

* adding zstd default codec for backward compatibility

* renaming to deprecated codec

* incorporating review comments

* nit fixes

---------

(cherry picked from commit 76f1b52)

(cherry picked from commit 90310c3)

Signed-off-by: Sarthak Aggarwal <sarthagg@amazon.com>
Signed-off-by: Prabhakar Sithanandam <backslasht@users.noreply.github.com>
Signed-off-by: Andrew Ross <andrross@amazon.com>
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: Sarthak Aggarwal <sarthagg@amazon.com>
Co-authored-by: Prabhakar Sithanandam <backslasht@users.noreply.github.com>

modules/reindex/build.gradle

@@ -69,6 +69,7 @@ dependencies {
   testImplementation project(':modules:transport-netty4')
   // for parent/child testing
   testImplementation project(':modules:parent-join')
+  testImplementation project(':plugins:custom-codecs')
 }
 
 restResources {
@@ -95,4 +96,5 @@ forbiddenPatterns {
 tasks.named("bundlePlugin").configure {
   dependsOn("copyParentJoinMetadata")
   dependsOn("copyTransportNetty4Metadata")
+  dependsOn("copyCustomCodecsMetadata")
 }

plugins/custom-codecs/build.gradle

@@ -0,0 +1,27 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+apply plugin: 'opensearch.opensearchplugin'
+apply plugin: 'opensearch.internal-cluster-test'
+
+opensearchplugin {
+  name 'custom-codecs'
+  description 'A plugin that implements custom compression codecs.'
+  classname 'org.opensearch.index.codec.customcodecs.CustomCodecPlugin'
+  licenseFile rootProject.file('licenses/APACHE-LICENSE-2.0.txt')
+  noticeFile rootProject.file('NOTICE.txt')
+}
+
+dependencies {
+  api "com.github.luben:zstd-jni:1.5.5-5"
+}
+
+testingConventions.enabled = false;

plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java

@@ -0,0 +1,47 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.codec.customcodecs;
+
+import org.opensearch.index.IndexSettings;
+import org.opensearch.index.codec.CodecServiceFactory;
+import org.opensearch.index.engine.EngineConfig;
+import org.opensearch.plugins.EnginePlugin;
+import org.opensearch.plugins.Plugin;
+
+import java.util.Optional;
+
+/**
+ * A plugin that implements custom codecs. Supports these codecs:
+ * <ul>
+ * <li>ZSTD
+ * <li>ZSTDNODICT
+ * </ul>
+ *
+ * @opensearch.internal
+ */
+public final class CustomCodecPlugin extends Plugin implements EnginePlugin {
+
+    /**
+     * Creates a new instance
+     */
+    public CustomCodecPlugin() {}
+
+    /**
+     * @param indexSettings is the default indexSettings
+     * @return the engine factory
+     */
+    @Override
+    public Optional<CodecServiceFactory> getCustomCodecServiceFactory(final IndexSettings indexSettings) {
+        String codecName = indexSettings.getValue(EngineConfig.INDEX_CODEC_SETTING);
+        if (codecName.equals(CustomCodecService.ZSTD_NO_DICT_CODEC) || codecName.equals(CustomCodecService.ZSTD_CODEC)) {
+            return Optional.of(new CustomCodecServiceFactory());
+        }
+        return Optional.empty();
+    }
+}

plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java

@@ -0,0 +1,72 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.codec.customcodecs;
+
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.codecs.Codec;
+import org.opensearch.common.collect.MapBuilder;
+import org.opensearch.index.IndexSettings;
+import org.opensearch.index.codec.CodecService;
+import org.opensearch.index.mapper.MapperService;
+
+import java.util.Arrays;
+import java.util.Map;
+import java.util.stream.Stream;
+
+import static org.opensearch.index.engine.EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING;
+
+/**
+ * CustomCodecService provides ZSTD and ZSTD_NO_DICT compression codecs.
+ */
+public class CustomCodecService extends CodecService {
+    private final Map<String, Codec> codecs;
+    /**
+     * ZStandard codec
+     */
+    public static final String ZSTD_CODEC = "zstd";
+    /**
+     * ZStandard without dictionary codec
+     */
+    public static final String ZSTD_NO_DICT_CODEC = "zstd_no_dict";
+
+    /**
+     * Creates a new CustomCodecService.
+     *
+     * @param mapperService The mapper service.
+     * @param indexSettings The index settings.
+     * @param logger        The logger.
+     */
+    public CustomCodecService(MapperService mapperService, IndexSettings indexSettings, Logger logger) {
+        super(mapperService, indexSettings, logger);
+        int compressionLevel = indexSettings.getValue(INDEX_CODEC_COMPRESSION_LEVEL_SETTING);
+        final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder();
+        if (mapperService == null) {
+            codecs.put(ZSTD_CODEC, new ZstdCodec(compressionLevel));
+            codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(compressionLevel));
+        } else {
+            codecs.put(ZSTD_CODEC, new ZstdCodec(mapperService, logger, compressionLevel));
+            codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(mapperService, logger, compressionLevel));
+        }
+        this.codecs = codecs.immutableMap();
+    }
+
+    @Override
+    public Codec codec(String name) {
+        Codec codec = codecs.get(name);
+        if (codec == null) {
+            return super.codec(name);
+        }
+        return codec;
+    }
+
+    @Override
+    public String[] availableCodecs() {
+        return Stream.concat(Arrays.stream(super.availableCodecs()), codecs.keySet().stream()).toArray(String[]::new);
+    }
+}

plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecServiceFactory.java

@@ -0,0 +1,27 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.codec.customcodecs;
+
+import org.opensearch.index.codec.CodecService;
+import org.opensearch.index.codec.CodecServiceConfig;
+import org.opensearch.index.codec.CodecServiceFactory;
+
+/**
+ * A factory for creating new {@link CodecService} instance
+ */
+public class CustomCodecServiceFactory implements CodecServiceFactory {
+
+    /** Creates a new instance. */
+    public CustomCodecServiceFactory() {}
+
+    @Override
+    public CodecService createCodecService(CodecServiceConfig config) {
+        return new CustomCodecService(config.getMapperService(), config.getIndexSettings(), config.getLogger());
+    }
+}

plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdDeprecatedCodec.java

@@ -0,0 +1,61 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.codec.customcodecs;
+
+import org.apache.logging.log4j.Logger;
+import org.opensearch.common.settings.Setting;
+import org.opensearch.index.codec.CodecSettings;
+import org.opensearch.index.engine.EngineConfig;
+import org.opensearch.index.mapper.MapperService;
+
+/**
+ * ZstdDeprecatedCodec provides ZSTD compressor using the <a href="https://github.com/luben/zstd-jni">zstd-jni</a> library.
+ * Added to support backward compatibility for indices created with Lucene95CustomCodec as codec name.
+ */
+@Deprecated(since = "2.10")
+public class ZstdDeprecatedCodec extends Lucene95CustomCodec implements CodecSettings {
+
+    /**
+     * Creates a new ZstdDefaultCodec instance with the default compression level.
+     */
+    public ZstdDeprecatedCodec() {
+        this(DEFAULT_COMPRESSION_LEVEL);
+    }
+
+    /**
+     * Creates a new ZstdDefaultCodec instance.
+     *
+     * @param compressionLevel The compression level.
+     */
+    public ZstdDeprecatedCodec(int compressionLevel) {
+        super(Mode.ZSTD_DEPRECATED, compressionLevel);
+    }
+
+    /**
+     * Creates a new ZstdDefaultCodec instance.
+     *
+     * @param mapperService The mapper service.
+     * @param logger The logger.
+     * @param compressionLevel The compression level.
+     */
+    public ZstdDeprecatedCodec(MapperService mapperService, Logger logger, int compressionLevel) {
+        super(Mode.ZSTD_DEPRECATED, compressionLevel, mapperService, logger);
+    }
+
+    /** The name for this codec. */
+    @Override
+    public String toString() {
+        return getClass().getSimpleName();
+    }
+
+    @Override
+    public boolean supports(Setting<?> setting) {
+        return setting.equals(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING);
+    }
+}

plugins/custom-codecs/src/main/plugin-metadata/plugin-security.policy

@@ -0,0 +1,11 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+grant codeBase "${codebase.zstd-jni}" {
+  permission java.lang.RuntimePermission "loadLibrary.*";
+};