diff --git a/CHANGELOG.md b/CHANGELOG.md index 581f5dd74617d..d1b5933aa809a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - GHA to verify checklist items completion in PR descriptions ([#10800](https://github.com/opensearch-project/OpenSearch/pull/10800)) - Allow to pass the list settings through environment variables (like [], ["a", "b", "c"], ...) ([#10625](https://github.com/opensearch-project/OpenSearch/pull/10625)) - [Admission Control] Integrate CPU AC with ResourceUsageCollector and add CPU AC stats to nodes/stats ([#10887](https://github.com/opensearch-project/OpenSearch/pull/10887)) +- Add support for dependencies in plugin descriptor properties with semver range ([#11441](https://github.com/opensearch-project/OpenSearch/pull/11441)) - [S3 Repository] Add setting to control connection count for sync client ([#12028](https://github.com/opensearch-project/OpenSearch/pull/12028)) ### Dependencies diff --git a/buildSrc/src/main/java/org/opensearch/gradle/RepositoriesSetupPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/RepositoriesSetupPlugin.java index 8ecfbf40b6c62..0c901b9726992 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/RepositoriesSetupPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/RepositoriesSetupPlugin.java @@ -94,7 +94,7 @@ public static void configureRepositories(Project project) { String revision = matcher.group(1); MavenArtifactRepository luceneRepo = repos.maven(repo -> { repo.setName("lucene-snapshots"); - repo.setUrl("https://artifacts.opensearch.org/snapshots/lucene/"); + repo.setUrl("https://ci.opensearch.org/ci/dbc/snapshots/lucene/"); }); repos.exclusiveContent(exclusiveRepo -> { exclusiveRepo.filter( diff --git a/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/ListPluginsCommand.java b/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/ListPluginsCommand.java index d269603656114..9ca42ac5f4ec1 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/ListPluginsCommand.java +++ b/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/ListPluginsCommand.java @@ -78,15 +78,14 @@ private void printPlugin(Environment env, Terminal terminal, Path plugin, String PluginInfo info = PluginInfo.readFromProperties(env.pluginsDir().resolve(plugin)); terminal.println(Terminal.Verbosity.SILENT, prefix + info.getName()); terminal.println(Terminal.Verbosity.VERBOSE, info.toString(prefix)); - if (info.getOpenSearchVersion().equals(Version.CURRENT) == false) { + if (!PluginsService.isPluginVersionCompatible(info, Version.CURRENT)) { terminal.errorPrintln( "WARNING: plugin [" + info.getName() + "] was built for OpenSearch version " - + info.getVersion() - + " but version " + + info.getOpenSearchVersionRangesString() + + " and is not compatible with " + Version.CURRENT - + " is required" ); } } diff --git a/distribution/tools/plugin-cli/src/test/java/org/opensearch/plugins/InstallPluginCommandTests.java b/distribution/tools/plugin-cli/src/test/java/org/opensearch/plugins/InstallPluginCommandTests.java index f4532f5f83cc4..c264788df20e8 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/opensearch/plugins/InstallPluginCommandTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/opensearch/plugins/InstallPluginCommandTests.java @@ -70,8 +70,10 @@ import org.opensearch.core.util.FileSystemUtils; import org.opensearch.env.Environment; import org.opensearch.env.TestEnvironment; +import org.opensearch.semver.SemverRange; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.PosixPermissionsResetter; +import org.opensearch.test.VersionUtils; import org.junit.After; import org.junit.Before; @@ -284,6 +286,35 @@ static void writePlugin(String name, Path structure, String... additionalProps) writeJar(structure.resolve("plugin.jar"), className); } + static void writePlugin(String name, Path structure, SemverRange opensearchVersionRange, String... additionalProps) throws IOException { + String[] properties = Stream.concat( + Stream.of( + "description", + "fake desc", + "name", + name, + "version", + "1.0", + "dependencies", + "{opensearch:\"" + opensearchVersionRange + "\"}", + "java.version", + System.getProperty("java.specification.version"), + "classname", + "FakePlugin" + ), + Arrays.stream(additionalProps) + ).toArray(String[]::new); + PluginTestUtil.writePluginProperties(structure, properties); + String className = name.substring(0, 1).toUpperCase(Locale.ENGLISH) + name.substring(1) + "Plugin"; + writeJar(structure.resolve("plugin.jar"), className); + } + + static Path createPlugin(String name, Path structure, SemverRange opensearchVersionRange, String... additionalProps) + throws IOException { + writePlugin(name, structure, opensearchVersionRange, additionalProps); + return writeZip(structure, null); + } + static void writePluginSecurityPolicy(Path pluginDir, String... permissions) throws IOException { StringBuilder securityPolicyContent = new StringBuilder("grant {\n "); for (String permission : permissions) { @@ -867,6 +898,32 @@ public void testInstallMisspelledOfficialPlugins() throws Exception { assertThat(e.getMessage(), containsString("Unknown plugin unknown_plugin")); } + public void testInstallPluginWithCompatibleDependencies() throws Exception { + Tuple env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); + String pluginZip = createPlugin("fake", pluginDir, SemverRange.fromString("~" + Version.CURRENT.toString())).toUri() + .toURL() + .toString(); + skipJarHellCommand.execute(terminal, Collections.singletonList(pluginZip), false, env.v2()); + assertThat(terminal.getOutput(), containsString("100%")); + } + + public void testInstallPluginWithIncompatibleDependencies() throws Exception { + Tuple env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); + // Core version is behind plugin version by one w.r.t patch, hence incompatible + Version coreVersion = Version.CURRENT; + Version pluginVersion = VersionUtils.getVersion(coreVersion.major, coreVersion.minor, (byte) (coreVersion.revision + 1)); + String pluginZip = createPlugin("fake", pluginDir, SemverRange.fromString("~" + pluginVersion.toString())).toUri() + .toURL() + .toString(); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> skipJarHellCommand.execute(terminal, Collections.singletonList(pluginZip), false, env.v2()) + ); + assertThat(e.getMessage(), containsString("Plugin [fake] was built for OpenSearch version ~" + pluginVersion)); + } + public void testBatchFlag() throws Exception { MockTerminal terminal = new MockTerminal(); installPlugin(terminal, true); diff --git a/distribution/tools/plugin-cli/src/test/java/org/opensearch/plugins/ListPluginsCommandTests.java b/distribution/tools/plugin-cli/src/test/java/org/opensearch/plugins/ListPluginsCommandTests.java index 7bbced38c7adb..6878efce4c804 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/opensearch/plugins/ListPluginsCommandTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/opensearch/plugins/ListPluginsCommandTests.java @@ -278,7 +278,7 @@ public void testExistingIncompatiblePlugin() throws Exception { buildFakePlugin(env, "fake desc 2", "fake_plugin2", "org.fake2"); MockTerminal terminal = listPlugins(home); - String message = "plugin [fake_plugin1] was built for OpenSearch version 1.0 but version " + Version.CURRENT + " is required"; + String message = "plugin [fake_plugin1] was built for OpenSearch version 5.0.0 and is not compatible with " + Version.CURRENT; assertEquals("fake_plugin1\nfake_plugin2\n", terminal.getOutput()); assertEquals("WARNING: " + message + "\n", terminal.getErrorOutput()); @@ -286,4 +286,41 @@ public void testExistingIncompatiblePlugin() throws Exception { terminal = listPlugins(home, params); assertEquals("fake_plugin1\nfake_plugin2\n", terminal.getOutput()); } + + public void testPluginWithDependencies() throws Exception { + PluginTestUtil.writePluginProperties( + env.pluginsDir().resolve("fake_plugin1"), + "description", + "fake desc 1", + "name", + "fake_plugin1", + "version", + "1.0", + "dependencies", + "{opensearch:\"" + Version.CURRENT + "\"}", + "java.version", + System.getProperty("java.specification.version"), + "classname", + "org.fake1" + ); + String[] params = { "-v" }; + MockTerminal terminal = listPlugins(home, params); + assertEquals( + buildMultiline( + "Plugins directory: " + env.pluginsDir(), + "fake_plugin1", + "- Plugin information:", + "Name: fake_plugin1", + "Description: fake desc 1", + "Version: 1.0", + "OpenSearch Version: " + Version.CURRENT.toString(), + "Java Version: " + System.getProperty("java.specification.version"), + "Native Controller: false", + "Extended Plugins: []", + " * Classname: org.fake1", + "Folder name: null" + ), + terminal.getOutput() + ); + } } diff --git a/gradle/code-coverage.gradle b/gradle/code-coverage.gradle index 822b471e2e034..3ca6b1fe84ea7 100644 --- a/gradle/code-coverage.gradle +++ b/gradle/code-coverage.gradle @@ -13,7 +13,7 @@ repositories { gradlePluginPortal() // TODO: Find the way to use the repositories from RepositoriesSetupPlugin maven { - url = "https://artifacts.opensearch.org/snapshots/lucene/" + url = "https://ci.opensearch.org/ci/dbc/snapshots/lucene/" } } @@ -37,7 +37,7 @@ tasks.withType(JacocoReport).configureEach { if (System.getProperty("tests.coverage")) { reporting { reports { - testCodeCoverageReport(JacocoCoverageReport) { + testCodeCoverageReport(JacocoCoverageReport) { testType = TestSuiteType.UNIT_TEST } } @@ -45,6 +45,6 @@ if (System.getProperty("tests.coverage")) { // Attach code coverage report task to Gradle check task project.getTasks().named(JavaBasePlugin.CHECK_TASK_NAME).configure { - dependsOn tasks.named('testCodeCoverageReport', JacocoReport) + dependsOn tasks.named('testCodeCoverageReport', JacocoReport) } } diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java index 3e996bdee83a2..ea23b3d81a775 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java @@ -56,6 +56,7 @@ import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.semver.SemverRange; import java.io.ByteArrayInputStream; import java.io.EOFException; @@ -750,6 +751,8 @@ public Object readGenericValue() throws IOException { return readCollection(StreamInput::readGenericValue, HashSet::new, Collections.emptySet()); case 26: return readBigInteger(); + case 27: + return readSemverRange(); default: throw new IOException("Can't read unknown type [" + type + "]"); } @@ -1090,6 +1093,10 @@ public Version readVersion() throws IOException { return Version.fromId(readVInt()); } + public SemverRange readSemverRange() throws IOException { + return SemverRange.fromString(readString()); + } + /** Reads the {@link Version} from the input stream */ public Build readBuild() throws IOException { // the following is new for opensearch: we write the distribution to support any "forks" diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java index 2d69e1c686df3..b7599265aece3 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java @@ -54,6 +54,7 @@ import org.opensearch.core.common.settings.SecureString; import org.opensearch.core.common.text.Text; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; +import org.opensearch.semver.SemverRange; import java.io.EOFException; import java.io.FileNotFoundException; @@ -784,6 +785,10 @@ public final void writeOptionalInstant(@Nullable Instant instant) throws IOExcep o.writeByte((byte) 26); o.writeString(v.toString()); }); + writers.put(SemverRange.class, (o, v) -> { + o.writeByte((byte) 27); + o.writeSemverRange((SemverRange) v); + }); WRITERS = Collections.unmodifiableMap(writers); } @@ -1101,6 +1106,10 @@ public void writeVersion(final Version version) throws IOException { writeVInt(version.id); } + public void writeSemverRange(final SemverRange range) throws IOException { + writeString(range.toString()); + } + /** Writes the OpenSearch {@link Build} informn to the output stream */ public void writeBuild(final Build build) throws IOException { // the following is new for opensearch: we write the distribution name to support any "forks" of the code diff --git a/libs/core/src/main/java/org/opensearch/semver/SemverRange.java b/libs/core/src/main/java/org/opensearch/semver/SemverRange.java new file mode 100644 index 0000000000000..da87acc7124aa --- /dev/null +++ b/libs/core/src/main/java/org/opensearch/semver/SemverRange.java @@ -0,0 +1,170 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.semver; + +import org.opensearch.Version; +import org.opensearch.common.Nullable; +import org.opensearch.core.xcontent.ToXContentFragment; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.semver.expr.Caret; +import org.opensearch.semver.expr.Equal; +import org.opensearch.semver.expr.Expression; +import org.opensearch.semver.expr.Tilde; + +import java.io.IOException; +import java.util.Objects; +import java.util.Optional; + +import static java.util.Arrays.stream; + +/** + * Represents a single semver range that allows for specifying which {@code org.opensearch.Version}s satisfy the range. + * It is composed of a range version and a range operator. Following are the supported operators: + * + */ +public class SemverRange implements ToXContentFragment { + + private final Version rangeVersion; + private final RangeOperator rangeOperator; + + public SemverRange(final Version rangeVersion, final RangeOperator rangeOperator) { + this.rangeVersion = rangeVersion; + this.rangeOperator = rangeOperator; + } + + /** + * Constructs a {@code SemverRange} from its string representation. + * @param range given range + * @return a {@code SemverRange} + */ + public static SemverRange fromString(final String range) { + RangeOperator rangeOperator = RangeOperator.fromRange(range); + String version = range.replaceFirst(rangeOperator.asEscapedString(), ""); + if (!Version.stringHasLength(version)) { + throw new IllegalArgumentException("Version cannot be empty"); + } + return new SemverRange(Version.fromString(version), rangeOperator); + } + + /** + * Return the range operator for this range. + * @return range operator + */ + public RangeOperator getRangeOperator() { + return rangeOperator; + } + + /** + * Return the version for this range. + * @return the range version + */ + public Version getRangeVersion() { + return rangeVersion; + } + + /** + * Check if range is satisfied by given version string. + * + * @param versionToEvaluate version to check + * @return {@code true} if range is satisfied by version, {@code false} otherwise + */ + public boolean isSatisfiedBy(final String versionToEvaluate) { + return isSatisfiedBy(Version.fromString(versionToEvaluate)); + } + + /** + * Check if range is satisfied by given version. + * + * @param versionToEvaluate version to check + * @return {@code true} if range is satisfied by version, {@code false} otherwise + * @see #isSatisfiedBy(String) + */ + public boolean isSatisfiedBy(final Version versionToEvaluate) { + return this.rangeOperator.expression.evaluate(this.rangeVersion, versionToEvaluate); + } + + @Override + public boolean equals(@Nullable final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + SemverRange range = (SemverRange) o; + return Objects.equals(rangeVersion, range.rangeVersion) && rangeOperator == range.rangeOperator; + } + + @Override + public int hashCode() { + return Objects.hash(rangeVersion, rangeOperator); + } + + @Override + public String toString() { + return rangeOperator.asString() + rangeVersion; + } + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + return builder.value(toString()); + } + + /** + * A range operator. + */ + public enum RangeOperator { + + EQ("=", new Equal()), + TILDE("~", new Tilde()), + CARET("^", new Caret()), + DEFAULT("", new Equal()); + + private final String operator; + private final Expression expression; + + RangeOperator(final String operator, final Expression expression) { + this.operator = operator; + this.expression = expression; + } + + /** + * String representation of the range operator. + * + * @return range operator as string + */ + public String asString() { + return operator; + } + + /** + * Escaped string representation of the range operator, + * if operator is a regex character. + * + * @return range operator as escaped string, if operator is a regex character + */ + public String asEscapedString() { + if (Objects.equals(operator, "^")) { + return "\\^"; + } + return operator; + } + + public static RangeOperator fromRange(final String range) { + Optional rangeOperator = stream(values()).filter( + operator -> operator != DEFAULT && range.startsWith(operator.asString()) + ).findFirst(); + return rangeOperator.orElse(DEFAULT); + } + } +} diff --git a/libs/core/src/main/java/org/opensearch/semver/expr/Caret.java b/libs/core/src/main/java/org/opensearch/semver/expr/Caret.java new file mode 100644 index 0000000000000..ce2b74dde0865 --- /dev/null +++ b/libs/core/src/main/java/org/opensearch/semver/expr/Caret.java @@ -0,0 +1,32 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.semver.expr; + +import org.opensearch.Version; + +/** + * Expression to evaluate version compatibility allowing for minor and patch version variability. + */ +public class Caret implements Expression { + + /** + * Checks if the given version is compatible with the range version allowing for minor and + * patch version variability. + * Allows all versions starting from the rangeVersion upto next major version (exclusive). + * @param rangeVersion the version specified in range + * @param versionToEvaluate the version to evaluate + * @return {@code true} if the versions are compatible {@code false} otherwise + */ + @Override + public boolean evaluate(final Version rangeVersion, final Version versionToEvaluate) { + Version lower = rangeVersion; + Version upper = Version.fromString((rangeVersion.major + 1) + ".0.0"); + return versionToEvaluate.onOrAfter(lower) && versionToEvaluate.before(upper); + } +} diff --git a/libs/core/src/main/java/org/opensearch/semver/expr/Equal.java b/libs/core/src/main/java/org/opensearch/semver/expr/Equal.java new file mode 100644 index 0000000000000..d3e1d63060b77 --- /dev/null +++ b/libs/core/src/main/java/org/opensearch/semver/expr/Equal.java @@ -0,0 +1,29 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.semver.expr; + +import org.opensearch.Version; + +/** + * Expression to evaluate equality of versions. + */ +public class Equal implements Expression { + + /** + * Checks if a given version matches a certain range version. + * + * @param rangeVersion the version specified in range + * @param versionToEvaluate the version to evaluate + * @return {@code true} if the versions are equal {@code false} otherwise + */ + @Override + public boolean evaluate(final Version rangeVersion, final Version versionToEvaluate) { + return versionToEvaluate.equals(rangeVersion); + } +} diff --git a/libs/core/src/main/java/org/opensearch/semver/expr/Expression.java b/libs/core/src/main/java/org/opensearch/semver/expr/Expression.java new file mode 100644 index 0000000000000..68bb4e249836a --- /dev/null +++ b/libs/core/src/main/java/org/opensearch/semver/expr/Expression.java @@ -0,0 +1,26 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.semver.expr; + +import org.opensearch.Version; + +/** + * An evaluation expression. + */ +public interface Expression { + + /** + * Evaluates an expression. + * + * @param rangeVersion the version specified in range + * @param versionToEvaluate the version to evaluate + * @return the result of the expression evaluation + */ + boolean evaluate(final Version rangeVersion, final Version versionToEvaluate); +} diff --git a/libs/core/src/main/java/org/opensearch/semver/expr/Tilde.java b/libs/core/src/main/java/org/opensearch/semver/expr/Tilde.java new file mode 100644 index 0000000000000..5f62ffe62ddeb --- /dev/null +++ b/libs/core/src/main/java/org/opensearch/semver/expr/Tilde.java @@ -0,0 +1,31 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.semver.expr; + +import org.opensearch.Version; + +/** + * Expression to evaluate version compatibility allowing patch version variability. + */ +public class Tilde implements Expression { + + /** + * Checks if the given version is compatible with a range version allowing for patch version variability. + * Allows all versions starting from the rangeVersion upto next minor version (exclusive). + * @param rangeVersion the version specified in range + * @param versionToEvaluate the version to evaluate + * @return {@code true} if the versions are compatible {@code false} otherwise + */ + @Override + public boolean evaluate(final Version rangeVersion, final Version versionToEvaluate) { + Version lower = rangeVersion; + Version upper = Version.fromString(rangeVersion.major + "." + (rangeVersion.minor + 1) + "." + 0); + return versionToEvaluate.onOrAfter(lower) && versionToEvaluate.before(upper); + } +} diff --git a/libs/core/src/main/java/org/opensearch/semver/expr/package-info.java b/libs/core/src/main/java/org/opensearch/semver/expr/package-info.java new file mode 100644 index 0000000000000..06cf9feaaaf8f --- /dev/null +++ b/libs/core/src/main/java/org/opensearch/semver/expr/package-info.java @@ -0,0 +1,9 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +/** Expressions library module */ +package org.opensearch.semver.expr; diff --git a/libs/core/src/main/java/org/opensearch/semver/package-info.java b/libs/core/src/main/java/org/opensearch/semver/package-info.java new file mode 100644 index 0000000000000..ada935582d408 --- /dev/null +++ b/libs/core/src/main/java/org/opensearch/semver/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** Semver library module */ +package org.opensearch.semver; diff --git a/libs/core/src/test/java/org/opensearch/semver/SemverRangeTests.java b/libs/core/src/test/java/org/opensearch/semver/SemverRangeTests.java new file mode 100644 index 0000000000000..af1d95b2561b7 --- /dev/null +++ b/libs/core/src/test/java/org/opensearch/semver/SemverRangeTests.java @@ -0,0 +1,105 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.semver; + +import org.opensearch.test.OpenSearchTestCase; + +public class SemverRangeTests extends OpenSearchTestCase { + + public void testRangeWithEqualsOperator() { + SemverRange range = SemverRange.fromString("=1.2.3"); + assertEquals(range.getRangeOperator(), SemverRange.RangeOperator.EQ); + assertTrue(range.isSatisfiedBy("1.2.3")); + assertFalse(range.isSatisfiedBy("1.2.4")); + assertFalse(range.isSatisfiedBy("1.3.3")); + assertFalse(range.isSatisfiedBy("2.2.3")); + } + + public void testRangeWithDefaultOperator() { + SemverRange range = SemverRange.fromString("1.2.3"); + assertEquals(range.getRangeOperator(), SemverRange.RangeOperator.DEFAULT); + assertTrue(range.isSatisfiedBy("1.2.3")); + assertFalse(range.isSatisfiedBy("1.2.4")); + assertFalse(range.isSatisfiedBy("1.3.3")); + assertFalse(range.isSatisfiedBy("2.2.3")); + } + + public void testRangeWithTildeOperator() { + SemverRange range = SemverRange.fromString("~2.3.4"); + assertEquals(range.getRangeOperator(), SemverRange.RangeOperator.TILDE); + assertTrue(range.isSatisfiedBy("2.3.4")); + assertTrue(range.isSatisfiedBy("2.3.5")); + assertTrue(range.isSatisfiedBy("2.3.12")); + + assertFalse(range.isSatisfiedBy("2.3.0")); + assertFalse(range.isSatisfiedBy("2.3.3")); + assertFalse(range.isSatisfiedBy("2.4.0")); + assertFalse(range.isSatisfiedBy("3.0.0")); + } + + public void testRangeWithCaretOperator() { + SemverRange range = SemverRange.fromString("^2.3.4"); + assertEquals(range.getRangeOperator(), SemverRange.RangeOperator.CARET); + assertTrue(range.isSatisfiedBy("2.3.4")); + assertTrue(range.isSatisfiedBy("2.3.5")); + assertTrue(range.isSatisfiedBy("2.4.12")); + + assertFalse(range.isSatisfiedBy("2.3.3")); + assertFalse(range.isSatisfiedBy("3.0.0")); + } + + public void testInvalidRanges() { + IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> SemverRange.fromString("")); + assertEquals("Version cannot be empty", ex.getMessage()); + + ex = expectThrows(IllegalArgumentException.class, () -> SemverRange.fromString("1")); + assertTrue(ex.getMessage().contains("the version needs to contain major, minor, and revision, and optionally the build")); + + ex = expectThrows(IllegalArgumentException.class, () -> SemverRange.fromString("1.2")); + assertTrue(ex.getMessage().contains("the version needs to contain major, minor, and revision, and optionally the build")); + + ex = expectThrows(IllegalArgumentException.class, () -> SemverRange.fromString("=")); + assertEquals("Version cannot be empty", ex.getMessage()); + + ex = expectThrows(IllegalArgumentException.class, () -> SemverRange.fromString("=1")); + assertTrue(ex.getMessage().contains("the version needs to contain major, minor, and revision, and optionally the build")); + + ex = expectThrows(IllegalArgumentException.class, () -> SemverRange.fromString("=1.2")); + assertTrue(ex.getMessage().contains("the version needs to contain major, minor, and revision, and optionally the build")); + + ex = expectThrows(IllegalArgumentException.class, () -> SemverRange.fromString("~")); + assertEquals("Version cannot be empty", ex.getMessage()); + + ex = expectThrows(IllegalArgumentException.class, () -> SemverRange.fromString("~1")); + assertTrue(ex.getMessage().contains("the version needs to contain major, minor, and revision, and optionally the build")); + + ex = expectThrows(IllegalArgumentException.class, () -> SemverRange.fromString("~1.2")); + assertTrue(ex.getMessage().contains("the version needs to contain major, minor, and revision, and optionally the build")); + + ex = expectThrows(IllegalArgumentException.class, () -> SemverRange.fromString("^")); + assertEquals("Version cannot be empty", ex.getMessage()); + + ex = expectThrows(IllegalArgumentException.class, () -> SemverRange.fromString("^1")); + assertTrue(ex.getMessage().contains("the version needs to contain major, minor, and revision, and optionally the build")); + + ex = expectThrows(IllegalArgumentException.class, () -> SemverRange.fromString("^1.2")); + assertTrue(ex.getMessage().contains("the version needs to contain major, minor, and revision, and optionally the build")); + + ex = expectThrows(IllegalArgumentException.class, () -> SemverRange.fromString("$")); + assertTrue(ex.getMessage().contains("the version needs to contain major, minor, and revision, and optionally the build")); + + ex = expectThrows(IllegalArgumentException.class, () -> SemverRange.fromString("$1")); + assertTrue(ex.getMessage().contains("the version needs to contain major, minor, and revision, and optionally the build")); + + ex = expectThrows(IllegalArgumentException.class, () -> SemverRange.fromString("$1.2")); + assertTrue(ex.getMessage().contains("the version needs to contain major, minor, and revision, and optionally the build")); + + expectThrows(NumberFormatException.class, () -> SemverRange.fromString("$1.2.3")); + } +} diff --git a/libs/core/src/test/java/org/opensearch/semver/expr/CaretTests.java b/libs/core/src/test/java/org/opensearch/semver/expr/CaretTests.java new file mode 100644 index 0000000000000..3cb168d42cda0 --- /dev/null +++ b/libs/core/src/test/java/org/opensearch/semver/expr/CaretTests.java @@ -0,0 +1,30 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.semver.expr; + +import org.opensearch.Version; +import org.opensearch.test.OpenSearchTestCase; + +public class CaretTests extends OpenSearchTestCase { + + public void testMinorAndPatchVersionVariability() { + Caret caretExpr = new Caret(); + Version rangeVersion = Version.fromString("1.2.3"); + + // Compatible versions + assertTrue(caretExpr.evaluate(rangeVersion, Version.fromString("1.2.3"))); + assertTrue(caretExpr.evaluate(rangeVersion, Version.fromString("1.2.4"))); + assertTrue(caretExpr.evaluate(rangeVersion, Version.fromString("1.3.3"))); + assertTrue(caretExpr.evaluate(rangeVersion, Version.fromString("1.9.9"))); + + // Incompatible versions + assertFalse(caretExpr.evaluate(rangeVersion, Version.fromString("1.2.2"))); + assertFalse(caretExpr.evaluate(rangeVersion, Version.fromString("2.0.0"))); + } +} diff --git a/libs/core/src/test/java/org/opensearch/semver/expr/EqualTests.java b/libs/core/src/test/java/org/opensearch/semver/expr/EqualTests.java new file mode 100644 index 0000000000000..fb090865157ed --- /dev/null +++ b/libs/core/src/test/java/org/opensearch/semver/expr/EqualTests.java @@ -0,0 +1,22 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.semver.expr; + +import org.opensearch.Version; +import org.opensearch.test.OpenSearchTestCase; + +public class EqualTests extends OpenSearchTestCase { + + public void testEquality() { + Equal equalExpr = new Equal(); + Version rangeVersion = Version.fromString("1.2.3"); + assertTrue(equalExpr.evaluate(rangeVersion, Version.fromString("1.2.3"))); + assertFalse(equalExpr.evaluate(rangeVersion, Version.fromString("1.2.4"))); + } +} diff --git a/libs/core/src/test/java/org/opensearch/semver/expr/TildeTests.java b/libs/core/src/test/java/org/opensearch/semver/expr/TildeTests.java new file mode 100644 index 0000000000000..8666611645c3a --- /dev/null +++ b/libs/core/src/test/java/org/opensearch/semver/expr/TildeTests.java @@ -0,0 +1,29 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.semver.expr; + +import org.opensearch.Version; +import org.opensearch.test.OpenSearchTestCase; + +public class TildeTests extends OpenSearchTestCase { + + public void testPatchVersionVariability() { + Tilde tildeExpr = new Tilde(); + Version rangeVersion = Version.fromString("1.2.3"); + + assertTrue(tildeExpr.evaluate(rangeVersion, Version.fromString("1.2.3"))); + assertTrue(tildeExpr.evaluate(rangeVersion, Version.fromString("1.2.4"))); + assertTrue(tildeExpr.evaluate(rangeVersion, Version.fromString("1.2.9"))); + + assertFalse(tildeExpr.evaluate(rangeVersion, Version.fromString("1.2.0"))); + assertFalse(tildeExpr.evaluate(rangeVersion, Version.fromString("1.2.2"))); + assertFalse(tildeExpr.evaluate(rangeVersion, Version.fromString("1.3.0"))); + assertFalse(tildeExpr.evaluate(rangeVersion, Version.fromString("2.0.0"))); + } +} diff --git a/qa/full-cluster-restart/src/test/java/org/opensearch/upgrades/PluginInfoIT.java b/qa/full-cluster-restart/src/test/java/org/opensearch/upgrades/PluginInfoIT.java new file mode 100644 index 0000000000000..d4e7017aab8c2 --- /dev/null +++ b/qa/full-cluster-restart/src/test/java/org/opensearch/upgrades/PluginInfoIT.java @@ -0,0 +1,27 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.upgrades; + +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.test.rest.yaml.ObjectPath; + +import java.util.Map; + +public class PluginInfoIT extends AbstractFullClusterRestartTestCase { + public void testPluginInfoSerialization() throws Exception { + // Ensure all nodes are able to come up, validate with GET _nodes. + Response response = client().performRequest(new Request("GET", "_nodes")); + ObjectPath objectPath = ObjectPath.createFromResponse(response); + final Map nodeMap = objectPath.evaluate("nodes"); + // Any issue in PluginInfo serialization logic will result into connection failures + // and hence reduced number of nodes. + assertEquals(2, nodeMap.keySet().size()); + } +} diff --git a/qa/mixed-cluster/src/test/java/org/opensearch/backwards/PluginInfoIT.java b/qa/mixed-cluster/src/test/java/org/opensearch/backwards/PluginInfoIT.java new file mode 100644 index 0000000000000..47e454a7549cb --- /dev/null +++ b/qa/mixed-cluster/src/test/java/org/opensearch/backwards/PluginInfoIT.java @@ -0,0 +1,26 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.backwards; + +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.test.rest.OpenSearchRestTestCase; +import org.opensearch.test.rest.yaml.ObjectPath; + +import java.util.Map; + +public class PluginInfoIT extends OpenSearchRestTestCase { + public void testPluginInfoSerialization() throws Exception { + // Ensure all nodes are able to come up, validate with GET _nodes. + Response response = client().performRequest(new Request("GET", "_nodes")); + ObjectPath objectPath = ObjectPath.createFromResponse(response); + final Map nodeMap = objectPath.evaluate("nodes"); + assertEquals(4, nodeMap.keySet().size()); + } +} diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationBaseIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationBaseIT.java index 641f714d33414..796f09cb9528f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationBaseIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationBaseIT.java @@ -19,7 +19,6 @@ import org.opensearch.common.lease.Releasable; import org.opensearch.common.settings.Settings; import org.opensearch.core.index.Index; -import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexService; import org.opensearch.index.SegmentReplicationShardStats; @@ -175,17 +174,6 @@ private IndexShard getIndexShard(ClusterState state, ShardRouting routing, Strin return getIndexShard(state.nodes().get(routing.currentNodeId()).getName(), routing.shardId(), indexName); } - /** - * Fetch IndexShard by shardId, multiple shards per node allowed. - */ - protected IndexShard getIndexShard(String node, ShardId shardId, String indexName) { - final Index index = resolveIndex(indexName); - IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node); - IndexService indexService = indicesService.indexServiceSafe(index); - final Optional id = indexService.shardIds().stream().filter(sid -> sid == shardId.id()).findFirst(); - return indexService.getShard(id.get()); - } - /** * Fetch IndexShard, assumes only a single shard per node. */ diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/stats/IndexStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/stats/IndexStatsIT.java index 21ab6e8a4f018..1d5da9370cce3 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/stats/IndexStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/stats/IndexStatsIT.java @@ -107,6 +107,7 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAllSuccessful; @@ -130,7 +131,8 @@ public IndexStatsIT(Settings settings) { public static Collection parameters() { return Arrays.asList( new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() }, + new Object[] { Settings.builder().put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT).build() } ); } @@ -175,7 +177,7 @@ public void testFieldDataStats() throws InterruptedException { ensureGreen(); client().prepareIndex("test").setId("1").setSource("field", "value1", "field2", "value1").execute().actionGet(); client().prepareIndex("test").setId("2").setSource("field", "value2", "field2", "value2").execute().actionGet(); - client().admin().indices().prepareRefresh().execute().actionGet(); + refreshAndWaitForReplication(); indexRandomForConcurrentSearch("test"); NodesStatsResponse nodesStats = client().admin().cluster().prepareNodesStats("data:true").setIndices(true).execute().actionGet(); @@ -299,7 +301,7 @@ public void testClearAllCaches() throws Exception { client().admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet(); client().prepareIndex("test").setId("1").setSource("field", "value1").execute().actionGet(); client().prepareIndex("test").setId("2").setSource("field", "value2").execute().actionGet(); - client().admin().indices().prepareRefresh().execute().actionGet(); + refreshAndWaitForReplication(); indexRandomForConcurrentSearch("test"); NodesStatsResponse nodesStats = client().admin().cluster().prepareNodesStats("data:true").setIndices(true).execute().actionGet(); @@ -667,7 +669,7 @@ public void testSimpleStats() throws Exception { client().prepareIndex("test1").setId(Integer.toString(1)).setSource("field", "value").execute().actionGet(); client().prepareIndex("test1").setId(Integer.toString(2)).setSource("field", "value").execute().actionGet(); client().prepareIndex("test2").setId(Integer.toString(1)).setSource("field", "value").execute().actionGet(); - refresh(); + refreshAndWaitForReplication(); NumShards test1 = getNumShards("test1"); long test1ExpectedWrites = 2 * test1.dataCopies; @@ -682,7 +684,13 @@ public void testSimpleStats() throws Exception { assertThat(stats.getPrimaries().getIndexing().getTotal().getIndexFailedCount(), equalTo(0L)); assertThat(stats.getPrimaries().getIndexing().getTotal().isThrottled(), equalTo(false)); assertThat(stats.getPrimaries().getIndexing().getTotal().getThrottleTime().millis(), equalTo(0L)); - assertThat(stats.getTotal().getIndexing().getTotal().getIndexCount(), equalTo(totalExpectedWrites)); + + // This assert should not be done on segrep enabled indices because we are asserting Indexing/Write operations count on + // all primary and replica shards. But in case of segrep, Indexing/Write operation don't happen on replica shards. So we can + // ignore this assert check for segrep enabled indices. + if (isSegmentReplicationEnabledForIndex("test1") == false && isSegmentReplicationEnabledForIndex("test2") == false) { + assertThat(stats.getTotal().getIndexing().getTotal().getIndexCount(), equalTo(totalExpectedWrites)); + } assertThat(stats.getTotal().getStore(), notNullValue()); assertThat(stats.getTotal().getMerge(), notNullValue()); assertThat(stats.getTotal().getFlush(), notNullValue()); @@ -825,6 +833,7 @@ public void testMergeStats() { client().admin().indices().prepareForceMerge().setMaxNumSegments(1).execute().actionGet(); stats = client().admin().indices().prepareStats().setMerge(true).execute().actionGet(); + refreshAndWaitForReplication(); assertThat(stats.getTotal().getMerge(), notNullValue()); assertThat(stats.getTotal().getMerge().getTotal(), greaterThan(0L)); } @@ -851,7 +860,7 @@ public void testSegmentsStats() { client().admin().indices().prepareFlush().get(); client().admin().indices().prepareForceMerge().setMaxNumSegments(1).execute().actionGet(); - client().admin().indices().prepareRefresh().get(); + refreshAndWaitForReplication(); stats = client().admin().indices().prepareStats().setSegments(true).get(); assertThat(stats.getTotal().getSegments(), notNullValue()); @@ -869,7 +878,7 @@ public void testAllFlags() throws Exception { client().prepareIndex("test_index").setId(Integer.toString(2)).setSource("field", "value").execute().actionGet(); client().prepareIndex("test_index_2").setId(Integer.toString(1)).setSource("field", "value").execute().actionGet(); - client().admin().indices().prepareRefresh().execute().actionGet(); + refreshAndWaitForReplication(); IndicesStatsRequestBuilder builder = client().admin().indices().prepareStats(); Flag[] values = CommonStatsFlags.Flag.values(); for (Flag flag : values) { @@ -1453,6 +1462,7 @@ public void testZeroRemoteStoreStatsOnNonRemoteStoreIndex() { .get() .status() ); + refreshAndWaitForReplication(); ShardStats shard = client().admin().indices().prepareStats(indexName).setSegments(true).setTranslog(true).get().getShards()[0]; RemoteSegmentStats remoteSegmentStatsFromIndexStats = shard.getStats().getSegments().getRemoteSegmentStats(); assertZeroRemoteSegmentStats(remoteSegmentStatsFromIndexStats); diff --git a/server/src/internalClusterTest/java/org/opensearch/plugins/PluginsServiceIT.java b/server/src/internalClusterTest/java/org/opensearch/plugins/PluginsServiceIT.java new file mode 100644 index 0000000000000..3cc10b0c0b858 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/plugins/PluginsServiceIT.java @@ -0,0 +1,115 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugins; + +import org.opensearch.Version; +import org.opensearch.common.settings.Settings; +import org.opensearch.env.Environment; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.VersionUtils; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.hamcrest.Matchers.containsString; + +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) +public class PluginsServiceIT extends OpenSearchIntegTestCase { + + public void testNodeBootstrapWithCompatiblePlugin() throws IOException { + // Prepare the plugins directory and then start a node + Path baseDir = createTempDir(); + Path pluginDir = baseDir.resolve("plugins/dummy-plugin"); + PluginTestUtil.writePluginProperties( + pluginDir, + "description", + "dummy desc", + "name", + "dummyPlugin", + "version", + "1.0", + "opensearch.version", + Version.CURRENT.toString(), + "java.version", + System.getProperty("java.specification.version"), + "classname", + "test.DummyPlugin" + ); + try (InputStream jar = PluginsServiceTests.class.getResourceAsStream("dummy-plugin.jar")) { + Files.copy(jar, pluginDir.resolve("dummy-plugin.jar")); + } + internalCluster().startNode(Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), baseDir)); + for (PluginsService pluginsService : internalCluster().getDataNodeInstances(PluginsService.class)) { + // Ensure plugins service was able to load the plugin + assertEquals(1, pluginsService.info().getPluginInfos().stream().filter(info -> info.getName().equals("dummyPlugin")).count()); + } + } + + public void testNodeBootstrapWithRangeCompatiblePlugin() throws IOException { + // Prepare the plugins directory and then start a node + Path baseDir = createTempDir(); + Path pluginDir = baseDir.resolve("plugins/dummy-plugin"); + PluginTestUtil.writePluginProperties( + pluginDir, + "description", + "dummy desc", + "name", + "dummyPlugin", + "version", + "1.0", + "dependencies", + "{opensearch:\"~" + Version.CURRENT + "\"}", + "java.version", + System.getProperty("java.specification.version"), + "classname", + "test.DummyPlugin" + ); + try (InputStream jar = PluginsServiceTests.class.getResourceAsStream("dummy-plugin.jar")) { + Files.copy(jar, pluginDir.resolve("dummy-plugin.jar")); + } + internalCluster().startNode(Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), baseDir)); + for (PluginsService pluginsService : internalCluster().getDataNodeInstances(PluginsService.class)) { + // Ensure plugins service was able to load the plugin + assertEquals(1, pluginsService.info().getPluginInfos().stream().filter(info -> info.getName().equals("dummyPlugin")).count()); + } + } + + public void testNodeBootstrapWithInCompatiblePlugin() throws IOException { + // Prepare the plugins directory with an incompatible plugin and attempt to start a node + Path baseDir = createTempDir(); + Path pluginDir = baseDir.resolve("plugins/dummy-plugin"); + String incompatibleRange = "~" + + VersionUtils.getVersion(Version.CURRENT.major, Version.CURRENT.minor, (byte) (Version.CURRENT.revision + 1)); + PluginTestUtil.writePluginProperties( + pluginDir, + "description", + "dummy desc", + "name", + "dummyPlugin", + "version", + "1.0", + "dependencies", + "{opensearch:\"" + incompatibleRange + "\"}", + "java.version", + System.getProperty("java.specification.version"), + "classname", + "test.DummyPlugin" + ); + try (InputStream jar = PluginsServiceTests.class.getResourceAsStream("dummy-plugin.jar")) { + Files.copy(jar, pluginDir.resolve("dummy-plugin.jar")); + } + IllegalArgumentException e = assertThrows( + IllegalArgumentException.class, + () -> internalCluster().startNode(Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), baseDir)) + ); + assertThat(e.getMessage(), containsString("Plugin [dummyPlugin] was built for OpenSearch version ")); + } +} diff --git a/server/src/internalClusterTest/java/org/opensearch/recovery/RecoveryWhileUnderLoadIT.java b/server/src/internalClusterTest/java/org/opensearch/recovery/RecoveryWhileUnderLoadIT.java index 30d5af58df545..eb293aeb6d490 100644 --- a/server/src/internalClusterTest/java/org/opensearch/recovery/RecoveryWhileUnderLoadIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/recovery/RecoveryWhileUnderLoadIT.java @@ -32,6 +32,8 @@ package org.opensearch.recovery; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.admin.indices.refresh.RefreshResponse; @@ -52,10 +54,11 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.DocsStats; import org.opensearch.index.translog.Translog; +import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.plugins.Plugin; import org.opensearch.search.sort.SortOrder; import org.opensearch.test.BackgroundIndexer; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase; import java.util.Arrays; import java.util.Collection; @@ -69,12 +72,26 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; +import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAllSuccessful; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertNoTimeout; -public class RecoveryWhileUnderLoadIT extends OpenSearchIntegTestCase { +public class RecoveryWhileUnderLoadIT extends ParameterizedStaticSettingsOpenSearchIntegTestCase { + + public RecoveryWhileUnderLoadIT(Settings settings) { + super(settings); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { Settings.builder().put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.DOCUMENT).build() }, + new Object[] { Settings.builder().put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT).build() } + ); + } + private final Logger logger = LogManager.getLogger(RecoveryWhileUnderLoadIT.class); public static final class RetentionLeaseSyncIntervalSettingPlugin extends Plugin { @@ -150,7 +167,7 @@ public void testRecoverWhileUnderLoadAllocateReplicasTest() throws Exception { logger.info("--> indexing threads stopped"); logger.info("--> refreshing the index"); - refreshAndAssert(); + assertAfterRefreshAndWaitForReplication(); logger.info("--> verifying indexed content"); iterateAssertCount(numberOfShards, 10, indexer.getIds()); } @@ -211,7 +228,7 @@ public void testRecoverWhileUnderLoadAllocateReplicasRelocatePrimariesTest() thr logger.info("--> indexing threads stopped"); logger.info("--> refreshing the index"); - refreshAndAssert(); + assertAfterRefreshAndWaitForReplication(); logger.info("--> verifying indexed content"); iterateAssertCount(numberOfShards, 10, indexer.getIds()); } @@ -325,7 +342,7 @@ public void testRecoverWhileUnderLoadWithReducedAllowedNodes() throws Exception ); logger.info("--> refreshing the index"); - refreshAndAssert(); + assertAfterRefreshAndWaitForReplication(); logger.info("--> verifying indexed content"); iterateAssertCount(numberOfShards, 10, indexer.getIds()); } @@ -375,7 +392,7 @@ public void testRecoverWhileRelocating() throws Exception { ensureGreen(TimeValue.timeValueMinutes(5)); logger.info("--> refreshing the index"); - refreshAndAssert(); + assertAfterRefreshAndWaitForReplication(); logger.info("--> verifying indexed content"); iterateAssertCount(numShards, 10, indexer.getIds()); } @@ -474,10 +491,11 @@ private void logSearchResponse(int numberOfShards, long numberOfDocs, int iterat ); } - private void refreshAndAssert() throws Exception { + private void assertAfterRefreshAndWaitForReplication() throws Exception { assertBusy(() -> { RefreshResponse actionGet = client().admin().indices().prepareRefresh().get(); assertAllSuccessful(actionGet); }, 5, TimeUnit.MINUTES); + waitForReplication(); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/sort/FieldSortIT.java b/server/src/internalClusterTest/java/org/opensearch/search/sort/FieldSortIT.java index 20458a4876dc7..e40928f15e8a8 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/sort/FieldSortIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/sort/FieldSortIT.java @@ -197,7 +197,8 @@ public void testIssue8226() throws InterruptedException { public void testIssue6614() throws ExecutionException, InterruptedException { List builders = new ArrayList<>(); boolean strictTimeBasedIndices = randomBoolean(); - final int numIndices = randomIntBetween(2, 25); // at most 25 days in the month + // consider only 15 days of the month to avoid hitting open file limit + final int numIndices = randomIntBetween(2, 15); int docs = 0; for (int i = 0; i < numIndices; i++) { final String indexId = strictTimeBasedIndices ? "idx_" + i : "idx"; diff --git a/server/src/main/java/org/opensearch/plugins/PluginInfo.java b/server/src/main/java/org/opensearch/plugins/PluginInfo.java index dc8fd6e604d72..79e57b3e8a0e8 100644 --- a/server/src/main/java/org/opensearch/plugins/PluginInfo.java +++ b/server/src/main/java/org/opensearch/plugins/PluginInfo.java @@ -32,20 +32,28 @@ package org.opensearch.plugins; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.json.JsonReadFeature; + import org.opensearch.Version; import org.opensearch.bootstrap.JarHell; import org.opensearch.common.annotation.PublicApi; +import org.opensearch.common.xcontent.json.JsonXContentParser; import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.semver.SemverRange; import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -65,11 +73,15 @@ public class PluginInfo implements Writeable, ToXContentObject { public static final String OPENSEARCH_PLUGIN_PROPERTIES = "plugin-descriptor.properties"; public static final String OPENSEARCH_PLUGIN_POLICY = "plugin-security.policy"; + private static final JsonFactory jsonFactory = new JsonFactory().configure( + JsonReadFeature.ALLOW_UNQUOTED_FIELD_NAMES.mappedFeature(), + true + ); private final String name; private final String description; private final String version; - private final Version opensearchVersion; + private final List opensearchVersionRanges; private final String javaVersion; private final String classname; private final String customFolderName; @@ -99,11 +111,41 @@ public PluginInfo( String customFolderName, List extendedPlugins, boolean hasNativeController + ) { + this( + name, + description, + version, + List.of(SemverRange.fromString(opensearchVersion.toString())), + javaVersion, + classname, + customFolderName, + extendedPlugins, + hasNativeController + ); + } + + public PluginInfo( + String name, + String description, + String version, + List opensearchVersionRanges, + String javaVersion, + String classname, + String customFolderName, + List extendedPlugins, + boolean hasNativeController ) { this.name = name; this.description = description; this.version = version; - this.opensearchVersion = opensearchVersion; + // Ensure only one range is specified (for now) + if (opensearchVersionRanges.size() != 1) { + throw new IllegalArgumentException( + "Exactly one range is allowed to be specified in dependencies for the plugin [" + name + "]" + ); + } + this.opensearchVersionRanges = opensearchVersionRanges; this.javaVersion = javaVersion; this.classname = classname; this.customFolderName = customFolderName; @@ -152,11 +194,16 @@ public PluginInfo( * @param in the stream * @throws IOException if an I/O exception occurred reading the plugin info from the stream */ + @SuppressWarnings("unchecked") public PluginInfo(final StreamInput in) throws IOException { this.name = in.readString(); this.description = in.readString(); this.version = in.readString(); - this.opensearchVersion = in.readVersion(); + if (in.getVersion().onOrAfter(Version.V_2_13_0)) { + this.opensearchVersionRanges = (List) in.readGenericValue(); + } else { + this.opensearchVersionRanges = List.of(new SemverRange(in.readVersion(), SemverRange.RangeOperator.DEFAULT)); + } this.javaVersion = in.readString(); this.classname = in.readString(); this.customFolderName = in.readString(); @@ -169,7 +216,15 @@ public void writeTo(final StreamOutput out) throws IOException { out.writeString(name); out.writeString(description); out.writeString(version); - out.writeVersion(opensearchVersion); + if (out.getVersion().onOrAfter(Version.V_2_13_0)) { + out.writeGenericValue(opensearchVersionRanges); + } else { + /* + This works for currently supported range notations (=,~) + As more notations get added, then a suitable version must be picked. + */ + out.writeVersion(opensearchVersionRanges.get(0).getRangeVersion()); + } out.writeString(javaVersion); out.writeString(classname); if (customFolderName != null) { @@ -214,10 +269,49 @@ public static PluginInfo readFromProperties(final Path path) throws IOException } final String opensearchVersionString = propsMap.remove("opensearch.version"); - if (opensearchVersionString == null) { - throw new IllegalArgumentException("property [opensearch.version] is missing for plugin [" + name + "]"); + final String dependenciesValue = propsMap.remove("dependencies"); + if (opensearchVersionString == null && dependenciesValue == null) { + throw new IllegalArgumentException( + "Either [opensearch.version] or [dependencies] property must be specified for the plugin [" + name + "]" + ); + } + if (opensearchVersionString != null && dependenciesValue != null) { + throw new IllegalArgumentException( + "Only one of [opensearch.version] or [dependencies] property can be specified for the plugin [" + name + "]" + ); + } + + final List opensearchVersionRanges = new ArrayList<>(); + if (opensearchVersionString != null) { + opensearchVersionRanges.add(SemverRange.fromString(opensearchVersionString)); + } else { + Map dependenciesMap; + try ( + final JsonXContentParser parser = new JsonXContentParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.IGNORE_DEPRECATIONS, + jsonFactory.createParser(dependenciesValue) + ) + ) { + dependenciesMap = parser.mapStrings(); + } + if (dependenciesMap.size() != 1) { + throw new IllegalArgumentException( + "Exactly one dependency is allowed to be specified in plugin descriptor properties: " + dependenciesMap + ); + } + if (dependenciesMap.keySet().stream().noneMatch(s -> s.equals("opensearch"))) { + throw new IllegalArgumentException("Only opensearch is allowed to be specified as a plugin dependency: " + dependenciesMap); + } + String[] ranges = dependenciesMap.get("opensearch").split(","); + if (ranges.length != 1) { + throw new IllegalArgumentException( + "Exactly one range is allowed to be specified in dependencies for the plugin [\" + name + \"]" + ); + } + opensearchVersionRanges.add(SemverRange.fromString(ranges[0].trim())); } - final Version opensearchVersion = Version.fromString(opensearchVersionString); + final String javaVersionString = propsMap.remove("java.version"); if (javaVersionString == null) { throw new IllegalArgumentException("property [java.version] is missing for plugin [" + name + "]"); @@ -273,7 +367,7 @@ public static PluginInfo readFromProperties(final Path path) throws IOException name, description, version, - opensearchVersion, + opensearchVersionRanges, javaVersionString, classname, customFolderName, @@ -337,12 +431,26 @@ public String getVersion() { } /** - * The version of OpenSearch the plugin was built for. + * The list of OpenSearch version ranges the plugin is compatible with. * - * @return an OpenSearch version + * @return a list of OpenSearch version ranges */ - public Version getOpenSearchVersion() { - return opensearchVersion; + public List getOpenSearchVersionRanges() { + return opensearchVersionRanges; + } + + /** + * Pretty print the semver ranges and return the string. + * @return semver ranges string + */ + public String getOpenSearchVersionRangesString() { + if (opensearchVersionRanges == null || opensearchVersionRanges.isEmpty()) { + return ""; + } + if (opensearchVersionRanges.size() == 1) { + return opensearchVersionRanges.get(0).toString(); + } + return opensearchVersionRanges.stream().map(Object::toString).collect(Collectors.joining(",", "[", "]")); } /** @@ -378,7 +486,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws { builder.field("name", name); builder.field("version", version); - builder.field("opensearch_version", opensearchVersion); + builder.field("opensearch_version", opensearchVersionRanges); builder.field("java_version", javaVersion); builder.field("description", description); builder.field("classname", classname); @@ -432,7 +540,7 @@ public String toString(String prefix) { .append("\n") .append(prefix) .append("OpenSearch Version: ") - .append(opensearchVersion) + .append(getOpenSearchVersionRangesString()) .append("\n") .append(prefix) .append("Java Version: ") diff --git a/server/src/main/java/org/opensearch/plugins/PluginsService.java b/server/src/main/java/org/opensearch/plugins/PluginsService.java index 590b70d3bfc53..a6eefd2f4fd17 100644 --- a/server/src/main/java/org/opensearch/plugins/PluginsService.java +++ b/server/src/main/java/org/opensearch/plugins/PluginsService.java @@ -52,6 +52,7 @@ import org.opensearch.core.common.Strings; import org.opensearch.core.service.ReportingService; import org.opensearch.index.IndexModule; +import org.opensearch.semver.SemverRange; import org.opensearch.threadpool.ExecutorBuilder; import org.opensearch.transport.TransportSettings; @@ -387,12 +388,12 @@ public static List findPluginDirs(final Path rootPath) throws IOException * Verify the given plugin is compatible with the current OpenSearch installation. */ static void verifyCompatibility(PluginInfo info) { - if (info.getOpenSearchVersion().equals(Version.CURRENT) == false) { + if (!isPluginVersionCompatible(info, Version.CURRENT)) { throw new IllegalArgumentException( "Plugin [" + info.getName() + "] was built for OpenSearch version " - + info.getOpenSearchVersion() + + info.getOpenSearchVersionRangesString() + " but version " + Version.CURRENT + " is running" @@ -401,6 +402,16 @@ static void verifyCompatibility(PluginInfo info) { JarHell.checkJavaVersion(info.getName(), info.getJavaVersion()); } + public static boolean isPluginVersionCompatible(final PluginInfo pluginInfo, final Version coreVersion) { + // Core version must satisfy the semver range in plugin info + for (SemverRange range : pluginInfo.getOpenSearchVersionRanges()) { + if (!range.isSatisfiedBy(coreVersion)) { + return false; + } + } + return true; + } + static void checkForFailedPluginRemovals(final Path pluginsDirectory) throws IOException { /* * Check for the existence of a marker file that indicates any plugins are in a garbage state from a failed attempt to remove the diff --git a/server/src/test/java/org/opensearch/plugins/PluginInfoTests.java b/server/src/test/java/org/opensearch/plugins/PluginInfoTests.java index b976704e8af57..7f55c9f5cc7f7 100644 --- a/server/src/test/java/org/opensearch/plugins/PluginInfoTests.java +++ b/server/src/test/java/org/opensearch/plugins/PluginInfoTests.java @@ -32,10 +32,13 @@ package org.opensearch.plugins; +import com.fasterxml.jackson.core.JsonParseException; + import org.opensearch.Version; import org.opensearch.action.admin.cluster.node.info.PluginsAndModules; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.ByteBufferStreamInput; +import org.opensearch.semver.SemverRange; import org.opensearch.test.OpenSearchTestCase; import java.nio.ByteBuffer; @@ -74,6 +77,33 @@ public void testReadFromProperties() throws Exception { assertEquals("fake desc", info.getDescription()); assertEquals("1.0", info.getVersion()); assertEquals("FakePlugin", info.getClassname()); + assertEquals(Version.CURRENT.toString(), info.getOpenSearchVersionRanges().get(0).toString()); + assertThat(info.getExtendedPlugins(), empty()); + } + + public void testReadFromPropertiesWithSingleOpenSearchRange() throws Exception { + Path pluginDir = createTempDir().resolve("fake-plugin"); + PluginTestUtil.writePluginProperties( + pluginDir, + "description", + "fake desc", + "name", + "my_plugin", + "version", + "1.0", + "dependencies", + "{opensearch:\"~" + Version.CURRENT.toString() + "\"}", + "java.version", + System.getProperty("java.specification.version"), + "classname", + "FakePlugin" + ); + PluginInfo info = PluginInfo.readFromProperties(pluginDir); + assertEquals("my_plugin", info.getName()); + assertEquals("fake desc", info.getDescription()); + assertEquals("1.0", info.getVersion()); + assertEquals("FakePlugin", info.getClassname()); + assertEquals("~" + Version.CURRENT.toString(), info.getOpenSearchVersionRanges().get(0).toString()); assertThat(info.getExtendedPlugins(), empty()); } @@ -102,6 +132,7 @@ public void testReadFromPropertiesWithFolderNameAndVersionAfter() throws Excepti assertEquals("1.0", info.getVersion()); assertEquals("FakePlugin", info.getClassname()); assertEquals("custom-folder", info.getTargetFolderName()); + assertEquals(Version.CURRENT.toString(), info.getOpenSearchVersionRanges().get(0).toString()); assertThat(info.getExtendedPlugins(), empty()); } @@ -130,11 +161,40 @@ public void testReadFromPropertiesVersionMissing() throws Exception { assertThat(e.getMessage(), containsString("[version] is missing")); } - public void testReadFromPropertiesOpenSearchVersionMissing() throws Exception { + public void testReadFromPropertiesOpenSearchVersionAndDependenciesMissing() throws Exception { Path pluginDir = createTempDir().resolve("fake-plugin"); PluginTestUtil.writePluginProperties(pluginDir, "description", "fake desc", "name", "my_plugin", "version", "1.0"); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> PluginInfo.readFromProperties(pluginDir)); - assertThat(e.getMessage(), containsString("[opensearch.version] is missing")); + assertThat( + e.getMessage(), + containsString("Either [opensearch.version] or [dependencies] property must be specified for the plugin ") + ); + } + + public void testReadFromPropertiesWithDependenciesAndOpenSearchVersion() throws Exception { + Path pluginDir = createTempDir().resolve("fake-plugin"); + PluginTestUtil.writePluginProperties( + pluginDir, + "description", + "fake desc", + "name", + "my_plugin", + "version", + "1.0", + "opensearch.version", + Version.CURRENT.toString(), + "dependencies", + "{opensearch:" + Version.CURRENT.toString() + "}", + "java.version", + System.getProperty("java.specification.version"), + "classname", + "FakePlugin" + ); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> PluginInfo.readFromProperties(pluginDir)); + assertThat( + e.getMessage(), + containsString("Only one of [opensearch.version] or [dependencies] property can be specified for the plugin") + ); } public void testReadFromPropertiesJavaVersionMissing() throws Exception { @@ -305,7 +365,6 @@ public void testSerialize() throws Exception { ByteBufferStreamInput input = new ByteBufferStreamInput(buffer); PluginInfo info2 = new PluginInfo(input); assertThat(info2.toString(), equalTo(info.toString())); - } public void testPluginListSorted() { @@ -347,4 +406,193 @@ public void testUnknownProperties() throws Exception { assertThat(e.getMessage(), containsString("Unknown properties in plugin descriptor")); } + public void testMultipleDependencies() throws Exception { + Path pluginDir = createTempDir().resolve("fake-plugin"); + PluginTestUtil.writePluginProperties( + pluginDir, + "description", + "fake desc", + "name", + "my_plugin", + "version", + "1.0", + "dependencies", + "{opensearch:\"~" + Version.CURRENT.toString() + "\", dependency2:\"1.0.0\"}", + "java.version", + System.getProperty("java.specification.version"), + "classname", + "FakePlugin" + ); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> PluginInfo.readFromProperties(pluginDir)); + assertThat(e.getMessage(), containsString("Exactly one dependency is allowed to be specified in plugin descriptor properties")); + } + + public void testNonOpenSearchDependency() throws Exception { + Path pluginDir = createTempDir().resolve("fake-plugin"); + PluginTestUtil.writePluginProperties( + pluginDir, + "description", + "fake desc", + "name", + "my_plugin", + "version", + "1.0", + "dependencies", + "{some_dependency:\"~" + Version.CURRENT.toString() + "\"}", + "java.version", + System.getProperty("java.specification.version"), + "classname", + "FakePlugin" + ); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> PluginInfo.readFromProperties(pluginDir)); + assertThat(e.getMessage(), containsString("Only opensearch is allowed to be specified as a plugin dependency")); + } + + public void testEmptyDependenciesProperty() throws Exception { + Path pluginDir = createTempDir().resolve("fake-plugin"); + PluginTestUtil.writePluginProperties( + pluginDir, + "description", + "fake desc", + "name", + "my_plugin", + "version", + "1.0", + "dependencies", + "{}", + "java.version", + System.getProperty("java.specification.version"), + "classname", + "FakePlugin" + ); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> PluginInfo.readFromProperties(pluginDir)); + assertThat(e.getMessage(), containsString("Exactly one dependency is allowed to be specified in plugin descriptor properties")); + } + + public void testInvalidDependenciesProperty() throws Exception { + Path pluginDir = createTempDir().resolve("fake-plugin"); + PluginTestUtil.writePluginProperties( + pluginDir, + "description", + "fake desc", + "name", + "my_plugin", + "version", + "1.0", + "dependencies", + "{invalid}", + "java.version", + System.getProperty("java.specification.version"), + "classname", + "FakePlugin" + ); + expectThrows(JsonParseException.class, () -> PluginInfo.readFromProperties(pluginDir)); + } + + public void testEmptyOpenSearchVersionInDependencies() throws Exception { + Path pluginDir = createTempDir().resolve("fake-plugin"); + PluginTestUtil.writePluginProperties( + pluginDir, + "description", + "fake desc", + "name", + "my_plugin", + "version", + "1.0", + "dependencies", + "{opensearch:\"\"}", + "java.version", + System.getProperty("java.specification.version"), + "classname", + "FakePlugin" + ); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> PluginInfo.readFromProperties(pluginDir)); + assertThat(e.getMessage(), containsString("Version cannot be empty")); + } + + public void testInvalidOpenSearchVersionInDependencies() throws Exception { + Path pluginDir = createTempDir().resolve("fake-plugin"); + PluginTestUtil.writePluginProperties( + pluginDir, + "description", + "fake desc", + "name", + "my_plugin", + "version", + "1.0", + "dependencies", + "{opensearch:\"1.2\"}", + "java.version", + System.getProperty("java.specification.version"), + "classname", + "FakePlugin" + ); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> PluginInfo.readFromProperties(pluginDir)); + assertThat( + e.getMessage(), + containsString("the version needs to contain major, minor, and revision, and optionally the build: 1.2") + ); + } + + public void testInvalidRangeInDependencies() throws Exception { + Path pluginDir = createTempDir().resolve("fake-plugin"); + PluginTestUtil.writePluginProperties( + pluginDir, + "description", + "fake desc", + "name", + "my_plugin", + "version", + "1.0", + "dependencies", + "{opensearch:\"<2.2.0\"}", + "java.version", + System.getProperty("java.specification.version"), + "classname", + "FakePlugin" + ); + expectThrows(NumberFormatException.class, () -> PluginInfo.readFromProperties(pluginDir)); + } + + public void testhMultipleOpenSearchRangesInDependencies() throws Exception { + Path pluginDir = createTempDir().resolve("fake-plugin"); + PluginTestUtil.writePluginProperties( + pluginDir, + "description", + "fake desc", + "name", + "my_plugin", + "version", + "1.0", + "dependencies", + "{opensearch:\"~1.2.3, =1.2.3\"}", + "java.version", + System.getProperty("java.specification.version"), + "classname", + "FakePlugin" + ); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> PluginInfo.readFromProperties(pluginDir)); + assertThat(e.getMessage(), containsString("Exactly one range is allowed to be specified in dependencies for the plugin")); + } + + public void testhMultipleOpenSearchRangesInConstructor() throws Exception { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> new PluginInfo( + "plugin_name", + "foo", + "dummy", + List.of( + new SemverRange(Version.CURRENT, SemverRange.RangeOperator.EQ), + new SemverRange(Version.CURRENT, SemverRange.RangeOperator.DEFAULT) + ), + "1.8", + "dummyclass", + null, + Collections.emptyList(), + randomBoolean() + ) + ); + assertThat(e.getMessage(), containsString("Exactly one range is allowed to be specified in dependencies for the plugin")); + } } diff --git a/server/src/test/java/org/opensearch/plugins/PluginsServiceTests.java b/server/src/test/java/org/opensearch/plugins/PluginsServiceTests.java index db276678ba4dd..bd9ee33856f14 100644 --- a/server/src/test/java/org/opensearch/plugins/PluginsServiceTests.java +++ b/server/src/test/java/org/opensearch/plugins/PluginsServiceTests.java @@ -45,8 +45,10 @@ import org.opensearch.env.Environment; import org.opensearch.env.TestEnvironment; import org.opensearch.index.IndexModule; +import org.opensearch.semver.SemverRange; import org.opensearch.test.MockLogAppender; import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.test.VersionUtils; import org.hamcrest.Matchers; import java.io.IOException; @@ -717,6 +719,45 @@ public void testIncompatibleOpenSearchVersion() throws Exception { assertThat(e.getMessage(), containsString("was built for OpenSearch version 6.0.0")); } + public void testCompatibleOpenSearchVersionRange() { + List pluginCompatibilityRange = List.of(new SemverRange(Version.CURRENT, SemverRange.RangeOperator.TILDE)); + PluginInfo info = new PluginInfo( + "my_plugin", + "desc", + "1.0", + pluginCompatibilityRange, + "1.8", + "FakePlugin", + null, + Collections.emptyList(), + false + ); + PluginsService.verifyCompatibility(info); + } + + public void testIncompatibleOpenSearchVersionRange() { + // Version.CURRENT is behind by one with respect to patch version in the range + List pluginCompatibilityRange = List.of( + new SemverRange( + VersionUtils.getVersion(Version.CURRENT.major, Version.CURRENT.minor, (byte) (Version.CURRENT.revision + 1)), + SemverRange.RangeOperator.TILDE + ) + ); + PluginInfo info = new PluginInfo( + "my_plugin", + "desc", + "1.0", + pluginCompatibilityRange, + "1.8", + "FakePlugin", + null, + Collections.emptyList(), + false + ); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> PluginsService.verifyCompatibility(info)); + assertThat(e.getMessage(), containsString("was built for OpenSearch version ")); + } + public void testIncompatibleJavaVersion() throws Exception { PluginInfo info = new PluginInfo( "my_plugin", @@ -891,7 +932,10 @@ public void testExtensiblePlugin() { TestExtensiblePlugin extensiblePlugin = new TestExtensiblePlugin(); PluginsService.loadExtensions( Collections.singletonList( - Tuple.tuple(new PluginInfo("extensible", null, null, null, null, null, Collections.emptyList(), false), extensiblePlugin) + Tuple.tuple( + new PluginInfo("extensible", null, null, Version.CURRENT, null, null, Collections.emptyList(), false), + extensiblePlugin + ) ) ); @@ -902,9 +946,12 @@ public void testExtensiblePlugin() { TestPlugin testPlugin = new TestPlugin(); PluginsService.loadExtensions( Arrays.asList( - Tuple.tuple(new PluginInfo("extensible", null, null, null, null, null, Collections.emptyList(), false), extensiblePlugin), Tuple.tuple( - new PluginInfo("test", null, null, null, null, null, Collections.singletonList("extensible"), false), + new PluginInfo("extensible", null, null, Version.CURRENT, null, null, Collections.emptyList(), false), + extensiblePlugin + ), + Tuple.tuple( + new PluginInfo("test", null, null, Version.CURRENT, null, null, Collections.singletonList("extensible"), false), testPlugin ) ) @@ -1036,6 +1083,40 @@ public void testThrowingConstructor() { assertThat(e.getCause().getCause(), hasToString(containsString("test constructor failure"))); } + public void testPluginCompatibilityWithSemverRange() { + // Compatible plugin and core versions + assertTrue(PluginsService.isPluginVersionCompatible(getPluginInfoWithWithSemverRange("1.0.0"), Version.fromString("1.0.0"))); + + assertTrue(PluginsService.isPluginVersionCompatible(getPluginInfoWithWithSemverRange("=1.0.0"), Version.fromString("1.0.0"))); + + assertTrue(PluginsService.isPluginVersionCompatible(getPluginInfoWithWithSemverRange("~1.0.0"), Version.fromString("1.0.0"))); + + assertTrue(PluginsService.isPluginVersionCompatible(getPluginInfoWithWithSemverRange("~1.0.1"), Version.fromString("1.0.2"))); + + // Incompatible plugin and core versions + assertFalse(PluginsService.isPluginVersionCompatible(getPluginInfoWithWithSemverRange("1.0.0"), Version.fromString("1.0.1"))); + + assertFalse(PluginsService.isPluginVersionCompatible(getPluginInfoWithWithSemverRange("=1.0.0"), Version.fromString("1.0.1"))); + + assertFalse(PluginsService.isPluginVersionCompatible(getPluginInfoWithWithSemverRange("~1.0.1"), Version.fromString("1.0.0"))); + + assertFalse(PluginsService.isPluginVersionCompatible(getPluginInfoWithWithSemverRange("~1.0.0"), Version.fromString("1.1.0"))); + } + + private PluginInfo getPluginInfoWithWithSemverRange(String semverRange) { + return new PluginInfo( + "my_plugin", + "desc", + "1.0", + List.of(SemverRange.fromString(semverRange)), + "1.8", + "FakePlugin", + null, + Collections.emptyList(), + false + ); + } + private static class TestExtensiblePlugin extends Plugin implements ExtensiblePlugin { private List extensions; diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index 0f9ab3aa3d64f..33d5669d33297 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -37,6 +37,7 @@ import com.carrotsearch.randomizedtesting.generators.RandomPicks; import org.apache.lucene.codecs.Codec; +import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.search.Sort; import org.apache.lucene.search.TotalHits; import org.apache.lucene.tests.util.LuceneTestCase; @@ -92,6 +93,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.Priority; import org.opensearch.common.collect.Tuple; +import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.network.NetworkModule; import org.opensearch.common.regex.Regex; import org.opensearch.common.settings.FeatureFlagSettings; @@ -114,6 +116,7 @@ import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; @@ -123,6 +126,7 @@ import org.opensearch.env.Environment; import org.opensearch.env.TestEnvironment; import org.opensearch.index.IndexModule; +import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; import org.opensearch.index.MergeSchedulerConfig; import org.opensearch.index.MockEngineFactoryPlugin; @@ -131,10 +135,12 @@ import org.opensearch.index.engine.Segment; import org.opensearch.index.mapper.CompletionFieldMapper; import org.opensearch.index.mapper.MockFieldFilterPlugin; +import org.opensearch.index.shard.IndexShard; import org.opensearch.index.store.Store; import org.opensearch.index.translog.Translog; import org.opensearch.indices.IndicesQueryCache; import org.opensearch.indices.IndicesRequestCache; +import org.opensearch.indices.IndicesService; import org.opensearch.indices.store.IndicesStore; import org.opensearch.monitor.os.OsInfo; import org.opensearch.node.NodeMocksPlugin; @@ -182,6 +188,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.Random; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; @@ -1556,14 +1563,17 @@ public void indexRandom(boolean forceRefresh, boolean dummyDocuments, boolean ma if (dummyDocuments) { indexRandomForMultipleSlices(indicesArray); } + if (forceRefresh) { + waitForReplication(); + } } /* - * This method ingests bogus documents for the given indices such that multiple slices - * are formed. This is useful for testing with the concurrent search use-case as it creates - * multiple slices based on segment count. - * @param indices the indices in which bogus documents should be ingested - * */ + * This method ingests bogus documents for the given indices such that multiple slices + * are formed. This is useful for testing with the concurrent search use-case as it creates + * multiple slices based on segment count. + * @param indices the indices in which bogus documents should be ingested + * */ protected void indexRandomForMultipleSlices(String... indices) throws InterruptedException { Set> bogusIds = new HashSet<>(); int refreshCount = randomIntBetween(2, 3); @@ -2357,4 +2367,96 @@ protected ClusterState getClusterState() { return client(internalCluster().getClusterManagerName()).admin().cluster().prepareState().get().getState(); } + /** + * Refreshes the indices in the cluster and waits until active/started replica shards + * are caught up with primary shard only when Segment Replication is enabled. + * This doesn't wait for inactive/non-started replica shards to become active/started. + */ + protected RefreshResponse refreshAndWaitForReplication(String... indices) { + RefreshResponse refreshResponse = refresh(indices); + waitForReplication(); + return refreshResponse; + } + + /** + * Waits until active/started replica shards are caught up with primary shard only when Segment Replication is enabled. + * This doesn't wait for inactive/non-started replica shards to become active/started. + */ + protected void waitForReplication(String... indices) { + if (indices.length == 0) { + indices = getClusterState().routingTable().indicesRouting().keySet().toArray(String[]::new); + } + try { + for (String index : indices) { + if (isSegmentReplicationEnabledForIndex(index)) { + if (isInternalCluster()) { + IndexRoutingTable indexRoutingTable = getClusterState().routingTable().index(index); + if (indexRoutingTable != null) { + assertBusy(() -> { + for (IndexShardRoutingTable shardRoutingTable : indexRoutingTable) { + final ShardRouting primaryRouting = shardRoutingTable.primaryShard(); + if (primaryRouting.state().toString().equals("STARTED")) { + if (isSegmentReplicationEnabledForIndex(index)) { + final List replicaRouting = shardRoutingTable.replicaShards(); + final IndexShard primaryShard = getIndexShard(primaryRouting, index); + for (ShardRouting replica : replicaRouting) { + if (replica.state().toString().equals("STARTED")) { + IndexShard replicaShard = getIndexShard(replica, index); + assertEquals( + "replica shards haven't caught up with primary", + getLatestSegmentInfoVersion(primaryShard), + getLatestSegmentInfoVersion(replicaShard) + ); + } + } + } + } + } + }, 30, TimeUnit.SECONDS); + } + } else { + throw new IllegalStateException( + "Segment Replication is not supported for testing tests using External Test Cluster" + ); + } + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * Checks if Segment Replication is enabled on Index. + */ + protected boolean isSegmentReplicationEnabledForIndex(String index) { + return clusterService().state().getMetadata().isSegmentReplicationEnabled(index); + } + + protected IndexShard getIndexShard(ShardRouting routing, String indexName) { + return getIndexShard(getClusterState().nodes().get(routing.currentNodeId()).getName(), routing.shardId(), indexName); + } + + /** + * Fetch IndexShard by shardId, multiple shards per node allowed. + */ + protected IndexShard getIndexShard(String node, ShardId shardId, String indexName) { + final Index index = resolveIndex(indexName); + IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node); + IndexService indexService = indicesService.indexServiceSafe(index); + final Optional id = indexService.shardIds().stream().filter(sid -> sid.equals(shardId.id())).findFirst(); + return indexService.getShard(id.get()); + } + + /** + * Fetch latest segment info snapshot version of an index. + */ + protected long getLatestSegmentInfoVersion(IndexShard shard) { + try (final GatedCloseable snapshot = shard.getSegmentInfosSnapshot()) { + return snapshot.get().version; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } diff --git a/test/framework/src/main/java/org/opensearch/test/ParameterizedOpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/ParameterizedOpenSearchIntegTestCase.java index 88a44f87952f5..23316adf6a2d7 100644 --- a/test/framework/src/main/java/org/opensearch/test/ParameterizedOpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/ParameterizedOpenSearchIntegTestCase.java @@ -35,7 +35,7 @@ abstract class ParameterizedOpenSearchIntegTestCase extends OpenSearchIntegTestC // This method shouldn't be called in setupSuiteScopeCluster(). Only call this method inside single test. public void indexRandomForConcurrentSearch(String... indices) throws InterruptedException { - if (settings.get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey()).equals("true")) { + if (CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.get(settings)) { indexRandomForMultipleSlices(indices); } } diff --git a/test/framework/src/main/java/org/opensearch/test/VersionUtils.java b/test/framework/src/main/java/org/opensearch/test/VersionUtils.java index 8fb9bc5cd7c1c..8ce5afab17c00 100644 --- a/test/framework/src/main/java/org/opensearch/test/VersionUtils.java +++ b/test/framework/src/main/java/org/opensearch/test/VersionUtils.java @@ -359,4 +359,14 @@ public static Version randomPreviousCompatibleVersion(Random random, Version ver // but 7.2.0 for minimum compat return randomVersionBetween(random, version.minimumIndexCompatibilityVersion(), getPreviousVersion(version)); } + + /** + * Returns a {@link Version} with a given major, minor and revision version. + * Build version is skipped for the sake of simplicity. + */ + public static Version getVersion(byte major, byte minor, byte revision) { + StringBuilder sb = new StringBuilder(); + sb.append(major).append('.').append(minor).append('.').append(revision); + return Version.fromString(sb.toString()); + } }