From a1c9def64ecd09afc80c181b65150375ad8fa23b Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Tue, 5 Jun 2018 15:00:34 -0400 Subject: [PATCH 01/26] [Rollup] Disallow index patterns that match the rollup index (#30491) We should not allow the user to configure index patterns that also match the index which stores the rollup index. For example, it is quite natural for a user to specify `metricbeat-*` as the index pattern, and then store the rollups in `metricbeat-rolled`. This will start throwing errors as soon as the rollup index is created because the indexer will try to search it. Note: this does not prevent the user from matching against existing rollup indices. That should be prevented by the field-level validation during job creation. --- .../core/rollup/job/RollupJobConfig.java | 14 ++++++++- .../xpack/core/rollup/ConfigTestHelpers.java | 5 +-- .../action/TransportPutRollupJobAction.java | 4 +-- .../xpack/rollup/config/ConfigTests.java | 31 +++++++++++++++++++ .../rest-api-spec/test/rollup/put_job.yml | 1 - 5 files changed, 49 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfig.java index a799cbe944715..3818ebcf44758 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfig.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContent; @@ -173,7 +174,7 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par builder.endObject(); return builder; } - + @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(id); @@ -336,6 +337,17 @@ public RollupJobConfig build() { if (indexPattern == null || indexPattern.isEmpty()) { throw new IllegalArgumentException("An index pattern is mandatory."); } + if (Regex.isMatchAllPattern(indexPattern)) { + throw new IllegalArgumentException("Index pattern must not match all indices (as it would match it's own rollup index"); + } + if (Regex.isSimpleMatchPattern(indexPattern)) { + if (Regex.simpleMatch(indexPattern, rollupIndex)) { + throw new IllegalArgumentException("Index pattern would match rollup index name which is not allowed."); + } + } + if (indexPattern.equals(rollupIndex)) { + throw new IllegalArgumentException("Rollup index may not be the same as the index pattern."); + } if (rollupIndex == null || rollupIndex.isEmpty()) { throw new IllegalArgumentException("A rollup index name is mandatory."); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java index 7522f474e77b2..3d82ac118f503 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java @@ -27,8 +27,9 @@ public static RollupJobConfig.Builder getRollupJob(String jobId) { builder.setId(jobId); builder.setCron(getCronString()); builder.setTimeout(new TimeValue(ESTestCase.randomIntBetween(1,100))); - builder.setIndexPattern(ESTestCase.randomAlphaOfLengthBetween(1,10)); - builder.setRollupIndex(ESTestCase.randomAlphaOfLengthBetween(1,10)); + String indexPattern = ESTestCase.randomAlphaOfLengthBetween(1,10); + builder.setIndexPattern(indexPattern); + builder.setRollupIndex("rollup_" + indexPattern); // to ensure the index pattern != rollup index builder.setGroupConfig(ConfigTestHelpers.getGroupConfig().build()); builder.setPageSize(ESTestCase.randomIntBetween(1,10)); if (ESTestCase.randomBoolean()) { diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java index 24dcb323e3dc6..819a8dfa3fe9f 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java @@ -95,8 +95,8 @@ protected void masterOperation(PutRollupJobAction.Request request, ClusterState XPackPlugin.checkReadyForXPackCustomMetadata(clusterState); FieldCapabilitiesRequest fieldCapsRequest = new FieldCapabilitiesRequest() - .indices(request.getConfig().getIndexPattern()) - .fields(request.getConfig().getAllFields().toArray(new String[0])); + .indices(request.getConfig().getIndexPattern()) + .fields(request.getConfig().getAllFields().toArray(new String[0])); client.fieldCaps(fieldCapsRequest, new ActionListener() { @Override diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/config/ConfigTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/config/ConfigTests.java index f22a6c87a3ff1..e465c7883cfdd 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/config/ConfigTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/config/ConfigTests.java @@ -122,6 +122,37 @@ public void testEmptyIndexPattern() { assertThat(e.getMessage(), equalTo("An index pattern is mandatory.")); } + public void testMatchAllIndexPattern() { + RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); + job.setIndexPattern("*"); + Exception e = expectThrows(IllegalArgumentException.class, job::build); + assertThat(e.getMessage(), equalTo("Index pattern must not match all indices (as it would match it's own rollup index")); + } + + public void testMatchOwnRollupPatternPrefix() { + RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); + job.setIndexPattern("foo-*"); + job.setRollupIndex("foo-rollup"); + Exception e = expectThrows(IllegalArgumentException.class, job::build); + assertThat(e.getMessage(), equalTo("Index pattern would match rollup index name which is not allowed.")); + } + + public void testMatchOwnRollupPatternSuffix() { + RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); + job.setIndexPattern("*-rollup"); + job.setRollupIndex("foo-rollup"); + Exception e = expectThrows(IllegalArgumentException.class, job::build); + assertThat(e.getMessage(), equalTo("Index pattern would match rollup index name which is not allowed.")); + } + + public void testIndexPatternIdenticalToRollup() { + RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); + job.setIndexPattern("foo"); + job.setRollupIndex("foo"); + Exception e = expectThrows(IllegalArgumentException.class, job::build); + assertThat(e.getMessage(), equalTo("Rollup index may not be the same as the index pattern.")); + } + public void testEmptyRollupIndex() { RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); job.setRollupIndex(""); diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml index 568a6261cda9b..717be0d6b250f 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml @@ -188,4 +188,3 @@ setup: ] } - From 7c05f69c390172f789e7bd7f13af19175cfb36fb Mon Sep 17 00:00:00 2001 From: lcawl Date: Tue, 5 Jun 2018 16:43:55 -0700 Subject: [PATCH 02/26] [DOCS] Creates rest-api folder in docs --- docs/reference/index.asciidoc | 2 +- docs/reference/rest-api/index.asciidoc | 29 ++++++++++++++++++++++++++ x-pack/docs/en/rest-api/index.asciidoc | 29 -------------------------- 3 files changed, 30 insertions(+), 30 deletions(-) create mode 100644 docs/reference/rest-api/index.asciidoc delete mode 100644 x-pack/docs/en/rest-api/index.asciidoc diff --git a/docs/reference/index.asciidoc b/docs/reference/index.asciidoc index 8567ed63b3418..b38a554d6815b 100644 --- a/docs/reference/index.asciidoc +++ b/docs/reference/index.asciidoc @@ -63,7 +63,7 @@ include::{xes-repo-dir}/monitoring/index.asciidoc[] include::{xes-repo-dir}/rollup/index.asciidoc[] -include::{xes-repo-dir}/rest-api/index.asciidoc[] +include::rest-api/index.asciidoc[] include::{xes-repo-dir}/commands/index.asciidoc[] diff --git a/docs/reference/rest-api/index.asciidoc b/docs/reference/rest-api/index.asciidoc new file mode 100644 index 0000000000000..8c58246a0a658 --- /dev/null +++ b/docs/reference/rest-api/index.asciidoc @@ -0,0 +1,29 @@ +[role="xpack"] +[[xpack-api]] += {xpack} APIs + +[partintro] +-- +{xpack} exposes REST APIs that are used by the UI components and can be called +directly to configure and access {xpack} features. + +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +-- + + +include::{xes-repo-dir}/rest-api/info.asciidoc[] +include::{xes-repo-dir}/rest-api/graph/explore.asciidoc[] +include::{xes-repo-dir}/rest-api/licensing.asciidoc[] +include::{xes-repo-dir}/rest-api/migration.asciidoc[] +include::{xes-repo-dir}/rest-api/ml-api.asciidoc[] +include::{xes-repo-dir}/rest-api/rollup-api.asciidoc[] +include::{xes-repo-dir}/rest-api/security.asciidoc[] +include::{xes-repo-dir}/rest-api/watcher.asciidoc[] +include::{xes-repo-dir}/rest-api/defs.asciidoc[] diff --git a/x-pack/docs/en/rest-api/index.asciidoc b/x-pack/docs/en/rest-api/index.asciidoc deleted file mode 100644 index 85c72a78d99fd..0000000000000 --- a/x-pack/docs/en/rest-api/index.asciidoc +++ /dev/null @@ -1,29 +0,0 @@ -[role="xpack"] -[[xpack-api]] -= {xpack} APIs - -[partintro] --- -{xpack} exposes REST APIs that are used by the UI components and can be called -directly to configure and access {xpack} features. - -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> --- - - -include::info.asciidoc[] -include::graph/explore.asciidoc[] -include::licensing.asciidoc[] -include::migration.asciidoc[] -include::ml-api.asciidoc[] -include::rollup-api.asciidoc[] -include::security.asciidoc[] -include::watcher.asciidoc[] -include::defs.asciidoc[] From 805648848d50b89e4c85d2f35a8de7456c67a694 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Tue, 5 Jun 2018 19:56:22 -0400 Subject: [PATCH 03/26] Add check for feature aware implementations (#31081) This commit adds a check that any class in X-Pack that is a feature aware custom also implements the appropriate mix-in interface in X-Pack. These interfaces provide a default implementation of FeatureAware#getRequiredFeature that returns that x-pack is the required feature. By implementing this interface, this gives a consistent way for X-Pack feature aware customs to return the appopriate required feature and this check enforces that all such feature aware customs return the appropriate required feature. --- build.gradle | 11 +- x-pack/plugin/build.gradle | 50 ++- x-pack/test/feature-aware/build.gradle | 16 + .../test/feature_aware/FeatureAwareCheck.java | 180 ++++++++++ .../feature_aware/FeatureAwareCheckTests.java | 323 ++++++++++++++++++ 5 files changed, 574 insertions(+), 6 deletions(-) create mode 100644 x-pack/test/feature-aware/build.gradle create mode 100644 x-pack/test/feature-aware/src/main/java/org/elasticsearch/xpack/test/feature_aware/FeatureAwareCheck.java create mode 100644 x-pack/test/feature-aware/src/test/java/org/elasticsearch/xpack/test/feature_aware/FeatureAwareCheckTests.java diff --git a/build.gradle b/build.gradle index 05ad5479e8dea..620e043d1c081 100644 --- a/build.gradle +++ b/build.gradle @@ -226,6 +226,7 @@ subprojects { "org.elasticsearch.distribution.deb:elasticsearch:${version}": ':distribution:packages:deb', "org.elasticsearch.distribution.deb:elasticsearch-oss:${version}": ':distribution:packages:oss-deb', "org.elasticsearch.test:logger-usage:${version}": ':test:logger-usage', + "org.elasticsearch.xpack.test:feature-aware:${version}": ':x-pack:test:feature-aware', // for transport client "org.elasticsearch.plugin:transport-netty4-client:${version}": ':modules:transport-netty4', "org.elasticsearch.plugin:reindex-client:${version}": ':modules:reindex', @@ -311,7 +312,15 @@ gradle.projectsEvaluated { // :test:framework:test cannot run before and after :server:test return } - configurations.all { + configurations.all { Configuration configuration -> + /* + * The featureAwarePlugin configuration has a dependency on x-pack:plugin:core and x-pack:plugin:core has a dependency on the + * featureAwarePlugin configuration. The below task ordering logic would force :x-pack:plugin:core:test + * :x-pack:test:feature-aware:test to depend on each other circularly. We break that cycle here. + */ + if (configuration.name == "featureAwarePlugin") { + return + } dependencies.all { Dependency dep -> Project upstreamProject = dependencyToProject(dep) if (upstreamProject != null) { diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 4a0b29c42582a..ac423c4281138 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -1,12 +1,8 @@ import org.elasticsearch.gradle.LoggedExec -import org.elasticsearch.gradle.MavenFilteringHack +import org.elasticsearch.gradle.plugin.PluginBuildPlugin import org.elasticsearch.gradle.test.NodeInfo import java.nio.charset.StandardCharsets -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.StandardCopyOption -import org.elasticsearch.gradle.test.RunTask; apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' @@ -17,6 +13,50 @@ dependencies { testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') } +subprojects { + afterEvaluate { + if (project.plugins.hasPlugin(PluginBuildPlugin)) { + // see the root Gradle file for additional logic regarding this configuration + project.configurations.create('featureAwarePlugin') + project.dependencies.add('featureAwarePlugin', project.configurations.compileClasspath) + project.dependencies.add( + 'featureAwarePlugin', + "org.elasticsearch.xpack.test:feature-aware:${org.elasticsearch.gradle.VersionProperties.elasticsearch}") + project.dependencies.add('featureAwarePlugin', project.sourceSets.main.output.getClassesDirs()) + + final Task featureAwareTask = project.tasks.create("featureAwareCheck", LoggedExec) { + description = "Runs FeatureAwareCheck on main classes." + dependsOn project.configurations.featureAwarePlugin + + final File successMarker = new File(project.buildDir, 'markers/featureAware') + outputs.file(successMarker) + + executable = new File(project.runtimeJavaHome, 'bin/java') + + // default to main class files if such a source set exists + final List files = [] + if (project.sourceSets.findByName("main")) { + files.add(project.sourceSets.main.output.classesDir) + dependsOn project.tasks.classes + } + // filter out non-existent classes directories from empty source sets + final FileCollection classDirectories = project.files(files).filter { it.exists() } + + doFirst { + args('-cp', project.configurations.featureAwarePlugin.asPath, 'org.elasticsearch.xpack.test.feature_aware.FeatureAwareCheck') + classDirectories.each { args it.getAbsolutePath() } + } + doLast { + successMarker.parentFile.mkdirs() + successMarker.setText("", 'UTF-8') + } + } + + project.precommit.dependsOn featureAwareTask + } + } +} + // https://github.com/elastic/x-plugins/issues/724 configurations { testArtifacts.extendsFrom testRuntime diff --git a/x-pack/test/feature-aware/build.gradle b/x-pack/test/feature-aware/build.gradle new file mode 100644 index 0000000000000..217ed25a2d4b1 --- /dev/null +++ b/x-pack/test/feature-aware/build.gradle @@ -0,0 +1,16 @@ +apply plugin: 'elasticsearch.build' + +dependencies { + compile 'org.ow2.asm:asm:6.2' + compile "org.elasticsearch:elasticsearch:${version}" + compile "org.elasticsearch.plugin:x-pack-core:${version}" + testCompile "org.elasticsearch.test:framework:${version}" +} + +forbiddenApisMain.enabled = true + +dependencyLicenses.enabled = false + +jarHell.enabled = false + +thirdPartyAudit.enabled = false diff --git a/x-pack/test/feature-aware/src/main/java/org/elasticsearch/xpack/test/feature_aware/FeatureAwareCheck.java b/x-pack/test/feature-aware/src/main/java/org/elasticsearch/xpack/test/feature_aware/FeatureAwareCheck.java new file mode 100644 index 0000000000000..7746692b408e9 --- /dev/null +++ b/x-pack/test/feature-aware/src/main/java/org/elasticsearch/xpack/test/feature_aware/FeatureAwareCheck.java @@ -0,0 +1,180 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.test.feature_aware; + +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.persistent.PersistentTaskParams; +import org.elasticsearch.xpack.core.XPackPlugin; +import org.objectweb.asm.ClassReader; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import java.util.function.Consumer; + +/** + * Used in the featureAwareCheck to check for classes in X-Pack that implement customs but do not extend the appropriate marker interface. + */ +public final class FeatureAwareCheck { + + /** + * Check the class directories specified by the arguments for classes in X-Pack that implement customs but do not extend the appropriate + * marker interface that provides a mix-in implementation of {@link ClusterState.FeatureAware#getRequiredFeature()}. + * + * @param args the class directories to check + * @throws IOException if an I/O exception is walking the class directories + */ + public static void main(final String[] args) throws IOException { + systemOutPrintln("checking for custom violations"); + final List violations = new ArrayList<>(); + checkDirectories(violations::add, args); + if (violations.isEmpty()) { + systemOutPrintln("no custom violations found"); + } else { + violations.forEach(violation -> + systemOutPrintln( + "class [" + violation.name + "] implements" + + " [" + violation.interfaceName + " but does not implement" + + " [" + violation.expectedInterfaceName + "]") + ); + throw new IllegalStateException( + "found custom" + (violations.size() == 1 ? "" : "s") + " in X-Pack not extending appropriate X-Pack mix-in"); + } + } + + @SuppressForbidden(reason = "System.out#println") + private static void systemOutPrintln(final String s) { + System.out.println(s); + } + + private static void checkDirectories( + final Consumer callback, + final String... classDirectories) throws IOException { + for (final String classDirectory : classDirectories) { + final Path root = pathsGet(classDirectory); + if (Files.isDirectory(root)) { + Files.walkFileTree(root, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException { + if (Files.isRegularFile(file) && file.getFileName().toString().endsWith(".class")) { + try (InputStream in = Files.newInputStream(file)) { + checkClass(in, callback); + } + } + return super.visitFile(file, attrs); + } + }); + } else { + throw new FileNotFoundException("class directory [" + classDirectory + "] should exist"); + } + } + } + + @SuppressForbidden(reason = "Paths#get") + private static Path pathsGet(final String pathString) { + return Paths.get(pathString); + } + + /** + * Represents a feature-aware violation. + */ + static class FeatureAwareViolation { + + final String name; + final String interfaceName; + final String expectedInterfaceName; + + /** + * Constructs a representation of a feature-aware violation. + * + * @param name the name of the custom class + * @param interfaceName the name of the feature-aware interface + * @param expectedInterfaceName the name of the expected mix-in class + */ + FeatureAwareViolation(final String name, final String interfaceName, final String expectedInterfaceName) { + this.name = name; + this.interfaceName = interfaceName; + this.expectedInterfaceName = expectedInterfaceName; + } + + } + + /** + * Loads a class from the specified input stream and checks that if it implements a feature-aware custom then it extends the appropriate + * mix-in interface from X-Pack. If the class does not, then the specified callback is invoked. + * + * @param in the input stream + * @param callback the callback to invoke + * @throws IOException if an I/O exception occurs loading the class hierarchy + */ + static void checkClass(final InputStream in, final Consumer callback) throws IOException { + // the class format only reports declared interfaces so we have to walk the hierarchy looking for all interfaces + final List interfaces = new ArrayList<>(); + ClassReader cr = new ClassReader(in); + final String name = cr.getClassName(); + do { + interfaces.addAll(Arrays.asList(cr.getInterfaces())); + final String superName = cr.getSuperName(); + if ("java/lang/Object".equals(superName)) { + break; + } + cr = new ClassReader(superName); + } while (true); + checkClass(name, interfaces, callback); + } + + private static void checkClass( + final String name, + final List interfaces, + final Consumer callback) { + checkCustomForClass(ClusterState.Custom.class, XPackPlugin.XPackClusterStateCustom.class, name, interfaces, callback); + checkCustomForClass(MetaData.Custom.class, XPackPlugin.XPackMetaDataCustom.class, name, interfaces, callback); + checkCustomForClass(PersistentTaskParams.class, XPackPlugin.XPackPersistentTaskParams.class, name, interfaces, callback); + } + + private static void checkCustomForClass( + final Class interfaceToCheck, + final Class expectedInterface, + final String name, + final List interfaces, + final Consumer callback) { + final Set interfaceSet = new TreeSet<>(interfaces); + final String interfaceToCheckName = formatClassName(interfaceToCheck); + final String expectedXPackInterfaceName = formatClassName(expectedInterface); + if (interfaceSet.contains(interfaceToCheckName) + && name.equals(expectedXPackInterfaceName) == false + && interfaceSet.contains(expectedXPackInterfaceName) == false) { + assert name.startsWith("org/elasticsearch/license") || name.startsWith("org/elasticsearch/xpack"); + callback.accept(new FeatureAwareViolation(name, interfaceToCheckName, expectedXPackInterfaceName)); + } + } + + /** + * Format the specified class to a name in the ASM format replacing all dots in the class name with forward-slashes. + * + * @param clazz the class whose name to format + * @return the formatted class name + */ + static String formatClassName(final Class clazz) { + return clazz.getName().replace(".", "/"); + } + +} diff --git a/x-pack/test/feature-aware/src/test/java/org/elasticsearch/xpack/test/feature_aware/FeatureAwareCheckTests.java b/x-pack/test/feature-aware/src/test/java/org/elasticsearch/xpack/test/feature_aware/FeatureAwareCheckTests.java new file mode 100644 index 0000000000000..2dde9efce42bb --- /dev/null +++ b/x-pack/test/feature-aware/src/test/java/org/elasticsearch/xpack/test/feature_aware/FeatureAwareCheckTests.java @@ -0,0 +1,323 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.test.feature_aware; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.Diff; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.persistent.PersistentTaskParams; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.XPackPlugin; + +import java.io.IOException; +import java.util.EnumSet; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; + +import static org.hamcrest.Matchers.equalTo; + +public class FeatureAwareCheckTests extends ESTestCase { + + public void testClusterStateCustomViolation() throws IOException { + runCustomViolationTest( + ClusterStateCustomViolation.class, + getClass(), + ClusterState.Custom.class, + XPackPlugin.XPackClusterStateCustom.class); + } + + public void testClusterStateCustom() throws IOException { + runCustomTest(XPackClusterStateCustom.class, getClass(), ClusterState.Custom.class, XPackPlugin.XPackClusterStateCustom.class); + } + + public void testClusterStateCustomMarkerInterface() throws IOException { + // marker interfaces do not implement the marker interface but should not fail the feature aware check + runCustomTest( + XPackPlugin.XPackClusterStateCustom.class, + XPackPlugin.class, + ClusterState.Custom.class, + XPackPlugin.XPackClusterStateCustom.class); + } + + public void testMetaDataCustomViolation() throws IOException { + runCustomViolationTest(MetaDataCustomViolation.class, getClass(), MetaData.Custom.class, XPackPlugin.XPackMetaDataCustom.class); + } + + public void testMetaDataCustom() throws IOException { + runCustomTest(XPackMetaDataCustom.class, getClass(), MetaData.Custom.class, XPackPlugin.XPackMetaDataCustom.class); + } + + public void testMetaDataCustomMarkerInterface() throws IOException { + // marker interfaces do not implement the marker interface but should not fail the feature aware check + runCustomTest( + XPackPlugin.XPackMetaDataCustom.class, + XPackPlugin.class, + MetaData.Custom.class, + XPackPlugin.XPackMetaDataCustom.class); + } + + public void testPersistentTaskParamsViolation() throws IOException { + runCustomViolationTest( + PersistentTaskParamsViolation.class, + getClass(), + PersistentTaskParams.class, + XPackPlugin.XPackPersistentTaskParams.class); + } + + public void testPersistentTaskParams() throws IOException { + runCustomTest(XPackPersistentTaskParams.class, getClass(), PersistentTaskParams.class, XPackPlugin.XPackPersistentTaskParams.class); + } + + public void testPersistentTaskParamsMarkerInterface() throws IOException { + // marker interfaces do not implement the marker interface but should not fail the feature aware check + runCustomTest( + XPackPlugin.XPackPersistentTaskParams.class, + XPackPlugin.class, + PersistentTaskParams.class, + XPackPlugin.XPackPersistentTaskParams.class); + } + + abstract class ClusterStateCustomFeatureAware implements ClusterState.Custom { + + private final String writeableName; + + ClusterStateCustomFeatureAware(final String writeableName) { + this.writeableName = writeableName; + } + + @Override + public Diff diff(ClusterState.Custom previousState) { + return null; + } + + @Override + public String getWriteableName() { + return writeableName; + } + + @Override + public Version getMinimalSupportedVersion() { + return Version.CURRENT.minimumCompatibilityVersion(); + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + + } + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + return builder; + } + + } + + class ClusterStateCustomViolation extends ClusterStateCustomFeatureAware { + + ClusterStateCustomViolation() { + super("cluster_state_custom_violation"); + } + } + + class XPackClusterStateCustom extends ClusterStateCustomFeatureAware implements XPackPlugin.XPackClusterStateCustom { + + XPackClusterStateCustom() { + super("x_pack_cluster_state_custom"); + } + + } + + abstract class MetaDataCustomFeatureAware implements MetaData.Custom { + + private final String writeableName; + + MetaDataCustomFeatureAware(final String writeableName) { + this.writeableName = writeableName; + } + + @Override + public EnumSet context() { + return MetaData.ALL_CONTEXTS; + } + + @Override + public Diff diff(MetaData.Custom previousState) { + return null; + } + + @Override + public String getWriteableName() { + return writeableName; + } + + @Override + public Version getMinimalSupportedVersion() { + return Version.CURRENT.minimumCompatibilityVersion(); + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + + } + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + return builder; + } + + } + + class MetaDataCustomViolation extends MetaDataCustomFeatureAware { + + MetaDataCustomViolation() { + super("meta_data_custom_violation"); + } + + } + + class XPackMetaDataCustom extends MetaDataCustomFeatureAware implements XPackPlugin.XPackMetaDataCustom { + + XPackMetaDataCustom() { + super("x_pack_meta_data_custom"); + } + + } + + abstract class PersistentTaskParamsFeatureAware implements PersistentTaskParams { + + private final String writeableName; + + PersistentTaskParamsFeatureAware(final String writeableName) { + this.writeableName = writeableName; + } + + @Override + public String getWriteableName() { + return writeableName; + } + + @Override + public Version getMinimalSupportedVersion() { + return Version.CURRENT.minimumCompatibilityVersion(); + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + + } + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + return builder; + } + + } + + class PersistentTaskParamsViolation extends PersistentTaskParamsFeatureAware { + + PersistentTaskParamsViolation() { + super("persistent_task_params_violation"); + } + + } + + class XPackPersistentTaskParams extends PersistentTaskParamsFeatureAware implements XPackPlugin.XPackPersistentTaskParams { + + XPackPersistentTaskParams() { + super("x_pack_persistent_task_params"); + } + + } + + private class FeatureAwareViolationConsumer implements Consumer { + + private final AtomicBoolean called = new AtomicBoolean(); + private final String name; + private final String interfaceName; + private final String expectedInterfaceName; + + FeatureAwareViolationConsumer(final String name, final String interfaceName, final String expectedInterfaceName) { + this.name = name; + this.interfaceName = interfaceName; + this.expectedInterfaceName = expectedInterfaceName; + } + + @Override + public void accept(final org.elasticsearch.xpack.test.feature_aware.FeatureAwareCheck.FeatureAwareViolation featureAwareViolation) { + called.set(true); + assertThat(featureAwareViolation.name, equalTo(name)); + assertThat(featureAwareViolation.interfaceName, equalTo(interfaceName)); + assertThat(featureAwareViolation.expectedInterfaceName, equalTo(expectedInterfaceName)); + } + + } + + /** + * Runs a test on an actual class implementing a custom interface and not the expected marker interface. + * + * @param clazz the custom implementation + * @param outerClazz the outer class to load the custom implementation relative to + * @param interfaceClazz the custom + * @param expectedInterfaceClazz the marker interface + * @throws IOException if an I/O error occurs reading the class + */ + private void runCustomViolationTest( + final Class clazz, + final Class outerClazz, + final Class interfaceClazz, + final Class expectedInterfaceClazz) throws IOException { + runTest(clazz, outerClazz, interfaceClazz, expectedInterfaceClazz, true); + } + + /** + * Runs a test on an actual class implementing a custom interface and the expected marker interface. + * + * @param clazz the custom implementation + * @param outerClazz the outer class to load the custom implementation relative to + * @param interfaceClazz the custom + * @param expectedInterfaceClazz the marker interface + * @throws IOException if an I/O error occurs reading the class + */ + private void runCustomTest( + final Class clazz, + final Class outerClazz, + final Class interfaceClazz, + final Class expectedInterfaceClazz) throws IOException { + runTest(clazz, outerClazz, interfaceClazz, expectedInterfaceClazz, false); + } + + /** + * Runs a test on an actual class implementing a custom interface and should implement the expected marker interface if and only if + * the specified violation parameter is false. + * + * @param clazz the custom implementation + * @param outerClazz the outer class to load the custom implementation relative to + * @param interfaceClazz the custom + * @param expectedInterfaceClazz the marker interface + * @param violation whether or not the actual class is expected to fail the feature aware check + * @throws IOException if an I/O error occurs reading the class + */ + private void runTest( + final Class clazz, + final Class outerClazz, + final Class interfaceClazz, + final Class expectedInterfaceClazz, + final boolean violation) throws IOException { + final String name = clazz.getName(); + final FeatureAwareViolationConsumer callback = + new FeatureAwareViolationConsumer( + FeatureAwareCheck.formatClassName(clazz), + FeatureAwareCheck.formatClassName(interfaceClazz), + FeatureAwareCheck.formatClassName(expectedInterfaceClazz)); + FeatureAwareCheck.checkClass(outerClazz.getResourceAsStream(name.substring(1 + name.lastIndexOf(".")) + ".class"), callback); + assertThat(callback.called.get(), equalTo(violation)); + } + +} From 735d0e671aa71a9c6ecf111eeca4d0355ea8167f Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 6 Jun 2018 07:40:21 +0200 Subject: [PATCH 04/26] Make PreBuiltAnalyzerProviderFactory plugable via AnalysisPlugin and move `finger_print`, `pattern` and `standard_html_strip` analyzers to analysis-common module. (both AnalysisProvider and PreBuiltAnalyzerProvider) Changed PreBuiltAnalyzerProviderFactory to extend from PreConfiguredAnalysisComponent and changed to make sure that predefined analyzers are always instantiated with the current ES version and if an instance is requested for a different version then delegate to PreBuiltCache. This is similar to the behaviour that exists today in AnalysisRegistry.PreBuiltAnalysis and PreBuiltAnalyzerProviderFactory. (#31095) Relates to #23658 --- .../analysis/common/CommonAnalysisPlugin.java | 23 +++++ .../analysis/common}/FingerprintAnalyzer.java | 4 +- .../common}/FingerprintAnalyzerProvider.java | 6 +- .../common/FingerprintTokenFilterFactory.java | 6 +- .../analysis/common}/PatternAnalyzer.java | 4 +- .../common}/PatternAnalyzerProvider.java | 6 +- .../common}/StandardHtmlStripAnalyzer.java | 4 +- .../StandardHtmlStripAnalyzerProvider.java | 6 +- .../common}/FingerprintAnalyzerTests.java | 2 +- .../common}/PatternAnalyzerTests.java | 2 +- .../test/analysis-common/20_analyzers.yml | 32 ++++++ .../index/analysis/AnalysisRegistry.java | 28 +++--- .../PreBuiltAnalyzerProviderFactory.java | 98 +++++++++++++++---- .../PreConfiguredAnalysisComponent.java | 9 +- .../indices/analysis/AnalysisModule.java | 20 ++-- .../indices/analysis/PreBuiltAnalyzers.java | 21 +--- .../analysis/PreBuiltCacheFactory.java | 21 ++++ .../elasticsearch/plugins/AnalysisPlugin.java | 8 ++ .../elasticsearch/index/IndexModuleTests.java | 2 +- .../index/analysis/AnalysisRegistryTests.java | 17 +++- .../highlight/HighlighterSearchIT.java | 34 ++++++- .../xpack/watcher/WatcherPluginTests.java | 2 +- 22 files changed, 265 insertions(+), 90 deletions(-) rename {server/src/main/java/org/elasticsearch/index/analysis => modules/analysis-common/src/main/java/org/elasticsearch/analysis/common}/FingerprintAnalyzer.java (94%) rename {server/src/main/java/org/elasticsearch/index/analysis => modules/analysis-common/src/main/java/org/elasticsearch/analysis/common}/FingerprintAnalyzerProvider.java (90%) rename {server/src/main/java/org/elasticsearch/index/analysis => modules/analysis-common/src/main/java/org/elasticsearch/analysis/common}/PatternAnalyzer.java (94%) rename {server/src/main/java/org/elasticsearch/index/analysis => modules/analysis-common/src/main/java/org/elasticsearch/analysis/common}/PatternAnalyzerProvider.java (88%) rename {server/src/main/java/org/elasticsearch/index/analysis => modules/analysis-common/src/main/java/org/elasticsearch/analysis/common}/StandardHtmlStripAnalyzer.java (95%) rename {server/src/main/java/org/elasticsearch/index/analysis => modules/analysis-common/src/main/java/org/elasticsearch/analysis/common}/StandardHtmlStripAnalyzerProvider.java (85%) rename {server/src/test/java/org/elasticsearch/index/analysis => modules/analysis-common/src/test/java/org/elasticsearch/analysis/common}/FingerprintAnalyzerTests.java (98%) rename {server/src/test/java/org/elasticsearch/index/analysis => modules/analysis-common/src/test/java/org/elasticsearch/analysis/common}/PatternAnalyzerTests.java (99%) diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java index 69c8afb3e2fc6..433bef902c1a1 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java @@ -19,6 +19,7 @@ package org.elasticsearch.analysis.common; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.CharArraySet; import org.apache.lucene.analysis.LowerCaseFilter; import org.apache.lucene.analysis.StopFilter; @@ -79,7 +80,9 @@ import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.index.analysis.AnalyzerProvider; import org.elasticsearch.index.analysis.CharFilterFactory; +import org.elasticsearch.index.analysis.PreBuiltAnalyzerProviderFactory; import org.elasticsearch.index.analysis.PreConfiguredCharFilter; import org.elasticsearch.index.analysis.PreConfiguredTokenFilter; import org.elasticsearch.index.analysis.PreConfiguredTokenizer; @@ -87,6 +90,7 @@ import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.index.analysis.TokenizerFactory; import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider; +import org.elasticsearch.indices.analysis.PreBuiltCacheFactory.CachingStrategy; import org.elasticsearch.plugins.AnalysisPlugin; import org.elasticsearch.plugins.Plugin; import org.tartarus.snowball.ext.DutchStemmer; @@ -103,6 +107,15 @@ public class CommonAnalysisPlugin extends Plugin implements AnalysisPlugin { private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(CommonAnalysisPlugin.class)); + @Override + public Map>> getAnalyzers() { + Map>> analyzers = new TreeMap<>(); + analyzers.put("fingerprint", FingerprintAnalyzerProvider::new); + analyzers.put("standard_html_strip", StandardHtmlStripAnalyzerProvider::new); + analyzers.put("pattern", PatternAnalyzerProvider::new); + return analyzers; + } + @Override public Map> getTokenFilters() { Map> filters = new TreeMap<>(); @@ -197,6 +210,16 @@ public Map> getTokenizers() { return tokenizers; } + @Override + public List getPreBuiltAnalyzerProviderFactories() { + List analyzers = new ArrayList<>(); + analyzers.add(new PreBuiltAnalyzerProviderFactory("standard_html_strip", CachingStrategy.LUCENE, + version -> new StandardHtmlStripAnalyzer(CharArraySet.EMPTY_SET))); + analyzers.add(new PreBuiltAnalyzerProviderFactory("pattern", CachingStrategy.ELASTICSEARCH, version -> + new PatternAnalyzer(Regex.compile("\\W+" /*PatternAnalyzer.NON_WORD_PATTERN*/, null), true, CharArraySet.EMPTY_SET))); + return analyzers; + } + @Override public List getPreConfiguredCharFilters() { List filters = new ArrayList<>(); diff --git a/server/src/main/java/org/elasticsearch/index/analysis/FingerprintAnalyzer.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FingerprintAnalyzer.java similarity index 94% rename from server/src/main/java/org/elasticsearch/index/analysis/FingerprintAnalyzer.java rename to modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FingerprintAnalyzer.java index 0a550f19aa7c1..d37239304cd2b 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/FingerprintAnalyzer.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FingerprintAnalyzer.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.index.analysis; +package org.elasticsearch.analysis.common; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.CharArraySet; @@ -35,7 +35,7 @@ public final class FingerprintAnalyzer extends Analyzer { private final int maxOutputSize; private final CharArraySet stopWords; - public FingerprintAnalyzer(CharArraySet stopWords, char separator, int maxOutputSize) { + FingerprintAnalyzer(CharArraySet stopWords, char separator, int maxOutputSize) { this.separator = separator; this.maxOutputSize = maxOutputSize; this.stopWords = stopWords; diff --git a/server/src/main/java/org/elasticsearch/index/analysis/FingerprintAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FingerprintAnalyzerProvider.java similarity index 90% rename from server/src/main/java/org/elasticsearch/index/analysis/FingerprintAnalyzerProvider.java rename to modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FingerprintAnalyzerProvider.java index 6a777e7c93187..f54b04bf30931 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/FingerprintAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FingerprintAnalyzerProvider.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.index.analysis; +package org.elasticsearch.analysis.common; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.CharArraySet; @@ -25,6 +25,8 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.analysis.AbstractIndexAnalyzerProvider; +import org.elasticsearch.index.analysis.Analysis; /** @@ -42,7 +44,7 @@ public class FingerprintAnalyzerProvider extends AbstractIndexAnalyzerProvider { private final StandardHtmlStripAnalyzer analyzer; - public StandardHtmlStripAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) { + StandardHtmlStripAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) { super(indexSettings, name, settings); final CharArraySet defaultStopwords = CharArraySet.EMPTY_SET; CharArraySet stopWords = Analysis.parseStopWords(env, settings, defaultStopwords); diff --git a/server/src/test/java/org/elasticsearch/index/analysis/FingerprintAnalyzerTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/FingerprintAnalyzerTests.java similarity index 98% rename from server/src/test/java/org/elasticsearch/index/analysis/FingerprintAnalyzerTests.java rename to modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/FingerprintAnalyzerTests.java index c5e854879e98e..0933f3bf13a43 100644 --- a/server/src/test/java/org/elasticsearch/index/analysis/FingerprintAnalyzerTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/FingerprintAnalyzerTests.java @@ -1,4 +1,4 @@ -package org.elasticsearch.index.analysis; +package org.elasticsearch.analysis.common; /* * Licensed to Elasticsearch under one or more contributor diff --git a/server/src/test/java/org/elasticsearch/index/analysis/PatternAnalyzerTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PatternAnalyzerTests.java similarity index 99% rename from server/src/test/java/org/elasticsearch/index/analysis/PatternAnalyzerTests.java rename to modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PatternAnalyzerTests.java index d80cbf66c34d0..d2d226d6250e8 100644 --- a/server/src/test/java/org/elasticsearch/index/analysis/PatternAnalyzerTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PatternAnalyzerTests.java @@ -1,4 +1,4 @@ -package org.elasticsearch.index.analysis; +package org.elasticsearch.analysis.common; /* * Licensed to Elasticsearch under one or more contributor diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/20_analyzers.yml b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/20_analyzers.yml index 6ff3b8c802735..d38f63f542979 100644 --- a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/20_analyzers.yml +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/20_analyzers.yml @@ -37,3 +37,35 @@ analyzer: bengali - length: { tokens: 1 } - match: { tokens.0.token: বার } + +--- +"fingerprint": + - do: + indices.analyze: + body: + text: A1 B2 A1 D4 C3 + analyzer: fingerprint + - length: { tokens: 1 } + - match: { tokens.0.token: a1 b2 c3 d4 } + +--- +"standard_html_strip": + - do: + indices.analyze: + body: + text: + analyzer: standard_html_strip + - length: { tokens: 2 } + - match: { tokens.0.token: bold } + - match: { tokens.1.token: italic } + +--- +"pattern": + - do: + indices.analyze: + body: + text: foo bar + analyzer: pattern + - length: { tokens: 2 } + - match: { tokens.0.token: foo } + - match: { tokens.1.token: bar } diff --git a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java index e421a19b2ac27..61b5cb9171244 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java @@ -22,7 +22,6 @@ import org.apache.lucene.analysis.core.WhitespaceTokenizer; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; @@ -70,14 +69,16 @@ public AnalysisRegistry(Environment environment, Map>> normalizers, Map preConfiguredCharFilters, Map preConfiguredTokenFilters, - Map preConfiguredTokenizers) { + Map preConfiguredTokenizers, + Map preConfiguredAnalyzers) { this.environment = environment; this.charFilters = unmodifiableMap(charFilters); this.tokenFilters = unmodifiableMap(tokenFilters); this.tokenizers = unmodifiableMap(tokenizers); this.analyzers = unmodifiableMap(analyzers); this.normalizers = unmodifiableMap(normalizers); - prebuiltAnalysis = new PrebuiltAnalysis(preConfiguredCharFilters, preConfiguredTokenFilters, preConfiguredTokenizers); + prebuiltAnalysis = + new PrebuiltAnalysis(preConfiguredCharFilters, preConfiguredTokenFilters, preConfiguredTokenizers, preConfiguredAnalyzers); } /** @@ -398,13 +399,15 @@ private static class PrebuiltAnalysis implements Closeable { private PrebuiltAnalysis( Map preConfiguredCharFilters, Map preConfiguredTokenFilters, - Map preConfiguredTokenizers) { - Map analyzerProviderFactories = new HashMap<>(); + Map preConfiguredTokenizers, + Map preConfiguredAnalyzers) { - // Analyzers + Map analyzerProviderFactories = new HashMap<>(); + analyzerProviderFactories.putAll(preConfiguredAnalyzers); + // Pre-build analyzers for (PreBuiltAnalyzers preBuiltAnalyzerEnum : PreBuiltAnalyzers.values()) { String name = preBuiltAnalyzerEnum.name().toLowerCase(Locale.ROOT); - analyzerProviderFactories.put(name, new PreBuiltAnalyzerProviderFactory(name, AnalyzerScope.INDICES, preBuiltAnalyzerEnum.getAnalyzer(Version.CURRENT))); + analyzerProviderFactories.put(name, new PreBuiltAnalyzerProviderFactory(name, preBuiltAnalyzerEnum)); } this.analyzerProviderFactories = Collections.unmodifiableMap(analyzerProviderFactories); @@ -429,17 +432,10 @@ public AnalysisModule.AnalysisProvider> getAnalyzerProvider( return analyzerProviderFactories.get(name); } - Analyzer analyzer(String name) { - PreBuiltAnalyzerProviderFactory analyzerProviderFactory = (PreBuiltAnalyzerProviderFactory) analyzerProviderFactories.get(name); - if (analyzerProviderFactory == null) { - return null; - } - return analyzerProviderFactory.analyzer(); - } - @Override public void close() throws IOException { - IOUtils.close(analyzerProviderFactories.values().stream().map((a) -> ((PreBuiltAnalyzerProviderFactory)a).analyzer()).collect(Collectors.toList())); + IOUtils.close(analyzerProviderFactories.values().stream() + .map((a) -> ((PreBuiltAnalyzerProviderFactory)a)).collect(Collectors.toList())); } } diff --git a/server/src/main/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerProviderFactory.java b/server/src/main/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerProviderFactory.java index 3e59377ecc288..9317f9fb1e4ab 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerProviderFactory.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerProviderFactory.java @@ -22,41 +22,101 @@ import org.apache.lucene.analysis.Analyzer; import org.elasticsearch.Version; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.indices.analysis.PreBuiltAnalyzers; +import org.elasticsearch.indices.analysis.PreBuiltCacheFactory; +import java.io.Closeable; import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; -public class PreBuiltAnalyzerProviderFactory implements AnalysisModule.AnalysisProvider> { +public class PreBuiltAnalyzerProviderFactory extends PreConfiguredAnalysisComponent> implements Closeable { - private final PreBuiltAnalyzerProvider analyzerProvider; + private final Function create; + private final PreBuiltAnalyzerProvider current; - public PreBuiltAnalyzerProviderFactory(String name, AnalyzerScope scope, Analyzer analyzer) { - analyzerProvider = new PreBuiltAnalyzerProvider(name, scope, analyzer); + /** + * This constructor only exists to expose analyzers defined in {@link PreBuiltAnalyzers} as {@link PreBuiltAnalyzerProviderFactory}. + */ + PreBuiltAnalyzerProviderFactory(String name, PreBuiltAnalyzers preBuiltAnalyzer) { + super(name, new PreBuiltAnalyzersDelegateCache(name, preBuiltAnalyzer)); + this.create = preBuiltAnalyzer::getAnalyzer; + current = new PreBuiltAnalyzerProvider(name, AnalyzerScope.INDICES, preBuiltAnalyzer.getAnalyzer(Version.CURRENT)); } - public AnalyzerProvider create(String name, Settings settings) { - Version indexVersion = Version.indexCreated(settings); - if (!Version.CURRENT.equals(indexVersion)) { - PreBuiltAnalyzers preBuiltAnalyzers = PreBuiltAnalyzers.getOrDefault(name, null); - if (preBuiltAnalyzers != null) { - Analyzer analyzer = preBuiltAnalyzers.getAnalyzer(indexVersion); - return new PreBuiltAnalyzerProvider(name, AnalyzerScope.INDICES, analyzer); - } + public PreBuiltAnalyzerProviderFactory(String name, PreBuiltCacheFactory.CachingStrategy cache, Function create) { + super(name, cache); + this.create = create; + this.current = new PreBuiltAnalyzerProvider(name, AnalyzerScope.INDICES, create.apply(Version.CURRENT)); + } + + @Override + public AnalyzerProvider get(IndexSettings indexSettings, + Environment environment, + String name, + Settings settings) throws IOException { + Version versionCreated = Version.indexCreated(settings); + if (Version.CURRENT.equals(versionCreated) == false) { + return super.get(indexSettings, environment, name, settings); + } else { + return current; } + } - return analyzerProvider; + @Override + protected AnalyzerProvider create(Version version) { + assert Version.CURRENT.equals(version) == false; + return new PreBuiltAnalyzerProvider(getName(), AnalyzerScope.INDICES, create.apply(version)); } @Override - public AnalyzerProvider get(IndexSettings indexSettings, Environment environment, String name, Settings settings) - throws IOException { - return create(name, settings); + public void close() throws IOException { + List closeables = cache.values().stream() + .map(AnalyzerProvider::get) + .collect(Collectors.toList()); + closeables.add(current.get()); + IOUtils.close(closeables); } - public Analyzer analyzer() { - return analyzerProvider.get(); + /** + * A special cache that closes the gap between PreBuiltAnalyzers and PreBuiltAnalyzerProviderFactory. + * + * This can be removed when all analyzers have been moved away from PreBuiltAnalyzers to + * PreBuiltAnalyzerProviderFactory either in server or analysis-common. + */ + static class PreBuiltAnalyzersDelegateCache implements PreBuiltCacheFactory.PreBuiltCache> { + + private final String name; + private final PreBuiltAnalyzers preBuiltAnalyzer; + + private PreBuiltAnalyzersDelegateCache(String name, PreBuiltAnalyzers preBuiltAnalyzer) { + this.name = name; + this.preBuiltAnalyzer = preBuiltAnalyzer; + } + + @Override + public AnalyzerProvider get(Version version) { + return new PreBuiltAnalyzerProvider(name, AnalyzerScope.INDICES, preBuiltAnalyzer.getAnalyzer(version)); + } + + @Override + public void put(Version version, AnalyzerProvider analyzerProvider) { + // No need to put, because we delegate in get() directly to PreBuiltAnalyzers which already caches. + } + + @Override + public Collection> values() { + return preBuiltAnalyzer.getCache().values().stream() + // Wrap the analyzer instance in a PreBuiltAnalyzerProvider, this is what PreBuiltAnalyzerProviderFactory#close expects + // (other caches are not directly caching analyzers, but analyzer provider instead. + .map(analyzer -> new PreBuiltAnalyzerProvider(name, AnalyzerScope.INDICES, analyzer)) + .collect(Collectors.toList()); + } + } } diff --git a/server/src/main/java/org/elasticsearch/index/analysis/PreConfiguredAnalysisComponent.java b/server/src/main/java/org/elasticsearch/index/analysis/PreConfiguredAnalysisComponent.java index fdd525d0c80dd..f7450c15ee9d9 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/PreConfiguredAnalysisComponent.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/PreConfiguredAnalysisComponent.java @@ -33,13 +33,18 @@ */ public abstract class PreConfiguredAnalysisComponent implements AnalysisModule.AnalysisProvider { private final String name; - private final PreBuiltCacheFactory.PreBuiltCache cache; + protected final PreBuiltCacheFactory.PreBuiltCache cache; - protected PreConfiguredAnalysisComponent(String name, PreBuiltCacheFactory.CachingStrategy cache) { + protected PreConfiguredAnalysisComponent(String name, PreBuiltCacheFactory.CachingStrategy cache) { this.name = name; this.cache = PreBuiltCacheFactory.getCache(cache); } + protected PreConfiguredAnalysisComponent(String name, PreBuiltCacheFactory.PreBuiltCache cache) { + this.name = name; + this.cache = cache; + } + @Override public T get(IndexSettings indexSettings, Environment environment, String name, Settings settings) throws IOException { Version versionCreated = Version.indexCreated(settings); diff --git a/server/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java b/server/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java index bc590381c3c7c..13aaf44c82ed9 100644 --- a/server/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java +++ b/server/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java @@ -43,7 +43,6 @@ import org.elasticsearch.index.analysis.DanishAnalyzerProvider; import org.elasticsearch.index.analysis.DutchAnalyzerProvider; import org.elasticsearch.index.analysis.EnglishAnalyzerProvider; -import org.elasticsearch.index.analysis.FingerprintAnalyzerProvider; import org.elasticsearch.index.analysis.FinnishAnalyzerProvider; import org.elasticsearch.index.analysis.FrenchAnalyzerProvider; import org.elasticsearch.index.analysis.GalicianAnalyzerProvider; @@ -59,9 +58,9 @@ import org.elasticsearch.index.analysis.LatvianAnalyzerProvider; import org.elasticsearch.index.analysis.LithuanianAnalyzerProvider; import org.elasticsearch.index.analysis.NorwegianAnalyzerProvider; -import org.elasticsearch.index.analysis.PatternAnalyzerProvider; import org.elasticsearch.index.analysis.PersianAnalyzerProvider; import org.elasticsearch.index.analysis.PortugueseAnalyzerProvider; +import org.elasticsearch.index.analysis.PreBuiltAnalyzerProviderFactory; import org.elasticsearch.index.analysis.PreConfiguredCharFilter; import org.elasticsearch.index.analysis.PreConfiguredTokenFilter; import org.elasticsearch.index.analysis.PreConfiguredTokenizer; @@ -73,7 +72,6 @@ import org.elasticsearch.index.analysis.SoraniAnalyzerProvider; import org.elasticsearch.index.analysis.SpanishAnalyzerProvider; import org.elasticsearch.index.analysis.StandardAnalyzerProvider; -import org.elasticsearch.index.analysis.StandardHtmlStripAnalyzerProvider; import org.elasticsearch.index.analysis.StandardTokenFilterFactory; import org.elasticsearch.index.analysis.StandardTokenizerFactory; import org.elasticsearch.index.analysis.StopAnalyzerProvider; @@ -122,11 +120,12 @@ public AnalysisModule(Environment environment, List plugins) thr Map preConfiguredCharFilters = setupPreConfiguredCharFilters(plugins); Map preConfiguredTokenFilters = setupPreConfiguredTokenFilters(plugins); Map preConfiguredTokenizers = setupPreConfiguredTokenizers(plugins); + Map preConfiguredAnalyzers = setupPreBuiltAnalyzerProviderFactories(plugins); analysisRegistry = new AnalysisRegistry(environment, charFilters.getRegistry(), tokenFilters.getRegistry(), tokenizers.getRegistry(), analyzers.getRegistry(), normalizers.getRegistry(), - preConfiguredCharFilters, preConfiguredTokenFilters, preConfiguredTokenizers); + preConfiguredCharFilters, preConfiguredTokenFilters, preConfiguredTokenizers, preConfiguredAnalyzers); } HunspellService getHunspellService() { @@ -162,6 +161,16 @@ private NamedRegistry> setupTokenFilters(Li return tokenFilters; } + static Map setupPreBuiltAnalyzerProviderFactories(List plugins) { + NamedRegistry preConfiguredCharFilters = new NamedRegistry<>("pre-built analyzer"); + for (AnalysisPlugin plugin : plugins) { + for (PreBuiltAnalyzerProviderFactory factory : plugin.getPreBuiltAnalyzerProviderFactories()) { + preConfiguredCharFilters.register(factory.getName(), factory); + } + } + return unmodifiableMap(preConfiguredCharFilters.getRegistry()); + } + static Map setupPreConfiguredCharFilters(List plugins) { NamedRegistry preConfiguredCharFilters = new NamedRegistry<>("pre-configured char_filter"); @@ -232,12 +241,10 @@ private NamedRegistry>> setupAnalyzers(List NamedRegistry>> analyzers = new NamedRegistry<>("analyzer"); analyzers.register("default", StandardAnalyzerProvider::new); analyzers.register("standard", StandardAnalyzerProvider::new); - analyzers.register("standard_html_strip", StandardHtmlStripAnalyzerProvider::new); analyzers.register("simple", SimpleAnalyzerProvider::new); analyzers.register("stop", StopAnalyzerProvider::new); analyzers.register("whitespace", WhitespaceAnalyzerProvider::new); analyzers.register("keyword", KeywordAnalyzerProvider::new); - analyzers.register("pattern", PatternAnalyzerProvider::new); analyzers.register("snowball", SnowballAnalyzerProvider::new); analyzers.register("arabic", ArabicAnalyzerProvider::new); analyzers.register("armenian", ArmenianAnalyzerProvider::new); @@ -274,7 +281,6 @@ private NamedRegistry>> setupAnalyzers(List analyzers.register("swedish", SwedishAnalyzerProvider::new); analyzers.register("turkish", TurkishAnalyzerProvider::new); analyzers.register("thai", ThaiAnalyzerProvider::new); - analyzers.register("fingerprint", FingerprintAnalyzerProvider::new); analyzers.extractAndRegister(plugins, AnalysisPlugin::getAnalyzers); return analyzers; } diff --git a/server/src/main/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzers.java b/server/src/main/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzers.java index 3c286f7dd5ec5..18cc247b84493 100644 --- a/server/src/main/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzers.java +++ b/server/src/main/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzers.java @@ -61,10 +61,7 @@ import org.apache.lucene.analysis.th.ThaiAnalyzer; import org.apache.lucene.analysis.tr.TurkishAnalyzer; import org.elasticsearch.Version; -import org.elasticsearch.common.regex.Regex; -import org.elasticsearch.index.analysis.PatternAnalyzer; import org.elasticsearch.index.analysis.SnowballAnalyzer; -import org.elasticsearch.index.analysis.StandardHtmlStripAnalyzer; import org.elasticsearch.indices.analysis.PreBuiltCacheFactory.CachingStrategy; import java.util.Locale; @@ -141,22 +138,6 @@ protected Analyzer create(Version version) { } }, - PATTERN(CachingStrategy.ELASTICSEARCH) { - @Override - protected Analyzer create(Version version) { - return new PatternAnalyzer(Regex.compile("\\W+" /*PatternAnalyzer.NON_WORD_PATTERN*/, null), true, CharArraySet.EMPTY_SET); - } - }, - - STANDARD_HTML_STRIP(CachingStrategy.ELASTICSEARCH) { - @Override - protected Analyzer create(Version version) { - final Analyzer analyzer = new StandardHtmlStripAnalyzer(CharArraySet.EMPTY_SET); - analyzer.setVersion(version.luceneVersion); - return analyzer; - } - }, - ARABIC { @Override protected Analyzer create(Version version) { @@ -484,7 +465,7 @@ protected Analyzer create(Version version) { cache = PreBuiltCacheFactory.getCache(cachingStrategy); } - PreBuiltCacheFactory.PreBuiltCache getCache() { + public PreBuiltCacheFactory.PreBuiltCache getCache() { return cache; } diff --git a/server/src/main/java/org/elasticsearch/indices/analysis/PreBuiltCacheFactory.java b/server/src/main/java/org/elasticsearch/indices/analysis/PreBuiltCacheFactory.java index 8636e04f20f10..22b5a8ffaf4a5 100644 --- a/server/src/main/java/org/elasticsearch/indices/analysis/PreBuiltCacheFactory.java +++ b/server/src/main/java/org/elasticsearch/indices/analysis/PreBuiltCacheFactory.java @@ -21,6 +21,8 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; +import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -36,8 +38,12 @@ public class PreBuiltCacheFactory { public enum CachingStrategy { ONE, LUCENE, ELASTICSEARCH }; public interface PreBuiltCache { + T get(Version version); + void put(Version version, T t); + + Collection values(); } private PreBuiltCacheFactory() {} @@ -71,6 +77,11 @@ public T get(Version version) { public void put(Version version, T model) { this.model = model; } + + @Override + public Collection values() { + return Collections.singleton(model); + } } /** @@ -89,6 +100,11 @@ public T get(Version version) { public void put(Version version, T model) { mapModel.put(version, model); } + + @Override + public Collection values() { + return mapModel.values(); + } } /** @@ -107,5 +123,10 @@ public T get(Version version) { public void put(org.elasticsearch.Version version, T model) { mapModel.put(version.luceneVersion, model); } + + @Override + public Collection values() { + return mapModel.values(); + } } } diff --git a/server/src/main/java/org/elasticsearch/plugins/AnalysisPlugin.java b/server/src/main/java/org/elasticsearch/plugins/AnalysisPlugin.java index cc04ed875d996..e740fddc6ece2 100644 --- a/server/src/main/java/org/elasticsearch/plugins/AnalysisPlugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/AnalysisPlugin.java @@ -28,6 +28,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalyzerProvider; import org.elasticsearch.index.analysis.CharFilterFactory; +import org.elasticsearch.index.analysis.PreBuiltAnalyzerProviderFactory; import org.elasticsearch.index.analysis.PreConfiguredCharFilter; import org.elasticsearch.index.analysis.PreConfiguredTokenFilter; import org.elasticsearch.index.analysis.PreConfiguredTokenizer; @@ -92,6 +93,13 @@ default Map>> getA return emptyMap(); } + /** + * Override to add additional pre-configured {@link Analyzer}s. + */ + default List getPreBuiltAnalyzerProviderFactories() { + return emptyList(); + } + /** * Override to add additional pre-configured {@link CharFilter}s. */ diff --git a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java index 008b05f6a1e95..2824b8caca12d 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java @@ -121,7 +121,7 @@ public void setUp() throws Exception { index = indexSettings.getIndex(); environment = TestEnvironment.newEnvironment(settings); emptyAnalysisRegistry = new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap(), - emptyMap(), emptyMap(), emptyMap()); + emptyMap(), emptyMap(), emptyMap(), emptyMap()); threadPool = new TestThreadPool("test"); circuitBreakerService = new NoneCircuitBreakerService(); PageCacheRecycler pageCacheRecycler = new PageCacheRecycler(settings); diff --git a/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java b/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java index 9c0f2b3c7a550..36da9761b978d 100644 --- a/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java +++ b/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java @@ -41,6 +41,7 @@ import org.elasticsearch.test.VersionUtils; import java.io.IOException; +import java.util.Collections; import java.util.Map; import static java.util.Collections.emptyMap; @@ -48,6 +49,8 @@ import static java.util.Collections.singletonMap; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; public class AnalysisRegistryTests extends ESTestCase { private AnalysisRegistry emptyRegistry; @@ -58,7 +61,7 @@ private static AnalyzerProvider analyzerProvider(final String name) { private static AnalysisRegistry emptyAnalysisRegistry(Settings settings) { return new AnalysisRegistry(TestEnvironment.newEnvironment(settings), emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap(), - emptyMap(), emptyMap(), emptyMap()); + emptyMap(), emptyMap(), emptyMap(), emptyMap()); } private static IndexSettings indexSettingsOfCurrentVersion(Settings.Builder settings) { @@ -224,4 +227,16 @@ public void testCloseIndexAnalyzersMultipleTimes() throws IOException { indexAnalyzers.close(); indexAnalyzers.close(); } + + public void testEnsureCloseInvocationProperlyDelegated() throws IOException { + Settings settings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .build(); + PreBuiltAnalyzerProviderFactory mock = mock(PreBuiltAnalyzerProviderFactory.class); + AnalysisRegistry registry = new AnalysisRegistry(TestEnvironment.newEnvironment(settings), emptyMap(), emptyMap(), + emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap(), Collections.singletonMap("key", mock)); + + registry.close(); + verify(mock).close(); + } } diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java index d1f91d60e2506..717bab12ea5cb 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java @@ -19,7 +19,9 @@ package org.elasticsearch.search.fetch.subphase.highlight; import com.carrotsearch.randomizedtesting.generators.RandomPicks; - +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequestBuilder; @@ -32,6 +34,8 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.analysis.AbstractIndexAnalyzerProvider; +import org.elasticsearch.index.analysis.AnalyzerProvider; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.IdsQueryBuilder; import org.elasticsearch.index.query.MatchQueryBuilder; @@ -41,6 +45,8 @@ import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; import org.elasticsearch.index.query.functionscore.RandomScoreFunctionBuilder; +import org.elasticsearch.indices.analysis.AnalysisModule; +import org.elasticsearch.plugins.AnalysisPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchHit; @@ -63,6 +69,7 @@ import java.util.Locale; import java.util.Map; +import static java.util.Collections.singletonMap; import static org.elasticsearch.client.Requests.searchRequest; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; @@ -106,7 +113,7 @@ public class HighlighterSearchIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { - return Arrays.asList(InternalSettingsPlugin.class, MockKeywordPlugin.class); + return Arrays.asList(InternalSettingsPlugin.class, MockKeywordPlugin.class, MockWhitespacePlugin.class); } public void testHighlightingWithStoredKeyword() throws IOException { @@ -1599,8 +1606,7 @@ public void testResetTwice() throws Exception { assertAcked(prepareCreate("test") .setSettings(Settings.builder() .put(indexSettings()) - .put("analysis.analyzer.my_analyzer.type", "pattern") - .put("analysis.analyzer.my_analyzer.pattern", "\\s+") + .put("analysis.analyzer.my_analyzer.type", "mock_whitespace") .build()) .addMapping("type", "text", "type=text,analyzer=my_analyzer")); ensureGreen(); @@ -1611,7 +1617,7 @@ public void testResetTwice() throws Exception { SearchResponse response = client().prepareSearch("test") .setQuery(QueryBuilders.matchQuery("text", "test")) .highlighter(new HighlightBuilder().field("text")).execute().actionGet(); - // PatternAnalyzer will throw an exception if it is resetted twice + // Mock tokenizer will throw an exception if it is resetted twice assertHitCount(response, 1L); } @@ -2976,4 +2982,22 @@ public void testWithNormalizer() throws Exception { assertThat(field.getFragments()[0].string(), equalTo("Hello World")); } } + + public static class MockWhitespacePlugin extends Plugin implements AnalysisPlugin { + + @Override + public Map>> getAnalyzers() { + return singletonMap("mock_whitespace", (indexSettings, environment, name, settings) -> { + return new AbstractIndexAnalyzerProvider(indexSettings, name, settings) { + + MockAnalyzer instance = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false); + + @Override + public Analyzer get() { + return instance; + } + }; + }); + } + } } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java index c7c2b59caaa50..786aae0c5218c 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java @@ -67,7 +67,7 @@ public void testWatcherDisabledTests() throws Exception { // ensure index module is not called, even if watches index is tried IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(Watch.INDEX, settings); AnalysisRegistry registry = new AnalysisRegistry(TestEnvironment.newEnvironment(settings), emptyMap(), emptyMap(), emptyMap(), - emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap()); + emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap()); IndexModule indexModule = new IndexModule(indexSettings, registry); // this will trip an assertion if the watcher indexing operation listener is null (which it is) but we try to add it watcher.onIndexModule(indexModule); From e9bd92fc764894445452b362b428aca2358288aa Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 6 Jun 2018 07:44:01 +0200 Subject: [PATCH 05/26] fixed typo --- .../index/analysis/PreBuiltAnalyzerProviderFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerProviderFactory.java b/server/src/main/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerProviderFactory.java index 9317f9fb1e4ab..eedff2c349ce4 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerProviderFactory.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerProviderFactory.java @@ -113,7 +113,7 @@ public void put(Version version, AnalyzerProvider analyzerProvider) { public Collection> values() { return preBuiltAnalyzer.getCache().values().stream() // Wrap the analyzer instance in a PreBuiltAnalyzerProvider, this is what PreBuiltAnalyzerProviderFactory#close expects - // (other caches are not directly caching analyzers, but analyzer provider instead. + // (other caches are not directly caching analyzers, but analyzer provider instead) .map(analyzer -> new PreBuiltAnalyzerProvider(name, AnalyzerScope.INDICES, analyzer)) .collect(Collectors.toList()); } From 1cee45e768b1b2d37961d5504563971b3bd0991d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 6 Jun 2018 09:53:14 +0200 Subject: [PATCH 06/26] [Docs] Delete superfluous callouts (#31111) Those callout create rendering problems on the subsequent page. Closes #30532 --- docs/reference/mapping/params/properties.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/mapping/params/properties.asciidoc b/docs/reference/mapping/params/properties.asciidoc index fa74bffd9d393..e50c0b3ac777f 100644 --- a/docs/reference/mapping/params/properties.asciidoc +++ b/docs/reference/mapping/params/properties.asciidoc @@ -78,7 +78,7 @@ GET my_index/_search { "query": { "match": { - "manager.name": "Alice White" <1> + "manager.name": "Alice White" } }, "aggs": { @@ -89,7 +89,7 @@ GET my_index/_search "aggs": { "Employee Ages": { "histogram": { - "field": "employees.age", <2> + "field": "employees.age", "interval": 5 } } From 0c9d4cb4178f55b88f0da1dfc41939405c3a8d00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 6 Jun 2018 09:58:16 +0200 Subject: [PATCH 07/26] Fix expectation on parsing exception (#31108) The structure of the expected exception slightly changed, the change adapts the assertions accordingly. Closes #31104 --- .../index/rankeval/RatedRequestsTests.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java index 1be1acb1317dc..f77951dd58b6e 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java @@ -131,22 +131,19 @@ public void testXContentRoundtrip() throws IOException { } } - @AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/31104") public void testXContentParsingIsNotLenient() throws IOException { RatedRequest testItem = createTestItem(randomBoolean()); XContentType xContentType = randomFrom(XContentType.values()); BytesReference originalBytes = toShuffledXContent(testItem, xContentType, ToXContent.EMPTY_PARAMS, randomBoolean()); BytesReference withRandomFields = insertRandomFields(xContentType, originalBytes, null, random()); try (XContentParser parser = createParser(xContentType.xContent(), withRandomFields)) { - Exception exception = expectThrows(Exception.class, () -> RatedRequest.fromXContent(parser)); - if (exception instanceof XContentParseException) { - XContentParseException xcpe = (XContentParseException) exception; - assertThat(xcpe.getCause().getMessage(), containsString("unknown field")); - assertThat(xcpe.getCause().getMessage(), containsString("parser not found")); - } - if (exception instanceof XContentParseException) { + Throwable exception = expectThrows(XContentParseException.class, () -> RatedRequest.fromXContent(parser)); + if (exception.getCause() != null) { assertThat(exception.getMessage(), containsString("[request] failed to parse field")); + exception = exception.getCause(); } + assertThat(exception.getMessage(), containsString("unknown field")); + assertThat(exception.getMessage(), containsString("parser not found")); } } From d09d60858a6acdaa794201d0fca0115a4d97cb5d Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Wed, 6 Jun 2018 09:32:45 +0100 Subject: [PATCH 08/26] [DOCS] Clarify nested datatype introduction (#31055) --- docs/reference/mapping/types/nested.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/mapping/types/nested.asciidoc b/docs/reference/mapping/types/nested.asciidoc index 804fb1c65080a..238e26bf3375b 100644 --- a/docs/reference/mapping/types/nested.asciidoc +++ b/docs/reference/mapping/types/nested.asciidoc @@ -2,8 +2,8 @@ === Nested datatype The `nested` type is a specialised version of the <> datatype -that allows arrays of objects to be indexed and queried independently of each -other. +that allows arrays of objects to be indexed in a way that they can be queried +independently of each other. ==== How arrays of objects are flattened From 23d156f02306c0373c0689c0004a53a9e7c1e0ee Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Wed, 6 Jun 2018 13:39:02 +0200 Subject: [PATCH 09/26] Move RestGetSettingsAction to RestToXContentListener (#31101) --- .../admin/indices/RestGetSettingsAction.java | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetSettingsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetSettingsAction.java index d9fa50cf9410d..6dead8060425e 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetSettingsAction.java @@ -20,23 +20,18 @@ package org.elasticsearch.rest.action.admin.indices; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; -import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.rest.BaseRestHandler; -import org.elasticsearch.rest.BytesRestResponse; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.RestResponse; -import org.elasticsearch.rest.action.RestBuilderListener; +import org.elasticsearch.rest.action.RestToXContentListener; import java.io.IOException; import static org.elasticsearch.rest.RestRequest.Method.GET; -import static org.elasticsearch.rest.RestStatus.OK; public class RestGetSettingsAction extends BaseRestHandler { @@ -68,15 +63,6 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC .names(names); getSettingsRequest.local(request.paramAsBoolean("local", getSettingsRequest.local())); getSettingsRequest.masterNodeTimeout(request.paramAsTime("master_timeout", getSettingsRequest.masterNodeTimeout())); - - return channel -> client.admin().indices().getSettings(getSettingsRequest, new RestBuilderListener(channel) { - - @Override - public RestResponse buildResponse(GetSettingsResponse getSettingsResponse, XContentBuilder builder) throws Exception { - getSettingsResponse.toXContent(builder, request); - return new BytesRestResponse(OK, builder); - } - }); + return channel -> client.admin().indices().getSettings(getSettingsRequest, new RestToXContentListener<>(channel)); } - } From 0c8c6191816e464c065672d1b4b9cd91b26bb0e5 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Wed, 6 Jun 2018 14:59:04 +0300 Subject: [PATCH 10/26] Fix audit index template upgrade loop (#30779) The Index Audit trail allows the override of the template index settings with settings specified on the conf file. A bug will manifest when such conf file settings are specified for templates that need to be upgraded. The bug is an endless upgrade loop because the upgrade, although successful, is not reckoned as such by the upgrade service. --- .../metadata/TemplateUpgradeService.java | 59 +++++-- .../metadata/TemplateUpgradeServiceTests.java | 166 ++++++++++++------ .../security/audit/index/IndexAuditTrail.java | 12 +- .../audit/index/IndexAuditTrailTests.java | 25 +++ 4 files changed, 186 insertions(+), 76 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/TemplateUpgradeService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/TemplateUpgradeService.java index 3bdc949752afa..024cc44dd6a8e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/TemplateUpgradeService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/TemplateUpgradeService.java @@ -19,7 +19,6 @@ package org.elasticsearch.cluster.metadata; -import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.Version; @@ -32,8 +31,6 @@ import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateListener; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.ImmutableOpenMap; @@ -57,6 +54,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.UnaryOperator; @@ -74,7 +72,7 @@ public class TemplateUpgradeService extends AbstractComponent implements Cluster public final Client client; - private final AtomicInteger updatesInProgress = new AtomicInteger(); + final AtomicInteger upgradesInProgress = new AtomicInteger(); private ImmutableOpenMap lastTemplateMetaData; @@ -103,8 +101,8 @@ public void clusterChanged(ClusterChangedEvent event) { return; } - if (updatesInProgress.get() > 0) { - // we are already running some updates - skip this cluster state update + if (upgradesInProgress.get() > 0) { + // we are already running some upgrades - skip this cluster state update return; } @@ -124,7 +122,7 @@ public void clusterChanged(ClusterChangedEvent event) { lastTemplateMetaData = templates; Optional, Set>> changes = calculateTemplateChanges(templates); if (changes.isPresent()) { - if (updatesInProgress.compareAndSet(0, changes.get().v1().size() + changes.get().v2().size())) { + if (upgradesInProgress.compareAndSet(0, changes.get().v1().size() + changes.get().v2().size() + 1)) { logger.info("Starting template upgrade to version {}, {} templates will be updated and {} will be removed", Version.CURRENT, changes.get().v1().size(), @@ -133,13 +131,14 @@ public void clusterChanged(ClusterChangedEvent event) { final ThreadContext threadContext = threadPool.getThreadContext(); try (ThreadContext.StoredContext ignore = threadContext.stashContext()) { threadContext.markAsSystemContext(); - threadPool.generic().execute(() -> updateTemplates(changes.get().v1(), changes.get().v2())); + threadPool.generic().execute(() -> upgradeTemplates(changes.get().v1(), changes.get().v2())); } } } } - void updateTemplates(Map changes, Set deletions) { + void upgradeTemplates(Map changes, Set deletions) { + final AtomicBoolean anyUpgradeFailed = new AtomicBoolean(false); if (threadPool.getThreadContext().isSystemContext() == false) { throw new IllegalStateException("template updates from the template upgrade service should always happen in a system context"); } @@ -151,20 +150,18 @@ void updateTemplates(Map changes, Set deletions) client.admin().indices().putTemplate(request, new ActionListener() { @Override public void onResponse(PutIndexTemplateResponse response) { - if (updatesInProgress.decrementAndGet() == 0) { - logger.info("Finished upgrading templates to version {}", Version.CURRENT); - } if (response.isAcknowledged() == false) { + anyUpgradeFailed.set(true); logger.warn("Error updating template [{}], request was not acknowledged", change.getKey()); } + tryFinishUpgrade(anyUpgradeFailed); } @Override public void onFailure(Exception e) { - if (updatesInProgress.decrementAndGet() == 0) { - logger.info("Templates were upgraded to version {}", Version.CURRENT); - } + anyUpgradeFailed.set(true); logger.warn(new ParameterizedMessage("Error updating template [{}]", change.getKey()), e); + tryFinishUpgrade(anyUpgradeFailed); } }); } @@ -175,27 +172,51 @@ public void onFailure(Exception e) { client.admin().indices().deleteTemplate(request, new ActionListener() { @Override public void onResponse(DeleteIndexTemplateResponse response) { - updatesInProgress.decrementAndGet(); if (response.isAcknowledged() == false) { + anyUpgradeFailed.set(true); logger.warn("Error deleting template [{}], request was not acknowledged", template); } + tryFinishUpgrade(anyUpgradeFailed); } @Override public void onFailure(Exception e) { - updatesInProgress.decrementAndGet(); + anyUpgradeFailed.set(true); if (e instanceof IndexTemplateMissingException == false) { // we might attempt to delete the same template from different nodes - so that's ok if template doesn't exist // otherwise we need to warn logger.warn(new ParameterizedMessage("Error deleting template [{}]", template), e); } + tryFinishUpgrade(anyUpgradeFailed); } }); } } - int getUpdatesInProgress() { - return updatesInProgress.get(); + void tryFinishUpgrade(AtomicBoolean anyUpgradeFailed) { + assert upgradesInProgress.get() > 0; + if (upgradesInProgress.decrementAndGet() == 1) { + try { + // this is the last upgrade, the templates should now be in the desired state + if (anyUpgradeFailed.get()) { + logger.info("Templates were partially upgraded to version {}", Version.CURRENT); + } else { + logger.info("Templates were upgraded successfuly to version {}", Version.CURRENT); + } + // Check upgraders are satisfied after the update completed. If they still + // report that changes are required, this might indicate a bug or that something + // else tinkering with the templates during the upgrade. + final ImmutableOpenMap upgradedTemplates = + clusterService.state().getMetaData().getTemplates(); + final boolean changesRequired = calculateTemplateChanges(upgradedTemplates).isPresent(); + if (changesRequired) { + logger.warn("Templates are still reported as out of date after the upgrade. The template upgrade will be retried."); + } + } finally { + final int noMoreUpgrades = upgradesInProgress.decrementAndGet(); + assert noMoreUpgrades == 0; + } + } } Optional, Set>> calculateTemplateChanges( diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceTests.java index e46f2e06fe16d..9ad4aeb69fb49 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceTests.java @@ -35,12 +35,16 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; +import org.junit.After; +import org.junit.Before; import java.util.ArrayList; import java.util.Arrays; @@ -52,13 +56,16 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import java.util.stream.IntStream; import static java.util.Collections.emptyMap; +import static org.elasticsearch.test.ClusterServiceUtils.createClusterService; +import static org.elasticsearch.test.ClusterServiceUtils.setState; import static org.elasticsearch.test.VersionUtils.randomVersion; import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.CoreMatchers.startsWith; @@ -75,8 +82,20 @@ public class TemplateUpgradeServiceTests extends ESTestCase { - private final ClusterService clusterService = new ClusterService(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, - ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null, Collections.emptyMap()); + private ThreadPool threadPool; + private ClusterService clusterService; + + @Before + public void setUpTest() throws Exception { + threadPool = new TestThreadPool("TemplateUpgradeServiceTests"); + clusterService = createClusterService(threadPool); + } + + @After + public void tearDownTest() throws Exception { + threadPool.shutdownNow(); + clusterService.close(); + } public void testCalculateChangesAddChangeAndDelete() { @@ -90,7 +109,7 @@ public void testCalculateChangesAddChangeAndDelete() { IndexTemplateMetaData.builder("changed_test_template").patterns(randomIndexPatterns()).build() ); - TemplateUpgradeService service = new TemplateUpgradeService(Settings.EMPTY, null, clusterService, null, + final TemplateUpgradeService service = new TemplateUpgradeService(Settings.EMPTY, null, clusterService, threadPool, Arrays.asList( templates -> { if (shouldAdd) { @@ -190,18 +209,18 @@ public void testUpdateTemplates() { additions.put("add_template_" + i, new BytesArray("{\"index_patterns\" : \"*\", \"order\" : " + i + "}")); } - ThreadPool threadPool = mock(ThreadPool.class); - ThreadContext threadContext = new ThreadContext(Settings.EMPTY); - when(threadPool.getThreadContext()).thenReturn(threadContext); - TemplateUpgradeService service = new TemplateUpgradeService(Settings.EMPTY, mockClient, clusterService, threadPool, + final TemplateUpgradeService service = new TemplateUpgradeService(Settings.EMPTY, mockClient, clusterService, threadPool, Collections.emptyList()); - IllegalStateException ise = expectThrows(IllegalStateException.class, () -> service.updateTemplates(additions, deletions)); + IllegalStateException ise = expectThrows(IllegalStateException.class, () -> service.upgradeTemplates(additions, deletions)); assertThat(ise.getMessage(), containsString("template upgrade service should always happen in a system context")); - threadContext.markAsSystemContext(); - service.updateTemplates(additions, deletions); - int updatesInProgress = service.getUpdatesInProgress(); + service.upgradesInProgress.set(additionsCount + deletionsCount + 2); // +2 to skip tryFinishUpgrade + final ThreadContext threadContext = threadPool.getThreadContext(); + try (ThreadContext.StoredContext ignore = threadContext.stashContext()) { + threadContext.markAsSystemContext(); + service.upgradeTemplates(additions, deletions); + } assertThat(putTemplateListeners, hasSize(additionsCount)); assertThat(deleteTemplateListeners, hasSize(deletionsCount)); @@ -218,30 +237,34 @@ public void testUpdateTemplates() { for (int i = 0; i < deletionsCount; i++) { if (randomBoolean()) { - int prevUpdatesInProgress = service.getUpdatesInProgress(); + int prevUpdatesInProgress = service.upgradesInProgress.get(); deleteTemplateListeners.get(i).onFailure(new RuntimeException("test - ignore")); - assertThat(prevUpdatesInProgress - service.getUpdatesInProgress(), equalTo(1)); + assertThat(prevUpdatesInProgress - service.upgradesInProgress.get(), equalTo(1)); } else { - int prevUpdatesInProgress = service.getUpdatesInProgress(); + int prevUpdatesInProgress = service.upgradesInProgress.get(); deleteTemplateListeners.get(i).onResponse(new DeleteIndexTemplateResponse(randomBoolean()) { }); - assertThat(prevUpdatesInProgress - service.getUpdatesInProgress(), equalTo(1)); + assertThat(prevUpdatesInProgress - service.upgradesInProgress.get(), equalTo(1)); } } - assertThat(updatesInProgress - service.getUpdatesInProgress(), equalTo(additionsCount + deletionsCount)); + // tryFinishUpgrade was skipped + assertThat(service.upgradesInProgress.get(), equalTo(2)); } private static final Set MASTER_DATA_ROLES = Collections.unmodifiableSet(EnumSet.of(DiscoveryNode.Role.MASTER, DiscoveryNode.Role.DATA)); @SuppressWarnings("unchecked") - public void testClusterStateUpdate() { + public void testClusterStateUpdate() throws InterruptedException { - AtomicReference> addedListener = new AtomicReference<>(); - AtomicReference> changedListener = new AtomicReference<>(); - AtomicReference> removedListener = new AtomicReference<>(); - AtomicInteger updateInvocation = new AtomicInteger(); + final AtomicReference> addedListener = new AtomicReference<>(); + final AtomicReference> changedListener = new AtomicReference<>(); + final AtomicReference> removedListener = new AtomicReference<>(); + final Semaphore updateInvocation = new Semaphore(0); + final Semaphore calculateInvocation = new Semaphore(0); + final Semaphore changedInvocation = new Semaphore(0); + final Semaphore finishInvocation = new Semaphore(0); MetaData metaData = randomMetaData( IndexTemplateMetaData.builder("user_template").patterns(randomIndexPatterns()).build(), @@ -249,21 +272,6 @@ public void testClusterStateUpdate() { IndexTemplateMetaData.builder("changed_test_template").patterns(randomIndexPatterns()).build() ); - ThreadPool threadPool = mock(ThreadPool.class); - ThreadContext threadContext = new ThreadContext(Settings.EMPTY); - when(threadPool.getThreadContext()).thenReturn(threadContext); - ExecutorService executorService = mock(ExecutorService.class); - when(threadPool.generic()).thenReturn(executorService); - doAnswer(invocation -> { - Object[] args = invocation.getArguments(); - assert args.length == 1; - assertTrue(threadContext.isSystemContext()); - Runnable runnable = (Runnable) args[0]; - runnable.run(); - updateInvocation.incrementAndGet(); - return null; - }).when(executorService).execute(any(Runnable.class)); - Client mockClient = mock(Client.class); AdminClient mockAdminClient = mock(AdminClient.class); IndicesAdminClient mockIndicesAdminClient = mock(IndicesAdminClient.class); @@ -293,7 +301,7 @@ public void testClusterStateUpdate() { return null; }).when(mockIndicesAdminClient).deleteTemplate(any(DeleteIndexTemplateRequest.class), any(ActionListener.class)); - TemplateUpgradeService service = new TemplateUpgradeService(Settings.EMPTY, mockClient, clusterService, threadPool, + final TemplateUpgradeService service = new TemplateUpgradeService(Settings.EMPTY, mockClient, clusterService, threadPool, Arrays.asList( templates -> { assertNull(templates.put("added_test_template", IndexTemplateMetaData.builder("added_test_template") @@ -309,26 +317,63 @@ public void testClusterStateUpdate() { .patterns(Collections.singletonList("*")).order(10).build())); return templates; } - )); + )) { + + @Override + void tryFinishUpgrade(AtomicBoolean anyUpgradeFailed) { + super.tryFinishUpgrade(anyUpgradeFailed); + finishInvocation.release(); + } + + @Override + void upgradeTemplates(Map changes, Set deletions) { + super.upgradeTemplates(changes, deletions); + updateInvocation.release(); + } + + @Override + Optional, Set>> + calculateTemplateChanges(ImmutableOpenMap templates) { + final Optional, Set>> ans = super.calculateTemplateChanges(templates); + calculateInvocation.release(); + return ans; + } + + @Override + public void clusterChanged(ClusterChangedEvent event) { + super.clusterChanged(event); + changedInvocation.release(); + } + }; ClusterState prevState = ClusterState.EMPTY_STATE; ClusterState state = ClusterState.builder(prevState).nodes(DiscoveryNodes.builder() .add(new DiscoveryNode("node1", "node1", buildNewFakeTransportAddress(), emptyMap(), MASTER_DATA_ROLES, Version.CURRENT) ).localNodeId("node1").masterNodeId("node1").build() ).metaData(metaData).build(); - service.clusterChanged(new ClusterChangedEvent("test", state, prevState)); - - assertThat(updateInvocation.get(), equalTo(1)); + setState(clusterService, state); + + changedInvocation.acquire(); + assertThat(changedInvocation.availablePermits(), equalTo(0)); + calculateInvocation.acquire(); + assertThat(calculateInvocation.availablePermits(), equalTo(0)); + updateInvocation.acquire(); + assertThat(updateInvocation.availablePermits(), equalTo(0)); + assertThat(finishInvocation.availablePermits(), equalTo(0)); assertThat(addedListener.get(), notNullValue()); assertThat(changedListener.get(), notNullValue()); assertThat(removedListener.get(), notNullValue()); prevState = state; state = ClusterState.builder(prevState).metaData(MetaData.builder(state.metaData()).removeTemplate("user_template")).build(); - service.clusterChanged(new ClusterChangedEvent("test 2", state, prevState)); + setState(clusterService, state); // Make sure that update wasn't invoked since we are still running - assertThat(updateInvocation.get(), equalTo(1)); + changedInvocation.acquire(); + assertThat(changedInvocation.availablePermits(), equalTo(0)); + assertThat(calculateInvocation.availablePermits(), equalTo(0)); + assertThat(updateInvocation.availablePermits(), equalTo(0)); + assertThat(finishInvocation.availablePermits(), equalTo(0)); addedListener.getAndSet(null).onResponse(new PutIndexTemplateResponse(true) { }); @@ -337,19 +382,40 @@ public void testClusterStateUpdate() { removedListener.getAndSet(null).onResponse(new DeleteIndexTemplateResponse(true) { }); - service.clusterChanged(new ClusterChangedEvent("test 3", state, prevState)); + // 3 upgrades should be completed, in addition to the final calculate + finishInvocation.acquire(3); + assertThat(finishInvocation.availablePermits(), equalTo(0)); + calculateInvocation.acquire(); + assertThat(calculateInvocation.availablePermits(), equalTo(0)); + + setState(clusterService, state); // Make sure that update was called this time since we are no longer running - assertThat(updateInvocation.get(), equalTo(2)); + changedInvocation.acquire(); + assertThat(changedInvocation.availablePermits(), equalTo(0)); + calculateInvocation.acquire(); + assertThat(calculateInvocation.availablePermits(), equalTo(0)); + updateInvocation.acquire(); + assertThat(updateInvocation.availablePermits(), equalTo(0)); + assertThat(finishInvocation.availablePermits(), equalTo(0)); addedListener.getAndSet(null).onFailure(new RuntimeException("test - ignore")); changedListener.getAndSet(null).onFailure(new RuntimeException("test - ignore")); removedListener.getAndSet(null).onFailure(new RuntimeException("test - ignore")); - service.clusterChanged(new ClusterChangedEvent("test 3", state, prevState)); + finishInvocation.acquire(3); + assertThat(finishInvocation.availablePermits(), equalTo(0)); + calculateInvocation.acquire(); + assertThat(calculateInvocation.availablePermits(), equalTo(0)); + + setState(clusterService, state); // Make sure that update wasn't called this time since the index template metadata didn't change - assertThat(updateInvocation.get(), equalTo(2)); + changedInvocation.acquire(); + assertThat(changedInvocation.availablePermits(), equalTo(0)); + assertThat(calculateInvocation.availablePermits(), equalTo(0)); + assertThat(updateInvocation.availablePermits(), equalTo(0)); + assertThat(finishInvocation.availablePermits(), equalTo(0)); } private static final int NODE_TEST_ITERS = 100; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java index db7475a89727f..1976722d65f36 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java @@ -992,24 +992,22 @@ Client initializeRemoteClient(Settings settings, Logger logger) { } public static Settings customAuditIndexSettings(Settings nodeSettings, Logger logger) { - Settings newSettings = Settings.builder() + final Settings newSettings = Settings.builder() .put(INDEX_SETTINGS.get(nodeSettings), false) + .normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX) .build(); if (newSettings.names().isEmpty()) { return Settings.EMPTY; } - // Filter out forbidden settings: - Settings.Builder builder = Settings.builder(); - builder.put(newSettings.filter(k -> { - String name = "index." + k; + // Filter out forbidden setting + return Settings.builder().put(newSettings.filter(name -> { if (FORBIDDEN_INDEX_SETTING.equals(name)) { logger.warn("overriding the default [{}} setting is forbidden. ignoring...", name); return false; } return true; - })); - return builder.build(); + })).build(); } private void putTemplate(Settings customSettings, Consumer consumer) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailTests.java index dab3d023f65d3..bc27e4cde40d4 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailTests.java @@ -6,10 +6,14 @@ package org.elasticsearch.xpack.security.audit.index; import org.apache.lucene.util.SetOnce; +import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequest; +import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.Client; @@ -17,6 +21,8 @@ import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; @@ -29,6 +35,7 @@ import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.plugins.MetaDataUpgrader; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.search.SearchHit; @@ -70,7 +77,9 @@ import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.concurrent.ExecutionException; import java.util.function.Function; +import static java.util.Collections.emptyMap; import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE; import static org.elasticsearch.test.InternalTestCluster.clusterName; @@ -85,6 +94,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.hasSize; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -360,6 +370,21 @@ List> remoteTransportClientPlugins() { auditor.start(); } + public void testIndexTemplateUpgrader() throws Exception { + final MetaDataUpgrader metaDataUpgrader = internalCluster().getInstance(MetaDataUpgrader.class); + final Map updatedTemplates = metaDataUpgrader.indexTemplateMetaDataUpgraders.apply(emptyMap()); + final IndexTemplateMetaData indexAuditTrailTemplate = updatedTemplates.get(IndexAuditTrail.INDEX_TEMPLATE_NAME); + assertThat(indexAuditTrailTemplate, notNullValue()); + // test custom index settings override template + assertThat(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.get(indexAuditTrailTemplate.settings()), is(numReplicas)); + assertThat(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(indexAuditTrailTemplate.settings()), is(numShards)); + // test upgrade template and installed template are equal + final GetIndexTemplatesRequest request = new GetIndexTemplatesRequest(IndexAuditTrail.INDEX_TEMPLATE_NAME); + final GetIndexTemplatesResponse response = client().admin().indices().getTemplates(request).get(); + assertThat(response.getIndexTemplates(), hasSize(1)); + assertThat(indexAuditTrailTemplate, is(response.getIndexTemplates().get(0))); + } + public void testProcessorsSetting() { final boolean explicitProcessors = randomBoolean(); final int processors; From 3767bdc98d974282e4b759fe9a072b0d188fe4f2 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Wed, 6 Jun 2018 13:21:16 +0100 Subject: [PATCH 11/26] [ML][DOCS] Add example of top N derivative aggregation (#31109) Add example of top N derivative aggregation to the ML datafeed docs --- x-pack/docs/build.gradle | 53 +++++++++++- x-pack/docs/en/ml/aggregations.asciidoc | 110 ++++++++++++++++++------ 2 files changed, 135 insertions(+), 28 deletions(-) diff --git a/x-pack/docs/build.gradle b/x-pack/docs/build.gradle index 17e0f2b70fd2a..3d799c8d0b548 100644 --- a/x-pack/docs/build.gradle +++ b/x-pack/docs/build.gradle @@ -16,7 +16,6 @@ buildRestTests.expectedUnconvertedCandidates = [ 'en/ml/functions/rare.asciidoc', 'en/ml/functions/sum.asciidoc', 'en/ml/functions/time.asciidoc', - 'en/ml/aggregations.asciidoc', 'en/ml/customurl.asciidoc', 'en/monitoring/indices.asciidoc', 'en/rest-api/security/ssl.asciidoc', @@ -281,6 +280,58 @@ setups['library'] = ''' {"name": "The Moon is a Harsh Mistress", "author": "Robert A. Heinlein", "release_date": "1966-04-01", "page_count": 288} ''' +setups['farequote_index'] = ''' + - do: + indices.create: + index: farequote + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + metric: + properties: + time: + type: date + responsetime: + type: float + airline: + type: keyword +''' +setups['farequote_data'] = setups['farequote_index'] + ''' + - do: + bulk: + index: farequote + type: metric + refresh: true + body: | + {"index": {"_id":"1"}} + {"airline":"JZA","responsetime":990.4628,"time":"2016-02-07T00:00:00+0000"} + {"index": {"_id":"2"}} + {"airline":"JBU","responsetime":877.5927,"time":"2016-02-07T00:00:00+0000"} + {"index": {"_id":"3"}} + {"airline":"KLM","responsetime":1355.4812,"time":"2016-02-07T00:00:00+0000"} +''' +setups['farequote_job'] = setups['farequote_data'] + ''' + - do: + xpack.ml.put_job: + job_id: "farequote" + body: > + { + "analysis_config": { + "bucket_span": "60m", + "detectors": [{ + "function": "mean", + "field_name": "responsetime", + "by_field_name": "airline" + }], + "summary_count_field_name": "doc_count" + }, + "data_description": { + "time_field": "time" + } + } +''' setups['server_metrics_index'] = ''' - do: indices.create: diff --git a/x-pack/docs/en/ml/aggregations.asciidoc b/x-pack/docs/en/ml/aggregations.asciidoc index cc98a45d11e50..f3b8e6b3e34d6 100644 --- a/x-pack/docs/en/ml/aggregations.asciidoc +++ b/x-pack/docs/en/ml/aggregations.asciidoc @@ -11,11 +11,12 @@ aggregated data into {xpackml} instead of raw results, which reduces the volume of data that must be considered while detecting anomalies. There are some limitations to using aggregations in {dfeeds}, however. -Your aggregation must include a buckets aggregation, which in turn must contain -a date histogram aggregation. This requirement ensures that the aggregated -data is a time series. If you use a terms aggregation and the cardinality of a -term is high, then the aggregation might not be effective and you might want -to just use the default search and scroll behavior. +Your aggregation must include a `date_histogram` aggregation, which in turn must +contain a `max` aggregation on the time field. This requirement ensures that the +aggregated data is a time series and the timestamp of each bucket is the time +of the last record in the bucket. If you use a terms aggregation and the +cardinality of a term is high, then the aggregation might not be effective and +you might want to just use the default search and scroll behavior. When you create or update a job, you can include the names of aggregations, for example: @@ -27,9 +28,9 @@ PUT _xpack/ml/anomaly_detectors/farequote "analysis_config": { "bucket_span": "60m", "detectors": [{ - "function":"mean", - "field_name":"responsetime", - "by_field_name":"airline" + "function": "mean", + "field_name": "responsetime", + "by_field_name": "airline" }], "summary_count_field_name": "doc_count" }, @@ -38,6 +39,8 @@ PUT _xpack/ml/anomaly_detectors/farequote } } ---------------------------------- +// CONSOLE +// TEST[setup:farequote_data] In this example, the `airline`, `responsetime`, and `time` fields are aggregations. @@ -85,7 +88,8 @@ PUT _xpack/ml/datafeeds/datafeed-farequote } } ---------------------------------- - +// CONSOLE +// TEST[setup:farequote_job] In this example, the aggregations have names that match the fields that they operate on. That is to say, the `max` aggregation is named `time` and its @@ -100,35 +104,86 @@ For all other aggregations, if the aggregation name doesn't match the field name there are limitations in the drill-down functionality within the {ml} page in {kib}. -When you define an aggregation in a {dfeed}, it must have the following form: +{dfeeds} support complex nested aggregations, this example uses the `derivative` +pipeline aggregation to find the 1st order derivative of the counter +`system.network.out.bytes` for each value of the field `beat.name`. [source,js] ---------------------------------- -"aggregations" : { - "buckets" : { - "date_histogram" : { - "time_zone": "UTC", ... +"aggregations": { + "beat.name": { + "terms": { + "field": "beat.name" }, "aggregations": { - "": { - "max": { - "field":"" + "buckets": { + "date_histogram": { + "field": "@timestamp", + "interval": "5m" + }, + "aggregations": { + "@timestamp": { + "max": { + "field": "@timestamp" + } + }, + "bytes_out_average": { + "avg": { + "field": "system.network.out.bytes" + } + }, + "bytes_out_derivative": { + "derivative": { + "buckets_path": "bytes_out_average" + } + } } } - [,"": { - "terms":{... + } + } +} +---------------------------------- +// NOTCONSOLE + +When you define an aggregation in a {dfeed}, it must have the following form: + +[source,js] +---------------------------------- +"aggregations": { + ["bucketing_aggregation": { + "bucket_agg": { + ... + }, + "aggregations": {] + "data_histogram_aggregation": { + "date_histogram": { + "field": "time", + }, + "aggregations": { + "timestamp": { + "max": { + "field": "time" + } + }, + [,"": { + "terms":{... + } + [,"aggregations" : { + []+ + } ] + }] } - [,"aggregations" : { - []+ - } ] - }] - } - } + } + } + } } ---------------------------------- +// NOTCONSOLE -You must specify `buckets` as the aggregation name and `date_histogram` as the -aggregation type. For more information, see +The top level aggregation must be either a {ref}/search-aggregations-bucket.html[Bucket Aggregation] +containing as single sub-aggregation that is a `date_histogram` or the top level aggregation +is the required `date_histogram`. There must be exactly 1 `date_histogram` aggregation. +For more information, see {ref}/search-aggregations-bucket-datehistogram-aggregation.html[Date Histogram Aggregation]. NOTE: The `time_zone` parameter in the date histogram aggregation must be set to `UTC`, @@ -163,6 +218,7 @@ GET .../_search { } } -------------------------------------------------- +// NOTCONSOLE By default, {es} limits the maximum number of terms returned to 10000. For high cardinality fields, the query might not run. It might return errors related to From a9af5ca6387cc0fa6d4e1425f57034d94713a328 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Wed, 6 Jun 2018 14:32:37 +0200 Subject: [PATCH 12/26] [TEST] Reenable UnicastZenPingTests.testSimplePings --- .../discovery/zen/UnicastZenPingTests.java | 62 ++++++++++++------- 1 file changed, 40 insertions(+), 22 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java b/server/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java index f209f771ab089..f71ffe28b50f6 100644 --- a/server/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java +++ b/server/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.discovery.zen; import org.apache.logging.log4j.Logger; +import org.apache.lucene.util.Constants; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterName; @@ -94,6 +95,7 @@ import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasSize; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; @@ -137,7 +139,6 @@ public void tearDown() throws Exception { private static final UnicastHostsProvider EMPTY_HOSTS_PROVIDER = Collections::emptyList; - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/28685") public void testSimplePings() throws IOException, InterruptedException, ExecutionException { // use ephemeral ports final Settings settings = Settings.builder().put("cluster.name", "test").put(TcpTransport.PORT.getKey(), 0).build(); @@ -233,9 +234,9 @@ protected Version getVersion() { ZenPing.PingResponse ping = pingResponses.iterator().next(); assertThat(ping.node().getId(), equalTo("UZP_B")); assertThat(ping.getClusterStateVersion(), equalTo(state.version())); - assertPingCount(handleA, handleB, 3); - assertPingCount(handleA, handleC, 0); // mismatch, shouldn't ping - assertPingCount(handleA, handleD, 0); // mismatch, shouldn't ping + assertPings(handleA, handleB); + assertNoPings(handleA, handleC); // mismatch, shouldn't ping + assertNoPings(handleA, handleD); // mismatch, shouldn't ping // ping again, this time from B, logger.info("ping from UZP_B"); @@ -244,23 +245,23 @@ protected Version getVersion() { ping = pingResponses.iterator().next(); assertThat(ping.node().getId(), equalTo("UZP_A")); assertThat(ping.getClusterStateVersion(), equalTo(ElectMasterService.MasterCandidate.UNRECOVERED_CLUSTER_VERSION)); - assertPingCount(handleB, handleA, 3); - assertPingCount(handleB, handleC, 0); // mismatch, shouldn't ping - assertPingCount(handleB, handleD, 0); // mismatch, shouldn't ping + assertPings(handleB, handleA); + assertNoPings(handleB, handleC); // mismatch, shouldn't ping + assertNoPings(handleB, handleD); // mismatch, shouldn't ping logger.info("ping from UZP_C"); pingResponses = zenPingC.pingAndWait().toList(); assertThat(pingResponses.size(), equalTo(1)); - assertPingCount(handleC, handleA, 0); - assertPingCount(handleC, handleB, 0); - assertPingCount(handleC, handleD, 3); + assertNoPings(handleC, handleA); + assertNoPings(handleC, handleB); + assertPings(handleC, handleD); logger.info("ping from UZP_D"); pingResponses = zenPingD.pingAndWait().toList(); assertThat(pingResponses.size(), equalTo(1)); - assertPingCount(handleD, handleA, 0); - assertPingCount(handleD, handleB, 0); - assertPingCount(handleD, handleC, 3); + assertNoPings(handleD, handleA); + assertNoPings(handleD, handleB); + assertPings(handleD, handleC); zenPingC.close(); handleD.counters.clear(); @@ -268,9 +269,9 @@ protected Version getVersion() { pingResponses = zenPingD.pingAndWait().toList(); // check that node does not respond to pings anymore after the ping service has been closed assertThat(pingResponses.size(), equalTo(0)); - assertPingCount(handleD, handleA, 0); - assertPingCount(handleD, handleB, 0); - assertPingCount(handleD, handleC, 3); + assertNoPings(handleD, handleA); + assertNoPings(handleD, handleB); + assertPings(handleD, handleC); } public void testUnknownHostNotCached() throws ExecutionException, InterruptedException { @@ -353,8 +354,8 @@ public TransportAddress[] addressesFromString(String address, int perAddressLimi ZenPing.PingResponse ping = pingResponses.iterator().next(); assertThat(ping.node().getId(), equalTo("UZP_C")); assertThat(ping.getClusterStateVersion(), equalTo(state.version())); - assertPingCount(handleA, handleB, 0); - assertPingCount(handleA, handleC, 3); + assertNoPings(handleA, handleB); + assertPings(handleA, handleC); assertNull(handleA.counters.get(handleB.address)); } @@ -377,8 +378,8 @@ public TransportAddress[] addressesFromString(String address, int perAddressLimi assertThat(secondPingResponses.size(), equalTo(2)); final Set ids = new HashSet<>(secondPingResponses.stream().map(p -> p.node().getId()).collect(Collectors.toList())); assertThat(ids, equalTo(new HashSet<>(Arrays.asList("UZP_B", "UZP_C")))); - assertPingCount(handleA, handleB, 3); - assertPingCount(handleA, handleC, 3); + assertPings(handleA, handleB); + assertPings(handleA, handleC); } } @@ -745,13 +746,30 @@ public BoundTransportAddress boundAddress() { verify(logger).warn(eq("failed to resolve host [127.0.0.1:9300:9300]"), Matchers.any(ExecutionException.class)); } - private void assertPingCount(final NetworkHandle fromNode, final NetworkHandle toNode, int expectedCount) { + private void assertNoPings(final NetworkHandle fromNode, final NetworkHandle toNode) { final AtomicInteger counter = fromNode.counters.getOrDefault(toNode.address, new AtomicInteger()); final String onNodeName = fromNode.node.getName(); assertNotNull("handle for [" + onNodeName + "] has no 'expected' counter", counter); final String forNodeName = toNode.node.getName(); assertThat("node [" + onNodeName + "] ping count to [" + forNodeName + "] is unexpected", - counter.get(), equalTo(expectedCount)); + counter.get(), equalTo(0)); + } + + private void assertPings(final NetworkHandle fromNode, final NetworkHandle toNode) { + final AtomicInteger counter = fromNode.counters.getOrDefault(toNode.address, new AtomicInteger()); + final String onNodeName = fromNode.node.getName(); + assertNotNull("handle for [" + onNodeName + "] has no 'expected' counter", counter); + final String forNodeName = toNode.node.getName(); + if (Constants.WINDOWS) { + // Some of the ping attempts seem to sporadically fail on Windows (see https://github.com/elastic/elasticsearch/issues/28685) + // Anyhow, the point of the test is not to assert the exact number of pings, but to check if pinging has taken place or not + assertThat("node [" + onNodeName + "] ping count to [" + forNodeName + "] is unexpected", + counter.get(), greaterThan(0)); + } else { + assertThat("node [" + onNodeName + "] ping count to [" + forNodeName + "] is unexpected", + counter.get(), equalTo(3)); + } + } private NetworkHandle startServices( From f4a412fe21b2ae595306e6bd261f5fc4d6374388 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Wed, 6 Jun 2018 16:13:02 +0200 Subject: [PATCH 13/26] Remove RestGetAllMappingsAction (#31129) We currently have a specific REST action to retrieve all indices and types mappings, which used internally the get index API. This doesn't seem to be required anymore though as the existing RestGetMappingAction could as well take the requests with no indices and types specified. This commit removes the RestGetAllMappingsAction in favour of using RestGetMappingAction also for requests that don't specify indices nor types. --- .../elasticsearch/action/ActionModule.java | 12 +- .../indices/RestGetAllMappingsAction.java | 109 ------------------ .../admin/indices/RestGetMappingAction.java | 5 +- 3 files changed, 7 insertions(+), 119 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAllMappingsAction.java diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java index fa4d751a54aed..235effdcf4492 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java @@ -206,6 +206,10 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.persistent.CompletionPersistentTaskAction; +import org.elasticsearch.persistent.RemovePersistentTaskAction; +import org.elasticsearch.persistent.StartPersistentTaskAction; +import org.elasticsearch.persistent.UpdatePersistentTaskStatusAction; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.ActionPlugin.ActionHandler; import org.elasticsearch.rest.RestController; @@ -241,7 +245,6 @@ import org.elasticsearch.rest.action.admin.cluster.RestRestoreSnapshotAction; import org.elasticsearch.rest.action.admin.cluster.RestSnapshotsStatusAction; import org.elasticsearch.rest.action.admin.cluster.RestVerifyRepositoryAction; -import org.elasticsearch.rest.action.admin.indices.RestResizeHandler; import org.elasticsearch.rest.action.admin.indices.RestAnalyzeAction; import org.elasticsearch.rest.action.admin.indices.RestClearIndicesCacheAction; import org.elasticsearch.rest.action.admin.indices.RestCloseIndexAction; @@ -252,7 +255,6 @@ import org.elasticsearch.rest.action.admin.indices.RestForceMergeAction; import org.elasticsearch.rest.action.admin.indices.RestGetAliasesAction; import org.elasticsearch.rest.action.admin.indices.RestGetAllAliasesAction; -import org.elasticsearch.rest.action.admin.indices.RestGetAllMappingsAction; import org.elasticsearch.rest.action.admin.indices.RestGetFieldMappingAction; import org.elasticsearch.rest.action.admin.indices.RestGetIndexTemplateAction; import org.elasticsearch.rest.action.admin.indices.RestGetIndicesAction; @@ -269,6 +271,7 @@ import org.elasticsearch.rest.action.admin.indices.RestPutMappingAction; import org.elasticsearch.rest.action.admin.indices.RestRecoveryAction; import org.elasticsearch.rest.action.admin.indices.RestRefreshAction; +import org.elasticsearch.rest.action.admin.indices.RestResizeHandler; import org.elasticsearch.rest.action.admin.indices.RestRolloverIndexAction; import org.elasticsearch.rest.action.admin.indices.RestSyncedFlushAction; import org.elasticsearch.rest.action.admin.indices.RestUpdateSettingsAction; @@ -313,10 +316,6 @@ import org.elasticsearch.rest.action.search.RestSearchScrollAction; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.usage.UsageService; -import org.elasticsearch.persistent.CompletionPersistentTaskAction; -import org.elasticsearch.persistent.RemovePersistentTaskAction; -import org.elasticsearch.persistent.StartPersistentTaskAction; -import org.elasticsearch.persistent.UpdatePersistentTaskStatusAction; import java.util.ArrayList; import java.util.Collections; @@ -556,7 +555,6 @@ public void initRestHandlers(Supplier nodesInCluster) { registerHandler.accept(new RestSnapshotsStatusAction(settings, restController)); registerHandler.accept(new RestGetAllAliasesAction(settings, restController)); - registerHandler.accept(new RestGetAllMappingsAction(settings, restController)); registerHandler.accept(new RestGetIndicesAction(settings, restController, indexScopedSettings, settingsFilter)); registerHandler.accept(new RestIndicesStatsAction(settings, restController)); registerHandler.accept(new RestIndicesSegmentsAction(settings, restController)); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAllMappingsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAllMappingsAction.java deleted file mode 100644 index 9892717cd779b..0000000000000 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAllMappingsAction.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.rest.action.admin.indices; - -import com.carrotsearch.hppc.cursors.ObjectObjectCursor; -import org.elasticsearch.action.admin.indices.get.GetIndexRequest; -import org.elasticsearch.action.admin.indices.get.GetIndexRequest.Feature; -import org.elasticsearch.action.admin.indices.get.GetIndexResponse; -import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.client.node.NodeClient; -import org.elasticsearch.cluster.metadata.AliasMetaData; -import org.elasticsearch.cluster.metadata.MappingMetaData; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.collect.ImmutableOpenMap; -import org.elasticsearch.common.settings.IndexScopedSettings; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.settings.SettingsFilter; -import org.elasticsearch.common.xcontent.ToXContent.Params; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.rest.BaseRestHandler; -import org.elasticsearch.rest.BytesRestResponse; -import org.elasticsearch.rest.RestController; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.RestResponse; -import org.elasticsearch.rest.action.RestBuilderListener; - -import java.io.IOException; -import java.util.List; -import java.util.Set; - -import static org.elasticsearch.rest.RestRequest.Method.GET; -import static org.elasticsearch.rest.RestRequest.Method.HEAD; -import static org.elasticsearch.rest.RestStatus.OK; - -/** - * The REST handler for retrieving all mappings - */ -public class RestGetAllMappingsAction extends BaseRestHandler { - - public RestGetAllMappingsAction(final Settings settings, final RestController controller) { - super(settings); - controller.registerHandler(GET, "/_mapping", this); - controller.registerHandler(GET, "/_mappings", this); - } - - @Override - public String getName() { - return "get_all_mappings_action"; - } - - @Override - public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - final GetIndexRequest getIndexRequest = new GetIndexRequest(); - getIndexRequest.indices(Strings.EMPTY_ARRAY); - getIndexRequest.features(Feature.MAPPINGS); - getIndexRequest.indicesOptions(IndicesOptions.fromRequest(request, getIndexRequest.indicesOptions())); - getIndexRequest.local(request.paramAsBoolean("local", getIndexRequest.local())); - getIndexRequest.humanReadable(request.paramAsBoolean("human", false)); - return channel -> client.admin().indices().getIndex(getIndexRequest, new RestBuilderListener(channel) { - - @Override - public RestResponse buildResponse(final GetIndexResponse response, final XContentBuilder builder) throws Exception { - builder.startObject(); - { - for (final String index : response.indices()) { - builder.startObject(index); - { - writeMappings(response.mappings().get(index), builder); - } - builder.endObject(); - } - } - builder.endObject(); - - return new BytesRestResponse(OK, builder); - } - - private void writeMappings(final ImmutableOpenMap mappings, - final XContentBuilder builder) throws IOException { - builder.startObject("mappings"); - { - for (final ObjectObjectCursor typeEntry : mappings) { - builder.field(typeEntry.key); - builder.map(typeEntry.value.sourceAsMap()); - } - } - builder.endObject(); - } - }); - } - -} diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java index 46388e6947f3e..08f8449b7017f 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java @@ -20,8 +20,6 @@ package org.elasticsearch.rest.action.admin.indices; import com.carrotsearch.hppc.cursors.ObjectCursor; -import com.carrotsearch.hppc.cursors.ObjectObjectCursor; - import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.support.IndicesOptions; @@ -56,12 +54,13 @@ import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.HEAD; -import static org.elasticsearch.rest.RestStatus.OK; public class RestGetMappingAction extends BaseRestHandler { public RestGetMappingAction(final Settings settings, final RestController controller) { super(settings); + controller.registerHandler(GET, "/_mapping", this); + controller.registerHandler(GET, "/_mappings", this); controller.registerHandler(GET, "/{index}/{type}/_mapping", this); controller.registerHandler(GET, "/{index}/_mappings", this); controller.registerHandler(GET, "/{index}/_mapping", this); From 8aa58887e2a15ebe49358b0a33f1f4ec81dd5836 Mon Sep 17 00:00:00 2001 From: Jay Modi Date: Wed, 6 Jun 2018 08:18:56 -0600 Subject: [PATCH 14/26] Security: make native realm usage stats accurate (#30824) The native realm's usage stats were previously pulled from the cache, which only contains the number of users that had authenticated in the past 20 minutes. This commit changes this so that we pull the current value from the security index by executing a search request. In order to support this, the usage stats for realms is now asynchronous so that we do not block while waiting on the search to complete. --- .../security/SecurityFeatureSetUsage.java | 4 + .../xpack/core/security/authc/Realm.java | 4 +- .../xpack/security/SecurityFeatureSet.java | 20 ++-- .../xpack/security/authc/Realms.java | 95 ++++++++++++------- .../security/authc/esnative/NativeRealm.java | 12 +++ .../authc/esnative/NativeUsersStore.java | 24 +++++ .../xpack/security/authc/file/FileRealm.java | 10 +- .../xpack/security/authc/ldap/LdapRealm.java | 14 +-- .../support/CachingUsernamePasswordRealm.java | 14 ++- .../security/authz/store/FileRolesStore.java | 2 +- .../authz/store/NativeRolesStore.java | 94 +++++++++--------- .../security/SecurityFeatureSetTests.java | 6 +- .../xpack/security/authc/RealmsTests.java | 15 ++- .../authc/esnative/NativeRealmIntegTests.java | 27 ++++++ .../security/authc/file/FileRealmTests.java | 4 +- .../authc/ldap/ActiveDirectoryRealmTests.java | 4 +- .../security/authc/ldap/LdapRealmTests.java | 4 +- 17 files changed, 237 insertions(+), 116 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityFeatureSetUsage.java index b549cffc0cc33..f615fbd0b5306 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityFeatureSetUsage.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityFeatureSetUsage.java @@ -92,4 +92,8 @@ protected void innerXContent(XContentBuilder builder, Params params) throws IOEx builder.field(ANONYMOUS_XFIELD, anonymousUsage); } } + + public Map getRealmsUsage() { + return Collections.unmodifiableMap(realmsUsage); + } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Realm.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Realm.java index 234141c77c9a7..3e92be2ef904d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Realm.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Realm.java @@ -119,11 +119,11 @@ public int compareTo(Realm other) { */ public abstract void lookupUser(String username, ActionListener listener); - public Map usageStats() { + public void usageStats(ActionListener> listener) { Map stats = new HashMap<>(); stats.put("name", name()); stats.put("order", order()); - return stats; + listener.onResponse(stats); } @Override diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityFeatureSet.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityFeatureSet.java index 1be3b4cd6795b..ab70b8513de31 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityFeatureSet.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityFeatureSet.java @@ -86,7 +86,6 @@ public Map nativeCodeInfo() { @Override public void usage(ActionListener listener) { - Map realmsUsage = buildRealmsUsage(realms); Map sslUsage = sslUsage(settings); Map auditUsage = auditUsage(settings); Map ipFilterUsage = ipFilterUsage(ipFilter); @@ -94,10 +93,11 @@ public void usage(ActionListener listener) { final AtomicReference> rolesUsageRef = new AtomicReference<>(); final AtomicReference> roleMappingUsageRef = new AtomicReference<>(); - final CountDown countDown = new CountDown(2); + final AtomicReference> realmsUsageRef = new AtomicReference<>(); + final CountDown countDown = new CountDown(3); final Runnable doCountDown = () -> { if (countDown.countDown()) { - listener.onResponse(new SecurityFeatureSetUsage(available(), enabled(), realmsUsage, + listener.onResponse(new SecurityFeatureSetUsage(available(), enabled(), realmsUsageRef.get(), rolesUsageRef.get(), roleMappingUsageRef.get(), sslUsage, auditUsage, ipFilterUsage, anonymousUsage)); } @@ -116,6 +116,12 @@ public void usage(ActionListener listener) { doCountDown.run(); }, listener::onFailure); + final ActionListener> realmsUsageListener = + ActionListener.wrap(realmsUsage -> { + realmsUsageRef.set(realmsUsage); + doCountDown.run(); + }, listener::onFailure); + if (rolesStore == null) { rolesStoreUsageListener.onResponse(Collections.emptyMap()); } else { @@ -126,13 +132,11 @@ public void usage(ActionListener listener) { } else { roleMappingStore.usageStats(roleMappingStoreUsageListener); } - } - - static Map buildRealmsUsage(Realms realms) { if (realms == null) { - return Collections.emptyMap(); + realmsUsageListener.onResponse(Collections.emptyMap()); + } else { + realms.usageStats(realmsUsageListener); } - return realms.usageStats(); } static Map sslUsage(Settings settings) { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/Realms.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/Realms.java index 3831959752382..0284ae9a05fa3 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/Realms.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/Realms.java @@ -15,12 +15,16 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.CountDown; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.env.Environment; import org.elasticsearch.license.XPackLicenseState; @@ -188,46 +192,67 @@ protected List initRealms() throws Exception { return realms; } - public Map usageStats() { + public void usageStats(ActionListener> listener) { Map realmMap = new HashMap<>(); - for (Realm realm : this) { - if (ReservedRealm.TYPE.equals(realm.type())) { - continue; - } - realmMap.compute(realm.type(), (key, value) -> { - if (value == null) { - Object realmTypeUsage = convertToMapOfLists(realm.usageStats()); - return realmTypeUsage; - } - assert value instanceof Map; - combineMaps((Map) value, realm.usageStats()); - return value; - }); - } + final AtomicBoolean failed = new AtomicBoolean(false); + final List realmList = asList().stream() + .filter(r -> ReservedRealm.TYPE.equals(r.type()) == false) + .collect(Collectors.toList()); + final CountDown countDown = new CountDown(realmList.size()); + final Runnable doCountDown = () -> { + if ((realmList.isEmpty() || countDown.countDown()) && failed.get() == false) { + final AllowedRealmType allowedRealmType = licenseState.allowedRealmType(); + // iterate over the factories so we can add enabled & available info + for (String type : factories.keySet()) { + assert ReservedRealm.TYPE.equals(type) == false; + realmMap.compute(type, (key, value) -> { + if (value == null) { + return MapBuilder.newMapBuilder() + .put("enabled", false) + .put("available", isRealmTypeAvailable(allowedRealmType, type)) + .map(); + } - final AllowedRealmType allowedRealmType = licenseState.allowedRealmType(); - // iterate over the factories so we can add enabled & available info - for (String type : factories.keySet()) { - assert ReservedRealm.TYPE.equals(type) == false; - realmMap.compute(type, (key, value) -> { - if (value == null) { - return MapBuilder.newMapBuilder() - .put("enabled", false) - .put("available", isRealmTypeAvailable(allowedRealmType, type)) - .map(); + assert value instanceof Map; + Map realmTypeUsage = (Map) value; + realmTypeUsage.put("enabled", true); + // the realms iterator returned this type so it must be enabled + assert isRealmTypeAvailable(allowedRealmType, type); + realmTypeUsage.put("available", true); + return value; + }); } + listener.onResponse(realmMap); + } + }; - assert value instanceof Map; - Map realmTypeUsage = (Map) value; - realmTypeUsage.put("enabled", true); - // the realms iterator returned this type so it must be enabled - assert isRealmTypeAvailable(allowedRealmType, type); - realmTypeUsage.put("available", true); - return value; - }); + if (realmList.isEmpty()) { + doCountDown.run(); + } else { + for (Realm realm : realmList) { + realm.usageStats(ActionListener.wrap(stats -> { + if (failed.get() == false) { + synchronized (realmMap) { + realmMap.compute(realm.type(), (key, value) -> { + if (value == null) { + Object realmTypeUsage = convertToMapOfLists(stats); + return realmTypeUsage; + } + assert value instanceof Map; + combineMaps((Map) value, stats); + return value; + }); + } + doCountDown.run(); + } + }, + e -> { + if (failed.compareAndSet(false, true)) { + listener.onFailure(e); + } + })); + } } - - return realmMap; } private void addNativeRealms(List realms) throws Exception { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealm.java index af2bfcf0d6c14..a84b76beab8bb 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealm.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealm.java @@ -15,6 +15,8 @@ import org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm; import org.elasticsearch.xpack.security.support.SecurityIndexManager; +import java.util.Map; + import static org.elasticsearch.xpack.security.support.SecurityIndexManager.isIndexDeleted; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.isMoveFromRedToNonRed; @@ -46,6 +48,16 @@ public void onSecurityIndexStateChange(SecurityIndexManager.State previousState, } } + @Override + public void usageStats(ActionListener> listener) { + super.usageStats(ActionListener.wrap(stats -> + userStore.getUserCount(ActionListener.wrap(size -> { + stats.put("size", size); + listener.onResponse(stats); + }, listener::onFailure)) + , listener::onFailure)); + } + // method is used for testing to verify cache expiration since expireAll is final void clearCache() { expireAll(); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java index 1477c6dc88045..72a65b8213f81 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java @@ -150,6 +150,30 @@ public void getUsers(String[] userNames, final ActionListener> } } + void getUserCount(final ActionListener listener) { + if (securityIndex.indexExists() == false) { + listener.onResponse(0L); + } else { + securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> + executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, + client.prepareSearch(SECURITY_INDEX_NAME) + .setQuery(QueryBuilders.termQuery(Fields.TYPE.getPreferredName(), USER_DOC_TYPE)) + .setSize(0) + .request(), + new ActionListener() { + @Override + public void onResponse(SearchResponse response) { + listener.onResponse(response.getHits().getTotalHits()); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }, client::search)); + } + } + /** * Async method to retrieve a user and their password */ diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileRealm.java index 88656b9e01e30..e2586ea836dec 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileRealm.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileRealm.java @@ -55,11 +55,11 @@ protected void doLookupUser(String username, ActionListener listener) { } @Override - public Map usageStats() { - Map stats = super.usageStats(); - // here we can determine the size based on the in mem user store - stats.put("size", userPasswdStore.usersCount()); - return stats; + public void usageStats(ActionListener> listener) { + super.usageStats(ActionListener.wrap(stats -> { + stats.put("size", userPasswdStore.usersCount()); + listener.onResponse(stats); + }, listener::onFailure)); } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealm.java index a7c6efdda3114..87749850141bf 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealm.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealm.java @@ -160,12 +160,14 @@ private ContextPreservingActionListener contextPreservingListener(L } @Override - public Map usageStats() { - Map usage = super.usageStats(); - usage.put("load_balance_type", LdapLoadBalancing.resolve(config.settings()).toString()); - usage.put("ssl", sessionFactory.isSslUsed()); - usage.put("user_search", LdapUserSearchSessionFactory.hasUserSearchSettings(config)); - return usage; + public void usageStats(ActionListener> listener) { + super.usageStats(ActionListener.wrap(usage -> { + usage.put("size", getCacheSize()); + usage.put("load_balance_type", LdapLoadBalancing.resolve(config.settings()).toString()); + usage.put("ssl", sessionFactory.isSslUsed()); + usage.put("user_search", LdapUserSearchSessionFactory.hasUserSearchSettings(config)); + listener.onResponse(usage); + }, listener::onFailure)); } private static void buildUser(LdapSession session, String username, ActionListener listener, diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/CachingUsernamePasswordRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/CachingUsernamePasswordRealm.java index 8dae5275eda14..e9c107abccef3 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/CachingUsernamePasswordRealm.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/CachingUsernamePasswordRealm.java @@ -22,6 +22,7 @@ import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.core.security.user.User; +import java.util.Collections; import java.util.Map; import java.util.Objects; import java.util.concurrent.ExecutionException; @@ -177,10 +178,15 @@ private void handleFailure(ListenableFuture usageStats() { - Map stats = super.usageStats(); - stats.put("size", cache.count()); - return stats; + public void usageStats(ActionListener> listener) { + super.usageStats(ActionListener.wrap(stats -> { + stats.put("cache", Collections.singletonMap("size", getCacheSize())); + listener.onResponse(stats); + }, listener::onFailure)); + } + + protected int getCacheSize() { + return cache.count(); } protected abstract void doAuthenticate(UsernamePasswordToken token, ActionListener listener); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/FileRolesStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/FileRolesStore.java index f2d78806da0dc..59bc8042fbaff 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/FileRolesStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/FileRolesStore.java @@ -87,7 +87,7 @@ public Set roleDescriptors(Set roleNames) { } public Map usageStats() { - Map usageStats = new HashMap<>(); + Map usageStats = new HashMap<>(3); usageStats.put("size", permissions.size()); boolean dls = false; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java index b1e5170a2027c..9093b6a66739e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java @@ -195,7 +195,7 @@ public void onFailure(Exception e) { } public void usageStats(ActionListener> listener) { - Map usageStats = new HashMap<>(); + Map usageStats = new HashMap<>(3); if (securityIndex.indexExists() == false) { usageStats.put("size", 0L); usageStats.put("fls", false); @@ -204,56 +204,56 @@ public void usageStats(ActionListener> listener) { } else { securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareMultiSearch() - .add(client.prepareSearch(SecurityIndexManager.SECURITY_INDEX_NAME) - .setQuery(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE)) - .setSize(0)) - .add(client.prepareSearch(SecurityIndexManager.SECURITY_INDEX_NAME) - .setQuery(QueryBuilders.boolQuery() - .must(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE)) - .must(QueryBuilders.boolQuery() - .should(existsQuery("indices.field_security.grant")) - .should(existsQuery("indices.field_security.except")) - // for backwardscompat with 2.x - .should(existsQuery("indices.fields")))) - .setSize(0) - .setTerminateAfter(1)) - .add(client.prepareSearch(SecurityIndexManager.SECURITY_INDEX_NAME) - .setQuery(QueryBuilders.boolQuery() - .must(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE)) - .filter(existsQuery("indices.query"))) - .setSize(0) - .setTerminateAfter(1)) - .request(), - new ActionListener() { - @Override - public void onResponse(MultiSearchResponse items) { - Item[] responses = items.getResponses(); - if (responses[0].isFailure()) { - usageStats.put("size", 0); - } else { - usageStats.put("size", responses[0].getResponse().getHits().getTotalHits()); - } - - if (responses[1].isFailure()) { - usageStats.put("fls", false); - } else { - usageStats.put("fls", responses[1].getResponse().getHits().getTotalHits() > 0L); - } + client.prepareMultiSearch() + .add(client.prepareSearch(SecurityIndexManager.SECURITY_INDEX_NAME) + .setQuery(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE)) + .setSize(0)) + .add(client.prepareSearch(SecurityIndexManager.SECURITY_INDEX_NAME) + .setQuery(QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE)) + .must(QueryBuilders.boolQuery() + .should(existsQuery("indices.field_security.grant")) + .should(existsQuery("indices.field_security.except")) + // for backwardscompat with 2.x + .should(existsQuery("indices.fields")))) + .setSize(0) + .setTerminateAfter(1)) + .add(client.prepareSearch(SecurityIndexManager.SECURITY_INDEX_NAME) + .setQuery(QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE)) + .filter(existsQuery("indices.query"))) + .setSize(0) + .setTerminateAfter(1)) + .request(), + new ActionListener() { + @Override + public void onResponse(MultiSearchResponse items) { + Item[] responses = items.getResponses(); + if (responses[0].isFailure()) { + usageStats.put("size", 0); + } else { + usageStats.put("size", responses[0].getResponse().getHits().getTotalHits()); + } - if (responses[2].isFailure()) { - usageStats.put("dls", false); - } else { - usageStats.put("dls", responses[2].getResponse().getHits().getTotalHits() > 0L); - } - listener.onResponse(usageStats); + if (responses[1].isFailure()) { + usageStats.put("fls", false); + } else { + usageStats.put("fls", responses[1].getResponse().getHits().getTotalHits() > 0L); } - @Override - public void onFailure(Exception e) { - listener.onFailure(e); + if (responses[2].isFailure()) { + usageStats.put("dls", false); + } else { + usageStats.put("dls", responses[2].getResponse().getHits().getTotalHits() > 0L); } - }, client::multiSearch)); + listener.onResponse(usageStats); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }, client::multiSearch)); } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityFeatureSetTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityFeatureSetTests.java index c169d62c6b17b..076ce6c9fcb56 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityFeatureSetTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityFeatureSetTests.java @@ -146,7 +146,11 @@ public void testUsage() throws Exception { realmUsage.put("key2", Arrays.asList(i)); realmUsage.put("key3", Arrays.asList(i % 2 == 0)); } - when(realms.usageStats()).thenReturn(realmsUsageStats); + doAnswer(invocationOnMock -> { + ActionListener> listener = (ActionListener) invocationOnMock.getArguments()[0]; + listener.onResponse(realmsUsageStats); + return Void.TYPE; + }).when(realms).usageStats(any(ActionListener.class)); final boolean anonymousEnabled = randomBoolean(); if (anonymousEnabled) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RealmsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RealmsTests.java index 2bc3d58471b15..ff4c30ddf8c0a 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RealmsTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RealmsTests.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.security.authc; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.env.Environment; @@ -454,9 +455,11 @@ public void testUsageStats() throws Exception { .put("xpack.security.authc.realms.bar.order", "1"); Settings settings = builder.build(); Environment env = TestEnvironment.newEnvironment(settings); - Realms realms = new Realms(settings, env, factories, licenseState, threadContext, reservedRealm ); + Realms realms = new Realms(settings, env, factories, licenseState, threadContext, reservedRealm); - Map usageStats = realms.usageStats(); + PlainActionFuture> future = new PlainActionFuture<>(); + realms.usageStats(future); + Map usageStats = future.get(); assertThat(usageStats.size(), is(factories.size())); // first check type_0 @@ -482,7 +485,9 @@ public void testUsageStats() throws Exception { // disable ALL using license when(licenseState.isAuthAllowed()).thenReturn(false); when(licenseState.allowedRealmType()).thenReturn(AllowedRealmType.NONE); - usageStats = realms.usageStats(); + future = new PlainActionFuture<>(); + realms.usageStats(future); + usageStats = future.get(); assertThat(usageStats.size(), is(factories.size())); for (Entry entry : usageStats.entrySet()) { Map typeMap = (Map) entry.getValue(); @@ -494,7 +499,9 @@ public void testUsageStats() throws Exception { // check native or internal realms enabled only when(licenseState.isAuthAllowed()).thenReturn(true); when(licenseState.allowedRealmType()).thenReturn(randomFrom(AllowedRealmType.NATIVE, AllowedRealmType.DEFAULT)); - usageStats = realms.usageStats(); + future = new PlainActionFuture<>(); + realms.usageStats(future); + usageStats = future.get(); assertThat(usageStats.size(), is(factories.size())); for (Entry entry : usageStats.entrySet()) { final String type = entry.getKey(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java index a238576e41323..a0550b4c1ce15 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java @@ -22,6 +22,10 @@ import org.elasticsearch.test.NativeRealmIntegTestCase; import org.elasticsearch.test.SecuritySettingsSource; import org.elasticsearch.test.SecuritySettingsSourceField; +import org.elasticsearch.xpack.core.XPackFeatureSet; +import org.elasticsearch.xpack.core.action.XPackUsageRequestBuilder; +import org.elasticsearch.xpack.core.action.XPackUsageResponse; +import org.elasticsearch.xpack.core.security.SecurityFeatureSetUsage; import org.elasticsearch.xpack.core.security.action.role.DeleteRoleResponse; import org.elasticsearch.xpack.core.security.action.role.GetRolesResponse; import org.elasticsearch.xpack.core.security.action.role.PutRoleResponse; @@ -49,6 +53,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.CountDownLatch; import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; @@ -662,6 +667,28 @@ public void testRolesUsageStats() throws Exception { assertThat(usage.get("dls"), is(dls)); } + public void testRealmUsageStats() { + final int numNativeUsers = scaledRandomIntBetween(1, 32); + SecurityClient securityClient = new SecurityClient(client()); + for (int i = 0; i < numNativeUsers; i++) { + securityClient.preparePutUser("joe" + i, "s3krit".toCharArray(), "superuser").get(); + } + + XPackUsageResponse response = new XPackUsageRequestBuilder(client()).get(); + Optional securityUsage = response.getUsages().stream() + .filter(usage -> usage instanceof SecurityFeatureSetUsage) + .findFirst(); + assertTrue(securityUsage.isPresent()); + SecurityFeatureSetUsage securityFeatureSetUsage = (SecurityFeatureSetUsage) securityUsage.get(); + Map realmsUsage = securityFeatureSetUsage.getRealmsUsage(); + assertNotNull(realmsUsage); + assertNotNull(realmsUsage.get("native")); + assertNotNull(((Map) realmsUsage.get("native")).get("size")); + List sizeList = (List) ((Map) realmsUsage.get("native")).get("size"); + assertEquals(1, sizeList.size()); + assertEquals(numNativeUsers, Math.toIntExact(sizeList.get(0))); + } + public void testSetEnabled() throws Exception { securityClient().preparePutUser("joe", "s3krit".toCharArray(), SecuritySettingsSource.TEST_ROLE).get(); final String token = basicAuthHeaderValue("joe", new SecureString("s3krit".toCharArray())); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileRealmTests.java index b0f53229377f0..7295e48d00394 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileRealmTests.java @@ -248,7 +248,9 @@ public void testUsageStats() throws Exception { threadContext); FileRealm realm = new FileRealm(config, userPasswdStore, userRolesStore, threadPool); - Map usage = realm.usageStats(); + PlainActionFuture> future = new PlainActionFuture<>(); + realm.usageStats(future); + Map usage = future.get(); assertThat(usage, is(notNullValue())); assertThat(usage, hasEntry("name", "file-realm")); assertThat(usage, hasEntry("order", order)); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java index 52026cc8af5d9..6ab4dbf3e0cfc 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java @@ -320,7 +320,9 @@ public void testRealmUsageStats() throws Exception { DnRoleMapper roleMapper = new DnRoleMapper(config, resourceWatcherService); LdapRealm realm = new LdapRealm(LdapRealmSettings.AD_TYPE, config, sessionFactory, roleMapper, threadPool); - Map stats = realm.usageStats(); + PlainActionFuture> future = new PlainActionFuture<>(); + realm.usageStats(future); + Map stats = future.get(); assertThat(stats, is(notNullValue())); assertThat(stats, hasEntry("name", realm.name())); assertThat(stats, hasEntry("order", realm.order())); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmTests.java index 042664fa6707d..ea1b9117922ce 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmTests.java @@ -360,7 +360,9 @@ public void testUsageStats() throws Exception { LdapRealm realm = new LdapRealm(LdapRealmSettings.LDAP_TYPE, config, ldapFactory, new DnRoleMapper(config, resourceWatcherService), threadPool); - Map stats = realm.usageStats(); + PlainActionFuture> future = new PlainActionFuture<>(); + realm.usageStats(future); + Map stats = future.get(); assertThat(stats, is(notNullValue())); assertThat(stats, hasEntry("name", "ldap-realm")); assertThat(stats, hasEntry("order", realm.order())); From 515a23360d4791499e7675c38bbad9e6e244fc7b Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Wed, 6 Jun 2018 16:38:06 +0200 Subject: [PATCH 15/26] Do not check for S3 blob to exist before writing (#31128) In #19749 an extra check was added before writing each blob to ensure that we would not be overriding an existing blob. Due to S3's weak consistency model, this check was best effort. To make matters worse, however, this resulted in a HEAD request to be done before every PUT, in particular also when PUTTING a new object. The approach taken in #19749 worsened our consistency guarantees for follow-up snapshot actions, as it made it less likely for new files that had been written to be available for reads. This commit therefore removes this extra check. Due to the weak consistency model, this check was a best effort thing anyway, and there's currently no way to prevent accidental overrides on S3. --- .../org/elasticsearch/repositories/s3/S3BlobContainer.java | 4 ---- .../repositories/s3/S3BlobStoreContainerTests.java | 5 +++++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java index 401ef0933a847..92050e34a5a62 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java @@ -96,10 +96,6 @@ public InputStream readBlob(String blobName) throws IOException { @Override public void writeBlob(String blobName, InputStream inputStream, long blobSize) throws IOException { - if (blobExists(blobName)) { - throw new FileAlreadyExistsException("Blob [" + blobName + "] already exists, cannot overwrite"); - } - SocketAccess.doPrivilegedIOException(() -> { if (blobSize <= blobStore.bufferSizeInBytes()) { executeSingleUpload(blobStore, buildKey(blobName), inputStream, blobSize); diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreContainerTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreContainerTests.java index 453ef3213f0b6..c760e86d1353f 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreContainerTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreContainerTests.java @@ -64,6 +64,11 @@ protected BlobStore newBlobStore() { return randomMockS3BlobStore(); } + @Override + public void testVerifyOverwriteFails() { + assumeFalse("not implemented because of S3's weak consistency model", true); + } + public void testExecuteSingleUploadBlobSizeTooLarge() { final long blobSize = ByteSizeUnit.GB.toBytes(randomIntBetween(6, 10)); final S3BlobStore blobStore = mock(S3BlobStore.class); From 1dca00deb93bdc104f6ebb8814bf43a42b9273d9 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Wed, 6 Jun 2018 16:38:37 +0200 Subject: [PATCH 16/26] Remove extra checks from HdfsBlobContainer (#31126) This commit saves one network roundtrip when reading or deleting files from an HDFS repository. --- .../repositories/hdfs/HdfsBlobContainer.java | 24 +++++++++++-------- .../ESBlobStoreContainerTestCase.java | 7 ++++++ 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java index 926cf0b2ad4af..1052dc9ded985 100644 --- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java +++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.blobstore.support.PlainBlobMetaData; import org.elasticsearch.repositories.hdfs.HdfsBlobStore.Operation; +import java.io.FileNotFoundException; import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; @@ -69,11 +70,13 @@ public boolean blobExists(String blobName) { @Override public void deleteBlob(String blobName) throws IOException { - if (!blobExists(blobName)) { - throw new NoSuchFileException("Blob [" + blobName + "] does not exist"); + try { + if (store.execute(fileContext -> fileContext.delete(new Path(path, blobName), true)) == false) { + throw new NoSuchFileException("Blob [" + blobName + "] does not exist"); + } + } catch (FileNotFoundException fnfe) { + throw new NoSuchFileException("[" + blobName + "] blob not found"); } - - store.execute(fileContext -> fileContext.delete(new Path(path, blobName), true)); } @Override @@ -86,16 +89,17 @@ public void move(String sourceBlobName, String targetBlobName) throws IOExceptio @Override public InputStream readBlob(String blobName) throws IOException { - if (!blobExists(blobName)) { - throw new NoSuchFileException("Blob [" + blobName + "] does not exist"); - } // FSDataInputStream does buffering internally // FSDataInputStream can open connections on read() or skip() so we wrap in // HDFSPrivilegedInputSteam which will ensure that underlying methods will // be called with the proper privileges. - return store.execute(fileContext -> - new HDFSPrivilegedInputSteam(fileContext.open(new Path(path, blobName), bufferSize), securityContext) - ); + try { + return store.execute(fileContext -> + new HDFSPrivilegedInputSteam(fileContext.open(new Path(path, blobName), bufferSize), securityContext) + ); + } catch (FileNotFoundException fnfe) { + throw new NoSuchFileException("[" + blobName + "] blob not found"); + } } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java index df2024de445c1..be06e09c6ff8b 100644 --- a/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java @@ -47,6 +47,13 @@ */ public abstract class ESBlobStoreContainerTestCase extends ESTestCase { + public void testReadNonExistingPath() throws IOException { + try(BlobStore store = newBlobStore()) { + final BlobContainer container = store.blobContainer(new BlobPath()); + expectThrows(NoSuchFileException.class, () -> container.readBlob("non-existing")); + } + } + public void testWriteRead() throws IOException { try(BlobStore store = newBlobStore()) { final BlobContainer container = store.blobContainer(new BlobPath()); From e9fe371e41bcb6f8bc7b90b9cbbd2c74e0145a79 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 6 Jun 2018 16:46:11 +0200 Subject: [PATCH 17/26] Give the engine the whole index buffer size on init. (#31105) Currently the engine is initialized with a hardcoded 256MB of RAM. Elasticsearch may never use more than that for a given shard, `IndexingMemoryController` only has the power to flush segments to disk earlier in case multiple shards are actively indexing and use too much memory. While this amount of memory is enough for an index with few fields and larger RAM buffers are not expected to improve indexing speed, this might actually be little for an index that has many fields. Kudos to @bleskes for finding it out when looking into a user who was reporting a **much** slower indexing speed when upgrading from 2.x to 5.6 with an index that has about 20,000 fields. --- .../index/engine/EngineConfig.java | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/engine/EngineConfig.java b/server/src/main/java/org/elasticsearch/index/engine/EngineConfig.java index b7c5a41691343..2deae61bd52e9 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/EngineConfig.java +++ b/server/src/main/java/org/elasticsearch/index/engine/EngineConfig.java @@ -29,8 +29,8 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; -import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.MemorySizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.codec.CodecService; @@ -139,10 +139,20 @@ public EngineConfig(ShardId shardId, String allocationId, ThreadPool threadPool, this.codecService = codecService; this.eventListener = eventListener; codecName = indexSettings.getValue(INDEX_CODEC_SETTING); - // We give IndexWriter a "huge" (256 MB) buffer, so it won't flush on its own unless the ES indexing buffer is also huge and/or - // there are not too many shards allocated to this node. Instead, IndexingMemoryController periodically checks - // and refreshes the most heap-consuming shards when total indexing heap usage across all shards is too high: - indexingBufferSize = new ByteSizeValue(256, ByteSizeUnit.MB); + // We need to make the indexing buffer for this shard at least as large + // as the amount of memory that is available for all engines on the + // local node so that decisions to flush segments to disk are made by + // IndexingMemoryController rather than Lucene. + // Add an escape hatch in case this change proves problematic - it used + // to be a fixed amound of RAM: 256 MB. + // TODO: Remove this escape hatch in 8.x + final String escapeHatchProperty = "es.index.memory.max_index_buffer_size"; + String maxBufferSize = System.getProperty(escapeHatchProperty); + if (maxBufferSize != null) { + indexingBufferSize = MemorySizeValue.parseBytesSizeValueOrHeapRatio(maxBufferSize, escapeHatchProperty); + } else { + indexingBufferSize = IndexingMemoryController.INDEX_BUFFER_SIZE_SETTING.get(indexSettings.getNodeSettings()); + } this.queryCache = queryCache; this.queryCachingPolicy = queryCachingPolicy; this.translogConfig = translogConfig; From 6fd4eb52b8261581cf3b7e5c3178fe7940d1b00f Mon Sep 17 00:00:00 2001 From: Lisa Cawley Date: Wed, 6 Jun 2018 07:49:15 -0700 Subject: [PATCH 18/26] [DOCS] Moves commands to docs folder (#31114) --- {x-pack/docs/en => docs/reference}/commands/certgen.asciidoc | 1 + {x-pack/docs/en => docs/reference}/commands/certutil.asciidoc | 1 + {x-pack/docs/en => docs/reference}/commands/index.asciidoc | 0 .../docs/en => docs/reference}/commands/migrate-tool.asciidoc | 1 + .../docs/en => docs/reference}/commands/saml-metadata.asciidoc | 1 + .../en => docs/reference}/commands/setup-passwords.asciidoc | 1 + {x-pack/docs/en => docs/reference}/commands/syskeygen.asciidoc | 1 + .../docs/en => docs/reference}/commands/users-command.asciidoc | 1 + docs/reference/index.asciidoc | 2 +- 9 files changed, 8 insertions(+), 1 deletion(-) rename {x-pack/docs/en => docs/reference}/commands/certgen.asciidoc (99%) rename {x-pack/docs/en => docs/reference}/commands/certutil.asciidoc (99%) rename {x-pack/docs/en => docs/reference}/commands/index.asciidoc (100%) rename {x-pack/docs/en => docs/reference}/commands/migrate-tool.asciidoc (99%) rename {x-pack/docs/en => docs/reference}/commands/saml-metadata.asciidoc (99%) rename {x-pack/docs/en => docs/reference}/commands/setup-passwords.asciidoc (99%) rename {x-pack/docs/en => docs/reference}/commands/syskeygen.asciidoc (98%) rename {x-pack/docs/en => docs/reference}/commands/users-command.asciidoc (99%) diff --git a/x-pack/docs/en/commands/certgen.asciidoc b/docs/reference/commands/certgen.asciidoc similarity index 99% rename from x-pack/docs/en/commands/certgen.asciidoc rename to docs/reference/commands/certgen.asciidoc index c2a00f11b6958..3a8b15fbd28c8 100644 --- a/x-pack/docs/en/commands/certgen.asciidoc +++ b/docs/reference/commands/certgen.asciidoc @@ -1,4 +1,5 @@ [role="xpack"] +[testenv="gold+"] [[certgen]] == certgen diff --git a/x-pack/docs/en/commands/certutil.asciidoc b/docs/reference/commands/certutil.asciidoc similarity index 99% rename from x-pack/docs/en/commands/certutil.asciidoc rename to docs/reference/commands/certutil.asciidoc index ad265c89f10e8..e0c6c701e31fe 100644 --- a/x-pack/docs/en/commands/certutil.asciidoc +++ b/docs/reference/commands/certutil.asciidoc @@ -1,4 +1,5 @@ [role="xpack"] +[testenv="gold+"] [[certutil]] == elasticsearch-certutil diff --git a/x-pack/docs/en/commands/index.asciidoc b/docs/reference/commands/index.asciidoc similarity index 100% rename from x-pack/docs/en/commands/index.asciidoc rename to docs/reference/commands/index.asciidoc diff --git a/x-pack/docs/en/commands/migrate-tool.asciidoc b/docs/reference/commands/migrate-tool.asciidoc similarity index 99% rename from x-pack/docs/en/commands/migrate-tool.asciidoc rename to docs/reference/commands/migrate-tool.asciidoc index 1d19452df8094..a1903ac69dacf 100644 --- a/x-pack/docs/en/commands/migrate-tool.asciidoc +++ b/docs/reference/commands/migrate-tool.asciidoc @@ -1,4 +1,5 @@ [role="xpack"] +[testenv="gold+"] [[migrate-tool]] == elasticsearch-migrate diff --git a/x-pack/docs/en/commands/saml-metadata.asciidoc b/docs/reference/commands/saml-metadata.asciidoc similarity index 99% rename from x-pack/docs/en/commands/saml-metadata.asciidoc rename to docs/reference/commands/saml-metadata.asciidoc index 1cd283fd77699..069c7135c0144 100644 --- a/x-pack/docs/en/commands/saml-metadata.asciidoc +++ b/docs/reference/commands/saml-metadata.asciidoc @@ -1,4 +1,5 @@ [role="xpack"] +[testenv="gold+"] [[saml-metadata]] == saml-metadata diff --git a/x-pack/docs/en/commands/setup-passwords.asciidoc b/docs/reference/commands/setup-passwords.asciidoc similarity index 99% rename from x-pack/docs/en/commands/setup-passwords.asciidoc rename to docs/reference/commands/setup-passwords.asciidoc index b323dc8e5c19e..a7dcd25d65e0b 100644 --- a/x-pack/docs/en/commands/setup-passwords.asciidoc +++ b/docs/reference/commands/setup-passwords.asciidoc @@ -1,4 +1,5 @@ [role="xpack"] +[testenv="gold+"] [[setup-passwords]] == elasticsearch-setup-passwords diff --git a/x-pack/docs/en/commands/syskeygen.asciidoc b/docs/reference/commands/syskeygen.asciidoc similarity index 98% rename from x-pack/docs/en/commands/syskeygen.asciidoc rename to docs/reference/commands/syskeygen.asciidoc index f4a198ff4bf22..3ae7456448d83 100644 --- a/x-pack/docs/en/commands/syskeygen.asciidoc +++ b/docs/reference/commands/syskeygen.asciidoc @@ -1,4 +1,5 @@ [role="xpack"] +[testenv="gold+"] [[syskeygen]] == elasticsearch-syskeygen diff --git a/x-pack/docs/en/commands/users-command.asciidoc b/docs/reference/commands/users-command.asciidoc similarity index 99% rename from x-pack/docs/en/commands/users-command.asciidoc rename to docs/reference/commands/users-command.asciidoc index ab1b89b149b90..e53e0815c5d7b 100644 --- a/x-pack/docs/en/commands/users-command.asciidoc +++ b/docs/reference/commands/users-command.asciidoc @@ -1,4 +1,5 @@ [role="xpack"] +[testenv="gold+"] [[users-command]] == Users Command ++++ diff --git a/docs/reference/index.asciidoc b/docs/reference/index.asciidoc index b38a554d6815b..11006d38976b5 100644 --- a/docs/reference/index.asciidoc +++ b/docs/reference/index.asciidoc @@ -65,7 +65,7 @@ include::{xes-repo-dir}/rollup/index.asciidoc[] include::rest-api/index.asciidoc[] -include::{xes-repo-dir}/commands/index.asciidoc[] +include::commands/index.asciidoc[] :edit_url: include::how-to.asciidoc[] From 7c59e7690e04b87c148d9b9cb1c0000f533a4da7 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 6 Jun 2018 11:59:16 -0400 Subject: [PATCH 19/26] QA: Switch xpack rolling upgrades to three nodes (#31112) This is much more realistic and can find more issues. This causes the "mixed cluster" tests to be run twice so I had to fix the tests to work in that case. In most cases I did as little as possible to get them working but in a few cases I went a little beyond that to make them easier for me to debug while getting them to work. My test changes: 1. Remove the "basic indexing" tests and replace them with a copy of the tests used in the OSS. We have no way of sharing code between these two projects so for now I copy. 2. Skip the a few tests in the "one third" upgraded scenario: * creating a scroll to be reused when the cluster is fully upgraded * creating some ml data to be used when the cluster is fully ugpraded 3. Drop many "assert yellow and that the cluster has two nodes" assertions. These assertions duplicate those made by the wait condition and they fail now that we have three nodes. 4. Switch many "assert green and that the cluster has two nodes" to 3 nodes. These assertions are unique from the wait condition and, while I imagine they aren't required in all cases, now is not the time to find that out. Thus, I made them work. 5. Rework the index audit trail test so it is more obvious that it is the same test expecting different numbers based on the shape of the cluster. The conditions for which number are expected are fairly complex because the index audit trail is shut down until the template for it is upgraded and the template is upgraded when a master node is elected that has the new version of the software. 6. Add some more information to debug the index audit trail test because it helped me figure out what was going on. I also dropped the `waitCondition` from the `rolling-upgrade-basic` tests because it wasn't needed. Closes #25336 --- .../elasticsearch/upgrades/IndexingIT.java | 4 + x-pack/qa/rolling-upgrade-basic/build.gradle | 144 ++++++---------- x-pack/qa/rolling-upgrade/build.gradle | 136 +++++++-------- .../upgrades/AbstractUpgradeTestCase.java | 6 +- .../upgrades/IndexAuditUpgradeIT.java | 83 +++++---- .../elasticsearch/upgrades/IndexingIT.java | 124 ++++++++++++++ .../TokenBackwardsCompatibilityIT.java | 8 +- .../test/mixed_cluster/10_basic.yml | 159 +----------------- .../test/mixed_cluster/20_security.yml | 11 -- .../test/mixed_cluster/30_ml_jobs_crud.yml | 7 - .../mixed_cluster/40_ml_datafeed_crud.yml | 6 - .../test/old_cluster/10_basic.yml | 31 ---- .../test/upgraded_cluster/10_basic.yml | 39 +---- .../test/upgraded_cluster/20_security.yml | 3 +- .../test/upgraded_cluster/30_ml_jobs_crud.yml | 2 +- .../upgraded_cluster/40_ml_datafeed_crud.yml | 3 +- 16 files changed, 314 insertions(+), 452 deletions(-) create mode 100644 x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java delete mode 100644 x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java index f1e01d24acff6..3898746e5c374 100644 --- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java @@ -30,6 +30,10 @@ * Basic test that indexed documents survive the rolling restart. See * {@link RecoveryIT} for much more in depth testing of the mechanism * by which they survive. + *

+ * This test is an almost exact copy of IndexingIT in the + * xpack rolling restart tests. We should work on a way to remove this + * duplication but for now we have no real way to share code. */ public class IndexingIT extends AbstractRollingTestCase { public void testIndexing() throws IOException { diff --git a/x-pack/qa/rolling-upgrade-basic/build.gradle b/x-pack/qa/rolling-upgrade-basic/build.gradle index 91a6d106c98f8..3592d34c9f4c4 100644 --- a/x-pack/qa/rolling-upgrade-basic/build.gradle +++ b/x-pack/qa/rolling-upgrade-basic/build.gradle @@ -8,62 +8,9 @@ apply plugin: 'elasticsearch.standalone-test' dependencies { testCompile project(path: xpackModule('core'), configuration: 'runtime') - testCompile project(path: xpackModule('security'), configuration: 'runtime') testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') // to be moved in a later commit } -Closure waitWithAuth = { NodeInfo node, AntBuilder ant -> - File tmpFile = new File(node.cwd, 'wait.success') - - // wait up to two minutes - final long stopTime = System.currentTimeMillis() + (2 * 60000L); - Exception lastException = null; - int lastResponseCode = 0 - - while (System.currentTimeMillis() < stopTime) { - - lastException = null; - // we use custom wait logic here as the elastic user is not available immediately and ant.get will fail when a 401 is returned - HttpURLConnection httpURLConnection = null; - try { - // TODO this sucks having to hardcode number of nodes, but node.config.numNodes isn't necessarily accurate for rolling - httpURLConnection = (HttpURLConnection) new URL("http://${node.httpUri()}/_cluster/health?wait_for_nodes=2&wait_for_status=yellow").openConnection(); - httpURLConnection.setRequestProperty("Authorization", "Basic " + - Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8))); - httpURLConnection.setRequestMethod("GET"); - httpURLConnection.setConnectTimeout(1000); - httpURLConnection.setReadTimeout(30000); // read needs to wait for nodes! - httpURLConnection.connect(); - lastResponseCode = httpURLConnection.getResponseCode() - if (lastResponseCode == 200) { - tmpFile.withWriter StandardCharsets.UTF_8.name(), { - it.write(httpURLConnection.getInputStream().getText(StandardCharsets.UTF_8.name())) - } - break; - } - } catch (Exception e) { - logger.debug("failed to call cluster health", e) - lastException = e - } finally { - if (httpURLConnection != null) { - httpURLConnection.disconnect(); - } - } - - // did not start, so wait a bit before trying again - Thread.sleep(500L); - } - if (tmpFile.exists() == false) { - final String message = "final attempt of calling cluster health failed [lastResponseCode=${lastResponseCode}]" - if (lastException != null) { - logger.error(message, lastException) - } else { - logger.error(message + " [no exception]") - } - } - return tmpFile.exists() -} - // This is a top level task which we will add dependencies to below. // It is a single task that can be used to backcompat tests against all versions. task bwcTest { @@ -82,14 +29,13 @@ for (Version version : bwcVersions.wireCompatible) { configure(extensions.findByName("${baseName}#oldClusterTestCluster")) { if (version.before('6.3.0')) { - mavenPlugin 'x-pack', "org.elasticsearch.plugin:x-pack:${version}" + mavenPlugin 'x-pack', "org.elasticsearch.plugin:x-pack:${version}" } bwcVersion = version - numBwcNodes = 2 - numNodes = 2 - minimumMasterNodes = { 2 } + numBwcNodes = 3 + numNodes = 3 + minimumMasterNodes = { 3 } clusterName = 'rolling-upgrade-basic' - waitCondition = waitWithAuth setting 'xpack.security.enabled', 'false' setting 'xpack.monitoring.enabled', 'false' setting 'xpack.ml.enabled', 'false' @@ -102,51 +48,62 @@ for (Version version : bwcVersions.wireCompatible) { systemProperty 'tests.rest.suite', 'old_cluster' } - Task mixedClusterTest = tasks.create(name: "${baseName}#mixedClusterTest", type: RestIntegTestTask) + Closure configureUpgradeCluster = {String name, Task lastRunner, int stopNode, Closure unicastSeed -> + configure(extensions.findByName("${baseName}#${name}")) { + dependsOn lastRunner, "${baseName}#oldClusterTestCluster#node${stopNode}.stop" + clusterName = 'rolling-upgrade-basic' + unicastTransportUri = { seedNode, node, ant -> unicastSeed() } + minimumMasterNodes = { 3 } + /* Override the data directory so the new node always gets the node we + * just stopped's data directory. */ + dataDir = { nodeNumber -> oldClusterTest.nodes[stopNode].dataDir } + setting 'repositories.url.allowed_urls', 'http://snapshot.test*' + setting 'xpack.security.enabled', 'false' + setting 'xpack.monitoring.enabled', 'false' + setting 'xpack.ml.enabled', 'false' + setting 'xpack.watcher.enabled', 'false' + setting 'xpack.license.self_generated.type', 'basic' + } + } + + Task oneThirdUpgradedTest = tasks.create(name: "${baseName}#oneThirdUpgradedTest", type: RestIntegTestTask) - configure(extensions.findByName("${baseName}#mixedClusterTestCluster")) { - dependsOn oldClusterTestRunner, "${baseName}#oldClusterTestCluster#node1.stop" - clusterName = 'rolling-upgrade-basic' - unicastTransportUri = { seedNode, node, ant -> oldClusterTest.nodes.get(0).transportUri() } - minimumMasterNodes = { 2 } - dataDir = { nodeNumber -> oldClusterTest.nodes[1].dataDir } - waitCondition = waitWithAuth - setting 'xpack.security.enabled', 'false' - setting 'xpack.monitoring.enabled', 'false' - setting 'xpack.ml.enabled', 'false' - setting 'xpack.watcher.enabled', 'false' - setting 'xpack.license.self_generated.type', 'basic' - setting 'node.name', 'mixed-node-0' + configureUpgradeCluster("oneThirdUpgradedTestCluster", oldClusterTestRunner, + 0, { oldClusterTest.nodes.get(1).transportUri() }) + + Task oneThirdUpgradedTestRunner = tasks.getByName("${baseName}#oneThirdUpgradedTestRunner") + oneThirdUpgradedTestRunner.configure { + systemProperty 'tests.rest.suite', 'mixed_cluster' + systemProperty 'tests.first_round', 'true' + finalizedBy "${baseName}#oldClusterTestCluster#node1.stop" } - Task mixedClusterTestRunner = tasks.getByName("${baseName}#mixedClusterTestRunner") - mixedClusterTestRunner.configure { + Task twoThirdsUpgradedTest = tasks.create(name: "${baseName}#twoThirdsUpgradedTest", type: RestIntegTestTask) + + configureUpgradeCluster("twoThirdsUpgradedTestCluster", oneThirdUpgradedTestRunner, + 1, { oneThirdUpgradedTest.nodes.get(0).transportUri() }) + + Task twoThirdsUpgradedTestRunner = tasks.getByName("${baseName}#twoThirdsUpgradedTestRunner") + twoThirdsUpgradedTestRunner.configure { systemProperty 'tests.rest.suite', 'mixed_cluster' - finalizedBy "${baseName}#oldClusterTestCluster#node0.stop" + systemProperty 'tests.first_round', 'false' + finalizedBy "${baseName}#oldClusterTestCluster#node2.stop" } Task upgradedClusterTest = tasks.create(name: "${baseName}#upgradedClusterTest", type: RestIntegTestTask) - configure(extensions.findByName("${baseName}#upgradedClusterTestCluster")) { - dependsOn(mixedClusterTestRunner, "${baseName}#oldClusterTestCluster#node0.stop") - clusterName = 'rolling-upgrade-basic' - unicastTransportUri = { seedNode, node, ant -> mixedClusterTest.nodes.get(0).transportUri() } - minimumMasterNodes = { 2 } - dataDir = { nodeNumber -> oldClusterTest.nodes[0].dataDir } - waitCondition = waitWithAuth - setting 'xpack.security.enabled', 'false' - setting 'xpack.monitoring.enabled', 'false' - setting 'xpack.ml.enabled', 'false' - setting 'xpack.watcher.enabled', 'false' - setting 'xpack.license.self_generated.type', 'basic' - setting 'node.name', 'upgraded-node-0' - } + configureUpgradeCluster("upgradedClusterTestCluster", twoThirdsUpgradedTestRunner, + 2, { twoThirdsUpgradedTest.nodes.get(0).transportUri() }) Task upgradedClusterTestRunner = tasks.getByName("${baseName}#upgradedClusterTestRunner") upgradedClusterTestRunner.configure { systemProperty 'tests.rest.suite', 'upgraded_cluster' - // only need to kill the mixed cluster tests node here because we explicitly told it to not stop nodes upon completion - finalizedBy "${baseName}#mixedClusterTestCluster#stop" + /* + * Force stopping all the upgraded nodes after the test runner + * so they are alive during the test. + */ + finalizedBy "${baseName}#oneThirdUpgradedTestCluster#stop" + finalizedBy "${baseName}#twoThirdsUpgradedTestCluster#stop" } Task versionBwcTest = tasks.create(name: "${baseName}#bwcTest") { @@ -170,11 +127,6 @@ task integTest { } check.dependsOn(integTest) -dependencies { - testCompile project(path: xpackModule('core'), configuration: 'runtime') - testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') -} - compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked" // copy x-pack plugin info so it is on the classpath and security manager has the right permissions diff --git a/x-pack/qa/rolling-upgrade/build.gradle b/x-pack/qa/rolling-upgrade/build.gradle index 6e93041e9a0f4..f11addb42b810 100644 --- a/x-pack/qa/rolling-upgrade/build.gradle +++ b/x-pack/qa/rolling-upgrade/build.gradle @@ -30,7 +30,7 @@ Closure waitWithAuth = { NodeInfo node, AntBuilder ant -> HttpURLConnection httpURLConnection = null; try { // TODO this sucks having to hardcode number of nodes, but node.config.numNodes isn't necessarily accurate for rolling - httpURLConnection = (HttpURLConnection) new URL("http://${node.httpUri()}/_cluster/health?wait_for_nodes=2&wait_for_status=yellow").openConnection(); + httpURLConnection = (HttpURLConnection) new URL("http://${node.httpUri()}/_cluster/health?wait_for_nodes=3&wait_for_status=yellow").openConnection(); httpURLConnection.setRequestProperty("Authorization", "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8))); httpURLConnection.setRequestMethod("GET"); @@ -128,9 +128,9 @@ subprojects { String usersCli = version.before('6.3.0') ? 'bin/x-pack/users' : 'bin/elasticsearch-users' setupCommand 'setupTestUser', usersCli, 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser' bwcVersion = version - numBwcNodes = 2 - numNodes = 2 - minimumMasterNodes = { 2 } + numBwcNodes = 3 + numNodes = 3 + minimumMasterNodes = { 3 } clusterName = 'rolling-upgrade' waitCondition = waitWithAuth setting 'xpack.monitoring.exporters._http.type', 'http' @@ -167,78 +167,84 @@ subprojects { systemProperty 'tests.rest.suite', 'old_cluster' } - Task mixedClusterTest = tasks.create(name: "${baseName}#mixedClusterTest", type: RestIntegTestTask) - - configure(extensions.findByName("${baseName}#mixedClusterTestCluster")) { - dependsOn oldClusterTestRunner, "${baseName}#oldClusterTestCluster#node1.stop" - setupCommand 'setupTestUser', 'bin/elasticsearch-users', 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser' - clusterName = 'rolling-upgrade' - unicastTransportUri = { seedNode, node, ant -> oldClusterTest.nodes.get(0).transportUri() } - minimumMasterNodes = { 2 } - dataDir = { nodeNumber -> oldClusterTest.nodes[1].dataDir } - waitCondition = waitWithAuth - setting 'xpack.monitoring.exporters._http.type', 'http' - setting 'xpack.monitoring.exporters._http.enabled', 'false' - setting 'xpack.monitoring.exporters._http.auth.username', 'test_user' - setting 'xpack.monitoring.exporters._http.auth.password', 'x-pack-test-password' - setting 'xpack.license.self_generated.type', 'trial' - setting 'xpack.security.enabled', 'true' - setting 'xpack.security.transport.ssl.enabled', 'true' - setting 'xpack.ssl.keystore.path', 'testnode.jks' - keystoreSetting 'xpack.ssl.keystore.secure_password', 'testnode' - setting 'node.attr.upgraded', 'first' - setting 'xpack.security.authc.token.enabled', 'true' - setting 'xpack.security.audit.enabled', 'true' - setting 'xpack.security.audit.outputs', 'index' - setting 'node.name', 'mixed-node-0' - dependsOn copyTestNodeKeystore - extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks') - if (withSystemKey) { - setting 'xpack.watcher.encrypt_sensitive_data', 'true' - keystoreFile 'xpack.watcher.encryption_key', "${mainProject.projectDir}/src/test/resources/system_key" + Closure configureUpgradeCluster = {String name, Task lastRunner, int stopNode, Closure unicastSeed -> + configure(extensions.findByName("${baseName}#${name}")) { + dependsOn lastRunner, "${baseName}#oldClusterTestCluster#node${stopNode}.stop" + setupCommand 'setupTestUser', 'bin/elasticsearch-users', 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser' + clusterName = 'rolling-upgrade' + unicastTransportUri = { seedNode, node, ant -> unicastSeed() } + minimumMasterNodes = { 3 } + /* Override the data directory so the new node always gets the node we + * just stopped's data directory. */ + dataDir = { nodeNumber -> oldClusterTest.nodes[stopNode].dataDir } + waitCondition = waitWithAuth + setting 'xpack.monitoring.exporters._http.type', 'http' + setting 'xpack.monitoring.exporters._http.enabled', 'false' + setting 'xpack.monitoring.exporters._http.auth.username', 'test_user' + setting 'xpack.monitoring.exporters._http.auth.password', 'x-pack-test-password' + setting 'xpack.license.self_generated.type', 'trial' + setting 'xpack.security.enabled', 'true' + setting 'xpack.security.transport.ssl.enabled', 'true' + setting 'xpack.ssl.keystore.path', 'testnode.jks' + keystoreSetting 'xpack.ssl.keystore.secure_password', 'testnode' + setting 'node.attr.upgraded', 'true' + setting 'xpack.security.authc.token.enabled', 'true' + setting 'xpack.security.audit.enabled', 'true' + setting 'xpack.security.audit.outputs', 'index' + setting 'node.name', "upgraded-node-${stopNode}" + dependsOn copyTestNodeKeystore + extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks') + if (withSystemKey) { + setting 'xpack.watcher.encrypt_sensitive_data', 'true' + keystoreFile 'xpack.watcher.encryption_key', "${mainProject.projectDir}/src/test/resources/system_key" + } } } - Task mixedClusterTestRunner = tasks.getByName("${baseName}#mixedClusterTestRunner") - mixedClusterTestRunner.configure { + Task oneThirdUpgradedTest = tasks.create(name: "${baseName}#oneThirdUpgradedTest", type: RestIntegTestTask) + + configureUpgradeCluster("oneThirdUpgradedTestCluster", oldClusterTestRunner, + 0, { oldClusterTest.nodes.get(1).transportUri() }) + + Task oneThirdUpgradedTestRunner = tasks.getByName("${baseName}#oneThirdUpgradedTestRunner") + oneThirdUpgradedTestRunner.configure { systemProperty 'tests.rest.suite', 'mixed_cluster' - finalizedBy "${baseName}#oldClusterTestCluster#node0.stop" + systemProperty 'tests.first_round', 'true' + // We only need to run these tests once so we may as well do it when we're two thirds upgraded + systemProperty 'tests.rest.blacklist', [ + 'mixed_cluster/10_basic/Start scroll in mixed cluster on upgraded node that we will continue after upgrade', + 'mixed_cluster/30_ml_jobs_crud/Create a job in the mixed cluster and write some data', + 'mixed_cluster/40_ml_datafeed_crud/Put job and datafeed in mixed cluster', + ].join(',') + finalizedBy "${baseName}#oldClusterTestCluster#node1.stop" } - Task upgradedClusterTest = tasks.create(name: "${baseName}#upgradedClusterTest", type: RestIntegTestTask) + Task twoThirdsUpgradedTest = tasks.create(name: "${baseName}#twoThirdsUpgradedTest", type: RestIntegTestTask) - configure(extensions.findByName("${baseName}#upgradedClusterTestCluster")) { - dependsOn(mixedClusterTestRunner, "${baseName}#oldClusterTestCluster#node0.stop") - setupCommand 'setupTestUser', 'bin/elasticsearch-users', 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser' - clusterName = 'rolling-upgrade' - unicastTransportUri = { seedNode, node, ant -> mixedClusterTest.nodes.get(0).transportUri() } - minimumMasterNodes = { 2 } - dataDir = { nodeNumber -> oldClusterTest.nodes[0].dataDir } - waitCondition = waitWithAuth - setting 'xpack.monitoring.exporters._http.type', 'http' - setting 'xpack.monitoring.exporters._http.enabled', 'false' - setting 'xpack.monitoring.exporters._http.auth.username', 'test_user' - setting 'xpack.monitoring.exporters._http.auth.password', 'x-pack-test-password' - setting 'xpack.license.self_generated.type', 'trial' - setting 'xpack.security.enabled', 'true' - setting 'xpack.security.transport.ssl.enabled', 'true' - setting 'xpack.ssl.keystore.path', 'testnode.jks' - keystoreSetting 'xpack.ssl.keystore.secure_password', 'testnode' - setting 'xpack.security.authc.token.enabled', 'true' - setting 'xpack.security.audit.enabled', 'true' - setting 'xpack.security.audit.outputs', 'index' - setting 'node.name', 'upgraded-node-0' - dependsOn copyTestNodeKeystore - extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks') - if (withSystemKey) { - setting 'xpack.watcher.encrypt_sensitive_data', 'true' - keystoreFile 'xpack.watcher.encryption_key', "${mainProject.projectDir}/src/test/resources/system_key" - } + configureUpgradeCluster("twoThirdsUpgradedTestCluster", oneThirdUpgradedTestRunner, + 1, { oneThirdUpgradedTest.nodes.get(0).transportUri() }) + + Task twoThirdsUpgradedTestRunner = tasks.getByName("${baseName}#twoThirdsUpgradedTestRunner") + twoThirdsUpgradedTestRunner.configure { + systemProperty 'tests.rest.suite', 'mixed_cluster' + systemProperty 'tests.first_round', 'false' + finalizedBy "${baseName}#oldClusterTestCluster#node2.stop" } + Task upgradedClusterTest = tasks.create(name: "${baseName}#upgradedClusterTest", type: RestIntegTestTask) + + configureUpgradeCluster("upgradedClusterTestCluster", twoThirdsUpgradedTestRunner, + 2, { twoThirdsUpgradedTest.nodes.get(0).transportUri() }) + Task upgradedClusterTestRunner = tasks.getByName("${baseName}#upgradedClusterTestRunner") upgradedClusterTestRunner.configure { systemProperty 'tests.rest.suite', 'upgraded_cluster' + /* + * Force stopping all the upgraded nodes after the test runner + * so they are alive during the test. + */ + finalizedBy "${baseName}#oneThirdUpgradedTestCluster#stop" + finalizedBy "${baseName}#twoThirdsUpgradedTestCluster#stop" // migration tests should only run when the original/old cluster nodes where versions < 5.2.0. // this stinks but we do the check here since our rest tests do not support conditionals @@ -251,8 +257,6 @@ subprojects { systemProperty 'tests.rest.blacklist', '/20_security/Verify default password migration results in upgraded cluster' } } - // only need to kill the mixed cluster tests node here because we explicitly told it to not stop nodes upon completion - finalizedBy "${baseName}#mixedClusterTestCluster#stop" } Task versionBwcTest = tasks.create(name: "${baseName}#bwcTest") { diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java index 65b1a7c85dcb1..a3576b7b8c3e8 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java @@ -37,12 +37,12 @@ protected boolean preserveTemplatesUponCompletion() { return true; } - enum CLUSTER_TYPE { + enum ClusterType { OLD, MIXED, UPGRADED; - public static CLUSTER_TYPE parse(String value) { + public static ClusterType parse(String value) { switch (value) { case "old_cluster": return OLD; @@ -56,7 +56,7 @@ public static CLUSTER_TYPE parse(String value) { } } - protected final CLUSTER_TYPE clusterType = CLUSTER_TYPE.parse(System.getProperty("tests.rest.suite")); + protected static final ClusterType CLUSTER_TYPE = ClusterType.parse(System.getProperty("tests.rest.suite")); @Override protected Settings restClientSettings() { diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexAuditUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexAuditUpgradeIT.java index 1f76e670854a2..da6f9133d033b 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexAuditUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexAuditUpgradeIT.java @@ -8,37 +8,48 @@ import org.apache.http.HttpEntity; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; +import org.elasticsearch.Version; import org.elasticsearch.client.Response; +import org.elasticsearch.common.Booleans; import org.hamcrest.Matchers; +import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.Matchers.hasSize; public class IndexAuditUpgradeIT extends AbstractUpgradeTestCase { - public void testDocsAuditedInOldCluster() throws Exception { - assumeTrue("only runs against old cluster", clusterType == CLUSTER_TYPE.OLD); + public void testAuditLogs() throws Exception { assertBusy(() -> { assertAuditDocsExist(); - assertNumUniqueNodeNameBuckets(2); - }); + assertNumUniqueNodeNameBuckets(expectedNumUniqueNodeNameBuckets()); + }, 1, TimeUnit.HOURS); } - public void testDocsAuditedInMixedCluster() throws Exception { - assumeTrue("only runs against mixed cluster", clusterType == CLUSTER_TYPE.MIXED); - assertBusy(() -> { - assertAuditDocsExist(); - assertNumUniqueNodeNameBuckets(2); - }); - } - - public void testDocsAuditedInUpgradedCluster() throws Exception { - assumeTrue("only runs against upgraded cluster", clusterType == CLUSTER_TYPE.UPGRADED); - assertBusy(() -> { - assertAuditDocsExist(); - assertNumUniqueNodeNameBuckets(4); - }); + private int expectedNumUniqueNodeNameBuckets() throws IOException { + switch (CLUSTER_TYPE) { + case OLD: + // There are three nodes in the initial test cluster + return 3; + case MIXED: + if (false == masterIsNewVersion()) { + return 3; + } + if (Booleans.parseBoolean(System.getProperty("tests.first_round"))) { + // One of the old nodes has been removed and we've added a new node + return 4; + } + // Two of the old nodes have been removed and we've added two new nodes + return 5; + case UPGRADED: + return 6; + default: + throw new IllegalArgumentException("Unsupported cluster type [" + CLUSTER_TYPE + "]"); + } } private void assertAuditDocsExist() throws Exception { @@ -51,26 +62,40 @@ private void assertAuditDocsExist() throws Exception { private void assertNumUniqueNodeNameBuckets(int numBuckets) throws Exception { // call API that will hit all nodes - assertEquals(200, client().performRequest("GET", "/_nodes").getStatusLine().getStatusCode()); + Map nodesResponse = entityAsMap(client().performRequest("GET", "/_nodes/_all/info/version")); + logger.info("all nodes {}", nodesResponse); HttpEntity httpEntity = new StringEntity( "{\n" + - " \"aggs\" : {\n" + - " \"nodes\" : {\n" + - " \"terms\" : { \"field\" : \"node_name\" }\n" + - " }\n" + - " }\n" + - "}", ContentType.APPLICATION_JSON); + " \"aggs\" : {\n" + + " \"nodes\" : {\n" + + " \"terms\" : { \"field\" : \"node_name\" }\n" + + " }\n" + + " }\n" + + "}", ContentType.APPLICATION_JSON); Response aggResponse = client().performRequest("GET", "/.security_audit_log*/_search", Collections.singletonMap("pretty", "true"), httpEntity); Map aggResponseMap = entityAsMap(aggResponse); logger.debug("aggResponse {}", aggResponseMap); - Map aggregations = (Map) aggResponseMap.get("aggregations"); + Map aggregations = (Map) aggResponseMap.get("aggregations"); assertNotNull(aggregations); - Map nodesAgg = (Map) aggregations.get("nodes"); + Map nodesAgg = (Map) aggregations.get("nodes"); assertNotNull(nodesAgg); - List> buckets = (List>) nodesAgg.get("buckets"); + List buckets = (List) nodesAgg.get("buckets"); assertNotNull(buckets); - assertEquals("Found node buckets " + buckets, numBuckets, buckets.size()); + assertThat("Found node buckets " + buckets, buckets, hasSize(numBuckets)); + } + + /** + * Has the master been upgraded to the new version? + * @throws IOException + */ + private boolean masterIsNewVersion() throws IOException { + Map map = entityAsMap(client().performRequest("GET", "/_nodes/_master")); + map = (Map) map.get("nodes"); + assertThat(map.values(), hasSize(1)); + map = (Map) map.values().iterator().next(); + Version masterVersion = Version.fromString(map.get("version").toString()); + return Version.CURRENT.equals(masterVersion); } } diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java new file mode 100644 index 0000000000000..3448117cd2c88 --- /dev/null +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.upgrades; + +import org.apache.http.util.EntityUtils; +import org.elasticsearch.common.Booleans; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +/** + * Basic test that indexed documents survive the rolling restart. + *

+ * This test is an almost exact copy of IndexingIT in the + * oss rolling restart tests. We should work on a way to remove this + * duplication but for now we have no real way to share code. + */ +public class IndexingIT extends AbstractUpgradeTestCase { + public void testIndexing() throws IOException { + switch (CLUSTER_TYPE) { + case OLD: + break; + case MIXED: + Request waitForYellow = new Request("GET", "/_cluster/health"); + waitForYellow.addParameter("wait_for_nodes", "3"); + waitForYellow.addParameter("wait_for_status", "yellow"); + client().performRequest(waitForYellow); + break; + case UPGRADED: + Request waitForGreen = new Request("GET", "/_cluster/health/test_index,index_with_replicas,empty_index"); + waitForGreen.addParameter("wait_for_nodes", "3"); + waitForGreen.addParameter("wait_for_status", "green"); + // wait for long enough that we give delayed unassigned shards to stop being delayed + waitForGreen.addParameter("timeout", "70s"); + waitForGreen.addParameter("level", "shards"); + client().performRequest(waitForGreen); + break; + default: + throw new UnsupportedOperationException("Unknown cluster type [" + CLUSTER_TYPE + "]"); + } + + if (CLUSTER_TYPE == ClusterType.OLD) { + Request createTestIndex = new Request("PUT", "/test_index"); + createTestIndex.setJsonEntity("{\"settings\": {\"index.number_of_replicas\": 0}}"); + client().performRequest(createTestIndex); + + String recoverQuickly = "{\"settings\": {\"index.unassigned.node_left.delayed_timeout\": \"100ms\"}}"; + Request createIndexWithReplicas = new Request("PUT", "/index_with_replicas"); + createIndexWithReplicas.setJsonEntity(recoverQuickly); + client().performRequest(createIndexWithReplicas); + + Request createEmptyIndex = new Request("PUT", "/empty_index"); + // Ask for recovery to be quick + createEmptyIndex.setJsonEntity(recoverQuickly); + client().performRequest(createEmptyIndex); + + bulk("test_index", "_OLD", 5); + bulk("index_with_replicas", "_OLD", 5); + } + + int expectedCount; + switch (CLUSTER_TYPE) { + case OLD: + expectedCount = 5; + break; + case MIXED: + if (Booleans.parseBoolean(System.getProperty("tests.first_round"))) { + expectedCount = 5; + } else { + expectedCount = 10; + } + break; + case UPGRADED: + expectedCount = 15; + break; + default: + throw new UnsupportedOperationException("Unknown cluster type [" + CLUSTER_TYPE + "]"); + } + + assertCount("test_index", expectedCount); + assertCount("index_with_replicas", 5); + assertCount("empty_index", 0); + + if (CLUSTER_TYPE != ClusterType.OLD) { + bulk("test_index", "_" + CLUSTER_TYPE, 5); + Request toBeDeleted = new Request("PUT", "/test_index/doc/to_be_deleted"); + toBeDeleted.addParameter("refresh", "true"); + toBeDeleted.setJsonEntity("{\"f1\": \"delete-me\"}"); + client().performRequest(toBeDeleted); + assertCount("test_index", expectedCount + 6); + + Request delete = new Request("DELETE", "/test_index/doc/to_be_deleted"); + delete.addParameter("refresh", "true"); + client().performRequest(delete); + + assertCount("test_index", expectedCount + 5); + } + } + + private void bulk(String index, String valueSuffix, int count) throws IOException { + StringBuilder b = new StringBuilder(); + for (int i = 0; i < count; i++) { + b.append("{\"index\": {\"_index\": \"").append(index).append("\", \"_type\": \"doc\"}}\n"); + b.append("{\"f1\": \"v").append(i).append(valueSuffix).append("\", \"f2\": ").append(i).append("}\n"); + } + Request bulk = new Request("POST", "/_bulk"); + bulk.addParameter("refresh", "true"); + bulk.setJsonEntity(b.toString()); + client().performRequest(bulk); + } + + private void assertCount(String index, int count) throws IOException { + Request searchTestIndexRequest = new Request("POST", "/" + index + "/_search"); + searchTestIndexRequest.addParameter("filter_path", "hits.total"); + Response searchTestIndexResponse = client().performRequest(searchTestIndexRequest); + assertEquals("{\"hits\":{\"total\":" + count + "}}", + EntityUtils.toString(searchTestIndexResponse.getEntity(), StandardCharsets.UTF_8)); + } +} diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java index 4fa0c9a535f6c..705122252e7c3 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java @@ -25,7 +25,7 @@ public class TokenBackwardsCompatibilityIT extends AbstractUpgradeTestCase { public void testGeneratingTokenInOldCluster() throws Exception { - assumeTrue("this test should only run against the old cluster", clusterType == CLUSTER_TYPE.OLD); + assumeTrue("this test should only run against the old cluster", CLUSTER_TYPE == ClusterType.OLD); final StringEntity tokenPostBody = new StringEntity("{\n" + " \"username\": \"test_user\",\n" + " \"password\": \"x-pack-test-password\",\n" + @@ -61,7 +61,7 @@ public void testGeneratingTokenInOldCluster() throws Exception { public void testTokenWorksInMixedOrUpgradedCluster() throws Exception { assumeTrue("this test should only run against the mixed or upgraded cluster", - clusterType == CLUSTER_TYPE.MIXED || clusterType == CLUSTER_TYPE.UPGRADED); + CLUSTER_TYPE == ClusterType.MIXED || CLUSTER_TYPE == ClusterType.UPGRADED); Response getResponse = client().performRequest("GET", "token_backwards_compatibility_it/doc/old_cluster_token1"); assertOK(getResponse); Map source = (Map) entityAsMap(getResponse).get("_source"); @@ -69,7 +69,7 @@ public void testTokenWorksInMixedOrUpgradedCluster() throws Exception { } public void testMixedCluster() throws Exception { - assumeTrue("this test should only run against the mixed cluster", clusterType == CLUSTER_TYPE.MIXED); + assumeTrue("this test should only run against the mixed cluster", CLUSTER_TYPE == ClusterType.MIXED); assumeTrue("the master must be on the latest version before we can write", isMasterOnLatestVersion()); Response getResponse = client().performRequest("GET", "token_backwards_compatibility_it/doc/old_cluster_token2"); assertOK(getResponse); @@ -117,7 +117,7 @@ public void testMixedCluster() throws Exception { } public void testUpgradedCluster() throws Exception { - assumeTrue("this test should only run against the mixed cluster", clusterType == CLUSTER_TYPE.UPGRADED); + assumeTrue("this test should only run against the mixed cluster", CLUSTER_TYPE == ClusterType.UPGRADED); Response getResponse = client().performRequest("GET", "token_backwards_compatibility_it/doc/old_cluster_token2"); assertOK(getResponse); Map source = (Map) entityAsMap(getResponse).get("_source"); diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml index 93db3996a6ba9..3dd1f708959ed 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml @@ -1,166 +1,13 @@ --- -setup: - - do: - cluster.health: - # if the primary shard of an index with (number_of_replicas > 0) ends up on the new node, the replica cannot be - # allocated to the old node (see NodeVersionAllocationDecider). x-pack automatically creates indices with - # replicas, for example monitoring-data-*. - wait_for_status: yellow - wait_for_nodes: 2 - ---- -"Index data and search on the mixed cluster": - - do: - search: - index: test_index - - - match: { hits.total: 5 } # no new indexed data, so expect the original 5 documents from the old cluster - - - do: - bulk: - refresh: true - body: - - '{"index": {"_index": "test_index", "_type": "test_type"}}' - - '{"f1": "v1_mixed", "f2": 5}' - - '{"index": {"_index": "test_index", "_type": "test_type"}}' - - '{"f1": "v2_mixed", "f2": 6}' - - '{"index": {"_index": "test_index", "_type": "test_type"}}' - - '{"f1": "v3_mixed", "f2": 7}' - - '{"index": {"_index": "test_index", "_type": "test_type"}}' - - '{"f1": "v4_mixed", "f2": 8}' - - '{"index": {"_index": "test_index", "_type": "test_type"}}' - - '{"f1": "v5_mixed", "f2": 9}' - - - do: - index: - index: test_index - type: test_type - id: d10 - body: {"f1": "v6_mixed", "f2": 10} - - - do: - index: - index: test_index - type: test_type - id: d11 - body: {"f1": "v7_mixed", "f2": 11} - - - do: - index: - index: test_index - type: test_type - id: d12 - body: {"f1": "v8_mixed", "f2": 12} - - - do: - indices.refresh: - index: test_index - - - do: - search: - index: test_index - - - match: { hits.total: 13 } # 5 docs from old cluster, 8 docs from mixed cluster - - - do: - delete: - index: test_index - type: test_type - id: d10 - - - do: - delete: - index: test_index - type: test_type - id: d11 - - - do: - delete: - index: test_index - type: test_type - id: d12 - - - do: - indices.refresh: - index: test_index - ---- -"Basic scroll mixed": - - do: - indices.create: - index: test_scroll - - do: - index: - index: test_scroll - type: test - id: 42 - body: { foo: 1 } - - - do: - index: - index: test_scroll - type: test - id: 43 - body: { foo: 2 } - - - do: - indices.refresh: {} - - - do: - search: - index: test_scroll - size: 1 - scroll: 1m - sort: foo - body: - query: - match_all: {} - - - set: {_scroll_id: scroll_id} - - match: {hits.total: 2 } - - length: {hits.hits: 1 } - - match: {hits.hits.0._id: "42" } - - - do: - index: - index: test_scroll - type: test - id: 44 - body: { foo: 3 } - - - do: - indices.refresh: {} - - - do: - scroll: - body: { "scroll_id": "$scroll_id", "scroll": "1m"} - - - match: {hits.total: 2 } - - length: {hits.hits: 1 } - - match: {hits.hits.0._id: "43" } - - - do: - scroll: - scroll_id: $scroll_id - scroll: 1m - - - match: {hits.total: 2 } - - length: {hits.hits: 0 } - - - do: - clear_scroll: - scroll_id: $scroll_id - ---- -"Start scroll in mixed cluster for upgraded": +"Start scroll in mixed cluster on upgraded node that we will continue after upgrade": - do: indices.create: index: upgraded_scroll wait_for_active_shards: all body: settings: - number_of_replicas: "0" - index.routing.allocation.include.upgraded: "first" + number_of_replicas: 0 + index.routing.allocation.include.upgraded: true - do: index: diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/20_security.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/20_security.yml index 750bedc4c6df9..cfe3ca9733044 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/20_security.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/20_security.yml @@ -1,13 +1,5 @@ --- "Verify user and role in mixed cluster": - - do: - headers: - Authorization: "Basic bmF0aXZlX3VzZXI6eC1wYWNrLXRlc3QtcGFzc3dvcmQ=" - cluster.health: - wait_for_status: yellow - wait_for_nodes: 2 - - match: { timed_out: false } - - do: xpack.security.get_user: username: "native_user" @@ -36,6 +28,3 @@ username: "kibana,logstash_system" - match: { kibana.enabled: false } - match: { logstash_system.enabled: true } - - - diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml index daf2f913fff0a..6ea8771c2374e 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml @@ -1,10 +1,3 @@ ---- -setup: - - do: - cluster.health: - wait_for_status: yellow - wait_for_nodes: 2 - --- "Test get old cluster job": - skip: diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/40_ml_datafeed_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/40_ml_datafeed_crud.yml index 8a06c91cc8a01..0ec288f90973c 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/40_ml_datafeed_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/40_ml_datafeed_crud.yml @@ -1,9 +1,3 @@ -setup: - - do: - cluster.health: - wait_for_status: yellow - wait_for_nodes: 2 - --- "Test old cluster datafeed": - do: diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml deleted file mode 100644 index a780709400a29..0000000000000 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- -"Index data and search on the old cluster": - - do: - indices.create: - index: test_index - wait_for_active_shards : all - body: - settings: - index: - number_of_replicas: 1 - - - do: - bulk: - refresh: true - body: - - '{"index": {"_index": "test_index", "_type": "test_type"}}' - - '{"f1": "v1_old", "f2": 0}' - - '{"index": {"_index": "test_index", "_type": "test_type"}}' - - '{"f1": "v2_old", "f2": 1}' - - '{"index": {"_index": "test_index", "_type": "test_type"}}' - - '{"f1": "v3_old", "f2": 2}' - - '{"index": {"_index": "test_index", "_type": "test_type"}}' - - '{"f1": "v4_old", "f2": 3}' - - '{"index": {"_index": "test_index", "_type": "test_type"}}' - - '{"f1": "v5_old", "f2": 4}' - - - do: - search: - index: test_index - - - match: { hits.total: 5 } diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml index 9c3443339a7cd..7249b4a32c755 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml @@ -1,42 +1,5 @@ --- -"Index data and search on the upgraded cluster": - - do: - cluster.health: - wait_for_status: green - wait_for_nodes: 2 - # wait for long enough that we give delayed unassigned shards to stop being delayed - timeout: 70s - level: shards - - - do: - search: - index: test_index - - - match: { hits.total: 10 } # no new indexed data, so expect the original 10 documents from the old and mixed clusters - - - do: - bulk: - refresh: true - body: - - '{"index": {"_index": "test_index", "_type": "test_type"}}' - - '{"f1": "v1_upgraded", "f2": 10}' - - '{"index": {"_index": "test_index", "_type": "test_type"}}' - - '{"f1": "v2_upgraded", "f2": 11}' - - '{"index": {"_index": "test_index", "_type": "test_type"}}' - - '{"f1": "v3_upgraded", "f2": 12}' - - '{"index": {"_index": "test_index", "_type": "test_type"}}' - - '{"f1": "v4_upgraded", "f2": 13}' - - '{"index": {"_index": "test_index", "_type": "test_type"}}' - - '{"f1": "v5_upgraded", "f2": 14}' - - - do: - search: - index: test_index - - - match: { hits.total: 15 } # 10 docs from previous clusters plus 5 new docs - ---- -"Get indexed scroll and execute scroll": +"Continue scroll after upgrade": - do: get: index: scroll_index diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/20_security.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/20_security.yml index 9c7097483914b..46ade4823a221 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/20_security.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/20_security.yml @@ -5,7 +5,7 @@ Authorization: "Basic bmF0aXZlX3VzZXI6eC1wYWNrLXRlc3QtcGFzc3dvcmQ=" cluster.health: wait_for_status: green - wait_for_nodes: 2 + wait_for_nodes: 3 # wait for long enough that we give delayed unassigned shards to stop being delayed timeout: 70s - match: { timed_out: false } @@ -22,4 +22,3 @@ - match: { native_role.cluster.0: "all" } - match: { native_role.indices.0.names.0: "test_index" } - match: { native_role.indices.0.privileges.0: "all" } - diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_ml_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_ml_jobs_crud.yml index 9520e954d7b48..91d294572894d 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_ml_jobs_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_ml_jobs_crud.yml @@ -2,7 +2,7 @@ setup: - do: cluster.health: wait_for_status: green - wait_for_nodes: 2 + wait_for_nodes: 3 # wait for long enough that we give delayed unassigned shards to stop being delayed timeout: 70s diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/40_ml_datafeed_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/40_ml_datafeed_crud.yml index ed6a66ae1a51f..6b4c963dd533b 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/40_ml_datafeed_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/40_ml_datafeed_crud.yml @@ -2,7 +2,7 @@ setup: - do: cluster.health: wait_for_status: green - wait_for_nodes: 2 + wait_for_nodes: 3 # wait for long enough that we give delayed unassigned shards to stop being delayed timeout: 70s @@ -97,4 +97,3 @@ setup: xpack.ml.delete_job: job_id: mixed-cluster-datafeed-job - match: { acknowledged: true } - From 45537c59e59287216099138f14cf58c5416503d0 Mon Sep 17 00:00:00 2001 From: Lisa Cawley Date: Wed, 6 Jun 2018 10:05:32 -0700 Subject: [PATCH 20/26] [DOCS] Moves X-Pack settings to docs folder (#31120) --- docs/reference/index.asciidoc | 2 +- .../reference}/settings/audit-settings.asciidoc | 0 .../reference}/settings/configuring-xes.asciidoc | 0 .../settings/images/monitoring-es-cgroup-true.png | Bin .../reference}/settings/license-settings.asciidoc | 0 .../reference}/settings/ml-settings.asciidoc | 0 .../settings/monitoring-settings.asciidoc | 0 .../settings/notification-settings.asciidoc | 0 .../reference}/settings/security-settings.asciidoc | 0 .../reference}/settings/sql-settings.asciidoc | 0 .../reference}/settings/ssl-settings.asciidoc | 0 .../en/monitoring/configuring-monitoring.asciidoc | 2 +- x-pack/docs/en/security/configuring-es.asciidoc | 4 ++-- 13 files changed, 4 insertions(+), 4 deletions(-) rename {x-pack/docs/en => docs/reference}/settings/audit-settings.asciidoc (100%) rename {x-pack/docs/en => docs/reference}/settings/configuring-xes.asciidoc (100%) rename {x-pack/docs/en => docs/reference}/settings/images/monitoring-es-cgroup-true.png (100%) rename {x-pack/docs/en => docs/reference}/settings/license-settings.asciidoc (100%) rename {x-pack/docs/en => docs/reference}/settings/ml-settings.asciidoc (100%) rename {x-pack/docs/en => docs/reference}/settings/monitoring-settings.asciidoc (100%) rename {x-pack/docs/en => docs/reference}/settings/notification-settings.asciidoc (100%) rename {x-pack/docs/en => docs/reference}/settings/security-settings.asciidoc (100%) rename {x-pack/docs/en => docs/reference}/settings/sql-settings.asciidoc (100%) rename {x-pack/docs/en => docs/reference}/settings/ssl-settings.asciidoc (100%) diff --git a/docs/reference/index.asciidoc b/docs/reference/index.asciidoc index 11006d38976b5..17cdde32b0716 100644 --- a/docs/reference/index.asciidoc +++ b/docs/reference/index.asciidoc @@ -22,7 +22,7 @@ include::{xes-repo-dir}/security/configuring-es.asciidoc[] include::{xes-repo-dir}/setup/setup-xclient.asciidoc[] -include::{xes-repo-dir}/settings/configuring-xes.asciidoc[] +include::settings/configuring-xes.asciidoc[] include::{xes-repo-dir}/setup/bootstrap-checks-xes.asciidoc[] diff --git a/x-pack/docs/en/settings/audit-settings.asciidoc b/docs/reference/settings/audit-settings.asciidoc similarity index 100% rename from x-pack/docs/en/settings/audit-settings.asciidoc rename to docs/reference/settings/audit-settings.asciidoc diff --git a/x-pack/docs/en/settings/configuring-xes.asciidoc b/docs/reference/settings/configuring-xes.asciidoc similarity index 100% rename from x-pack/docs/en/settings/configuring-xes.asciidoc rename to docs/reference/settings/configuring-xes.asciidoc diff --git a/x-pack/docs/en/settings/images/monitoring-es-cgroup-true.png b/docs/reference/settings/images/monitoring-es-cgroup-true.png similarity index 100% rename from x-pack/docs/en/settings/images/monitoring-es-cgroup-true.png rename to docs/reference/settings/images/monitoring-es-cgroup-true.png diff --git a/x-pack/docs/en/settings/license-settings.asciidoc b/docs/reference/settings/license-settings.asciidoc similarity index 100% rename from x-pack/docs/en/settings/license-settings.asciidoc rename to docs/reference/settings/license-settings.asciidoc diff --git a/x-pack/docs/en/settings/ml-settings.asciidoc b/docs/reference/settings/ml-settings.asciidoc similarity index 100% rename from x-pack/docs/en/settings/ml-settings.asciidoc rename to docs/reference/settings/ml-settings.asciidoc diff --git a/x-pack/docs/en/settings/monitoring-settings.asciidoc b/docs/reference/settings/monitoring-settings.asciidoc similarity index 100% rename from x-pack/docs/en/settings/monitoring-settings.asciidoc rename to docs/reference/settings/monitoring-settings.asciidoc diff --git a/x-pack/docs/en/settings/notification-settings.asciidoc b/docs/reference/settings/notification-settings.asciidoc similarity index 100% rename from x-pack/docs/en/settings/notification-settings.asciidoc rename to docs/reference/settings/notification-settings.asciidoc diff --git a/x-pack/docs/en/settings/security-settings.asciidoc b/docs/reference/settings/security-settings.asciidoc similarity index 100% rename from x-pack/docs/en/settings/security-settings.asciidoc rename to docs/reference/settings/security-settings.asciidoc diff --git a/x-pack/docs/en/settings/sql-settings.asciidoc b/docs/reference/settings/sql-settings.asciidoc similarity index 100% rename from x-pack/docs/en/settings/sql-settings.asciidoc rename to docs/reference/settings/sql-settings.asciidoc diff --git a/x-pack/docs/en/settings/ssl-settings.asciidoc b/docs/reference/settings/ssl-settings.asciidoc similarity index 100% rename from x-pack/docs/en/settings/ssl-settings.asciidoc rename to docs/reference/settings/ssl-settings.asciidoc diff --git a/x-pack/docs/en/monitoring/configuring-monitoring.asciidoc b/x-pack/docs/en/monitoring/configuring-monitoring.asciidoc index 99c69eeea8aec..1712c88380b7a 100644 --- a/x-pack/docs/en/monitoring/configuring-monitoring.asciidoc +++ b/x-pack/docs/en/monitoring/configuring-monitoring.asciidoc @@ -142,4 +142,4 @@ stored, that is to say the monitoring cluster. To grant all of the necessary per <>. include::indices.asciidoc[] -include::{xes-repo-dir}/settings/monitoring-settings.asciidoc[] \ No newline at end of file +include::{es-repo-dir}/settings/monitoring-settings.asciidoc[] \ No newline at end of file diff --git a/x-pack/docs/en/security/configuring-es.asciidoc b/x-pack/docs/en/security/configuring-es.asciidoc index d8ef6c2809b34..5e8f1adbc7aa8 100644 --- a/x-pack/docs/en/security/configuring-es.asciidoc +++ b/x-pack/docs/en/security/configuring-es.asciidoc @@ -142,5 +142,5 @@ include::authentication/configuring-ldap-realm.asciidoc[] include::authentication/configuring-native-realm.asciidoc[] include::authentication/configuring-pki-realm.asciidoc[] include::authentication/configuring-saml-realm.asciidoc[] -include::{xes-repo-dir}/settings/security-settings.asciidoc[] -include::{xes-repo-dir}/settings/audit-settings.asciidoc[] +include::{es-repo-dir}/settings/security-settings.asciidoc[] +include::{es-repo-dir}/settings/audit-settings.asciidoc[] From dc4bb62a7893f15cd5d48eb710bf95359a3c6177 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 6 Jun 2018 13:51:54 -0400 Subject: [PATCH 21/26] QA: Remove mistaken timeout I pushed a test that `assertBusy`s for a whole hour accidentally. I was testing something and forgot to revert my local hack but caught it on backport. This removes it. --- .../java/org/elasticsearch/upgrades/IndexAuditUpgradeIT.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexAuditUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexAuditUpgradeIT.java index da6f9133d033b..7c81a7141a991 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexAuditUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexAuditUpgradeIT.java @@ -17,7 +17,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.concurrent.TimeUnit; import static org.hamcrest.Matchers.hasSize; @@ -27,7 +26,7 @@ public void testAuditLogs() throws Exception { assertBusy(() -> { assertAuditDocsExist(); assertNumUniqueNodeNameBuckets(expectedNumUniqueNodeNameBuckets()); - }, 1, TimeUnit.HOURS); + }); } private int expectedNumUniqueNodeNameBuckets() throws IOException { From 67e73b4df481f3902a4aeee78d34532297ff7b6b Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Wed, 6 Jun 2018 11:59:54 -0600 Subject: [PATCH 22/26] Combine accepting selector and socket selector (#31115) This is related to #27260. This commit combines the AcceptingSelector and SocketSelector classes into a single NioSelector. This change allows the same selector to handle both server and socket channels. This is valuable as we do not necessarily want a dedicated thread running for accepting channels. With this change, this commit removes the configuration for dedicated accepting selectors for the normal transport class. The accepting workload for new node connections is likely low, meaning that there is no need to dedicate a thread to this process. --- .../elasticsearch/nio/AcceptingSelector.java | 98 ---- .../nio/AcceptorEventHandler.java | 81 ---- .../nio/BytesChannelContext.java | 2 +- .../org/elasticsearch/nio/ChannelContext.java | 2 +- .../org/elasticsearch/nio/ChannelFactory.java | 24 +- .../org/elasticsearch/nio/ESSelector.java | 248 ---------- .../org/elasticsearch/nio/EventHandler.java | 152 ++++++- .../org/elasticsearch/nio/NioChannel.java | 3 +- .../java/org/elasticsearch/nio/NioGroup.java | 109 +++-- .../org/elasticsearch/nio/NioSelector.java | 428 ++++++++++++++++++ .../elasticsearch/nio/RoundRobinSupplier.java | 28 +- .../nio/ServerChannelContext.java | 8 +- .../nio/SocketChannelContext.java | 8 +- .../elasticsearch/nio/SocketEventHandler.java | 150 ------ .../org/elasticsearch/nio/SocketSelector.java | 224 --------- .../nio/AcceptingSelectorTests.java | 129 ------ .../nio/AcceptorEventHandlerTests.java | 123 ----- .../nio/BytesChannelContextTests.java | 4 +- .../nio/ChannelContextTests.java | 2 +- .../nio/ChannelFactoryTests.java | 16 +- .../elasticsearch/nio/ESSelectorTests.java | 114 ----- ...ndlerTests.java => EventHandlerTests.java} | 107 ++++- .../org/elasticsearch/nio/NioGroupTests.java | 9 +- ...lectorTests.java => NioSelectorTests.java} | 198 ++++++-- .../nio/SocketChannelContextTests.java | 6 +- .../http/nio/NioHttpServerTransport.java | 13 +- .../transport/nio/NioTransport.java | 28 +- .../transport/nio/NioTransportPlugin.java | 3 +- .../nio/TcpNioServerSocketChannel.java | 2 - .../transport/nio/TcpNioSocketChannel.java | 1 - .../elasticsearch/transport/Transports.java | 4 +- .../transport/nio/MockNioTransport.java | 26 +- .../nio/TestingSocketEventHandler.java | 10 +- .../transport/nio/SSLChannelContext.java | 6 +- .../transport/nio/SecurityNioTransport.java | 7 +- .../transport/nio/SSLChannelContextTests.java | 6 +- 36 files changed, 987 insertions(+), 1392 deletions(-) delete mode 100644 libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/AcceptingSelector.java delete mode 100644 libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/AcceptorEventHandler.java delete mode 100644 libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/ESSelector.java create mode 100644 libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/NioSelector.java delete mode 100644 libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/SocketEventHandler.java delete mode 100644 libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/SocketSelector.java delete mode 100644 libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/AcceptingSelectorTests.java delete mode 100644 libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/AcceptorEventHandlerTests.java delete mode 100644 libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/ESSelectorTests.java rename libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/{SocketEventHandlerTests.java => EventHandlerTests.java} (66%) rename libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/{SocketSelectorTests.java => NioSelectorTests.java} (61%) diff --git a/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/AcceptingSelector.java b/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/AcceptingSelector.java deleted file mode 100644 index da64020daa81d..0000000000000 --- a/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/AcceptingSelector.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.nio; - -import java.io.IOException; -import java.nio.channels.ClosedChannelException; -import java.nio.channels.SelectionKey; -import java.nio.channels.Selector; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.stream.Collectors; - -/** - * Selector implementation that handles {@link NioServerSocketChannel}. It's main piece of functionality is - * accepting new channels. - */ -public class AcceptingSelector extends ESSelector { - - private final AcceptorEventHandler eventHandler; - private final ConcurrentLinkedQueue newChannels = new ConcurrentLinkedQueue<>(); - - public AcceptingSelector(AcceptorEventHandler eventHandler) throws IOException { - super(eventHandler); - this.eventHandler = eventHandler; - } - - public AcceptingSelector(AcceptorEventHandler eventHandler, Selector selector) throws IOException { - super(eventHandler, selector); - this.eventHandler = eventHandler; - } - - @Override - void processKey(SelectionKey selectionKey) { - ServerChannelContext channelContext = (ServerChannelContext) selectionKey.attachment(); - if (selectionKey.isAcceptable()) { - try { - eventHandler.acceptChannel(channelContext); - } catch (IOException e) { - eventHandler.acceptException(channelContext, e); - } - } - } - - @Override - void preSelect() { - setUpNewServerChannels(); - } - - @Override - void cleanup() { - channelsToClose.addAll(newChannels.stream().map(NioServerSocketChannel::getContext).collect(Collectors.toList())); - } - - /** - * Schedules a NioServerSocketChannel to be registered with this selector. The channel will by queued and - * eventually registered next time through the event loop. - * - * @param serverSocketChannel the channel to register - */ - public void scheduleForRegistration(NioServerSocketChannel serverSocketChannel) { - newChannels.add(serverSocketChannel); - ensureSelectorOpenForEnqueuing(newChannels, serverSocketChannel); - wakeup(); - } - - private void setUpNewServerChannels() { - NioServerSocketChannel newChannel; - while ((newChannel = this.newChannels.poll()) != null) { - ServerChannelContext context = newChannel.getContext(); - assert context.getSelector() == this : "The channel must be registered with the selector with which it was created"; - try { - if (context.isOpen()) { - eventHandler.handleRegistration(context); - } else { - eventHandler.registrationException(context, new ClosedChannelException()); - } - } catch (Exception e) { - eventHandler.registrationException(context, e); - } - } - } -} diff --git a/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/AcceptorEventHandler.java b/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/AcceptorEventHandler.java deleted file mode 100644 index f3aab9c9be125..0000000000000 --- a/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/AcceptorEventHandler.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.nio; - -import java.io.IOException; -import java.nio.channels.SelectionKey; -import java.util.function.Consumer; -import java.util.function.Supplier; - -/** - * Event handler designed to handle events from server sockets - */ -public class AcceptorEventHandler extends EventHandler { - - private final Supplier selectorSupplier; - - public AcceptorEventHandler(Supplier selectorSupplier, Consumer exceptionHandler) { - super(exceptionHandler); - this.selectorSupplier = selectorSupplier; - } - - /** - * This method is called when a NioServerSocketChannel is being registered with the selector. It should - * only be called once per channel. - * - * @param context that was registered - */ - protected void handleRegistration(ServerChannelContext context) throws IOException { - context.register(); - SelectionKey selectionKey = context.getSelectionKey(); - selectionKey.attach(context); - SelectionKeyUtils.setAcceptInterested(selectionKey); - } - - /** - * This method is called when an attempt to register a server channel throws an exception. - * - * @param context that was registered - * @param exception that occurred - */ - protected void registrationException(ServerChannelContext context, Exception exception) { - context.handleException(exception); - } - - /** - * This method is called when a server channel signals it is ready to accept a connection. All of the - * accept logic should occur in this call. - * - * @param context that can accept a connection - */ - protected void acceptChannel(ServerChannelContext context) throws IOException { - context.acceptChannels(selectorSupplier); - } - - /** - * This method is called when an attempt to accept a connection throws an exception. - * - * @param context that accepting a connection - * @param exception that occurred - */ - protected void acceptException(ServerChannelContext context, Exception exception) { - context.handleException(exception); - } -} diff --git a/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/BytesChannelContext.java b/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/BytesChannelContext.java index ef1e188a22e0a..a82d381951b76 100644 --- a/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/BytesChannelContext.java +++ b/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/BytesChannelContext.java @@ -24,7 +24,7 @@ public class BytesChannelContext extends SocketChannelContext { - public BytesChannelContext(NioSocketChannel channel, SocketSelector selector, Consumer exceptionHandler, + public BytesChannelContext(NioSocketChannel channel, NioSelector selector, Consumer exceptionHandler, ReadWriteHandler handler, InboundChannelBuffer channelBuffer) { super(channel, selector, exceptionHandler, handler, channelBuffer); } diff --git a/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/ChannelContext.java b/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/ChannelContext.java index 93930bbabf058..e3702c2880a18 100644 --- a/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/ChannelContext.java +++ b/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/ChannelContext.java @@ -105,7 +105,7 @@ void handleException(Exception e) { */ public abstract void closeChannel(); - public abstract ESSelector getSelector(); + public abstract NioSelector getSelector(); public abstract NioChannel getChannel(); diff --git a/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/ChannelFactory.java b/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/ChannelFactory.java index a03c4bcc15bdc..77443d948d9a6 100644 --- a/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/ChannelFactory.java +++ b/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/ChannelFactory.java @@ -42,30 +42,30 @@ protected ChannelFactory(RawChannelFactory rawChannelFactory) { this.rawChannelFactory = rawChannelFactory; } - public Socket openNioChannel(InetSocketAddress remoteAddress, Supplier supplier) throws IOException { + public Socket openNioChannel(InetSocketAddress remoteAddress, Supplier supplier) throws IOException { SocketChannel rawChannel = rawChannelFactory.openNioChannel(remoteAddress); - SocketSelector selector = supplier.get(); + NioSelector selector = supplier.get(); Socket channel = internalCreateChannel(selector, rawChannel); scheduleChannel(channel, selector); return channel; } - public Socket acceptNioChannel(ServerChannelContext serverContext, Supplier supplier) throws IOException { + public Socket acceptNioChannel(ServerChannelContext serverContext, Supplier supplier) throws IOException { SocketChannel rawChannel = rawChannelFactory.acceptNioChannel(serverContext); // Null is returned if there are no pending sockets to accept if (rawChannel == null) { return null; } else { - SocketSelector selector = supplier.get(); + NioSelector selector = supplier.get(); Socket channel = internalCreateChannel(selector, rawChannel); scheduleChannel(channel, selector); return channel; } } - public ServerSocket openNioServerSocketChannel(InetSocketAddress address, Supplier supplier) throws IOException { + public ServerSocket openNioServerSocketChannel(InetSocketAddress address, Supplier supplier) throws IOException { ServerSocketChannel rawChannel = rawChannelFactory.openNioServerSocketChannel(address); - AcceptingSelector selector = supplier.get(); + NioSelector selector = supplier.get(); ServerSocket serverChannel = internalCreateServerChannel(selector, rawChannel); scheduleServerChannel(serverChannel, selector); return serverChannel; @@ -81,7 +81,7 @@ public ServerSocket openNioServerSocketChannel(InetSocketAddress address, Suppli * @return the channel * @throws IOException related to the creation of the channel */ - public abstract Socket createChannel(SocketSelector selector, SocketChannel channel) throws IOException; + public abstract Socket createChannel(NioSelector selector, SocketChannel channel) throws IOException; /** * This method should return a new {@link NioServerSocketChannel} implementation. When this method has @@ -92,9 +92,9 @@ public ServerSocket openNioServerSocketChannel(InetSocketAddress address, Suppli * @return the server channel * @throws IOException related to the creation of the channel */ - public abstract ServerSocket createServerChannel(AcceptingSelector selector, ServerSocketChannel channel) throws IOException; + public abstract ServerSocket createServerChannel(NioSelector selector, ServerSocketChannel channel) throws IOException; - private Socket internalCreateChannel(SocketSelector selector, SocketChannel rawChannel) throws IOException { + private Socket internalCreateChannel(NioSelector selector, SocketChannel rawChannel) throws IOException { try { Socket channel = createChannel(selector, rawChannel); assert channel.getContext() != null : "channel context should have been set on channel"; @@ -105,7 +105,7 @@ private Socket internalCreateChannel(SocketSelector selector, SocketChannel rawC } } - private ServerSocket internalCreateServerChannel(AcceptingSelector selector, ServerSocketChannel rawChannel) throws IOException { + private ServerSocket internalCreateServerChannel(NioSelector selector, ServerSocketChannel rawChannel) throws IOException { try { return createServerChannel(selector, rawChannel); } catch (Exception e) { @@ -114,7 +114,7 @@ private ServerSocket internalCreateServerChannel(AcceptingSelector selector, Ser } } - private void scheduleChannel(Socket channel, SocketSelector selector) { + private void scheduleChannel(Socket channel, NioSelector selector) { try { selector.scheduleForRegistration(channel); } catch (IllegalStateException e) { @@ -123,7 +123,7 @@ private void scheduleChannel(Socket channel, SocketSelector selector) { } } - private void scheduleServerChannel(ServerSocket channel, AcceptingSelector selector) { + private void scheduleServerChannel(ServerSocket channel, NioSelector selector) { try { selector.scheduleForRegistration(channel); } catch (IllegalStateException e) { diff --git a/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/ESSelector.java b/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/ESSelector.java deleted file mode 100644 index c6cf97d10d38e..0000000000000 --- a/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/ESSelector.java +++ /dev/null @@ -1,248 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.nio; - -import java.io.Closeable; -import java.io.IOException; -import java.nio.channels.CancelledKeyException; -import java.nio.channels.ClosedSelectorException; -import java.nio.channels.SelectionKey; -import java.nio.channels.Selector; -import java.util.Iterator; -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Future; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.locks.ReentrantLock; -import java.util.stream.Collectors; - -/** - * This is a basic selector abstraction. This selector wraps a raw nio {@link Selector}. When you call - * {@link #runLoop()}, the selector will run until {@link #close()} is called. This instance handles closing - * of channels. Users should call {@link #queueChannelClose(NioChannel)} to schedule a channel for close by - * this selector. - *

- * Children of this class should implement the specific {@link #processKey(SelectionKey)}, - * {@link #preSelect()}, and {@link #cleanup()} functionality. - */ -public abstract class ESSelector implements Closeable { - - final Selector selector; - final ConcurrentLinkedQueue> channelsToClose = new ConcurrentLinkedQueue<>(); - - private final EventHandler eventHandler; - private final ReentrantLock runLock = new ReentrantLock(); - private final CountDownLatch exitedLoop = new CountDownLatch(1); - private final AtomicBoolean isClosed = new AtomicBoolean(false); - private final CompletableFuture isRunningFuture = new CompletableFuture<>(); - private volatile Thread thread; - - ESSelector(EventHandler eventHandler) throws IOException { - this(eventHandler, Selector.open()); - } - - ESSelector(EventHandler eventHandler, Selector selector) { - this.eventHandler = eventHandler; - this.selector = selector; - } - - /** - * Starts this selector. The selector will run until {@link #close()} is called. - */ - public void runLoop() { - if (runLock.tryLock()) { - isRunningFuture.complete(null); - try { - setThread(); - while (isOpen()) { - singleLoop(); - } - } finally { - try { - cleanupAndCloseChannels(); - } finally { - try { - selector.close(); - } catch (IOException e) { - eventHandler.selectorException(e); - } finally { - runLock.unlock(); - exitedLoop.countDown(); - } - } - } - } else { - throw new IllegalStateException("selector is already running"); - } - } - - void singleLoop() { - try { - closePendingChannels(); - preSelect(); - - int ready = selector.select(300); - if (ready > 0) { - Set selectionKeys = selector.selectedKeys(); - Iterator keyIterator = selectionKeys.iterator(); - while (keyIterator.hasNext()) { - SelectionKey sk = keyIterator.next(); - keyIterator.remove(); - if (sk.isValid()) { - try { - processKey(sk); - } catch (CancelledKeyException cke) { - eventHandler.genericChannelException((ChannelContext) sk.attachment(), cke); - } - } else { - eventHandler.genericChannelException((ChannelContext) sk.attachment(), new CancelledKeyException()); - } - } - } - } catch (ClosedSelectorException e) { - if (isOpen()) { - throw e; - } - } catch (IOException e) { - eventHandler.selectorException(e); - } catch (Exception e) { - eventHandler.uncaughtException(e); - } - } - - void cleanupAndCloseChannels() { - cleanup(); - channelsToClose.addAll(selector.keys().stream().map(sk -> (ChannelContext) sk.attachment()).collect(Collectors.toList())); - closePendingChannels(); - } - - /** - * Called by the base {@link ESSelector} class when there is a {@link SelectionKey} to be handled. - * - * @param selectionKey the key to be handled - * @throws CancelledKeyException thrown when the key has already been cancelled - */ - abstract void processKey(SelectionKey selectionKey) throws CancelledKeyException; - - /** - * Called immediately prior to a raw {@link Selector#select()} call. Should be used to implement - * channel registration, handling queued writes, and other work that is not specifically processing - * a selection key. - */ - abstract void preSelect(); - - /** - * Called once as the selector is being closed. - */ - abstract void cleanup(); - - void setThread() { - thread = Thread.currentThread(); - } - - public boolean isOnCurrentThread() { - return Thread.currentThread() == thread; - } - - public void assertOnSelectorThread() { - assert isOnCurrentThread() : "Must be on selector thread to perform this operation. Currently on thread [" - + Thread.currentThread().getName() + "]."; - } - - void wakeup() { - // TODO: Do we need the wakeup optimizations that some other libraries use? - selector.wakeup(); - } - - @Override - public void close() throws IOException { - if (isClosed.compareAndSet(false, true)) { - wakeup(); - if (isRunning()) { - try { - exitedLoop.await(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new IllegalStateException("Thread was interrupted while waiting for selector to close", e); - } - } else if (selector.isOpen()) { - selector.close(); - } - } - } - - public void queueChannelClose(NioChannel channel) { - ChannelContext context = channel.getContext(); - assert context.getSelector() == this : "Must schedule a channel for closure with its selector"; - channelsToClose.offer(context); - ensureSelectorOpenForEnqueuing(channelsToClose, context); - wakeup(); - } - - public Selector rawSelector() { - return selector; - } - - public boolean isOpen() { - return isClosed.get() == false; - } - - public boolean isRunning() { - return runLock.isLocked(); - } - - public Future isRunningFuture() { - return isRunningFuture; - } - - /** - * This is a convenience method to be called after some object (normally channels) are enqueued with this - * selector. This method will check if the selector is still open. If it is open, normal operation can - * proceed. - * - * If the selector is closed, then we attempt to remove the object from the queue. If the removal - * succeeds then we throw an {@link IllegalStateException} indicating that normal operation failed. If - * the object cannot be removed from the queue, then the object has already been handled by the selector - * and operation can proceed normally. - * - * If this method is called from the selector thread, we will not throw an exception as the selector - * thread can manipulate its queues internally even if it is no longer open. - * - * @param queue the queue to which the object was added - * @param objectAdded the objected added - * @param the object type - */ - void ensureSelectorOpenForEnqueuing(ConcurrentLinkedQueue queue, O objectAdded) { - if (isOpen() == false && isOnCurrentThread() == false) { - if (queue.remove(objectAdded)) { - throw new IllegalStateException("selector is already closed"); - } - } - } - - private void closePendingChannels() { - ChannelContext channelContext; - while ((channelContext = channelsToClose.poll()) != null) { - eventHandler.handleClose(channelContext); - } - } -} diff --git a/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/EventHandler.java b/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/EventHandler.java index cb4d43af4fdc3..3c52423c7aff3 100644 --- a/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/EventHandler.java +++ b/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/EventHandler.java @@ -20,15 +20,163 @@ package org.elasticsearch.nio; import java.io.IOException; +import java.nio.channels.SelectionKey; import java.nio.channels.Selector; import java.util.function.Consumer; +import java.util.function.Supplier; -public abstract class EventHandler { +public class EventHandler { protected final Consumer exceptionHandler; + private final Supplier selectorSupplier; - protected EventHandler(Consumer exceptionHandler) { + public EventHandler(Consumer exceptionHandler, Supplier selectorSupplier) { this.exceptionHandler = exceptionHandler; + this.selectorSupplier = selectorSupplier; + } + + /** + * This method is called when a server channel signals it is ready to accept a connection. All of the + * accept logic should occur in this call. + * + * @param context that can accept a connection + */ + protected void acceptChannel(ServerChannelContext context) throws IOException { + context.acceptChannels(selectorSupplier); + } + + /** + * This method is called when an attempt to accept a connection throws an exception. + * + * @param context that accepting a connection + * @param exception that occurred + */ + protected void acceptException(ServerChannelContext context, Exception exception) { + context.handleException(exception); + } + + /** + * This method is called when a NioChannel is being registered with the selector. It should + * only be called once per channel. + * + * @param context that was registered + */ + protected void handleRegistration(ChannelContext context) throws IOException { + context.register(); + SelectionKey selectionKey = context.getSelectionKey(); + selectionKey.attach(context); + if (context instanceof SocketChannelContext) { + if (((SocketChannelContext) context).readyForFlush()) { + SelectionKeyUtils.setConnectReadAndWriteInterested(context.getSelectionKey()); + } else { + SelectionKeyUtils.setConnectAndReadInterested(context.getSelectionKey()); + } + } else { + assert context instanceof ServerChannelContext : "If not SocketChannelContext the context must be a ServerChannelContext"; + SelectionKeyUtils.setAcceptInterested(context.getSelectionKey()); + } + } + + /** + * This method is called when an attempt to register a channel throws an exception. + * + * @param context that was registered + * @param exception that occurred + */ + protected void registrationException(ChannelContext context, Exception exception) { + context.handleException(exception); + } + + /** + * This method is called when a NioSocketChannel has just been accepted or if it has receive an + * OP_CONNECT event. + * + * @param context that was registered + */ + protected void handleConnect(SocketChannelContext context) throws IOException { + if (context.connect()) { + SelectionKeyUtils.removeConnectInterested(context.getSelectionKey()); + } + } + + /** + * This method is called when an attempt to connect a channel throws an exception. + * + * @param context that was connecting + * @param exception that occurred + */ + protected void connectException(SocketChannelContext context, Exception exception) { + context.handleException(exception); + } + + /** + * This method is called when a channel signals it is ready for be read. All of the read logic should + * occur in this call. + * + * @param context that can be read + */ + protected void handleRead(SocketChannelContext context) throws IOException { + context.read(); + } + + /** + * This method is called when an attempt to read from a channel throws an exception. + * + * @param context that was being read + * @param exception that occurred + */ + protected void readException(SocketChannelContext context, Exception exception) { + context.handleException(exception); + } + + /** + * This method is called when a channel signals it is ready to receive writes. All of the write logic + * should occur in this call. + * + * @param context that can be written to + */ + protected void handleWrite(SocketChannelContext context) throws IOException { + context.flushChannel(); + } + + /** + * This method is called when an attempt to write to a channel throws an exception. + * + * @param context that was being written to + * @param exception that occurred + */ + protected void writeException(SocketChannelContext context, Exception exception) { + context.handleException(exception); + } + + /** + * This method is called when a listener attached to a channel operation throws an exception. + * + * @param exception that occurred + */ + protected void listenerException(Exception exception) { + exceptionHandler.accept(exception); + } + + /** + * This method is called after ready events (READ, ACCEPT, WRITE, CONNECT) have been handled for a + * channel. + * + * @param context that was handled + */ + protected void postHandling(SocketChannelContext context) { + if (context.selectorShouldClose()) { + handleClose(context); + } else { + SelectionKey selectionKey = context.getSelectionKey(); + boolean currentlyWriteInterested = SelectionKeyUtils.isWriteInterested(selectionKey); + boolean pendingWrites = context.readyForFlush(); + if (currentlyWriteInterested == false && pendingWrites) { + SelectionKeyUtils.setWriteInterested(selectionKey); + } else if (currentlyWriteInterested && pendingWrites == false) { + SelectionKeyUtils.removeWriteInterested(selectionKey); + } + } } /** diff --git a/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/NioChannel.java b/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/NioChannel.java index ea633bd3276a3..8262d9c87e3ef 100644 --- a/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/NioChannel.java +++ b/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/NioChannel.java @@ -22,10 +22,11 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.nio.channels.NetworkChannel; +import java.nio.channels.SocketChannel; import java.util.function.BiConsumer; /** - * This is a basic channel abstraction used by the {@link ESSelector}. + * This is a basic channel abstraction used by the {@link NioSelector}. *

* A channel is open once it is constructed. The channel remains open and {@link #isOpen()} will return * true until the channel is explicitly closed. diff --git a/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/NioGroup.java b/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/NioGroup.java index 3f2fd44259c64..fe1bc1cf4043e 100644 --- a/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/NioGroup.java +++ b/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/NioGroup.java @@ -35,10 +35,9 @@ /** * The NioGroup is a group of selectors for interfacing with java nio. When it is started it will create the - * configured number of socket and acceptor selectors. Each selector will be running in a dedicated thread. - * Server connections can be bound using the {@link #bindServerChannel(InetSocketAddress, ChannelFactory)} - * method. Client connections can be opened using the {@link #openChannel(InetSocketAddress, ChannelFactory)} - * method. + * configured number of selectors. Each selector will be running in a dedicated thread. Server connections + * can be bound using the {@link #bindServerChannel(InetSocketAddress, ChannelFactory)} method. Client + * connections can be opened using the {@link #openChannel(InetSocketAddress, ChannelFactory)} method. *

* The logic specific to a particular channel is provided by the {@link ChannelFactory} passed to the method * when the channel is created. This is what allows an NioGroup to support different channel types. @@ -46,35 +45,75 @@ public class NioGroup implements AutoCloseable { - private final ArrayList acceptors; - private final RoundRobinSupplier acceptorSupplier; + private final List dedicatedAcceptors; + private final RoundRobinSupplier acceptorSupplier; - private final ArrayList socketSelectors; - private final RoundRobinSupplier socketSelectorSupplier; + private final List selectors; + private final RoundRobinSupplier selectorSupplier; private final AtomicBoolean isOpen = new AtomicBoolean(true); - public NioGroup(ThreadFactory acceptorThreadFactory, int acceptorCount, - Function, AcceptorEventHandler> acceptorEventHandlerFunction, - ThreadFactory socketSelectorThreadFactory, int socketSelectorCount, - Supplier socketEventHandlerFunction) throws IOException { - acceptors = new ArrayList<>(acceptorCount); - socketSelectors = new ArrayList<>(socketSelectorCount); + /** + * This will create an NioGroup with no dedicated acceptors. All server channels will be handled by the + * same selectors that are handling child channels. + * + * @param threadFactory factory to create selector threads + * @param selectorCount the number of selectors to be created + * @param eventHandlerFunction function for creating event handlers + * @throws IOException occurs if there is a problem while opening a java.nio.Selector + */ + public NioGroup(ThreadFactory threadFactory, int selectorCount, Function, EventHandler> eventHandlerFunction) + throws IOException { + this(null, 0, threadFactory, selectorCount, eventHandlerFunction); + } + + /** + * This will create an NioGroup with dedicated acceptors. All server channels will be handled by a group + * of selectors dedicated to accepting channels. These accepted channels will be handed off the + * non-server selectors. + * + * @param acceptorThreadFactory factory to create acceptor selector threads + * @param dedicatedAcceptorCount the number of dedicated acceptor selectors to be created + * @param selectorThreadFactory factory to create non-acceptor selector threads + * @param selectorCount the number of non-acceptor selectors to be created + * @param eventHandlerFunction function for creating event handlers + * @throws IOException occurs if there is a problem while opening a java.nio.Selector + */ + public NioGroup(ThreadFactory acceptorThreadFactory, int dedicatedAcceptorCount, ThreadFactory selectorThreadFactory, int selectorCount, + Function, EventHandler> eventHandlerFunction) throws IOException { + dedicatedAcceptors = new ArrayList<>(dedicatedAcceptorCount); + selectors = new ArrayList<>(selectorCount); try { - for (int i = 0; i < socketSelectorCount; ++i) { - SocketSelector selector = new SocketSelector(socketEventHandlerFunction.get()); - socketSelectors.add(selector); + List> suppliersToSet = new ArrayList<>(selectorCount); + for (int i = 0; i < selectorCount; ++i) { + RoundRobinSupplier supplier = new RoundRobinSupplier<>(); + suppliersToSet.add(supplier); + NioSelector selector = new NioSelector(eventHandlerFunction.apply(supplier)); + selectors.add(selector); + } + for (RoundRobinSupplier supplierToSet : suppliersToSet) { + supplierToSet.setSelectors(selectors.toArray(new NioSelector[0])); + assert supplierToSet.count() == selectors.size() : "Supplier should have same count as selector list."; } - startSelectors(socketSelectors, socketSelectorThreadFactory); - for (int i = 0; i < acceptorCount; ++i) { - SocketSelector[] childSelectors = this.socketSelectors.toArray(new SocketSelector[this.socketSelectors.size()]); - Supplier selectorSupplier = new RoundRobinSupplier<>(childSelectors); - AcceptingSelector acceptor = new AcceptingSelector(acceptorEventHandlerFunction.apply(selectorSupplier)); - acceptors.add(acceptor); + for (int i = 0; i < dedicatedAcceptorCount; ++i) { + RoundRobinSupplier supplier = new RoundRobinSupplier<>(selectors.toArray(new NioSelector[0])); + NioSelector acceptor = new NioSelector(eventHandlerFunction.apply(supplier)); + dedicatedAcceptors.add(acceptor); } - startSelectors(acceptors, acceptorThreadFactory); + + if (dedicatedAcceptorCount != 0) { + acceptorSupplier = new RoundRobinSupplier<>(dedicatedAcceptors.toArray(new NioSelector[0])); + } else { + acceptorSupplier = new RoundRobinSupplier<>(selectors.toArray(new NioSelector[0])); + } + selectorSupplier = new RoundRobinSupplier<>(selectors.toArray(new NioSelector[0])); + assert selectorCount == selectors.size() : "We need to have created all the selectors at this point."; + assert dedicatedAcceptorCount == dedicatedAcceptors.size() : "We need to have created all the acceptors at this point."; + + startSelectors(selectors, selectorThreadFactory); + startSelectors(dedicatedAcceptors, acceptorThreadFactory); } catch (Exception e) { try { close(); @@ -83,31 +122,25 @@ public NioGroup(ThreadFactory acceptorThreadFactory, int acceptorCount, } throw e; } - - socketSelectorSupplier = new RoundRobinSupplier<>(socketSelectors.toArray(new SocketSelector[socketSelectors.size()])); - acceptorSupplier = new RoundRobinSupplier<>(acceptors.toArray(new AcceptingSelector[acceptors.size()])); } public S bindServerChannel(InetSocketAddress address, ChannelFactory factory) throws IOException { ensureOpen(); - if (acceptors.isEmpty()) { - throw new IllegalArgumentException("There are no acceptors configured. Without acceptors, server channels are not supported."); - } return factory.openNioServerSocketChannel(address, acceptorSupplier); } public S openChannel(InetSocketAddress address, ChannelFactory factory) throws IOException { ensureOpen(); - return factory.openNioChannel(address, socketSelectorSupplier); + return factory.openNioChannel(address, selectorSupplier); } @Override public void close() throws IOException { if (isOpen.compareAndSet(true, false)) { - List toClose = Stream.concat(acceptors.stream(), socketSelectors.stream()).collect(Collectors.toList()); + List toClose = Stream.concat(dedicatedAcceptors.stream(), selectors.stream()).collect(Collectors.toList()); List closingExceptions = new ArrayList<>(); - for (ESSelector selector : toClose) { + for (NioSelector selector : toClose) { try { selector.close(); } catch (IOException e) { @@ -118,12 +151,12 @@ public void close() throws IOException { } } - private static void startSelectors(Iterable selectors, ThreadFactory threadFactory) { - for (ESSelector acceptor : selectors) { - if (acceptor.isRunning() == false) { - threadFactory.newThread(acceptor::runLoop).start(); + private static void startSelectors(Iterable selectors, ThreadFactory threadFactory) { + for (NioSelector selector : selectors) { + if (selector.isRunning() == false) { + threadFactory.newThread(selector::runLoop).start(); try { - acceptor.isRunningFuture().get(); + selector.isRunningFuture().get(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new IllegalStateException("Interrupted while waiting for selector to start.", e); diff --git a/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/NioSelector.java b/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/NioSelector.java new file mode 100644 index 0000000000000..ab6709bcc5bd4 --- /dev/null +++ b/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/NioSelector.java @@ -0,0 +1,428 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.nio; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.channels.CancelledKeyException; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.ClosedSelectorException; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.util.Iterator; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.BiConsumer; +import java.util.stream.Collectors; + +/** + * This is a nio selector implementation. This selector wraps a raw nio {@link Selector}. When you call + * {@link #runLoop()}, the selector will run until {@link #close()} is called. This instance handles closing + * of channels. Users should call {@link #queueChannelClose(NioChannel)} to schedule a channel for close by + * this selector. + *

+ * Children of this class should implement the specific {@link #processKey(SelectionKey)}, + * {@link #preSelect()}, and {@link #cleanup()} functionality. + */ +public class NioSelector implements Closeable { + + private final ConcurrentLinkedQueue queuedWrites = new ConcurrentLinkedQueue<>(); + private final ConcurrentLinkedQueue> channelsToClose = new ConcurrentLinkedQueue<>(); + private final ConcurrentLinkedQueue> channelsToRegister = new ConcurrentLinkedQueue<>(); + private final EventHandler eventHandler; + private final Selector selector; + + private final ReentrantLock runLock = new ReentrantLock(); + private final CountDownLatch exitedLoop = new CountDownLatch(1); + private final AtomicBoolean isClosed = new AtomicBoolean(false); + private final CompletableFuture isRunningFuture = new CompletableFuture<>(); + private final AtomicReference thread = new AtomicReference<>(null); + + public NioSelector(EventHandler eventHandler) throws IOException { + this(eventHandler, Selector.open()); + } + + public NioSelector(EventHandler eventHandler, Selector selector) throws IOException { + this.selector = selector; + this.eventHandler = eventHandler; + } + + public Selector rawSelector() { + return selector; + } + + public boolean isOpen() { + return isClosed.get() == false; + } + + public boolean isRunning() { + return runLock.isLocked(); + } + + Future isRunningFuture() { + return isRunningFuture; + } + + void setThread() { + boolean wasSet = thread.compareAndSet(null, Thread.currentThread()); + assert wasSet : "Failed to set thread as it was already set. Should only set once."; + } + + public boolean isOnCurrentThread() { + return Thread.currentThread() == thread.get(); + } + + public void assertOnSelectorThread() { + assert isOnCurrentThread() : "Must be on selector thread [" + thread.get().getName() + "} to perform this operation. " + + "Currently on thread [" + Thread.currentThread().getName() + "]."; + } + + /** + * Starts this selector. The selector will run until {@link #close()} is called. + */ + public void runLoop() { + if (runLock.tryLock()) { + isRunningFuture.complete(null); + try { + setThread(); + while (isOpen()) { + singleLoop(); + } + } finally { + try { + cleanupAndCloseChannels(); + } finally { + try { + selector.close(); + } catch (IOException e) { + eventHandler.selectorException(e); + } finally { + runLock.unlock(); + exitedLoop.countDown(); + } + } + } + } else { + throw new IllegalStateException("selector is already running"); + } + } + + void singleLoop() { + try { + closePendingChannels(); + preSelect(); + + int ready = selector.select(300); + if (ready > 0) { + Set selectionKeys = selector.selectedKeys(); + Iterator keyIterator = selectionKeys.iterator(); + while (keyIterator.hasNext()) { + SelectionKey sk = keyIterator.next(); + keyIterator.remove(); + if (sk.isValid()) { + try { + processKey(sk); + } catch (CancelledKeyException cke) { + eventHandler.genericChannelException((ChannelContext) sk.attachment(), cke); + } + } else { + eventHandler.genericChannelException((ChannelContext) sk.attachment(), new CancelledKeyException()); + } + } + } + } catch (ClosedSelectorException e) { + if (isOpen()) { + throw e; + } + } catch (IOException e) { + eventHandler.selectorException(e); + } catch (Exception e) { + eventHandler.uncaughtException(e); + } + } + + void cleanupAndCloseChannels() { + cleanup(); + channelsToClose.addAll(channelsToRegister); + channelsToRegister.clear(); + channelsToClose.addAll(selector.keys().stream().map(sk -> (ChannelContext) sk.attachment()).collect(Collectors.toList())); + closePendingChannels(); + } + + @Override + public void close() throws IOException { + if (isClosed.compareAndSet(false, true)) { + wakeup(); + if (isRunning()) { + try { + exitedLoop.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Thread was interrupted while waiting for selector to close", e); + } + } else if (selector.isOpen()) { + selector.close(); + } + } + } + + void processKey(SelectionKey selectionKey) { + ChannelContext context = (ChannelContext) selectionKey.attachment(); + if (selectionKey.isAcceptable()) { + assert context instanceof ServerChannelContext : "Only server channels can receive accept events"; + ServerChannelContext serverChannelContext = (ServerChannelContext) context; + int ops = selectionKey.readyOps(); + if ((ops & SelectionKey.OP_ACCEPT) != 0) { + try { + eventHandler.acceptChannel(serverChannelContext); + } catch (IOException e) { + eventHandler.acceptException(serverChannelContext, e); + } + } + } else { + assert context instanceof SocketChannelContext : "Only sockets channels can receive non-accept events"; + SocketChannelContext channelContext = (SocketChannelContext) context; + int ops = selectionKey.readyOps(); + if ((ops & SelectionKey.OP_CONNECT) != 0) { + attemptConnect(channelContext, true); + } + + if (channelContext.isConnectComplete()) { + if ((ops & SelectionKey.OP_WRITE) != 0) { + handleWrite(channelContext); + } + + if ((ops & SelectionKey.OP_READ) != 0) { + handleRead(channelContext); + } + } + eventHandler.postHandling(channelContext); + } + + } + + /** + * Called immediately prior to a raw {@link Selector#select()} call. Should be used to implement + * channel registration, handling queued writes, and other work that is not specifically processing + * a selection key. + */ + void preSelect() { + setUpNewChannels(); + handleQueuedWrites(); + } + + /** + * Called once as the selector is being closed. + */ + void cleanup() { + WriteOperation op; + while ((op = queuedWrites.poll()) != null) { + executeFailedListener(op.getListener(), new ClosedSelectorException()); + } + } + + /** + * Queues a write operation to be handled by the event loop. This can be called by any thread and is the + * api available for non-selector threads to schedule writes. + * + * @param writeOperation to be queued + */ + public void queueWrite(WriteOperation writeOperation) { + queuedWrites.offer(writeOperation); + if (isOpen() == false) { + boolean wasRemoved = queuedWrites.remove(writeOperation); + if (wasRemoved) { + writeOperation.getListener().accept(null, new ClosedSelectorException()); + } + } else { + wakeup(); + } + } + + public void queueChannelClose(NioChannel channel) { + ChannelContext context = channel.getContext(); + assert context.getSelector() == this : "Must schedule a channel for closure with its selector"; + channelsToClose.offer(context); + ensureSelectorOpenForEnqueuing(channelsToClose, context); + wakeup(); + } + + /** + * Schedules a NioChannel to be registered with this selector. The channel will by queued and + * eventually registered next time through the event loop. + * + * @param channel to register + */ + public void scheduleForRegistration(NioChannel channel) { + ChannelContext context = channel.getContext(); + channelsToRegister.add(context); + ensureSelectorOpenForEnqueuing(channelsToRegister, context); + wakeup(); + } + + /** + * Queues a write operation directly in a channel's buffer. Channel buffers are only safe to be accessed + * by the selector thread. As a result, this method should only be called by the selector thread. + * + * @param writeOperation to be queued in a channel's buffer + */ + public void queueWriteInChannelBuffer(WriteOperation writeOperation) { + assertOnSelectorThread(); + SocketChannelContext context = writeOperation.getChannel(); + try { + SelectionKeyUtils.setWriteInterested(context.getSelectionKey()); + context.queueWriteOperation(writeOperation); + } catch (Exception e) { + executeFailedListener(writeOperation.getListener(), e); + } + } + + /** + * Executes a success listener with consistent exception handling. This can only be called from current + * selector thread. + * + * @param listener to be executed + * @param value to provide to listener + */ + public void executeListener(BiConsumer listener, V value) { + assertOnSelectorThread(); + try { + listener.accept(value, null); + } catch (Exception e) { + eventHandler.listenerException(e); + } + } + + /** + * Executes a failed listener with consistent exception handling. This can only be called from current + * selector thread. + * + * @param listener to be executed + * @param exception to provide to listener + */ + public void executeFailedListener(BiConsumer listener, Exception exception) { + assertOnSelectorThread(); + try { + listener.accept(null, exception); + } catch (Exception e) { + eventHandler.listenerException(e); + } + } + + private void wakeup() { + // TODO: Do we need the wakeup optimizations that some other libraries use? + selector.wakeup(); + } + + private void handleWrite(SocketChannelContext context) { + try { + eventHandler.handleWrite(context); + } catch (Exception e) { + eventHandler.writeException(context, e); + } + } + + private void handleRead(SocketChannelContext context) { + try { + eventHandler.handleRead(context); + } catch (Exception e) { + eventHandler.readException(context, e); + } + } + + private void attemptConnect(SocketChannelContext context, boolean connectEvent) { + try { + eventHandler.handleConnect(context); + if (connectEvent && context.isConnectComplete() == false) { + eventHandler.connectException(context, new IOException("Received OP_CONNECT but connect failed")); + } + } catch (Exception e) { + eventHandler.connectException(context, e); + } + } + + private void setUpNewChannels() { + ChannelContext newChannel; + while ((newChannel = this.channelsToRegister.poll()) != null) { + assert newChannel.getSelector() == this : "The channel must be registered with the selector with which it was created"; + try { + if (newChannel.isOpen()) { + eventHandler.handleRegistration(newChannel); + if (newChannel instanceof SocketChannelContext) { + attemptConnect((SocketChannelContext) newChannel, false); + } + } else { + eventHandler.registrationException(newChannel, new ClosedChannelException()); + } + } catch (Exception e) { + eventHandler.registrationException(newChannel, e); + } + } + } + + private void closePendingChannels() { + ChannelContext channelContext; + while ((channelContext = channelsToClose.poll()) != null) { + eventHandler.handleClose(channelContext); + } + } + + private void handleQueuedWrites() { + WriteOperation writeOperation; + while ((writeOperation = queuedWrites.poll()) != null) { + if (writeOperation.getChannel().isOpen()) { + queueWriteInChannelBuffer(writeOperation); + } else { + executeFailedListener(writeOperation.getListener(), new ClosedChannelException()); + } + } + } + + /** + * This is a convenience method to be called after some object (normally channels) are enqueued with this + * selector. This method will check if the selector is still open. If it is open, normal operation can + * proceed. + * + * If the selector is closed, then we attempt to remove the object from the queue. If the removal + * succeeds then we throw an {@link IllegalStateException} indicating that normal operation failed. If + * the object cannot be removed from the queue, then the object has already been handled by the selector + * and operation can proceed normally. + * + * If this method is called from the selector thread, we will not allow the queuing to occur as the + * selector thread can manipulate its queues internally even if it is no longer open. + * + * @param queue the queue to which the object was added + * @param objectAdded the objected added + * @param the object type + */ + private void ensureSelectorOpenForEnqueuing(ConcurrentLinkedQueue queue, O objectAdded) { + if (isOpen() == false && isOnCurrentThread() == false) { + if (queue.remove(objectAdded)) { + throw new IllegalStateException("selector is already closed"); + } + } + } +} diff --git a/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/RoundRobinSupplier.java b/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/RoundRobinSupplier.java index 311403a48857b..4d4de689fe7f8 100644 --- a/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/RoundRobinSupplier.java +++ b/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/RoundRobinSupplier.java @@ -19,21 +19,39 @@ package org.elasticsearch.nio; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; -public class RoundRobinSupplier implements Supplier { +final class RoundRobinSupplier implements Supplier { - private final S[] selectors; - private final int count; + private final AtomicBoolean selectorsSet = new AtomicBoolean(false); + private volatile S[] selectors; private AtomicInteger counter = new AtomicInteger(0); + RoundRobinSupplier() { + this.selectors = null; + } + RoundRobinSupplier(S[] selectors) { - this.count = selectors.length; this.selectors = selectors; + this.selectorsSet.set(true); } public S get() { - return selectors[counter.getAndIncrement() % count]; + S[] selectors = this.selectors; + return selectors[counter.getAndIncrement() % selectors.length]; + } + + void setSelectors(S[] selectors) { + if (selectorsSet.compareAndSet(false, true)) { + this.selectors = selectors; + } else { + throw new AssertionError("Selectors already set. Should only be set once."); + } + } + + int count() { + return selectors.length; } } diff --git a/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/ServerChannelContext.java b/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/ServerChannelContext.java index 4b47ce063f9b3..9e1af3e9973fb 100644 --- a/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/ServerChannelContext.java +++ b/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/ServerChannelContext.java @@ -28,12 +28,12 @@ public class ServerChannelContext extends ChannelContext { private final NioServerSocketChannel channel; - private final AcceptingSelector selector; + private final NioSelector selector; private final Consumer acceptor; private final AtomicBoolean isClosing = new AtomicBoolean(false); private final ChannelFactory channelFactory; - public ServerChannelContext(NioServerSocketChannel channel, ChannelFactory channelFactory, AcceptingSelector selector, + public ServerChannelContext(NioServerSocketChannel channel, ChannelFactory channelFactory, NioSelector selector, Consumer acceptor, Consumer exceptionHandler) { super(channel.getRawChannel(), exceptionHandler); this.channel = channel; @@ -42,7 +42,7 @@ public ServerChannelContext(NioServerSocketChannel channel, ChannelFactory this.acceptor = acceptor; } - public void acceptChannels(Supplier selectorSupplier) throws IOException { + public void acceptChannels(Supplier selectorSupplier) throws IOException { NioSocketChannel acceptedChannel; while ((acceptedChannel = channelFactory.acceptNioChannel(this, selectorSupplier)) != null) { acceptor.accept(acceptedChannel); @@ -57,7 +57,7 @@ public void closeChannel() { } @Override - public AcceptingSelector getSelector() { + public NioSelector getSelector() { return selector; } diff --git a/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/SocketChannelContext.java b/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/SocketChannelContext.java index 6a769b4d17381..53be0e7f89fe0 100644 --- a/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/SocketChannelContext.java +++ b/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/SocketChannelContext.java @@ -47,14 +47,14 @@ public abstract class SocketChannelContext extends ChannelContext protected final InboundChannelBuffer channelBuffer; protected final AtomicBoolean isClosing = new AtomicBoolean(false); private final ReadWriteHandler readWriteHandler; - private final SocketSelector selector; + private final NioSelector selector; private final CompletableContext connectContext = new CompletableContext<>(); private final LinkedList pendingFlushes = new LinkedList<>(); private boolean ioException; private boolean peerClosed; private Exception connectException; - protected SocketChannelContext(NioSocketChannel channel, SocketSelector selector, Consumer exceptionHandler, + protected SocketChannelContext(NioSocketChannel channel, NioSelector selector, Consumer exceptionHandler, ReadWriteHandler readWriteHandler, InboundChannelBuffer channelBuffer) { super(channel.getRawChannel(), exceptionHandler); this.selector = selector; @@ -64,7 +64,7 @@ protected SocketChannelContext(NioSocketChannel channel, SocketSelector selector } @Override - public SocketSelector getSelector() { + public NioSelector getSelector() { return selector; } @@ -129,7 +129,7 @@ public void sendMessage(Object message, BiConsumer listener) { WriteOperation writeOperation = readWriteHandler.createWriteOperation(this, message, listener); - SocketSelector selector = getSelector(); + NioSelector selector = getSelector(); if (selector.isOnCurrentThread() == false) { selector.queueWrite(writeOperation); return; diff --git a/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/SocketEventHandler.java b/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/SocketEventHandler.java deleted file mode 100644 index b486243f2197d..0000000000000 --- a/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/SocketEventHandler.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.nio; - -import java.io.IOException; -import java.nio.channels.SelectionKey; -import java.util.function.Consumer; - -/** - * Event handler designed to handle events from non-server sockets - */ -public class SocketEventHandler extends EventHandler { - - public SocketEventHandler(Consumer exceptionHandler) { - super(exceptionHandler); - } - - /** - * This method is called when a NioSocketChannel is successfully registered. It should only be called - * once per channel. - * - * @param context that was registered - */ - protected void handleRegistration(SocketChannelContext context) throws IOException { - context.register(); - SelectionKey selectionKey = context.getSelectionKey(); - selectionKey.attach(context); - if (context.readyForFlush()) { - SelectionKeyUtils.setConnectReadAndWriteInterested(selectionKey); - } else { - SelectionKeyUtils.setConnectAndReadInterested(selectionKey); - } - } - - /** - * This method is called when an attempt to register a channel throws an exception. - * - * @param context that was registered - * @param exception that occurred - */ - protected void registrationException(SocketChannelContext context, Exception exception) { - context.handleException(exception); - } - - /** - * This method is called when a NioSocketChannel has just been accepted or if it has receive an - * OP_CONNECT event. - * - * @param context that was registered - */ - protected void handleConnect(SocketChannelContext context) throws IOException { - if (context.connect()) { - SelectionKeyUtils.removeConnectInterested(context.getSelectionKey()); - } - } - - /** - * This method is called when an attempt to connect a channel throws an exception. - * - * @param context that was connecting - * @param exception that occurred - */ - protected void connectException(SocketChannelContext context, Exception exception) { - context.handleException(exception); - } - - /** - * This method is called when a channel signals it is ready for be read. All of the read logic should - * occur in this call. - * - * @param context that can be read - */ - protected void handleRead(SocketChannelContext context) throws IOException { - context.read(); - } - - /** - * This method is called when an attempt to read from a channel throws an exception. - * - * @param context that was being read - * @param exception that occurred - */ - protected void readException(SocketChannelContext context, Exception exception) { - context.handleException(exception); - } - - /** - * This method is called when a channel signals it is ready to receive writes. All of the write logic - * should occur in this call. - * - * @param context that can be written to - */ - protected void handleWrite(SocketChannelContext context) throws IOException { - context.flushChannel(); - } - - /** - * This method is called when an attempt to write to a channel throws an exception. - * - * @param context that was being written to - * @param exception that occurred - */ - protected void writeException(SocketChannelContext context, Exception exception) { - context.handleException(exception); - } - - /** - * This method is called when a listener attached to a channel operation throws an exception. - * - * @param exception that occurred - */ - protected void listenerException(Exception exception) { - exceptionHandler.accept(exception); - } - - /** - * @param context that was handled - */ - protected void postHandling(SocketChannelContext context) { - if (context.selectorShouldClose()) { - handleClose(context); - } else { - SelectionKey selectionKey = context.getSelectionKey(); - boolean currentlyWriteInterested = SelectionKeyUtils.isWriteInterested(selectionKey); - boolean pendingWrites = context.readyForFlush(); - if (currentlyWriteInterested == false && pendingWrites) { - SelectionKeyUtils.setWriteInterested(selectionKey); - } else if (currentlyWriteInterested && pendingWrites == false) { - SelectionKeyUtils.removeWriteInterested(selectionKey); - } - } - } -} diff --git a/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/SocketSelector.java b/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/SocketSelector.java deleted file mode 100644 index 30ef7b317a3f8..0000000000000 --- a/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/SocketSelector.java +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.nio; - -import java.io.IOException; -import java.nio.channels.ClosedChannelException; -import java.nio.channels.ClosedSelectorException; -import java.nio.channels.SelectionKey; -import java.nio.channels.Selector; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.function.BiConsumer; - -/** - * Selector implementation that handles {@link NioSocketChannel}. It's main piece of functionality is - * handling connect, read, and write events. - */ -public class SocketSelector extends ESSelector { - - private final ConcurrentLinkedQueue newChannels = new ConcurrentLinkedQueue<>(); - private final ConcurrentLinkedQueue queuedWrites = new ConcurrentLinkedQueue<>(); - private final SocketEventHandler eventHandler; - - public SocketSelector(SocketEventHandler eventHandler) throws IOException { - super(eventHandler); - this.eventHandler = eventHandler; - } - - public SocketSelector(SocketEventHandler eventHandler, Selector selector) throws IOException { - super(eventHandler, selector); - this.eventHandler = eventHandler; - } - - @Override - void processKey(SelectionKey selectionKey) { - SocketChannelContext channelContext = (SocketChannelContext) selectionKey.attachment(); - int ops = selectionKey.readyOps(); - if ((ops & SelectionKey.OP_CONNECT) != 0) { - attemptConnect(channelContext, true); - } - - if (channelContext.isConnectComplete()) { - if ((ops & SelectionKey.OP_WRITE) != 0) { - handleWrite(channelContext); - } - - if ((ops & SelectionKey.OP_READ) != 0) { - handleRead(channelContext); - } - } - - eventHandler.postHandling(channelContext); - } - - @Override - void preSelect() { - setUpNewChannels(); - handleQueuedWrites(); - } - - @Override - void cleanup() { - WriteOperation op; - while ((op = queuedWrites.poll()) != null) { - executeFailedListener(op.getListener(), new ClosedSelectorException()); - } - channelsToClose.addAll(newChannels); - } - - /** - * Schedules a NioSocketChannel to be registered by this selector. The channel will by queued and eventually - * registered next time through the event loop. - * @param nioSocketChannel the channel to register - */ - public void scheduleForRegistration(NioSocketChannel nioSocketChannel) { - SocketChannelContext channelContext = nioSocketChannel.getContext(); - newChannels.offer(channelContext); - ensureSelectorOpenForEnqueuing(newChannels, channelContext); - wakeup(); - } - - - /** - * Queues a write operation to be handled by the event loop. This can be called by any thread and is the - * api available for non-selector threads to schedule writes. - * - * @param writeOperation to be queued - */ - public void queueWrite(WriteOperation writeOperation) { - queuedWrites.offer(writeOperation); - if (isOpen() == false) { - boolean wasRemoved = queuedWrites.remove(writeOperation); - if (wasRemoved) { - writeOperation.getListener().accept(null, new ClosedSelectorException()); - } - } else { - wakeup(); - } - } - - /** - * Queues a write operation directly in a channel's buffer. Channel buffers are only safe to be accessed - * by the selector thread. As a result, this method should only be called by the selector thread. - * - * @param writeOperation to be queued in a channel's buffer - */ - public void queueWriteInChannelBuffer(WriteOperation writeOperation) { - assertOnSelectorThread(); - SocketChannelContext context = writeOperation.getChannel(); - try { - SelectionKeyUtils.setWriteInterested(context.getSelectionKey()); - context.queueWriteOperation(writeOperation); - } catch (Exception e) { - executeFailedListener(writeOperation.getListener(), e); - } - } - - /** - * Executes a success listener with consistent exception handling. This can only be called from current - * selector thread. - * - * @param listener to be executed - * @param value to provide to listener - */ - public void executeListener(BiConsumer listener, V value) { - assertOnSelectorThread(); - try { - listener.accept(value, null); - } catch (Exception e) { - eventHandler.listenerException(e); - } - } - - /** - * Executes a failed listener with consistent exception handling. This can only be called from current - * selector thread. - * - * @param listener to be executed - * @param exception to provide to listener - */ - public void executeFailedListener(BiConsumer listener, Exception exception) { - assertOnSelectorThread(); - try { - listener.accept(null, exception); - } catch (Exception e) { - eventHandler.listenerException(e); - } - } - - private void handleWrite(SocketChannelContext context) { - try { - eventHandler.handleWrite(context); - } catch (Exception e) { - eventHandler.writeException(context, e); - } - } - - private void handleRead(SocketChannelContext context) { - try { - eventHandler.handleRead(context); - } catch (Exception e) { - eventHandler.readException(context, e); - } - } - - private void handleQueuedWrites() { - WriteOperation writeOperation; - while ((writeOperation = queuedWrites.poll()) != null) { - if (writeOperation.getChannel().isOpen()) { - queueWriteInChannelBuffer(writeOperation); - } else { - executeFailedListener(writeOperation.getListener(), new ClosedChannelException()); - } - } - } - - private void setUpNewChannels() { - SocketChannelContext channelContext; - while ((channelContext = this.newChannels.poll()) != null) { - setupChannel(channelContext); - } - } - - private void setupChannel(SocketChannelContext context) { - assert context.getSelector() == this : "The channel must be registered with the selector with which it was created"; - try { - if (context.isOpen()) { - eventHandler.handleRegistration(context); - attemptConnect(context, false); - } else { - eventHandler.registrationException(context, new ClosedChannelException()); - } - } catch (Exception e) { - eventHandler.registrationException(context, e); - } - } - - private void attemptConnect(SocketChannelContext context, boolean connectEvent) { - try { - eventHandler.handleConnect(context); - if (connectEvent && context.isConnectComplete() == false) { - eventHandler.connectException(context, new IOException("Received OP_CONNECT but connect failed")); - } - } catch (Exception e) { - eventHandler.connectException(context, e); - } - } -} diff --git a/libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/AcceptingSelectorTests.java b/libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/AcceptingSelectorTests.java deleted file mode 100644 index 7536ad9d1e1a9..0000000000000 --- a/libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/AcceptingSelectorTests.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.nio; - -import org.elasticsearch.test.ESTestCase; -import org.junit.Before; - -import java.io.IOException; -import java.nio.channels.ClosedChannelException; -import java.nio.channels.SelectionKey; -import java.nio.channels.Selector; -import java.security.PrivilegedActionException; -import java.util.Collections; -import java.util.HashSet; - -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.same; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -public class AcceptingSelectorTests extends ESTestCase { - - private AcceptingSelector selector; - private NioServerSocketChannel serverChannel; - private AcceptorEventHandler eventHandler; - private TestSelectionKey selectionKey; - private Selector rawSelector; - private ServerChannelContext context; - - @Before - public void setUp() throws Exception { - super.setUp(); - - eventHandler = mock(AcceptorEventHandler.class); - serverChannel = mock(NioServerSocketChannel.class); - - rawSelector = mock(Selector.class); - selector = new AcceptingSelector(eventHandler, rawSelector); - this.selector.setThread(); - - context = mock(ServerChannelContext.class); - selectionKey = new TestSelectionKey(0); - selectionKey.attach(context); - when(context.getSelectionKey()).thenReturn(selectionKey); - when(context.getSelector()).thenReturn(selector); - when(context.isOpen()).thenReturn(true); - when(serverChannel.getContext()).thenReturn(context); - } - - public void testRegisteredChannel() throws IOException { - selector.scheduleForRegistration(serverChannel); - - selector.preSelect(); - - verify(eventHandler).handleRegistration(context); - } - - public void testClosedChannelWillNotBeRegistered() { - when(context.isOpen()).thenReturn(false); - selector.scheduleForRegistration(serverChannel); - - selector.preSelect(); - - verify(eventHandler).registrationException(same(context), any(ClosedChannelException.class)); - } - - public void testRegisterChannelFailsDueToException() throws Exception { - selector.scheduleForRegistration(serverChannel); - - ClosedChannelException closedChannelException = new ClosedChannelException(); - doThrow(closedChannelException).when(eventHandler).handleRegistration(context); - - selector.preSelect(); - - verify(eventHandler).registrationException(context, closedChannelException); - } - - public void testAcceptEvent() throws IOException { - selectionKey.setReadyOps(SelectionKey.OP_ACCEPT); - - selector.processKey(selectionKey); - - verify(eventHandler).acceptChannel(context); - } - - public void testAcceptException() throws IOException { - selectionKey.setReadyOps(SelectionKey.OP_ACCEPT); - IOException ioException = new IOException(); - - doThrow(ioException).when(eventHandler).acceptChannel(context); - - selector.processKey(selectionKey); - - verify(eventHandler).acceptException(context, ioException); - } - - public void testCleanup() throws IOException { - selector.scheduleForRegistration(serverChannel); - - selector.preSelect(); - - TestSelectionKey key = new TestSelectionKey(0); - key.attach(context); - when(rawSelector.keys()).thenReturn(new HashSet<>(Collections.singletonList(key))); - - selector.cleanupAndCloseChannels(); - - verify(eventHandler).handleClose(context); - } -} diff --git a/libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/AcceptorEventHandlerTests.java b/libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/AcceptorEventHandlerTests.java deleted file mode 100644 index a162a8e234c21..0000000000000 --- a/libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/AcceptorEventHandlerTests.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.nio; - -import org.elasticsearch.test.ESTestCase; -import org.junit.Before; - -import java.io.IOException; -import java.nio.channels.SelectionKey; -import java.nio.channels.ServerSocketChannel; -import java.nio.channels.SocketChannel; -import java.util.ArrayList; -import java.util.function.Consumer; - -import static org.mockito.Matchers.same; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -public class AcceptorEventHandlerTests extends ESTestCase { - - private AcceptorEventHandler handler; - private ChannelFactory channelFactory; - private NioServerSocketChannel channel; - private DoNotRegisterContext context; - private RoundRobinSupplier selectorSupplier; - - @Before - @SuppressWarnings("unchecked") - public void setUpHandler() throws IOException { - channelFactory = mock(ChannelFactory.class); - ArrayList selectors = new ArrayList<>(); - selectors.add(mock(SocketSelector.class)); - selectorSupplier = new RoundRobinSupplier<>(selectors.toArray(new SocketSelector[selectors.size()])); - handler = new AcceptorEventHandler(selectorSupplier, mock(Consumer.class)); - - channel = new NioServerSocketChannel(mock(ServerSocketChannel.class)); - context = new DoNotRegisterContext(channel, mock(AcceptingSelector.class), mock(Consumer.class)); - channel.setContext(context); - } - - public void testHandleRegisterSetsOP_ACCEPTInterest() throws IOException { - assertNull(context.getSelectionKey()); - - handler.handleRegistration(context); - - assertEquals(SelectionKey.OP_ACCEPT, channel.getContext().getSelectionKey().interestOps()); - } - - public void testRegisterAddsAttachment() throws IOException { - assertNull(context.getSelectionKey()); - - handler.handleRegistration(context); - - assertEquals(context, context.getSelectionKey().attachment()); - } - - public void testHandleAcceptCallsChannelFactory() throws IOException { - NioSocketChannel childChannel = new NioSocketChannel(mock(SocketChannel.class)); - NioSocketChannel nullChannel = null; - when(channelFactory.acceptNioChannel(same(context), same(selectorSupplier))).thenReturn(childChannel, nullChannel); - - handler.acceptChannel(context); - - verify(channelFactory, times(2)).acceptNioChannel(same(context), same(selectorSupplier)); - } - - @SuppressWarnings("unchecked") - public void testHandleAcceptCallsServerAcceptCallback() throws IOException { - NioSocketChannel childChannel = new NioSocketChannel(mock(SocketChannel.class)); - SocketChannelContext childContext = mock(SocketChannelContext.class); - childChannel.setContext(childContext); - ServerChannelContext serverChannelContext = mock(ServerChannelContext.class); - channel = new NioServerSocketChannel(mock(ServerSocketChannel.class)); - channel.setContext(serverChannelContext); - when(serverChannelContext.getChannel()).thenReturn(channel); - when(channelFactory.acceptNioChannel(same(context), same(selectorSupplier))).thenReturn(childChannel); - - handler.acceptChannel(serverChannelContext); - - verify(serverChannelContext).acceptChannels(selectorSupplier); - } - - public void testAcceptExceptionCallsExceptionHandler() throws IOException { - ServerChannelContext serverChannelContext = mock(ServerChannelContext.class); - IOException exception = new IOException(); - handler.acceptException(serverChannelContext, exception); - - verify(serverChannelContext).handleException(exception); - } - - private class DoNotRegisterContext extends ServerChannelContext { - - - @SuppressWarnings("unchecked") - DoNotRegisterContext(NioServerSocketChannel channel, AcceptingSelector selector, Consumer acceptor) { - super(channel, channelFactory, selector, acceptor, mock(Consumer.class)); - } - - @Override - public void register() { - setSelectionKey(new TestSelectionKey(0)); - } - } -} diff --git a/libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/BytesChannelContextTests.java b/libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/BytesChannelContextTests.java index e5c236e48a89c..2ab20522db65b 100644 --- a/libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/BytesChannelContextTests.java +++ b/libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/BytesChannelContextTests.java @@ -44,7 +44,7 @@ public class BytesChannelContextTests extends ESTestCase { private SocketChannel rawChannel; private BytesChannelContext context; private InboundChannelBuffer channelBuffer; - private SocketSelector selector; + private NioSelector selector; private BiConsumer listener; private int messageLength; @@ -54,7 +54,7 @@ public void init() { readConsumer = mock(CheckedFunction.class); messageLength = randomInt(96) + 20; - selector = mock(SocketSelector.class); + selector = mock(NioSelector.class); listener = mock(BiConsumer.class); channel = mock(NioSocketChannel.class); rawChannel = mock(SocketChannel.class); diff --git a/libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/ChannelContextTests.java b/libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/ChannelContextTests.java index 586dae83d082d..13372055668ed 100644 --- a/libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/ChannelContextTests.java +++ b/libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/ChannelContextTests.java @@ -115,7 +115,7 @@ public void closeChannel() { } @Override - public ESSelector getSelector() { + public NioSelector getSelector() { throw new UnsupportedOperationException("not implemented"); } diff --git a/libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/ChannelFactoryTests.java b/libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/ChannelFactoryTests.java index 858f547f8a6c1..8ff0cfcd0c876 100644 --- a/libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/ChannelFactoryTests.java +++ b/libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/ChannelFactoryTests.java @@ -43,18 +43,18 @@ public class ChannelFactoryTests extends ESTestCase { private ChannelFactory.RawChannelFactory rawChannelFactory; private SocketChannel rawChannel; private ServerSocketChannel rawServerChannel; - private SocketSelector socketSelector; - private Supplier socketSelectorSupplier; - private Supplier acceptingSelectorSupplier; - private AcceptingSelector acceptingSelector; + private NioSelector socketSelector; + private Supplier socketSelectorSupplier; + private Supplier acceptingSelectorSupplier; + private NioSelector acceptingSelector; @Before @SuppressWarnings("unchecked") public void setupFactory() throws IOException { rawChannelFactory = mock(ChannelFactory.RawChannelFactory.class); channelFactory = new TestChannelFactory(rawChannelFactory); - socketSelector = mock(SocketSelector.class); - acceptingSelector = mock(AcceptingSelector.class); + socketSelector = mock(NioSelector.class); + acceptingSelector = mock(NioSelector.class); socketSelectorSupplier = mock(Supplier.class); acceptingSelectorSupplier = mock(Supplier.class); rawChannel = SocketChannel.open(); @@ -139,14 +139,14 @@ private static class TestChannelFactory extends ChannelFactory channelExceptionHandler; private Consumer genericExceptionHandler; private ReadWriteHandler readWriteHandler; - private SocketEventHandler handler; - private NioSocketChannel channel; - private SocketChannel rawChannel; - private DoNotRegisterContext context; + private EventHandler handler; + private DoNotRegisterSocketContext context; + private DoNotRegisterServerContext serverContext; + private ChannelFactory channelFactory; + private RoundRobinSupplier selectorSupplier; @Before @SuppressWarnings("unchecked") @@ -51,16 +55,24 @@ public void setUpHandler() throws IOException { channelExceptionHandler = mock(Consumer.class); genericExceptionHandler = mock(Consumer.class); readWriteHandler = mock(ReadWriteHandler.class); - SocketSelector selector = mock(SocketSelector.class); - handler = new SocketEventHandler(genericExceptionHandler); - rawChannel = mock(SocketChannel.class); - channel = new NioSocketChannel(rawChannel); + channelFactory = mock(ChannelFactory.class); + NioSelector selector = mock(NioSelector.class); + ArrayList selectors = new ArrayList<>(); + selectors.add(selector); + selectorSupplier = new RoundRobinSupplier<>(selectors.toArray(new NioSelector[0])); + handler = new EventHandler(genericExceptionHandler, selectorSupplier); + + SocketChannel rawChannel = mock(SocketChannel.class); when(rawChannel.finishConnect()).thenReturn(true); - - context = new DoNotRegisterContext(channel, selector, channelExceptionHandler, new TestSelectionKey(0), readWriteHandler); + NioSocketChannel channel = new NioSocketChannel(rawChannel); + context = new DoNotRegisterSocketContext(channel, selector, channelExceptionHandler, readWriteHandler); channel.setContext(context); handler.handleRegistration(context); + NioServerSocketChannel serverChannel = new NioServerSocketChannel(mock(ServerSocketChannel.class)); + serverContext = new DoNotRegisterServerContext(serverChannel, mock(NioSelector.class), mock(Consumer.class)); + serverChannel.setContext(serverContext); + when(selector.isOnCurrentThread()).thenReturn(true); } @@ -73,7 +85,7 @@ public void testRegisterCallsContext() throws IOException { verify(channelContext).register(); } - public void testRegisterAddsOP_CONNECTAndOP_READInterest() throws IOException { + public void testRegisterNonServerAddsOP_CONNECTAndOP_READInterest() throws IOException { SocketChannelContext context = mock(SocketChannelContext.class); when(context.getSelectionKey()).thenReturn(new TestSelectionKey(0)); handler.handleRegistration(context); @@ -81,16 +93,55 @@ public void testRegisterAddsOP_CONNECTAndOP_READInterest() throws IOException { } public void testRegisterAddsAttachment() throws IOException { - SocketChannelContext context = mock(SocketChannelContext.class); + ChannelContext context = randomBoolean() ? mock(SocketChannelContext.class) : mock(ServerChannelContext.class); when(context.getSelectionKey()).thenReturn(new TestSelectionKey(0)); handler.handleRegistration(context); assertEquals(context, context.getSelectionKey().attachment()); } + public void testHandleServerRegisterSetsOP_ACCEPTInterest() throws IOException { + assertNull(serverContext.getSelectionKey()); + + handler.handleRegistration(serverContext); + + assertEquals(SelectionKey.OP_ACCEPT, serverContext.getSelectionKey().interestOps()); + } + + public void testHandleAcceptCallsChannelFactory() throws IOException { + NioSocketChannel childChannel = new NioSocketChannel(mock(SocketChannel.class)); + NioSocketChannel nullChannel = null; + when(channelFactory.acceptNioChannel(same(serverContext), same(selectorSupplier))).thenReturn(childChannel, nullChannel); + + handler.acceptChannel(serverContext); + + verify(channelFactory, times(2)).acceptNioChannel(same(serverContext), same(selectorSupplier)); + } + + @SuppressWarnings("unchecked") + public void testHandleAcceptCallsServerAcceptCallback() throws IOException { + NioSocketChannel childChannel = new NioSocketChannel(mock(SocketChannel.class)); + SocketChannelContext childContext = mock(SocketChannelContext.class); + childChannel.setContext(childContext); + ServerChannelContext serverChannelContext = mock(ServerChannelContext.class); + when(channelFactory.acceptNioChannel(same(serverContext), same(selectorSupplier))).thenReturn(childChannel); + + handler.acceptChannel(serverChannelContext); + + verify(serverChannelContext).acceptChannels(selectorSupplier); + } + + public void testAcceptExceptionCallsExceptionHandler() throws IOException { + ServerChannelContext serverChannelContext = mock(ServerChannelContext.class); + IOException exception = new IOException(); + handler.acceptException(serverChannelContext, exception); + + verify(serverChannelContext).handleException(exception); + } + public void testRegisterWithPendingWritesAddsOP_CONNECTAndOP_READAndOP_WRITEInterest() throws IOException { FlushReadyWrite flushReadyWrite = mock(FlushReadyWrite.class); when(readWriteHandler.writeToBytes(flushReadyWrite)).thenReturn(Collections.singletonList(flushReadyWrite)); - channel.getContext().queueWriteOperation(flushReadyWrite); + context.queueWriteOperation(flushReadyWrite); handler.handleRegistration(context); assertEquals(SelectionKey.OP_READ | SelectionKey.OP_CONNECT | SelectionKey.OP_WRITE, context.getSelectionKey().interestOps()); } @@ -120,11 +171,7 @@ public void testConnectExceptionCallsExceptionHandler() throws IOException { } public void testHandleReadDelegatesToContext() throws IOException { - NioSocketChannel channel = new NioSocketChannel(rawChannel); SocketChannelContext context = mock(SocketChannelContext.class); - channel.setContext(context); - - when(context.read()).thenReturn(1); handler.handleRead(context); verify(context).read(); } @@ -200,19 +247,31 @@ public void testListenerExceptionCallsGenericExceptionHandler() throws IOExcepti verify(genericExceptionHandler).accept(listenerException); } - private class DoNotRegisterContext extends BytesChannelContext { + private class DoNotRegisterSocketContext extends BytesChannelContext { - private final TestSelectionKey selectionKey; - DoNotRegisterContext(NioSocketChannel channel, SocketSelector selector, Consumer exceptionHandler, - TestSelectionKey selectionKey, ReadWriteHandler handler) { + DoNotRegisterSocketContext(NioSocketChannel channel, NioSelector selector, Consumer exceptionHandler, + ReadWriteHandler handler) { super(channel, selector, exceptionHandler, handler, InboundChannelBuffer.allocatingInstance()); - this.selectionKey = selectionKey; } @Override public void register() { - setSelectionKey(selectionKey); + setSelectionKey(new TestSelectionKey(0)); + } + } + + private class DoNotRegisterServerContext extends ServerChannelContext { + + + @SuppressWarnings("unchecked") + DoNotRegisterServerContext(NioServerSocketChannel channel, NioSelector selector, Consumer acceptor) { + super(channel, channelFactory, selector, acceptor, mock(Consumer.class)); + } + + @Override + public void register() { + setSelectionKey(new TestSelectionKey(0)); } } } diff --git a/libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/NioGroupTests.java b/libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/NioGroupTests.java index 13ce2c136544e..027f1255a590c 100644 --- a/libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/NioGroupTests.java +++ b/libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/NioGroupTests.java @@ -38,9 +38,8 @@ public class NioGroupTests extends ESTestCase { @SuppressWarnings("unchecked") public void setUp() throws Exception { super.setUp(); - nioGroup = new NioGroup(daemonThreadFactory(Settings.EMPTY, "acceptor"), 1, - (s) -> new AcceptorEventHandler(s, mock(Consumer.class)), daemonThreadFactory(Settings.EMPTY, "selector"), 1, - () -> new SocketEventHandler(mock(Consumer.class))); + nioGroup = new NioGroup(daemonThreadFactory(Settings.EMPTY, "acceptor"), 1, daemonThreadFactory(Settings.EMPTY, "selector"), 1, + (s) -> new EventHandler(mock(Consumer.class), s)); } @Override @@ -76,8 +75,8 @@ public void testCanCloseTwice() throws IOException { public void testExceptionAtStartIsHandled() throws IOException { RuntimeException ex = new RuntimeException(); CheckedRunnable ctor = () -> new NioGroup(r -> {throw ex;}, 1, - (s) -> new AcceptorEventHandler(s, mock(Consumer.class)), daemonThreadFactory(Settings.EMPTY, "selector"), - 1, () -> new SocketEventHandler(mock(Consumer.class))); + daemonThreadFactory(Settings.EMPTY, "selector"), + 1, (s) -> new EventHandler(mock(Consumer.class), s)); RuntimeException runtimeException = expectThrows(RuntimeException.class, ctor::run); assertSame(ex, runtimeException); // ctor starts threads. So we are testing that a failure to construct will stop threads. Our thread diff --git a/libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/SocketSelectorTests.java b/libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/NioSelectorTests.java similarity index 61% rename from libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/SocketSelectorTests.java rename to libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/NioSelectorTests.java index f8775d03b4277..dd3fea8bf50e8 100644 --- a/libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/SocketSelectorTests.java +++ b/libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/NioSelectorTests.java @@ -43,13 +43,15 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -public class SocketSelectorTests extends ESTestCase { +public class NioSelectorTests extends ESTestCase { - private SocketSelector socketSelector; - private SocketEventHandler eventHandler; + private NioSelector selector; + private EventHandler eventHandler; private NioSocketChannel channel; + private NioServerSocketChannel serverChannel; private TestSelectionKey selectionKey; private SocketChannelContext channelContext; + private ServerChannelContext serverChannelContext; private BiConsumer listener; private ByteBuffer[] buffers = {ByteBuffer.allocate(1)}; private Selector rawSelector; @@ -59,75 +61,172 @@ public class SocketSelectorTests extends ESTestCase { public void setUp() throws Exception { super.setUp(); rawSelector = mock(Selector.class); - eventHandler = mock(SocketEventHandler.class); + eventHandler = mock(EventHandler.class); channel = mock(NioSocketChannel.class); channelContext = mock(SocketChannelContext.class); + serverChannel = mock(NioServerSocketChannel.class); + serverChannelContext = mock(ServerChannelContext.class); listener = mock(BiConsumer.class); selectionKey = new TestSelectionKey(0); - selectionKey.attach(channelContext); - this.socketSelector = new SocketSelector(eventHandler, rawSelector); - this.socketSelector.setThread(); + this.selector = new NioSelector(eventHandler, rawSelector); + this.selector.setThread(); when(channel.getContext()).thenReturn(channelContext); when(channelContext.isOpen()).thenReturn(true); - when(channelContext.getSelector()).thenReturn(socketSelector); + when(channelContext.getSelector()).thenReturn(selector); when(channelContext.getSelectionKey()).thenReturn(selectionKey); when(channelContext.isConnectComplete()).thenReturn(true); + + when(serverChannel.getContext()).thenReturn(serverChannelContext); + when(serverChannelContext.isOpen()).thenReturn(true); + when(serverChannelContext.getSelector()).thenReturn(selector); + when(serverChannelContext.getSelectionKey()).thenReturn(selectionKey); } - public void testRegisterChannel() throws Exception { - socketSelector.scheduleForRegistration(channel); + @SuppressWarnings({"unchecked", "rawtypes"}) + public void testQueueChannelForClosed() throws IOException { + NioChannel channel = mock(NioChannel.class); + ChannelContext context = mock(ChannelContext.class); + when(channel.getContext()).thenReturn(context); + when(context.getSelector()).thenReturn(selector); - socketSelector.preSelect(); + selector.queueChannelClose(channel); - verify(eventHandler).handleRegistration(channelContext); + selector.singleLoop(); + + verify(eventHandler).handleClose(context); + } + + public void testSelectorClosedExceptionIsNotCaughtWhileRunning() throws IOException { + boolean closedSelectorExceptionCaught = false; + when(rawSelector.select(anyInt())).thenThrow(new ClosedSelectorException()); + try { + this.selector.singleLoop(); + } catch (ClosedSelectorException e) { + closedSelectorExceptionCaught = true; + } + + assertTrue(closedSelectorExceptionCaught); + } + + public void testIOExceptionWhileSelect() throws IOException { + IOException ioException = new IOException(); + + when(rawSelector.select(anyInt())).thenThrow(ioException); + + this.selector.singleLoop(); + + verify(eventHandler).selectorException(ioException); } - public void testClosedChannelWillNotBeRegistered() throws Exception { + public void testSelectorClosedIfOpenAndEventLoopNotRunning() throws IOException { + when(rawSelector.isOpen()).thenReturn(true); + selector.close(); + verify(rawSelector).close(); + } + + public void testRegisteredChannel() throws IOException { + selector.scheduleForRegistration(serverChannel); + + selector.preSelect(); + + verify(eventHandler).handleRegistration(serverChannelContext); + } + + public void testClosedServerChannelWillNotBeRegistered() { + when(serverChannelContext.isOpen()).thenReturn(false); + selector.scheduleForRegistration(serverChannel); + + selector.preSelect(); + + verify(eventHandler).registrationException(same(serverChannelContext), any(ClosedChannelException.class)); + } + + public void testRegisterServerChannelFailsDueToException() throws Exception { + selector.scheduleForRegistration(serverChannel); + + ClosedChannelException closedChannelException = new ClosedChannelException(); + doThrow(closedChannelException).when(eventHandler).handleRegistration(serverChannelContext); + + selector.preSelect(); + + verify(eventHandler).registrationException(serverChannelContext, closedChannelException); + } + + public void testClosedSocketChannelWillNotBeRegistered() throws Exception { when(channelContext.isOpen()).thenReturn(false); - socketSelector.scheduleForRegistration(channel); + selector.scheduleForRegistration(channel); - socketSelector.preSelect(); + selector.preSelect(); verify(eventHandler).registrationException(same(channelContext), any(ClosedChannelException.class)); verify(eventHandler, times(0)).handleConnect(channelContext); } - public void testRegisterChannelFailsDueToException() throws Exception { - socketSelector.scheduleForRegistration(channel); + public void testRegisterSocketChannelFailsDueToException() throws Exception { + selector.scheduleForRegistration(channel); ClosedChannelException closedChannelException = new ClosedChannelException(); doThrow(closedChannelException).when(eventHandler).handleRegistration(channelContext); - socketSelector.preSelect(); + selector.preSelect(); verify(eventHandler).registrationException(channelContext, closedChannelException); verify(eventHandler, times(0)).handleConnect(channelContext); } + public void testAcceptEvent() throws IOException { + selectionKey.setReadyOps(SelectionKey.OP_ACCEPT); + + selectionKey.attach(serverChannelContext); + selector.processKey(selectionKey); + + verify(eventHandler).acceptChannel(serverChannelContext); + } + + public void testAcceptException() throws IOException { + selectionKey.setReadyOps(SelectionKey.OP_ACCEPT); + IOException ioException = new IOException(); + + doThrow(ioException).when(eventHandler).acceptChannel(serverChannelContext); + + selectionKey.attach(serverChannelContext); + selector.processKey(selectionKey); + + verify(eventHandler).acceptException(serverChannelContext, ioException); + } + + public void testRegisterChannel() throws Exception { + selector.scheduleForRegistration(channel); + + selector.preSelect(); + + verify(eventHandler).handleRegistration(channelContext); + } + public void testSuccessfullyRegisterChannelWillAttemptConnect() throws Exception { - socketSelector.scheduleForRegistration(channel); + selector.scheduleForRegistration(channel); - socketSelector.preSelect(); + selector.preSelect(); verify(eventHandler).handleConnect(channelContext); } public void testQueueWriteWhenNotRunning() throws Exception { - socketSelector.close(); + selector.close(); - socketSelector.queueWrite(new FlushReadyWrite(channelContext, buffers, listener)); + selector.queueWrite(new FlushReadyWrite(channelContext, buffers, listener)); verify(listener).accept(isNull(Void.class), any(ClosedSelectorException.class)); } public void testQueueWriteChannelIsClosed() throws Exception { WriteOperation writeOperation = new FlushReadyWrite(channelContext, buffers, listener); - socketSelector.queueWrite(writeOperation); + selector.queueWrite(writeOperation); when(channelContext.isOpen()).thenReturn(false); - socketSelector.preSelect(); + selector.preSelect(); verify(channelContext, times(0)).queueWriteOperation(writeOperation); verify(listener).accept(isNull(Void.class), any(ClosedChannelException.class)); @@ -138,11 +237,11 @@ public void testQueueWriteSelectionKeyThrowsException() throws Exception { WriteOperation writeOperation = new FlushReadyWrite(channelContext, buffers, listener); CancelledKeyException cancelledKeyException = new CancelledKeyException(); - socketSelector.queueWrite(writeOperation); + selector.queueWrite(writeOperation); when(channelContext.getSelectionKey()).thenReturn(selectionKey); when(selectionKey.interestOps(anyInt())).thenThrow(cancelledKeyException); - socketSelector.preSelect(); + selector.preSelect(); verify(channelContext, times(0)).queueWriteOperation(writeOperation); verify(listener).accept(null, cancelledKeyException); @@ -150,11 +249,11 @@ public void testQueueWriteSelectionKeyThrowsException() throws Exception { public void testQueueWriteSuccessful() throws Exception { WriteOperation writeOperation = new FlushReadyWrite(channelContext, buffers, listener); - socketSelector.queueWrite(writeOperation); + selector.queueWrite(writeOperation); assertTrue((selectionKey.interestOps() & SelectionKey.OP_WRITE) == 0); - socketSelector.preSelect(); + selector.preSelect(); verify(channelContext).queueWriteOperation(writeOperation); assertTrue((selectionKey.interestOps() & SelectionKey.OP_WRITE) != 0); @@ -165,7 +264,7 @@ public void testQueueDirectlyInChannelBufferSuccessful() throws Exception { assertTrue((selectionKey.interestOps() & SelectionKey.OP_WRITE) == 0); - socketSelector.queueWriteInChannelBuffer(writeOperation); + selector.queueWriteInChannelBuffer(writeOperation); verify(channelContext).queueWriteOperation(writeOperation); assertTrue((selectionKey.interestOps() & SelectionKey.OP_WRITE) != 0); @@ -179,7 +278,7 @@ public void testQueueDirectlyInChannelBufferSelectionKeyThrowsException() throws when(channelContext.getSelectionKey()).thenReturn(selectionKey); when(selectionKey.interestOps(anyInt())).thenThrow(cancelledKeyException); - socketSelector.queueWriteInChannelBuffer(writeOperation); + selector.queueWriteInChannelBuffer(writeOperation); verify(channelContext, times(0)).queueWriteOperation(writeOperation); verify(listener).accept(null, cancelledKeyException); @@ -188,7 +287,8 @@ public void testQueueDirectlyInChannelBufferSelectionKeyThrowsException() throws public void testConnectEvent() throws Exception { selectionKey.setReadyOps(SelectionKey.OP_CONNECT); - socketSelector.processKey(selectionKey); + selectionKey.attach(channelContext); + selector.processKey(selectionKey); verify(eventHandler).handleConnect(channelContext); } @@ -199,7 +299,8 @@ public void testConnectEventFinishThrowException() throws Exception { selectionKey.setReadyOps(SelectionKey.OP_CONNECT); doThrow(ioException).when(eventHandler).handleConnect(channelContext); - socketSelector.processKey(selectionKey); + selectionKey.attach(channelContext); + selector.processKey(selectionKey); verify(eventHandler).connectException(channelContext, ioException); } @@ -212,7 +313,8 @@ public void testWillNotConsiderWriteOrReadUntilConnectionComplete() throws Excep doThrow(ioException).when(eventHandler).handleWrite(channelContext); when(channelContext.isConnectComplete()).thenReturn(false); - socketSelector.processKey(selectionKey); + selectionKey.attach(channelContext); + selector.processKey(selectionKey); verify(eventHandler, times(0)).handleWrite(channelContext); verify(eventHandler, times(0)).handleRead(channelContext); @@ -221,7 +323,8 @@ public void testWillNotConsiderWriteOrReadUntilConnectionComplete() throws Excep public void testSuccessfulWriteEvent() throws Exception { selectionKey.setReadyOps(SelectionKey.OP_WRITE); - socketSelector.processKey(selectionKey); + selectionKey.attach(channelContext); + selector.processKey(selectionKey); verify(eventHandler).handleWrite(channelContext); } @@ -229,11 +332,13 @@ public void testSuccessfulWriteEvent() throws Exception { public void testWriteEventWithException() throws Exception { IOException ioException = new IOException(); + selectionKey.attach(channelContext); selectionKey.setReadyOps(SelectionKey.OP_WRITE); doThrow(ioException).when(eventHandler).handleWrite(channelContext); - socketSelector.processKey(selectionKey); + selectionKey.attach(channelContext); + selector.processKey(selectionKey); verify(eventHandler).writeException(channelContext, ioException); } @@ -241,7 +346,8 @@ public void testWriteEventWithException() throws Exception { public void testSuccessfulReadEvent() throws Exception { selectionKey.setReadyOps(SelectionKey.OP_READ); - socketSelector.processKey(selectionKey); + selectionKey.attach(channelContext); + selector.processKey(selectionKey); verify(eventHandler).handleRead(channelContext); } @@ -253,7 +359,8 @@ public void testReadEventWithException() throws Exception { doThrow(ioException).when(eventHandler).handleRead(channelContext); - socketSelector.processKey(selectionKey); + selectionKey.attach(channelContext); + selector.processKey(selectionKey); verify(eventHandler).readException(channelContext, ioException); } @@ -261,7 +368,8 @@ public void testReadEventWithException() throws Exception { public void testWillCallPostHandleAfterChannelHandling() throws Exception { selectionKey.setReadyOps(SelectionKey.OP_WRITE | SelectionKey.OP_READ); - socketSelector.processKey(selectionKey); + selectionKey.attach(channelContext); + selector.processKey(selectionKey); verify(eventHandler).handleWrite(channelContext); verify(eventHandler).handleRead(channelContext); @@ -273,18 +381,18 @@ public void testCleanup() throws Exception { SocketChannelContext unregisteredContext = mock(SocketChannelContext.class); when(unregisteredChannel.getContext()).thenReturn(unregisteredContext); - socketSelector.scheduleForRegistration(channel); + selector.scheduleForRegistration(channel); - socketSelector.preSelect(); + selector.preSelect(); - socketSelector.queueWrite(new FlushReadyWrite(channelContext, buffers, listener)); - socketSelector.scheduleForRegistration(unregisteredChannel); + selector.queueWrite(new FlushReadyWrite(channelContext, buffers, listener)); + selector.scheduleForRegistration(unregisteredChannel); TestSelectionKey testSelectionKey = new TestSelectionKey(0); testSelectionKey.attach(channelContext); when(rawSelector.keys()).thenReturn(new HashSet<>(Collections.singletonList(testSelectionKey))); - socketSelector.cleanupAndCloseChannels(); + selector.cleanupAndCloseChannels(); verify(listener).accept(isNull(Void.class), any(ClosedSelectorException.class)); verify(eventHandler).handleClose(channelContext); @@ -295,7 +403,7 @@ public void testExecuteListenerWillHandleException() throws Exception { RuntimeException exception = new RuntimeException(); doThrow(exception).when(listener).accept(null, null); - socketSelector.executeListener(listener, null); + selector.executeListener(listener, null); verify(eventHandler).listenerException(exception); } @@ -305,7 +413,7 @@ public void testExecuteFailedListenerWillHandleException() throws Exception { RuntimeException exception = new RuntimeException(); doThrow(exception).when(listener).accept(null, ioException); - socketSelector.executeFailedListener(listener, ioException); + selector.executeFailedListener(listener, ioException); verify(eventHandler).listenerException(exception); } diff --git a/libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/SocketChannelContextTests.java b/libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/SocketChannelContextTests.java index f27052ac5d5f4..fdb4a77b922e2 100644 --- a/libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/SocketChannelContextTests.java +++ b/libs/elasticsearch-nio/src/test/java/org/elasticsearch/nio/SocketChannelContextTests.java @@ -51,7 +51,7 @@ public class SocketChannelContextTests extends ESTestCase { private Consumer exceptionHandler; private NioSocketChannel channel; private BiConsumer listener; - private SocketSelector selector; + private NioSelector selector; private ReadWriteHandler readWriteHandler; @SuppressWarnings("unchecked") @@ -64,7 +64,7 @@ public void setup() throws Exception { listener = mock(BiConsumer.class); when(channel.getRawChannel()).thenReturn(rawChannel); exceptionHandler = mock(Consumer.class); - selector = mock(SocketSelector.class); + selector = mock(NioSelector.class); readWriteHandler = mock(ReadWriteHandler.class); InboundChannelBuffer channelBuffer = InboundChannelBuffer.allocatingInstance(); context = new TestSocketChannelContext(channel, selector, exceptionHandler, readWriteHandler, channelBuffer); @@ -275,7 +275,7 @@ public void testCloseClosesChannelBuffer() throws IOException { private static class TestSocketChannelContext extends SocketChannelContext { - private TestSocketChannelContext(NioSocketChannel channel, SocketSelector selector, Consumer exceptionHandler, + private TestSocketChannelContext(NioSocketChannel channel, NioSelector selector, Consumer exceptionHandler, ReadWriteHandler readWriteHandler, InboundChannelBuffer channelBuffer) { super(channel, selector, exceptionHandler, readWriteHandler, channelBuffer); } diff --git a/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/NioHttpServerTransport.java b/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/NioHttpServerTransport.java index ce0ed83aad420..495dbf9729391 100644 --- a/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/NioHttpServerTransport.java +++ b/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/NioHttpServerTransport.java @@ -47,10 +47,9 @@ import org.elasticsearch.http.HttpStats; import org.elasticsearch.http.nio.cors.NioCorsConfig; import org.elasticsearch.http.nio.cors.NioCorsConfigBuilder; -import org.elasticsearch.nio.AcceptingSelector; -import org.elasticsearch.nio.AcceptorEventHandler; import org.elasticsearch.nio.BytesChannelContext; import org.elasticsearch.nio.ChannelFactory; +import org.elasticsearch.nio.EventHandler; import org.elasticsearch.nio.InboundChannelBuffer; import org.elasticsearch.nio.NioChannel; import org.elasticsearch.nio.NioGroup; @@ -58,8 +57,7 @@ import org.elasticsearch.nio.NioSocketChannel; import org.elasticsearch.nio.ServerChannelContext; import org.elasticsearch.nio.SocketChannelContext; -import org.elasticsearch.nio.SocketEventHandler; -import org.elasticsearch.nio.SocketSelector; +import org.elasticsearch.nio.NioSelector; import org.elasticsearch.rest.RestUtils; import org.elasticsearch.threadpool.ThreadPool; @@ -180,9 +178,8 @@ protected void doStart() { int acceptorCount = NIO_HTTP_ACCEPTOR_COUNT.get(settings); int workerCount = NIO_HTTP_WORKER_COUNT.get(settings); nioGroup = new NioGroup(daemonThreadFactory(this.settings, TRANSPORT_ACCEPTOR_THREAD_NAME_PREFIX), acceptorCount, - (s) -> new AcceptorEventHandler(s, this::nonChannelExceptionCaught), daemonThreadFactory(this.settings, TRANSPORT_WORKER_THREAD_NAME_PREFIX), workerCount, - () -> new SocketEventHandler(this::nonChannelExceptionCaught)); + (s) -> new EventHandler(this::nonChannelExceptionCaught, s)); channelFactory = new HttpChannelFactory(); this.boundAddress = createBoundHttpAddress(); @@ -360,7 +357,7 @@ private HttpChannelFactory() { } @Override - public NioSocketChannel createChannel(SocketSelector selector, SocketChannel channel) throws IOException { + public NioSocketChannel createChannel(NioSelector selector, SocketChannel channel) throws IOException { NioSocketChannel nioChannel = new NioSocketChannel(channel); HttpReadWriteHandler httpReadWritePipeline = new HttpReadWriteHandler(nioChannel,NioHttpServerTransport.this, httpHandlingSettings, xContentRegistry, corsConfig, threadPool.getThreadContext()); @@ -372,7 +369,7 @@ public NioSocketChannel createChannel(SocketSelector selector, SocketChannel cha } @Override - public NioServerSocketChannel createServerChannel(AcceptingSelector selector, ServerSocketChannel channel) throws IOException { + public NioServerSocketChannel createServerChannel(NioSelector selector, ServerSocketChannel channel) throws IOException { NioServerSocketChannel nioChannel = new NioServerSocketChannel(channel); Consumer exceptionHandler = (e) -> logger.error(() -> new ParameterizedMessage("exception from server channel caught on transport layer [{}]", channel), e); diff --git a/plugins/transport-nio/src/main/java/org/elasticsearch/transport/nio/NioTransport.java b/plugins/transport-nio/src/main/java/org/elasticsearch/transport/nio/NioTransport.java index 2ef49d7791234..b85d707dcd934 100644 --- a/plugins/transport-nio/src/main/java/org/elasticsearch/transport/nio/NioTransport.java +++ b/plugins/transport-nio/src/main/java/org/elasticsearch/transport/nio/NioTransport.java @@ -31,16 +31,14 @@ import org.elasticsearch.common.util.PageCacheRecycler; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.indices.breaker.CircuitBreakerService; -import org.elasticsearch.nio.AcceptingSelector; -import org.elasticsearch.nio.AcceptorEventHandler; import org.elasticsearch.nio.BytesChannelContext; import org.elasticsearch.nio.ChannelFactory; +import org.elasticsearch.nio.EventHandler; import org.elasticsearch.nio.InboundChannelBuffer; import org.elasticsearch.nio.NioGroup; +import org.elasticsearch.nio.NioSelector; import org.elasticsearch.nio.NioSocketChannel; import org.elasticsearch.nio.ServerChannelContext; -import org.elasticsearch.nio.SocketEventHandler; -import org.elasticsearch.nio.SocketSelector; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TcpChannel; import org.elasticsearch.transport.TcpTransport; @@ -55,23 +53,18 @@ import java.util.function.Consumer; import java.util.function.Supplier; -import static org.elasticsearch.common.settings.Setting.intSetting; import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.newConcurrentMap; import static org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadFactory; public class NioTransport extends TcpTransport { private static final String TRANSPORT_WORKER_THREAD_NAME_PREFIX = Transports.NIO_TRANSPORT_WORKER_THREAD_NAME_PREFIX; - private static final String TRANSPORT_ACCEPTOR_THREAD_NAME_PREFIX = Transports.NIO_TRANSPORT_ACCEPTOR_THREAD_NAME_PREFIX; public static final Setting NIO_WORKER_COUNT = new Setting<>("transport.nio.worker_count", (s) -> Integer.toString(EsExecutors.numberOfProcessors(s) * 2), (s) -> Setting.parseInt(s, 1, "transport.nio.worker_count"), Setting.Property.NodeScope); - public static final Setting NIO_ACCEPTOR_COUNT = - intSetting("transport.nio.acceptor_count", 1, 1, Setting.Property.NodeScope); - protected final PageCacheRecycler pageCacheRecycler; private final ConcurrentMap profileToChannelFactory = newConcurrentMap(); private volatile NioGroup nioGroup; @@ -101,20 +94,13 @@ protected TcpNioSocketChannel initiateChannel(InetSocketAddress address, ActionL protected void doStart() { boolean success = false; try { - int acceptorCount = 0; - boolean useNetworkServer = NetworkService.NETWORK_SERVER.get(settings); - if (useNetworkServer) { - acceptorCount = NioTransport.NIO_ACCEPTOR_COUNT.get(settings); - } - nioGroup = new NioGroup(daemonThreadFactory(this.settings, TRANSPORT_ACCEPTOR_THREAD_NAME_PREFIX), acceptorCount, - (s) -> new AcceptorEventHandler(s, this::onNonChannelException), - daemonThreadFactory(this.settings, TRANSPORT_WORKER_THREAD_NAME_PREFIX), NioTransport.NIO_WORKER_COUNT.get(settings), - () -> new SocketEventHandler(this::onNonChannelException)); + nioGroup = new NioGroup(daemonThreadFactory(this.settings, TRANSPORT_WORKER_THREAD_NAME_PREFIX), + NioTransport.NIO_WORKER_COUNT.get(settings), (s) -> new EventHandler(this::onNonChannelException, s)); ProfileSettings clientProfileSettings = new ProfileSettings(settings, "default"); clientChannelFactory = channelFactory(clientProfileSettings, true); - if (useNetworkServer) { + if (NetworkService.NETWORK_SERVER.get(settings)) { // loop through all profiles and start them up, special handling for default one for (ProfileSettings profileSettings : profileSettings) { String profileName = profileSettings.profileName; @@ -178,7 +164,7 @@ private TcpChannelFactoryImpl(ProfileSettings profileSettings) { } @Override - public TcpNioSocketChannel createChannel(SocketSelector selector, SocketChannel channel) throws IOException { + public TcpNioSocketChannel createChannel(NioSelector selector, SocketChannel channel) throws IOException { TcpNioSocketChannel nioChannel = new TcpNioSocketChannel(profileName, channel); Supplier pageSupplier = () -> { Recycler.V bytes = pageCacheRecycler.bytePage(false); @@ -193,7 +179,7 @@ public TcpNioSocketChannel createChannel(SocketSelector selector, SocketChannel } @Override - public TcpNioServerSocketChannel createServerChannel(AcceptingSelector selector, ServerSocketChannel channel) throws IOException { + public TcpNioServerSocketChannel createServerChannel(NioSelector selector, ServerSocketChannel channel) throws IOException { TcpNioServerSocketChannel nioChannel = new TcpNioServerSocketChannel(profileName, channel); Consumer exceptionHandler = (e) -> logger.error(() -> new ParameterizedMessage("exception from server channel caught on transport layer [{}]", channel), e); diff --git a/plugins/transport-nio/src/main/java/org/elasticsearch/transport/nio/NioTransportPlugin.java b/plugins/transport-nio/src/main/java/org/elasticsearch/transport/nio/NioTransportPlugin.java index 422e3e9b83330..1cc94f18dd3c1 100644 --- a/plugins/transport-nio/src/main/java/org/elasticsearch/transport/nio/NioTransportPlugin.java +++ b/plugins/transport-nio/src/main/java/org/elasticsearch/transport/nio/NioTransportPlugin.java @@ -50,8 +50,7 @@ public List> getSettings() { return Arrays.asList( NioHttpServerTransport.NIO_HTTP_ACCEPTOR_COUNT, NioHttpServerTransport.NIO_HTTP_WORKER_COUNT, - NioTransport.NIO_WORKER_COUNT, - NioTransport.NIO_ACCEPTOR_COUNT + NioTransport.NIO_WORKER_COUNT ); } diff --git a/plugins/transport-nio/src/main/java/org/elasticsearch/transport/nio/TcpNioServerSocketChannel.java b/plugins/transport-nio/src/main/java/org/elasticsearch/transport/nio/TcpNioServerSocketChannel.java index c63acc9f4decc..946563225c66c 100644 --- a/plugins/transport-nio/src/main/java/org/elasticsearch/transport/nio/TcpNioServerSocketChannel.java +++ b/plugins/transport-nio/src/main/java/org/elasticsearch/transport/nio/TcpNioServerSocketChannel.java @@ -21,8 +21,6 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.nio.AcceptingSelector; -import org.elasticsearch.nio.ChannelFactory; import org.elasticsearch.nio.NioServerSocketChannel; import org.elasticsearch.transport.TcpChannel; diff --git a/plugins/transport-nio/src/main/java/org/elasticsearch/transport/nio/TcpNioSocketChannel.java b/plugins/transport-nio/src/main/java/org/elasticsearch/transport/nio/TcpNioSocketChannel.java index 44ab17457e8fb..ef2bc875aa994 100644 --- a/plugins/transport-nio/src/main/java/org/elasticsearch/transport/nio/TcpNioSocketChannel.java +++ b/plugins/transport-nio/src/main/java/org/elasticsearch/transport/nio/TcpNioSocketChannel.java @@ -22,7 +22,6 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.nio.NioSocketChannel; -import org.elasticsearch.nio.SocketSelector; import org.elasticsearch.transport.TcpChannel; import java.io.IOException; diff --git a/server/src/main/java/org/elasticsearch/transport/Transports.java b/server/src/main/java/org/elasticsearch/transport/Transports.java index d07846835c23f..2adf07bad2356 100644 --- a/server/src/main/java/org/elasticsearch/transport/Transports.java +++ b/server/src/main/java/org/elasticsearch/transport/Transports.java @@ -30,7 +30,6 @@ public enum Transports { public static final String TEST_MOCK_TRANSPORT_THREAD_PREFIX = "__mock_network_thread"; public static final String NIO_TRANSPORT_WORKER_THREAD_NAME_PREFIX = "es_nio_transport_worker"; - public static final String NIO_TRANSPORT_ACCEPTOR_THREAD_NAME_PREFIX = "es_nio_transport_acceptor"; /** * Utility method to detect whether a thread is a network thread. Typically @@ -44,8 +43,7 @@ public static final boolean isTransportThread(Thread t) { TcpTransport.TRANSPORT_SERVER_WORKER_THREAD_NAME_PREFIX, TcpTransport.TRANSPORT_CLIENT_BOSS_THREAD_NAME_PREFIX, TEST_MOCK_TRANSPORT_THREAD_PREFIX, - NIO_TRANSPORT_WORKER_THREAD_NAME_PREFIX, - NIO_TRANSPORT_ACCEPTOR_THREAD_NAME_PREFIX)) { + NIO_TRANSPORT_WORKER_THREAD_NAME_PREFIX)) { if (threadName.contains(s)) { return true; } diff --git a/test/framework/src/main/java/org/elasticsearch/transport/nio/MockNioTransport.java b/test/framework/src/main/java/org/elasticsearch/transport/nio/MockNioTransport.java index 9481f60d93384..cb9e243660a8e 100644 --- a/test/framework/src/main/java/org/elasticsearch/transport/nio/MockNioTransport.java +++ b/test/framework/src/main/java/org/elasticsearch/transport/nio/MockNioTransport.java @@ -30,17 +30,15 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.PageCacheRecycler; import org.elasticsearch.indices.breaker.CircuitBreakerService; -import org.elasticsearch.nio.AcceptingSelector; -import org.elasticsearch.nio.AcceptorEventHandler; import org.elasticsearch.nio.BytesChannelContext; import org.elasticsearch.nio.BytesWriteHandler; import org.elasticsearch.nio.ChannelFactory; import org.elasticsearch.nio.InboundChannelBuffer; import org.elasticsearch.nio.NioGroup; +import org.elasticsearch.nio.NioSelector; import org.elasticsearch.nio.NioServerSocketChannel; import org.elasticsearch.nio.NioSocketChannel; import org.elasticsearch.nio.ServerChannelContext; -import org.elasticsearch.nio.SocketSelector; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TcpChannel; import org.elasticsearch.transport.TcpTransport; @@ -62,7 +60,6 @@ public class MockNioTransport extends TcpTransport { private static final String TRANSPORT_WORKER_THREAD_NAME_PREFIX = Transports.NIO_TRANSPORT_WORKER_THREAD_NAME_PREFIX; - private static final String TRANSPORT_ACCEPTOR_THREAD_NAME_PREFIX = Transports.NIO_TRANSPORT_ACCEPTOR_THREAD_NAME_PREFIX; private final PageCacheRecycler pageCacheRecycler; private final ConcurrentMap profileToChannelFactory = newConcurrentMap(); @@ -93,20 +90,13 @@ protected MockSocketChannel initiateChannel(InetSocketAddress address, ActionLis protected void doStart() { boolean success = false; try { - int acceptorCount = 0; - boolean useNetworkServer = NetworkService.NETWORK_SERVER.get(settings); - if (useNetworkServer) { - acceptorCount = 1; - } - nioGroup = new NioGroup(daemonThreadFactory(this.settings, TRANSPORT_ACCEPTOR_THREAD_NAME_PREFIX), acceptorCount, - (s) -> new AcceptorEventHandler(s, this::onNonChannelException), - daemonThreadFactory(this.settings, TRANSPORT_WORKER_THREAD_NAME_PREFIX), 2, - () -> new TestingSocketEventHandler(this::onNonChannelException)); + nioGroup = new NioGroup(daemonThreadFactory(this.settings, TRANSPORT_WORKER_THREAD_NAME_PREFIX), 2, + (s) -> new TestingSocketEventHandler(this::onNonChannelException, s)); ProfileSettings clientProfileSettings = new ProfileSettings(settings, "default"); clientChannelFactory = new MockTcpChannelFactory(clientProfileSettings, "client"); - if (useNetworkServer) { + if (NetworkService.NETWORK_SERVER.get(settings)) { // loop through all profiles and start them up, special handling for default one for (ProfileSettings profileSettings : profileSettings) { String profileName = profileSettings.profileName; @@ -159,7 +149,7 @@ private MockTcpChannelFactory(ProfileSettings profileSettings, String profileNam } @Override - public MockSocketChannel createChannel(SocketSelector selector, SocketChannel channel) throws IOException { + public MockSocketChannel createChannel(NioSelector selector, SocketChannel channel) throws IOException { MockSocketChannel nioChannel = new MockSocketChannel(profileName, channel, selector); Supplier pageSupplier = () -> { Recycler.V bytes = pageCacheRecycler.bytePage(false); @@ -173,7 +163,7 @@ public MockSocketChannel createChannel(SocketSelector selector, SocketChannel ch } @Override - public MockServerChannel createServerChannel(AcceptingSelector selector, ServerSocketChannel channel) throws IOException { + public MockServerChannel createServerChannel(NioSelector selector, ServerSocketChannel channel) throws IOException { MockServerChannel nioServerChannel = new MockServerChannel(profileName, channel, this, selector); Consumer exceptionHandler = (e) -> logger.error(() -> new ParameterizedMessage("exception from server channel caught on transport layer [{}]", channel), e); @@ -205,7 +195,7 @@ private static class MockServerChannel extends NioServerSocketChannel implements private final String profile; - MockServerChannel(String profile, ServerSocketChannel channel, ChannelFactory channelFactory, AcceptingSelector selector) + MockServerChannel(String profile, ServerSocketChannel channel, ChannelFactory channelFactory, NioSelector selector) throws IOException { super(channel); this.profile = profile; @@ -246,7 +236,7 @@ private static class MockSocketChannel extends NioSocketChannel implements TcpCh private final String profile; - private MockSocketChannel(String profile, java.nio.channels.SocketChannel socketChannel, SocketSelector selector) + private MockSocketChannel(String profile, java.nio.channels.SocketChannel socketChannel, NioSelector selector) throws IOException { super(socketChannel); this.profile = profile; diff --git a/test/framework/src/main/java/org/elasticsearch/transport/nio/TestingSocketEventHandler.java b/test/framework/src/main/java/org/elasticsearch/transport/nio/TestingSocketEventHandler.java index 810e42010224c..cecd3c606138a 100644 --- a/test/framework/src/main/java/org/elasticsearch/transport/nio/TestingSocketEventHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/transport/nio/TestingSocketEventHandler.java @@ -19,21 +19,23 @@ package org.elasticsearch.transport.nio; +import org.elasticsearch.nio.EventHandler; +import org.elasticsearch.nio.NioSelector; import org.elasticsearch.nio.SocketChannelContext; -import org.elasticsearch.nio.SocketEventHandler; import java.io.IOException; import java.util.Collections; import java.util.Set; import java.util.WeakHashMap; import java.util.function.Consumer; +import java.util.function.Supplier; -public class TestingSocketEventHandler extends SocketEventHandler { +public class TestingSocketEventHandler extends EventHandler { private Set hasConnectedMap = Collections.newSetFromMap(new WeakHashMap<>()); - public TestingSocketEventHandler(Consumer exceptionHandler) { - super(exceptionHandler); + public TestingSocketEventHandler(Consumer exceptionHandler, Supplier selectorSupplier) { + super(exceptionHandler, selectorSupplier); } public void handleConnect(SocketChannelContext context) throws IOException { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SSLChannelContext.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SSLChannelContext.java index 171507de7413e..95af766515777 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SSLChannelContext.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SSLChannelContext.java @@ -11,7 +11,7 @@ import org.elasticsearch.nio.NioSocketChannel; import org.elasticsearch.nio.ReadWriteHandler; import org.elasticsearch.nio.SocketChannelContext; -import org.elasticsearch.nio.SocketSelector; +import org.elasticsearch.nio.NioSelector; import org.elasticsearch.nio.WriteOperation; import java.io.IOException; @@ -28,7 +28,7 @@ public final class SSLChannelContext extends SocketChannelContext { private final SSLDriver sslDriver; - SSLChannelContext(NioSocketChannel channel, SocketSelector selector, Consumer exceptionHandler, SSLDriver sslDriver, + SSLChannelContext(NioSocketChannel channel, NioSelector selector, Consumer exceptionHandler, SSLDriver sslDriver, ReadWriteHandler readWriteHandler, InboundChannelBuffer channelBuffer) { super(channel, selector, exceptionHandler, readWriteHandler, channelBuffer); this.sslDriver = sslDriver; @@ -140,7 +140,7 @@ public boolean selectorShouldClose() { public void closeChannel() { if (isClosing.compareAndSet(false, true)) { WriteOperation writeOperation = new CloseNotifyOperation(this); - SocketSelector selector = getSelector(); + NioSelector selector = getSelector(); if (selector.isOnCurrentThread() == false) { selector.queueWrite(writeOperation); return; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SecurityNioTransport.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SecurityNioTransport.java index 1c9d779c2cc37..39ce1a0150c4f 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SecurityNioTransport.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SecurityNioTransport.java @@ -13,11 +13,10 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.PageCacheRecycler; import org.elasticsearch.indices.breaker.CircuitBreakerService; -import org.elasticsearch.nio.AcceptingSelector; import org.elasticsearch.nio.InboundChannelBuffer; import org.elasticsearch.nio.NioSocketChannel; import org.elasticsearch.nio.ServerChannelContext; -import org.elasticsearch.nio.SocketSelector; +import org.elasticsearch.nio.NioSelector; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TcpTransport; import org.elasticsearch.transport.nio.NioTransport; @@ -117,7 +116,7 @@ private SecurityTcpChannelFactory(ProfileSettings profileSettings, boolean isCli } @Override - public TcpNioSocketChannel createChannel(SocketSelector selector, SocketChannel channel) throws IOException { + public TcpNioSocketChannel createChannel(NioSelector selector, SocketChannel channel) throws IOException { SSLConfiguration defaultConfig = profileConfiguration.get(TcpTransport.DEFAULT_PROFILE); SSLEngine sslEngine = sslService.createSSLEngine(profileConfiguration.getOrDefault(profileName, defaultConfig), null, -1); SSLDriver sslDriver = new SSLDriver(sslEngine, isClient); @@ -136,7 +135,7 @@ public TcpNioSocketChannel createChannel(SocketSelector selector, SocketChannel } @Override - public TcpNioServerSocketChannel createServerChannel(AcceptingSelector selector, ServerSocketChannel channel) throws IOException { + public TcpNioServerSocketChannel createServerChannel(NioSelector selector, ServerSocketChannel channel) throws IOException { TcpNioServerSocketChannel nioChannel = new TcpNioServerSocketChannel(profileName, channel); Consumer exceptionHandler = (e) -> logger.error(() -> new ParameterizedMessage("exception from server channel caught on transport layer [{}]", channel), e); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SSLChannelContextTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SSLChannelContextTests.java index 168dcd64e6c5f..14a22d300d12d 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SSLChannelContextTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SSLChannelContextTests.java @@ -11,7 +11,7 @@ import org.elasticsearch.nio.FlushReadyWrite; import org.elasticsearch.nio.InboundChannelBuffer; import org.elasticsearch.nio.NioSocketChannel; -import org.elasticsearch.nio.SocketSelector; +import org.elasticsearch.nio.NioSelector; import org.elasticsearch.nio.WriteOperation; import org.elasticsearch.test.ESTestCase; import org.junit.Before; @@ -40,7 +40,7 @@ public class SSLChannelContextTests extends ESTestCase { private SocketChannel rawChannel; private SSLChannelContext context; private InboundChannelBuffer channelBuffer; - private SocketSelector selector; + private NioSelector selector; private BiConsumer listener; private Consumer exceptionHandler; private SSLDriver sslDriver; @@ -55,7 +55,7 @@ public void init() { TestReadWriteHandler readWriteHandler = new TestReadWriteHandler(readConsumer); messageLength = randomInt(96) + 20; - selector = mock(SocketSelector.class); + selector = mock(NioSelector.class); listener = mock(BiConsumer.class); channel = mock(NioSocketChannel.class); rawChannel = mock(SocketChannel.class); From b4514d3cc12c18cea5395c3dcb7f41455ec4ec6b Mon Sep 17 00:00:00 2001 From: Lisa Cawley Date: Wed, 6 Jun 2018 12:39:24 -0700 Subject: [PATCH 23/26] [DOCS] Moves ML node info to docs (#31142) --- .../en/node.asciidoc => docs/reference/modules/ml-node.asciidoc | 0 docs/reference/modules/node.asciidoc | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename x-pack/docs/en/node.asciidoc => docs/reference/modules/ml-node.asciidoc (100%) diff --git a/x-pack/docs/en/node.asciidoc b/docs/reference/modules/ml-node.asciidoc similarity index 100% rename from x-pack/docs/en/node.asciidoc rename to docs/reference/modules/ml-node.asciidoc diff --git a/docs/reference/modules/node.asciidoc b/docs/reference/modules/node.asciidoc index cf053df1818fa..f772977e3f02f 100644 --- a/docs/reference/modules/node.asciidoc +++ b/docs/reference/modules/node.asciidoc @@ -325,5 +325,5 @@ the <>, the <> and the <>. ifdef::include-xpack[] -include::{xes-repo-dir}/node.asciidoc[] +include::ml-node.asciidoc[] endif::include-xpack[] From 68827fc0465abe1e365815fd344e900f593e5008 Mon Sep 17 00:00:00 2001 From: Lisa Cawley Date: Wed, 6 Jun 2018 13:25:36 -0700 Subject: [PATCH 24/26] [DOCS] Enables testing for monitoring examples (#31119) --- x-pack/docs/build.gradle | 1 - x-pack/docs/en/monitoring/indices.asciidoc | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/docs/build.gradle b/x-pack/docs/build.gradle index 3d799c8d0b548..a0af24d6cc645 100644 --- a/x-pack/docs/build.gradle +++ b/x-pack/docs/build.gradle @@ -17,7 +17,6 @@ buildRestTests.expectedUnconvertedCandidates = [ 'en/ml/functions/sum.asciidoc', 'en/ml/functions/time.asciidoc', 'en/ml/customurl.asciidoc', - 'en/monitoring/indices.asciidoc', 'en/rest-api/security/ssl.asciidoc', 'en/rest-api/security/users.asciidoc', 'en/rest-api/security/tokens.asciidoc', diff --git a/x-pack/docs/en/monitoring/indices.asciidoc b/x-pack/docs/en/monitoring/indices.asciidoc index 10d2c212de273..efa9836daa2e9 100644 --- a/x-pack/docs/en/monitoring/indices.asciidoc +++ b/x-pack/docs/en/monitoring/indices.asciidoc @@ -11,6 +11,8 @@ You can retrieve the templates through the `_template` API: ---------------------------------- GET /_template/.monitoring-* ---------------------------------- +// CONSOLE +// TEST[catch:missing] By default, the template configures one shard and one replica for the monitoring indices. To override the default settings, add your own template: @@ -36,6 +38,7 @@ PUT /_template/custom_monitoring } } ---------------------------------- +// CONSOLE IMPORTANT: Only set the `number_of_shards` and `number_of_replicas` in the settings section. Overriding other monitoring template settings could cause From be4a101ea1824d4e6be1dec7341ddbe1d620d1c5 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Wed, 6 Jun 2018 23:17:45 +0200 Subject: [PATCH 25/26] Add high-level client methods that accept RequestOptions (#31069) With #30490 we have introduced a new way to provide request options whenever sending a request using the high-level REST client. Before you could provide headers as the last argument varargs of each API method, now you can provide `RequestOptions` that in the future will allow to provide more options which can be specified per request. This commit deprecates all of the client methods that accept a `Header` varargs argument in favour of new methods that accept `RequestOptions` instead. For some API we don't even go through deprecation given that they were not released since they were added, hence in that case we can just move them to the new method. --- .../elasticsearch/client/ClusterClient.java | 36 +- .../elasticsearch/client/IndicesClient.java | 698 +++++++++++++++--- .../elasticsearch/client/IngestClient.java | 76 +- .../client/RequestConverters.java | 2 +- .../client/RestHighLevelClient.java | 455 ++++++++++-- .../elasticsearch/client/SnapshotClient.java | 93 ++- .../org/elasticsearch/client/TasksClient.java | 28 +- .../elasticsearch/client/BulkProcessorIT.java | 13 +- .../client/BulkProcessorRetryIT.java | 4 +- .../elasticsearch/client/ClusterClientIT.java | 3 + .../java/org/elasticsearch/client/CrudIT.java | 171 +++-- .../client/ESRestHighLevelClientTestCase.java | 45 +- .../elasticsearch/client/IndicesClientIT.java | 102 ++- .../elasticsearch/client/PingAndInfoIT.java | 4 +- .../org/elasticsearch/client/RankEvalIT.java | 7 +- .../client/RestHighLevelClientTests.java | 26 +- .../org/elasticsearch/client/SearchIT.java | 56 +- .../documentation/CRUDDocumentationIT.java | 102 +-- .../ClusterClientDocumentationIT.java | 16 +- .../IndicesClientDocumentationIT.java | 161 ++-- .../IngestClientDocumentationIT.java | 13 +- .../MigrationDocumentationIT.java | 21 +- .../MiscellaneousDocumentationIT.java | 5 +- .../documentation/SearchDocumentationIT.java | 73 +- .../SnapshotClientDocumentationIT.java | 19 +- .../TasksClientDocumentationIT.java | 5 +- ...rossClusterSearchUnavailableClusterIT.java | 30 +- 27 files changed, 1677 insertions(+), 587 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ClusterClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ClusterClient.java index f3c84db79d65f..488579785e0f7 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ClusterClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ClusterClient.java @@ -41,11 +41,28 @@ public final class ClusterClient { } /** - * Updates cluster wide specific settings using the Cluster Update Settings API + * Updates cluster wide specific settings using the Cluster Update Settings API. + * See Cluster Update Settings + * API on elastic.co + * @param clusterUpdateSettingsRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public ClusterUpdateSettingsResponse putSettings(ClusterUpdateSettingsRequest clusterUpdateSettingsRequest, RequestOptions options) + throws IOException { + return restHighLevelClient.performRequestAndParseEntity(clusterUpdateSettingsRequest, RequestConverters::clusterPutSettings, + options, ClusterUpdateSettingsResponse::fromXContent, emptySet()); + } + + /** + * Updates cluster wide specific settings using the Cluster Update Settings API. *

* See Cluster Update Settings * API on elastic.co + * @deprecated Prefer {@link #putSettings(ClusterUpdateSettingsRequest, RequestOptions)} */ + @Deprecated public ClusterUpdateSettingsResponse putSettings(ClusterUpdateSettingsRequest clusterUpdateSettingsRequest, Header... headers) throws IOException { return restHighLevelClient.performRequestAndParseEntity(clusterUpdateSettingsRequest, RequestConverters::clusterPutSettings, @@ -53,11 +70,26 @@ public ClusterUpdateSettingsResponse putSettings(ClusterUpdateSettingsRequest cl } /** - * Asynchronously updates cluster wide specific settings using the Cluster Update Settings API + * Asynchronously updates cluster wide specific settings using the Cluster Update Settings API. + * See Cluster Update Settings + * API on elastic.co + * @param clusterUpdateSettingsRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void putSettingsAsync(ClusterUpdateSettingsRequest clusterUpdateSettingsRequest, RequestOptions options, + ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(clusterUpdateSettingsRequest, RequestConverters::clusterPutSettings, + options, ClusterUpdateSettingsResponse::fromXContent, listener, emptySet()); + } + /** + * Asynchronously updates cluster wide specific settings using the Cluster Update Settings API. *

* See Cluster Update Settings * API on elastic.co + * @deprecated Prefer {@link #putSettingsAsync(ClusterUpdateSettingsRequest, RequestOptions, ActionListener)} */ + @Deprecated public void putSettingsAsync(ClusterUpdateSettingsRequest clusterUpdateSettingsRequest, ActionListener listener, Header... headers) { restHighLevelClient.performRequestAsyncAndParseEntity(clusterUpdateSettingsRequest, RequestConverters::clusterPutSettings, diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java index d51a92ea00fc5..fa7eb9ab9ec8a 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java @@ -20,7 +20,6 @@ package org.elasticsearch.client; import org.apache.http.Header; -import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; @@ -47,10 +46,10 @@ import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; -import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; -import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsResponse; import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; @@ -76,66 +75,159 @@ public final class IndicesClient { } /** - * Deletes an index using the Delete Index API + * Deletes an index using the Delete Index API. + * See + * Delete Index API on elastic.co + * @param deleteIndexRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public DeleteIndexResponse delete(DeleteIndexRequest deleteIndexRequest, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(deleteIndexRequest, RequestConverters::deleteIndex, options, + DeleteIndexResponse::fromXContent, emptySet()); + } + + /** + * Deletes an index using the Delete Index API. *

* See * Delete Index API on elastic.co + * @deprecated Prefer {@link #delete(DeleteIndexRequest, RequestOptions)} */ + @Deprecated public DeleteIndexResponse delete(DeleteIndexRequest deleteIndexRequest, Header... headers) throws IOException { return restHighLevelClient.performRequestAndParseEntity(deleteIndexRequest, RequestConverters::deleteIndex, DeleteIndexResponse::fromXContent, emptySet(), headers); } /** - * Asynchronously deletes an index using the Delete Index API + * Asynchronously deletes an index using the Delete Index API. + * See + * Delete Index API on elastic.co + * @param deleteIndexRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void deleteAsync(DeleteIndexRequest deleteIndexRequest, RequestOptions options, ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(deleteIndexRequest, RequestConverters::deleteIndex, options, + DeleteIndexResponse::fromXContent, listener, emptySet()); + } + + /** + * Asynchronously deletes an index using the Delete Index API. *

* See * Delete Index API on elastic.co + * @deprecated Prefer {@link #deleteAsync(DeleteIndexRequest, RequestOptions, ActionListener)} */ + @Deprecated public void deleteAsync(DeleteIndexRequest deleteIndexRequest, ActionListener listener, Header... headers) { restHighLevelClient.performRequestAsyncAndParseEntity(deleteIndexRequest, RequestConverters::deleteIndex, DeleteIndexResponse::fromXContent, listener, emptySet(), headers); } /** - * Creates an index using the Create Index API + * Creates an index using the Create Index API. + * See + * Create Index API on elastic.co + * @param createIndexRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public CreateIndexResponse create(CreateIndexRequest createIndexRequest, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(createIndexRequest, RequestConverters::createIndex, options, + CreateIndexResponse::fromXContent, emptySet()); + } + + /** + * Creates an index using the Create Index API. *

* See * Create Index API on elastic.co + * @deprecated Prefer {@link #create(CreateIndexRequest, RequestOptions)} */ + @Deprecated public CreateIndexResponse create(CreateIndexRequest createIndexRequest, Header... headers) throws IOException { return restHighLevelClient.performRequestAndParseEntity(createIndexRequest, RequestConverters::createIndex, CreateIndexResponse::fromXContent, emptySet(), headers); } /** - * Asynchronously creates an index using the Create Index API + * Asynchronously creates an index using the Create Index API. + * See + * Create Index API on elastic.co + * @param createIndexRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void createAsync(CreateIndexRequest createIndexRequest, RequestOptions options, ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(createIndexRequest, RequestConverters::createIndex, options, + CreateIndexResponse::fromXContent, listener, emptySet()); + } + + /** + * Asynchronously creates an index using the Create Index API. *

* See * Create Index API on elastic.co + * @deprecated Prefer {@link #createAsync(CreateIndexRequest, RequestOptions, ActionListener)} */ + @Deprecated public void createAsync(CreateIndexRequest createIndexRequest, ActionListener listener, Header... headers) { restHighLevelClient.performRequestAsyncAndParseEntity(createIndexRequest, RequestConverters::createIndex, CreateIndexResponse::fromXContent, listener, emptySet(), headers); } /** - * Updates the mappings on an index using the Put Mapping API + * Updates the mappings on an index using the Put Mapping API. + * See + * Put Mapping API on elastic.co + * @param putMappingRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public PutMappingResponse putMapping(PutMappingRequest putMappingRequest, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(putMappingRequest, RequestConverters::putMapping, options, + PutMappingResponse::fromXContent, emptySet()); + } + + /** + * Updates the mappings on an index using the Put Mapping API. *

* See * Put Mapping API on elastic.co + * @deprecated Prefer {@link #putMapping(PutMappingRequest, RequestOptions)} */ + @Deprecated public PutMappingResponse putMapping(PutMappingRequest putMappingRequest, Header... headers) throws IOException { return restHighLevelClient.performRequestAndParseEntity(putMappingRequest, RequestConverters::putMapping, PutMappingResponse::fromXContent, emptySet(), headers); } /** - * Asynchronously updates the mappings on an index using the Put Mapping API + * Asynchronously updates the mappings on an index using the Put Mapping API. + * See + * Put Mapping API on elastic.co + * @param putMappingRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void putMappingAsync(PutMappingRequest putMappingRequest, RequestOptions options, ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(putMappingRequest, RequestConverters::putMapping, options, + PutMappingResponse::fromXContent, listener, emptySet()); + } + + /** + * Asynchronously updates the mappings on an index using the Put Mapping API. *

* See * Put Mapping API on elastic.co + * @deprecated Prefer {@link #putMappingAsync(PutMappingRequest, RequestOptions, ActionListener)} */ + @Deprecated public void putMappingAsync(PutMappingRequest putMappingRequest, ActionListener listener, Header... headers) { restHighLevelClient.performRequestAsyncAndParseEntity(putMappingRequest, RequestConverters::putMapping, @@ -143,242 +235,507 @@ public void putMappingAsync(PutMappingRequest putMappingRequest, ActionListener< } /** - * Retrieves the mappings on an index or indices using the Get Mapping API - *

+ * Retrieves the mappings on an index or indices using the Get Mapping API. * See * Get Mapping API on elastic.co + * @param getMappingsRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response */ - public GetMappingsResponse getMappings(GetMappingsRequest getMappingsRequest, Header... headers) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(getMappingsRequest, RequestConverters::getMappings, - GetMappingsResponse::fromXContent, emptySet(), headers); + public GetMappingsResponse getMappings(GetMappingsRequest getMappingsRequest, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(getMappingsRequest, RequestConverters::getMappings, options, + GetMappingsResponse::fromXContent, emptySet()); } /** - * Asynchronously retrieves the mappings on an index on indices using the Get Mapping API - *

+ * Asynchronously retrieves the mappings on an index on indices using the Get Mapping API. * See * Get Mapping API on elastic.co + * @param getMappingsRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion */ - public void getMappingsAsync(GetMappingsRequest getMappingsRequest, ActionListener listener, - Header... headers) { - restHighLevelClient.performRequestAsyncAndParseEntity(getMappingsRequest, RequestConverters::getMappings, - GetMappingsResponse::fromXContent, listener, emptySet(), headers); + public void getMappingsAsync(GetMappingsRequest getMappingsRequest, RequestOptions options, + ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(getMappingsRequest, RequestConverters::getMappings, options, + GetMappingsResponse::fromXContent, listener, emptySet()); } /** - * Updates aliases using the Index Aliases API + * Updates aliases using the Index Aliases API. + * See + * Index Aliases API on elastic.co + * @param indicesAliasesRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public IndicesAliasesResponse updateAliases(IndicesAliasesRequest indicesAliasesRequest, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(indicesAliasesRequest, RequestConverters::updateAliases, options, + IndicesAliasesResponse::fromXContent, emptySet()); + } + + /** + * Updates aliases using the Index Aliases API. *

* See * Index Aliases API on elastic.co + * @deprecated {@link #updateAliases(IndicesAliasesRequest, RequestOptions)} */ + @Deprecated public IndicesAliasesResponse updateAliases(IndicesAliasesRequest indicesAliasesRequest, Header... headers) throws IOException { return restHighLevelClient.performRequestAndParseEntity(indicesAliasesRequest, RequestConverters::updateAliases, IndicesAliasesResponse::fromXContent, emptySet(), headers); } /** - * Asynchronously updates aliases using the Index Aliases API + * Asynchronously updates aliases using the Index Aliases API. + * See + * Index Aliases API on elastic.co + * @param indicesAliasesRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void updateAliasesAsync(IndicesAliasesRequest indicesAliasesRequest, RequestOptions options, + ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(indicesAliasesRequest, RequestConverters::updateAliases, options, + IndicesAliasesResponse::fromXContent, listener, emptySet()); + } + + /** + * Asynchronously updates aliases using the Index Aliases API. *

* See * Index Aliases API on elastic.co + * @deprecated Prefer {@link #updateAliasesAsync(IndicesAliasesRequest, RequestOptions, ActionListener)} */ + @Deprecated public void updateAliasesAsync(IndicesAliasesRequest indicesAliasesRequest, ActionListener listener, - Header... headers) { + Header... headers) { restHighLevelClient.performRequestAsyncAndParseEntity(indicesAliasesRequest, RequestConverters::updateAliases, IndicesAliasesResponse::fromXContent, listener, emptySet(), headers); } /** - * Opens an index using the Open Index API + * Opens an index using the Open Index API. + * See + * Open Index API on elastic.co + * @param openIndexRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public OpenIndexResponse open(OpenIndexRequest openIndexRequest, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(openIndexRequest, RequestConverters::openIndex, options, + OpenIndexResponse::fromXContent, emptySet()); + } + + /** + * Opens an index using the Open Index API. *

* See * Open Index API on elastic.co + * @deprecated Prefer {@link #open(OpenIndexRequest, RequestOptions)} */ + @Deprecated public OpenIndexResponse open(OpenIndexRequest openIndexRequest, Header... headers) throws IOException { return restHighLevelClient.performRequestAndParseEntity(openIndexRequest, RequestConverters::openIndex, OpenIndexResponse::fromXContent, emptySet(), headers); } /** - * Asynchronously opens an index using the Open Index API + * Asynchronously opens an index using the Open Index API. + * See + * Open Index API on elastic.co + * @param openIndexRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void openAsync(OpenIndexRequest openIndexRequest, RequestOptions options, ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(openIndexRequest, RequestConverters::openIndex, options, + OpenIndexResponse::fromXContent, listener, emptySet()); + } + + /** + * Asynchronously opens an index using the Open Index API. *

* See * Open Index API on elastic.co + * @deprecated Prefer {@link #openAsync(OpenIndexRequest, RequestOptions, ActionListener)} */ + @Deprecated public void openAsync(OpenIndexRequest openIndexRequest, ActionListener listener, Header... headers) { restHighLevelClient.performRequestAsyncAndParseEntity(openIndexRequest, RequestConverters::openIndex, OpenIndexResponse::fromXContent, listener, emptySet(), headers); } /** - * Closes an index using the Close Index API + * Closes an index using the Close Index API. + * See + * Close Index API on elastic.co + * @param closeIndexRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public CloseIndexResponse close(CloseIndexRequest closeIndexRequest, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(closeIndexRequest, RequestConverters::closeIndex, options, + CloseIndexResponse::fromXContent, emptySet()); + } + + /** + * Closes an index using the Close Index API. *

* See * Close Index API on elastic.co + * @deprecated Prefer {@link #close(CloseIndexRequest, RequestOptions)} */ + @Deprecated public CloseIndexResponse close(CloseIndexRequest closeIndexRequest, Header... headers) throws IOException { return restHighLevelClient.performRequestAndParseEntity(closeIndexRequest, RequestConverters::closeIndex, CloseIndexResponse::fromXContent, emptySet(), headers); } /** - * Asynchronously closes an index using the Close Index API + * Asynchronously closes an index using the Close Index API. + * See + * Close Index API on elastic.co + * @param closeIndexRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void closeAsync(CloseIndexRequest closeIndexRequest, RequestOptions options, ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(closeIndexRequest, RequestConverters::closeIndex, options, + CloseIndexResponse::fromXContent, listener, emptySet()); + } + + + /** + * Asynchronously closes an index using the Close Index API. *

* See * Close Index API on elastic.co + * @deprecated Prefer {@link #closeAsync(CloseIndexRequest, RequestOptions, ActionListener)} */ + @Deprecated public void closeAsync(CloseIndexRequest closeIndexRequest, ActionListener listener, Header... headers) { restHighLevelClient.performRequestAsyncAndParseEntity(closeIndexRequest, RequestConverters::closeIndex, CloseIndexResponse::fromXContent, listener, emptySet(), headers); } /** - * Checks if one or more aliases exist using the Aliases Exist API + * Checks if one or more aliases exist using the Aliases Exist API. + * See + * Indices Aliases API on elastic.co + * @param getAliasesRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request + */ + public boolean existsAlias(GetAliasesRequest getAliasesRequest, RequestOptions options) throws IOException { + return restHighLevelClient.performRequest(getAliasesRequest, RequestConverters::existsAlias, options, + RestHighLevelClient::convertExistsResponse, emptySet()); + } + + /** + * Checks if one or more aliases exist using the Aliases Exist API. *

* See * Indices Aliases API on elastic.co + * @deprecated Prefer {@link #existsAlias(GetAliasesRequest, RequestOptions)} */ + @Deprecated public boolean existsAlias(GetAliasesRequest getAliasesRequest, Header... headers) throws IOException { return restHighLevelClient.performRequest(getAliasesRequest, RequestConverters::existsAlias, RestHighLevelClient::convertExistsResponse, emptySet(), headers); } /** - * Asynchronously checks if one or more aliases exist using the Aliases Exist API + * Asynchronously checks if one or more aliases exist using the Aliases Exist API. + * See + * Indices Aliases API on elastic.co + * @param getAliasesRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void existsAliasAsync(GetAliasesRequest getAliasesRequest, RequestOptions options, ActionListener listener) { + restHighLevelClient.performRequestAsync(getAliasesRequest, RequestConverters::existsAlias, options, + RestHighLevelClient::convertExistsResponse, listener, emptySet()); + } + + /** + * Asynchronously checks if one or more aliases exist using the Aliases Exist API. *

* See * Indices Aliases API on elastic.co + * @deprecated Prefer {@link #existsAliasAsync(GetAliasesRequest, RequestOptions, ActionListener)} */ + @Deprecated public void existsAliasAsync(GetAliasesRequest getAliasesRequest, ActionListener listener, Header... headers) { restHighLevelClient.performRequestAsync(getAliasesRequest, RequestConverters::existsAlias, RestHighLevelClient::convertExistsResponse, listener, emptySet(), headers); } /** - * Refresh one or more indices using the Refresh API + * Refresh one or more indices using the Refresh API. + * See Refresh API on elastic.co + * @param refreshRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public RefreshResponse refresh(RefreshRequest refreshRequest, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(refreshRequest, RequestConverters::refresh, options, + RefreshResponse::fromXContent, emptySet()); + } + + /** + * Refresh one or more indices using the Refresh API. *

* See Refresh API on elastic.co + * @deprecated Prefer {@link #refresh(RefreshRequest, RequestOptions)} */ + @Deprecated public RefreshResponse refresh(RefreshRequest refreshRequest, Header... headers) throws IOException { return restHighLevelClient.performRequestAndParseEntity(refreshRequest, RequestConverters::refresh, RefreshResponse::fromXContent, emptySet(), headers); } /** - * Asynchronously refresh one or more indices using the Refresh API + * Asynchronously refresh one or more indices using the Refresh API. + * See Refresh API on elastic.co + * @param refreshRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void refreshAsync(RefreshRequest refreshRequest, RequestOptions options, ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(refreshRequest, RequestConverters::refresh, options, + RefreshResponse::fromXContent, listener, emptySet()); + } + + /** + * Asynchronously refresh one or more indices using the Refresh API. *

* See Refresh API on elastic.co + * @deprecated Prefer {@link #refreshAsync(RefreshRequest, RequestOptions, ActionListener)} */ + @Deprecated public void refreshAsync(RefreshRequest refreshRequest, ActionListener listener, Header... headers) { restHighLevelClient.performRequestAsyncAndParseEntity(refreshRequest, RequestConverters::refresh, RefreshResponse::fromXContent, listener, emptySet(), headers); } /** - * Flush one or more indices using the Flush API + * Flush one or more indices using the Flush API. + * See Flush API on elastic.co + * @param flushRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public FlushResponse flush(FlushRequest flushRequest, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(flushRequest, RequestConverters::flush, options, + FlushResponse::fromXContent, emptySet()); + } + + /** + * Flush one or more indices using the Flush API. *

* See Flush API on elastic.co + * @deprecated Prefer {@link #flush(FlushRequest, RequestOptions)} */ + @Deprecated public FlushResponse flush(FlushRequest flushRequest, Header... headers) throws IOException { return restHighLevelClient.performRequestAndParseEntity(flushRequest, RequestConverters::flush, FlushResponse::fromXContent, emptySet(), headers); } /** - * Asynchronously flush one or more indices using the Flush API + * Asynchronously flush one or more indices using the Flush API. + * See Flush API on elastic.co + * @param flushRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void flushAsync(FlushRequest flushRequest, RequestOptions options, ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(flushRequest, RequestConverters::flush, options, + FlushResponse::fromXContent, listener, emptySet()); + } + + /** + * Asynchronously flush one or more indices using the Flush API. *

* See Flush API on elastic.co + * @deprecated Prefer {@link #flushAsync(FlushRequest, RequestOptions, ActionListener)} */ + @Deprecated public void flushAsync(FlushRequest flushRequest, ActionListener listener, Header... headers) { restHighLevelClient.performRequestAsyncAndParseEntity(flushRequest, RequestConverters::flush, FlushResponse::fromXContent, listener, emptySet(), headers); } - /** Initiate a synced flush manually using the synced flush API - *

- * See - * Synced flush API on elastic.co - */ - public SyncedFlushResponse flushSynced(SyncedFlushRequest syncedFlushRequest, Header... headers) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(syncedFlushRequest, RequestConverters::flushSynced, - SyncedFlushResponse::fromXContent, emptySet(), headers); + /** + * Initiate a synced flush manually using the synced flush API. + * See + * Synced flush API on elastic.co + * @param syncedFlushRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public SyncedFlushResponse flushSynced(SyncedFlushRequest syncedFlushRequest, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(syncedFlushRequest, RequestConverters::flushSynced, options, + SyncedFlushResponse::fromXContent, emptySet()); } /** - * Asynchronously initiate a synced flush manually using the synced flush API - *

+ * Asynchronously initiate a synced flush manually using the synced flush API. * See * Synced flush API on elastic.co + * @param syncedFlushRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion */ - public void flushSyncedAsync(SyncedFlushRequest syncedFlushRequest, ActionListener listener, Header... headers) { - restHighLevelClient.performRequestAsyncAndParseEntity(syncedFlushRequest, RequestConverters::flushSynced, - SyncedFlushResponse::fromXContent, listener, emptySet(), headers); + public void flushSyncedAsync(SyncedFlushRequest syncedFlushRequest, RequestOptions options, + ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(syncedFlushRequest, RequestConverters::flushSynced, options, + SyncedFlushResponse::fromXContent, listener, emptySet()); } - /** - * Retrieve the settings of one or more indices - *

+ * Retrieve the settings of one or more indices. * See * Indices Get Settings API on elastic.co + * @param getSettingsRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response */ - public GetSettingsResponse getSettings(GetSettingsRequest getSettingsRequest, Header... headers) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(getSettingsRequest, RequestConverters::getSettings, - GetSettingsResponse::fromXContent, emptySet(), headers); + public GetSettingsResponse getSettings(GetSettingsRequest getSettingsRequest, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(getSettingsRequest, RequestConverters::getSettings, options, + GetSettingsResponse::fromXContent, emptySet()); } /** - * Asynchronously retrieve the settings of one or more indices - *

+ * Asynchronously retrieve the settings of one or more indices. * See * Indices Get Settings API on elastic.co + * @param getSettingsRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void getSettingsAsync(GetSettingsRequest getSettingsRequest, RequestOptions options, + ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(getSettingsRequest, RequestConverters::getSettings, options, + GetSettingsResponse::fromXContent, listener, emptySet()); + } + + /** + * Force merge one or more indices using the Force Merge API. + * See + * Force Merge API on elastic.co + * @param forceMergeRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response */ - public void getSettingsAsync(GetSettingsRequest getSettingsRequest, ActionListener listener, Header... headers) { - restHighLevelClient.performRequestAsyncAndParseEntity(getSettingsRequest, RequestConverters::getSettings, - GetSettingsResponse::fromXContent, listener, emptySet(), headers); + public ForceMergeResponse forceMerge(ForceMergeRequest forceMergeRequest, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(forceMergeRequest, RequestConverters::forceMerge, options, + ForceMergeResponse::fromXContent, emptySet()); } /** - * Force merge one or more indices using the Force Merge API + * Force merge one or more indices using the Force Merge API. *

* See * Force Merge API on elastic.co + * @deprecated Prefer {@link #forceMerge(ForceMergeRequest, RequestOptions)} */ + @Deprecated public ForceMergeResponse forceMerge(ForceMergeRequest forceMergeRequest, Header... headers) throws IOException { return restHighLevelClient.performRequestAndParseEntity(forceMergeRequest, RequestConverters::forceMerge, ForceMergeResponse::fromXContent, emptySet(), headers); } /** - * Asynchronously force merge one or more indices using the Force Merge API + * Asynchronously force merge one or more indices using the Force Merge API. + * See + * Force Merge API on elastic.co + * @param forceMergeRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void forceMergeAsync(ForceMergeRequest forceMergeRequest, RequestOptions options, ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(forceMergeRequest, RequestConverters::forceMerge, options, + ForceMergeResponse::fromXContent, listener, emptySet()); + } + + /** + * Asynchronously force merge one or more indices using the Force Merge API. *

* See * Force Merge API on elastic.co + * @deprecated Prefer {@link #forceMergeAsync(ForceMergeRequest, RequestOptions, ActionListener)} */ + @Deprecated public void forceMergeAsync(ForceMergeRequest forceMergeRequest, ActionListener listener, Header... headers) { restHighLevelClient.performRequestAsyncAndParseEntity(forceMergeRequest, RequestConverters::forceMerge, ForceMergeResponse::fromXContent, listener, emptySet(), headers); } /** - * Clears the cache of one or more indices using the Clear Cache API + * Clears the cache of one or more indices using the Clear Cache API. + * See + * Clear Cache API on elastic.co + * @param clearIndicesCacheRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public ClearIndicesCacheResponse clearCache(ClearIndicesCacheRequest clearIndicesCacheRequest, + RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(clearIndicesCacheRequest, RequestConverters::clearCache, options, + ClearIndicesCacheResponse::fromXContent, emptySet()); + } + + /** + * Clears the cache of one or more indices using the Clear Cache API. *

* See * Clear Cache API on elastic.co + * @deprecated Prefer {@link #clearCache(ClearIndicesCacheRequest, RequestOptions)} */ + @Deprecated public ClearIndicesCacheResponse clearCache(ClearIndicesCacheRequest clearIndicesCacheRequest, Header... headers) throws IOException { return restHighLevelClient.performRequestAndParseEntity(clearIndicesCacheRequest, RequestConverters::clearCache, ClearIndicesCacheResponse::fromXContent, emptySet(), headers); } /** - * Asynchronously clears the cache of one or more indices using the Clear Cache API + * Asynchronously clears the cache of one or more indices using the Clear Cache API. + * See + * Clear Cache API on elastic.co + * @param clearIndicesCacheRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void clearCacheAsync(ClearIndicesCacheRequest clearIndicesCacheRequest, RequestOptions options, + ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(clearIndicesCacheRequest, RequestConverters::clearCache, options, + ClearIndicesCacheResponse::fromXContent, listener, emptySet()); + } + + /** + * Asynchronously clears the cache of one or more indices using the Clear Cache API. *

* See * Clear Cache API on elastic.co + * @deprecated Prefer {@link #clearCacheAsync(ClearIndicesCacheRequest, RequestOptions, ActionListener)} */ + @Deprecated public void clearCacheAsync(ClearIndicesCacheRequest clearIndicesCacheRequest, ActionListener listener, Header... headers) { restHighLevelClient.performRequestAsyncAndParseEntity(clearIndicesCacheRequest, RequestConverters::clearCache, @@ -387,17 +744,57 @@ public void clearCacheAsync(ClearIndicesCacheRequest clearIndicesCacheRequest, A /** * Checks if the index (indices) exists or not. - *

* See * Indices Exists API on elastic.co + * @param request the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request */ - public boolean exists(GetIndexRequest request, Header... headers) throws IOException { + public boolean exists(GetIndexRequest request, RequestOptions options) throws IOException { return restHighLevelClient.performRequest( request, RequestConverters::indicesExist, + options, RestHighLevelClient::convertExistsResponse, - Collections.emptySet(), - headers + Collections.emptySet() + ); + } + + /** + * Checks if the index (indices) exists or not. + *

+ * See + * Indices Exists API on elastic.co + * @deprecated Prefer {@link #exists(GetIndexRequest, RequestOptions)} + */ + @Deprecated + public boolean exists(GetIndexRequest request, Header... headers) throws IOException { + return restHighLevelClient.performRequest( + request, + RequestConverters::indicesExist, + RestHighLevelClient::convertExistsResponse, + Collections.emptySet(), + headers + ); + } + + /** + * Asynchronously checks if the index (indices) exists or not. + * See + * Indices Exists API on elastic.co + * @param request the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void existsAsync(GetIndexRequest request, RequestOptions options, ActionListener listener) { + restHighLevelClient.performRequestAsync( + request, + RequestConverters::indicesExist, + options, + RestHighLevelClient::convertExistsResponse, + listener, + Collections.emptySet() ); } @@ -406,7 +803,9 @@ public boolean exists(GetIndexRequest request, Header... headers) throws IOExcep *

* See * Indices Exists API on elastic.co + * @deprecated Prefer {@link #existsAsync(GetIndexRequest, RequestOptions, ActionListener)} */ + @Deprecated public void existsAsync(GetIndexRequest request, ActionListener listener, Header... headers) { restHighLevelClient.performRequestAsync( request, @@ -419,88 +818,213 @@ public void existsAsync(GetIndexRequest request, ActionListener listene } /** - * Shrinks an index using the Shrink Index API + * Shrinks an index using the Shrink Index API. + * See + * Shrink Index API on elastic.co + * @param resizeRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public ResizeResponse shrink(ResizeRequest resizeRequest, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(resizeRequest, RequestConverters::shrink, options, + ResizeResponse::fromXContent, emptySet()); + } + + /** + * Shrinks an index using the Shrink Index API. *

* See * Shrink Index API on elastic.co + * @deprecated Prefer {@link #shrink(ResizeRequest, RequestOptions)} */ + @Deprecated public ResizeResponse shrink(ResizeRequest resizeRequest, Header... headers) throws IOException { return restHighLevelClient.performRequestAndParseEntity(resizeRequest, RequestConverters::shrink, ResizeResponse::fromXContent, emptySet(), headers); } /** - * Asynchronously shrinks an index using the Shrink index API + * Asynchronously shrinks an index using the Shrink index API. + * See + * Shrink Index API on elastic.co + * @param resizeRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void shrinkAsync(ResizeRequest resizeRequest, RequestOptions options, ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(resizeRequest, RequestConverters::shrink, options, + ResizeResponse::fromXContent, listener, emptySet()); + } + + /** + * Asynchronously shrinks an index using the Shrink index API. *

* See * Shrink Index API on elastic.co + * @deprecated Prefer {@link #shrinkAsync(ResizeRequest, RequestOptions, ActionListener)} */ + @Deprecated public void shrinkAsync(ResizeRequest resizeRequest, ActionListener listener, Header... headers) { restHighLevelClient.performRequestAsyncAndParseEntity(resizeRequest, RequestConverters::shrink, ResizeResponse::fromXContent, listener, emptySet(), headers); } /** - * Splits an index using the Split Index API + * Splits an index using the Split Index API. + * See + * Split Index API on elastic.co + * @param resizeRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public ResizeResponse split(ResizeRequest resizeRequest, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(resizeRequest, RequestConverters::split, options, + ResizeResponse::fromXContent, emptySet()); + } + + /** + * Splits an index using the Split Index API. *

* See * Split Index API on elastic.co + * @deprecated {@link #split(ResizeRequest, RequestOptions)} */ + @Deprecated public ResizeResponse split(ResizeRequest resizeRequest, Header... headers) throws IOException { return restHighLevelClient.performRequestAndParseEntity(resizeRequest, RequestConverters::split, ResizeResponse::fromXContent, emptySet(), headers); } /** - * Asynchronously splits an index using the Split Index API + * Asynchronously splits an index using the Split Index API. + * See + * Split Index API on elastic.co + * @param resizeRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void splitAsync(ResizeRequest resizeRequest, RequestOptions options, ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(resizeRequest, RequestConverters::split, options, + ResizeResponse::fromXContent, listener, emptySet()); + } + + /** + * Asynchronously splits an index using the Split Index API. *

* See * Split Index API on elastic.co + * @deprecated Prefer {@link #splitAsync(ResizeRequest, RequestOptions, ActionListener)} */ + @Deprecated public void splitAsync(ResizeRequest resizeRequest, ActionListener listener, Header... headers) { restHighLevelClient.performRequestAsyncAndParseEntity(resizeRequest, RequestConverters::split, ResizeResponse::fromXContent, listener, emptySet(), headers); } /** - * Rolls over an index using the Rollover Index API + * Rolls over an index using the Rollover Index API. + * See + * Rollover Index API on elastic.co + * @param rolloverRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public RolloverResponse rollover(RolloverRequest rolloverRequest, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(rolloverRequest, RequestConverters::rollover, options, + RolloverResponse::fromXContent, emptySet()); + } + + /** + * Rolls over an index using the Rollover Index API. *

* See * Rollover Index API on elastic.co + * @deprecated Prefer {@link #rollover(RolloverRequest, RequestOptions)} */ + @Deprecated public RolloverResponse rollover(RolloverRequest rolloverRequest, Header... headers) throws IOException { return restHighLevelClient.performRequestAndParseEntity(rolloverRequest, RequestConverters::rollover, RolloverResponse::fromXContent, emptySet(), headers); } /** - * Asynchronously rolls over an index using the Rollover Index API + * Asynchronously rolls over an index using the Rollover Index API. + * See + * Rollover Index API on elastic.co + * @param rolloverRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void rolloverAsync(RolloverRequest rolloverRequest, RequestOptions options, ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(rolloverRequest, RequestConverters::rollover, options, + RolloverResponse::fromXContent, listener, emptySet()); + } + + /** + * Asynchronously rolls over an index using the Rollover Index API. *

* See * Rollover Index API on elastic.co + * @deprecated Prefer {@link #rolloverAsync(RolloverRequest, RequestOptions, ActionListener)} */ + @Deprecated public void rolloverAsync(RolloverRequest rolloverRequest, ActionListener listener, Header... headers) { restHighLevelClient.performRequestAsyncAndParseEntity(rolloverRequest, RequestConverters::rollover, RolloverResponse::fromXContent, listener, emptySet(), headers); } /** - * Updates specific index level settings using the Update Indices Settings API + * Updates specific index level settings using the Update Indices Settings API. + * See Update Indices Settings + * API on elastic.co + * @param updateSettingsRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public UpdateSettingsResponse putSettings(UpdateSettingsRequest updateSettingsRequest, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(updateSettingsRequest, RequestConverters::indexPutSettings, options, + UpdateSettingsResponse::fromXContent, emptySet()); + } + + /** + * Updates specific index level settings using the Update Indices Settings API. *

* See Update Indices Settings * API on elastic.co + * @deprecated Prefer {@link #putSettings(UpdateSettingsRequest, RequestOptions)} */ + @Deprecated public UpdateSettingsResponse putSettings(UpdateSettingsRequest updateSettingsRequest, Header... headers) throws IOException { return restHighLevelClient.performRequestAndParseEntity(updateSettingsRequest, RequestConverters::indexPutSettings, UpdateSettingsResponse::fromXContent, emptySet(), headers); } /** - * Asynchronously updates specific index level settings using the Update Indices Settings API + * Asynchronously updates specific index level settings using the Update Indices Settings API. + * See Update Indices Settings + * API on elastic.co + * @param updateSettingsRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void putSettingsAsync(UpdateSettingsRequest updateSettingsRequest, RequestOptions options, + ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(updateSettingsRequest, RequestConverters::indexPutSettings, options, + UpdateSettingsResponse::fromXContent, listener, emptySet()); + } + + /** + * Asynchronously updates specific index level settings using the Update Indices Settings API. *

* See Update Indices Settings * API on elastic.co + * @deprecated Prefer {@link #putSettingsAsync(UpdateSettingsRequest, RequestOptions, ActionListener)} */ + @Deprecated public void putSettingsAsync(UpdateSettingsRequest updateSettingsRequest, ActionListener listener, Header... headers) { restHighLevelClient.performRequestAsyncAndParseEntity(updateSettingsRequest, RequestConverters::indexPutSettings, @@ -508,25 +1032,31 @@ public void putSettingsAsync(UpdateSettingsRequest updateSettingsRequest, Action } /** - * Puts an index template using the Index Templates API - *

+ * Puts an index template using the Index Templates API. * See Index Templates API * on elastic.co + * @param putIndexTemplateRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response */ - public PutIndexTemplateResponse putTemplate(PutIndexTemplateRequest putIndexTemplateRequest, Header... headers) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(putIndexTemplateRequest, RequestConverters::putTemplate, - PutIndexTemplateResponse::fromXContent, emptySet(), headers); + public PutIndexTemplateResponse putTemplate(PutIndexTemplateRequest putIndexTemplateRequest, + RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(putIndexTemplateRequest, RequestConverters::putTemplate, options, + PutIndexTemplateResponse::fromXContent, emptySet()); } /** - * Asynchronously puts an index template using the Index Templates API - *

+ * Asynchronously puts an index template using the Index Templates API. * See Index Templates API * on elastic.co + * @param putIndexTemplateRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion */ - public void putTemplateAsync(PutIndexTemplateRequest putIndexTemplateRequest, - ActionListener listener, Header... headers) { - restHighLevelClient.performRequestAsyncAndParseEntity(putIndexTemplateRequest, RequestConverters::putTemplate, - PutIndexTemplateResponse::fromXContent, listener, emptySet(), headers); + public void putTemplateAsync(PutIndexTemplateRequest putIndexTemplateRequest, RequestOptions options, + ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(putIndexTemplateRequest, RequestConverters::putTemplate, options, + PutIndexTemplateResponse::fromXContent, listener, emptySet()); } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IngestClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IngestClient.java index 72b1813f93909..5c5a82b52f438 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IngestClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IngestClient.java @@ -19,7 +19,6 @@ package org.elasticsearch.client; -import org.apache.http.Header; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ingest.DeletePipelineRequest; import org.elasticsearch.action.ingest.GetPipelineRequest; @@ -45,70 +44,85 @@ public final class IngestClient { } /** - * Add a pipeline or update an existing pipeline - *

+ * Add a pipeline or update an existing pipeline. * See * Put Pipeline API on elastic.co + * @param request the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response */ - public WritePipelineResponse putPipeline(PutPipelineRequest request, Header... headers) throws IOException { - return restHighLevelClient.performRequestAndParseEntity( request, RequestConverters::putPipeline, - WritePipelineResponse::fromXContent, emptySet(), headers); + public WritePipelineResponse putPipeline(PutPipelineRequest request, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity( request, RequestConverters::putPipeline, options, + WritePipelineResponse::fromXContent, emptySet()); } /** - * Asynchronously add a pipeline or update an existing pipeline - *

+ * Asynchronously add a pipeline or update an existing pipeline. * See * Put Pipeline API on elastic.co + * @param request the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion */ - public void putPipelineAsync(PutPipelineRequest request, ActionListener listener, Header... headers) { - restHighLevelClient.performRequestAsyncAndParseEntity( request, RequestConverters::putPipeline, - WritePipelineResponse::fromXContent, listener, emptySet(), headers); + public void putPipelineAsync(PutPipelineRequest request, RequestOptions options, ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity( request, RequestConverters::putPipeline, options, + WritePipelineResponse::fromXContent, listener, emptySet()); } /** - * Get an existing pipeline - *

+ * Get an existing pipeline. * See * Get Pipeline API on elastic.co + * @param request the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response */ - public GetPipelineResponse getPipeline(GetPipelineRequest request, Header... headers) throws IOException { - return restHighLevelClient.performRequestAndParseEntity( request, RequestConverters::getPipeline, - GetPipelineResponse::fromXContent, emptySet(), headers); + public GetPipelineResponse getPipeline(GetPipelineRequest request, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity( request, RequestConverters::getPipeline, options, + GetPipelineResponse::fromXContent, emptySet()); } /** - * Asynchronously get an existing pipeline - *

+ * Asynchronously get an existing pipeline. * See * Get Pipeline API on elastic.co + * @param request the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion */ - public void getPipelineAsync(GetPipelineRequest request, ActionListener listener, Header... headers) { - restHighLevelClient.performRequestAsyncAndParseEntity( request, RequestConverters::getPipeline, - GetPipelineResponse::fromXContent, listener, emptySet(), headers); + public void getPipelineAsync(GetPipelineRequest request, RequestOptions options, ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity( request, RequestConverters::getPipeline, options, + GetPipelineResponse::fromXContent, listener, emptySet()); } /** - * Delete an existing pipeline - *

+ * Delete an existing pipeline. * See * * Delete Pipeline API on elastic.co + * @param request the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response */ - public WritePipelineResponse deletePipeline(DeletePipelineRequest request, Header... headers) throws IOException { - return restHighLevelClient.performRequestAndParseEntity( request, RequestConverters::deletePipeline, - WritePipelineResponse::fromXContent, emptySet(), headers); + public WritePipelineResponse deletePipeline(DeletePipelineRequest request, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity( request, RequestConverters::deletePipeline, options, + WritePipelineResponse::fromXContent, emptySet()); } /** - * Asynchronously delete an existing pipeline - *

+ * Asynchronously delete an existing pipeline. * See * * Delete Pipeline API on elastic.co + * @param request the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion */ - public void deletePipelineAsync(DeletePipelineRequest request, ActionListener listener, Header... headers) { - restHighLevelClient.performRequestAsyncAndParseEntity( request, RequestConverters::deletePipeline, - WritePipelineResponse::fromXContent, listener, emptySet(), headers); + public void deletePipelineAsync(DeletePipelineRequest request, RequestOptions options, ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity( request, RequestConverters::deletePipeline, options, + WritePipelineResponse::fromXContent, listener, emptySet()); } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index e5a45e19fe0d3..b061289888c0c 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -710,7 +710,7 @@ static Request rollover(RolloverRequest rolloverRequest) throws IOException { return request; } - static Request getSettings(GetSettingsRequest getSettingsRequest) throws IOException { + static Request getSettings(GetSettingsRequest getSettingsRequest) { String[] indices = getSettingsRequest.indices() == null ? Strings.EMPTY_ARRAY : getSettingsRequest.indices(); String[] names = getSettingsRequest.names() == null ? Strings.EMPTY_ARRAY : getSettingsRequest.names(); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java index a9587b73c1959..8980508c48738 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java @@ -285,16 +285,19 @@ public final TasksClient tasks() { } /** - * Executes a bulk request using the Bulk API - * + * Executes a bulk request using the Bulk API. * See Bulk API on elastic.co + * @param bulkRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response */ public final BulkResponse bulk(BulkRequest bulkRequest, RequestOptions options) throws IOException { return performRequestAndParseEntity(bulkRequest, RequestConverters::bulk, options, BulkResponse::fromXContent, emptySet()); } /** - * Executes a bulk request using the Bulk API + * Executes a bulk request using the Bulk API. * * See Bulk API on elastic.co * @deprecated Prefer {@link #bulk(BulkRequest, RequestOptions)} @@ -305,16 +308,18 @@ public final BulkResponse bulk(BulkRequest bulkRequest, Header... headers) throw } /** - * Asynchronously executes a bulk request using the Bulk API - * + * Asynchronously executes a bulk request using the Bulk API. * See Bulk API on elastic.co + * @param bulkRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion */ public final void bulkAsync(BulkRequest bulkRequest, RequestOptions options, ActionListener listener) { performRequestAsyncAndParseEntity(bulkRequest, RequestConverters::bulk, options, BulkResponse::fromXContent, listener, emptySet()); } /** - * Asynchronously executes a bulk request using the Bulk API + * Asynchronously executes a bulk request using the Bulk API. * * See Bulk API on elastic.co * @deprecated Prefer {@link #bulkAsync(BulkRequest, RequestOptions, ActionListener)} @@ -326,194 +331,482 @@ public final void bulkAsync(BulkRequest bulkRequest, ActionListenertrue if the ping succeeded, false otherwise + * @throws IOException in case there is a problem sending the request + */ + public final boolean ping(RequestOptions options) throws IOException { + return performRequest(new MainRequest(), (request) -> RequestConverters.ping(), options, RestHighLevelClient::convertExistsResponse, + emptySet()); + } + + /** + * Pings the remote Elasticsearch cluster and returns true if the ping succeeded, false otherwise + * @deprecated Prefer {@link #ping(RequestOptions)} */ + @Deprecated public final boolean ping(Header... headers) throws IOException { return performRequest(new MainRequest(), (request) -> RequestConverters.ping(), RestHighLevelClient::convertExistsResponse, emptySet(), headers); } + /** + * Get the cluster info otherwise provided when sending an HTTP request to '/' + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public final MainResponse info(RequestOptions options) throws IOException { + return performRequestAndParseEntity(new MainRequest(), (request) -> RequestConverters.info(), options, + MainResponse::fromXContent, emptySet()); + } + /** * Get the cluster info otherwise provided when sending an HTTP request to port 9200 + * @deprecated Prefer {@link #info(RequestOptions)} */ + @Deprecated public final MainResponse info(Header... headers) throws IOException { return performRequestAndParseEntity(new MainRequest(), (request) -> RequestConverters.info(), MainResponse::fromXContent, emptySet(), headers); } /** - * Retrieves a document by id using the Get API + * Retrieves a document by id using the Get API. + * See Get API on elastic.co + * @param getRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public final GetResponse get(GetRequest getRequest, RequestOptions options) throws IOException { + return performRequestAndParseEntity(getRequest, RequestConverters::get, options, GetResponse::fromXContent, singleton(404)); + } + + /** + * Retrieves a document by id using the Get API. * * See Get API on elastic.co + * @deprecated Prefer {@link #get(GetRequest, RequestOptions)} */ + @Deprecated public final GetResponse get(GetRequest getRequest, Header... headers) throws IOException { return performRequestAndParseEntity(getRequest, RequestConverters::get, GetResponse::fromXContent, singleton(404), headers); } /** - * Asynchronously retrieves a document by id using the Get API + * Asynchronously retrieves a document by id using the Get API. + * See Get API on elastic.co + * @param getRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public final void getAsync(GetRequest getRequest, RequestOptions options, ActionListener listener) { + performRequestAsyncAndParseEntity(getRequest, RequestConverters::get, options, GetResponse::fromXContent, listener, + singleton(404)); + } + + /** + * Asynchronously retrieves a document by id using the Get API. * * See Get API on elastic.co + * @deprecated Prefer {@link #getAsync(GetRequest, RequestOptions, ActionListener)} */ + @Deprecated public final void getAsync(GetRequest getRequest, ActionListener listener, Header... headers) { performRequestAsyncAndParseEntity(getRequest, RequestConverters::get, GetResponse::fromXContent, listener, singleton(404), headers); } /** - * Retrieves multiple documents by id using the Multi Get API + * Retrieves multiple documents by id using the Multi Get API. + * See Multi Get API on elastic.co + * @param multiGetRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public final MultiGetResponse multiGet(MultiGetRequest multiGetRequest, RequestOptions options) throws IOException { + return performRequestAndParseEntity(multiGetRequest, RequestConverters::multiGet, options, MultiGetResponse::fromXContent, + singleton(404)); + } + + /** + * Retrieves multiple documents by id using the Multi Get API. * * See Multi Get API on elastic.co + * @deprecated Prefer {@link #multiGet(MultiGetRequest, RequestOptions)} */ + @Deprecated public final MultiGetResponse multiGet(MultiGetRequest multiGetRequest, Header... headers) throws IOException { return performRequestAndParseEntity(multiGetRequest, RequestConverters::multiGet, MultiGetResponse::fromXContent, singleton(404), headers); } /** - * Asynchronously retrieves multiple documents by id using the Multi Get API + * Asynchronously retrieves multiple documents by id using the Multi Get API. + * See Multi Get API on elastic.co + * @param multiGetRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public final void multiGetAsync(MultiGetRequest multiGetRequest, RequestOptions options, ActionListener listener) { + performRequestAsyncAndParseEntity(multiGetRequest, RequestConverters::multiGet, options, MultiGetResponse::fromXContent, listener, + singleton(404)); + } + + /** + * Asynchronously retrieves multiple documents by id using the Multi Get API. * * See Multi Get API on elastic.co + * @deprecated Prefer {@link #multiGetAsync(MultiGetRequest, RequestOptions, ActionListener)} */ + @Deprecated public final void multiGetAsync(MultiGetRequest multiGetRequest, ActionListener listener, Header... headers) { performRequestAsyncAndParseEntity(multiGetRequest, RequestConverters::multiGet, MultiGetResponse::fromXContent, listener, singleton(404), headers); } /** - * Checks for the existence of a document. Returns true if it exists, false otherwise + * Checks for the existence of a document. Returns true if it exists, false otherwise. + * See Get API on elastic.co + * @param getRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return true if the document exists, false otherwise + * @throws IOException in case there is a problem sending the request + */ + public final boolean exists(GetRequest getRequest, RequestOptions options) throws IOException { + return performRequest(getRequest, RequestConverters::exists, options, RestHighLevelClient::convertExistsResponse, emptySet()); + } + + /** + * Checks for the existence of a document. Returns true if it exists, false otherwise. * * See Get API on elastic.co + * @deprecated Prefer {@link #exists(GetRequest, RequestOptions)} */ + @Deprecated public final boolean exists(GetRequest getRequest, Header... headers) throws IOException { return performRequest(getRequest, RequestConverters::exists, RestHighLevelClient::convertExistsResponse, emptySet(), headers); } /** - * Asynchronously checks for the existence of a document. Returns true if it exists, false otherwise + * Asynchronously checks for the existence of a document. Returns true if it exists, false otherwise. + * See Get API on elastic.co + * @param getRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public final void existsAsync(GetRequest getRequest, RequestOptions options, ActionListener listener) { + performRequestAsync(getRequest, RequestConverters::exists, options, RestHighLevelClient::convertExistsResponse, listener, + emptySet()); + } + + /** + * Asynchronously checks for the existence of a document. Returns true if it exists, false otherwise. * * See Get API on elastic.co + * @deprecated Prefer {@link #existsAsync(GetRequest, RequestOptions, ActionListener)} */ + @Deprecated public final void existsAsync(GetRequest getRequest, ActionListener listener, Header... headers) { performRequestAsync(getRequest, RequestConverters::exists, RestHighLevelClient::convertExistsResponse, listener, emptySet(), headers); } /** - * Index a document using the Index API + * Index a document using the Index API. + * See Index API on elastic.co + * @param indexRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public final IndexResponse index(IndexRequest indexRequest, RequestOptions options) throws IOException { + return performRequestAndParseEntity(indexRequest, RequestConverters::index, options, IndexResponse::fromXContent, emptySet()); + } + + /** + * Index a document using the Index API. * * See Index API on elastic.co + * @deprecated Prefer {@link #index(IndexRequest, RequestOptions)} */ + @Deprecated public final IndexResponse index(IndexRequest indexRequest, Header... headers) throws IOException { return performRequestAndParseEntity(indexRequest, RequestConverters::index, IndexResponse::fromXContent, emptySet(), headers); } /** - * Asynchronously index a document using the Index API + * Asynchronously index a document using the Index API. + * See Index API on elastic.co + * @param indexRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public final void indexAsync(IndexRequest indexRequest, RequestOptions options, ActionListener listener) { + performRequestAsyncAndParseEntity(indexRequest, RequestConverters::index, options, IndexResponse::fromXContent, listener, + emptySet()); + } + + /** + * Asynchronously index a document using the Index API. * * See Index API on elastic.co + * @deprecated Prefer {@link #indexAsync(IndexRequest, RequestOptions, ActionListener)} */ + @Deprecated public final void indexAsync(IndexRequest indexRequest, ActionListener listener, Header... headers) { performRequestAsyncAndParseEntity(indexRequest, RequestConverters::index, IndexResponse::fromXContent, listener, emptySet(), headers); } /** - * Updates a document using the Update API + * Updates a document using the Update API. + * See Update API on elastic.co + * @param updateRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public final UpdateResponse update(UpdateRequest updateRequest, RequestOptions options) throws IOException { + return performRequestAndParseEntity(updateRequest, RequestConverters::update, options, UpdateResponse::fromXContent, emptySet()); + } + + /** + * Updates a document using the Update API. *

* See Update API on elastic.co + * @deprecated Prefer {@link #update(UpdateRequest, RequestOptions)} */ + @Deprecated public final UpdateResponse update(UpdateRequest updateRequest, Header... headers) throws IOException { return performRequestAndParseEntity(updateRequest, RequestConverters::update, UpdateResponse::fromXContent, emptySet(), headers); } /** - * Asynchronously updates a document using the Update API + * Asynchronously updates a document using the Update API. + * See Update API on elastic.co + * @param updateRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public final void updateAsync(UpdateRequest updateRequest, RequestOptions options, ActionListener listener) { + performRequestAsyncAndParseEntity(updateRequest, RequestConverters::update, options, UpdateResponse::fromXContent, listener, + emptySet()); + } + + /** + * Asynchronously updates a document using the Update API. *

* See Update API on elastic.co + * @deprecated Prefer {@link #updateAsync(UpdateRequest, RequestOptions, ActionListener)} */ + @Deprecated public final void updateAsync(UpdateRequest updateRequest, ActionListener listener, Header... headers) { performRequestAsyncAndParseEntity(updateRequest, RequestConverters::update, UpdateResponse::fromXContent, listener, emptySet(), headers); } /** - * Deletes a document by id using the Delete API + * Deletes a document by id using the Delete API. + * See Delete API on elastic.co + * @param deleteRequest the reuqest + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public final DeleteResponse delete(DeleteRequest deleteRequest, RequestOptions options) throws IOException { + return performRequestAndParseEntity(deleteRequest, RequestConverters::delete, options, DeleteResponse::fromXContent, + singleton(404)); + } + + /** + * Deletes a document by id using the Delete API. * * See Delete API on elastic.co + * @deprecated Prefer {@link #delete(DeleteRequest, RequestOptions)} */ + @Deprecated public final DeleteResponse delete(DeleteRequest deleteRequest, Header... headers) throws IOException { return performRequestAndParseEntity(deleteRequest, RequestConverters::delete, DeleteResponse::fromXContent, singleton(404), headers); } /** - * Asynchronously deletes a document by id using the Delete API + * Asynchronously deletes a document by id using the Delete API. + * See Delete API on elastic.co + * @param deleteRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public final void deleteAsync(DeleteRequest deleteRequest, RequestOptions options, ActionListener listener) { + performRequestAsyncAndParseEntity(deleteRequest, RequestConverters::delete, options, DeleteResponse::fromXContent, listener, + Collections.singleton(404)); + } + + /** + * Asynchronously deletes a document by id using the Delete API. * * See Delete API on elastic.co + * @deprecated Prefer {@link #deleteAsync(DeleteRequest, RequestOptions, ActionListener)} */ + @Deprecated public final void deleteAsync(DeleteRequest deleteRequest, ActionListener listener, Header... headers) { performRequestAsyncAndParseEntity(deleteRequest, RequestConverters::delete, DeleteResponse::fromXContent, listener, - Collections.singleton(404), headers); + Collections.singleton(404), headers); + } + + /** + * Executes a search request using the Search API. + * See Search API on elastic.co + * @param searchRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public final SearchResponse search(SearchRequest searchRequest, RequestOptions options) throws IOException { + return performRequestAndParseEntity(searchRequest, RequestConverters::search, options, SearchResponse::fromXContent, emptySet()); } /** - * Executes a search using the Search API + * Executes a search using the Search API. * * See Search API on elastic.co + * @deprecated Prefer {@link #search(SearchRequest, RequestOptions)} */ + @Deprecated public final SearchResponse search(SearchRequest searchRequest, Header... headers) throws IOException { return performRequestAndParseEntity(searchRequest, RequestConverters::search, SearchResponse::fromXContent, emptySet(), headers); } /** - * Asynchronously executes a search using the Search API + * Asynchronously executes a search using the Search API. + * See Search API on elastic.co + * @param searchRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public final void searchAsync(SearchRequest searchRequest, RequestOptions options, ActionListener listener) { + performRequestAsyncAndParseEntity(searchRequest, RequestConverters::search, options, SearchResponse::fromXContent, listener, + emptySet()); + } + + /** + * Asynchronously executes a search using the Search API. * * See Search API on elastic.co + * @deprecated Prefer {@link #searchAsync(SearchRequest, RequestOptions, ActionListener)} */ + @Deprecated public final void searchAsync(SearchRequest searchRequest, ActionListener listener, Header... headers) { performRequestAsyncAndParseEntity(searchRequest, RequestConverters::search, SearchResponse::fromXContent, listener, emptySet(), headers); } /** - * Executes a multi search using the msearch API + * Executes a multi search using the msearch API. + * See Multi search API on + * elastic.co + * @param multiSearchRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public final MultiSearchResponse multiSearch(MultiSearchRequest multiSearchRequest, RequestOptions options) throws IOException { + return performRequestAndParseEntity(multiSearchRequest, RequestConverters::multiSearch, options, MultiSearchResponse::fromXContext, + emptySet()); + } + + /** + * Executes a multi search using the msearch API. * * See Multi search API on * elastic.co + * @deprecated Prefer {@link #multiSearch(MultiSearchRequest, RequestOptions)} */ + @Deprecated public final MultiSearchResponse multiSearch(MultiSearchRequest multiSearchRequest, Header... headers) throws IOException { return performRequestAndParseEntity(multiSearchRequest, RequestConverters::multiSearch, MultiSearchResponse::fromXContext, emptySet(), headers); } /** - * Asynchronously executes a multi search using the msearch API + * Asynchronously executes a multi search using the msearch API. + * See Multi search API on + * elastic.co + * @param searchRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public final void multiSearchAsync(MultiSearchRequest searchRequest, RequestOptions options, + ActionListener listener) { + performRequestAsyncAndParseEntity(searchRequest, RequestConverters::multiSearch, options, MultiSearchResponse::fromXContext, + listener, emptySet()); + } + + /** + * Asynchronously executes a multi search using the msearch API. * * See Multi search API on * elastic.co + * @deprecated Prefer {@link #multiSearchAsync(MultiSearchRequest, RequestOptions, ActionListener)} */ + @Deprecated public final void multiSearchAsync(MultiSearchRequest searchRequest, ActionListener listener, Header... headers) { performRequestAsyncAndParseEntity(searchRequest, RequestConverters::multiSearch, MultiSearchResponse::fromXContext, listener, emptySet(), headers); } /** - * Executes a search using the Search Scroll API + * Executes a search using the Search Scroll API. + * See Search Scroll + * API on elastic.co + * @param searchScrollRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public final SearchResponse searchScroll(SearchScrollRequest searchScrollRequest, RequestOptions options) throws IOException { + return performRequestAndParseEntity(searchScrollRequest, RequestConverters::searchScroll, options, SearchResponse::fromXContent, + emptySet()); + } + + /** + * Executes a search using the Search Scroll API. * * See Search Scroll * API on elastic.co + * @deprecated Prefer {@link #searchScroll(SearchScrollRequest, RequestOptions)} */ + @Deprecated public final SearchResponse searchScroll(SearchScrollRequest searchScrollRequest, Header... headers) throws IOException { return performRequestAndParseEntity(searchScrollRequest, RequestConverters::searchScroll, SearchResponse::fromXContent, emptySet(), headers); } /** - * Asynchronously executes a search using the Search Scroll API + * Asynchronously executes a search using the Search Scroll API. + * See Search Scroll + * API on elastic.co + * @param searchScrollRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public final void searchScrollAsync(SearchScrollRequest searchScrollRequest, RequestOptions options, + ActionListener listener) { + performRequestAsyncAndParseEntity(searchScrollRequest, RequestConverters::searchScroll, options, SearchResponse::fromXContent, + listener, emptySet()); + } + + /** + * Asynchronously executes a search using the Search Scroll API. * * See Search Scroll * API on elastic.co + * @deprecated Prefer {@link #searchScrollAsync(SearchScrollRequest, RequestOptions, ActionListener)} */ + @Deprecated public final void searchScrollAsync(SearchScrollRequest searchScrollRequest, ActionListener listener, Header... headers) { performRequestAsyncAndParseEntity(searchScrollRequest, RequestConverters::searchScroll, SearchResponse::fromXContent, @@ -521,22 +814,54 @@ public final void searchScrollAsync(SearchScrollRequest searchScrollRequest, } /** - * Clears one or more scroll ids using the Clear Scroll API + * Clears one or more scroll ids using the Clear Scroll API. + * See + * Clear Scroll API on elastic.co + * @param clearScrollRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public final ClearScrollResponse clearScroll(ClearScrollRequest clearScrollRequest, RequestOptions options) throws IOException { + return performRequestAndParseEntity(clearScrollRequest, RequestConverters::clearScroll, options, ClearScrollResponse::fromXContent, + emptySet()); + } + + /** + * Clears one or more scroll ids using the Clear Scroll API. * * See * Clear Scroll API on elastic.co + * @deprecated Prefer {@link #clearScroll(ClearScrollRequest, RequestOptions)} */ + @Deprecated public final ClearScrollResponse clearScroll(ClearScrollRequest clearScrollRequest, Header... headers) throws IOException { return performRequestAndParseEntity(clearScrollRequest, RequestConverters::clearScroll, ClearScrollResponse::fromXContent, emptySet(), headers); } /** - * Asynchronously clears one or more scroll ids using the Clear Scroll API + * Asynchronously clears one or more scroll ids using the Clear Scroll API. + * See + * Clear Scroll API on elastic.co + * @param clearScrollRequest the reuqest + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public final void clearScrollAsync(ClearScrollRequest clearScrollRequest, RequestOptions options, + ActionListener listener) { + performRequestAsyncAndParseEntity(clearScrollRequest, RequestConverters::clearScroll, options, ClearScrollResponse::fromXContent, + listener, emptySet()); + } + + /** + * Asynchronously clears one or more scroll ids using the Clear Scroll API. * * See * Clear Scroll API on elastic.co + * @deprecated Prefer {@link #clearScrollAsync(ClearScrollRequest, RequestOptions, ActionListener)} */ + @Deprecated public final void clearScrollAsync(ClearScrollRequest clearScrollRequest, ActionListener listener, Header... headers) { performRequestAsyncAndParseEntity(clearScrollRequest, RequestConverters::clearScroll, ClearScrollResponse::fromXContent, @@ -545,47 +870,79 @@ public final void clearScrollAsync(ClearScrollRequest clearScrollRequest, /** * Executes a request using the Search Template API. - * * See Search Template API * on elastic.co. + * @param searchTemplateRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response */ public final SearchTemplateResponse searchTemplate(SearchTemplateRequest searchTemplateRequest, - Header... headers) throws IOException { - return performRequestAndParseEntity(searchTemplateRequest, RequestConverters::searchTemplate, - SearchTemplateResponse::fromXContent, emptySet(), headers); + RequestOptions options) throws IOException { + return performRequestAndParseEntity(searchTemplateRequest, RequestConverters::searchTemplate, options, + SearchTemplateResponse::fromXContent, emptySet()); } /** - * Asynchronously executes a request using the Search Template API + * Asynchronously executes a request using the Search Template API. * * See Search Template API * on elastic.co. */ - public final void searchTemplateAsync(SearchTemplateRequest searchTemplateRequest, - ActionListener listener, - Header... headers) { - performRequestAsyncAndParseEntity(searchTemplateRequest, RequestConverters::searchTemplate, - SearchTemplateResponse::fromXContent, listener, emptySet(), headers); + public final void searchTemplateAsync(SearchTemplateRequest searchTemplateRequest, RequestOptions options, + ActionListener listener) { + performRequestAsyncAndParseEntity(searchTemplateRequest, RequestConverters::searchTemplate, options, + SearchTemplateResponse::fromXContent, listener, emptySet()); } + /** + * Executes a request using the Ranking Evaluation API. + * See Ranking Evaluation API + * on elastic.co + * @param rankEvalRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public final RankEvalResponse rankEval(RankEvalRequest rankEvalRequest, RequestOptions options) throws IOException { + return performRequestAndParseEntity(rankEvalRequest, RequestConverters::rankEval, options, RankEvalResponse::fromXContent, + emptySet()); + } /** * Executes a request using the Ranking Evaluation API. * * See Ranking Evaluation API * on elastic.co + * @deprecated Prefer {@link #rankEval(RankEvalRequest, RequestOptions)} */ + @Deprecated public final RankEvalResponse rankEval(RankEvalRequest rankEvalRequest, Header... headers) throws IOException { return performRequestAndParseEntity(rankEvalRequest, RequestConverters::rankEval, RankEvalResponse::fromXContent, emptySet(), headers); } + /** + * Asynchronously executes a request using the Ranking Evaluation API. + * See Ranking Evaluation API + * on elastic.co + * @param rankEvalRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public final void rankEvalAsync(RankEvalRequest rankEvalRequest, RequestOptions options, ActionListener listener) { + performRequestAsyncAndParseEntity(rankEvalRequest, RequestConverters::rankEval, options, RankEvalResponse::fromXContent, listener, + emptySet()); + } + /** * Asynchronously executes a request using the Ranking Evaluation API. * * See Ranking Evaluation API * on elastic.co + * @deprecated Prefer {@link #rankEvalAsync(RankEvalRequest, RequestOptions, ActionListener)} */ + @Deprecated public final void rankEvalAsync(RankEvalRequest rankEvalRequest, ActionListener listener, Header... headers) { performRequestAsyncAndParseEntity(rankEvalRequest, RequestConverters::rankEval, RankEvalResponse::fromXContent, listener, emptySet(), headers); @@ -593,27 +950,31 @@ public final void rankEvalAsync(RankEvalRequest rankEvalRequest, ActionListener< /** * Executes a request using the Field Capabilities API. - * * See Field Capabilities API * on elastic.co. + * @param fieldCapabilitiesRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response */ public final FieldCapabilitiesResponse fieldCaps(FieldCapabilitiesRequest fieldCapabilitiesRequest, - Header... headers) throws IOException { - return performRequestAndParseEntity(fieldCapabilitiesRequest, RequestConverters::fieldCaps, - FieldCapabilitiesResponse::fromXContent, emptySet(), headers); + RequestOptions options) throws IOException { + return performRequestAndParseEntity(fieldCapabilitiesRequest, RequestConverters::fieldCaps, options, + FieldCapabilitiesResponse::fromXContent, emptySet()); } /** * Asynchronously executes a request using the Field Capabilities API. - * * See Field Capabilities API * on elastic.co. + * @param fieldCapabilitiesRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion */ - public final void fieldCapsAsync(FieldCapabilitiesRequest fieldCapabilitiesRequest, - ActionListener listener, - Header... headers) { - performRequestAsyncAndParseEntity(fieldCapabilitiesRequest, RequestConverters::fieldCaps, - FieldCapabilitiesResponse::fromXContent, listener, emptySet(), headers); + public final void fieldCapsAsync(FieldCapabilitiesRequest fieldCapabilitiesRequest, RequestOptions options, + ActionListener listener) { + performRequestAsyncAndParseEntity(fieldCapabilitiesRequest, RequestConverters::fieldCaps, options, + FieldCapabilitiesResponse::fromXContent, listener, emptySet()); } @Deprecated diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java index 104bc91271148..b7cd2d52732cc 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java @@ -19,7 +19,6 @@ package org.elasticsearch.client; -import org.apache.http.Header; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryRequest; import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryResponse; @@ -49,97 +48,117 @@ public final class SnapshotClient { /** * Gets a list of snapshot repositories. If the list of repositories is empty or it contains a single element "_all", all * registered repositories are returned. - *

* See Snapshot and Restore * API on elastic.co + * @param getRepositoriesRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response */ - public GetRepositoriesResponse getRepositories(GetRepositoriesRequest getRepositoriesRequest, Header... headers) + public GetRepositoriesResponse getRepositories(GetRepositoriesRequest getRepositoriesRequest, RequestOptions options) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(getRepositoriesRequest, RequestConverters::getRepositories, - GetRepositoriesResponse::fromXContent, emptySet(), headers); + return restHighLevelClient.performRequestAndParseEntity(getRepositoriesRequest, RequestConverters::getRepositories, options, + GetRepositoriesResponse::fromXContent, emptySet()); } /** * Asynchronously gets a list of snapshot repositories. If the list of repositories is empty or it contains a single element "_all", all * registered repositories are returned. - *

* See Snapshot and Restore * API on elastic.co + * @param getRepositoriesRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion */ - public void getRepositoriesAsync(GetRepositoriesRequest getRepositoriesRequest, - ActionListener listener, Header... headers) { - restHighLevelClient.performRequestAsyncAndParseEntity(getRepositoriesRequest, RequestConverters::getRepositories, - GetRepositoriesResponse::fromXContent, listener, emptySet(), headers); + public void getRepositoriesAsync(GetRepositoriesRequest getRepositoriesRequest, RequestOptions options, + ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(getRepositoriesRequest, RequestConverters::getRepositories, options, + GetRepositoriesResponse::fromXContent, listener, emptySet()); } /** * Creates a snapshot repository. - *

* See Snapshot and Restore * API on elastic.co + * @param putRepositoryRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response */ - public PutRepositoryResponse createRepository(PutRepositoryRequest putRepositoryRequest, Header... headers) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(putRepositoryRequest, RequestConverters::createRepository, - PutRepositoryResponse::fromXContent, emptySet(), headers); + public PutRepositoryResponse createRepository(PutRepositoryRequest putRepositoryRequest, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(putRepositoryRequest, RequestConverters::createRepository, options, + PutRepositoryResponse::fromXContent, emptySet()); } /** * Asynchronously creates a snapshot repository. - *

* See Snapshot and Restore * API on elastic.co + * @param putRepositoryRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion */ - public void createRepositoryAsync(PutRepositoryRequest putRepositoryRequest, - ActionListener listener, Header... headers) { - restHighLevelClient.performRequestAsyncAndParseEntity(putRepositoryRequest, RequestConverters::createRepository, - PutRepositoryResponse::fromXContent, listener, emptySet(), headers); + public void createRepositoryAsync(PutRepositoryRequest putRepositoryRequest, RequestOptions options, + ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(putRepositoryRequest, RequestConverters::createRepository, options, + PutRepositoryResponse::fromXContent, listener, emptySet()); } /** * Deletes a snapshot repository. - *

* See Snapshot and Restore * API on elastic.co + * @param deleteRepositoryRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response */ - public DeleteRepositoryResponse deleteRepository(DeleteRepositoryRequest deleteRepositoryRequest, Header... headers) + public DeleteRepositoryResponse deleteRepository(DeleteRepositoryRequest deleteRepositoryRequest, RequestOptions options) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(deleteRepositoryRequest, RequestConverters::deleteRepository, - DeleteRepositoryResponse::fromXContent, emptySet(), headers); + return restHighLevelClient.performRequestAndParseEntity(deleteRepositoryRequest, RequestConverters::deleteRepository, options, + DeleteRepositoryResponse::fromXContent, emptySet()); } /** * Asynchronously deletes a snapshot repository. - *

* See Snapshot and Restore * API on elastic.co + * @param deleteRepositoryRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion */ - public void deleteRepositoryAsync(DeleteRepositoryRequest deleteRepositoryRequest, - ActionListener listener, Header... headers) { - restHighLevelClient.performRequestAsyncAndParseEntity(deleteRepositoryRequest, RequestConverters::deleteRepository, - DeleteRepositoryResponse::fromXContent, listener, emptySet(), headers); + public void deleteRepositoryAsync(DeleteRepositoryRequest deleteRepositoryRequest, RequestOptions options, + ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(deleteRepositoryRequest, RequestConverters::deleteRepository, options, + DeleteRepositoryResponse::fromXContent, listener, emptySet()); } /** * Verifies a snapshot repository. - *

* See Snapshot and Restore * API on elastic.co + * @param verifyRepositoryRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response */ - public VerifyRepositoryResponse verifyRepository(VerifyRepositoryRequest verifyRepositoryRequest, Header... headers) + public VerifyRepositoryResponse verifyRepository(VerifyRepositoryRequest verifyRepositoryRequest, RequestOptions options) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(verifyRepositoryRequest, RequestConverters::verifyRepository, - VerifyRepositoryResponse::fromXContent, emptySet(), headers); + return restHighLevelClient.performRequestAndParseEntity(verifyRepositoryRequest, RequestConverters::verifyRepository, options, + VerifyRepositoryResponse::fromXContent, emptySet()); } /** * Asynchronously verifies a snapshot repository. - *

* See Snapshot and Restore * API on elastic.co + * @param verifyRepositoryRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion */ - public void verifyRepositoryAsync(VerifyRepositoryRequest verifyRepositoryRequest, - ActionListener listener, Header... headers) { - restHighLevelClient.performRequestAsyncAndParseEntity(verifyRepositoryRequest, RequestConverters::verifyRepository, - VerifyRepositoryResponse::fromXContent, listener, emptySet(), headers); + public void verifyRepositoryAsync(VerifyRepositoryRequest verifyRepositoryRequest, RequestOptions options, + ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(verifyRepositoryRequest, RequestConverters::verifyRepository, options, + VerifyRepositoryResponse::fromXContent, listener, emptySet()); } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/TasksClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/TasksClient.java index 214f1e7884a2a..f4a76e78b946b 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/TasksClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/TasksClient.java @@ -19,7 +19,6 @@ package org.elasticsearch.client; -import org.apache.http.Header; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; @@ -33,7 +32,7 @@ *

* See Task Management API on elastic.co */ -public class TasksClient { +public final class TasksClient { private final RestHighLevelClient restHighLevelClient; TasksClient(RestHighLevelClient restHighLevelClient) { @@ -41,24 +40,29 @@ public class TasksClient { } /** - * Get current tasks using the Task Management API - *

+ * Get current tasks using the Task Management API. * See * Task Management API on elastic.co + * @param request the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response */ - public ListTasksResponse list(ListTasksRequest request, Header... headers) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(request, RequestConverters::listTasks, ListTasksResponse::fromXContent, - emptySet(), headers); + public ListTasksResponse list(ListTasksRequest request, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(request, RequestConverters::listTasks, options, + ListTasksResponse::fromXContent, emptySet()); } /** - * Asynchronously get current tasks using the Task Management API - *

+ * Asynchronously get current tasks using the Task Management API. * See * Task Management API on elastic.co + * @param request the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion */ - public void listAsync(ListTasksRequest request, ActionListener listener, Header... headers) { - restHighLevelClient.performRequestAsyncAndParseEntity(request, RequestConverters::listTasks, ListTasksResponse::fromXContent, - listener, emptySet(), headers); + public void listAsync(ListTasksRequest request, RequestOptions options, ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(request, RequestConverters::listTasks, options, + ListTasksResponse::fromXContent, listener, emptySet()); } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorIT.java index 9782b1016b421..d41c47177f968 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorIT.java @@ -20,8 +20,6 @@ package org.elasticsearch.client; import com.carrotsearch.randomizedtesting.generators.RandomPicks; -import org.apache.http.entity.ContentType; -import org.apache.http.nio.entity.NStringEntity; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkProcessor; import org.elasticsearch.action.bulk.BulkRequest; @@ -39,7 +37,6 @@ import org.elasticsearch.common.xcontent.json.JsonXContent; import java.util.Arrays; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -81,7 +78,7 @@ public void testThatBulkProcessorCountIsCorrect() throws Exception { assertThat(listener.afterCounts.get(), equalTo(1)); assertThat(listener.bulkFailures.size(), equalTo(0)); assertResponseItems(listener.bulkItems, numDocs); - assertMultiGetResponse(highLevelClient().multiGet(multiGetRequest), numDocs); + assertMultiGetResponse(highLevelClient().multiGet(multiGetRequest, RequestOptions.DEFAULT), numDocs); } } @@ -107,7 +104,7 @@ public void testBulkProcessorFlush() throws Exception { assertThat(listener.afterCounts.get(), equalTo(1)); assertThat(listener.bulkFailures.size(), equalTo(0)); assertResponseItems(listener.bulkItems, numDocs); - assertMultiGetResponse(highLevelClient().multiGet(multiGetRequest), numDocs); + assertMultiGetResponse(highLevelClient().multiGet(multiGetRequest, RequestOptions.DEFAULT), numDocs); } } @@ -159,7 +156,7 @@ public void testBulkProcessorConcurrentRequests() throws Exception { assertThat(ids.add(bulkItemResponse.getId()), equalTo(true)); } - assertMultiGetResponse(highLevelClient().multiGet(multiGetRequest), numDocs); + assertMultiGetResponse(highLevelClient().multiGet(multiGetRequest, RequestOptions.DEFAULT), numDocs); } public void testBulkProcessorWaitOnClose() throws Exception { @@ -190,7 +187,7 @@ public void testBulkProcessorWaitOnClose() throws Exception { } assertThat(listener.bulkFailures.size(), equalTo(0)); assertResponseItems(listener.bulkItems, numDocs); - assertMultiGetResponse(highLevelClient().multiGet(multiGetRequest), numDocs); + assertMultiGetResponse(highLevelClient().multiGet(multiGetRequest, RequestOptions.DEFAULT), numDocs); } public void testBulkProcessorConcurrentRequestsReadOnlyIndex() throws Exception { @@ -267,7 +264,7 @@ public void testBulkProcessorConcurrentRequestsReadOnlyIndex() throws Exception } } - assertMultiGetResponse(highLevelClient().multiGet(multiGetRequest), testDocs); + assertMultiGetResponse(highLevelClient().multiGet(multiGetRequest, RequestOptions.DEFAULT), testDocs); } private static MultiGetRequest indexDocs(BulkProcessor processor, int numDocs) throws Exception { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorRetryIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorRetryIT.java index 597d35a99967b..fe6aa6b1017ee 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorRetryIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorRetryIT.java @@ -127,8 +127,8 @@ public void afterBulk(long executionId, BulkRequest request, Throwable failure) } } - highLevelClient().indices().refresh(new RefreshRequest()); - int multiGetResponsesCount = highLevelClient().multiGet(multiGetRequest).getResponses().length; + highLevelClient().indices().refresh(new RefreshRequest(), RequestOptions.DEFAULT); + int multiGetResponsesCount = highLevelClient().multiGet(multiGetRequest, RequestOptions.DEFAULT).getResponses().length; if (rejectedExecutionExpected) { assertThat(multiGetResponsesCount, lessThanOrEqualTo(numberOfAsyncOps)); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterClientIT.java index 9314bb2e36cea..f1110163b2517 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterClientIT.java @@ -57,6 +57,7 @@ public void testClusterPutSettings() throws IOException { setRequest.persistentSettings(map); ClusterUpdateSettingsResponse setResponse = execute(setRequest, highLevelClient().cluster()::putSettings, + highLevelClient().cluster()::putSettingsAsync, highLevelClient().cluster()::putSettings, highLevelClient().cluster()::putSettingsAsync); assertAcked(setResponse); @@ -79,6 +80,7 @@ public void testClusterPutSettings() throws IOException { resetRequest.persistentSettings("{\"" + persistentSettingKey + "\": null }", XContentType.JSON); ClusterUpdateSettingsResponse resetResponse = execute(resetRequest, highLevelClient().cluster()::putSettings, + highLevelClient().cluster()::putSettingsAsync, highLevelClient().cluster()::putSettings, highLevelClient().cluster()::putSettingsAsync); assertThat(resetResponse.getTransientSettings().get(transientSettingKey), equalTo(null)); @@ -100,6 +102,7 @@ public void testClusterUpdateSettingNonExistent() { clusterUpdateSettingsRequest.transientSettings(Settings.builder().put(setting, value).build()); ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> execute(clusterUpdateSettingsRequest, + highLevelClient().cluster()::putSettings, highLevelClient().cluster()::putSettingsAsync, highLevelClient().cluster()::putSettings, highLevelClient().cluster()::putSettingsAsync)); assertThat(exception.status(), equalTo(RestStatus.BAD_REQUEST)); assertThat(exception.getMessage(), equalTo( diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java index f384e5706b09a..81c894f242fe4 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java @@ -68,12 +68,14 @@ public void testDelete() throws IOException { { // Testing deletion String docId = "id"; - highLevelClient().index(new IndexRequest("index", "type", docId).source(Collections.singletonMap("foo", "bar"))); + highLevelClient().index( + new IndexRequest("index", "type", docId).source(Collections.singletonMap("foo", "bar")), RequestOptions.DEFAULT); DeleteRequest deleteRequest = new DeleteRequest("index", "type", docId); if (randomBoolean()) { deleteRequest.version(1L); } - DeleteResponse deleteResponse = execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync); + DeleteResponse deleteResponse = execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync, + highLevelClient()::delete, highLevelClient()::deleteAsync); assertEquals("index", deleteResponse.getIndex()); assertEquals("type", deleteResponse.getType()); assertEquals(docId, deleteResponse.getId()); @@ -83,7 +85,8 @@ public void testDelete() throws IOException { // Testing non existing document String docId = "does_not_exist"; DeleteRequest deleteRequest = new DeleteRequest("index", "type", docId); - DeleteResponse deleteResponse = execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync); + DeleteResponse deleteResponse = execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync, + highLevelClient()::delete, highLevelClient()::deleteAsync); assertEquals("index", deleteResponse.getIndex()); assertEquals("type", deleteResponse.getType()); assertEquals(docId, deleteResponse.getId()); @@ -92,10 +95,12 @@ public void testDelete() throws IOException { { // Testing version conflict String docId = "version_conflict"; - highLevelClient().index(new IndexRequest("index", "type", docId).source(Collections.singletonMap("foo", "bar"))); + highLevelClient().index( + new IndexRequest("index", "type", docId).source(Collections.singletonMap("foo", "bar")), RequestOptions.DEFAULT); DeleteRequest deleteRequest = new DeleteRequest("index", "type", docId).version(2); ElasticsearchException exception = expectThrows(ElasticsearchException.class, - () -> execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync)); + () -> execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync, + highLevelClient()::delete, highLevelClient()::deleteAsync)); assertEquals(RestStatus.CONFLICT, exception.status()); assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][" + docId + "]: " + "version conflict, current version [1] is different than the one provided [2]]", exception.getMessage()); @@ -104,10 +109,12 @@ public void testDelete() throws IOException { { // Testing version type String docId = "version_type"; - highLevelClient().index(new IndexRequest("index", "type", docId).source(Collections.singletonMap("foo", "bar")) - .versionType(VersionType.EXTERNAL).version(12)); + highLevelClient().index( + new IndexRequest("index", "type", docId).source(Collections.singletonMap("foo", "bar")) + .versionType(VersionType.EXTERNAL).version(12), RequestOptions.DEFAULT); DeleteRequest deleteRequest = new DeleteRequest("index", "type", docId).versionType(VersionType.EXTERNAL).version(13); - DeleteResponse deleteResponse = execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync); + DeleteResponse deleteResponse = execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync, + highLevelClient()::delete, highLevelClient()::deleteAsync); assertEquals("index", deleteResponse.getIndex()); assertEquals("type", deleteResponse.getType()); assertEquals(docId, deleteResponse.getId()); @@ -116,11 +123,13 @@ public void testDelete() throws IOException { { // Testing version type with a wrong version String docId = "wrong_version"; - highLevelClient().index(new IndexRequest("index", "type", docId).source(Collections.singletonMap("foo", "bar")) - .versionType(VersionType.EXTERNAL).version(12)); + highLevelClient().index( + new IndexRequest("index", "type", docId).source(Collections.singletonMap("foo", "bar")) + .versionType(VersionType.EXTERNAL).version(12), RequestOptions.DEFAULT); ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, () -> { DeleteRequest deleteRequest = new DeleteRequest("index", "type", docId).versionType(VersionType.EXTERNAL).version(10); - execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync); + execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync, + highLevelClient()::delete, highLevelClient()::deleteAsync); }); assertEquals(RestStatus.CONFLICT, exception.status()); assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][" + @@ -130,9 +139,11 @@ public void testDelete() throws IOException { { // Testing routing String docId = "routing"; - highLevelClient().index(new IndexRequest("index", "type", docId).source(Collections.singletonMap("foo", "bar")).routing("foo")); + highLevelClient().index(new IndexRequest("index", "type", docId).source(Collections.singletonMap("foo", "bar")).routing("foo"), + RequestOptions.DEFAULT); DeleteRequest deleteRequest = new DeleteRequest("index", "type", docId).routing("foo"); - DeleteResponse deleteResponse = execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync); + DeleteResponse deleteResponse = execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync, + highLevelClient()::delete, highLevelClient()::deleteAsync); assertEquals("index", deleteResponse.getIndex()); assertEquals("type", deleteResponse.getType()); assertEquals(docId, deleteResponse.getId()); @@ -143,23 +154,27 @@ public void testDelete() throws IOException { public void testExists() throws IOException { { GetRequest getRequest = new GetRequest("index", "type", "id"); - assertFalse(execute(getRequest, highLevelClient()::exists, highLevelClient()::existsAsync)); + assertFalse(execute(getRequest, highLevelClient()::exists, highLevelClient()::existsAsync, + highLevelClient()::exists, highLevelClient()::existsAsync)); } IndexRequest index = new IndexRequest("index", "type", "id"); index.source("{\"field1\":\"value1\",\"field2\":\"value2\"}", XContentType.JSON); index.setRefreshPolicy(RefreshPolicy.IMMEDIATE); - highLevelClient().index(index); + highLevelClient().index(index, RequestOptions.DEFAULT); { GetRequest getRequest = new GetRequest("index", "type", "id"); - assertTrue(execute(getRequest, highLevelClient()::exists, highLevelClient()::existsAsync)); + assertTrue(execute(getRequest, highLevelClient()::exists, highLevelClient()::existsAsync, + highLevelClient()::exists, highLevelClient()::existsAsync)); } { GetRequest getRequest = new GetRequest("index", "type", "does_not_exist"); - assertFalse(execute(getRequest, highLevelClient()::exists, highLevelClient()::existsAsync)); + assertFalse(execute(getRequest, highLevelClient()::exists, highLevelClient()::existsAsync, + highLevelClient()::exists, highLevelClient()::existsAsync)); } { GetRequest getRequest = new GetRequest("index", "type", "does_not_exist").version(1); - assertFalse(execute(getRequest, highLevelClient()::exists, highLevelClient()::existsAsync)); + assertFalse(execute(getRequest, highLevelClient()::exists, highLevelClient()::existsAsync, + highLevelClient()::exists, highLevelClient()::existsAsync)); } } @@ -167,7 +182,8 @@ public void testGet() throws IOException { { GetRequest getRequest = new GetRequest("index", "type", "id"); ElasticsearchException exception = expectThrows(ElasticsearchException.class, - () -> execute(getRequest, highLevelClient()::get, highLevelClient()::getAsync)); + () -> execute(getRequest, highLevelClient()::get, highLevelClient()::getAsync, + highLevelClient()::get, highLevelClient()::getAsync)); assertEquals(RestStatus.NOT_FOUND, exception.status()); assertEquals("Elasticsearch exception [type=index_not_found_exception, reason=no such index]", exception.getMessage()); assertEquals("index", exception.getMetadata("es.index").get(0)); @@ -176,11 +192,12 @@ public void testGet() throws IOException { String document = "{\"field1\":\"value1\",\"field2\":\"value2\"}"; index.source(document, XContentType.JSON); index.setRefreshPolicy(RefreshPolicy.IMMEDIATE); - highLevelClient().index(index); + highLevelClient().index(index, RequestOptions.DEFAULT); { GetRequest getRequest = new GetRequest("index", "type", "id").version(2); ElasticsearchException exception = expectThrows(ElasticsearchException.class, - () -> execute(getRequest, highLevelClient()::get, highLevelClient()::getAsync)); + () -> execute(getRequest, highLevelClient()::get, highLevelClient()::getAsync, + highLevelClient()::get, highLevelClient()::getAsync)); assertEquals(RestStatus.CONFLICT, exception.status()); assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, " + "reason=[type][id]: " + "version conflict, current version [1] is different than the one provided [2]]", exception.getMessage()); @@ -191,7 +208,8 @@ public void testGet() throws IOException { if (randomBoolean()) { getRequest.version(1L); } - GetResponse getResponse = execute(getRequest, highLevelClient()::get, highLevelClient()::getAsync); + GetResponse getResponse = execute(getRequest, highLevelClient()::get, highLevelClient()::getAsync, + highLevelClient()::get, highLevelClient()::getAsync); assertEquals("index", getResponse.getIndex()); assertEquals("type", getResponse.getType()); assertEquals("id", getResponse.getId()); @@ -202,7 +220,8 @@ public void testGet() throws IOException { } { GetRequest getRequest = new GetRequest("index", "type", "does_not_exist"); - GetResponse getResponse = execute(getRequest, highLevelClient()::get, highLevelClient()::getAsync); + GetResponse getResponse = execute(getRequest, highLevelClient()::get, highLevelClient()::getAsync, + highLevelClient()::get, highLevelClient()::getAsync); assertEquals("index", getResponse.getIndex()); assertEquals("type", getResponse.getType()); assertEquals("does_not_exist", getResponse.getId()); @@ -214,7 +233,8 @@ public void testGet() throws IOException { { GetRequest getRequest = new GetRequest("index", "type", "id"); getRequest.fetchSourceContext(new FetchSourceContext(false, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY)); - GetResponse getResponse = execute(getRequest, highLevelClient()::get, highLevelClient()::getAsync); + GetResponse getResponse = execute(getRequest, highLevelClient()::get, highLevelClient()::getAsync, + highLevelClient()::get, highLevelClient()::getAsync); assertEquals("index", getResponse.getIndex()); assertEquals("type", getResponse.getType()); assertEquals("id", getResponse.getId()); @@ -230,7 +250,8 @@ public void testGet() throws IOException { } else { getRequest.fetchSourceContext(new FetchSourceContext(true, Strings.EMPTY_ARRAY, new String[]{"field2"})); } - GetResponse getResponse = execute(getRequest, highLevelClient()::get, highLevelClient()::getAsync); + GetResponse getResponse = execute(getRequest, highLevelClient()::get, highLevelClient()::getAsync, + highLevelClient()::get, highLevelClient()::getAsync); assertEquals("index", getResponse.getIndex()); assertEquals("type", getResponse.getType()); assertEquals("id", getResponse.getId()); @@ -248,7 +269,8 @@ public void testMultiGet() throws IOException { MultiGetRequest multiGetRequest = new MultiGetRequest(); multiGetRequest.add("index", "type", "id1"); multiGetRequest.add("index", "type", "id2"); - MultiGetResponse response = execute(multiGetRequest, highLevelClient()::multiGet, highLevelClient()::multiGetAsync); + MultiGetResponse response = execute(multiGetRequest, highLevelClient()::multiGet, highLevelClient()::multiGetAsync, + highLevelClient()::multiGet, highLevelClient()::multiGetAsync); assertEquals(2, response.getResponses().length); assertTrue(response.getResponses()[0].isFailed()); @@ -275,12 +297,13 @@ public void testMultiGet() throws IOException { index = new IndexRequest("index", "type", "id2"); index.source("{\"field\":\"value2\"}", XContentType.JSON); bulk.add(index); - highLevelClient().bulk(bulk); + highLevelClient().bulk(bulk, RequestOptions.DEFAULT); { MultiGetRequest multiGetRequest = new MultiGetRequest(); multiGetRequest.add("index", "type", "id1"); multiGetRequest.add("index", "type", "id2"); - MultiGetResponse response = execute(multiGetRequest, highLevelClient()::multiGet, highLevelClient()::multiGetAsync); + MultiGetResponse response = execute(multiGetRequest, highLevelClient()::multiGet, highLevelClient()::multiGetAsync, + highLevelClient()::multiGet, highLevelClient()::multiGetAsync); assertEquals(2, response.getResponses().length); assertFalse(response.getResponses()[0].isFailed()); @@ -305,7 +328,8 @@ public void testIndex() throws IOException { IndexRequest indexRequest = new IndexRequest("index", "type"); indexRequest.source(XContentBuilder.builder(xContentType.xContent()).startObject().field("test", "test").endObject()); - IndexResponse indexResponse = execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync); + IndexResponse indexResponse = execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync, + highLevelClient()::index, highLevelClient()::indexAsync); assertEquals(RestStatus.CREATED, indexResponse.status()); assertEquals(DocWriteResponse.Result.CREATED, indexResponse.getResult()); assertEquals("index", indexResponse.getIndex()); @@ -326,7 +350,8 @@ public void testIndex() throws IOException { IndexRequest indexRequest = new IndexRequest("index", "type", "id"); indexRequest.source(XContentBuilder.builder(xContentType.xContent()).startObject().field("version", 1).endObject()); - IndexResponse indexResponse = execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync); + IndexResponse indexResponse = execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync, + highLevelClient()::index, highLevelClient()::indexAsync); assertEquals(RestStatus.CREATED, indexResponse.status()); assertEquals("index", indexResponse.getIndex()); assertEquals("type", indexResponse.getType()); @@ -336,7 +361,8 @@ public void testIndex() throws IOException { indexRequest = new IndexRequest("index", "type", "id"); indexRequest.source(XContentBuilder.builder(xContentType.xContent()).startObject().field("version", 2).endObject()); - indexResponse = execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync); + indexResponse = execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync, + highLevelClient()::index, highLevelClient()::indexAsync); assertEquals(RestStatus.OK, indexResponse.status()); assertEquals("index", indexResponse.getIndex()); assertEquals("type", indexResponse.getType()); @@ -348,7 +374,8 @@ public void testIndex() throws IOException { wrongRequest.source(XContentBuilder.builder(xContentType.xContent()).startObject().field("field", "test").endObject()); wrongRequest.version(5L); - execute(wrongRequest, highLevelClient()::index, highLevelClient()::indexAsync); + execute(wrongRequest, highLevelClient()::index, highLevelClient()::indexAsync, + highLevelClient()::index, highLevelClient()::indexAsync); }); assertEquals(RestStatus.CONFLICT, exception.status()); assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][id]: " + @@ -361,7 +388,8 @@ public void testIndex() throws IOException { indexRequest.source(XContentBuilder.builder(xContentType.xContent()).startObject().field("field", "test").endObject()); indexRequest.setPipeline("missing"); - execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync); + execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync, + highLevelClient()::index, highLevelClient()::indexAsync); }); assertEquals(RestStatus.BAD_REQUEST, exception.status()); @@ -374,7 +402,8 @@ public void testIndex() throws IOException { indexRequest.version(12L); indexRequest.versionType(VersionType.EXTERNAL); - IndexResponse indexResponse = execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync); + IndexResponse indexResponse = execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync, + highLevelClient()::index, highLevelClient()::indexAsync); assertEquals(RestStatus.CREATED, indexResponse.status()); assertEquals("index", indexResponse.getIndex()); assertEquals("type", indexResponse.getType()); @@ -386,14 +415,16 @@ public void testIndex() throws IOException { indexRequest.source(XContentBuilder.builder(xContentType.xContent()).startObject().field("field", "test").endObject()); indexRequest.opType(DocWriteRequest.OpType.CREATE); - IndexResponse indexResponse = execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync); + IndexResponse indexResponse = execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync, + highLevelClient()::index, highLevelClient()::indexAsync); assertEquals(RestStatus.CREATED, indexResponse.status()); assertEquals("index", indexResponse.getIndex()); assertEquals("type", indexResponse.getType()); assertEquals("with_create_op_type", indexResponse.getId()); ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, () -> { - execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync); + execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync, + highLevelClient()::index, highLevelClient()::indexAsync); }); assertEquals(RestStatus.CONFLICT, exception.status()); @@ -408,7 +439,8 @@ public void testUpdate() throws IOException { updateRequest.doc(singletonMap("field", "value"), randomFrom(XContentType.values())); ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, () -> - execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync)); + execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync, + highLevelClient()::update, highLevelClient()::updateAsync)); assertEquals(RestStatus.NOT_FOUND, exception.status()); assertEquals("Elasticsearch exception [type=document_missing_exception, reason=[type][does_not_exist]: document missing]", exception.getMessage()); @@ -416,7 +448,7 @@ public void testUpdate() throws IOException { { IndexRequest indexRequest = new IndexRequest("index", "type", "id"); indexRequest.source(singletonMap("field", "value")); - IndexResponse indexResponse = highLevelClient().index(indexRequest); + IndexResponse indexResponse = highLevelClient().index(indexRequest, RequestOptions.DEFAULT); assertEquals(RestStatus.CREATED, indexResponse.status()); UpdateRequest updateRequest = new UpdateRequest("index", "type", "id"); @@ -431,7 +463,8 @@ public void testUpdate() throws IOException { updateRequestConflict.version(indexResponse.getVersion()); ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, () -> - execute(updateRequestConflict, highLevelClient()::update, highLevelClient()::updateAsync)); + execute(updateRequestConflict, highLevelClient()::update, highLevelClient()::updateAsync, + highLevelClient()::update, highLevelClient()::updateAsync)); assertEquals(RestStatus.CONFLICT, exception.status()); assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][id]: version conflict, " + "current version [2] is different than the one provided [1]]", exception.getMessage()); @@ -439,7 +472,7 @@ public void testUpdate() throws IOException { { IndexRequest indexRequest = new IndexRequest("index", "type", "with_script"); indexRequest.source(singletonMap("counter", 12)); - IndexResponse indexResponse = highLevelClient().index(indexRequest); + IndexResponse indexResponse = highLevelClient().index(indexRequest, RequestOptions.DEFAULT); assertEquals(RestStatus.CREATED, indexResponse.status()); UpdateRequest updateRequest = new UpdateRequest("index", "type", "with_script"); @@ -447,7 +480,8 @@ public void testUpdate() throws IOException { updateRequest.script(script); updateRequest.fetchSource(true); - UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync); + UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync, + highLevelClient()::update, highLevelClient()::updateAsync); assertEquals(RestStatus.OK, updateResponse.status()); assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult()); assertEquals(2L, updateResponse.getVersion()); @@ -459,7 +493,7 @@ public void testUpdate() throws IOException { indexRequest.source("field_1", "one", "field_3", "three"); indexRequest.version(12L); indexRequest.versionType(VersionType.EXTERNAL); - IndexResponse indexResponse = highLevelClient().index(indexRequest); + IndexResponse indexResponse = highLevelClient().index(indexRequest, RequestOptions.DEFAULT); assertEquals(RestStatus.CREATED, indexResponse.status()); assertEquals(12L, indexResponse.getVersion()); @@ -467,7 +501,8 @@ public void testUpdate() throws IOException { updateRequest.doc(singletonMap("field_2", "two"), randomFrom(XContentType.values())); updateRequest.fetchSource("field_*", "field_3"); - UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync); + UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync, + highLevelClient()::update, highLevelClient()::updateAsync); assertEquals(RestStatus.OK, updateResponse.status()); assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult()); assertEquals(13L, updateResponse.getVersion()); @@ -481,14 +516,15 @@ public void testUpdate() throws IOException { { IndexRequest indexRequest = new IndexRequest("index", "type", "noop"); indexRequest.source("field", "value"); - IndexResponse indexResponse = highLevelClient().index(indexRequest); + IndexResponse indexResponse = highLevelClient().index(indexRequest, RequestOptions.DEFAULT); assertEquals(RestStatus.CREATED, indexResponse.status()); assertEquals(1L, indexResponse.getVersion()); UpdateRequest updateRequest = new UpdateRequest("index", "type", "noop"); updateRequest.doc(singletonMap("field", "value"), randomFrom(XContentType.values())); - UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync); + UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync, + highLevelClient()::update, highLevelClient()::updateAsync); assertEquals(RestStatus.OK, updateResponse.status()); assertEquals(DocWriteResponse.Result.NOOP, updateResponse.getResult()); assertEquals(1L, updateResponse.getVersion()); @@ -506,7 +542,8 @@ public void testUpdate() throws IOException { updateRequest.doc(singletonMap("doc_status", "updated")); updateRequest.fetchSource(true); - UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync); + UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync, + highLevelClient()::update, highLevelClient()::updateAsync); assertEquals(RestStatus.CREATED, updateResponse.status()); assertEquals("index", updateResponse.getIndex()); assertEquals("type", updateResponse.getType()); @@ -521,7 +558,8 @@ public void testUpdate() throws IOException { updateRequest.fetchSource(true); updateRequest.docAsUpsert(true); - UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync); + UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync, + highLevelClient()::update, highLevelClient()::updateAsync); assertEquals(RestStatus.CREATED, updateResponse.status()); assertEquals("index", updateResponse.getIndex()); assertEquals("type", updateResponse.getType()); @@ -537,7 +575,8 @@ public void testUpdate() throws IOException { updateRequest.scriptedUpsert(true); updateRequest.upsert(singletonMap("level", "A")); - UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync); + UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync, + highLevelClient()::update, highLevelClient()::updateAsync); assertEquals(RestStatus.CREATED, updateResponse.status()); assertEquals("index", updateResponse.getIndex()); assertEquals("type", updateResponse.getType()); @@ -552,7 +591,8 @@ public void testUpdate() throws IOException { UpdateRequest updateRequest = new UpdateRequest("index", "type", "id"); updateRequest.doc(new IndexRequest().source(Collections.singletonMap("field", "doc"), XContentType.JSON)); updateRequest.upsert(new IndexRequest().source(Collections.singletonMap("field", "upsert"), XContentType.YAML)); - execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync); + execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync, + highLevelClient()::update, highLevelClient()::updateAsync); }); assertEquals("Update request cannot have different content types for doc [JSON] and upsert [YAML] documents", exception.getMessage()); @@ -575,7 +615,8 @@ public void testBulk() throws IOException { if (opType == DocWriteRequest.OpType.DELETE) { if (erroneous == false) { assertEquals(RestStatus.CREATED, - highLevelClient().index(new IndexRequest("index", "test", id).source("field", -1)).status()); + highLevelClient().index( + new IndexRequest("index", "test", id).source("field", -1), RequestOptions.DEFAULT).status()); } DeleteRequest deleteRequest = new DeleteRequest("index", "test", id); bulkRequest.add(deleteRequest); @@ -593,7 +634,7 @@ public void testBulk() throws IOException { } else if (opType == DocWriteRequest.OpType.CREATE) { IndexRequest createRequest = new IndexRequest("index", "test", id).source(source, xContentType).create(true); if (erroneous) { - assertEquals(RestStatus.CREATED, highLevelClient().index(createRequest).status()); + assertEquals(RestStatus.CREATED, highLevelClient().index(createRequest, RequestOptions.DEFAULT).status()); } bulkRequest.add(createRequest); @@ -602,14 +643,16 @@ public void testBulk() throws IOException { .doc(new IndexRequest().source(source, xContentType)); if (erroneous == false) { assertEquals(RestStatus.CREATED, - highLevelClient().index(new IndexRequest("index", "test", id).source("field", -1)).status()); + highLevelClient().index( + new IndexRequest("index", "test", id).source("field", -1), RequestOptions.DEFAULT).status()); } bulkRequest.add(updateRequest); } } } - BulkResponse bulkResponse = execute(bulkRequest, highLevelClient()::bulk, highLevelClient()::bulkAsync); + BulkResponse bulkResponse = execute(bulkRequest, highLevelClient()::bulk, highLevelClient()::bulkAsync, + highLevelClient()::bulk, highLevelClient()::bulkAsync); assertEquals(RestStatus.OK, bulkResponse.status()); assertTrue(bulkResponse.getTook().getMillis() > 0); assertEquals(nbItems, bulkResponse.getItems().length); @@ -662,7 +705,8 @@ public void afterBulk(long executionId, BulkRequest request, Throwable failure) if (opType == DocWriteRequest.OpType.DELETE) { if (erroneous == false) { assertEquals(RestStatus.CREATED, - highLevelClient().index(new IndexRequest("index", "test", id).source("field", -1)).status()); + highLevelClient().index( + new IndexRequest("index", "test", id).source("field", -1), RequestOptions.DEFAULT).status()); } DeleteRequest deleteRequest = new DeleteRequest("index", "test", id); processor.add(deleteRequest); @@ -678,7 +722,7 @@ public void afterBulk(long executionId, BulkRequest request, Throwable failure) } else if (opType == DocWriteRequest.OpType.CREATE) { IndexRequest createRequest = new IndexRequest("index", "test", id).source(xContentType, "id", i).create(true); if (erroneous) { - assertEquals(RestStatus.CREATED, highLevelClient().index(createRequest).status()); + assertEquals(RestStatus.CREATED, highLevelClient().index(createRequest, RequestOptions.DEFAULT).status()); } processor.add(createRequest); @@ -687,7 +731,8 @@ public void afterBulk(long executionId, BulkRequest request, Throwable failure) .doc(new IndexRequest().source(xContentType, "id", i)); if (erroneous == false) { assertEquals(RestStatus.CREATED, - highLevelClient().index(new IndexRequest("index", "test", id).source("field", -1)).status()); + highLevelClient().index( + new IndexRequest("index", "test", id).source("field", -1), RequestOptions.DEFAULT).status()); } processor.add(updateRequest); } @@ -739,14 +784,14 @@ public void testUrlEncode() throws IOException { { IndexRequest indexRequest = new IndexRequest(indexPattern, "type", "id#1"); indexRequest.source("field", "value"); - IndexResponse indexResponse = highLevelClient().index(indexRequest); + IndexResponse indexResponse = highLevelClient().index(indexRequest, RequestOptions.DEFAULT); assertEquals(expectedIndex, indexResponse.getIndex()); assertEquals("type", indexResponse.getType()); assertEquals("id#1", indexResponse.getId()); } { GetRequest getRequest = new GetRequest(indexPattern, "type", "id#1"); - GetResponse getResponse = highLevelClient().get(getRequest); + GetResponse getResponse = highLevelClient().get(getRequest, RequestOptions.DEFAULT); assertTrue(getResponse.isExists()); assertEquals(expectedIndex, getResponse.getIndex()); assertEquals("type", getResponse.getType()); @@ -757,21 +802,21 @@ public void testUrlEncode() throws IOException { { IndexRequest indexRequest = new IndexRequest("index", "type", docId); indexRequest.source("field", "value"); - IndexResponse indexResponse = highLevelClient().index(indexRequest); + IndexResponse indexResponse = highLevelClient().index(indexRequest, RequestOptions.DEFAULT); assertEquals("index", indexResponse.getIndex()); assertEquals("type", indexResponse.getType()); assertEquals(docId, indexResponse.getId()); } { GetRequest getRequest = new GetRequest("index", "type", docId); - GetResponse getResponse = highLevelClient().get(getRequest); + GetResponse getResponse = highLevelClient().get(getRequest, RequestOptions.DEFAULT); assertTrue(getResponse.isExists()); assertEquals("index", getResponse.getIndex()); assertEquals("type", getResponse.getType()); assertEquals(docId, getResponse.getId()); } - assertTrue(highLevelClient().indices().exists(new GetIndexRequest().indices(indexPattern, "index"))); + assertTrue(highLevelClient().indices().exists(new GetIndexRequest().indices(indexPattern, "index"), RequestOptions.DEFAULT)); } public void testParamsEncode() throws IOException { @@ -781,14 +826,14 @@ public void testParamsEncode() throws IOException { IndexRequest indexRequest = new IndexRequest("index", "type", "id"); indexRequest.source("field", "value"); indexRequest.routing(routing); - IndexResponse indexResponse = highLevelClient().index(indexRequest); + IndexResponse indexResponse = highLevelClient().index(indexRequest, RequestOptions.DEFAULT); assertEquals("index", indexResponse.getIndex()); assertEquals("type", indexResponse.getType()); assertEquals("id", indexResponse.getId()); } { GetRequest getRequest = new GetRequest("index", "type", "id").routing(routing); - GetResponse getResponse = highLevelClient().get(getRequest); + GetResponse getResponse = highLevelClient().get(getRequest, RequestOptions.DEFAULT); assertTrue(getResponse.isExists()); assertEquals("index", getResponse.getIndex()); assertEquals("type", getResponse.getType()); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java index f7a934405c2ae..14fe0e01d31f9 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java @@ -60,23 +60,60 @@ protected static RestHighLevelClient highLevelClient() { * Executes the provided request using either the sync method or its async variant, both provided as functions */ protected static Resp execute(Req request, SyncMethod syncMethod, - AsyncMethod asyncMethod, Header... headers) throws IOException { + AsyncMethod asyncMethod) throws IOException { if (randomBoolean()) { - return syncMethod.execute(request, headers); + return syncMethod.execute(request, RequestOptions.DEFAULT); } else { PlainActionFuture future = PlainActionFuture.newFuture(); - asyncMethod.execute(request, future, headers); + asyncMethod.execute(request, RequestOptions.DEFAULT, future); return future.actionGet(); } } @FunctionalInterface protected interface SyncMethod { - Response execute(Request request, Header... headers) throws IOException; + Response execute(Request request, RequestOptions options) throws IOException; } @FunctionalInterface protected interface AsyncMethod { + void execute(Request request, RequestOptions options, ActionListener listener); + } + + /** + * Executes the provided request using either the sync method or its async variant, both provided as functions + */ + @Deprecated + protected static Resp execute(Req request, SyncMethod syncMethod, AsyncMethod asyncMethod, + SyncMethodWithHeaders syncMethodWithHeaders, + AsyncMethodWithHeaders asyncMethodWithHeaders) throws IOException { + switch(randomIntBetween(0, 3)) { + case 0: + return syncMethod.execute(request, RequestOptions.DEFAULT); + case 1: + PlainActionFuture future = PlainActionFuture.newFuture(); + asyncMethod.execute(request, RequestOptions.DEFAULT, future); + return future.actionGet(); + case 2: + return syncMethodWithHeaders.execute(request); + case 3: + PlainActionFuture futureWithHeaders = PlainActionFuture.newFuture(); + asyncMethodWithHeaders.execute(request, futureWithHeaders); + return futureWithHeaders.actionGet(); + default: + throw new UnsupportedOperationException(); + } + } + + @Deprecated + @FunctionalInterface + protected interface SyncMethodWithHeaders { + Response execute(Request request, Header... headers) throws IOException; + } + + @Deprecated + @FunctionalInterface + protected interface AsyncMethodWithHeaders { void execute(Request request, ActionListener listener, Header... headers); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java index 55357e06ab299..986c3380ff3c8 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java @@ -110,6 +110,8 @@ public void testIndicesExists() throws IOException { boolean response = execute( request, highLevelClient().indices()::exists, + highLevelClient().indices()::existsAsync, + highLevelClient().indices()::exists, highLevelClient().indices()::existsAsync ); assertTrue(response); @@ -125,6 +127,8 @@ public void testIndicesExists() throws IOException { boolean response = execute( request, highLevelClient().indices()::exists, + highLevelClient().indices()::existsAsync, + highLevelClient().indices()::exists, highLevelClient().indices()::existsAsync ); assertFalse(response); @@ -143,6 +147,8 @@ public void testIndicesExists() throws IOException { boolean response = execute( request, highLevelClient().indices()::exists, + highLevelClient().indices()::existsAsync, + highLevelClient().indices()::exists, highLevelClient().indices()::existsAsync ); assertFalse(response); @@ -160,7 +166,8 @@ public void testCreateIndex() throws IOException { CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName); CreateIndexResponse createIndexResponse = - execute(createIndexRequest, highLevelClient().indices()::create, highLevelClient().indices()::createAsync); + execute(createIndexRequest, highLevelClient().indices()::create, highLevelClient().indices()::createAsync, + highLevelClient().indices()::create, highLevelClient().indices()::createAsync); assertTrue(createIndexResponse.isAcknowledged()); assertTrue(indexExists(indexName)); @@ -188,7 +195,8 @@ public void testCreateIndex() throws IOException { createIndexRequest.mapping("type_name", mappingBuilder); CreateIndexResponse createIndexResponse = - execute(createIndexRequest, highLevelClient().indices()::create, highLevelClient().indices()::createAsync); + execute(createIndexRequest, highLevelClient().indices()::create, highLevelClient().indices()::createAsync, + highLevelClient().indices()::create, highLevelClient().indices()::createAsync); assertTrue(createIndexResponse.isAcknowledged()); Map getIndexResponse = getAsMap(indexName); @@ -323,7 +331,8 @@ public void testPutMapping() throws IOException { putMappingRequest.source(mappingBuilder); PutMappingResponse putMappingResponse = - execute(putMappingRequest, highLevelClient().indices()::putMapping, highLevelClient().indices()::putMappingAsync); + execute(putMappingRequest, highLevelClient().indices()::putMapping, highLevelClient().indices()::putMappingAsync, + highLevelClient().indices()::putMapping, highLevelClient().indices()::putMappingAsync); assertTrue(putMappingResponse.isAcknowledged()); Map getIndexResponse = getAsMap(indexName); @@ -375,7 +384,8 @@ public void testDeleteIndex() throws IOException { DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(indexName); DeleteIndexResponse deleteIndexResponse = - execute(deleteIndexRequest, highLevelClient().indices()::delete, highLevelClient().indices()::deleteAsync); + execute(deleteIndexRequest, highLevelClient().indices()::delete, highLevelClient().indices()::deleteAsync, + highLevelClient().indices()::delete, highLevelClient().indices()::deleteAsync); assertTrue(deleteIndexResponse.isAcknowledged()); assertFalse(indexExists(indexName)); @@ -388,7 +398,8 @@ public void testDeleteIndex() throws IOException { DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(nonExistentIndex); ElasticsearchException exception = expectThrows(ElasticsearchException.class, - () -> execute(deleteIndexRequest, highLevelClient().indices()::delete, highLevelClient().indices()::deleteAsync)); + () -> execute(deleteIndexRequest, highLevelClient().indices()::delete, highLevelClient().indices()::deleteAsync, + highLevelClient().indices()::delete, highLevelClient().indices()::deleteAsync)); assertEquals(RestStatus.NOT_FOUND, exception.status()); } } @@ -407,6 +418,7 @@ public void testUpdateAliases() throws IOException { addAction.routing("routing").searchRouting("search_routing").filter("{\"term\":{\"year\":2016}}"); aliasesAddRequest.addAliasAction(addAction); IndicesAliasesResponse aliasesAddResponse = execute(aliasesAddRequest, highLevelClient().indices()::updateAliases, + highLevelClient().indices()::updateAliasesAsync, highLevelClient().indices()::updateAliases, highLevelClient().indices()::updateAliasesAsync); assertTrue(aliasesAddResponse.isAcknowledged()); assertThat(aliasExists(alias), equalTo(true)); @@ -425,6 +437,7 @@ public void testUpdateAliases() throws IOException { AliasActions removeAction = new AliasActions(AliasActions.Type.REMOVE).index(index).alias(alias); aliasesAddRemoveRequest.addAliasAction(removeAction); IndicesAliasesResponse aliasesAddRemoveResponse = execute(aliasesAddRemoveRequest, highLevelClient().indices()::updateAliases, + highLevelClient().indices()::updateAliasesAsync, highLevelClient().indices()::updateAliases, highLevelClient().indices()::updateAliasesAsync); assertTrue(aliasesAddRemoveResponse.isAcknowledged()); assertThat(aliasExists(alias), equalTo(false)); @@ -436,6 +449,7 @@ public void testUpdateAliases() throws IOException { AliasActions removeIndexAction = new AliasActions(AliasActions.Type.REMOVE_INDEX).index(index); aliasesRemoveIndexRequest.addAliasAction(removeIndexAction); IndicesAliasesResponse aliasesRemoveIndexResponse = execute(aliasesRemoveIndexRequest, highLevelClient().indices()::updateAliases, + highLevelClient().indices()::updateAliasesAsync, highLevelClient().indices()::updateAliases, highLevelClient().indices()::updateAliasesAsync); assertTrue(aliasesRemoveIndexResponse.isAcknowledged()); assertThat(aliasExists(alias), equalTo(false)); @@ -453,7 +467,9 @@ public void testAliasesNonExistentIndex() throws IOException { IndicesAliasesRequest nonExistentIndexRequest = new IndicesAliasesRequest(); nonExistentIndexRequest.addAliasAction(new AliasActions(AliasActions.Type.ADD).index(nonExistentIndex).alias(alias)); ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> execute(nonExistentIndexRequest, - highLevelClient().indices()::updateAliases, highLevelClient().indices()::updateAliasesAsync)); + highLevelClient().indices()::updateAliases, highLevelClient().indices()::updateAliasesAsync, + highLevelClient().indices()::updateAliases, + highLevelClient().indices()::updateAliasesAsync)); assertThat(exception.status(), equalTo(RestStatus.NOT_FOUND)); assertThat(exception.getMessage(), equalTo("Elasticsearch exception [type=index_not_found_exception, reason=no such index]")); assertThat(exception.getMetadata("es.index"), hasItem(nonExistentIndex)); @@ -463,7 +479,8 @@ public void testAliasesNonExistentIndex() throws IOException { mixedRequest.addAliasAction(new AliasActions(AliasActions.Type.ADD).indices(index).aliases(alias)); mixedRequest.addAliasAction(new AliasActions(AliasActions.Type.REMOVE).indices(nonExistentIndex).alias(alias)); exception = expectThrows(ElasticsearchStatusException.class, - () -> execute(mixedRequest, highLevelClient().indices()::updateAliases, highLevelClient().indices()::updateAliasesAsync)); + () -> execute(mixedRequest, highLevelClient().indices()::updateAliases, highLevelClient().indices()::updateAliasesAsync, + highLevelClient().indices()::updateAliases, highLevelClient().indices()::updateAliasesAsync)); assertThat(exception.status(), equalTo(RestStatus.NOT_FOUND)); assertThat(exception.getMessage(), equalTo("Elasticsearch exception [type=index_not_found_exception, reason=no such index]")); assertThat(exception.getMetadata("es.index"), hasItem(nonExistentIndex)); @@ -475,6 +492,7 @@ public void testAliasesNonExistentIndex() throws IOException { removeIndexRequest.addAliasAction(new AliasActions(AliasActions.Type.ADD).index(nonExistentIndex).alias(alias)); removeIndexRequest.addAliasAction(new AliasActions(AliasActions.Type.REMOVE_INDEX).indices(nonExistentIndex)); exception = expectThrows(ElasticsearchException.class, () -> execute(removeIndexRequest, highLevelClient().indices()::updateAliases, + highLevelClient().indices()::updateAliasesAsync, highLevelClient().indices()::updateAliases, highLevelClient().indices()::updateAliasesAsync)); assertThat(exception.status(), equalTo(RestStatus.NOT_FOUND)); assertThat(exception.getMessage(), equalTo("Elasticsearch exception [type=index_not_found_exception, reason=no such index]")); @@ -495,6 +513,7 @@ public void testOpenExistingIndex() throws IOException { OpenIndexRequest openIndexRequest = new OpenIndexRequest(index); OpenIndexResponse openIndexResponse = execute(openIndexRequest, highLevelClient().indices()::open, + highLevelClient().indices()::openAsync, highLevelClient().indices()::open, highLevelClient().indices()::openAsync); assertTrue(openIndexResponse.isAcknowledged()); @@ -508,19 +527,22 @@ public void testOpenNonExistentIndex() throws IOException { OpenIndexRequest openIndexRequest = new OpenIndexRequest(nonExistentIndex); ElasticsearchException exception = expectThrows(ElasticsearchException.class, - () -> execute(openIndexRequest, highLevelClient().indices()::open, highLevelClient().indices()::openAsync)); + () -> execute(openIndexRequest, highLevelClient().indices()::open, highLevelClient().indices()::openAsync, + highLevelClient().indices()::open, highLevelClient().indices()::openAsync)); assertEquals(RestStatus.NOT_FOUND, exception.status()); OpenIndexRequest lenientOpenIndexRequest = new OpenIndexRequest(nonExistentIndex); lenientOpenIndexRequest.indicesOptions(IndicesOptions.lenientExpandOpen()); OpenIndexResponse lenientOpenIndexResponse = execute(lenientOpenIndexRequest, highLevelClient().indices()::open, + highLevelClient().indices()::openAsync, highLevelClient().indices()::open, highLevelClient().indices()::openAsync); assertThat(lenientOpenIndexResponse.isAcknowledged(), equalTo(true)); OpenIndexRequest strictOpenIndexRequest = new OpenIndexRequest(nonExistentIndex); strictOpenIndexRequest.indicesOptions(IndicesOptions.strictExpandOpen()); ElasticsearchException strictException = expectThrows(ElasticsearchException.class, - () -> execute(openIndexRequest, highLevelClient().indices()::open, highLevelClient().indices()::openAsync)); + () -> execute(openIndexRequest, highLevelClient().indices()::open, highLevelClient().indices()::openAsync, + highLevelClient().indices()::open, highLevelClient().indices()::openAsync)); assertEquals(RestStatus.NOT_FOUND, strictException.status()); } @@ -532,6 +554,7 @@ public void testCloseExistingIndex() throws IOException { CloseIndexRequest closeIndexRequest = new CloseIndexRequest(index); CloseIndexResponse closeIndexResponse = execute(closeIndexRequest, highLevelClient().indices()::close, + highLevelClient().indices()::closeAsync, highLevelClient().indices()::close, highLevelClient().indices()::closeAsync); assertTrue(closeIndexResponse.isAcknowledged()); @@ -547,7 +570,8 @@ public void testCloseNonExistentIndex() throws IOException { CloseIndexRequest closeIndexRequest = new CloseIndexRequest(nonExistentIndex); ElasticsearchException exception = expectThrows(ElasticsearchException.class, - () -> execute(closeIndexRequest, highLevelClient().indices()::close, highLevelClient().indices()::closeAsync)); + () -> execute(closeIndexRequest, highLevelClient().indices()::close, highLevelClient().indices()::closeAsync, + highLevelClient().indices()::close, highLevelClient().indices()::closeAsync)); assertEquals(RestStatus.NOT_FOUND, exception.status()); } @@ -561,7 +585,8 @@ public void testRefresh() throws IOException { createIndex(index, settings); RefreshRequest refreshRequest = new RefreshRequest(index); RefreshResponse refreshResponse = - execute(refreshRequest, highLevelClient().indices()::refresh, highLevelClient().indices()::refreshAsync); + execute(refreshRequest, highLevelClient().indices()::refresh, highLevelClient().indices()::refreshAsync, + highLevelClient().indices()::refresh, highLevelClient().indices()::refreshAsync); assertThat(refreshResponse.getTotalShards(), equalTo(1)); assertThat(refreshResponse.getSuccessfulShards(), equalTo(1)); assertThat(refreshResponse.getFailedShards(), equalTo(0)); @@ -572,7 +597,8 @@ public void testRefresh() throws IOException { assertFalse(indexExists(nonExistentIndex)); RefreshRequest refreshRequest = new RefreshRequest(nonExistentIndex); ElasticsearchException exception = expectThrows(ElasticsearchException.class, - () -> execute(refreshRequest, highLevelClient().indices()::refresh, highLevelClient().indices()::refreshAsync)); + () -> execute(refreshRequest, highLevelClient().indices()::refresh, highLevelClient().indices()::refreshAsync, + highLevelClient().indices()::refresh, highLevelClient().indices()::refreshAsync)); assertEquals(RestStatus.NOT_FOUND, exception.status()); } } @@ -587,7 +613,8 @@ public void testFlush() throws IOException { createIndex(index, settings); FlushRequest flushRequest = new FlushRequest(index); FlushResponse flushResponse = - execute(flushRequest, highLevelClient().indices()::flush, highLevelClient().indices()::flushAsync); + execute(flushRequest, highLevelClient().indices()::flush, highLevelClient().indices()::flushAsync, + highLevelClient().indices()::flush, highLevelClient().indices()::flushAsync); assertThat(flushResponse.getTotalShards(), equalTo(1)); assertThat(flushResponse.getSuccessfulShards(), equalTo(1)); assertThat(flushResponse.getFailedShards(), equalTo(0)); @@ -598,7 +625,8 @@ public void testFlush() throws IOException { assertFalse(indexExists(nonExistentIndex)); FlushRequest flushRequest = new FlushRequest(nonExistentIndex); ElasticsearchException exception = expectThrows(ElasticsearchException.class, - () -> execute(flushRequest, highLevelClient().indices()::flush, highLevelClient().indices()::flushAsync)); + () -> execute(flushRequest, highLevelClient().indices()::flush, highLevelClient().indices()::flushAsync, + highLevelClient().indices()::flush, highLevelClient().indices()::flushAsync)); assertEquals(RestStatus.NOT_FOUND, exception.status()); } } @@ -646,7 +674,8 @@ public void testClearCache() throws IOException { createIndex(index, settings); ClearIndicesCacheRequest clearCacheRequest = new ClearIndicesCacheRequest(index); ClearIndicesCacheResponse clearCacheResponse = - execute(clearCacheRequest, highLevelClient().indices()::clearCache, highLevelClient().indices()::clearCacheAsync); + execute(clearCacheRequest, highLevelClient().indices()::clearCache, highLevelClient().indices()::clearCacheAsync, + highLevelClient().indices()::clearCache, highLevelClient().indices()::clearCacheAsync); assertThat(clearCacheResponse.getTotalShards(), equalTo(1)); assertThat(clearCacheResponse.getSuccessfulShards(), equalTo(1)); assertThat(clearCacheResponse.getFailedShards(), equalTo(0)); @@ -657,8 +686,8 @@ public void testClearCache() throws IOException { assertFalse(indexExists(nonExistentIndex)); ClearIndicesCacheRequest clearCacheRequest = new ClearIndicesCacheRequest(nonExistentIndex); ElasticsearchException exception = expectThrows(ElasticsearchException.class, - () -> execute(clearCacheRequest, highLevelClient().indices()::clearCache, - highLevelClient().indices()::clearCacheAsync)); + () -> execute(clearCacheRequest, highLevelClient().indices()::clearCache, highLevelClient().indices()::clearCacheAsync, + highLevelClient().indices()::clearCache, highLevelClient().indices()::clearCacheAsync)); assertEquals(RestStatus.NOT_FOUND, exception.status()); } } @@ -673,7 +702,8 @@ public void testForceMerge() throws IOException { createIndex(index, settings); ForceMergeRequest forceMergeRequest = new ForceMergeRequest(index); ForceMergeResponse forceMergeResponse = - execute(forceMergeRequest, highLevelClient().indices()::forceMerge, highLevelClient().indices()::forceMergeAsync); + execute(forceMergeRequest, highLevelClient().indices()::forceMerge, highLevelClient().indices()::forceMergeAsync, + highLevelClient().indices()::forceMerge, highLevelClient().indices()::forceMergeAsync); assertThat(forceMergeResponse.getTotalShards(), equalTo(1)); assertThat(forceMergeResponse.getSuccessfulShards(), equalTo(1)); assertThat(forceMergeResponse.getFailedShards(), equalTo(0)); @@ -684,25 +714,30 @@ public void testForceMerge() throws IOException { assertFalse(indexExists(nonExistentIndex)); ForceMergeRequest forceMergeRequest = new ForceMergeRequest(nonExistentIndex); ElasticsearchException exception = expectThrows(ElasticsearchException.class, - () -> execute(forceMergeRequest, highLevelClient().indices()::forceMerge, highLevelClient().indices()::forceMergeAsync)); + () -> execute(forceMergeRequest, highLevelClient().indices()::forceMerge, highLevelClient().indices()::forceMergeAsync, + highLevelClient().indices()::forceMerge, highLevelClient().indices()::forceMergeAsync)); assertEquals(RestStatus.NOT_FOUND, exception.status()); } } public void testExistsAlias() throws IOException { GetAliasesRequest getAliasesRequest = new GetAliasesRequest("alias"); - assertFalse(execute(getAliasesRequest, highLevelClient().indices()::existsAlias, highLevelClient().indices()::existsAliasAsync)); + assertFalse(execute(getAliasesRequest, highLevelClient().indices()::existsAlias, highLevelClient().indices()::existsAliasAsync, + highLevelClient().indices()::existsAlias, highLevelClient().indices()::existsAliasAsync)); createIndex("index", Settings.EMPTY); client().performRequest(HttpPut.METHOD_NAME, "/index/_alias/alias"); - assertTrue(execute(getAliasesRequest, highLevelClient().indices()::existsAlias, highLevelClient().indices()::existsAliasAsync)); + assertTrue(execute(getAliasesRequest, highLevelClient().indices()::existsAlias, highLevelClient().indices()::existsAliasAsync, + highLevelClient().indices()::existsAlias, highLevelClient().indices()::existsAliasAsync)); GetAliasesRequest getAliasesRequest2 = new GetAliasesRequest(); getAliasesRequest2.aliases("alias"); getAliasesRequest2.indices("index"); - assertTrue(execute(getAliasesRequest2, highLevelClient().indices()::existsAlias, highLevelClient().indices()::existsAliasAsync)); + assertTrue(execute(getAliasesRequest2, highLevelClient().indices()::existsAlias, highLevelClient().indices()::existsAliasAsync, + highLevelClient().indices()::existsAlias, highLevelClient().indices()::existsAliasAsync)); getAliasesRequest2.indices("does_not_exist"); - assertFalse(execute(getAliasesRequest2, highLevelClient().indices()::existsAlias, highLevelClient().indices()::existsAliasAsync)); + assertFalse(execute(getAliasesRequest2, highLevelClient().indices()::existsAlias, highLevelClient().indices()::existsAliasAsync, + highLevelClient().indices()::existsAlias, highLevelClient().indices()::existsAliasAsync)); } @SuppressWarnings("unchecked") @@ -722,7 +757,8 @@ public void testShrink() throws IOException { .putNull("index.routing.allocation.require._name") .build(); resizeRequest.setTargetIndex(new CreateIndexRequest("target").settings(targetSettings).alias(new Alias("alias"))); - ResizeResponse resizeResponse = highLevelClient().indices().shrink(resizeRequest); + ResizeResponse resizeResponse = execute(resizeRequest, highLevelClient().indices()::shrink, + highLevelClient().indices()::shrinkAsync, highLevelClient().indices()::shrink, highLevelClient().indices()::shrinkAsync); assertTrue(resizeResponse.isAcknowledged()); assertTrue(resizeResponse.isShardsAcknowledged()); Map getIndexResponse = getAsMap("target"); @@ -744,7 +780,8 @@ public void testSplit() throws IOException { resizeRequest.setResizeType(ResizeType.SPLIT); Settings targetSettings = Settings.builder().put("index.number_of_shards", 4).put("index.number_of_replicas", 0).build(); resizeRequest.setTargetIndex(new CreateIndexRequest("target").settings(targetSettings).alias(new Alias("alias"))); - ResizeResponse resizeResponse = highLevelClient().indices().split(resizeRequest); + ResizeResponse resizeResponse = execute(resizeRequest, highLevelClient().indices()::split, highLevelClient().indices()::splitAsync, + highLevelClient().indices()::split, highLevelClient().indices()::splitAsync); assertTrue(resizeResponse.isAcknowledged()); assertTrue(resizeResponse.isShardsAcknowledged()); Map getIndexResponse = getAsMap("target"); @@ -757,12 +794,13 @@ public void testSplit() throws IOException { } public void testRollover() throws IOException { - highLevelClient().indices().create(new CreateIndexRequest("test").alias(new Alias("alias"))); + highLevelClient().indices().create(new CreateIndexRequest("test").alias(new Alias("alias")), RequestOptions.DEFAULT); RolloverRequest rolloverRequest = new RolloverRequest("alias", "test_new"); rolloverRequest.addMaxIndexDocsCondition(1); { RolloverResponse rolloverResponse = execute(rolloverRequest, highLevelClient().indices()::rollover, + highLevelClient().indices()::rolloverAsync, highLevelClient().indices()::rollover, highLevelClient().indices()::rolloverAsync); assertFalse(rolloverResponse.isRolledOver()); assertFalse(rolloverResponse.isDryRun()); @@ -773,15 +811,16 @@ public void testRollover() throws IOException { assertEquals("test_new", rolloverResponse.getNewIndex()); } - highLevelClient().index(new IndexRequest("test", "type", "1").source("field", "value")); + highLevelClient().index(new IndexRequest("test", "type", "1").source("field", "value"), RequestOptions.DEFAULT); highLevelClient().index(new IndexRequest("test", "type", "2").source("field", "value") - .setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL)); + .setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL), RequestOptions.DEFAULT); //without the refresh the rollover may not happen as the number of docs seen may be off { rolloverRequest.addMaxIndexAgeCondition(new TimeValue(1)); rolloverRequest.dryRun(true); RolloverResponse rolloverResponse = execute(rolloverRequest, highLevelClient().indices()::rollover, + highLevelClient().indices()::rolloverAsync, highLevelClient().indices()::rollover, highLevelClient().indices()::rolloverAsync); assertFalse(rolloverResponse.isRolledOver()); assertTrue(rolloverResponse.isDryRun()); @@ -796,6 +835,7 @@ public void testRollover() throws IOException { rolloverRequest.dryRun(false); rolloverRequest.addMaxIndexSizeCondition(new ByteSizeValue(1, ByteSizeUnit.MB)); RolloverResponse rolloverResponse = execute(rolloverRequest, highLevelClient().indices()::rollover, + highLevelClient().indices()::rolloverAsync, highLevelClient().indices()::rollover, highLevelClient().indices()::rolloverAsync); assertTrue(rolloverResponse.isRolledOver()); assertFalse(rolloverResponse.isDryRun()); @@ -830,6 +870,7 @@ public void testIndexPutSettings() throws IOException { UpdateSettingsRequest dynamicSettingRequest = new UpdateSettingsRequest(); dynamicSettingRequest.settings(Settings.builder().put(dynamicSettingKey, dynamicSettingValue).build()); UpdateSettingsResponse response = execute(dynamicSettingRequest, highLevelClient().indices()::putSettings, + highLevelClient().indices()::putSettingsAsync, highLevelClient().indices()::putSettings, highLevelClient().indices()::putSettingsAsync); assertTrue(response.isAcknowledged()); @@ -840,6 +881,7 @@ public void testIndexPutSettings() throws IOException { UpdateSettingsRequest staticSettingRequest = new UpdateSettingsRequest(); staticSettingRequest.settings(Settings.builder().put(staticSettingKey, staticSettingValue).build()); ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> execute(staticSettingRequest, + highLevelClient().indices()::putSettings, highLevelClient().indices()::putSettingsAsync, highLevelClient().indices()::putSettings, highLevelClient().indices()::putSettingsAsync)); assertThat(exception.getMessage(), startsWith("Elasticsearch exception [type=illegal_argument_exception, " @@ -850,6 +892,7 @@ public void testIndexPutSettings() throws IOException { closeIndex(index); response = execute(staticSettingRequest, highLevelClient().indices()::putSettings, + highLevelClient().indices()::putSettingsAsync, highLevelClient().indices()::putSettings, highLevelClient().indices()::putSettingsAsync); assertTrue(response.isAcknowledged()); openIndex(index); @@ -860,6 +903,7 @@ public void testIndexPutSettings() throws IOException { UpdateSettingsRequest unmodifiableSettingRequest = new UpdateSettingsRequest(); unmodifiableSettingRequest.settings(Settings.builder().put(unmodifiableSettingKey, unmodifiableSettingValue).build()); exception = expectThrows(ElasticsearchException.class, () -> execute(unmodifiableSettingRequest, + highLevelClient().indices()::putSettings, highLevelClient().indices()::putSettingsAsync, highLevelClient().indices()::putSettings, highLevelClient().indices()::putSettingsAsync)); assertThat(exception.getMessage(), startsWith( "Elasticsearch exception [type=illegal_argument_exception, " @@ -887,12 +931,14 @@ public void testIndexPutSettingNonExistent() throws IOException { indexUpdateSettingsRequest.settings(Settings.builder().put(setting, value).build()); ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> execute(indexUpdateSettingsRequest, + highLevelClient().indices()::putSettings, highLevelClient().indices()::putSettingsAsync, highLevelClient().indices()::putSettings, highLevelClient().indices()::putSettingsAsync)); assertEquals(RestStatus.NOT_FOUND, exception.status()); assertThat(exception.getMessage(), equalTo("Elasticsearch exception [type=index_not_found_exception, reason=no such index]")); createIndex(index, Settings.EMPTY); exception = expectThrows(ElasticsearchException.class, () -> execute(indexUpdateSettingsRequest, + highLevelClient().indices()::putSettings, highLevelClient().indices()::putSettingsAsync, highLevelClient().indices()::putSettings, highLevelClient().indices()::putSettingsAsync)); assertThat(exception.status(), equalTo(RestStatus.BAD_REQUEST)); assertThat(exception.getMessage(), equalTo( diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/PingAndInfoIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/PingAndInfoIT.java index b4d8828eb7e6f..057ea49f9a969 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/PingAndInfoIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/PingAndInfoIT.java @@ -28,12 +28,12 @@ public class PingAndInfoIT extends ESRestHighLevelClientTestCase { public void testPing() throws IOException { - assertTrue(highLevelClient().ping()); + assertTrue(highLevelClient().ping(RequestOptions.DEFAULT)); } @SuppressWarnings("unchecked") public void testInfo() throws IOException { - MainResponse info = highLevelClient().info(); + MainResponse info = highLevelClient().info(RequestOptions.DEFAULT); // compare with what the low level client outputs Map infoAsMap = entityAsMap(adminClient().performRequest(HttpGet.METHOD_NAME, "/")); assertEquals(infoAsMap.get("cluster_name"), info.getClusterName().value()); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java index 9497bdded0549..1e12f3f5e62f6 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java @@ -82,8 +82,8 @@ public void testRankEvalRequest() throws IOException { RankEvalSpec spec = new RankEvalSpec(specifications, metric); RankEvalRequest rankEvalRequest = new RankEvalRequest(spec, new String[] { "index", "index2" }); - RankEvalResponse response = execute(rankEvalRequest, highLevelClient()::rankEval, - highLevelClient()::rankEvalAsync); + RankEvalResponse response = execute(rankEvalRequest, highLevelClient()::rankEval, highLevelClient()::rankEvalAsync, + highLevelClient()::rankEval, highLevelClient()::rankEvalAsync); // the expected Prec@ for the first query is 5/7 and the expected Prec@ for the second is 1/7, divided by 2 to get the average double expectedPrecision = (1.0 / 7.0 + 5.0 / 7.0) / 2.0; assertEquals(expectedPrecision, response.getEvaluationResult(), Double.MIN_VALUE); @@ -117,7 +117,8 @@ public void testRankEvalRequest() throws IOException { // now try this when test2 is closed client().performRequest("POST", "index2/_close", Collections.emptyMap()); rankEvalRequest.indicesOptions(IndicesOptions.fromParameters(null, "true", null, SearchRequest.DEFAULT_INDICES_OPTIONS)); - response = execute(rankEvalRequest, highLevelClient()::rankEval, highLevelClient()::rankEvalAsync); + response = execute(rankEvalRequest, highLevelClient()::rankEval, highLevelClient()::rankEvalAsync, + highLevelClient()::rankEval, highLevelClient()::rankEvalAsync); } private static List createRelevant(String indexName, String... docs) { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java index 5ca9b05f73adf..9084a547c162d 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java @@ -20,7 +20,6 @@ package org.elasticsearch.client; import com.fasterxml.jackson.core.JsonParseException; - import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; @@ -28,10 +27,7 @@ import org.apache.http.ProtocolVersion; import org.apache.http.RequestLine; import org.apache.http.StatusLine; -import org.apache.http.client.methods.HttpDelete; import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpHead; -import org.apache.http.client.methods.HttpPost; import org.apache.http.entity.ByteArrayEntity; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; @@ -77,9 +73,6 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.InternalAggregationTestCase; import org.junit.Before; -import org.mockito.ArgumentMatcher; -import org.mockito.internal.matchers.ArrayEquals; -import org.mockito.internal.matchers.VarargMatcher; import java.io.IOException; import java.net.SocketTimeoutException; @@ -124,25 +117,22 @@ public void testCloseIsIdempotent() throws IOException { } public void testPingSuccessful() throws IOException { - Header[] headers = randomHeaders(random(), "Header"); Response response = mock(Response.class); when(response.getStatusLine()).thenReturn(newStatusLine(RestStatus.OK)); when(restClient.performRequest(any(Request.class))).thenReturn(response); - assertTrue(restHighLevelClient.ping(headers)); + assertTrue(restHighLevelClient.ping(RequestOptions.DEFAULT)); } public void testPing404NotFound() throws IOException { - Header[] headers = randomHeaders(random(), "Header"); Response response = mock(Response.class); when(response.getStatusLine()).thenReturn(newStatusLine(RestStatus.NOT_FOUND)); when(restClient.performRequest(any(Request.class))).thenReturn(response); - assertFalse(restHighLevelClient.ping(headers)); + assertFalse(restHighLevelClient.ping(RequestOptions.DEFAULT)); } public void testPingSocketTimeout() throws IOException { - Header[] headers = randomHeaders(random(), "Header"); when(restClient.performRequest(any(Request.class))).thenThrow(new SocketTimeoutException()); - expectThrows(SocketTimeoutException.class, () -> restHighLevelClient.ping(headers)); + expectThrows(SocketTimeoutException.class, () -> restHighLevelClient.ping(RequestOptions.DEFAULT)); } public void testInfo() throws IOException { @@ -150,18 +140,17 @@ public void testInfo() throws IOException { MainResponse testInfo = new MainResponse("nodeName", Version.CURRENT, new ClusterName("clusterName"), "clusterUuid", Build.CURRENT); mockResponse(testInfo); - MainResponse receivedInfo = restHighLevelClient.info(headers); + MainResponse receivedInfo = restHighLevelClient.info(RequestOptions.DEFAULT); assertEquals(testInfo, receivedInfo); } public void testSearchScroll() throws IOException { - Header[] headers = randomHeaders(random(), "Header"); SearchResponse mockSearchResponse = new SearchResponse(new SearchResponseSections(SearchHits.empty(), InternalAggregations.EMPTY, null, false, false, null, 1), randomAlphaOfLengthBetween(5, 10), 5, 5, 0, 100, ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY); mockResponse(mockSearchResponse); - SearchResponse searchResponse = restHighLevelClient.searchScroll(new SearchScrollRequest(randomAlphaOfLengthBetween(5, 10)), - headers); + SearchResponse searchResponse = restHighLevelClient.searchScroll( + new SearchScrollRequest(randomAlphaOfLengthBetween(5, 10)), RequestOptions.DEFAULT); assertEquals(mockSearchResponse.getScrollId(), searchResponse.getScrollId()); assertEquals(0, searchResponse.getHits().totalHits); assertEquals(5, searchResponse.getTotalShards()); @@ -170,12 +159,11 @@ public void testSearchScroll() throws IOException { } public void testClearScroll() throws IOException { - Header[] headers = randomHeaders(random(), "Header"); ClearScrollResponse mockClearScrollResponse = new ClearScrollResponse(randomBoolean(), randomIntBetween(0, Integer.MAX_VALUE)); mockResponse(mockClearScrollResponse); ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); clearScrollRequest.addScrollId(randomAlphaOfLengthBetween(5, 10)); - ClearScrollResponse clearScrollResponse = restHighLevelClient.clearScroll(clearScrollRequest, headers); + ClearScrollResponse clearScrollResponse = restHighLevelClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); assertEquals(mockClearScrollResponse.isSucceeded(), clearScrollResponse.isSucceeded()); assertEquals(mockClearScrollResponse.getNumFreed(), clearScrollResponse.getNumFreed()); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java index e147642fc73bd..80d09acf2817d 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java @@ -164,7 +164,8 @@ public void testSearchNoQuery() throws IOException { public void testSearchMatchQuery() throws IOException { SearchRequest searchRequest = new SearchRequest("index"); searchRequest.source(new SearchSourceBuilder().query(new MatchQueryBuilder("num", 10))); - SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync); + SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync, + highLevelClient()::search, highLevelClient()::searchAsync); assertSearchHeader(searchResponse); assertNull(searchResponse.getAggregations()); assertNull(searchResponse.getSuggest()); @@ -190,7 +191,8 @@ public void testSearchWithTermsAgg() throws IOException { searchSourceBuilder.aggregation(new TermsAggregationBuilder("agg1", ValueType.STRING).field("type.keyword")); searchSourceBuilder.size(0); searchRequest.source(searchSourceBuilder); - SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync); + SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync, + highLevelClient()::search, highLevelClient()::searchAsync); assertSearchHeader(searchResponse); assertNull(searchResponse.getSuggest()); assertEquals(Collections.emptyMap(), searchResponse.getProfileResults()); @@ -216,7 +218,8 @@ public void testSearchWithRangeAgg() throws IOException { searchRequest.source(searchSourceBuilder); ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, - () -> execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync)); + () -> execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync, + highLevelClient()::search, highLevelClient()::searchAsync)); assertEquals(RestStatus.BAD_REQUEST, exception.status()); } @@ -226,7 +229,8 @@ public void testSearchWithRangeAgg() throws IOException { .addRange("first", 0, 30).addRange("second", 31, 200)); searchSourceBuilder.size(0); searchRequest.source(searchSourceBuilder); - SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync); + SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync, + highLevelClient()::search, highLevelClient()::searchAsync); assertSearchHeader(searchResponse); assertNull(searchResponse.getSuggest()); assertEquals(Collections.emptyMap(), searchResponse.getProfileResults()); @@ -257,7 +261,8 @@ public void testSearchWithTermsAndRangeAgg() throws IOException { searchSourceBuilder.aggregation(agg); searchSourceBuilder.size(0); searchRequest.source(searchSourceBuilder); - SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync); + SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync, + highLevelClient()::search, highLevelClient()::searchAsync); assertSearchHeader(searchResponse); assertNull(searchResponse.getSuggest()); assertEquals(Collections.emptyMap(), searchResponse.getProfileResults()); @@ -308,7 +313,8 @@ public void testSearchWithMatrixStats() throws IOException { searchSourceBuilder.aggregation(new MatrixStatsAggregationBuilder("agg1").fields(Arrays.asList("num", "num2"))); searchSourceBuilder.size(0); searchRequest.source(searchSourceBuilder); - SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync); + SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync, + highLevelClient()::search, highLevelClient()::searchAsync); assertSearchHeader(searchResponse); assertNull(searchResponse.getSuggest()); assertEquals(Collections.emptyMap(), searchResponse.getProfileResults()); @@ -397,7 +403,8 @@ public void testSearchWithParentJoin() throws IOException { SearchRequest searchRequest = new SearchRequest(indexName); searchRequest.source(searchSourceBuilder); - SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync); + SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync, + highLevelClient()::search, highLevelClient()::searchAsync); assertSearchHeader(searchResponse); assertNull(searchResponse.getSuggest()); assertEquals(Collections.emptyMap(), searchResponse.getProfileResults()); @@ -437,7 +444,8 @@ public void testSearchWithSuggest() throws IOException { searchSourceBuilder.size(0); searchRequest.source(searchSourceBuilder); - SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync); + SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync, + highLevelClient()::search, highLevelClient()::searchAsync); assertSearchHeader(searchResponse); assertNull(searchResponse.getAggregations()); assertEquals(Collections.emptyMap(), searchResponse.getProfileResults()); @@ -469,7 +477,8 @@ public void testSearchWithWeirdScriptFields() throws Exception { { SearchRequest searchRequest = new SearchRequest("test").source(SearchSourceBuilder.searchSource() .scriptField("result", new Script("null"))); - SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync); + SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync, + highLevelClient()::search, highLevelClient()::searchAsync); SearchHit searchHit = searchResponse.getHits().getAt(0); List values = searchHit.getFields().get("result").getValues(); assertNotNull(values); @@ -479,7 +488,8 @@ public void testSearchWithWeirdScriptFields() throws Exception { { SearchRequest searchRequest = new SearchRequest("test").source(SearchSourceBuilder.searchSource() .scriptField("result", new Script("new HashMap()"))); - SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync); + SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync, + highLevelClient()::search, highLevelClient()::searchAsync); SearchHit searchHit = searchResponse.getHits().getAt(0); List values = searchHit.getFields().get("result").getValues(); assertNotNull(values); @@ -491,7 +501,8 @@ public void testSearchWithWeirdScriptFields() throws Exception { { SearchRequest searchRequest = new SearchRequest("test").source(SearchSourceBuilder.searchSource() .scriptField("result", new Script("new String[]{}"))); - SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync); + SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync, + highLevelClient()::search, highLevelClient()::searchAsync); SearchHit searchHit = searchResponse.getHits().getAt(0); List values = searchHit.getFields().get("result").getValues(); assertNotNull(values); @@ -513,7 +524,8 @@ public void testSearchScroll() throws Exception { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().size(35).sort("field", SortOrder.ASC); SearchRequest searchRequest = new SearchRequest("test").scroll(TimeValue.timeValueMinutes(2)).source(searchSourceBuilder); - SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync); + SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync, + highLevelClient()::search, highLevelClient()::searchAsync); try { long counter = 0; @@ -525,6 +537,7 @@ public void testSearchScroll() throws Exception { } searchResponse = execute(new SearchScrollRequest(searchResponse.getScrollId()).scroll(TimeValue.timeValueMinutes(2)), + highLevelClient()::searchScroll, highLevelClient()::searchScrollAsync, highLevelClient()::searchScroll, highLevelClient()::searchScrollAsync); assertThat(searchResponse.getHits().getTotalHits(), equalTo(100L)); @@ -534,6 +547,7 @@ public void testSearchScroll() throws Exception { } searchResponse = execute(new SearchScrollRequest(searchResponse.getScrollId()).scroll(TimeValue.timeValueMinutes(2)), + highLevelClient()::searchScroll, highLevelClient()::searchScrollAsync, highLevelClient()::searchScroll, highLevelClient()::searchScrollAsync); assertThat(searchResponse.getHits().getTotalHits(), equalTo(100L)); @@ -545,14 +559,14 @@ public void testSearchScroll() throws Exception { ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); clearScrollRequest.addScrollId(searchResponse.getScrollId()); ClearScrollResponse clearScrollResponse = execute(clearScrollRequest, - // Not using a method reference to work around https://bugs.eclipse.org/bugs/show_bug.cgi?id=517951 - (request, headers) -> highLevelClient().clearScroll(request, headers), - (request, listener, headers) -> highLevelClient().clearScrollAsync(request, listener, headers)); + highLevelClient()::clearScroll, highLevelClient()::clearScrollAsync, + highLevelClient()::clearScroll, highLevelClient()::clearScrollAsync); assertThat(clearScrollResponse.getNumFreed(), greaterThan(0)); assertTrue(clearScrollResponse.isSucceeded()); SearchScrollRequest scrollRequest = new SearchScrollRequest(searchResponse.getScrollId()).scroll(TimeValue.timeValueMinutes(2)); ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, () -> execute(scrollRequest, + highLevelClient()::searchScroll, highLevelClient()::searchScrollAsync, highLevelClient()::searchScroll, highLevelClient()::searchScrollAsync)); assertEquals(RestStatus.NOT_FOUND, exception.status()); assertThat(exception.getRootCause(), instanceOf(ElasticsearchException.class)); @@ -574,7 +588,8 @@ public void testMultiSearch() throws Exception { multiSearchRequest.add(searchRequest3); MultiSearchResponse multiSearchResponse = - execute(multiSearchRequest, highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync); + execute(multiSearchRequest, highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync, + highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync); assertThat(multiSearchResponse.getTook().millis(), Matchers.greaterThanOrEqualTo(0L)); assertThat(multiSearchResponse.getResponses().length, Matchers.equalTo(3)); @@ -616,7 +631,8 @@ public void testMultiSearch_withAgg() throws Exception { multiSearchRequest.add(searchRequest3); MultiSearchResponse multiSearchResponse = - execute(multiSearchRequest, highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync); + execute(multiSearchRequest, highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync, + highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync); assertThat(multiSearchResponse.getTook().millis(), Matchers.greaterThanOrEqualTo(0L)); assertThat(multiSearchResponse.getResponses().length, Matchers.equalTo(3)); @@ -664,7 +680,8 @@ public void testMultiSearch_withQuery() throws Exception { multiSearchRequest.add(searchRequest3); MultiSearchResponse multiSearchResponse = - execute(multiSearchRequest, highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync); + execute(multiSearchRequest, highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync, + highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync); assertThat(multiSearchResponse.getTook().millis(), Matchers.greaterThanOrEqualTo(0L)); assertThat(multiSearchResponse.getResponses().length, Matchers.equalTo(3)); @@ -727,7 +744,8 @@ public void testMultiSearch_failure() throws Exception { multiSearchRequest.add(searchRequest2); MultiSearchResponse multiSearchResponse = - execute(multiSearchRequest, highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync); + execute(multiSearchRequest, highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync, + highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync); assertThat(multiSearchResponse.getTook().millis(), Matchers.greaterThanOrEqualTo(0L)); assertThat(multiSearchResponse.getResponses().length, Matchers.equalTo(2)); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java index 6641aa2fc7d25..ef92e28a07280 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java @@ -48,6 +48,7 @@ import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.ESRestHighLevelClientTestCase; import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.common.Strings; @@ -72,13 +73,12 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import static java.util.Collections.singletonMap; import static org.hamcrest.Matchers.arrayWithSize; -import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.not; -import static java.util.Collections.emptyMap; -import static java.util.Collections.singletonMap; /** * This class is used to generate the Java CRUD API documentation. @@ -112,7 +112,7 @@ public void testIndex() throws Exception { IndexRequest indexRequest = new IndexRequest("posts", "doc", "1") .source(jsonMap); // <1> //end::index-request-map - IndexResponse indexResponse = client.index(indexRequest); + IndexResponse indexResponse = client.index(indexRequest, RequestOptions.DEFAULT); assertEquals(indexResponse.getResult(), DocWriteResponse.Result.CREATED); } { @@ -128,7 +128,7 @@ public void testIndex() throws Exception { IndexRequest indexRequest = new IndexRequest("posts", "doc", "1") .source(builder); // <1> //end::index-request-xcontent - IndexResponse indexResponse = client.index(indexRequest); + IndexResponse indexResponse = client.index(indexRequest, RequestOptions.DEFAULT); assertEquals(indexResponse.getResult(), DocWriteResponse.Result.UPDATED); } { @@ -138,7 +138,7 @@ public void testIndex() throws Exception { "postDate", new Date(), "message", "trying out Elasticsearch"); // <1> //end::index-request-shortcut - IndexResponse indexResponse = client.index(indexRequest); + IndexResponse indexResponse = client.index(indexRequest, RequestOptions.DEFAULT); assertEquals(indexResponse.getResult(), DocWriteResponse.Result.UPDATED); } { @@ -156,7 +156,7 @@ public void testIndex() throws Exception { //end::index-request-string // tag::index-execute - IndexResponse indexResponse = client.index(request); + IndexResponse indexResponse = client.index(request, RequestOptions.DEFAULT); // end::index-execute assertEquals(indexResponse.getResult(), DocWriteResponse.Result.UPDATED); @@ -214,7 +214,7 @@ public void testIndex() throws Exception { .source("field", "value") .version(1); try { - IndexResponse response = client.index(request); + IndexResponse response = client.index(request, RequestOptions.DEFAULT); } catch(ElasticsearchException e) { if (e.status() == RestStatus.CONFLICT) { // <1> @@ -228,7 +228,7 @@ public void testIndex() throws Exception { .source("field", "value") .opType(DocWriteRequest.OpType.CREATE); try { - IndexResponse response = client.index(request); + IndexResponse response = client.index(request, RequestOptions.DEFAULT); } catch(ElasticsearchException e) { if (e.status() == RestStatus.CONFLICT) { // <1> @@ -257,7 +257,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::index-execute-async - client.indexAsync(request, listener); // <1> + client.indexAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::index-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -268,7 +268,7 @@ public void testUpdate() throws Exception { RestHighLevelClient client = highLevelClient(); { IndexRequest indexRequest = new IndexRequest("posts", "doc", "1").source("field", 0); - IndexResponse indexResponse = client.index(indexRequest); + IndexResponse indexResponse = client.index(indexRequest, RequestOptions.DEFAULT); assertSame(indexResponse.status(), RestStatus.CREATED); Request request = new Request("POST", "/_scripts/increment-field"); @@ -297,7 +297,7 @@ public void testUpdate() throws Exception { "ctx._source.field += params.count", parameters); // <2> request.script(inline); // <3> //end::update-request-with-inline-script - UpdateResponse updateResponse = client.update(request); + UpdateResponse updateResponse = client.update(request, RequestOptions.DEFAULT); assertEquals(updateResponse.getResult(), DocWriteResponse.Result.UPDATED); assertEquals(4, updateResponse.getGetResult().getSource().get("field")); @@ -307,7 +307,7 @@ public void testUpdate() throws Exception { new Script(ScriptType.STORED, null, "increment-field", parameters); // <1> request.script(stored); // <2> //end::update-request-with-stored-script - updateResponse = client.update(request); + updateResponse = client.update(request, RequestOptions.DEFAULT); assertEquals(updateResponse.getResult(), DocWriteResponse.Result.UPDATED); assertEquals(8, updateResponse.getGetResult().getSource().get("field")); } @@ -319,7 +319,7 @@ public void testUpdate() throws Exception { UpdateRequest request = new UpdateRequest("posts", "doc", "1") .doc(jsonMap); // <1> //end::update-request-with-doc-as-map - UpdateResponse updateResponse = client.update(request); + UpdateResponse updateResponse = client.update(request, RequestOptions.DEFAULT); assertEquals(updateResponse.getResult(), DocWriteResponse.Result.UPDATED); } { @@ -334,7 +334,7 @@ public void testUpdate() throws Exception { UpdateRequest request = new UpdateRequest("posts", "doc", "1") .doc(builder); // <1> //end::update-request-with-doc-as-xcontent - UpdateResponse updateResponse = client.update(request); + UpdateResponse updateResponse = client.update(request, RequestOptions.DEFAULT); assertEquals(updateResponse.getResult(), DocWriteResponse.Result.UPDATED); } { @@ -343,7 +343,7 @@ public void testUpdate() throws Exception { .doc("updated", new Date(), "reason", "daily update"); // <1> //end::update-request-shortcut - UpdateResponse updateResponse = client.update(request); + UpdateResponse updateResponse = client.update(request, RequestOptions.DEFAULT); assertEquals(updateResponse.getResult(), DocWriteResponse.Result.UPDATED); } { @@ -357,7 +357,7 @@ public void testUpdate() throws Exception { //end::update-request-with-doc-as-string request.fetchSource(true); // tag::update-execute - UpdateResponse updateResponse = client.update(request); + UpdateResponse updateResponse = client.update(request, RequestOptions.DEFAULT); // end::update-execute assertEquals(updateResponse.getResult(), DocWriteResponse.Result.UPDATED); @@ -406,7 +406,7 @@ public void testUpdate() throws Exception { UpdateRequest request = new UpdateRequest("posts", "type", "does_not_exist") .doc("field", "value"); try { - UpdateResponse updateResponse = client.update(request); + UpdateResponse updateResponse = client.update(request, RequestOptions.DEFAULT); } catch (ElasticsearchException e) { if (e.status() == RestStatus.NOT_FOUND) { // <1> @@ -420,7 +420,7 @@ public void testUpdate() throws Exception { .doc("field", "value") .version(1); try { - UpdateResponse updateResponse = client.update(request); + UpdateResponse updateResponse = client.update(request, RequestOptions.DEFAULT); } catch(ElasticsearchException e) { if (e.status() == RestStatus.CONFLICT) { // <1> @@ -433,7 +433,7 @@ public void testUpdate() throws Exception { //tag::update-request-no-source request.fetchSource(true); // <1> //end::update-request-no-source - UpdateResponse updateResponse = client.update(request); + UpdateResponse updateResponse = client.update(request, RequestOptions.DEFAULT); assertEquals(updateResponse.getResult(), DocWriteResponse.Result.UPDATED); assertNotNull(updateResponse.getGetResult()); assertEquals(3, updateResponse.getGetResult().sourceAsMap().size()); @@ -445,7 +445,7 @@ public void testUpdate() throws Exception { String[] excludes = Strings.EMPTY_ARRAY; request.fetchSource(new FetchSourceContext(true, includes, excludes)); // <1> //end::update-request-source-include - UpdateResponse updateResponse = client.update(request); + UpdateResponse updateResponse = client.update(request, RequestOptions.DEFAULT); assertEquals(updateResponse.getResult(), DocWriteResponse.Result.UPDATED); Map sourceAsMap = updateResponse.getGetResult().sourceAsMap(); assertEquals(2, sourceAsMap.size()); @@ -459,7 +459,7 @@ public void testUpdate() throws Exception { String[] excludes = new String[]{"updated"}; request.fetchSource(new FetchSourceContext(true, includes, excludes)); // <1> //end::update-request-source-exclude - UpdateResponse updateResponse = client.update(request); + UpdateResponse updateResponse = client.update(request, RequestOptions.DEFAULT); assertEquals(updateResponse.getResult(), DocWriteResponse.Result.UPDATED); Map sourceAsMap = updateResponse.getGetResult().sourceAsMap(); assertEquals(2, sourceAsMap.size()); @@ -525,7 +525,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::update-execute-async - client.updateAsync(request, listener); // <1> + client.updateAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::update-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -537,7 +537,7 @@ public void testDelete() throws Exception { { IndexRequest indexRequest = new IndexRequest("posts", "doc", "1").source("field", "value"); - IndexResponse indexResponse = client.index(indexRequest); + IndexResponse indexResponse = client.index(indexRequest, RequestOptions.DEFAULT); assertSame(indexResponse.status(), RestStatus.CREATED); } @@ -550,7 +550,7 @@ public void testDelete() throws Exception { // end::delete-request // tag::delete-execute - DeleteResponse deleteResponse = client.delete(request); + DeleteResponse deleteResponse = client.delete(request, RequestOptions.DEFAULT); // end::delete-execute assertSame(deleteResponse.getResult(), DocWriteResponse.Result.DELETED); @@ -595,7 +595,7 @@ public void testDelete() throws Exception { { // tag::delete-notfound DeleteRequest request = new DeleteRequest("posts", "doc", "does_not_exist"); - DeleteResponse deleteResponse = client.delete(request); + DeleteResponse deleteResponse = client.delete(request, RequestOptions.DEFAULT); if (deleteResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { // <1> } @@ -603,13 +603,14 @@ public void testDelete() throws Exception { } { - IndexResponse indexResponse = client.index(new IndexRequest("posts", "doc", "1").source("field", "value")); + IndexResponse indexResponse = client.index(new IndexRequest("posts", "doc", "1").source("field", "value") + , RequestOptions.DEFAULT); assertSame(indexResponse.status(), RestStatus.CREATED); // tag::delete-conflict try { DeleteRequest request = new DeleteRequest("posts", "doc", "1").version(2); - DeleteResponse deleteResponse = client.delete(request); + DeleteResponse deleteResponse = client.delete(request, RequestOptions.DEFAULT); } catch (ElasticsearchException exception) { if (exception.status() == RestStatus.CONFLICT) { // <1> @@ -618,7 +619,8 @@ public void testDelete() throws Exception { // end::delete-conflict } { - IndexResponse indexResponse = client.index(new IndexRequest("posts", "doc", "async").source("field", "value")); + IndexResponse indexResponse = client.index(new IndexRequest("posts", "doc", "async").source("field", "value"), + RequestOptions.DEFAULT); assertSame(indexResponse.status(), RestStatus.CREATED); DeleteRequest request = new DeleteRequest("posts", "doc", "async"); @@ -642,7 +644,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::delete-execute-async - client.deleteAsync(request, listener); // <1> + client.deleteAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::delete-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -662,7 +664,7 @@ public void testBulk() throws Exception { .source(XContentType.JSON,"field", "baz")); // end::bulk-request // tag::bulk-execute - BulkResponse bulkResponse = client.bulk(request); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); // end::bulk-execute assertSame(bulkResponse.status(), RestStatus.OK); assertFalse(bulkResponse.hasFailures()); @@ -676,7 +678,7 @@ public void testBulk() throws Exception { request.add(new IndexRequest("posts", "doc", "4") // <3> .source(XContentType.JSON,"field", "baz")); // end::bulk-request-with-mixed-operations - BulkResponse bulkResponse = client.bulk(request); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); assertSame(bulkResponse.status(), RestStatus.OK); assertFalse(bulkResponse.hasFailures()); @@ -775,7 +777,7 @@ public void testGet() throws Exception { .source("user", "kimchy", "postDate", new Date(), "message", "trying out Elasticsearch"); - IndexResponse indexResponse = client.index(indexRequest); + IndexResponse indexResponse = client.index(indexRequest, RequestOptions.DEFAULT); assertEquals(indexResponse.getResult(), DocWriteResponse.Result.CREATED); } { @@ -787,7 +789,7 @@ public void testGet() throws Exception { //end::get-request //tag::get-execute - GetResponse getResponse = client.get(getRequest); + GetResponse getResponse = client.get(getRequest, RequestOptions.DEFAULT); //end::get-execute assertTrue(getResponse.isExists()); assertEquals(3, getResponse.getSourceAsMap().size()); @@ -810,7 +812,7 @@ public void testGet() throws Exception { //tag::get-request-no-source request.fetchSourceContext(FetchSourceContext.DO_NOT_FETCH_SOURCE); // <1> //end::get-request-no-source - GetResponse getResponse = client.get(request); + GetResponse getResponse = client.get(request, RequestOptions.DEFAULT); assertNull(getResponse.getSourceInternal()); } { @@ -822,7 +824,7 @@ public void testGet() throws Exception { new FetchSourceContext(true, includes, excludes); request.fetchSourceContext(fetchSourceContext); // <1> //end::get-request-source-include - GetResponse getResponse = client.get(request); + GetResponse getResponse = client.get(request, RequestOptions.DEFAULT); Map sourceAsMap = getResponse.getSourceAsMap(); assertEquals(2, sourceAsMap.size()); assertEquals("trying out Elasticsearch", sourceAsMap.get("message")); @@ -837,7 +839,7 @@ public void testGet() throws Exception { new FetchSourceContext(true, includes, excludes); request.fetchSourceContext(fetchSourceContext); // <1> //end::get-request-source-exclude - GetResponse getResponse = client.get(request); + GetResponse getResponse = client.get(request, RequestOptions.DEFAULT); Map sourceAsMap = getResponse.getSourceAsMap(); assertEquals(2, sourceAsMap.size()); assertEquals("kimchy", sourceAsMap.get("user")); @@ -847,7 +849,7 @@ public void testGet() throws Exception { GetRequest request = new GetRequest("posts", "doc", "1"); //tag::get-request-stored request.storedFields("message"); // <1> - GetResponse getResponse = client.get(request); + GetResponse getResponse = client.get(request, RequestOptions.DEFAULT); String message = getResponse.getField("message").getValue(); // <2> //end::get-request-stored assertEquals("trying out Elasticsearch", message); @@ -897,7 +899,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); //tag::get-execute-async - client.getAsync(request, listener); // <1> + client.getAsync(request, RequestOptions.DEFAULT, listener); // <1> //end::get-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -906,7 +908,7 @@ public void onFailure(Exception e) { //tag::get-indexnotfound GetRequest request = new GetRequest("does_not_exist", "doc", "1"); try { - GetResponse getResponse = client.get(request); + GetResponse getResponse = client.get(request, RequestOptions.DEFAULT); } catch (ElasticsearchException e) { if (e.status() == RestStatus.NOT_FOUND) { // <1> @@ -918,7 +920,7 @@ public void onFailure(Exception e) { // tag::get-conflict try { GetRequest request = new GetRequest("posts", "doc", "1").version(2); - GetResponse getResponse = client.get(request); + GetResponse getResponse = client.get(request, RequestOptions.DEFAULT); } catch (ElasticsearchException exception) { if (exception.status() == RestStatus.CONFLICT) { // <1> @@ -940,7 +942,7 @@ public void testExists() throws Exception { // end::exists-request { // tag::exists-execute - boolean exists = client.exists(getRequest); + boolean exists = client.exists(getRequest, RequestOptions.DEFAULT); // end::exists-execute assertFalse(exists); } @@ -964,7 +966,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::exists-execute-async - client.existsAsync(getRequest, listener); // <1> + client.existsAsync(getRequest, RequestOptions.DEFAULT, listener); // <1> // end::exists-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -1091,7 +1093,7 @@ public void testMultiGet() throws Exception { source.put("baz", "val3"); client.index(new IndexRequest("index", "type", "example_id") .source(source) - .setRefreshPolicy(RefreshPolicy.IMMEDIATE)); + .setRefreshPolicy(RefreshPolicy.IMMEDIATE), RequestOptions.DEFAULT); { // tag::multi-get-request @@ -1120,7 +1122,7 @@ public void testMultiGet() throws Exception { // end::multi-get-request-top-level-extras // tag::multi-get-execute - MultiGetResponse response = client.multiGet(request); + MultiGetResponse response = client.multiGet(request, RequestOptions.DEFAULT); // end::multi-get-execute // tag::multi-get-response @@ -1184,7 +1186,7 @@ public void onFailure(Exception e) { request.add(new MultiGetRequest.Item("index", "type", "example_id") .fetchSourceContext(FetchSourceContext.DO_NOT_FETCH_SOURCE)); // <1> // end::multi-get-request-no-source - MultiGetItemResponse item = unwrapAndAssertExample(client.multiGet(request)); + MultiGetItemResponse item = unwrapAndAssertExample(client.multiGet(request, RequestOptions.DEFAULT)); assertNull(item.getResponse().getSource()); } { @@ -1197,7 +1199,7 @@ public void onFailure(Exception e) { request.add(new MultiGetRequest.Item("index", "type", "example_id") .fetchSourceContext(fetchSourceContext)); // <1> // end::multi-get-request-source-include - MultiGetItemResponse item = unwrapAndAssertExample(client.multiGet(request)); + MultiGetItemResponse item = unwrapAndAssertExample(client.multiGet(request, RequestOptions.DEFAULT)); assertThat(item.getResponse().getSource(), hasEntry("foo", "val1")); assertThat(item.getResponse().getSource(), hasEntry("bar", "val2")); assertThat(item.getResponse().getSource(), not(hasKey("baz"))); @@ -1212,7 +1214,7 @@ public void onFailure(Exception e) { request.add(new MultiGetRequest.Item("index", "type", "example_id") .fetchSourceContext(fetchSourceContext)); // <1> // end::multi-get-request-source-exclude - MultiGetItemResponse item = unwrapAndAssertExample(client.multiGet(request)); + MultiGetItemResponse item = unwrapAndAssertExample(client.multiGet(request, RequestOptions.DEFAULT)); assertThat(item.getResponse().getSource(), not(hasKey("foo"))); assertThat(item.getResponse().getSource(), not(hasKey("bar"))); assertThat(item.getResponse().getSource(), hasEntry("baz", "val3")); @@ -1222,7 +1224,7 @@ public void onFailure(Exception e) { // tag::multi-get-request-stored request.add(new MultiGetRequest.Item("index", "type", "example_id") .storedFields("foo")); // <1> - MultiGetResponse response = client.multiGet(request); + MultiGetResponse response = client.multiGet(request, RequestOptions.DEFAULT); MultiGetItemResponse item = response.getResponses()[0]; String value = item.getResponse().getField("foo").getValue(); // <2> // end::multi-get-request-stored @@ -1234,7 +1236,7 @@ public void onFailure(Exception e) { MultiGetRequest request = new MultiGetRequest(); request.add(new MultiGetRequest.Item("index", "type", "example_id") .version(1000L)); - MultiGetResponse response = client.multiGet(request); + MultiGetResponse response = client.multiGet(request, RequestOptions.DEFAULT); MultiGetItemResponse item = response.getResponses()[0]; assertNull(item.getResponse()); // <1> Exception e = item.getFailure().getFailure(); // <2> diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/ClusterClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/ClusterClientDocumentationIT.java index 304c5010a47e3..e8dd4025ba94e 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/ClusterClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/ClusterClientDocumentationIT.java @@ -23,27 +23,19 @@ import org.elasticsearch.action.LatchedActionListener; import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; -import org.elasticsearch.action.ingest.GetPipelineRequest; -import org.elasticsearch.action.ingest.GetPipelineResponse; -import org.elasticsearch.action.ingest.PutPipelineRequest; -import org.elasticsearch.action.ingest.DeletePipelineRequest; -import org.elasticsearch.action.ingest.WritePipelineResponse; import org.elasticsearch.client.ESRestHighLevelClientTestCase; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.indices.recovery.RecoverySettings; -import org.elasticsearch.ingest.PipelineConfiguration; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; -import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -134,7 +126,7 @@ public void testClusterPutSettings() throws IOException { // end::put-settings-request-masterTimeout // tag::put-settings-execute - ClusterUpdateSettingsResponse response = client.cluster().putSettings(request); + ClusterUpdateSettingsResponse response = client.cluster().putSettings(request, RequestOptions.DEFAULT); // end::put-settings-execute // tag::put-settings-response @@ -150,7 +142,7 @@ public void testClusterPutSettings() throws IOException { request.transientSettings(Settings.builder().putNull(transientSettingKey).build()); // <1> // tag::put-settings-request-reset-transient request.persistentSettings(Settings.builder().putNull(persistentSettingKey)); - ClusterUpdateSettingsResponse resetResponse = client.cluster().putSettings(request); + ClusterUpdateSettingsResponse resetResponse = client.cluster().putSettings(request, RequestOptions.DEFAULT); assertTrue(resetResponse.isAcknowledged()); } @@ -180,7 +172,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::put-settings-execute-async - client.cluster().putSettingsAsync(request, listener); // <1> + client.cluster().putSettingsAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::put-settings-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java index c3decd93a174c..2b81e4a4adce9 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java @@ -64,6 +64,7 @@ import org.elasticsearch.action.support.DefaultShardOperationFailedException; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.ESRestHighLevelClientTestCase; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.client.SyncedFlushResponse; import org.elasticsearch.cluster.metadata.MappingMetaData; @@ -111,7 +112,7 @@ public void testIndicesExist() throws IOException { RestHighLevelClient client = highLevelClient(); { - CreateIndexResponse createIndexResponse = client.indices().create(new CreateIndexRequest("twitter")); + CreateIndexResponse createIndexResponse = client.indices().create(new CreateIndexRequest("twitter"), RequestOptions.DEFAULT); assertTrue(createIndexResponse.isAcknowledged()); } @@ -130,7 +131,7 @@ public void testIndicesExist() throws IOException { // end::indices-exists-request-optionals // tag::indices-exists-response - boolean exists = client.indices().exists(request); + boolean exists = client.indices().exists(request, RequestOptions.DEFAULT); // end::indices-exists-response assertTrue(exists); } @@ -140,7 +141,7 @@ public void testIndicesExistAsync() throws Exception { RestHighLevelClient client = highLevelClient(); { - CreateIndexResponse createIndexResponse = client.indices().create(new CreateIndexRequest("twitter")); + CreateIndexResponse createIndexResponse = client.indices().create(new CreateIndexRequest("twitter"), RequestOptions.DEFAULT); assertTrue(createIndexResponse.isAcknowledged()); } @@ -167,7 +168,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::indices-exists-async - client.indices().existsAsync(request, listener); // <1> + client.indices().existsAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::indices-exists-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -177,7 +178,7 @@ public void testDeleteIndex() throws IOException { RestHighLevelClient client = highLevelClient(); { - CreateIndexResponse createIndexResponse = client.indices().create(new CreateIndexRequest("posts")); + CreateIndexResponse createIndexResponse = client.indices().create(new CreateIndexRequest("posts"), RequestOptions.DEFAULT); assertTrue(createIndexResponse.isAcknowledged()); } @@ -199,7 +200,7 @@ public void testDeleteIndex() throws IOException { // end::delete-index-request-indicesOptions // tag::delete-index-execute - DeleteIndexResponse deleteIndexResponse = client.indices().delete(request); + DeleteIndexResponse deleteIndexResponse = client.indices().delete(request, RequestOptions.DEFAULT); // end::delete-index-execute // tag::delete-index-response @@ -212,7 +213,7 @@ public void testDeleteIndex() throws IOException { // tag::delete-index-notfound try { DeleteIndexRequest request = new DeleteIndexRequest("does_not_exist"); - client.indices().delete(request); + client.indices().delete(request, RequestOptions.DEFAULT); } catch (ElasticsearchException exception) { if (exception.status() == RestStatus.NOT_FOUND) { // <1> @@ -226,7 +227,7 @@ public void testDeleteIndexAsync() throws Exception { final RestHighLevelClient client = highLevelClient(); { - CreateIndexResponse createIndexResponse = client.indices().create(new CreateIndexRequest("posts")); + CreateIndexResponse createIndexResponse = client.indices().create(new CreateIndexRequest("posts"), RequestOptions.DEFAULT); assertTrue(createIndexResponse.isAcknowledged()); } @@ -253,7 +254,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::delete-index-execute-async - client.indices().deleteAsync(request, listener); // <1> + client.indices().deleteAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::delete-index-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -289,7 +290,7 @@ public void testCreateIndex() throws IOException { "}", // <2> XContentType.JSON); // end::create-index-request-mappings - CreateIndexResponse createIndexResponse = client.indices().create(request); + CreateIndexResponse createIndexResponse = client.indices().create(request, RequestOptions.DEFAULT); assertTrue(createIndexResponse.isAcknowledged()); } @@ -306,7 +307,7 @@ public void testCreateIndex() throws IOException { jsonMap.put("tweet", tweet); request.mapping("tweet", jsonMap); // <1> //end::create-index-mappings-map - CreateIndexResponse createIndexResponse = client.indices().create(request); + CreateIndexResponse createIndexResponse = client.indices().create(request, RequestOptions.DEFAULT); assertTrue(createIndexResponse.isAcknowledged()); } { @@ -332,7 +333,7 @@ public void testCreateIndex() throws IOException { builder.endObject(); request.mapping("tweet", builder); // <1> //end::create-index-mappings-xcontent - CreateIndexResponse createIndexResponse = client.indices().create(request); + CreateIndexResponse createIndexResponse = client.indices().create(request, RequestOptions.DEFAULT); assertTrue(createIndexResponse.isAcknowledged()); } { @@ -340,7 +341,7 @@ public void testCreateIndex() throws IOException { //tag::create-index-mappings-shortcut request.mapping("tweet", "message", "type=text"); // <1> //end::create-index-mappings-shortcut - CreateIndexResponse createIndexResponse = client.indices().create(request); + CreateIndexResponse createIndexResponse = client.indices().create(request, RequestOptions.DEFAULT); assertTrue(createIndexResponse.isAcknowledged()); } @@ -362,7 +363,7 @@ public void testCreateIndex() throws IOException { request.waitForActiveShards(ActiveShardCount.DEFAULT); // <2> // end::create-index-request-waitForActiveShards { - CreateIndexResponse createIndexResponse = client.indices().create(request); + CreateIndexResponse createIndexResponse = client.indices().create(request, RequestOptions.DEFAULT); assertTrue(createIndexResponse.isAcknowledged()); } @@ -387,7 +388,7 @@ public void testCreateIndex() throws IOException { // end::create-index-whole-source // tag::create-index-execute - CreateIndexResponse createIndexResponse = client.indices().create(request); + CreateIndexResponse createIndexResponse = client.indices().create(request, RequestOptions.DEFAULT); // end::create-index-execute // tag::create-index-response @@ -426,7 +427,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::create-index-execute-async - client.indices().createAsync(request, listener); // <1> + client.indices().createAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::create-index-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -437,7 +438,7 @@ public void testPutMapping() throws IOException { RestHighLevelClient client = highLevelClient(); { - CreateIndexResponse createIndexResponse = client.indices().create(new CreateIndexRequest("twitter")); + CreateIndexResponse createIndexResponse = client.indices().create(new CreateIndexRequest("twitter"), RequestOptions.DEFAULT); assertTrue(createIndexResponse.isAcknowledged()); } @@ -459,7 +460,7 @@ public void testPutMapping() throws IOException { "}", // <1> XContentType.JSON); // end::put-mapping-request-source - PutMappingResponse putMappingResponse = client.indices().putMapping(request); + PutMappingResponse putMappingResponse = client.indices().putMapping(request, RequestOptions.DEFAULT); assertTrue(putMappingResponse.isAcknowledged()); } @@ -473,7 +474,7 @@ public void testPutMapping() throws IOException { jsonMap.put("properties", properties); request.source(jsonMap); // <1> //end::put-mapping-map - PutMappingResponse putMappingResponse = client.indices().putMapping(request); + PutMappingResponse putMappingResponse = client.indices().putMapping(request, RequestOptions.DEFAULT); assertTrue(putMappingResponse.isAcknowledged()); } { @@ -494,14 +495,14 @@ public void testPutMapping() throws IOException { builder.endObject(); request.source(builder); // <1> //end::put-mapping-xcontent - PutMappingResponse putMappingResponse = client.indices().putMapping(request); + PutMappingResponse putMappingResponse = client.indices().putMapping(request, RequestOptions.DEFAULT); assertTrue(putMappingResponse.isAcknowledged()); } { //tag::put-mapping-shortcut request.source("message", "type=text"); // <1> //end::put-mapping-shortcut - PutMappingResponse putMappingResponse = client.indices().putMapping(request); + PutMappingResponse putMappingResponse = client.indices().putMapping(request, RequestOptions.DEFAULT); assertTrue(putMappingResponse.isAcknowledged()); } @@ -515,7 +516,7 @@ public void testPutMapping() throws IOException { // end::put-mapping-request-masterTimeout // tag::put-mapping-execute - PutMappingResponse putMappingResponse = client.indices().putMapping(request); + PutMappingResponse putMappingResponse = client.indices().putMapping(request, RequestOptions.DEFAULT); // end::put-mapping-execute // tag::put-mapping-response @@ -529,7 +530,7 @@ public void testPutMappingAsync() throws Exception { final RestHighLevelClient client = highLevelClient(); { - CreateIndexResponse createIndexResponse = client.indices().create(new CreateIndexRequest("twitter")); + CreateIndexResponse createIndexResponse = client.indices().create(new CreateIndexRequest("twitter"), RequestOptions.DEFAULT); assertTrue(createIndexResponse.isAcknowledged()); } @@ -556,7 +557,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::put-mapping-execute-async - client.indices().putMappingAsync(request, listener); // <1> + client.indices().putMappingAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::put-mapping-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -601,7 +602,7 @@ public void testGetMapping() throws IOException { // end::get-mapping-request-indicesOptions // tag::get-mapping-execute - GetMappingsResponse getMappingResponse = client.indices().getMappings(request); + GetMappingsResponse getMappingResponse = client.indices().getMappings(request, RequestOptions.DEFAULT); // end::get-mapping-execute // tag::get-mapping-response @@ -683,7 +684,7 @@ public void onFailure(Exception e) { }); // tag::get-mapping-execute-async - client.indices().getMappingsAsync(request, listener); // <1> + client.indices().getMappingsAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::get-mapping-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -694,7 +695,7 @@ public void testOpenIndex() throws Exception { RestHighLevelClient client = highLevelClient(); { - CreateIndexResponse createIndexResponse = client.indices().create(new CreateIndexRequest("index")); + CreateIndexResponse createIndexResponse = client.indices().create(new CreateIndexRequest("index"), RequestOptions.DEFAULT); assertTrue(createIndexResponse.isAcknowledged()); } @@ -721,7 +722,7 @@ public void testOpenIndex() throws Exception { // end::open-index-request-indicesOptions // tag::open-index-execute - OpenIndexResponse openIndexResponse = client.indices().open(request); + OpenIndexResponse openIndexResponse = client.indices().open(request, RequestOptions.DEFAULT); // end::open-index-execute // tag::open-index-response @@ -751,7 +752,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::open-index-execute-async - client.indices().openAsync(request, listener); // <1> + client.indices().openAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::open-index-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -761,7 +762,7 @@ public void onFailure(Exception e) { // tag::open-index-notfound try { OpenIndexRequest request = new OpenIndexRequest("does_not_exist"); - client.indices().open(request); + client.indices().open(request, RequestOptions.DEFAULT); } catch (ElasticsearchException exception) { if (exception.status() == RestStatus.BAD_REQUEST) { // <1> @@ -790,7 +791,7 @@ public void testRefreshIndex() throws Exception { // end::refresh-request-indicesOptions // tag::refresh-execute - RefreshResponse refreshResponse = client.indices().refresh(request); + RefreshResponse refreshResponse = client.indices().refresh(request, RequestOptions.DEFAULT); // end::refresh-execute // tag::refresh-response @@ -819,7 +820,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::refresh-execute-async - client.indices().refreshAsync(request, listener); // <1> + client.indices().refreshAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::refresh-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -829,7 +830,7 @@ public void onFailure(Exception e) { // tag::refresh-notfound try { RefreshRequest request = new RefreshRequest("does_not_exist"); - client.indices().refresh(request); + client.indices().refresh(request, RequestOptions.DEFAULT); } catch (ElasticsearchException exception) { if (exception.status() == RestStatus.NOT_FOUND) { // <1> @@ -866,7 +867,7 @@ public void testFlushIndex() throws Exception { // end::flush-request-force // tag::flush-execute - FlushResponse flushResponse = client.indices().flush(request); + FlushResponse flushResponse = client.indices().flush(request, RequestOptions.DEFAULT); // end::flush-execute // tag::flush-response @@ -895,7 +896,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::flush-execute-async - client.indices().flushAsync(request, listener); // <1> + client.indices().flushAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::flush-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -905,7 +906,7 @@ public void onFailure(Exception e) { // tag::flush-notfound try { FlushRequest request = new FlushRequest("does_not_exist"); - client.indices().flush(request); + client.indices().flush(request, RequestOptions.DEFAULT); } catch (ElasticsearchException exception) { if (exception.status() == RestStatus.NOT_FOUND) { // <1> @@ -934,7 +935,7 @@ public void testSyncedFlushIndex() throws Exception { // end::flush-synced-request-indicesOptions // tag::flush-synced-execute - SyncedFlushResponse flushSyncedResponse = client.indices().flushSynced(request); + SyncedFlushResponse flushSyncedResponse = client.indices().flushSynced(request, RequestOptions.DEFAULT); // end::flush-synced-execute // tag::flush-synced-response @@ -978,7 +979,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::flush-synced-execute-async - client.indices().flushSyncedAsync(request, listener); // <1> + client.indices().flushSyncedAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::flush-synced-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -988,7 +989,7 @@ public void onFailure(Exception e) { // tag::flush-synced-notfound try { SyncedFlushRequest request = new SyncedFlushRequest("does_not_exist"); - client.indices().flushSynced(request); + client.indices().flushSynced(request, RequestOptions.DEFAULT); } catch (ElasticsearchException exception) { if (exception.status() == RestStatus.NOT_FOUND) { // <1> @@ -1003,7 +1004,8 @@ public void testGetSettings() throws Exception { { Settings settings = Settings.builder().put("number_of_shards", 3).build(); - CreateIndexResponse createIndexResponse = client.indices().create(new CreateIndexRequest("index", settings)); + CreateIndexResponse createIndexResponse = client.indices().create( + new CreateIndexRequest("index", settings), RequestOptions.DEFAULT); assertTrue(createIndexResponse.isAcknowledged()); } @@ -1020,7 +1022,7 @@ public void testGetSettings() throws Exception { // end::get-settings-request-indicesOptions // tag::get-settings-execute - GetSettingsResponse getSettingsResponse = client.indices().getSettings(request); + GetSettingsResponse getSettingsResponse = client.indices().getSettings(request, RequestOptions.DEFAULT); // end::get-settings-execute // tag::get-settings-response @@ -1055,7 +1057,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::get-settings-execute-async - client.indices().getSettingsAsync(request, listener); // <1> + client.indices().getSettingsAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::get-settings-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -1066,7 +1068,8 @@ public void testGetSettingsWithDefaults() throws Exception { { Settings settings = Settings.builder().put("number_of_shards", 3).build(); - CreateIndexResponse createIndexResponse = client.indices().create(new CreateIndexRequest("index", settings)); + CreateIndexResponse createIndexResponse = client.indices().create( + new CreateIndexRequest("index", settings), RequestOptions.DEFAULT); assertTrue(createIndexResponse.isAcknowledged()); } @@ -1077,7 +1080,7 @@ public void testGetSettingsWithDefaults() throws Exception { request.includeDefaults(true); // <1> // end::get-settings-request-include-defaults - GetSettingsResponse getSettingsResponse = client.indices().getSettings(request); + GetSettingsResponse getSettingsResponse = client.indices().getSettings(request, RequestOptions.DEFAULT); String numberOfShardsString = getSettingsResponse.getSetting("index", "index.number_of_shards"); Settings indexSettings = getSettingsResponse.getIndexToSettings().get("index"); Integer numberOfShards = indexSettings.getAsInt("index.number_of_shards", null); @@ -1107,7 +1110,7 @@ public void onFailure(Exception e) { final CountDownLatch latch = new CountDownLatch(1); listener = new LatchedActionListener<>(listener, latch); - client.indices().getSettingsAsync(request, listener); + client.indices().getSettingsAsync(request, RequestOptions.DEFAULT, listener); assertTrue(latch.await(30L, TimeUnit.SECONDS)); } @@ -1142,7 +1145,7 @@ public void testForceMergeIndex() throws Exception { // end::force-merge-request-flush // tag::force-merge-execute - ForceMergeResponse forceMergeResponse = client.indices().forceMerge(request); + ForceMergeResponse forceMergeResponse = client.indices().forceMerge(request, RequestOptions.DEFAULT); // end::force-merge-execute // tag::force-merge-response @@ -1167,14 +1170,14 @@ public void onFailure(Exception e) { // end::force-merge-execute-listener // tag::force-merge-execute-async - client.indices().forceMergeAsync(request, listener); // <1> + client.indices().forceMergeAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::force-merge-execute-async } { // tag::force-merge-notfound try { ForceMergeRequest request = new ForceMergeRequest("does_not_exist"); - client.indices().forceMerge(request); + client.indices().forceMerge(request, RequestOptions.DEFAULT); } catch (ElasticsearchException exception) { if (exception.status() == RestStatus.NOT_FOUND) { // <1> @@ -1219,7 +1222,7 @@ public void testClearCache() throws Exception { // end::clear-cache-request-fields // tag::clear-cache-execute - ClearIndicesCacheResponse clearCacheResponse = client.indices().clearCache(request); + ClearIndicesCacheResponse clearCacheResponse = client.indices().clearCache(request, RequestOptions.DEFAULT); // end::clear-cache-execute // tag::clear-cache-response @@ -1248,7 +1251,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::clear-cache-execute-async - client.indices().clearCacheAsync(request, listener); // <1> + client.indices().clearCacheAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::clear-cache-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -1258,7 +1261,7 @@ public void onFailure(Exception e) { // tag::clear-cache-notfound try { ClearIndicesCacheRequest request = new ClearIndicesCacheRequest("does_not_exist"); - client.indices().clearCache(request); + client.indices().clearCache(request, RequestOptions.DEFAULT); } catch (ElasticsearchException exception) { if (exception.status() == RestStatus.NOT_FOUND) { // <1> @@ -1272,7 +1275,7 @@ public void testCloseIndex() throws Exception { RestHighLevelClient client = highLevelClient(); { - CreateIndexResponse createIndexResponse = client.indices().create(new CreateIndexRequest("index")); + CreateIndexResponse createIndexResponse = client.indices().create(new CreateIndexRequest("index"), RequestOptions.DEFAULT); assertTrue(createIndexResponse.isAcknowledged()); } @@ -1295,7 +1298,7 @@ public void testCloseIndex() throws Exception { // end::close-index-request-indicesOptions // tag::close-index-execute - CloseIndexResponse closeIndexResponse = client.indices().close(request); + CloseIndexResponse closeIndexResponse = client.indices().close(request, RequestOptions.DEFAULT); // end::close-index-execute // tag::close-index-response @@ -1323,7 +1326,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::close-index-execute-async - client.indices().closeAsync(request, listener); // <1> + client.indices().closeAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::close-index-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -1335,7 +1338,7 @@ public void testExistsAlias() throws Exception { { CreateIndexResponse createIndexResponse = client.indices().create(new CreateIndexRequest("index") - .alias(new Alias("alias"))); + .alias(new Alias("alias")), RequestOptions.DEFAULT); assertTrue(createIndexResponse.isAcknowledged()); } @@ -1363,7 +1366,7 @@ public void testExistsAlias() throws Exception { // end::exists-alias-request-local // tag::exists-alias-execute - boolean exists = client.indices().existsAlias(request); + boolean exists = client.indices().existsAlias(request, RequestOptions.DEFAULT); // end::exists-alias-execute assertTrue(exists); @@ -1386,7 +1389,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::exists-alias-execute-async - client.indices().existsAliasAsync(request, listener); // <1> + client.indices().existsAliasAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::exists-alias-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -1397,13 +1400,13 @@ public void testUpdateAliases() throws Exception { RestHighLevelClient client = highLevelClient(); { - CreateIndexResponse createIndexResponse = client.indices().create(new CreateIndexRequest("index1")); + CreateIndexResponse createIndexResponse = client.indices().create(new CreateIndexRequest("index1"), RequestOptions.DEFAULT); assertTrue(createIndexResponse.isAcknowledged()); - createIndexResponse = client.indices().create(new CreateIndexRequest("index2")); + createIndexResponse = client.indices().create(new CreateIndexRequest("index2"), RequestOptions.DEFAULT); assertTrue(createIndexResponse.isAcknowledged()); - createIndexResponse = client.indices().create(new CreateIndexRequest("index3")); + createIndexResponse = client.indices().create(new CreateIndexRequest("index3"), RequestOptions.DEFAULT); assertTrue(createIndexResponse.isAcknowledged()); - createIndexResponse = client.indices().create(new CreateIndexRequest("index4")); + createIndexResponse = client.indices().create(new CreateIndexRequest("index4"), RequestOptions.DEFAULT); assertTrue(createIndexResponse.isAcknowledged()); } @@ -1448,7 +1451,7 @@ public void testUpdateAliases() throws Exception { // tag::update-aliases-execute IndicesAliasesResponse indicesAliasesResponse = - client.indices().updateAliases(request); + client.indices().updateAliases(request, RequestOptions.DEFAULT); // end::update-aliases-execute // tag::update-aliases-response @@ -1482,7 +1485,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::update-aliases-execute-async - client.indices().updateAliasesAsync(request, listener); // <1> + client.indices().updateAliasesAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::update-aliases-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -1527,7 +1530,7 @@ public void testShrinkIndex() throws Exception { // end::shrink-index-request-aliases // tag::shrink-index-execute - ResizeResponse resizeResponse = client.indices().shrink(request); + ResizeResponse resizeResponse = client.indices().shrink(request, RequestOptions.DEFAULT); // end::shrink-index-execute // tag::shrink-index-response @@ -1556,7 +1559,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::shrink-index-execute-async - client.indices().shrinkAsync(request, listener); // <1> + client.indices().shrinkAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::shrink-index-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -1597,7 +1600,7 @@ public void testSplitIndex() throws Exception { // end::split-index-request-aliases // tag::split-index-execute - ResizeResponse resizeResponse = client.indices().split(request); + ResizeResponse resizeResponse = client.indices().split(request, RequestOptions.DEFAULT); // end::split-index-execute // tag::split-index-response @@ -1626,7 +1629,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::split-index-execute-async - client.indices().splitAsync(request,listener); // <1> + client.indices().splitAsync(request, RequestOptions.DEFAULT,listener); // <1> // end::split-index-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -1636,7 +1639,7 @@ public void testRolloverIndex() throws Exception { RestHighLevelClient client = highLevelClient(); { - client.indices().create(new CreateIndexRequest("index-1").alias(new Alias("alias"))); + client.indices().create(new CreateIndexRequest("index-1").alias(new Alias("alias")), RequestOptions.DEFAULT); } // tag::rollover-request @@ -1673,7 +1676,7 @@ public void testRolloverIndex() throws Exception { // end::rollover-request-alias // tag::rollover-execute - RolloverResponse rolloverResponse = client.indices().rollover(request); + RolloverResponse rolloverResponse = client.indices().rollover(request, RequestOptions.DEFAULT); // end::rollover-execute // tag::rollover-response @@ -1712,7 +1715,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::rollover-execute-async - client.indices().rolloverAsync(request,listener); // <1> + client.indices().rolloverAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::rollover-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -1722,7 +1725,7 @@ public void testIndexPutSettings() throws Exception { RestHighLevelClient client = highLevelClient(); { - CreateIndexResponse createIndexResponse = client.indices().create(new CreateIndexRequest("index")); + CreateIndexResponse createIndexResponse = client.indices().create(new CreateIndexRequest("index"), RequestOptions.DEFAULT); assertTrue(createIndexResponse.isAcknowledged()); } @@ -1785,7 +1788,7 @@ public void testIndexPutSettings() throws Exception { // tag::put-settings-execute UpdateSettingsResponse updateSettingsResponse = - client.indices().putSettings(request); + client.indices().putSettings(request, RequestOptions.DEFAULT); // end::put-settings-execute // tag::put-settings-response @@ -1814,7 +1817,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::put-settings-execute-async - client.indices().putSettingsAsync(request,listener); // <1> + client.indices().putSettingsAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::put-settings-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -1849,7 +1852,7 @@ public void testPutTemplate() throws Exception { "}", // <2> XContentType.JSON); // end::put-template-request-mappings-json - assertTrue(client.indices().putTemplate(request).isAcknowledged()); + assertTrue(client.indices().putTemplate(request, RequestOptions.DEFAULT).isAcknowledged()); } { //tag::put-template-request-mappings-map @@ -1863,7 +1866,7 @@ public void testPutTemplate() throws Exception { jsonMap.put("tweet", tweet); request.mapping("tweet", jsonMap); // <1> //end::put-template-request-mappings-map - assertTrue(client.indices().putTemplate(request).isAcknowledged()); + assertTrue(client.indices().putTemplate(request, RequestOptions.DEFAULT).isAcknowledged()); } { //tag::put-template-request-mappings-xcontent @@ -1887,13 +1890,13 @@ public void testPutTemplate() throws Exception { builder.endObject(); request.mapping("tweet", builder); // <1> //end::put-template-request-mappings-xcontent - assertTrue(client.indices().putTemplate(request).isAcknowledged()); + assertTrue(client.indices().putTemplate(request, RequestOptions.DEFAULT).isAcknowledged()); } { //tag::put-template-request-mappings-shortcut request.mapping("tweet", "message", "type=text"); // <1> //end::put-template-request-mappings-shortcut - assertTrue(client.indices().putTemplate(request).isAcknowledged()); + assertTrue(client.indices().putTemplate(request, RequestOptions.DEFAULT).isAcknowledged()); } // tag::put-template-request-aliases @@ -1947,7 +1950,7 @@ public void testPutTemplate() throws Exception { request.create(false); // make test happy // tag::put-template-execute - PutIndexTemplateResponse putTemplateResponse = client.indices().putTemplate(request); + PutIndexTemplateResponse putTemplateResponse = client.indices().putTemplate(request, RequestOptions.DEFAULT); // end::put-template-execute // tag::put-template-response @@ -1975,7 +1978,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::put-template-execute-async - client.indices().putTemplateAsync(request, listener); // <1> + client.indices().putTemplateAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::put-template-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java index 7971e49da44f4..f5bdc9f2f3ee5 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java @@ -27,6 +27,7 @@ import org.elasticsearch.action.ingest.PutPipelineRequest; import org.elasticsearch.action.ingest.WritePipelineResponse; import org.elasticsearch.client.ESRestHighLevelClientTestCase; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.unit.TimeValue; @@ -86,7 +87,7 @@ public void testPutPipeline() throws IOException { // end::put-pipeline-request-masterTimeout // tag::put-pipeline-execute - WritePipelineResponse response = client.ingest().putPipeline(request); // <1> + WritePipelineResponse response = client.ingest().putPipeline(request, RequestOptions.DEFAULT); // <1> // end::put-pipeline-execute // tag::put-pipeline-response @@ -129,7 +130,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::put-pipeline-execute-async - client.ingest().putPipelineAsync(request, listener); // <1> + client.ingest().putPipelineAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::put-pipeline-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -154,7 +155,7 @@ public void testGetPipeline() throws IOException { // end::get-pipeline-request-masterTimeout // tag::get-pipeline-execute - GetPipelineResponse response = client.ingest().getPipeline(request); // <1> + GetPipelineResponse response = client.ingest().getPipeline(request, RequestOptions.DEFAULT); // <1> // end::get-pipeline-execute // tag::get-pipeline-response @@ -199,7 +200,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::get-pipeline-execute-async - client.ingest().getPipelineAsync(request, listener); // <1> + client.ingest().getPipelineAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::get-pipeline-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -229,7 +230,7 @@ public void testDeletePipeline() throws IOException { // end::delete-pipeline-request-masterTimeout // tag::delete-pipeline-execute - WritePipelineResponse response = client.ingest().deletePipeline(request); // <1> + WritePipelineResponse response = client.ingest().deletePipeline(request, RequestOptions.DEFAULT); // <1> // end::delete-pipeline-execute // tag::delete-pipeline-response @@ -269,7 +270,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::delete-pipeline-execute-async - client.ingest().deletePipelineAsync(request, listener); // <1> + client.ingest().deletePipelineAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::delete-pipeline-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MigrationDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MigrationDocumentationIT.java index 489d4d9b1ed5f..b56fb3359ffae 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MigrationDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MigrationDocumentationIT.java @@ -19,10 +19,6 @@ package org.elasticsearch.client.documentation; -import org.apache.http.HttpEntity; -import org.apache.http.HttpStatus; -import org.apache.http.entity.ContentType; -import org.apache.http.nio.entity.NStringEntity; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteResponse; @@ -31,12 +27,10 @@ import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.client.ESRestHighLevelClientTestCase; import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.RestStatus; @@ -45,11 +39,6 @@ import java.io.InputStream; import java.util.Map; -import static java.util.Collections.emptyMap; -import static java.util.Collections.singletonMap; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; - /** * This class is used to generate the documentation for the * docs/java-rest/high-level/migration.asciidoc page. @@ -98,14 +87,14 @@ public void testRequests() throws Exception { //end::migration-request-ctor //tag::migration-request-ctor-execution - IndexResponse response = client.index(request); + IndexResponse response = client.index(request, RequestOptions.DEFAULT); //end::migration-request-ctor-execution assertEquals(RestStatus.CREATED, response.status()); } { //tag::migration-request-async-execution DeleteRequest request = new DeleteRequest("index", "doc", "id"); // <1> - client.deleteAsync(request, new ActionListener() { // <2> + client.deleteAsync(request, RequestOptions.DEFAULT, new ActionListener() { // <2> @Override public void onResponse(DeleteResponse deleteResponse) { // <3> @@ -117,12 +106,12 @@ public void onFailure(Exception e) { } }); //end::migration-request-async-execution - assertBusy(() -> assertFalse(client.exists(new GetRequest("index", "doc", "id")))); + assertBusy(() -> assertFalse(client.exists(new GetRequest("index", "doc", "id"), RequestOptions.DEFAULT))); } { //tag::migration-request-sync-execution DeleteRequest request = new DeleteRequest("index", "doc", "id"); - DeleteResponse response = client.delete(request); // <1> + DeleteResponse response = client.delete(request, RequestOptions.DEFAULT); // <1> //end::migration-request-sync-execution assertEquals(RestStatus.NOT_FOUND, response.status()); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MiscellaneousDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MiscellaneousDocumentationIT.java index 504ea797c35f6..2186bd8ebfd30 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MiscellaneousDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MiscellaneousDocumentationIT.java @@ -24,6 +24,7 @@ import org.elasticsearch.Version; import org.elasticsearch.action.main.MainResponse; import org.elasticsearch.client.ESRestHighLevelClientTestCase; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.cluster.ClusterName; @@ -40,7 +41,7 @@ public void testMain() throws IOException { RestHighLevelClient client = highLevelClient(); { //tag::main-execute - MainResponse response = client.info(); + MainResponse response = client.info(RequestOptions.DEFAULT); //end::main-execute //tag::main-response ClusterName clusterName = response.getClusterName(); // <1> @@ -60,7 +61,7 @@ public void testMain() throws IOException { public void testPing() throws IOException { RestHighLevelClient client = highLevelClient(); //tag::ping-execute - boolean response = client.ping(); + boolean response = client.ping(RequestOptions.DEFAULT); //end::ping-execute assertTrue(response); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java index 463c5f7d12f5e..cf6409bab6444 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java @@ -42,6 +42,7 @@ import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.ESRestHighLevelClientTestCase; import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestHighLevelClient; @@ -143,7 +144,7 @@ public void testSearch() throws Exception { // tag::search-request-preference searchRequest.preference("_local"); // <1> // end::search-request-preference - assertNotNull(client.search(searchRequest)); + assertNotNull(client.search(searchRequest, RequestOptions.DEFAULT)); } { // tag::search-source-basics @@ -176,7 +177,7 @@ public void testSearch() throws Exception { // end::search-source-setter // tag::search-execute - SearchResponse searchResponse = client.search(searchRequest); + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); // end::search-execute // tag::search-execute-listener @@ -198,7 +199,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::search-execute-async - client.searchAsync(searchRequest, listener); // <1> + client.searchAsync(searchRequest, RequestOptions.DEFAULT, listener); // <1> // end::search-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -296,7 +297,7 @@ public void testSearchRequestAggregations() throws IOException { request.add(new IndexRequest("posts", "doc", "3") .source(XContentType.JSON, "company", "Elastic", "age", 40)); request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - BulkResponse bulkResponse = client.bulk(request); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); assertSame(RestStatus.OK, bulkResponse.status()); assertFalse(bulkResponse.hasFailures()); } @@ -312,7 +313,7 @@ public void testSearchRequestAggregations() throws IOException { // end::search-request-aggregations searchSourceBuilder.query(QueryBuilders.matchAllQuery()); searchRequest.source(searchSourceBuilder); - SearchResponse searchResponse = client.search(searchRequest); + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); { // tag::search-request-aggregations-get Aggregations aggregations = searchResponse.getAggregations(); @@ -369,7 +370,7 @@ public void testSearchRequestSuggestions() throws IOException { request.add(new IndexRequest("posts", "doc", "3").source(XContentType.JSON, "user", "tlrx")); request.add(new IndexRequest("posts", "doc", "4").source(XContentType.JSON, "user", "cbuescher")); request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - BulkResponse bulkResponse = client.bulk(request); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); assertSame(RestStatus.OK, bulkResponse.status()); assertFalse(bulkResponse.hasFailures()); } @@ -384,7 +385,7 @@ public void testSearchRequestSuggestions() throws IOException { searchSourceBuilder.suggest(suggestBuilder); // end::search-request-suggestion searchRequest.source(searchSourceBuilder); - SearchResponse searchResponse = client.search(searchRequest); + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); { // tag::search-request-suggestion-get Suggest suggest = searchResponse.getSuggest(); // <1> @@ -416,7 +417,7 @@ public void testSearchRequestHighlighting() throws IOException { .source(XContentType.JSON, "title", "The Future of Federated Search in Elasticsearch", "user", Arrays.asList("kimchy", "tanguy"), "innerObject", Collections.singletonMap("key", "value"))); request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - BulkResponse bulkResponse = client.bulk(request); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); assertSame(RestStatus.OK, bulkResponse.status()); assertFalse(bulkResponse.hasFailures()); } @@ -437,7 +438,7 @@ public void testSearchRequestHighlighting() throws IOException { .should(matchQuery("title", "Elasticsearch")) .should(matchQuery("user", "kimchy"))); searchRequest.source(searchSourceBuilder); - SearchResponse searchResponse = client.search(searchRequest); + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); { // tag::search-request-highlighting-get SearchHits hits = searchResponse.getHits(); @@ -472,7 +473,7 @@ public void testSearchRequestProfiling() throws IOException { IndexRequest request = new IndexRequest("posts", "doc", "1") .source(XContentType.JSON, "tags", "elasticsearch", "comments", 123); request.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); - IndexResponse indexResponse = client.index(request); + IndexResponse indexResponse = client.index(request, RequestOptions.DEFAULT); assertSame(RestStatus.CREATED, indexResponse.status()); } { @@ -485,7 +486,7 @@ public void testSearchRequestProfiling() throws IOException { searchSourceBuilder.aggregation(AggregationBuilders.histogram("by_comments").field("comments").interval(100)); searchRequest.source(searchSourceBuilder); - SearchResponse searchResponse = client.search(searchRequest); + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); // tag::search-request-profiling-get Map profilingResults = searchResponse.getProfileResults(); // <1> @@ -548,7 +549,7 @@ public void testScroll() throws Exception { request.add(new IndexRequest("posts", "doc", "3") .source(XContentType.JSON, "title", "The Future of Federated Search in Elasticsearch")); request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - BulkResponse bulkResponse = client.bulk(request); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); assertSame(RestStatus.OK, bulkResponse.status()); assertFalse(bulkResponse.hasFailures()); } @@ -561,7 +562,7 @@ public void testScroll() throws Exception { searchSourceBuilder.size(size); // <1> searchRequest.source(searchSourceBuilder); searchRequest.scroll(TimeValue.timeValueMinutes(1L)); // <2> - SearchResponse searchResponse = client.search(searchRequest); + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); String scrollId = searchResponse.getScrollId(); // <3> SearchHits hits = searchResponse.getHits(); // <4> // end::search-scroll-init @@ -572,7 +573,7 @@ public void testScroll() throws Exception { // tag::search-scroll2 SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); // <1> scrollRequest.scroll(TimeValue.timeValueSeconds(30)); - SearchResponse searchScrollResponse = client.searchScroll(scrollRequest); + SearchResponse searchScrollResponse = client.searchScroll(scrollRequest, RequestOptions.DEFAULT); scrollId = searchScrollResponse.getScrollId(); // <2> hits = searchScrollResponse.getHits(); // <3> assertEquals(3, hits.getTotalHits()); @@ -582,14 +583,14 @@ public void testScroll() throws Exception { ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); clearScrollRequest.addScrollId(scrollId); - ClearScrollResponse clearScrollResponse = client.clearScroll(clearScrollRequest); + ClearScrollResponse clearScrollResponse = client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); assertTrue(clearScrollResponse.isSucceeded()); } { SearchRequest searchRequest = new SearchRequest(); searchRequest.scroll("60s"); - SearchResponse initialSearchResponse = client.search(searchRequest); + SearchResponse initialSearchResponse = client.search(searchRequest, RequestOptions.DEFAULT); String scrollId = initialSearchResponse.getScrollId(); SearchScrollRequest scrollRequest = new SearchScrollRequest(); @@ -601,7 +602,7 @@ public void testScroll() throws Exception { // end::scroll-request-arguments // tag::search-scroll-execute-sync - SearchResponse searchResponse = client.searchScroll(scrollRequest); + SearchResponse searchResponse = client.searchScroll(scrollRequest, RequestOptions.DEFAULT); // end::search-scroll-execute-sync assertEquals(0, searchResponse.getFailedShards()); @@ -648,7 +649,7 @@ public void onFailure(Exception e) { // end::clear-scroll-add-scroll-ids // tag::clear-scroll-execute - ClearScrollResponse response = client.clearScroll(request); + ClearScrollResponse response = client.clearScroll(request, RequestOptions.DEFAULT); // end::clear-scroll-execute // tag::clear-scroll-response @@ -678,7 +679,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, clearScrollLatch); // tag::clear-scroll-execute-async - client.clearScrollAsync(request, listener); // <1> + client.clearScrollAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::clear-scroll-execute-async assertTrue(clearScrollLatch.await(30L, TimeUnit.SECONDS)); @@ -692,14 +693,14 @@ public void onFailure(Exception e) { searchSourceBuilder.query(matchQuery("title", "Elasticsearch")); searchRequest.source(searchSourceBuilder); - SearchResponse searchResponse = client.search(searchRequest); // <1> + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); // <1> String scrollId = searchResponse.getScrollId(); SearchHit[] searchHits = searchResponse.getHits().getHits(); while (searchHits != null && searchHits.length > 0) { // <2> SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); // <3> scrollRequest.scroll(scroll); - searchResponse = client.searchScroll(scrollRequest); + searchResponse = client.searchScroll(scrollRequest, RequestOptions.DEFAULT); scrollId = searchResponse.getScrollId(); searchHits = searchResponse.getHits().getHits(); // <4> @@ -707,7 +708,7 @@ public void onFailure(Exception e) { ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); // <5> clearScrollRequest.addScrollId(scrollId); - ClearScrollResponse clearScrollResponse = client.clearScroll(clearScrollRequest); + ClearScrollResponse clearScrollResponse = client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); boolean succeeded = clearScrollResponse.isSucceeded(); // end::search-scroll-example assertTrue(succeeded); @@ -737,7 +738,7 @@ public void testSearchTemplateWithInlineScript() throws Exception { // end::search-template-request-inline // tag::search-template-response - SearchTemplateResponse response = client.searchTemplate(request); + SearchTemplateResponse response = client.searchTemplate(request, RequestOptions.DEFAULT); SearchResponse searchResponse = response.getResponse(); // end::search-template-response @@ -749,7 +750,7 @@ public void testSearchTemplateWithInlineScript() throws Exception { // end::render-search-template-request // tag::render-search-template-response - SearchTemplateResponse renderResponse = client.searchTemplate(request); + SearchTemplateResponse renderResponse = client.searchTemplate(request, RequestOptions.DEFAULT); BytesReference source = renderResponse.getSource(); // <1> // end::render-search-template-response @@ -802,7 +803,7 @@ public void testSearchTemplateWithStoredScript() throws Exception { // end::search-template-request-options // tag::search-template-execute - SearchTemplateResponse response = client.searchTemplate(request); + SearchTemplateResponse response = client.searchTemplate(request, RequestOptions.DEFAULT); // end::search-template-execute SearchResponse searchResponse = response.getResponse(); @@ -828,7 +829,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::search-template-execute-async - client.searchTemplateAsync(request, listener); // <1> + client.searchTemplateAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::search-template-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -849,7 +850,7 @@ public void testFieldCaps() throws Exception { // end::field-caps-request-indicesOptions // tag::field-caps-execute - FieldCapabilitiesResponse response = client.fieldCaps(request); + FieldCapabilitiesResponse response = client.fieldCaps(request, RequestOptions.DEFAULT); // end::field-caps-execute // tag::field-caps-response @@ -892,7 +893,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::field-caps-execute-async - client.fieldCapsAsync(request, listener); // <1> + client.fieldCapsAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::field-caps-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -918,7 +919,7 @@ public void testRankEval() throws Exception { // end::rank-eval-request-basic // tag::rank-eval-execute - RankEvalResponse response = client.rankEval(request); + RankEvalResponse response = client.rankEval(request, RequestOptions.DEFAULT); // end::rank-eval-execute // tag::rank-eval-response @@ -962,7 +963,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::rank-eval-execute-async - client.rankEvalAsync(request, listener); // <1> + client.rankEvalAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::rank-eval-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -987,7 +988,7 @@ public void testMultiSearch() throws Exception { request.add(secondSearchRequest); // end::multi-search-request-basic // tag::multi-search-execute - MultiSearchResponse response = client.multiSearch(request); + MultiSearchResponse response = client.multiSearch(request, RequestOptions.DEFAULT); // end::multi-search-execute // tag::multi-search-response MultiSearchResponse.Item firstResponse = response.getResponses()[0]; // <1> @@ -1019,7 +1020,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::multi-search-execute-async - client.multiSearchAsync(request, listener); // <1> + client.multiSearchAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::multi-search-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -1030,7 +1031,7 @@ public void onFailure(Exception e) { request.add(new SearchRequest("posts") // <1> .types("doc")); // <2> // end::multi-search-request-index - MultiSearchResponse response = client.multiSearch(request); + MultiSearchResponse response = client.multiSearch(request, RequestOptions.DEFAULT); MultiSearchResponse.Item firstResponse = response.getResponses()[0]; assertNull(firstResponse.getFailure()); SearchResponse searchResponse = firstResponse.getResponse(); @@ -1041,12 +1042,12 @@ public void onFailure(Exception e) { private void indexSearchTestData() throws IOException { CreateIndexRequest authorsRequest = new CreateIndexRequest("authors") .mapping("doc", "user", "type=keyword,doc_values=false"); - CreateIndexResponse authorsResponse = highLevelClient().indices().create(authorsRequest); + CreateIndexResponse authorsResponse = highLevelClient().indices().create(authorsRequest, RequestOptions.DEFAULT); assertTrue(authorsResponse.isAcknowledged()); CreateIndexRequest reviewersRequest = new CreateIndexRequest("contributors") .mapping("doc", "user", "type=keyword"); - CreateIndexResponse reviewersResponse = highLevelClient().indices().create(reviewersRequest); + CreateIndexResponse reviewersResponse = highLevelClient().indices().create(reviewersRequest, RequestOptions.DEFAULT); assertTrue(reviewersResponse.isAcknowledged()); BulkRequest bulkRequest = new BulkRequest(); @@ -1067,7 +1068,7 @@ private void indexSearchTestData() throws IOException { bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - BulkResponse bulkResponse = highLevelClient().bulk(bulkRequest); + BulkResponse bulkResponse = highLevelClient().bulk(bulkRequest, RequestOptions.DEFAULT); assertSame(RestStatus.OK, bulkResponse.status()); assertFalse(bulkResponse.hasFailures()); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java index 2890ad50c2666..8c158a91a5111 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java @@ -30,6 +30,7 @@ import org.elasticsearch.action.admin.cluster.repositories.verify.VerifyRepositoryRequest; import org.elasticsearch.action.admin.cluster.repositories.verify.VerifyRepositoryResponse; import org.elasticsearch.client.ESRestHighLevelClientTestCase; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.cluster.metadata.RepositoryMetaData; import org.elasticsearch.common.settings.Settings; @@ -134,7 +135,7 @@ public void testSnapshotCreateRepository() throws IOException { // end::create-repository-request-verify // tag::create-repository-execute - PutRepositoryResponse response = client.snapshot().createRepository(request); + PutRepositoryResponse response = client.snapshot().createRepository(request, RequestOptions.DEFAULT); // end::create-repository-execute // tag::create-repository-response @@ -168,7 +169,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::create-repository-execute-async - client.snapshot().createRepositoryAsync(request, listener); // <1> + client.snapshot().createRepositoryAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::create-repository-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -197,7 +198,7 @@ public void testSnapshotGetRepository() throws IOException { // end::get-repository-request-masterTimeout // tag::get-repository-execute - GetRepositoriesResponse response = client.snapshot().getRepositories(request); + GetRepositoriesResponse response = client.snapshot().getRepositories(request, RequestOptions.DEFAULT); // end::get-repository-execute // tag::get-repository-response @@ -232,7 +233,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::get-repository-execute-async - client.snapshot().getRepositoriesAsync(request, listener); // <1> + client.snapshot().getRepositoriesAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::get-repository-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -258,7 +259,7 @@ public void testSnapshotDeleteRepository() throws IOException { // end::delete-repository-request-timeout // tag::delete-repository-execute - DeleteRepositoryResponse response = client.snapshot().deleteRepository(request); + DeleteRepositoryResponse response = client.snapshot().deleteRepository(request, RequestOptions.DEFAULT); // end::delete-repository-execute // tag::delete-repository-response @@ -292,7 +293,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::delete-repository-execute-async - client.snapshot().deleteRepositoryAsync(request, listener); // <1> + client.snapshot().deleteRepositoryAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::delete-repository-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -317,7 +318,7 @@ public void testSnapshotVerifyRepository() throws IOException { // end::verify-repository-request-timeout // tag::verify-repository-execute - VerifyRepositoryResponse response = client.snapshot().verifyRepository(request); + VerifyRepositoryResponse response = client.snapshot().verifyRepository(request, RequestOptions.DEFAULT); // end::verify-repository-execute // tag::verify-repository-response @@ -352,7 +353,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::verify-repository-execute-async - client.snapshot().verifyRepositoryAsync(request, listener); // <1> + client.snapshot().verifyRepositoryAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::verify-repository-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -363,6 +364,6 @@ private void createTestRepositories() throws IOException { PutRepositoryRequest request = new PutRepositoryRequest(repositoryName); request.type(FsRepository.TYPE); request.settings("{\"location\": \".\"}", XContentType.JSON); - assertTrue(highLevelClient().snapshot().createRepository(request).isAcknowledged()); + assertTrue(highLevelClient().snapshot().createRepository(request, RequestOptions.DEFAULT).isAcknowledged()); } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/TasksClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/TasksClientDocumentationIT.java index faf447a4143b1..0d62a2d29a03b 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/TasksClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/TasksClientDocumentationIT.java @@ -27,6 +27,7 @@ import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskGroup; import org.elasticsearch.client.ESRestHighLevelClientTestCase; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.tasks.TaskId; @@ -90,7 +91,7 @@ public void testListTasks() throws IOException { ListTasksRequest request = new ListTasksRequest(); // tag::list-tasks-execute - ListTasksResponse response = client.tasks().list(request); + ListTasksResponse response = client.tasks().list(request, RequestOptions.DEFAULT); // end::list-tasks-execute assertThat(response, notNullValue()); @@ -139,7 +140,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::list-tasks-execute-async - client.tasks().listAsync(request, listener); // <1> + client.tasks().listAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::list-tasks-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); diff --git a/qa/ccs-unavailable-clusters/src/test/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java b/qa/ccs-unavailable-clusters/src/test/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java index f7b87905b24d5..73df782c92049 100644 --- a/qa/ccs-unavailable-clusters/src/test/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java +++ b/qa/ccs-unavailable-clusters/src/test/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java @@ -36,6 +36,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequest; import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; @@ -133,19 +134,20 @@ public void testSearchSkipUnavailable() throws IOException { updateRemoteClusterSettings(Collections.singletonMap("seeds", remoteNode.getAddress().toString())); for (int i = 0; i < 10; i++) { - restHighLevelClient.index(new IndexRequest("index", "doc", String.valueOf(i)).source("field", "value")); + restHighLevelClient.index( + new IndexRequest("index", "doc", String.valueOf(i)).source("field", "value"), RequestOptions.DEFAULT); } Response refreshResponse = client().performRequest(new Request("POST", "/index/_refresh")); assertEquals(200, refreshResponse.getStatusLine().getStatusCode()); { - SearchResponse response = restHighLevelClient.search(new SearchRequest("index")); + SearchResponse response = restHighLevelClient.search(new SearchRequest("index"), RequestOptions.DEFAULT); assertSame(SearchResponse.Clusters.EMPTY, response.getClusters()); assertEquals(10, response.getHits().totalHits); assertEquals(10, response.getHits().getHits().length); } { - SearchResponse response = restHighLevelClient.search(new SearchRequest("index", "remote1:index")); + SearchResponse response = restHighLevelClient.search(new SearchRequest("index", "remote1:index"), RequestOptions.DEFAULT); assertEquals(2, response.getClusters().getTotal()); assertEquals(2, response.getClusters().getSuccessful()); assertEquals(0, response.getClusters().getSkipped()); @@ -153,7 +155,7 @@ public void testSearchSkipUnavailable() throws IOException { assertEquals(10, response.getHits().getHits().length); } { - SearchResponse response = restHighLevelClient.search(new SearchRequest("remote1:index")); + SearchResponse response = restHighLevelClient.search(new SearchRequest("remote1:index"), RequestOptions.DEFAULT); assertEquals(1, response.getClusters().getTotal()); assertEquals(1, response.getClusters().getSuccessful()); assertEquals(0, response.getClusters().getSkipped()); @@ -161,14 +163,15 @@ public void testSearchSkipUnavailable() throws IOException { } { - SearchResponse response = restHighLevelClient.search(new SearchRequest("index", "remote1:index").scroll("1m")); + SearchResponse response = restHighLevelClient.search(new SearchRequest("index", "remote1:index").scroll("1m"), + RequestOptions.DEFAULT); assertEquals(2, response.getClusters().getTotal()); assertEquals(2, response.getClusters().getSuccessful()); assertEquals(0, response.getClusters().getSkipped()); assertEquals(10, response.getHits().totalHits); assertEquals(10, response.getHits().getHits().length); String scrollId = response.getScrollId(); - SearchResponse scrollResponse = restHighLevelClient.searchScroll(new SearchScrollRequest(scrollId)); + SearchResponse scrollResponse = restHighLevelClient.searchScroll(new SearchScrollRequest(scrollId), RequestOptions.DEFAULT); assertSame(SearchResponse.Clusters.EMPTY, scrollResponse.getClusters()); assertEquals(10, scrollResponse.getHits().totalHits); assertEquals(0, scrollResponse.getHits().getHits().length); @@ -179,7 +182,7 @@ public void testSearchSkipUnavailable() throws IOException { updateRemoteClusterSettings(Collections.singletonMap("skip_unavailable", true)); { - SearchResponse response = restHighLevelClient.search(new SearchRequest("index", "remote1:index")); + SearchResponse response = restHighLevelClient.search(new SearchRequest("index", "remote1:index"), RequestOptions.DEFAULT); assertEquals(2, response.getClusters().getTotal()); assertEquals(1, response.getClusters().getSuccessful()); assertEquals(1, response.getClusters().getSkipped()); @@ -187,7 +190,7 @@ public void testSearchSkipUnavailable() throws IOException { assertEquals(10, response.getHits().getHits().length); } { - SearchResponse response = restHighLevelClient.search(new SearchRequest("remote1:index")); + SearchResponse response = restHighLevelClient.search(new SearchRequest("remote1:index"), RequestOptions.DEFAULT); assertEquals(1, response.getClusters().getTotal()); assertEquals(0, response.getClusters().getSuccessful()); assertEquals(1, response.getClusters().getSkipped()); @@ -195,14 +198,15 @@ public void testSearchSkipUnavailable() throws IOException { } { - SearchResponse response = restHighLevelClient.search(new SearchRequest("index", "remote1:index").scroll("1m")); + SearchResponse response = restHighLevelClient.search(new SearchRequest("index", "remote1:index").scroll("1m"), + RequestOptions.DEFAULT); assertEquals(2, response.getClusters().getTotal()); assertEquals(1, response.getClusters().getSuccessful()); assertEquals(1, response.getClusters().getSkipped()); assertEquals(10, response.getHits().totalHits); assertEquals(10, response.getHits().getHits().length); String scrollId = response.getScrollId(); - SearchResponse scrollResponse = restHighLevelClient.searchScroll(new SearchScrollRequest(scrollId)); + SearchResponse scrollResponse = restHighLevelClient.searchScroll(new SearchScrollRequest(scrollId), RequestOptions.DEFAULT); assertSame(SearchResponse.Clusters.EMPTY, scrollResponse.getClusters()); assertEquals(10, scrollResponse.getHits().totalHits); assertEquals(0, scrollResponse.getHits().getHits().length); @@ -266,19 +270,19 @@ public void testSkipUnavailableDependsOnSeeds() throws IOException { private static void assertSearchConnectFailure() { { ElasticsearchException exception = expectThrows(ElasticsearchException.class, - () -> restHighLevelClient.search(new SearchRequest("index", "remote1:index"))); + () -> restHighLevelClient.search(new SearchRequest("index", "remote1:index"), RequestOptions.DEFAULT)); ElasticsearchException rootCause = (ElasticsearchException)exception.getRootCause(); assertThat(rootCause.getMessage(), containsString("connect_exception")); } { ElasticsearchException exception = expectThrows(ElasticsearchException.class, - () -> restHighLevelClient.search(new SearchRequest("remote1:index"))); + () -> restHighLevelClient.search(new SearchRequest("remote1:index"), RequestOptions.DEFAULT)); ElasticsearchException rootCause = (ElasticsearchException)exception.getRootCause(); assertThat(rootCause.getMessage(), containsString("connect_exception")); } { ElasticsearchException exception = expectThrows(ElasticsearchException.class, - () -> restHighLevelClient.search(new SearchRequest("remote1:index").scroll("1m"))); + () -> restHighLevelClient.search(new SearchRequest("remote1:index").scroll("1m"), RequestOptions.DEFAULT)); ElasticsearchException rootCause = (ElasticsearchException)exception.getRootCause(); assertThat(rootCause.getMessage(), containsString("connect_exception")); } From 4158387554ff2ff18fd0fbb10f8e6f2de55d35c8 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Wed, 6 Jun 2018 15:36:13 -0600 Subject: [PATCH 26/26] Cleanup nio http thread names (#31148) This is related to #28898. This commit adds the acceptor thread name to the method checking if this thread is a transport thread. Additionally, it modifies the nio http transport to use the same worker name as the netty4 http server transport. --- .../java/org/elasticsearch/nio/RoundRobinSupplier.java | 1 + .../org/elasticsearch/http/nio/NioHttpServerTransport.java | 7 ++----- .../java/org/elasticsearch/http/HttpServerTransport.java | 2 ++ .../main/java/org/elasticsearch/transport/Transports.java | 1 + 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/RoundRobinSupplier.java b/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/RoundRobinSupplier.java index 4d4de689fe7f8..c35dca1c9599a 100644 --- a/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/RoundRobinSupplier.java +++ b/libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/RoundRobinSupplier.java @@ -38,6 +38,7 @@ final class RoundRobinSupplier implements Supplier { this.selectorsSet.set(true); } + @Override public S get() { S[] selectors = this.selectors; return selectors[counter.getAndIncrement() % selectors.length]; diff --git a/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/NioHttpServerTransport.java b/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/NioHttpServerTransport.java index 495dbf9729391..076c4ca84d515 100644 --- a/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/NioHttpServerTransport.java +++ b/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/NioHttpServerTransport.java @@ -108,9 +108,6 @@ public class NioHttpServerTransport extends AbstractHttpServerTransport { (s) -> Integer.toString(EsExecutors.numberOfProcessors(s) * 2), (s) -> Setting.parseInt(s, 1, "http.nio.worker_count"), Setting.Property.NodeScope); - private static final String TRANSPORT_WORKER_THREAD_NAME_PREFIX = "http_nio_transport_worker"; - private static final String TRANSPORT_ACCEPTOR_THREAD_NAME_PREFIX = "http_nio_transport_acceptor"; - private final BigArrays bigArrays; private final ThreadPool threadPool; private final NamedXContentRegistry xContentRegistry; @@ -177,8 +174,8 @@ protected void doStart() { try { int acceptorCount = NIO_HTTP_ACCEPTOR_COUNT.get(settings); int workerCount = NIO_HTTP_WORKER_COUNT.get(settings); - nioGroup = new NioGroup(daemonThreadFactory(this.settings, TRANSPORT_ACCEPTOR_THREAD_NAME_PREFIX), acceptorCount, - daemonThreadFactory(this.settings, TRANSPORT_WORKER_THREAD_NAME_PREFIX), workerCount, + nioGroup = new NioGroup(daemonThreadFactory(this.settings, HTTP_SERVER_ACCEPTOR_THREAD_NAME_PREFIX), acceptorCount, + daemonThreadFactory(this.settings, HTTP_SERVER_WORKER_THREAD_NAME_PREFIX), workerCount, (s) -> new EventHandler(this::nonChannelExceptionCaught, s)); channelFactory = new HttpChannelFactory(); this.boundAddress = createBoundHttpAddress(); diff --git a/server/src/main/java/org/elasticsearch/http/HttpServerTransport.java b/server/src/main/java/org/elasticsearch/http/HttpServerTransport.java index de345a39fd6d9..0ce8edcf3b662 100644 --- a/server/src/main/java/org/elasticsearch/http/HttpServerTransport.java +++ b/server/src/main/java/org/elasticsearch/http/HttpServerTransport.java @@ -29,6 +29,8 @@ public interface HttpServerTransport extends LifecycleComponent { String HTTP_SERVER_WORKER_THREAD_NAME_PREFIX = "http_server_worker"; + String HTTP_SERVER_ACCEPTOR_THREAD_NAME_PREFIX = "http_server_acceptor"; + BoundTransportAddress boundAddress(); HttpInfo info(); diff --git a/server/src/main/java/org/elasticsearch/transport/Transports.java b/server/src/main/java/org/elasticsearch/transport/Transports.java index 2adf07bad2356..ddd22a7fb868c 100644 --- a/server/src/main/java/org/elasticsearch/transport/Transports.java +++ b/server/src/main/java/org/elasticsearch/transport/Transports.java @@ -40,6 +40,7 @@ public static final boolean isTransportThread(Thread t) { final String threadName = t.getName(); for (String s : Arrays.asList( HttpServerTransport.HTTP_SERVER_WORKER_THREAD_NAME_PREFIX, + HttpServerTransport.HTTP_SERVER_ACCEPTOR_THREAD_NAME_PREFIX, TcpTransport.TRANSPORT_SERVER_WORKER_THREAD_NAME_PREFIX, TcpTransport.TRANSPORT_CLIENT_BOSS_THREAD_NAME_PREFIX, TEST_MOCK_TRANSPORT_THREAD_PREFIX,