Merge remote-tracking branch 'elastic/master' into avx-512-jdk-10-only

* elastic/master: (36 commits)
  Remove unneded cluster config from test (elastic#40856)
  Make Fuzziness reject illegal values earlier (elastic#33511)
  Remove test-only customisation from TransReplAct (elastic#40863)
  Fix dense/sparse vector limit documentation (elastic#40852)
  Make -try xlint warning disabled by default. (elastic#40833)
  Async Snapshot Repository Deletes (elastic#40144)
  Revert "Replace usages RandomizedTestingTask with built-in Gradle Test (elastic#40564)"
  Init global checkpoint after copy commit in peer recovery (elastic#40823)
  Replace usages RandomizedTestingTask with built-in Gradle Test (elastic#40564)
  [DOCS] Removed redundant (not quite right) information about upgrades.
  Remove string usages of old transport settings (elastic#40818)
  Rollup ignores time_zone on date histogram (elastic#40844)
  HLRC: fix uri encode bug when url path starts with '/' (elastic#34436)
  Mutes GatewayIndexStateIT.testRecoverBrokenIndexMetadata
  Docs: Pin two IDs in the rest client (elastic#40785)
  Adds version 6.7.2
  [DOCS] Remind users to include @ symbol when applying license file (elastic#40688)
  HLRC: Convert xpack methods to client side objects (elastic#40705)
  Allow ILM to stop if indices have nonexistent policies (elastic#40820)
  Add an OpenID Connect authentication realm (elastic#40674)
  ...

benchmarks/build.gradle

@@ -39,7 +39,7 @@ dependencies {
     runtime 'org.apache.commons:commons-math3:3.2'
 }
 
-compileJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked,-processing"
+compileJava.options.compilerArgs << "-Xlint:-cast,-rawtypes,-unchecked,-processing"
 // enable the JMH's BenchmarkProcessor to generate the final benchmark classes
 // needs to be added separately otherwise Gradle will quote it and javac will fail
 compileJava.options.compilerArgs.addAll(["-processor", "org.openjdk.jmh.generators.BenchmarkProcessor"])

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -752,7 +752,7 @@ class BuildPlugin implements Plugin<Project> {
                  */
                 // don't even think about passing args with -J-xxx, oracle will ask you to submit a bug report :)
                 // fail on all javac warnings
-                options.compilerArgs << '-Werror' << '-Xlint:all,-path,-serial,-options,-deprecation' << '-Xdoclint:all' << '-Xdoclint:-missing'
+                options.compilerArgs << '-Werror' << '-Xlint:all,-path,-serial,-options,-deprecation,-try' << '-Xdoclint:all' << '-Xdoclint:-missing'
 
                 // either disable annotation processor completely (default) or allow to enable them if an annotation processor is explicitly defined
                 if (options.compilerArgs.contains("-processor") == false) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -22,13 +22,18 @@ import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin
 import nebula.plugin.publishing.maven.MavenScmPlugin
 import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.NoticeTask
+import org.elasticsearch.gradle.Version
+import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.test.RestIntegTestTask
 import org.elasticsearch.gradle.test.RunTask
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin
+import org.gradle.api.InvalidUserDataException
 import org.gradle.api.Project
+import org.gradle.api.Task
 import org.gradle.api.publish.maven.MavenPublication
 import org.gradle.api.publish.maven.plugins.MavenPublishPlugin
 import org.gradle.api.publish.maven.tasks.GenerateMavenPom
+import org.gradle.api.tasks.Copy
 import org.gradle.api.tasks.SourceSet
 import org.gradle.api.tasks.bundling.Zip
 import org.gradle.jvm.tasks.Jar
@@ -38,25 +43,29 @@ import java.util.regex.Pattern
 /**
  * Encapsulates build configuration for an Elasticsearch plugin.
  */
-public class PluginBuildPlugin extends BuildPlugin {
+class PluginBuildPlugin extends BuildPlugin {
+
+    public static final String PLUGIN_EXTENSION_NAME = 'esplugin'
 
     @Override
-    public void apply(Project project) {
+    void apply(Project project) {
         super.apply(project)
+
+        PluginPropertiesExtension extension = project.extensions.create(PLUGIN_EXTENSION_NAME, PluginPropertiesExtension, project)
         configureDependencies(project)
 
         // this afterEvaluate must happen before the afterEvaluate added by integTest creation,
         // so that the file name resolution for installing the plugin will be setup
         project.afterEvaluate {
             boolean isXPackModule = project.path.startsWith(':x-pack:plugin')
             boolean isModule = project.path.startsWith(':modules:') || isXPackModule
-            String name = project.pluginProperties.extension.name
+            String name = extension.name
             project.archivesBaseName = name
 
             // set the project description so it will be picked up by publishing
-            project.description = project.pluginProperties.extension.description
+            project.description = extension.description
 
-            configurePublishing(project)
+            configurePublishing(project, extension)
 
             if (project.plugins.hasPlugin(TestClustersPlugin.class) == false) {
                 project.integTestCluster.dependsOn(project.tasks.bundlePlugin)
@@ -68,12 +77,23 @@ public class PluginBuildPlugin extends BuildPlugin {
             } else {
                 project.tasks.integTest.dependsOn(project.tasks.bundlePlugin)
                 if (isModule) {
-                    throw new RuntimeException("Testclusters does not support modules yet");
+                    project.testClusters.integTest.module(
+                            project.file(project.tasks.bundlePlugin.archiveFile)
+                    )
                 } else {
                     project.testClusters.integTest.plugin(
                             project.file(project.tasks.bundlePlugin.archiveFile)
                     )
                 }
+
+                project.extensions.getByType(PluginPropertiesExtension).extendedPlugins.each { pluginName ->
+                    // Auto add dependent modules to the test cluster
+                    if (project.findProject(":modules:${pluginName}") != null) {
+                        project.testClusters.integTest.module(
+                                project.file(project.project(":modules:${pluginName}").tasks.bundlePlugin.archiveFile)
+                        )
+                    }
+                }
             }
 
             project.tasks.run.dependsOn(project.tasks.bundlePlugin)
@@ -87,7 +107,7 @@ public class PluginBuildPlugin extends BuildPlugin {
             }
 
             if (isModule == false || isXPackModule) {
-                addNoticeGeneration(project)
+                addNoticeGeneration(project, extension)
             }
         }
         project.testingConventions {
@@ -104,32 +124,28 @@ public class PluginBuildPlugin extends BuildPlugin {
             }
         }
         createIntegTestTask(project)
-        createBundleTask(project)
+        createBundleTasks(project, extension)
         project.configurations.getByName('default').extendsFrom(project.configurations.getByName('runtime'))
         project.tasks.create('run', RunTask) // allow running ES with this plugin in the foreground of a build
     }
 
-    private void configurePublishing(Project project) {
+    private void configurePublishing(Project project, PluginPropertiesExtension extension) {
         // Only configure publishing if applied externally
-        if (project.pluginProperties.extension.hasClientJar) {
+        if (extension.hasClientJar) {
             project.plugins.apply(MavenScmPlugin.class)
             // Only change Jar tasks, we don't want a -client zip so we can't change archivesBaseName
             project.tasks.withType(Jar) {
                 baseName = baseName + "-client"
             }
             // always configure publishing for client jars
             project.plugins.apply(MavenScmPlugin.class)
-            project.publishing.publications.nebula(MavenPublication).artifactId(
-                    project.pluginProperties.extension.name + "-client"
-            )
+            project.publishing.publications.nebula(MavenPublication).artifactId(extension.name + "-client")
             project.tasks.withType(GenerateMavenPom.class) { GenerateMavenPom generatePOMTask ->
                 generatePOMTask.ext.pomFileName = "${project.archivesBaseName}-client-${project.versions.elasticsearch}.pom"
             }
         } else {
             if (project.plugins.hasPlugin(MavenPublishPlugin)) {
-                project.publishing.publications.nebula(MavenPublication).artifactId(
-                        project.pluginProperties.extension.name
-                )
+                project.publishing.publications.nebula(MavenPublication).artifactId(extension.name)
             }
 
         }
@@ -164,24 +180,64 @@ public class PluginBuildPlugin extends BuildPlugin {
      * Adds a bundlePlugin task which builds the zip containing the plugin jars,
      * metadata, properties, and packaging files
      */
-    private static void createBundleTask(Project project) {
+    private static void createBundleTasks(Project project, PluginPropertiesExtension extension) {
         File pluginMetadata = project.file('src/main/plugin-metadata')
+        File templateFile = new File(project.buildDir, "templates/plugin-descriptor.properties")
+
+        // create tasks to build the properties file for this plugin
+        Task copyPluginPropertiesTemplate = project.tasks.create('copyPluginPropertiesTemplate') {
+            outputs.file(templateFile)
+            doLast {
+                InputStream resourceTemplate = PluginBuildPlugin.getResourceAsStream("/${templateFile.name}")
+                templateFile.setText(resourceTemplate.getText('UTF-8'), 'UTF-8')
+            }
+        }
 
-        // create a task to build the properties file for this plugin
-        PluginPropertiesTask buildProperties = project.tasks.create('pluginProperties', PluginPropertiesTask.class)
+        Copy buildProperties = project.tasks.create('pluginProperties', Copy) {
+            dependsOn(copyPluginPropertiesTemplate)
+            from(templateFile)
+            into("${project.buildDir}/generated-resources")
+        }
+
+        project.afterEvaluate {
+            // check require properties are set
+            if (extension.name == null) {
+                throw new InvalidUserDataException('name is a required setting for esplugin')
+            }
+            if (extension.description == null) {
+                throw new InvalidUserDataException('description is a required setting for esplugin')
+            }
+            if (extension.classname == null) {
+                throw new InvalidUserDataException('classname is a required setting for esplugin')
+            }
+
+            Map<String, String> properties = [
+                    'name': extension.name,
+                    'description': extension.description,
+                    'version': extension.version,
+                    'elasticsearchVersion': Version.fromString(VersionProperties.elasticsearch).toString(),
+                    'javaVersion': project.targetCompatibility as String,
+                    'classname': extension.classname,
+                    'extendedPlugins': extension.extendedPlugins.join(','),
+                    'hasNativeController': extension.hasNativeController,
+                    'requiresKeystore': extension.requiresKeystore
+            ]
+
+            buildProperties.configure {
+                expand(properties)
+                inputs.properties(properties)
+            }
+        }
 
         // add the plugin properties and metadata to test resources, so unit tests can
         // know about the plugin (used by test security code to statically initialize the plugin in unit tests)
         SourceSet testSourceSet = project.sourceSets.test
-        testSourceSet.output.dir(buildProperties.descriptorOutput.parentFile, builtBy: 'pluginProperties')
+        testSourceSet.output.dir(buildProperties.destinationDir, builtBy: buildProperties)
         testSourceSet.resources.srcDir(pluginMetadata)
 
         // create the actual bundle task, which zips up all the files for the plugin
-        Zip bundle = project.tasks.create(name: 'bundlePlugin', type: Zip, dependsOn: [project.jar, buildProperties]) {
-            from(buildProperties.descriptorOutput.parentFile) {
-                // plugin properties file
-                include(buildProperties.descriptorOutput.name)
-            }
+        Zip bundle = project.tasks.create(name: 'bundlePlugin', type: Zip) {
+            from buildProperties
             from pluginMetadata // metadata (eg custom security policy)
             /*
              * If the plugin is using the shadow plugin then we need to bundle
@@ -223,23 +279,17 @@ public class PluginBuildPlugin extends BuildPlugin {
         }
     }
 
-    /** Adds nebula publishing task to generate a pom file for the plugin. */
-    protected static void addClientJarPomGeneration(Project project) {
-        project.plugins.apply(MavenScmPlugin.class)
-        project.description = project.pluginProperties.extension.description
-    }
-
     /** Configure the pom for the main jar of this plugin */
 
-    protected void addNoticeGeneration(Project project) {
-        File licenseFile = project.pluginProperties.extension.licenseFile
+    protected void addNoticeGeneration(Project project, PluginPropertiesExtension extension) {
+        File licenseFile = extension.licenseFile
         if (licenseFile != null) {
             project.tasks.bundlePlugin.from(licenseFile.parentFile) {
                 include(licenseFile.name)
                 rename { 'LICENSE.txt' }
             }
         }
-        File noticeFile = project.pluginProperties.extension.noticeFile
+        File noticeFile = extension.noticeFile
         if (noticeFile != null) {
             NoticeTask generateNotice = project.tasks.create('generateNotice', NoticeTask.class)
             generateNotice.inputFile = noticeFile

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -384,7 +384,11 @@ class ClusterFormationTasks {
         ]
         esConfig['node.max_local_storage_nodes'] = node.config.numNodes
         esConfig['http.port'] = node.config.httpPort
-        esConfig['transport.tcp.port'] =  node.config.transportPort
+        if (node.nodeVersion.onOrAfter('6.7.0')) {
+            esConfig['transport.port'] =  node.config.transportPort
+        } else {
+            esConfig['transport.tcp.port'] =  node.config.transportPort
+        }
         // Default the watermarks to absurdly low to prevent the tests from failing on nodes without enough disk space
         esConfig['cluster.routing.allocation.disk.watermark.low'] = '1b'
         esConfig['cluster.routing.allocation.disk.watermark.high'] = '1b'

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java

@@ -128,6 +128,11 @@ public void plugin(File plugin) {
         nodes.all(each -> each.plugin(plugin));
     }
 
+    @Override
+    public void module(File module) {
+        nodes.all(each -> each.module(module));
+    }
+
     @Override
     public void keystore(String key, String value) {
         nodes.all(each -> each.keystore(key, value));
@@ -198,6 +203,11 @@ public void start() {
         }
     }
 
+    @Override
+    public void extraConfigFile(String destination, File from) {
+        nodes.all(node -> node.extraConfigFile(destination, from));
+    }
+
     private void writeUnicastHostsFiles() {
         String unicastUris = nodes.stream().flatMap(node -> node.getAllTransportPortURI().stream()).collect(Collectors.joining("\n"));
         nodes.forEach(node -> {