[7.17] Muted test automation (#106784)

build-tools-internal/muted-tests.yml

@@ -0,0 +1,24 @@
+tests:
+# Examples:
+#
+#  Mute a single test case in a YAML test suite:
+#  - class: org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT
+#    method: test {yaml=analysis-common/30_tokenizers/letter}
+#    issue: https://github.com/elastic/elasticsearch/...
+#
+#  Mute several methods of a Java test:
+#  - class: org.elasticsearch.common.CharArraysTests
+#    methods:
+#      - testCharsBeginsWith
+#      - testCharsToBytes
+#      - testConstantTimeEquals
+#    issue: https://github.com/elastic/elasticsearch/...
+#
+#  Mute an entire test class:
+#  - class: org.elasticsearch.common.unit.TimeValueTests
+#    issue: https://github.com/elastic/elasticsearch/...
+#
+#  Mute a single method in a test class:
+#  - class: org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToIPTests
+#    method: testCrankyEvaluateBlockWithoutNulls
+#    issue: https://github.com/elastic/elasticsearch/...

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java

@@ -12,6 +12,7 @@
 import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitTaskPlugin;
 import org.elasticsearch.gradle.internal.info.BuildParams;
 import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin;
+import org.elasticsearch.gradle.internal.test.MutedTestPlugin;
 import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.JavaVersion;
 import org.gradle.api.Plugin;
@@ -43,6 +44,7 @@ public void apply(Project project) {
         project.getPluginManager().apply(RepositoriesSetupPlugin.class);
         project.getPluginManager().apply(ElasticsearchTestBasePlugin.class);
         project.getPluginManager().apply(PrecommitTaskPlugin.class);
+        project.getPluginManager().apply(MutedTestPlugin.class);
 
         configureConfigurations(project);
         configureCompile(project);

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestPlugin.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.test;
+
+import org.elasticsearch.gradle.internal.conventions.util.Util;
+import org.elasticsearch.gradle.internal.info.BuildParams;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.provider.Provider;
+import org.gradle.api.tasks.testing.Test;
+
+import java.io.File;
+
+public class MutedTestPlugin implements Plugin<Project> {
+    @Override
+    public void apply(Project project) {
+        File infoPath = new File(Util.locateElasticsearchWorkspace(project.getGradle()), "build-tools-internal");
+        Provider<MutedTestsBuildService> mutedTestsProvider = project.getGradle()
+            .getSharedServices()
+            .registerIfAbsent("mutedTests", MutedTestsBuildService.class, spec -> { spec.getParameters().getInfoPath().set(infoPath); });
+
+        project.getTasks().withType(Test.class).configureEach(test -> {
+            test.filter(filter -> {
+                for (String exclude : mutedTestsProvider.get().getExcludePatterns()) {
+                    filter.excludeTestsMatching(exclude);
+                }
+
+                // Don't fail when all tests are ignored when running in CI
+                filter.setFailOnNoMatchingTests(BuildParams.isCi() == false);
+            });
+        });
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestsBuildService.java

@@ -0,0 +1,141 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.test;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+
+import org.gradle.api.file.RegularFileProperty;
+import org.gradle.api.services.BuildService;
+import org.gradle.api.services.BuildServiceParameters;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public abstract class MutedTestsBuildService implements BuildService<MutedTestsBuildService.Params> {
+    private final List<String> excludePatterns;
+
+    public MutedTestsBuildService() {
+        File infoPath = getParameters().getInfoPath().get().getAsFile();
+        File mutedTestsFile = new File(infoPath, "muted-tests.yml");
+        try (InputStream is = new BufferedInputStream(new FileInputStream(mutedTestsFile))) {
+            ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory());
+            List<MutedTest> mutedTests = objectMapper.readValue(is, MutedTests.class).getTests();
+            excludePatterns = buildExcludePatterns(mutedTests == null ? Collections.emptyList() : mutedTests);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    public List<String> getExcludePatterns() {
+        return excludePatterns;
+    }
+
+    private static List<String> buildExcludePatterns(List<MutedTest> mutedTests) {
+        List<String> excludes = new ArrayList<>();
+        if (mutedTests.isEmpty() == false) {
+            for (MutedTestsBuildService.MutedTest mutedTest : mutedTests) {
+                if (mutedTest.getClassName() != null && mutedTest.getMethods().isEmpty() == false) {
+                    for (String method : mutedTest.getMethods()) {
+                        // Tests that use the randomized runner and parameters end up looking like this:
+                        // test {yaml=analysis-common/30_tokenizers/letter}
+                        // We need to detect this and handle them a little bit different than non-parameterized tests, because of some
+                        // quirks in the randomized runner
+                        int index = method.indexOf(" {");
+                        String methodWithoutParams = index >= 0 ? method.substring(0, index) : method;
+                        String paramString = index >= 0 ? method.substring(index) : null;
+
+                        excludes.add(mutedTest.getClassName() + "." + method);
+
+                        if (paramString != null) {
+                            // Because of randomized runner quirks, we need skip the test method by itself whenever we want to skip a test
+                            // that has parameters
+                            // This is because the runner has *two* separate checks that can cause the test to end up getting executed, so
+                            // we need filters that cover both checks
+                            excludes.add(mutedTest.getClassName() + "." + methodWithoutParams);
+                        } else {
+                            // We need to add the following, in case we're skipping an entire class of parameterized tests
+                            excludes.add(mutedTest.getClassName() + "." + method + " *");
+                        }
+                    }
+                } else if (mutedTest.getClassName() != null) {
+                    excludes.add(mutedTest.getClassName() + ".*");
+                }
+            }
+        }
+
+        return excludes;
+    }
+
+    public interface Params extends BuildServiceParameters {
+        RegularFileProperty getInfoPath();
+    }
+
+    public static class MutedTest {
+        private final String className;
+        private final String method;
+        private final List<String> methods;
+        private final String issue;
+
+        @JsonCreator
+        public MutedTest(
+            @JsonProperty("class") String className,
+            @JsonProperty("method") String method,
+            @JsonProperty("methods") List<String> methods,
+            @JsonProperty("issue") String issue
+        ) {
+            this.className = className;
+            this.method = method;
+            this.methods = methods;
+            this.issue = issue;
+        }
+
+        public List<String> getMethods() {
+            List<String> allMethods = new ArrayList<>();
+            if (methods != null) {
+                allMethods.addAll(methods);
+            }
+            if (method != null) {
+                allMethods.add(method);
+            }
+
+            return allMethods;
+        }
+
+        public String getClassName() {
+            return className;
+        }
+
+        public String getIssue() {
+            return issue;
+        }
+    }
+
+    private static class MutedTests {
+        private final List<MutedTest> tests;
+
+        @JsonCreator
+        MutedTests(@JsonProperty("tests") List<MutedTest> tests) {
+            this.tests = tests;
+        }
+
+        public List<MutedTest> getTests() {
+            return tests;
+        }
+    }
+}

build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy

@@ -20,6 +20,8 @@ import spock.lang.Specification
 import spock.lang.TempDir
 
 import java.lang.management.ManagementFactory
+import java.nio.file.Files
+import java.nio.file.Path
 import java.util.jar.JarEntry
 import java.util.jar.JarOutputStream
 
@@ -47,7 +49,12 @@ abstract class AbstractGradleFuncTest extends Specification {
         buildFile = testProjectDir.newFile('build.gradle')
         propertiesFile = testProjectDir.newFile('gradle.properties')
         propertiesFile <<
-                "org.gradle.java.installations.fromEnv=JAVA_HOME,RUNTIME_JAVA_HOME,JAVA15_HOME,JAVA14_HOME,JAVA13_HOME,JAVA12_HOME,JAVA11_HOME,JAVA8_HOME"
+            "org.gradle.java.installations.fromEnv=JAVA_HOME,RUNTIME_JAVA_HOME,JAVA15_HOME,JAVA14_HOME,JAVA13_HOME,JAVA12_HOME,JAVA11_HOME,JAVA8_HOME"
+
+        def mutedTestsFile = Files.createFile(Path.of(testProjectDir.newFolder("build-tools-internal").path, "muted-tests.yml"))
+        mutedTestsFile << """
+            tests: []
+        """
     }
 
     def cleanup() {