diff --git a/build-tools-internal/muted-tests.yml b/build-tools-internal/muted-tests.yml new file mode 100644 index 0000000000000..341d127c7b64a --- /dev/null +++ b/build-tools-internal/muted-tests.yml @@ -0,0 +1,24 @@ +tests: +# Examples: +# +# Mute a single test case in a YAML test suite: +# - class: org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT +# method: test {yaml=analysis-common/30_tokenizers/letter} +# issue: https://github.com/elastic/elasticsearch/... +# +# Mute several methods of a Java test: +# - class: org.elasticsearch.common.CharArraysTests +# methods: +# - testCharsBeginsWith +# - testCharsToBytes +# - testConstantTimeEquals +# issue: https://github.com/elastic/elasticsearch/... +# +# Mute an entire test class: +# - class: org.elasticsearch.common.unit.TimeValueTests +# issue: https://github.com/elastic/elasticsearch/... +# +# Mute a single method in a test class: +# - class: org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToIPTests +# method: testCrankyEvaluateBlockWithoutNulls +# issue: https://github.com/elastic/elasticsearch/... diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java index fb6ac34819270..a1fb922c0dabb 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java @@ -12,6 +12,7 @@ import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitTaskPlugin; import org.elasticsearch.gradle.internal.info.BuildParams; import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; +import org.elasticsearch.gradle.internal.test.MutedTestPlugin; import org.elasticsearch.gradle.util.GradleUtils; import org.gradle.api.JavaVersion; import org.gradle.api.Plugin; @@ -43,6 +44,7 @@ public void apply(Project project) { project.getPluginManager().apply(RepositoriesSetupPlugin.class); project.getPluginManager().apply(ElasticsearchTestBasePlugin.class); project.getPluginManager().apply(PrecommitTaskPlugin.class); + project.getPluginManager().apply(MutedTestPlugin.class); configureConfigurations(project); configureCompile(project); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestPlugin.java new file mode 100644 index 0000000000000..c71facf4f87d8 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestPlugin.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test; + +import org.elasticsearch.gradle.internal.conventions.util.Util; +import org.elasticsearch.gradle.internal.info.BuildParams; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.provider.Provider; +import org.gradle.api.tasks.testing.Test; + +import java.io.File; + +public class MutedTestPlugin implements Plugin { + @Override + public void apply(Project project) { + File infoPath = new File(Util.locateElasticsearchWorkspace(project.getGradle()), "build-tools-internal"); + Provider mutedTestsProvider = project.getGradle() + .getSharedServices() + .registerIfAbsent("mutedTests", MutedTestsBuildService.class, spec -> { spec.getParameters().getInfoPath().set(infoPath); }); + + project.getTasks().withType(Test.class).configureEach(test -> { + test.filter(filter -> { + for (String exclude : mutedTestsProvider.get().getExcludePatterns()) { + filter.excludeTestsMatching(exclude); + } + + // Don't fail when all tests are ignored when running in CI + filter.setFailOnNoMatchingTests(BuildParams.isCi() == false); + }); + }); + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestsBuildService.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestsBuildService.java new file mode 100644 index 0000000000000..9e4a92f26d4dd --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestsBuildService.java @@ -0,0 +1,141 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; + +import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.services.BuildService; +import org.gradle.api.services.BuildServiceParameters; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public abstract class MutedTestsBuildService implements BuildService { + private final List excludePatterns; + + public MutedTestsBuildService() { + File infoPath = getParameters().getInfoPath().get().getAsFile(); + File mutedTestsFile = new File(infoPath, "muted-tests.yml"); + try (InputStream is = new BufferedInputStream(new FileInputStream(mutedTestsFile))) { + ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory()); + List mutedTests = objectMapper.readValue(is, MutedTests.class).getTests(); + excludePatterns = buildExcludePatterns(mutedTests == null ? Collections.emptyList() : mutedTests); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public List getExcludePatterns() { + return excludePatterns; + } + + private static List buildExcludePatterns(List mutedTests) { + List excludes = new ArrayList<>(); + if (mutedTests.isEmpty() == false) { + for (MutedTestsBuildService.MutedTest mutedTest : mutedTests) { + if (mutedTest.getClassName() != null && mutedTest.getMethods().isEmpty() == false) { + for (String method : mutedTest.getMethods()) { + // Tests that use the randomized runner and parameters end up looking like this: + // test {yaml=analysis-common/30_tokenizers/letter} + // We need to detect this and handle them a little bit different than non-parameterized tests, because of some + // quirks in the randomized runner + int index = method.indexOf(" {"); + String methodWithoutParams = index >= 0 ? method.substring(0, index) : method; + String paramString = index >= 0 ? method.substring(index) : null; + + excludes.add(mutedTest.getClassName() + "." + method); + + if (paramString != null) { + // Because of randomized runner quirks, we need skip the test method by itself whenever we want to skip a test + // that has parameters + // This is because the runner has *two* separate checks that can cause the test to end up getting executed, so + // we need filters that cover both checks + excludes.add(mutedTest.getClassName() + "." + methodWithoutParams); + } else { + // We need to add the following, in case we're skipping an entire class of parameterized tests + excludes.add(mutedTest.getClassName() + "." + method + " *"); + } + } + } else if (mutedTest.getClassName() != null) { + excludes.add(mutedTest.getClassName() + ".*"); + } + } + } + + return excludes; + } + + public interface Params extends BuildServiceParameters { + RegularFileProperty getInfoPath(); + } + + public static class MutedTest { + private final String className; + private final String method; + private final List methods; + private final String issue; + + @JsonCreator + public MutedTest( + @JsonProperty("class") String className, + @JsonProperty("method") String method, + @JsonProperty("methods") List methods, + @JsonProperty("issue") String issue + ) { + this.className = className; + this.method = method; + this.methods = methods; + this.issue = issue; + } + + public List getMethods() { + List allMethods = new ArrayList<>(); + if (methods != null) { + allMethods.addAll(methods); + } + if (method != null) { + allMethods.add(method); + } + + return allMethods; + } + + public String getClassName() { + return className; + } + + public String getIssue() { + return issue; + } + } + + private static class MutedTests { + private final List tests; + + @JsonCreator + MutedTests(@JsonProperty("tests") List tests) { + this.tests = tests; + } + + public List getTests() { + return tests; + } + } +} diff --git a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy index c1d0528c9d02b..39e008fdeea31 100644 --- a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy +++ b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy @@ -20,6 +20,8 @@ import spock.lang.Specification import spock.lang.TempDir import java.lang.management.ManagementFactory +import java.nio.file.Files +import java.nio.file.Path import java.util.jar.JarEntry import java.util.jar.JarOutputStream @@ -47,7 +49,12 @@ abstract class AbstractGradleFuncTest extends Specification { buildFile = testProjectDir.newFile('build.gradle') propertiesFile = testProjectDir.newFile('gradle.properties') propertiesFile << - "org.gradle.java.installations.fromEnv=JAVA_HOME,RUNTIME_JAVA_HOME,JAVA15_HOME,JAVA14_HOME,JAVA13_HOME,JAVA12_HOME,JAVA11_HOME,JAVA8_HOME" + "org.gradle.java.installations.fromEnv=JAVA_HOME,RUNTIME_JAVA_HOME,JAVA15_HOME,JAVA14_HOME,JAVA13_HOME,JAVA12_HOME,JAVA11_HOME,JAVA8_HOME" + + def mutedTestsFile = Files.createFile(Path.of(testProjectDir.newFolder("build-tools-internal").path, "muted-tests.yml")) + mutedTestsFile << """ + tests: [] + """ } def cleanup() {