Adapt test runner for all JUnit 5 Platform Engines

.gitignore

@@ -39,6 +39,7 @@ bin/
 **/lib/
 out/
 server/
+.jqwik-database
 
 jdtls/
 

java-extension/com.microsoft.java.test.plugin/src/main/java/com/microsoft/java/test/plugin/searcher/JUnit5TestSearcher.java

@@ -34,16 +34,24 @@
 
 public class JUnit5TestSearcher extends BaseFrameworkSearcher {
 
-    public static final String NESTED = "org.junit.jupiter.api.Nested";
+    public static final String JUPITER_NESTED = "org.junit.jupiter.api.Nested";
+    public static final String JUNIT_PLATFORM_TESTABLE = "org.junit.platform.commons.annotation.Testable";
+
+    // TODO: Remove the following annotations once we can find tests without the search engine
+    //       - The search engine cannot find the meta-annotation and testable annotated ones.
+    public static final String JUPITER_TEST = "org.junit.jupiter.api.Test";
+    public static final String JUPITER_PARAMETERIZED_TEST = "org.junit.jupiter.params.ParameterizedTest";
+    public static final String JUPITER_REPEATED_TEST = "org.junit.jupiter.api.RepeatedTest";
+    public static final String JUPITER_TEST_FACTORY = "org.junit.jupiter.api.TestFactory";
+    public static final String JUPITER_TEST_TEMPLATE = "org.junit.jupiter.api.TestTemplate";
 
     protected static final String DISPLAY_NAME_ANNOTATION_JUNIT5 = "org.junit.jupiter.api.DisplayName";
 
     public JUnit5TestSearcher() {
         super();
-        this.testMethodAnnotations = new String[] { "org.junit.jupiter.api.Test",
-            "org.junit.jupiter.params.ParameterizedTest", "org.junit.jupiter.api.RepeatedTest",
-            "org.junit.jupiter.api.TestFactory", "org.junit.jupiter.api.TestTemplate" };
-        this.testClassAnnotations = new String[] { NESTED };
+        this.testMethodAnnotations = new String[] { JUPITER_TEST, JUPITER_PARAMETERIZED_TEST,
+            JUPITER_REPEATED_TEST, JUPITER_TEST_FACTORY, JUPITER_TEST_TEMPLATE, JUNIT_PLATFORM_TESTABLE };
+        this.testClassAnnotations = new String[] { JUNIT_PLATFORM_TESTABLE, JUPITER_NESTED };
     }
 
     @Override
@@ -63,12 +71,7 @@ public boolean isTestMethod(IMethod method) {
             }
             for (final String annotation : this.testMethodAnnotations) {
                 if (TestFrameworkUtils.hasAnnotation(method, annotation, true /*checkHierarchy*/)) {
-                    if ("org.junit.jupiter.api.TestFactory".equals(annotation)) {
-                        return true;
-                    } else if ("V".equals(method.getReturnType())) {
-                        // Other annotations need the return type to be void
-                        return true;
-                    }
+                    return true;
                 }
             }
             return false;

java-extension/com.microsoft.java.test.plugin/src/main/java/com/microsoft/java/test/plugin/util/TestSearchUtils.java

@@ -116,6 +116,11 @@ public static List<TestItem> searchCodeLens(List<Object> arguments, IProgressMon
                 resultList.add(parent);
                 continue;
             }
+            // JUnit 5 supports nested test classes
+            if (isJunit5TestableClass(type)) {
+                resultList.add(TestItemUtils.constructTestItem(type, TestLevel.CLASS, TestKind.JUnit));
+            }
+
             // Class annotated by @RunWith should be considered as a Suite even it has no test method children
             if (TestFrameworkUtils.hasAnnotation(type, JUnit4TestSearcher.RUN_WITH, false /*checkHierarchy*/)) {
                 resultList.add(TestItemUtils.constructTestItem(type, TestLevel.CLASS, TestKind.JUnit));
@@ -413,11 +418,38 @@ private static boolean isTestableClass(IType type) throws JavaModelException {
             return false;
         }
 
-        if (TestFrameworkUtils.hasAnnotation(type, JUnit5TestSearcher.NESTED, false /*checkHierarchy*/) ||
+        if (isJunit5TestableClass(type)) {
+            return true;
+        }
+
+        return false;
+    }
+
+    private static boolean isJunit5TestableClass(IType type) throws JavaModelException {
+        final int flags = type.getFlags();
+
+        // Classes with Testable annotation are testable
+        if (TestFrameworkUtils.hasAnnotation(
+                type,
+                JUnit5TestSearcher.JUNIT_PLATFORM_TESTABLE,
+                true /*checkHierarchy*/
+        )) {
+            return true;
+        }
+
+        // Jupiter's Nested annotation does not have Testable as meta annotation
+        if (TestFrameworkUtils.hasAnnotation(type, JUnit5TestSearcher.JUPITER_NESTED, true /*checkHierarchy*/) ||
                 (Flags.isStatic(flags) && Flags.isPublic(flags))) {
             return true;
         }
 
+        // Classes whose inner classes are testable should also be testable
+        for (final IJavaElement child : type.getChildren())  {
+            if (child instanceof IType && isJunit5TestableClass((IType) child)) {
+                return true;
+            }
+        }
+
         return false;
     }
 

test/gradle-junit5-suite/codelens.test.ts

@@ -38,6 +38,31 @@ suite('Code Lens Tests', function() {
         assert.ok(passedDetail!.duration !== undefined, 'Should have execution time');
     });
 
+    test("Can run test method annotated with @Testable", async function() {
+        const document: TextDocument = await workspace.openTextDocument(Uris.GRADLE_JUNIT5_PROPERTY_TEST);
+        await window.showTextDocument(document);
+
+        const codeLensProvider: TestCodeLensProvider = new TestCodeLensProvider();
+        const codeLens: CodeLens[] = await codeLensProvider.provideCodeLenses(document, Token.cancellationToken);
+        assert.equal(codeLens.length, 4, 'Code Lens should appear for @ParameterizedTest annotation');
+
+        const command: Command | undefined = codeLens[0].command;
+        assert.notEqual(command, undefined, 'Command inside Code Lens should not be undefined');
+        assert.notEqual(command, null, 'Command inside Code Lens should not be null');
+
+        const testItem: ITestItem[] = command!.arguments as ITestItem[];
+        assert.equal(testItem.length, 1);
+        assert.equal(testItem[0].paramTypes[0], 'int');
+
+        await commands.executeCommand(command!.command, testItem[0]);
+
+        const projectName: string = testItem[0].project;
+
+        const failedDetail: ITestResult| undefined = testResultManager.getResultById(`${projectName}@junit5.PropertyTest#absoluteValueOfIntegerAlwaysPositive`);
+        assert.equal(failedDetail!.status, TestStatus.Fail);
+        assert.ok(failedDetail!.duration !== undefined, 'Should have execution time');
+    });
+
     teardown(async function() {
         // Clear the result cache
         testResultManager.dispose();

test/shared.ts

@@ -38,6 +38,7 @@ export namespace Uris {
     // Gradle JUnit5
     const GRADLE_JUNIT5_TEST_PACKAGE: string = path.join('junit5', 'src', 'test', 'java', 'junit5');
     export const GRADLE_JUNIT5_PARAMETERIZED_TEST: Uri = Uri.file(path.join(TEST_PROJECT_BASE_PATH, GRADLE_JUNIT5_TEST_PACKAGE, 'ParameterizedAnnotationTest.java'));
+    export const GRADLE_JUNIT5_PROPERTY_TEST: Uri = Uri.file(path.join(TEST_PROJECT_BASE_PATH, GRADLE_JUNIT5_TEST_PACKAGE, 'PropertyTest.java'));
 }
 
 export async function getJavaVersion(): Promise<number> {

test/test-projects/junit5/build.gradle

@@ -3,13 +3,20 @@ plugins {
 }
 
 repositories {
-    jcenter()
+    mavenCentral()
 }
 
 dependencies {
     testImplementation 'org.junit.jupiter:junit-jupiter:5.6.0'
+    testImplementation "net.jqwik:jqwik:1.2.7"
 }
 
 test {
-    useJUnitPlatform()
+    useJUnitPlatform {
+        includeEngines 'jqwik', 'junit-jupiter', 'junit-vintage'
+    }
+
+    include '**/*Properties.class'
+	include '**/*Test.class'
+	include '**/*Tests.class'
 }

test/test-projects/junit5/src/test/java/junit5/PropertyTest.java

@@ -0,0 +1,11 @@
+package junit5;
+
+import net.jqwik.api.ForAll;
+import net.jqwik.api.Property;
+
+public class PropertyTest {
+    @Property
+    boolean absoluteValueOfIntegerAlwaysPositive(@ForAll int anInteger) {
+        return Math.abs(anInteger) >= 0;
+    }
+}