test: move test generation code to separate files

Safe-DS · Aug 8, 2023 · c5afb8b · c5afb8b
1 parent 53a9674
commit c5afb8b
Show file tree

Hide file tree

Showing 8 changed files with 344 additions and 208 deletions.
diff --git a/DSL/src/language-server/formatting/safe-ds-formatter.ts b/DSL/src/language-server/formatting/safe-ds-formatter.ts
@@ -905,7 +905,7 @@ export class SafeDSFormatter extends AbstractFormatter {
             formatter.keyword('}').prepend(noSpace());
         } else {
             formatter.nodes(...columns).prepend(indent());
-            formatter.keywords(',').prepend(noSpace()).append(newLine());
+            formatter.keywords(',').prepend(noSpace());
             formatter.keyword('}').prepend(newLine());
         }
     }

diff --git a/DSL/tests/formatting/creator.ts b/DSL/tests/formatting/creator.ts
@@ -0,0 +1,79 @@
+import {listTestResources, resolvePathRelativeToResources} from "../helpers/testResources";
+import path from "path";
+import fs from "fs";
+import {validationHelper} from "langium/test";
+import {Diagnostic} from "vscode-languageserver-types";
+import {createSafeDsServices} from "../../src/language-server/safe-ds-module";
+import {EmptyFileSystem} from "langium";
+
+const services = createSafeDsServices(EmptyFileSystem).SafeDs;
+const separator = '// -----------------------------------------------------------------------------';
+
+export const createFormatterTests = async (): Promise<FormatterTest[]> => {
+    const testCases = listTestResources('formatting').map(async (pathRelativeToResources): Promise<FormatterTest> => {
+        const absolutePath = resolvePathRelativeToResources(path.join('formatting', pathRelativeToResources));
+        const program = fs.readFileSync(absolutePath).toString();
+        const parts = program.split(separator);
+
+        // Must contain exactly one separator
+        if (parts.length !== 2) {
+            return {
+                testName: `INVALID TEST FILE [${pathRelativeToResources}]`,
+                originalCode: '',
+                expectedFormattedCode: '',
+                error: new SeparatorError(parts.length - 1),
+            };
+        }
+
+        // Original code must not contain syntax errors
+        const originalCode = normalizeLineBreaks(parts[0]).trimEnd();
+        const expectedFormattedCode = normalizeLineBreaks(parts[1]).trim();
+
+        const validationResult = await validationHelper(services)(parts[0]);
+        const syntaxErrors = validationResult.diagnostics.filter(
+            (d) => d.severity === 1 && (d.code === 'lexing-error' || d.code === 'parsing-error'),
+        );
+
+        if (syntaxErrors.length > 0) {
+            return {
+                testName: `INVALID TEST FILE [${pathRelativeToResources}]`,
+                originalCode,
+                expectedFormattedCode,
+                error: new SyntaxErrorsInOriginalCodeError(syntaxErrors),
+            };
+        }
+
+        return {
+            testName: `${pathRelativeToResources} should be formatted correctly`,
+            originalCode,
+            expectedFormattedCode,
+        };
+    });
+
+    return Promise.all(testCases);
+};
+
+const normalizeLineBreaks = (code: string): string => {
+    return code.replace(/\r\n?/gu, '\n');
+};
+
+interface FormatterTest {
+    testName: string;
+    originalCode: string;
+    expectedFormattedCode: string;
+    error?: Error;
+}
+
+class SeparatorError extends Error {
+    constructor(readonly number_of_separators: number) {
+        super(`Expected exactly one separator but found ${number_of_separators}.`);
+    }
+}
+
+class SyntaxErrorsInOriginalCodeError extends Error {
+    constructor(readonly syntaxErrors: Diagnostic[]) {
+        const syntaxErrorsAsString = syntaxErrors.map((e) => `- ${e.message}`).join(`\n`);
+
+        super(`Original code has syntax errors:\n${syntaxErrorsAsString}`);
+    }
+}
diff --git a/DSL/tests/formatting/testFormatter.test.ts b/DSL/tests/formatting/testFormatter.test.ts
@@ -1,93 +1,42 @@
 import { createSafeDsServices } from '../../src/language-server/safe-ds-module';
-import { expectFormatting, validationHelper } from 'langium/test';
+import {clearDocuments, expectFormatting} from 'langium/test';
 import { describe, it } from 'vitest';
 import { EmptyFileSystem } from 'langium';
-import { listTestResources, resolvePathRelativeToResources } from '../helpers/testResources';
-import path from 'path';
-import fs from 'fs';
-import { Diagnostic } from 'vscode-languageserver-types';
+import { createFormatterTests } from './creator';
 
-const services = createSafeDsServices({ ...EmptyFileSystem }).SafeDs;
-const separator = '// -----------------------------------------------------------------------------';
+const services = createSafeDsServices(EmptyFileSystem).SafeDs;
+const formatterTests = createFormatterTests();
 
 describe('formatter', async () => {
-    it.each(await createFormatterTest())('$testName', async (test) => {
+    it.each(await formatterTests)('$testName', async (test) => {
+        // Test is invalid
         if (test.error) {
             throw test.error;
         }
 
+        // Formatting original code must result in expected formatted code
         await expectFormatting(services)({
             before: test.originalCode,
             after: test.expectedFormattedCode,
         });
-    });
-});
 
-const createFormatterTest = async (): Promise<FormatterTest[]> => {
-    const testCases = listTestResources('formatting').map(async (pathRelativeToResources): Promise<FormatterTest> => {
-        const absolutePath = resolvePathRelativeToResources(path.join('formatting', pathRelativeToResources));
-        const program = fs.readFileSync(absolutePath).toString();
-        const parts = program.split(separator);
+        // Clear loaded documents to avoid colliding URIs (https://github.com/langium/langium/issues/1146)
+        await clearDocuments(services);
+    });
 
-        // Must contain exactly one separator
-        if (parts.length !== 2) {
-            return {
-                testName: `INVALID TEST FILE [${pathRelativeToResources}]`,
-                originalCode: '',
-                expectedFormattedCode: '',
-                error: new SeparatorError(parts.length - 1),
-            };
+    it.each(await formatterTests)('$testName (idempotence)', async (test) => {
+        // Test is invalid
+        if (test.error) {
+            throw test.error;
         }
 
-        // Original code must not contain syntax errors
-        const originalCode = normalizeLineBreaks(parts[0]).trimEnd();
-        const expectedFormattedCode = normalizeLineBreaks(parts[1]).trim();
-
-        const validationResult = await validationHelper(services)(parts[0]);
-        const syntaxErrors = validationResult.diagnostics.filter(
-            (d) => d.severity === 1 && (d.code === 'lexing-error' || d.code === 'parsing-error'),
-        );
-
-        if (syntaxErrors.length > 0) {
-            return {
-                testName: `INVALID TEST FILE [${pathRelativeToResources}]`,
-                originalCode,
-                expectedFormattedCode,
-                error: new SyntaxErrorsInOriginalCodeError(syntaxErrors),
-            };
-        }
+        // Formatting must be idempotent
+        await expectFormatting(services)({
+            before: test.expectedFormattedCode,
+            after: test.expectedFormattedCode,
+        });
 
-        return {
-            testName: `${pathRelativeToResources} should be formatted correctly`,
-            originalCode,
-            expectedFormattedCode,
-        };
+        // Clear loaded documents to avoid colliding URIs (https://github.com/langium/langium/issues/1146)
+        await clearDocuments(services);
     });
-
-    return Promise.all(testCases);
-};
-
-const normalizeLineBreaks = (code: string): string => {
-    return code.replace(/\r\n?/gu, '\n');
-};
-
-interface FormatterTest {
-    testName: string;
-    originalCode: string;
-    expectedFormattedCode: string;
-    error?: Error;
-}
-
-class SeparatorError extends Error {
-    constructor(readonly number_of_separators: number) {
-        super(`Expected exactly one separator but found ${number_of_separators}.`);
-    }
-}
-
-class SyntaxErrorsInOriginalCodeError extends Error {
-    constructor(readonly syntaxErrors: Diagnostic[]) {
-        const syntaxErrorsAsString = syntaxErrors.map((e) => `- ${e.message}`).join(`\n`);
-
-        super(`Original code has syntax errors:\n${syntaxErrorsAsString}`);
-    }
-}
+});
diff --git a/DSL/tests/grammar/creator.ts b/DSL/tests/grammar/creator.ts
@@ -0,0 +1,86 @@
+import { listTestResources, resolvePathRelativeToResources } from '../helpers/testResources';
+import path from 'path';
+import fs from 'fs';
+import { findTestComments } from '../helpers/testComments';
+import { NoCommentsError } from '../helpers/testChecks';
+
+export const createGrammarTests = (): GrammarTest[] => {
+    return listTestResources('grammar').map((pathRelativeToResources): GrammarTest => {
+        const absolutePath = resolvePathRelativeToResources(path.join('grammar', pathRelativeToResources));
+        const program = fs.readFileSync(absolutePath).toString();
+        const comments = findTestComments(program);
+
+        // Must contain at least one comment
+        if (comments.length === 0) {
+            return {
+                testName: `INVALID TEST FILE [${pathRelativeToResources}]`,
+                program,
+                expectedResult: 'invalid',
+                error: new NoCommentsError(),
+            };
+        }
+
+        // Must contain no more than one comment
+        if (comments.length > 1) {
+            return {
+                testName: `INVALID TEST FILE [${pathRelativeToResources}]`,
+                program,
+                expectedResult: 'invalid',
+                error: new MultipleCommentsError(comments),
+            };
+        }
+
+        const comment = comments[0];
+
+        // Must contain a valid comment
+        if (comment !== 'syntax_error' && comment !== 'no_syntax_error') {
+            return {
+                testName: `INVALID TEST FILE [${pathRelativeToResources}]`,
+                program,
+                expectedResult: 'invalid',
+                error: new InvalidCommentError(comment),
+            };
+        }
+
+        let testName: string;
+        if (comment === 'syntax_error') {
+            testName = `[${pathRelativeToResources}] should have syntax errors`;
+        } else {
+            testName = `[${pathRelativeToResources}] should not have syntax errors`;
+        }
+
+        return {
+            testName,
+            program,
+            expectedResult: comment,
+        };
+    });
+};
+
+/**
+ * A description of a grammar test.
+ */
+interface GrammarTest {
+    testName: string;
+    program: string;
+    expectedResult: 'syntax_error' | 'no_syntax_error' | 'invalid';
+    error?: Error;
+}
+
+/**
+ * Found multiple test comments.
+ */
+class MultipleCommentsError extends Error {
+    constructor(readonly comments: string[]) {
+        super(`Found multiple test comments (grammar tests expect only one): ${comments}`);
+    }
+}
+
+/**
+ * Found one test comment but it was invalid.
+ */
+class InvalidCommentError extends Error {
+    constructor(readonly comment: string) {
+        super(`Invalid test comment (valid values are 'syntax_error' and 'no_syntax_error'): ${comment}`);
+    }
+}