Upgrade runner for type-tests (#73)

* Upgrade runner for type-tests wip

* Make .ts import work again

* Fix fallback check, fix tsconfig unlinking

* Clean-up formatting

* Implement tstyche support

* Use tsconfig.solutions.json

* Update expected results

* Inline the config files

* Support for skipping only when the exercise allows it

* Add flags to todos

package.json

@@ -24,7 +24,7 @@
     "prepublish": "corepack yarn test:bare && corepack yarn lint",
     "lint": "corepack yarn eslint src -c eslint.config.mjs && corepack yarn eslint test -c eslint.config.mjs",
     "test": "corepack yarn build && corepack yarn test:bare",
-    "test:bare": "corepack yarn node test/smoke.test.mjs && corepack yarn node test/skip.test.mjs && corepack yarn node test/import.test.mjs"
+    "test:bare": "corepack yarn node test/smoke.test.mjs && corepack yarn node test/skip.test.mjs && corepack yarn node test/import.test.mjs && corepack yarn node test/types.test.mjs"
   },
   "dependencies": {
     "@exercism/babel-preset-typescript": "^0.5.0",
@@ -40,6 +40,7 @@
     "core-js": "^3.37.1",
     "jest": "^29.7.0",
     "shelljs": "^0.8.5",
+    "tstyche": "^2.1.1",
     "typescript": "~5.5.4"
   },
   "devDependencies": {

src/output.ts

@@ -31,6 +31,8 @@ type ExerciseConfig = {
     'flag.tests.task-per-describe': boolean
     'flag.tests.may-run-long': boolean
     'flag.tests.includes-optional': boolean
+    'flag.tests.jest'?: boolean
+    'flag.tests.tstyche'?: boolean
   }
 }
 
@@ -66,11 +68,9 @@ export class Output {
       this.results.status =
         aggregatedResults.numRuntimeErrorTestSuites === 0 &&
         aggregatedResults.numFailedTestSuites === 0 &&
-        // Pending tests are skipped tests. test.skip tests are fine in our
-        // reporter and should not be forced to have ran here. So the next
-        // line is commented out.
+        // Pending tests are skipped tests. test.skip tests are fine if the
+        // exercise reports that there are optional tests.
         //
-        // aggregatedResults.numPendingTests === 0 &&
         aggregatedResults.numFailedTests === 0
           ? 'pass'
           : 'fail'
@@ -88,6 +88,24 @@ export class Output {
             'an issue if the problem persists.'
         )
       }
+
+      if (
+        this.results.status === 'pass' &&
+        (aggregatedResults.numPendingTests !== 0 ||
+          aggregatedResults.numPendingTestSuites !== 0)
+      ) {
+        if (
+          !this.configFlag('flag.tests.includes-optional') &&
+          this.config?.custom
+        ) {
+          this.results.status = 'fail'
+          this.error(
+            'Expected to see 0 skipped tests and 0 skipped test suites. ' +
+              'None of the tests in this exercise are optional. The skipped ' +
+              'tests will not show up, but were found during the last run.'
+          )
+        }
+      }
     }
 
     const parsedSources: Record<

test/fixtures/tstyche/documentation/.meta/config.json

@@ -0,0 +1,14 @@
+{
+  "authors": [
+    "SleeplessByte"
+  ],
+  "files": {
+    "solution": [
+    ],
+    "test": [
+      "__typetests__/docs.tst.ts"
+    ],
+    "example": [
+    ]
+  }
+}

test/fixtures/tstyche/documentation/__typetests__/docs.tst.ts

@@ -0,0 +1,19 @@
+import { expect, test } from "tstyche";
+
+function firstItem<T>(target: Array<T>): T | undefined {
+  return target[0];
+}
+
+test("first item requires a parameter", () => {
+  expect(firstItem(["a", "b", "c"])).type.toBe<string | undefined>();
+
+  expect(firstItem()).type.toRaiseError("Expected 1 argument");
+});
+
+function secondItem<T>(target: Array<T>): T | undefined {
+  return target[1];
+}
+
+test("handles numbers", () => {
+  expect(secondItem([1, 2, 3])).type.toBe<number | undefined>();
+});

test/fixtures/tstyche/documentation/__typetests__/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "extends": "../tsconfig.json",
+  "compilerOptions": {
+    "noEmit": true,
+    "strict": true,
+    "types": []
+  },
+  "include": ["./"],
+  "exclude": []
+}

test/fixtures/tstyche/documentation/empty.ts

@@ -0,0 +1 @@
+export {}

test/fixtures/tstyche/documentation/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "compilerOptions": {
+    "target": "es2016",
+    "module": "commonjs",
+    "forceConsistentCasingInFileNames": true,
+    "strict": true,
+    "skipLibCheck": true
+  },
+  "exclude": ["./__typetests__"]
+}

test/fixtures/tstyche/fire/.meta/config.json

@@ -0,0 +1,15 @@
+{
+  "authors": [
+    "SleeplessByte"
+  ],
+  "files": {
+    "solution": [
+      "fire.ts"
+    ],
+    "test": [
+      "__typetests__/fire.tst.ts"
+    ],
+    "example": [
+    ]
+  }
+}

test/fixtures/tstyche/fire/__typetests__/fire.tst.ts

@@ -0,0 +1,14 @@
+import { test, expect, describe } from 'tstyche'
+import type { MethodLikeKeys } from '../fire.js'
+
+interface Sample {
+  description: string
+  getLength: () => number
+  getWidth?: () => number
+}
+
+describe('fire', () => {
+  test('all method keys are found', () => {
+    expect<MethodLikeKeys<Sample>>().type.toBe<'getLength' | 'getWidth'>()
+  })
+})

test/fixtures/tstyche/fire/__typetests__/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "extends": "../tsconfig.json",
+  "compilerOptions": {
+    "noEmit": true,
+    "strict": true,
+    "types": []
+  },
+  "include": ["./"],
+  "exclude": []
+}

test/fixtures/tstyche/fire/fire.ts

@@ -0,0 +1,5 @@
+type MethodLike = (...args: any) => any;
+
+export type MethodLikeKeys<T> = keyof {
+  [K in keyof T as T[K] extends MethodLike ? K : never]: T[K];
+};

test/fixtures/tstyche/fire/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "compilerOptions": {
+    "target": "es2016",
+    "module": "commonjs",
+    "forceConsistentCasingInFileNames": true,
+    "strict": true,
+    "skipLibCheck": true
+  },
+  "exclude": ["./__typetests__"]
+}

test/fixtures/tstyche/firefought/.meta/config.json

@@ -0,0 +1,15 @@
+{
+  "authors": [
+    "SleeplessByte"
+  ],
+  "files": {
+    "solution": [
+      "fire.ts"
+    ],
+    "test": [
+      "__typetests__/fire.tst.ts"
+    ],
+    "example": [
+    ]
+  }
+}

test/fixtures/tstyche/firefought/__typetests__/fire.tst.ts

@@ -0,0 +1,14 @@
+import { describe, test, expect } from "tstyche";
+import type { MethodLikeKeys } from "../fire.js";
+
+interface Sample {
+  description: string;
+  getLength: () => number;
+  getWidth?: () => number;
+}
+
+describe('fire', () => {
+  test('all method keys are found', () => {
+    expect<MethodLikeKeys<Sample>>().type.toBe<"getLength" | "getWidth">();
+  })
+})

test/fixtures/tstyche/firefought/__typetests__/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "extends": "../tsconfig.json",
+  "compilerOptions": {
+    "noEmit": true,
+    "strict": true,
+    "types": []
+  },
+  "include": ["./"],
+  "exclude": []
+}

test/fixtures/tstyche/firefought/fire.ts

@@ -0,0 +1,5 @@
+type MethodLike = (...args: any) => any;
+
+export type MethodLikeKeys<T> = keyof {
+  [K in keyof T as Required<T>[K] extends MethodLike ? K : never]: T[K];
+};

test/fixtures/tstyche/firefought/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "compilerOptions": {
+    "target": "es2016",
+    "module": "commonjs",
+    "forceConsistentCasingInFileNames": true,
+    "strict": true,
+    "skipLibCheck": true
+  },
+  "exclude": ["./__typetests__"]
+}

test/fixtures/two-fer/error/syntax/expected_results.json

@@ -1 +1 @@
-{ "version": 1, "status": "error", "message": "The submitted code didn't compile. We have collected the errors encountered during compilation. At this moment the error messages are not very read-friendly, but it's a start. We are working on a more helpful output.\n-------------------------------\ntwo-fer.ts(1,14): error TS1389: 'const' is not allowed as a variable declaration name.\ntwo-fer.ts(1,20): error TS1134: Variable declaration expected.\ntwo-fer.ts(1,29): error TS1109: Expression expected.\ntwo-fer.ts(2,1): error TS1005: ')' expected." }
+{ "version": 1, "status": "error", "message": "The submitted code didn't compile. We have collected the errors encountered during compilation. At this moment the error messages are not very read-friendly, but it's a start. We are working on a more helpful output.\n-------------------------------\ntwo-fer.ts(1,14): error TS1389: 'const' is not allowed as a variable declaration name.\ntwo-fer.ts(1,20): error TS1134: Variable declaration expected.\ntwo-fer.ts(1,29): error TS1109: Expression expected.\ntwo-fer.ts(2,1): error TS1005: ')' expected.\n" }

test/smoke.test.mjs

@@ -12,9 +12,9 @@ assertPass(
 )
 
 shelljs.echo(
-  'typescript-test-runner > passing solution > with output directory'
+  'typescript-test-runner > failing solution > with output directory'
 )
-assertPass('clock', join(fixtures, 'clock', 'pass'))
+assertError('clock', join(fixtures, 'clock', 'fail'))
 
 /** Test failures */
 const failures = ['tests', 'empty']

test/types.test.mjs

@@ -0,0 +1,13 @@
+import { join } from 'node:path'
+import shelljs from 'shelljs'
+import { assertError, assertPass } from './asserts.mjs'
+import { fixtures } from './paths.mjs'
+
+shelljs.echo('type tests (only) > documentation solution (smoke test)')
+assertPass('tstyche', join(fixtures, 'tstyche', 'documentation'))
+
+shelljs.echo('type tests (only) > failing solution')
+assertError('tstyche', join(fixtures, 'tstyche', 'fire'))
+
+shelljs.echo('type tests (only) > passing solution')
+assertPass('tstyche', join(fixtures, 'tstyche', 'firefought'))

tsconfig.solutions.json

@@ -0,0 +1,31 @@
+{
+  "extends": "@tsconfig/node20/tsconfig.json",
+  "compilerOptions": {
+    // Allows you to use the newest syntax, and have access to console.log
+    // https://www.typescriptlang.org/tsconfig#lib
+    "lib": ["ES2020", "dom"],
+    // Make sure typescript is configured to output ESM
+    // https://gist.github.com/sindresorhus/a39789f98801d908bbc7ff3ecc99d99c#how-can-i-make-my-typescript-project-output-esm
+    "module": "Node16",
+    // Since this project is using babel, TypeScript may target something very
+    // high, and babel will make sure it runs on your local Node version.
+    // https://babeljs.io/docs/en/
+    "target": "ES2020", // ESLint doesn't support this yet: "es2022",
+
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+
+    // Because jest-resolve isn't like node resolve, the absolute path must be .ts
+    "allowImportingTsExtensions": true,
+    "noEmit": true,
+
+    // Because we'll be using babel: ensure that Babel can safely transpile
+    // files in the TypeScript project.
+    //
+    // https://babeljs.io/docs/en/babel-plugin-transform-typescript/#caveats
+    "isolatedModules": true
+  },
+  "exclude": [".meta/*", "__typetests__/*", "*.test.ts", "*.tst.ts"]
+}

yarn.lock

@@ -1546,6 +1546,7 @@ __metadata:
     prettier: "npm:^3.3.3"
     rimraf: "npm:^6.0.1"
     shelljs: "npm:^0.8.5"
+    tstyche: "npm:^2.1.1"
     typescript: "npm:~5.5.4"
   bin:
     typescript-test-runner: bin/run.sh
@@ -6274,6 +6275,20 @@ __metadata:
   languageName: node
   linkType: hard
 
+"tstyche@npm:^2.1.1":
+  version: 2.1.1
+  resolution: "tstyche@npm:2.1.1"
+  peerDependencies:
+    typescript: 4.x || 5.x
+  peerDependenciesMeta:
+    typescript:
+      optional: true
+  bin:
+    tstyche: ./build/bin.js
+  checksum: 10/f30e7d782e51c262528ededf383c9daf39af8dea063d483667e3ff9f4800434891589c294c4b4f69802dd06daf8fb1d2a10553316d2f4631ba1413d3e48dab81
+  languageName: node
+  linkType: hard
+
 "type-check@npm:^0.4.0, type-check@npm:~0.4.0":
   version: 0.4.0
   resolution: "type-check@npm:0.4.0"