Require soft assertions (#118)

* Better scope for no-skipped-tests * Add examples folder for testing * Better scope for no-useless-not * max-nested-describe * missing-playwright-await * no-focused-test * no-force-option * no-page-pause * no-restricted-matchers * Update ecma version * no-wait-for-timeout * prefer-lowercase-title * prefer-strict-equal * prefer-to-be * prefer-to-have-length * require-top-level-describe * valid-expect * no-conditional-in-test * Update example eslintrc * no-eval * Cleanup * no-element-handle * Order * Use dedent * Update examples * Docs * Finish rule * Update docs
playwright-community · Jan 17, 2023 · 5c031ef · 5c031ef
1 parent e5958a0
commit 5c031ef
Show file tree

Hide file tree

Showing 6 changed files with 115 additions and 21 deletions.
diff --git a/README.md b/README.md
@@ -49,23 +49,24 @@ command line option.\
 💡: Some problems reported by this rule are manually fixable by editor
 [suggestions](https://eslint.org/docs/latest/developer-guide/working-with-rules#providing-suggestions).
 
-|  ✔  | 🔧  | 💡  | Rule                                                                                                                                              | Description                                                       |
-| :-: | :-: | :-: | ------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------- |
-|  ✔  |     |     | [max-nested-describe](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/max-nested-describe.md)               | Enforces a maximum depth to nested describe calls                 |
-|  ✔  | 🔧  |     | [missing-playwright-await](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/missing-playwright-await.md)     | Enforce Playwright APIs to be awaited                             |
-|  ✔  |     |     | [no-conditional-in-test](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/no-conditional-in-test.md)         | Disallow conditional logic in tests                               |
-|  ✔  |     | 💡  | [no-element-handle](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/no-element-handle.md)                   | Disallow usage of element handles                                 |
-|  ✔  |     |     | [no-eval](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/no-eval.md)                                       | Disallow usage of `page.$eval` and `page.$$eval`                  |
-|  ✔  |     | 💡  | [no-focused-test](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/no-focused-test.md)                       | Disallow usage of `.only` annotation                              |
-|  ✔  |     |     | [no-force-option](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/no-force-option.md)                       | Disallow usage of the `{ force: true }` option                    |
-|  ✔  |     |     | [no-page-pause](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/no-page-pause.md)                           | Disallow using `page.pause`                                       |
-|     |     |     | [no-restricted-matchers](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/no-restricted-matchers.md)         | Disallow specific matchers & modifiers                            |
-|  ✔  |     | 💡  | [no-skipped-test](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/no-skipped-test.md)                       | Disallow usage of the `.skip` annotation                          |
-|  ✔  | 🔧  |     | [no-useless-not](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/no-useless-not.md)                         | Disallow usage of `not` matchers when a specific matcher exists   |
-|  ✔  |     | 💡  | [no-wait-for-timeout](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/no-wait-for-timeout.md)               | Disallow usage of `page.waitForTimeout`                           |
-|     |     | 💡  | [prefer-strict-equal](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/prefer-strict-equal.md)               | Suggest using `toStrictEqual()`                                   |
-|     | 🔧  |     | [prefer-lowercase-title](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/prefer-lowercase-title.md)         | Enforce lowercase test names                                      |
-|     | 🔧  |     | [prefer-to-be](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/prefer-to-be.md)                             | Suggest using `toBe()`                                            |
-|     | 🔧  |     | [prefer-to-have-length](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/prefer-to-have-length.md)           | Suggest using `toHaveLength()`                                    |
-|     |     |     | [require-top-level-describe](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/require-top-level-describe.md) | Require test cases and hooks to be inside a `test.describe` block |
-|  ✔  |     |     | [valid-expect](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/valid-expect.md)                             | Enforce valid `expect()` usage                                    |
+|  ✔  | 🔧  | 💡  | Rule                                                                                                                                                        | Description                                                       |
+| :-: | :-: | :-: | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------- |
+|  ✔  |     |     | [max-nested-describe](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/max-nested-describe.md)                         | Enforces a maximum depth to nested describe calls                 |
+|  ✔  | 🔧  |     | [missing-playwright-await](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/missing-playwright-await.md)               | Enforce Playwright APIs to be awaited                             |
+|  ✔  |     |     | [no-conditional-in-test](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/no-conditional-in-test.md)                   | Disallow conditional logic in tests                               |
+|  ✔  |     | 💡  | [no-element-handle](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/no-element-handle.md)                             | Disallow usage of element handles                                 |
+|  ✔  |     |     | [no-eval](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/no-eval.md)                                                 | Disallow usage of `page.$eval` and `page.$$eval`                  |
+|  ✔  |     | 💡  | [no-focused-test](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/no-focused-test.md)                                 | Disallow usage of `.only` annotation                              |
+|  ✔  |     |     | [no-force-option](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/no-force-option.md)                                 | Disallow usage of the `{ force: true }` option                    |
+|  ✔  |     |     | [no-page-pause](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/no-page-pause.md)                                     | Disallow using `page.pause`                                       |
+|     |     |     | [no-restricted-matchers](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/no-restricted-matchers.md)                   | Disallow specific matchers & modifiers                            |
+|  ✔  |     | 💡  | [no-skipped-test](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/no-skipped-test.md)                                 | Disallow usage of the `.skip` annotation                          |
+|  ✔  | 🔧  |     | [no-useless-not](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/no-useless-not.md)                                   | Disallow usage of `not` matchers when a specific matcher exists   |
+|  ✔  |     | 💡  | [no-wait-for-timeout](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/no-wait-for-timeout.md)                         | Disallow usage of `page.waitForTimeout`                           |
+|     |     | 💡  | [prefer-strict-equal](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/prefer-strict-equal.md)                         | Suggest using `toStrictEqual()`                                   |
+|     | 🔧  |     | [prefer-lowercase-title](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/prefer-lowercase-title.md)                   | Enforce lowercase test names                                      |
+|     | 🔧  |     | [prefer-to-be](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/prefer-to-be.md)                                       | Suggest using `toBe()`                                            |
+|     | 🔧  |     | [prefer-to-have-length](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/prefer-to-have-length.md)                     | Suggest using `toHaveLength()`                                    |
+|     |     |     | [require-top-level-describe](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/require-top-level-describe.md)           | Require test cases and hooks to be inside a `test.describe` block |
+|     | 🔧  |     | [require-require-soft-assertions](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/require-require-soft-assertions.md) | Require assertions to use `expect.soft()`                         |
+|  ✔  |     |     | [valid-expect](https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/valid-expect.md)                                       | Enforce valid `expect()` usage                                    |
diff --git a/docs/rules/require-soft-assertions.md b/docs/rules/require-soft-assertions.md
@@ -0,0 +1,26 @@
+# Require soft assertions (`require-soft-assertions`)
+
+Some find it easier to write longer test that perform more assertions per test.
+In cases like these, it can be helpful to require
+[soft assertions](https://playwright.dev/docs/test-assertions#soft-assertions)
+in your tests.
+
+This rule is not enabled by default and is only intended to be used it if fits
+your workflow. If you aren't sure if you should use this rule, you probably
+shouldn't 🙂.
+
+## Rule Details
+
+Examples of **incorrect** code for this rule:
+
+```javascript
+await expect(page.locator('foo')).toHaveText('bar');
+await expect(page).toHaveTitle('baz');
+```
+
+Examples of **correct** code for this rule:
+
+```javascript
+await expect.soft(page.locator('foo')).toHaveText('bar');
+await expect.soft(page).toHaveTitle('baz');
+```
diff --git a/src/index.ts b/src/index.ts
@@ -14,6 +14,7 @@ import preferLowercaseTitle from './rules/prefer-lowercase-title';
 import preferToBe from './rules/prefer-to-be';
 import preferToHaveLength from './rules/prefer-to-have-length';
 import preferStrictEqual from './rules/prefer-strict-equal';
+import requireSoftAssertions from './rules/require-soft-assertions';
 import requireTopLevelDescribe from './rules/require-top-level-describe';
 import validExpect from './rules/valid-expect';
 
@@ -91,6 +92,7 @@ export = {
     'prefer-to-be': preferToBe,
     'prefer-to-have-length': preferToHaveLength,
     'require-top-level-describe': requireTopLevelDescribe,
+    'require-soft-assertions': requireSoftAssertions,
     'valid-expect': validExpect,
   },
 };
diff --git a/src/rules/require-soft-assertions.ts b/src/rules/require-soft-assertions.ts
@@ -0,0 +1,31 @@
+import { Rule } from 'eslint';
+import { getExpectType } from '../utils/ast';
+
+export default {
+  create(context) {
+    return {
+      CallExpression(node) {
+        if (getExpectType(node) === 'standalone') {
+          context.report({
+            node: node.callee,
+            messageId: 'requireSoft',
+            fix: (fixer) => fixer.insertTextAfter(node.callee, '.soft'),
+          });
+        }
+      },
+    };
+  },
+  meta: {
+    docs: {
+      description: 'Require all assertions to use `expect.soft`',
+      recommended: false,
+      url: 'https://github.com/playwright-community/eslint-plugin-playwright/tree/main/docs/rules/require-soft-assertions.md',
+    },
+    messages: {
+      requireSoft: 'Unexpected non-soft assertion',
+    },
+    fixable: 'code',
+    type: 'suggestion',
+    schema: [],
+  },
+} as Rule.RuleModule;
diff --git a/test/spec/require-soft-assertions.spec.ts b/test/spec/require-soft-assertions.spec.ts
@@ -0,0 +1,33 @@
+import rule from '../../src/rules/require-soft-assertions';
+import { runRuleTester } from '../utils/rule-tester';
+
+const messageId = 'requireSoft';
+
+runRuleTester('require-soft-assertions', rule, {
+  valid: [
+    'expect.soft(page).toHaveTitle("baz")',
+    'expect.soft(page.locator("foo")).toHaveText("bar")',
+    'expect["soft"](foo).toBe("bar")',
+    'expect[`soft`](bar).toHaveText("bar")',
+    'expect.poll(() => foo).toBe("bar")',
+    'expect["poll"](() => foo).toBe("bar")',
+    'expect[`poll`](() => foo).toBe("bar")',
+  ],
+  invalid: [
+    {
+      code: 'expect(page).toHaveTitle("baz")',
+      output: 'expect.soft(page).toHaveTitle("baz")',
+      errors: [{ messageId, line: 1, column: 1, endColumn: 7 }],
+    },
+    {
+      code: 'expect(page.locator("foo")).toHaveText("bar")',
+      output: 'expect.soft(page.locator("foo")).toHaveText("bar")',
+      errors: [{ messageId, line: 1, column: 1, endColumn: 7 }],
+    },
+    {
+      code: 'await expect(page.locator("foo")).toHaveText("bar")',
+      output: 'await expect.soft(page.locator("foo")).toHaveText("bar")',
+      errors: [{ messageId, line: 1, column: 7, endColumn: 13 }],
+    },
+  ],
+});
diff --git a/test/utils/rule-tester.ts b/test/utils/rule-tester.ts
@@ -12,7 +12,8 @@ import { RuleTester } from 'eslint';
 export function runRuleTester(...args: Parameters<RuleTester['run']>) {
   const config = {
     parserOptions: {
-      ecmaVersion: 2018,
+      ecmaVersion: 2022,
+      sourceType: 'module',
     },
   };