test(e2e): Add tests for Account Portal flows with both clerk v4 and …

…v5 (#2998)

* test(e2e): Add tests for Account Portal flows with both clerk v4 and v5

* test(e2e): Add fallback to npmjs when Verdaccio cannot find a package

* chore(e2e): Publish on `e2e` tag for tests

* chore(e2e): Exit pre mode to release on verdaccio

* chore(e2e): Exit pre mode to release on verdaccio

* chore(e2e): Use different config for Verdaccio when publishing

* test(e2e): Group duplicated code on AP tests

* test(e2e): Add webkit project and increase the timeout to 90s

* fix(shared): Fix issue when clearing the hash DB JWT in Webkit

* chore(e2e): Init the webkit browser

* chore(e2e): Clear Action cache

* chore(e2e): Fix tests that break on webkit

* chore(e2e): Run webkit tests in different jobs

* chore(e2e): Exclude some webkit tests

* chore(e2e): Exclude all webkit tests

* chore(e2e): Remove webkit from installing on CI

.changeset/chilled-apples-exercise.md

@@ -0,0 +1,5 @@
+---
+'@clerk/shared': patch
+---
+
+Fix issue when clearing the hash DB JWT in Webkit

.changeset/fuzzy-months-join.md

@@ -0,0 +1,2 @@
+---
+---

.github/actions/init/action.yml

@@ -113,9 +113,10 @@ runs:
       id: playwright-cache
       with:
         path: ~/.cache/ms-playwright
-        key: ${{ runner.os }}-node-${{ env.NODE_VERSION }}-playwright-${{ steps.playwright-version.outputs.VERSION }}
+        key: ${{ runner.os }}-node-${{ env.NODE_VERSION }}-playwright-${{ steps.playwright-version.outputs.VERSION }}-v2
 
     - name: Install Playwright Browsers
       if: inputs.playwright-enabled == 'true' && steps.playwright-cache.outputs.cache-hit != 'true'
       shell: bash
-      run: npx playwright install chromium
+      run: |
+        npx playwright install chromium

.github/actions/verdaccio/action.yml

@@ -28,13 +28,25 @@ runs:
       shell: bash
       run: npm set registry ${{ inputs.registry }}
 
-    - name: Run Verdaccio
+    - name: Run Verdaccio (using publish config)
       shell: bash
       run: |
-        nohup ./node_modules/.bin/verdaccio --config ./verdaccio.yaml & sleep 5
+        nohup ./node_modules/.bin/verdaccio --config ./verdaccio.publish.yaml & echo "VERDACCIO_PID=$!" >> $GITHUB_ENV
+        sleep 5
         ./node_modules/.bin/npm-cli-adduser -u ${{ inputs.auth-user }} -p ${{ inputs.auth-password }} -e ${{ inputs.auth-email }} -r ${{ inputs.registry }}
         ./node_modules/.bin/npm-cli-login -u ${{ inputs.auth-user }} -p ${{ inputs.auth-password }} -e ${{ inputs.auth-email }} -r ${{ inputs.registry }}
 
     - name: Publish to Verdaccio
       shell: bash
       run: ${{ inputs.publish-cmd }}
+
+    - name: Stop Verdaccio
+      shell: bash
+      run: kill -9 $VERDACCIO_PID
+
+    - name: Run Verdaccio (using install config)
+      shell: bash
+      run: |
+        nohup ./node_modules/.bin/verdaccio --config ./verdaccio.install.yaml & sleep 5
+        ./node_modules/.bin/npm-cli-adduser -u ${{ inputs.auth-user }} -p ${{ inputs.auth-password }} -e ${{ inputs.auth-email }} -r ${{ inputs.registry }}
+        ./node_modules/.bin/npm-cli-login -u ${{ inputs.auth-user }} -p ${{ inputs.auth-password }} -e ${{ inputs.auth-email }} -r ${{ inputs.registry }}

.github/workflows/ci.yml

@@ -138,7 +138,8 @@ jobs:
 
     strategy:
       matrix:
-        test-name: [ 'generic', 'nextjs', 'express', 'quickstart']
+        test-name: [ 'generic', 'nextjs', 'express', 'quickstart', 'ap-flows' ]
+        test-project: [ 'chrome' ]
 
     steps:
       - name: Checkout Repo
@@ -171,7 +172,7 @@ jobs:
         run: mkdir clerk-js && cd clerk-js && npm init -y && npm install @clerk/clerk-js
 
       - name: Run Integration Tests
-        run: npm run test:integration:${{ matrix.test-name }}
+        run: npm run test:integration:${{ matrix.test-name }} -- --project=${{ matrix.test-project }}
         env:
           E2E_APP_CLERK_JS_DIR: ${{runner.temp}}
           E2E_CLERK_VERSION: 'latest'

integration/README.md

@@ -18,6 +18,7 @@ Before you begin writing tests, you should already have:
 - Access to Clerk's 1Password
 - Access to the **Integration testing** organization on Clerk (it owns the Clerk instances used in the tests)
 - Chromium installed. Run `npx playwright install chromium`.
+- WebKit installed. Run `npx playwright install webkit`.
 
 ## Initial setup
 

integration/playwright.config.ts

@@ -16,7 +16,7 @@ export const common: PlaywrightTestConfig = {
   fullyParallel: true,
   forbidOnly: !!process.env.CI,
   retries: process.env.CI ? 2 : 0,
-  timeout: process.env.CI ? 90000 : 30000,
+  timeout: 90000,
   maxFailures: process.env.CI ? 1 : undefined,
   workers: process.env.CI ? numAvailableWorkers : '70%',
   reporter: process.env.CI ? 'line' : 'list',
@@ -43,5 +43,10 @@ export default defineConfig({
       use: { ...devices['Desktop Chrome'], channel: 'chrome' },
       dependencies: ['setup'],
     },
+    {
+      name: 'webkit',
+      use: { ...devices['Desktop Safari'] },
+      dependencies: ['setup'],
+    },
   ],
 });

integration/presets/envs.ts

@@ -55,9 +55,39 @@ const withEmailCodesQuickstart = withEmailCodes
   .removeEnvVariable('public', 'CLERK_SIGN_IN_URL')
   .removeEnvVariable('public', 'CLERK_SIGN_UP_URL');
 
+const withAPCore1ClerkLatest = environmentConfig()
+  .setId('withAPCore1ClerkLatest')
+  .setEnvVariable('public', 'CLERK_TELEMETRY_DISABLED', true)
+  .setEnvVariable('private', 'CLERK_SECRET_KEY', envKeys['with-email-codes'].sk)
+  .setEnvVariable('public', 'CLERK_PUBLISHABLE_KEY', envKeys['with-email-codes'].pk)
+  .setEnvVariable('public', 'CLERK_JS_URL', constants.E2E_APP_CLERK_JS || 'http://localhost:18211/clerk.browser.js');
+
+const withAPCore1ClerkV4 = environmentConfig()
+  .setId('withAPCore1ClerkV4')
+  .setEnvVariable('public', 'CLERK_TELEMETRY_DISABLED', true)
+  .setEnvVariable('private', 'CLERK_SECRET_KEY', envKeys['with-email-codes'].sk)
+  .setEnvVariable('public', 'CLERK_PUBLISHABLE_KEY', envKeys['with-email-codes'].pk);
+
+const withAPCore2ClerkLatest = environmentConfig()
+  .setId('withAPCore2ClerkLatest')
+  .setEnvVariable('public', 'CLERK_TELEMETRY_DISABLED', true)
+  .setEnvVariable('private', 'CLERK_SECRET_KEY', envKeys['core-2-all-enabled'].sk)
+  .setEnvVariable('public', 'CLERK_PUBLISHABLE_KEY', envKeys['core-2-all-enabled'].pk)
+  .setEnvVariable('public', 'CLERK_JS_URL', constants.E2E_APP_CLERK_JS || 'http://localhost:18211/clerk.browser.js');
+
+const withAPCore2ClerkV4 = environmentConfig()
+  .setId('withAPCore2ClerkV4')
+  .setEnvVariable('public', 'CLERK_TELEMETRY_DISABLED', true)
+  .setEnvVariable('private', 'CLERK_SECRET_KEY', envKeys['core-2-all-enabled'].sk)
+  .setEnvVariable('public', 'CLERK_PUBLISHABLE_KEY', envKeys['core-2-all-enabled'].pk);
+
 export const envs = {
   withEmailCodes,
   withEmailLinks,
   withCustomRoles,
   withEmailCodesQuickstart,
+  withAPCore1ClerkLatest,
+  withAPCore1ClerkV4,
+  withAPCore2ClerkLatest,
+  withAPCore2ClerkV4,
 } as const;

integration/presets/next.ts

@@ -24,8 +24,30 @@ const appRouterQuickstart = appRouter
   .setName('next-app-router-quickstart')
   .useTemplate(templates['next-app-router-quickstart']);
 
+const appRouterAPWithClerkNextLatest = appRouterQuickstart.clone().setName('next-app-router-ap-clerk-next-latest');
+
+const appRouterAPWithClerkNextV4 = appRouterQuickstart
+  .clone()
+  .setName('next-app-router-ap-clerk-next-v4')
+  .addDependency('@clerk/nextjs', '4')
+  .addFile(
+    'src/middleware.ts',
+    () => `import { authMiddleware } from '@clerk/nextjs';
+
+    export default authMiddleware({
+      publicRoutes: ['/']
+    });
+
+    export const config = {
+      matcher: ['/((?!.+\\.[\\w]+$|_next).*)', '/', '/(api|trpc)(.*)'],
+    };
+    `,
+  );
+
 export const next = {
   appRouter,
   appRouterTurbo,
   appRouterQuickstart,
+  appRouterAPWithClerkNextLatest,
+  appRouterAPWithClerkNextV4,
 } as const;

integration/templates/next-app-router-quickstart/src/app/page.tsx

@@ -1,12 +1,15 @@
-import { SignedIn, SignedOut, SignInButton, UserButton } from '@clerk/nextjs';
+import { SignedIn, SignedOut, SignInButton, SignUpButton, UserButton } from '@clerk/nextjs';
 
 export default function Home() {
   return (
     <header>
       <SignedOut>
+        <p>signed-out-state</p>
         <SignInButton />
+        <SignUpButton />
       </SignedOut>
       <SignedIn>
+        <p>signed-in-state</p>
         <UserButton />
       </SignedIn>
     </header>

integration/tests/next-account-portal/clerk-v4-ap-core-1.test.ts

@@ -0,0 +1,40 @@
+import { test } from '@playwright/test';
+
+import type { Application } from '../../models/application';
+import { appConfigs } from '../../presets';
+import type { FakeUser } from '../../testUtils';
+import { createTestUtils } from '../../testUtils';
+import { testSignIn, testSignUp, testSSR } from './common';
+
+test.describe('Next with ClerkJS V4 <-> Account Portal Core 1 @ap-flows', () => {
+  test.describe.configure({ mode: 'serial' });
+  let app: Application;
+  let fakeUser: FakeUser;
+
+  test.beforeAll(async () => {
+    app = await appConfigs.next.appRouterAPWithClerkNextV4.clone().commit();
+    await app.setup();
+    await app.withEnv(appConfigs.envs.withAPCore1ClerkV4);
+    await app.dev();
+    const u = createTestUtils({ app });
+    fakeUser = u.services.users.createFakeUser();
+    await u.services.users.createBapiUser(fakeUser);
+  });
+
+  test.afterAll(async () => {
+    await fakeUser.deleteIfExists();
+    await app.teardown();
+  });
+
+  test('sign in', async ({ page, context }) => {
+    await testSignIn({ app, page, context, fakeUser });
+  });
+
+  test('sign up', async ({ page, context }) => {
+    await testSignUp({ app, page, context, fakeUser });
+  });
+
+  test('ssr', async ({ page, context }) => {
+    await testSSR({ app, page, context, fakeUser });
+  });
+});

integration/tests/next-account-portal/clerk-v4-ap-core-2.test.ts

@@ -0,0 +1,40 @@
+import { test } from '@playwright/test';
+
+import type { Application } from '../../models/application';
+import { appConfigs } from '../../presets';
+import type { FakeUser } from '../../testUtils';
+import { createTestUtils } from '../../testUtils';
+import { testSignIn, testSignUp, testSSR } from './common';
+
+test.describe('Next with ClerkJS V4 <-> Account Portal Core 2 @ap-flows', () => {
+  test.describe.configure({ mode: 'serial' });
+  let app: Application;
+  let fakeUser: FakeUser;
+
+  test.beforeAll(async () => {
+    app = await appConfigs.next.appRouterAPWithClerkNextV4.clone().commit();
+    await app.setup();
+    await app.withEnv(appConfigs.envs.withAPCore2ClerkV4);
+    await app.dev();
+    const u = createTestUtils({ app });
+    fakeUser = u.services.users.createFakeUser();
+    await u.services.users.createBapiUser(fakeUser);
+  });
+
+  test.afterAll(async () => {
+    await fakeUser.deleteIfExists();
+    await app.teardown();
+  });
+
+  test('sign in', async ({ page, context }) => {
+    await testSignIn({ app, page, context, fakeUser });
+  });
+
+  test('sign up', async ({ page, context }) => {
+    await testSignUp({ app, page, context, fakeUser });
+  });
+
+  test('ssr', async ({ page, context }) => {
+    await testSSR({ app, page, context, fakeUser });
+  });
+});

integration/tests/next-account-portal/clerk-v5-ap-core-1.test.ts

@@ -0,0 +1,40 @@
+import { test } from '@playwright/test';
+
+import type { Application } from '../../models/application';
+import { appConfigs } from '../../presets';
+import type { FakeUser } from '../../testUtils';
+import { createTestUtils } from '../../testUtils';
+import { testSignIn, testSignUp, testSSR } from './common';
+
+test.describe('Next with ClerkJS V5 <-> Account Portal Core 1 @ap-flows', () => {
+  test.describe.configure({ mode: 'serial' });
+  let app: Application;
+  let fakeUser: FakeUser;
+
+  test.beforeAll(async () => {
+    app = await appConfigs.next.appRouterAPWithClerkNextLatest.clone().commit();
+    await app.setup();
+    await app.withEnv(appConfigs.envs.withAPCore1ClerkLatest);
+    await app.dev();
+    const u = createTestUtils({ app });
+    fakeUser = u.services.users.createFakeUser();
+    await u.services.users.createBapiUser(fakeUser);
+  });
+
+  test.afterAll(async () => {
+    await fakeUser.deleteIfExists();
+    await app.teardown();
+  });
+
+  test('sign in', async ({ page, context }) => {
+    await testSignIn({ app, page, context, fakeUser });
+  });
+
+  test('sign up', async ({ page, context }) => {
+    await testSignUp({ app, page, context, fakeUser });
+  });
+
+  test('ssr', async ({ page, context }) => {
+    await testSSR({ app, page, context, fakeUser });
+  });
+});

integration/tests/next-account-portal/clerk-v5-ap-core-2.test.ts

@@ -0,0 +1,40 @@
+import { test } from '@playwright/test';
+
+import type { Application } from '../../models/application';
+import { appConfigs } from '../../presets';
+import type { FakeUser } from '../../testUtils';
+import { createTestUtils } from '../../testUtils';
+import { testSignIn, testSignUp, testSSR } from './common';
+
+test.describe('Next with ClerkJS V5 <-> Account Portal Core 2 @ap-flows', () => {
+  test.describe.configure({ mode: 'serial' });
+  let app: Application;
+  let fakeUser: FakeUser;
+
+  test.beforeAll(async () => {
+    app = await appConfigs.next.appRouterAPWithClerkNextLatest.clone().commit();
+    await app.setup();
+    await app.withEnv(appConfigs.envs.withAPCore2ClerkLatest);
+    await app.dev();
+    const u = createTestUtils({ app });
+    fakeUser = u.services.users.createFakeUser();
+    await u.services.users.createBapiUser(fakeUser);
+  });
+
+  test.afterAll(async () => {
+    await fakeUser.deleteIfExists();
+    await app.teardown();
+  });
+
+  test('sign in', async ({ page, context }) => {
+    await testSignIn({ app, page, context, fakeUser });
+  });
+
+  test('sign up', async ({ page, context }) => {
+    await testSignUp({ app, page, context, fakeUser });
+  });
+
+  test('ssr', async ({ page, context }) => {
+    await testSSR({ app, page, context, fakeUser });
+  });
+});