test: roll test runner to 0.3.18

.github/workflows/tests.yml

@@ -53,6 +53,7 @@ jobs:
       fail-fast: false
       matrix:
         browser: [chromium, firefox, webkit]
+        shard: ['1', '2']
     runs-on: macos-latest
     steps:
     - uses: actions/checkout@v2
@@ -62,14 +63,14 @@ jobs:
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
     - run: npm run build
-    - run: npx test-runner test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json
+    - run: npx test-runner test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json --shard=${{ matrix.shard }}/2
       env:
         BROWSER: ${{ matrix.browser }}
         PWRUNNER_JSON_REPORT: "test-results/report.json"
     - uses: actions/upload-artifact@v1
       if: ${{ always() }}
       with:
-        name: ${{ matrix.browser }}-mac-test-results
+        name: ${{ matrix.browser }}-mac-test-results-${{ matrix.shard }}
         path: test-results
 
   test_win:

package.json

@@ -49,7 +49,7 @@
     "ws": "^7.3.1"
   },
   "devDependencies": {
-    "@playwright/test-runner": "^0.3.17",
+    "@playwright/test-runner": "^0.3.18",
     "@types/debug": "^4.1.5",
     "@types/extract-zip": "^1.6.2",
     "@types/mime": "^2.0.3",

src/client/browser.ts

@@ -26,7 +26,6 @@ import { headersObjectToArray } from '../utils/utils';
 export class Browser extends ChannelOwner<channels.BrowserChannel, channels.BrowserInitializer> {
   readonly _contexts = new Set<BrowserContext>();
   private _isConnected = true;
-  private _isClosedOrClosing = false;
   private _closedPromise: Promise<void>;
   _isRemote = false;
 
@@ -83,18 +82,22 @@ export class Browser extends ChannelOwner<channels.BrowserChannel, channels.Brow
   }
 
   async close(): Promise<void> {
-    return this._wrapApiCall('browser.close', async () => {
-      if (!this._isClosedOrClosing) {
-        this._isClosedOrClosing = true;
+    try {
+      await this._wrapApiCall('browser.close', async () => {
         await this._channel.close();
-      }
-      await this._closedPromise;
-    });
+        await this._closedPromise;
+      });
+    } catch (e) {
+      if (e.message === 'browser.close: Browser has been closed')
+        return;
+      if (e.message === 'browser.close: Target browser or context has been closed')
+        return;
+      throw e;
+    }
   }
 
   _didClose() {
     this._isConnected = false;
     this.emit(Events.Browser.Disconnected);
-    this._isClosedOrClosing = true;
   }
 }

test/browsertype-connect.spec.ts

@@ -202,4 +202,14 @@ describe('connect', suite => {
 
     await browser1.close();
   });
+
+  it('should not throw on close after disconnect', async ({browserType, remoteServer, server}) => {
+    const remote = await browserType.connect({ wsEndpoint: remoteServer.wsEndpoint() });
+    await remote.newPage();
+    await Promise.all([
+      new Promise(f => remote.on('disconnected', f)),
+      remoteServer.close()
+    ]);
+    await remote.close();
+  });
 });

test/browsertype-launch-server.spec.ts

@@ -26,9 +26,9 @@ describe('lauch server', suite => {
     await browserServer.close();
   });
 
-  it('should work with port', async ({browserType, defaultBrowserOptions, parallelIndex}) => {
-    const browserServer = await browserType.launchServer({ ...defaultBrowserOptions, port: 8800 + parallelIndex });
-    expect(browserServer.wsEndpoint()).toContain(String(8800 + parallelIndex));
+  it('should work with port', async ({browserType, defaultBrowserOptions, workerIndex}) => {
+    const browserServer = await browserType.launchServer({ ...defaultBrowserOptions, port: 8800 + workerIndex });
+    expect(browserServer.wsEndpoint()).toContain(String(8800 + workerIndex));
     await browserServer.close();
   });
 

test/playwright.fixtures.ts

@@ -97,11 +97,11 @@ export const options = {
   TRACING: valueFromEnv('TRACING', false),
 };
 
-defineWorkerFixture('httpService', async ({parallelIndex}, test) => {
+defineWorkerFixture('httpService', async ({workerIndex}, test) => {
   const assetsPath = path.join(__dirname, 'assets');
   const cachedPath = path.join(__dirname, 'assets', 'cached');
 
-  const port = 8907 + parallelIndex * 2;
+  const port = 8907 + workerIndex * 2;
   const server = await TestServer.create(assetsPath, port);
   server.enableHTTPCache(cachedPath);
 
@@ -139,7 +139,7 @@ defineWorkerFixture('defaultBrowserOptions', async ({browserName}, runTest, conf
   });
 });
 
-defineWorkerFixture('playwright', async ({browserName, parallelIndex, platform}, test) => {
+defineWorkerFixture('playwright', async ({browserName, workerIndex, platform}, test) => {
   assert(platform); // Depend on platform to generate all tests.
   const {coverage, uninstall} = installCoverageHooks(browserName);
   if (options.WIRE) {
@@ -172,7 +172,7 @@ defineWorkerFixture('playwright', async ({browserName, parallelIndex, platform},
 
   async function teardownCoverage() {
     uninstall();
-    const coveragePath = path.join(__dirname, 'coverage-report', parallelIndex + '.json');
+    const coveragePath = path.join(__dirname, 'coverage-report', workerIndex + '.json');
     const coverageJSON = [...coverage.keys()].filter(key => coverage.get(key));
     await fs.promises.mkdir(path.dirname(coveragePath), { recursive: true });
     await fs.promises.writeFile(coveragePath, JSON.stringify(coverageJSON, undefined, 2), 'utf8');
@@ -260,10 +260,9 @@ defineWorkerFixture('expectedSSLError', async ({browserName, platform}, runTest)
 });
 
 defineTestFixture('testOutputDir', async ({}, runTest, info) => {
-  const { spec, config } = info;
-  const relativePath = path.relative(config.testDir, spec.file).replace(/\.spec\.[jt]s/, '');
-  const sanitizedTitle = spec.title.replace(/[^\w\d]+/g, '_');
-  const testOutputDir = path.join(config.outputDir, relativePath, sanitizedTitle);
+  const relativePath = path.relative(info.config.testDir, info.file).replace(/\.spec\.[jt]s/, '');
+  const sanitizedTitle = info.title.replace(/[^\w\d]+/g, '_');
+  const testOutputDir = path.join(info.config.outputDir, relativePath, sanitizedTitle);
   await fs.promises.mkdir(testOutputDir, { recursive: true });
   await runTest(testOutputDir);
   const files = await fs.promises.readdir(testOutputDir);
@@ -291,8 +290,7 @@ defineTestFixture('context', async ({browser, testOutputDir}, runTest, info) =>
 defineTestFixture('page', async ({context, testOutputDir}, runTest, info) => {
   const page = await context.newPage();
   await runTest(page);
-  const { testRun } = info;
-  if (testRun.status === 'failed' || testRun.status === 'timedOut')
+  if (info.status === 'failed' || info.status === 'timedOut')
     await page.screenshot({ timeout: 5000, path: path.join(testOutputDir, 'test-failed.png') });
 });
 

test/proxy.spec.ts

@@ -121,7 +121,7 @@ it('should exclude patterns', (test, parameters) => {
 
 it('should use socks proxy', (test, parameters) => {
   test.flaky(options.MAC(parameters) && options.WEBKIT(parameters), 'Intermittent page.goto: The network connection was lost error on bots');
-}, async ({ browserType, defaultBrowserOptions, parallelIndex }) => {
+}, async ({ browserType, defaultBrowserOptions, workerIndex }) => {
   const server = socks.createServer((info, accept, deny) => {
     let socket;
     if ((socket = accept(true))) {
@@ -138,7 +138,7 @@ it('should use socks proxy', (test, parameters) => {
       ].join('\r\n'));
     }
   });
-  const socksPort = 9107 + parallelIndex * 2;
+  const socksPort = 9107 + workerIndex * 2;
   server.listen(socksPort, 'localhost');
   server.useAuth(socks.auth.None());
 

test/screencast.spec.ts

@@ -58,8 +58,7 @@ defineTestFixture('videoPlayer', async ({videoPlayerBrowser, server}, test) => {
 });
 
 defineTestFixture('relativeArtifactsPath', async ({browserType}, runTest, info) => {
-  const { spec } = info;
-  const sanitizedTitle = spec.title.replace(/[^\w\d]+/g, '_');
+  const sanitizedTitle = info.title.replace(/[^\w\d]+/g, '_');
   const relativeArtifactsPath = `${browserType.name()}-${sanitizedTitle}`;
   await runTest(relativeArtifactsPath);
 });