Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: convert puppeteer to TS #37

Merged
merged 8 commits into from
Aug 10, 2021
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .npmrc
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
legacy-peer-deps=true
szmarczak marked this conversation as resolved.
Show resolved Hide resolved
4 changes: 2 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
"ow": "^0.23.0",
"proxy-chain": "^1.0.2",
"tiny-typed-emitter": "^2.1.0",
"tslib": "^2.2.0"
"tslib": "^2.3.0"
},
"devDependencies": {
"@apify/eslint-config-ts": "^0.1.1",
Expand All @@ -69,6 +69,6 @@
"puppeteer": "^9.0.0",
"ts-jest": "^26.5.5",
"ts-node": "^9.1.1",
"typescript": "^4.2.4"
"typescript": "^4.3.5"
}
}
6 changes: 3 additions & 3 deletions src/abstract-classes/browser-controller.ts
Original file line number Diff line number Diff line change
Expand Up @@ -103,12 +103,12 @@ export abstract class BrowserController<
/**
* Browser representation of the underlying automation library.
*/
browser?: LaunchResult = undefined;
browser: LaunchResult = undefined!;

/**
* The configuration the browser was launched with.
*/
launchContext?: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult> = undefined;
launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult> = undefined!;

isActive = false;

Expand Down Expand Up @@ -204,7 +204,7 @@ export abstract class BrowserController<
* Opens new browser page.
* @ignore
*/
async newPage(pageOptions: NewPageOptions): Promise<NewPageResult> {
async newPage(pageOptions?: NewPageOptions): Promise<NewPageResult> {
this.activePages++;
this.totalPages++;
await this.isActivePromise;
Expand Down
3 changes: 2 additions & 1 deletion src/index.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
const BrowserPool = require('./browser-pool');
const PuppeteerPlugin = require('./puppeteer/puppeteer-plugin');
// eslint-disable-next-line import/extensions
const { PuppeteerPlugin } = require('./puppeteer/puppeteer-plugin');
const PlaywrightPlugin = require('./playwright/playwright-plugin');

/**
Expand Down
3 changes: 3 additions & 0 deletions src/launch-context.ts
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,9 @@ export class LaunchContext<

private readonly _reservedFieldNames = [...Reflect.ownKeys(this), 'extend'];

// TODO: change this to PropertyKey when TypeScript 4.4 releases
[K: string]: unknown;

constructor(options: LaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>) {
const {
id,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,17 +1,18 @@
const _ = require('lodash');

const { BrowserController } = require('../abstract-classes/browser-controller'); // eslint-disable-line import/extensions
import * as Puppeteer from 'puppeteer';
import { BrowserController, Cookie } from '../abstract-classes/browser-controller';
import { log } from '../logger';
import { noop } from '../utils';

const PROCESS_KILL_TIMEOUT_MILLIS = 5000;

/**
* puppeteer
* Puppeteer
*/
class PuppeteerController extends BrowserController {
async _newPage() {
export class PuppeteerController extends BrowserController<typeof Puppeteer> {
protected async _newPage(): Promise<Puppeteer.Page> {
const { useIncognitoPages } = this.launchContext;
let page;
let context;
let page: Puppeteer.Page;
let context: Puppeteer.BrowserContext;

if (useIncognitoPages) {
context = await this.browser.createIncognitoBrowserContext();
Expand All @@ -24,23 +25,23 @@ class PuppeteerController extends BrowserController {
this.activePages--;

if (useIncognitoPages) {
context.close().catch(_.noop);
context.close().catch(noop);
}
});

page.once('error', (error) => {
this.log.exception(error, 'Page crashed.');
page.close().catch(_.noop);
log.exception(error, 'Page crashed.');
page.close().catch(noop);
});

return page;
}

async _close() {
protected async _close(): Promise<void> {
await this.browser.close();
}

async _kill() {
protected async _kill(): Promise<void> {
const browserProcess = this.browser.process();

if (!browserProcess) {
Expand All @@ -52,7 +53,7 @@ class PuppeteerController extends BrowserController {
// This is here because users reported that it happened
// that error `TypeError: Cannot read property 'kill' of null` was thrown.
// Likely Chrome process wasn't started due to some error ...
if (browserProcess) browserProcess.kill('SIGKILL');
browserProcess?.kill('SIGKILL');
}, PROCESS_KILL_TIMEOUT_MILLIS);

try {
Expand All @@ -63,13 +64,11 @@ class PuppeteerController extends BrowserController {
}
}

async _getCookies(page) {
protected _getCookies(page: Puppeteer.Page): Promise<Cookie[]> {
return page.cookies();
}

async _setCookies(page, cookies) {
protected _setCookies(page: Puppeteer.Page, cookies: Cookie[]): Promise<void> {
return page.setCookie(...cookies);
}
}

module.exports = PuppeteerController;
Original file line number Diff line number Diff line change
@@ -1,18 +1,16 @@
const { BrowserPlugin } = require('../abstract-classes/browser-plugin'); // eslint-disable-line import/extensions
const PuppeteerController = require('./puppeteer-controller');
import * as Puppeteer from 'puppeteer';
import { BrowserController } from '../abstract-classes/browser-controller';
import { BrowserPlugin } from '../abstract-classes/browser-plugin';
import { LaunchContext } from '../launch-context';
import { PuppeteerController } from './puppeteer-controller';

const PROXY_SERVER_ARG = '--proxy-server=';

/**
* puppeteer
* Puppeteer
*/
class PuppeteerPlugin extends BrowserPlugin {
/**
* @param {LaunchContext} launchContext
* @return {Promise<Browser>}
* @private
*/
async _launch(launchContext) {
export class PuppeteerPlugin extends BrowserPlugin<typeof Puppeteer> {
protected async _launch(launchContext: LaunchContext<typeof Puppeteer>): Promise<Puppeteer.Browser> {
const {
launchOptions,
anonymizedProxyUrl,
Expand All @@ -21,39 +19,33 @@ class PuppeteerPlugin extends BrowserPlugin {

const finalLaunchOptions = {
...launchOptions,
userDataDir: launchOptions.userDataDir || userDataDir,
userDataDir: launchOptions?.userDataDir ?? userDataDir,
};

const browser = await this.library.launch(finalLaunchOptions);

if (anonymizedProxyUrl) {
browser.once('disconnected', () => {
this._closeAnonymizedProxy(anonymizedProxyUrl);
this._closeAnonymizedProxy(anonymizedProxyUrl as string);
});
}

return browser;
}

/**
* @return {PuppeteerController}
* @private
*/
_createController() {
protected _createController(): BrowserController<typeof Puppeteer> {
return new PuppeteerController(this);
}

/**
*
* @param launchContext {object}
* @return {Promise<void>}
* @private
*/
async _addProxyToLaunchOptions(launchContext) {
protected async _addProxyToLaunchOptions(
launchContext: LaunchContext<typeof Puppeteer>,
): Promise<void> {
launchContext.launchOptions ??= {};

const { launchOptions, proxyUrl } = launchContext;
let finalProxyUrl = proxyUrl;

if (this._shouldAnonymizeProxy(proxyUrl)) {
if (proxyUrl && this._shouldAnonymizeProxy(proxyUrl)) {
finalProxyUrl = await this._getAnonymizedProxyUrl(proxyUrl);
launchContext.anonymizedProxyUrl = finalProxyUrl;
}
Expand All @@ -67,5 +59,3 @@ class PuppeteerPlugin extends BrowserPlugin {
}
}
}

module.exports = PuppeteerPlugin;
3 changes: 3 additions & 0 deletions src/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,6 @@ export function addTimeoutToPromise<T>(promise: Promise<T>, timeoutMillis: numbe
};

export type UnwrapPromise<T> = T extends PromiseLike<infer R> ? UnwrapPromise<R> : T;

// eslint-disable-next-line @typescript-eslint/no-empty-function, @typescript-eslint/no-unused-vars
export function noop(..._args: unknown[]): void {}
5 changes: 3 additions & 2 deletions test/browser-plugins/plugins.test.js
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
/* eslint-disable import/extensions */
const puppeteer = require('puppeteer');
const playwright = require('playwright');
const fs = require('fs');

const PuppeteerPlugin = require('../../src/puppeteer/puppeteer-plugin');
const PuppeteerController = require('../../src/puppeteer/puppeteer-controller');
const { PuppeteerPlugin } = require('../../src/puppeteer/puppeteer-plugin');
const { PuppeteerController } = require('../../src/puppeteer/puppeteer-controller');

const PlaywrightPlugin = require('../../src/playwright/playwright-plugin');
const PlaywrightController = require('../../src/playwright/playwright-controller');
Expand Down
3 changes: 2 additions & 1 deletion test/browser-pool.test.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
/* eslint-disable import/extensions */
const puppeteer = require('puppeteer');
const playwright = require('playwright');
const BrowserPool = require('../src/browser-pool');
const PuppeteerPlugin = require('../src/puppeteer/puppeteer-plugin');
const { PuppeteerPlugin } = require('../src/puppeteer/puppeteer-plugin');
const PlaywrightPlugin = require('../src/playwright/playwright-plugin');
const {
BROWSER_POOL_EVENTS: {
Expand Down
3 changes: 2 additions & 1 deletion test/index.test.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
/* eslint-disable import/extensions */
const BrowserPool = require('../src/browser-pool');
const PuppeteerPlugin = require('../src/puppeteer/puppeteer-plugin');
const { PuppeteerPlugin } = require('../src/puppeteer/puppeteer-plugin');
const PlaywrightPlugin = require('../src/playwright/playwright-plugin');

const modules = require('../src/index');
Expand Down