diff --git a/.babelrc.json b/.babelrc.json index 7c2cc4b..9eabc79 100644 --- a/.babelrc.json +++ b/.babelrc.json @@ -3,7 +3,7 @@ "@babel/preset-typescript", ["@babel/preset-env", { "targets": { - "node": "10" + "node": "current" } }] ] diff --git a/.circleci/config.yml b/.circleci/config.yml index 7a4abdf..6c9a8cc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,9 +18,17 @@ jobs: command: npm run test -- --ci --reporters="default" --reporters="jest-junit" environment: JEST_JUNIT_OUTPUT: "/tmp/junit/jest.xml" + - run: npm run build + - run: + name: "Prepare Integration Tests" + command: npm install ../../ + working_directory: docs/example + - run: + name: "Run integration tests" + command: node_modules/.bin/nightcrawler --config nightcrawler.js crawl + working_directory: docs/example - store_test_results: path: /tmp/junit - - run: npm run build - persist_to_workspace: root: /srv paths: diff --git a/.eslintrc.js b/.eslintrc.js index 69127e8..8b23650 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -1,5 +1,8 @@ module.exports = { root: true, + env: { + "node": true + }, parser: '@typescript-eslint/parser', plugins: ['@typescript-eslint'], extends: [ diff --git a/.flowconfig b/.flowconfig deleted file mode 100644 index 1fed445..0000000 --- a/.flowconfig +++ /dev/null @@ -1,11 +0,0 @@ -[ignore] - -[include] - -[libs] - -[lints] - -[options] - -[strict] diff --git a/.gitignore b/.gitignore index f27c470..f74da62 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ node_modules/ yarn.lock *.log -nightcrawler.js -dist/ \ No newline at end of file +/nightcrawler.js +dist/ diff --git a/README.md b/README.md index e47509d..2c0a232 100644 --- a/README.md +++ b/README.md @@ -12,24 +12,21 @@ yarn add lastcall-nightcrawler Define your crawler by creating a `nightcrawler.js` file, like this: ```js # nightcrawler.js -const Crawler = require('lastcall-nightcrawler'); -const Number = Crawler.metrics.Number; - -const myCrawler = new Crawler('My Crawler'); - -myCrawler.on('setup', function(crawler) { - // On setup, give the crawler a list of URLs to crawl. - crawler.enqueue('http://localhost/'); - crawler.enqueue('http://localhost/foo'); +const {crawl, test} = require('./dist'); +const expect = require('expect'); + +module.exports = crawl('Response code validation', function() { + test('Should return 2xx', function(unit) { + expect(unit.response.statusCode).toBeGreaterThanOrEqual(200); + expect(unit.response.statusCode).toBeLessThan(300); + }); + + return [ + {url: 'https://example.com'}, + {url: 'https://example.com?q=1'}, + {url: 'https://example.com?q=2'} + ]; }); - -myCrawler.on('analyze', function(crawlReport, analysis) { - // On analysis, derive the metrics you need from the - // array of collected data. - analysis.addMetric('count', new Number('Total Requests', 0, crawlReport.data.length)); -}); - -module.exports = myCrawler; ``` Run your crawler: ```bash @@ -37,116 +34,92 @@ Run your crawler: node_modules/.bin/nightcrawler crawl ``` -Queueing Requests ------------------ -Requests can be queued during the `setup` event. You can queue a new request by calling the `enqueue()` method, using either a string (representing the URL) or an object containing a `url` property. If you pass an object, you will have access to that object's properties later on during analysis. +Specifying what URLs to crawl +----------------------------- +The `crawl` function expects a return value of an iterable (or async iterable) containing "requests". The simplest version of this is just an array of objects that have a `url` property. Eg: ```js -myCrawler.on('setup', function(crawler) { - // This works - crawler.enqueue('http://localhost/'); - // So does this: - crawler.enqueue({ - url: 'http://localhost/foo', - group: 'awesome' - }); -}); +module.exports = crawl('Crawl a static list of URLs', function() { -myCrawler.on('analyze', function(crawlReport, analysis) { - var awesomeRequests = crawlReport.data.filter(function(point) { - // *group property is only available if you added it during queuing. - return point.group === 'awesome'; - }); - // Do additional analysis only on pages in the awesome group. - analysis.addMetric('awesome.count', new Number('Awesome Requests', 0, awesomeRequests.length)); -}) + return [ + {url: 'https://example.com'} + ] +}); ``` -Collecting data ---------------- -By default, only the following information is collected for each response: -* `url` (string) : The URL that was crawled. -* `error` (bool) : Whether the response was determined to be an error response. -* `status` (int): The HTTP status code received. -* `backendResponseTime` (int): The duration of HTTP server response (see the [request module's documentation](https://github.com/request/request) on `timingPhases.firstByte`). +For more advanced use cases, you may want to use async generators to fetch a list of URLs from somewhere else (eg: a database). Eg: -If there is other data you're interested in knowing, you can collect it like this: ```js -// Collect the `Expires` header for each request. -myCrawler.on('response', function(response, data) { - data.expires = response.headers['expires']; -}); +async function* getURLs() { + const result = await queryDB(); + for(const url of result) { + yield {url: url}; + } +} + +module.exports = crawl('Crawl a dynamic list of URLs', function() { + + return getURLs(); +}) ``` -The response event is triggered on request success or error, as long as the server sends a response. Anything put into the `data` object will end up in the final JSON report. +Performing assertions on responses +---------------------------------- -Dynamic Crawling ----------------- -You may wish to be able to crawl a list of URLs that isn't static (it's determined at runtime). For example, you may want to query a remote API or a database and enqueue a list of URLs based on that data. To support this, the `setup` event allows you to return a promise. +One of the primary goals of Nightcrawler is to detect URLs that don't meet your expectations. To achieve this, you can use the `test` function within a `crawl` to make assertions about the response received. ```js -// Fetch a list of URLs from a remote API, then enqueue them all. -myCrawler.on('setup', function(crawler) { - return fetchData().then(function(myData) { - myData.forEach(function(url) { - crawler.enqueue(url); - }) - }) -}) -``` +const {crawl, test} = require('./dist'); +// Use the expect module from NPM for assertions. +// You can use any assertion library, including the built-in assert module. +const expect = require('expect'); -Analysis --------- -Once the crawl has been completed, you will probably want to analyze the data in some way. Data analysis in Nightcrawler is intentionally loose - the crawler fires an `analyze` event with an array of collected data, and you are responsible for analyzing your own data. Here are some examples of things you might do during analysis: - - ```js -const Crawler = require('lastcall-nightcrawler'); -const Number = Crawler.metrics.Number; -const Milliseconds = Crawler.metrics.Milliseconds; -const Percent = Crawler.metrics.Percent; - -myCrawler.on('analyze', function(crawlReport, analysis) { - var data = crawlReport.data; +module.exports = crawl('Check that the homepage is cacheable', function() { - // Calculate the number of requests that were made: - analysis.addMetric('count', new Number('Total Requests', 0, data.length)); - - // Calculate the average response time: - var avgTime = data.reduce(function(sum, dataPoint) { - return sum + dataPoint.backendTime - }, 0) / data.length; - analysis.addMetric('time', new Milliseconds('Avg Response Time', 0, avgTime)); - - // Calculate the percent of requests that were marked failed: - var failRatio = data.filter(function(dataPoint) { - return dataPoint.fail === true; - }).length / data.length; - var level = failRatio > 0 ? 2 : 0; - analysis.addMetric('fail', new Percent('% Failed', level, failRatio)); - - // Calculate the percent of requests that resulted in a 500 response. - var serverErrorRatio = data.filter(function(dataPoint) { - return dataPoint.statusCode >= 500; - }).length / data.length; - var level = serverErrorRatio > 0 ? 2 : 0; - analysis.add('500', new Percent('% 500', level, serverErrorRatio)); + test('Should have cache-control header', function(unit) { + expect(unit.response.headers).toHaveProperty('cache-control'); + expect(unit.response.headers['cache-control']).toBe('public; max-age: 1800'); + }) + + return [{url: 'https://example.com/'}] }); ``` -The [`analysis`](./src/analysis.js) object can consist of many metrics, added through the `add` method. See [`src/metrics.js`](./src/metrics.js) for more information about metrics. -Analysis can also be performed on individual requests to mark them passed or failed. +The `test` function will receive a `unit` of crawler work, which includes the following properties: + +* `request`: The request, as you passed it into the Crawler. This will include any additional properties you passed in, and you can use those properties to do conditional checking of units of work. +* `response`: The response object, as returned by the Driver. The default `NativeDriver` will produce a response in the shape of a Node [`http.IncomingMessage`](https://nodejs.org/api/http.html#http_class_http_incomingmessage) object. All `response` objects are guaranteed to have both a `statusCode` and a `time` property. + +Performing assertions about the overall status of the crawl +----------------------------------------------------------- + +For some use cases, you will want make assertions about many requests. For example, checking the average response type of all requests. To do this, you may use the `after` function to perform assertions after all the URLs have been requested. Just use the `test` function to collect the data you need from each request, then perform the final assertion in `after`: ```js -myCrawler.on('analyze', function(crawlReport, analysis) { - var data = crawlReport.data; +const {crawl, test, after} = require('./dist'); +const expect = require('expect'); - data.forEach(function(request) { - var level = request.statusCode > 499 ? 2 : 0 - analysis.addResult(request.url, level) - }); -}) +module.exports = crawl('Check that pages load quickly', function() { + const times = []; + + test('Collect response time', function(unit) { + times.push(unit.response.time); + }) + + after('Response time should be less than 500ms', function() { + const sum = times.reduce((total, value) => total + value, 0); + expect(sum / times.length).toBeLessThan(500); + }) + + return [{url: 'https://example.com/'}] +}); ``` +Drivers +------- + +Right now, there is only one "Driver" available for making requests. It uses Node's built-in `http` and `https` modules to issue HTTP requests to the target URL. In the future, we may have additional drivers available. + CI Setup -------- To add Nightcrawler to CircleCI make sure to the following steps are done: diff --git a/bin/nightcrawler b/bin/nightcrawler index 952b3db..09ce553 100755 --- a/bin/nightcrawler +++ b/bin/nightcrawler @@ -1,3 +1,11 @@ #!/usr/bin/env node -require('../dist/cli'); \ No newline at end of file +const cli = require('../dist/cli').default; +const {FailedAnalysisError} = require('../dist/cli/errors') +cli(process.argv, process.stdout, process.cwd()).then( + () => process.exit(0), + (err) => { + console.error(err instanceof FailedAnalysisError ? err.message : err); + process.exit(1); + } +) diff --git a/docs/example/.gitignore b/docs/example/.gitignore new file mode 100644 index 0000000..15813be --- /dev/null +++ b/docs/example/.gitignore @@ -0,0 +1,2 @@ +package-lock.json +node_modules/ diff --git a/docs/example/nightcrawler.js b/docs/example/nightcrawler.js new file mode 100644 index 0000000..7f17c23 --- /dev/null +++ b/docs/example/nightcrawler.js @@ -0,0 +1,34 @@ + +const {crawl, test, after} = require('lastcall-nightcrawler'); +const expect = require('expect'); + +module.exports = crawl('Homepage', function() { + const times = []; + + // Tests run for every request/response cycle. + test('Status code is 2xx', function(unit) { + expect(unit.response.statusCode).toBeGreaterThanOrEqual(200); + expect(unit.response.statusCode).toBeLessThan(300); + }); + + test('Has a long cache lifetime', function(unit) { + expect(unit.response.headers).toHaveProperty('cache-control', 'max-age=604800'); + }) + + test('Collect response time', function(unit) { + times.push(unit.response.time); + }) + + // After functions run after all responses have been received. + after('Average response time should be < 200ms', function() { + const sum = times.reduce((total, value) => total + value, 0); + expect(sum / times.length).toBeLessThan(200); + }) + + // Return any iterable/async iterable filled with request-shaped objects. + return [ + // options can be used to pass options to the driver. + // For example, passing {auth: 'foo:bar'} will enable basic auth. + {url: 'https://www.example.com/', options: {}} + ] +}) diff --git a/docs/example/package.json b/docs/example/package.json new file mode 100644 index 0000000..ebc4fda --- /dev/null +++ b/docs/example/package.json @@ -0,0 +1,7 @@ +{ + "name": "lastcall-nightcrawler-examples", + "dependencies": { + "expect": "^25.1.0", + "lastcall-nightcrawler": "^2.0.0" + } +} diff --git a/package.json b/package.json index 1a939d6..c63ed32 100644 --- a/package.json +++ b/package.json @@ -7,10 +7,10 @@ "chalk": "^2.0.0", "debug": "^3.0.0", "indent-string": "^4.0.0", + "minimist": "^1.2.5", "strip-ansi": "^6.0.0", "wrap-ansi": "^6.0.0", - "xml": "^1.0.0", - "yargs": "^15.0.0" + "xml": "^1.0.0" }, "bin": { "nightcrawler": "bin/nightcrawler" @@ -24,6 +24,7 @@ "@babel/preset-typescript": "^7.8.3", "@types/debug": "^4.1.5", "@types/jest": "^25.1.2", + "@types/minimist": "^1.2.0", "@types/node": "^13.7.2", "@types/tmp": "^0.1.0", "@types/wrap-ansi": "^3.0.0", @@ -43,8 +44,8 @@ "scripts": { "prettier": "prettier --single-quote --write './src/**/*.ts'", "test": "jest", - "check-types": "tsc", - "build": "babel src/ --out-dir=dist --extensions '.ts' --ignore 'src/**/__tests__/**'", + "check-types": "tsc --noEmit", + "build": "babel src/ --out-dir=dist --extensions '.ts' --ignore 'src/**/__tests__/**' --ignore 'src/**/__mocks__/**' --ignore 'src/**/__stubs__/**' && tsc --emitDeclarationOnly", "lint": "eslint ./src/ --ext .js,.jsx,.ts,.tsx" }, "files": [ diff --git a/src/Crawler.ts b/src/Crawler.ts index a4643c4..0689094 100644 --- a/src/Crawler.ts +++ b/src/Crawler.ts @@ -1,17 +1,25 @@ import debug from 'debug'; -import NativeDriver from './driver/native'; -import { isCrawlerRequest, toAsyncIterable } from './util'; +import native from './driver/native'; +import { toAsyncIterable } from './util'; const log = debug('nightcrawler:info'); const error = debug('nightcrawler:error'); import { RequestIterable, Driver, CrawlerRequest, CrawlerUnit } from './types'; +function isCrawlerRequest(request: unknown): request is CrawlerRequest { + return ( + request !== null && + typeof request === 'object' && + typeof (request as CrawlerRequest).url === 'string' + ); +} + export default class Crawler { driver: Driver; iterator: AsyncIterable; - constructor(requests: RequestIterable, driver: Driver = new NativeDriver()) { + constructor(requests: RequestIterable, driver: Driver = native) { this.iterator = toAsyncIterable(requests); this.driver = driver; } @@ -71,11 +79,11 @@ export default class Crawler { * @param req * @returns {Promise.} */ - async _fetch(req: CrawlerRequest): Promise { + private async _fetch(req: CrawlerRequest): Promise { log(`Fetching ${req.url}`); try { - const res = await this.driver.fetch(req); + const res = await this.driver(req.url, req.options); return { request: req, response: res diff --git a/src/__tests__/crawler.ts b/src/__tests__/crawler.ts index af2016b..93650b0 100644 --- a/src/__tests__/crawler.ts +++ b/src/__tests__/crawler.ts @@ -1,40 +1,49 @@ import Crawler from '../Crawler'; -import DummyDriver from '../driver/dummy'; import { performance } from 'perf_hooks'; -import { CrawlerRequest } from '../types'; -import NativeDriver from '../driver/native'; +import { CrawlerRequest, Driver } from '../types'; +import native from '../driver/native'; +import { all } from '../util'; -async function all( - iterator: AsyncIterable -): Promise { - const collected = []; - for await (const i of iterator) { - collected.push(i); - } - return collected; +function delay(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); } +const dummy: Driver = ( + url: string, + options: { delay?: number; shouldFail?: string } = {} +) => { + const start = performance.now(); + const delayMS = options.delay ?? 0; + if (options.shouldFail !== undefined) { + return delay(delayMS).then(() => Promise.reject(options.shouldFail)); + } + return delay(delayMS).then(() => ({ + statusCode: 200, + time: performance.now() - start + })); +}; + describe('Crawler', () => { it('Should default to the native driver', async () => { const c = new Crawler([]); - expect(c.driver).toBeInstanceOf(NativeDriver); + expect(c.driver).toBe(native); }); it('Should crawl an array of URLs', async () => { - const c = new Crawler([{ url: 'foo' }], new DummyDriver()); + const c = new Crawler([{ url: 'foo' }], dummy); await expect(all(c.crawl())).resolves.toHaveLength(1); }); it('Should crawl a generator of URLs', async () => { const requests = (function*(): Iterable { yield { url: 'foo' }; })(); - const c = new Crawler(requests, new DummyDriver()); + const c = new Crawler(requests, dummy); await expect(all(c.crawl())).resolves.toHaveLength(1); }); it('Should crawl an async generator of URLs', async () => { const requests = (async function*(): AsyncIterable { yield { url: 'foo' }; })(); - const c = new Crawler(requests, new DummyDriver()); + const c = new Crawler(requests, dummy); await expect(all(c.crawl())).resolves.toHaveLength(1); }); it('Should throw a meaningful error when an invalid iterator is given', function() { @@ -54,13 +63,13 @@ describe('Crawler', () => { for (let i = 0; i < 4; i++) { yield { url: i.toString(), - delay: 200, + options: { delay: 200 }, created: performance.now() as number }; } })(); const start = performance.now(); - const c = new Crawler(requests, new DummyDriver()); + const c = new Crawler(requests, dummy); const crawl = await all(c.crawl(2)); expect(crawl).toHaveLength(4); expect(crawl[0].request.created).toBeLessThan(start + 10); @@ -69,7 +78,7 @@ describe('Crawler', () => { expect(crawl[3].request.created).toBeGreaterThan(start + 200); }); - it('Should respect concurrency in crawling', async () => { + it('Should respect concurrency in crawling - 2', async () => { /** * This test checks to make sure we are respecting concurrency settings. * It uses the request iterable as a proxy for making sure we don't have @@ -86,7 +95,7 @@ describe('Crawler', () => { }; } })(); - const c = new Crawler(requests, new DummyDriver()); + const c = new Crawler(requests, dummy); const iterator = c.crawl(2); await iterator.next(); // When we've consumed 1 item, there should be 1 resolved, 2 in the pool. @@ -100,8 +109,8 @@ describe('Crawler', () => { }); it('Should return errors via the generator', async () => { - const request = { url: 'foo', shouldFail: 'foo' }; - const c = new Crawler([request], new DummyDriver()); + const request = { url: 'foo', options: { shouldFail: 'foo' } }; + const c = new Crawler([request], dummy); const result = await all(c.crawl()); expect(result).toEqual([ { diff --git a/src/__tests__/index.ts b/src/__tests__/index.ts new file mode 100644 index 0000000..66d6089 --- /dev/null +++ b/src/__tests__/index.ts @@ -0,0 +1,28 @@ +import * as index from '../'; + +describe('Index', function() { + it('Should export the crawl function', function() { + expect(index).toHaveProperty('crawl'); + expect(typeof index.crawl).toBe('function'); + }); + it('Should export the test function', function() { + expect(index).toHaveProperty('test'); + expect(typeof index.test).toBe('function'); + }); + + it('Should export the after function', function() { + expect(index).toHaveProperty('after'); + expect(typeof index.after).toBe('function'); + }); + + it('Should export the crawler function', function() { + expect(index).toHaveProperty('Crawler'); + expect(typeof index.Crawler).toBe('function'); + }); + + it('Should not export any unknown properties', function() { + const knowns = ['test', 'after', 'crawl', 'Crawler']; + const unknowns = Object.keys(index).filter(k => !knowns.includes(k)); + expect(unknowns).toEqual([]); + }); +}); diff --git a/src/cli/__stubs__/noexport.js b/src/cli/__stubs__/noexport.js new file mode 100644 index 0000000..eff670a --- /dev/null +++ b/src/cli/__stubs__/noexport.js @@ -0,0 +1 @@ +/* istanbul ignore file */ diff --git a/src/cli/__stubs__/ok.js b/src/cli/__stubs__/ok.js new file mode 100644 index 0000000..54873d3 --- /dev/null +++ b/src/cli/__stubs__/ok.js @@ -0,0 +1,4 @@ +/* istanbul ignore file */ +const TestContext = require('../../testing/TestContext').default; + +module.exports = new TestContext(''); diff --git a/src/cli/__tests__/cli.ts b/src/cli/__tests__/cli.ts new file mode 100644 index 0000000..9b07830 --- /dev/null +++ b/src/cli/__tests__/cli.ts @@ -0,0 +1,142 @@ +import cli from '../'; +import help from '../commands/help'; +import version from '../commands/version'; +import crawl from '../commands/crawl'; +import { loadContext } from '../util'; +import stream from 'stream'; +import { mocked } from 'ts-jest/utils'; +import TestContext from '../../testing/TestContext'; + +jest.mock('../util'); +jest.mock('../commands/help'); +jest.mock('../commands/version'); +jest.mock('../commands/crawl'); +jest.mock('../../testing/TestContext'); + +class MockTTY extends stream.PassThrough { + columns: number; + constructor() { + super(); + this.columns = 60; + } +} + +describe('CLI', function() { + let stdout: MockTTY; + const cwd = '/foo'; + + beforeEach(() => { + stdout = new MockTTY(); + mocked(loadContext).mockClear(); + mocked(crawl).mockClear(); + }); + + it('Should trigger help if the --help flag is passed', async function() { + await cli(['--help'], stdout, cwd); + expect(help).toHaveBeenCalledWith(stdout); + }); + + it('Should trigger version if the --version flag is passed', async function() { + await cli(['--version'], stdout, cwd); + expect(version).toHaveBeenCalledWith(stdout); + }); + + it('Should should attempt to load the configuration from the default location', async function() { + await cli([], stdout, cwd); + expect(loadContext).toHaveBeenCalledWith('./nightcrawler.js', cwd); + }); + + it('Should accept a flag for the configuration to load', async function() { + await cli(['--config', 'foo.js'], stdout, cwd); + expect(loadContext).toHaveBeenCalledTimes(1); + expect(loadContext).toHaveBeenCalledWith('foo.js', cwd); + }); + + it('Should should pass through errors when loading context.', async function() { + const mockedLoadContext = mocked(loadContext); + mockedLoadContext.mockImplementationOnce(() => { + throw new Error('foo'); + }); + await expect(cli([], stdout, cwd)).rejects.toThrow('foo'); + }); + + it('Should pass test context through to crawl', async function() { + const ctx = new TestContext(''); + const mockedLoadContext = mocked(loadContext); + mockedLoadContext.mockImplementationOnce(() => ctx); + await cli([], stdout, cwd); + expect(crawl).toHaveBeenCalledWith(expect.objectContaining({ + context: ctx + }), expect.anything()); + }) + + it('Should pass default values to crawl', async function() { + await cli([], stdout, cwd); + expect(crawl).toHaveBeenCalledWith( + expect.objectContaining({ + concurrency: 5, + silent: false + }), + expect.anything() + ); + expect(crawl).toHaveBeenCalledWith( + expect.not.objectContaining({ + junit: expect.anything(), + json: expect.anything() + }), + expect.anything() + ); + }); + + it('Should pass stdout to crawl', async function() { + await cli([], stdout, cwd); + expect(crawl).toHaveBeenCalledWith(expect.anything(), stdout); + }); + + it('Should massage concurrency values', async function() { + await cli(['--concurrency', '15'], stdout, cwd); + expect(crawl).toHaveBeenCalledWith( + expect.objectContaining({ + concurrency: 15 + }), + expect.anything() + ); + await cli(['--concurrency'], stdout, cwd); + expect(crawl).toHaveBeenCalledWith( + expect.objectContaining({ + concurrency: 5 + }), + expect.anything() + ); + }); + + it('Should pass junit flag', async function() { + await cli(['--junit', 'test.xml'], stdout, cwd); + expect(crawl).toHaveBeenCalledWith( + expect.objectContaining({ + junit: 'test.xml' + }), + expect.anything() + ); + }); + + it('Should pass json flag', async function() { + await cli(['--json', 'test.json'], stdout, cwd); + expect(crawl).toHaveBeenCalledWith( + expect.objectContaining({ + json: 'test.json' + }), + expect.anything() + ); + }); + + it('Should pass silent flag', async function() { + await cli(['--silent'], stdout, cwd); + expect(crawl).toHaveBeenCalledWith( + expect.objectContaining({ + silent: true + }), + expect.anything() + ); + }); +}); diff --git a/src/cli/__tests__/util.ts b/src/cli/__tests__/util.ts index cd8b841..fd8b2e9 100644 --- a/src/cli/__tests__/util.ts +++ b/src/cli/__tests__/util.ts @@ -1,5 +1,6 @@ import { TestResultMap, TestResult } from '../../testing/TestContext'; -import { hasFailure } from '../util'; +import { hasFailure, loadContext } from '../util'; +import path from 'path'; function r(obj: { [k: string]: TestResult }): TestResultMap { return new Map(Object.entries(obj)); @@ -26,3 +27,25 @@ describe('hasFailure', function() { expect(hasFailure(result)).toEqual(true); }); }); + +describe('loadContext', function() { + const cwd = path.join(__dirname, '..', '__stubs__'); + + it('Should fail when context is not the default export', function() { + expect(() => { + loadContext('./noexport.js', cwd); + }).toThrow( + 'The configuration file at ./noexport.js does not export a valid test context.' + ); + }); + + it('Should load when context is the primary export', function() { + expect(loadContext('./ok.js', cwd)).toBeTruthy(); + }); + + it('Should fail when the config file does not exist', function() { + expect(() => { + loadContext('./nonexistent.js', cwd); + }).toThrow('Unable to find configuration file at ./nonexistent.js.'); + }); +}); diff --git a/src/cli/commands/__tests__/crawl.ts b/src/cli/commands/__tests__/crawl.ts index bfb6623..5c6c4c9 100644 --- a/src/cli/commands/__tests__/crawl.ts +++ b/src/cli/commands/__tests__/crawl.ts @@ -1,17 +1,13 @@ -import { CrawlCommandArgs, handler } from '../crawl'; +import handler from '../crawl'; import stream from 'stream'; import { FailedAnalysisError } from '../../errors'; -import yargs from 'yargs'; - -import * as crawlModule from '../crawl'; import TestContext from '../../../testing/TestContext'; import ConsoleReporter from '../../formatters/ConsoleReporter'; import JUnitReporter from '../../formatters/JUnitReporter'; import JSONReporter from '../../formatters/JSONReporter'; - -import { mocked } from 'ts-jest/utils'; import { makeResult } from '../../util'; +import { mocked } from 'ts-jest/utils'; jest.mock('../../../testing/TestContext'); jest.mock('../../formatters/JUnitReporter'); @@ -22,59 +18,6 @@ const MockedConsoleReporter = mocked(ConsoleReporter); const MockedJUnitReporter = mocked(JUnitReporter); const MockedJSONReporter = mocked(JSONReporter); -function runWithHandler( - argv: string, - handler: (argv: CrawlCommandArgs) => void -): Promise> { - let invoked = 0; - const cmd = Object.assign({}, crawlModule, { - handler: (argv: CrawlCommandArgs) => { - invoked++; - handler(argv); - } - }); - - return new Promise((res, rej) => { - yargs - .command(cmd) - .parse(argv, (err: Error, argv: Record) => { - if (err) return rej(err); - if (!invoked) return rej(new Error('handler was not invoked')); - res(argv); - }); - }); -} -describe('Crawl Command', function() { - it('Has defaults', function() { - return runWithHandler('crawl', argv => { - expect(argv.silent).toEqual(false); - expect(argv.json).toEqual(''); - expect(argv.junit).toEqual(''); - expect(argv.concurrency).toEqual(3); - }); - }); - it('Passes silent', function() { - return runWithHandler('crawl --silent', argv => { - expect(argv.silent).toEqual(true); - }); - }); - it('Passes junit', function() { - return runWithHandler('crawl --junit foo/bar.xml', argv => { - expect(argv.junit).toEqual('foo/bar.xml'); - }); - }); - it('Passes json', function() { - return runWithHandler('crawl --json foo/bar.json', argv => { - expect(argv.json).toEqual('foo/bar.json'); - }); - }); - it('Passes concurrency', function() { - return runWithHandler('crawl --concurrency 5', argv => { - expect(argv.concurrency).toBe(5); - }); - }); -}); - class MockTTY extends stream.PassThrough { columns: number; constructor() { @@ -96,14 +39,8 @@ describe('Crawl Handler', function() { it('Executes the crawl', async function() { // eslint-disable-next-line const context = new TestContext(''); - context.crawl = jest.fn(async function*(): ReturnType< - TestContext['crawl'] - > { - // No-op. - }); - const mockedContext = mocked(context); - await handler({ stdout, context, concurrency: 1 }); - expect(mockedContext.crawl).toHaveBeenCalledWith(1); + await handler({ context, concurrency: 1 }, stdout); + expect(context.crawl).toHaveBeenCalledWith(1); }); it('Displays console output.', async function() { @@ -115,7 +52,7 @@ describe('Crawl Handler', function() { ]; }; try { - await handler({ stdout, context }); + await handler({ context }, stdout); } catch (e) { // no-op - we don't care. } @@ -135,7 +72,7 @@ describe('Crawl Handler', function() { ]; }; try { - await handler({ stdout, context, silent: true }); + await handler({ context, silent: true }, stdout); } catch (e) { // no-op - we don't care. } @@ -151,7 +88,7 @@ describe('Crawl Handler', function() { ]; }; try { - await handler({ stdout, context, junit: 'test.xml' }); + await handler({ context, junit: 'test.xml' }, stdout); } catch (e) { // no-op - we don't care. } @@ -171,7 +108,7 @@ describe('Crawl Handler', function() { ]; }; try { - await handler({ stdout, context, json: 'test.json' }); + await handler({ context, json: 'test.json' }, stdout); } catch (e) { // no-op - we don't care. } @@ -190,7 +127,7 @@ describe('Crawl Handler', function() { makeResult({ Testing: { pass: false, message: 'Test' } }) ]; }; - await expect(handler({ stdout, context })).rejects.toBeInstanceOf( + await expect(handler({ context }, stdout)).rejects.toBeInstanceOf( FailedAnalysisError ); }); @@ -200,10 +137,12 @@ describe('Crawl Handler', function() { context.crawl = (): ReturnType => { throw new Error('Oh no!'); }; - const p = handler({ - stdout, - context - }); + const p = handler( + { + context + }, + stdout + ); return expect(p).rejects.toThrow('Oh no!'); }); }); diff --git a/src/cli/commands/__tests__/help.ts b/src/cli/commands/__tests__/help.ts new file mode 100644 index 0000000..21d41d1 --- /dev/null +++ b/src/cli/commands/__tests__/help.ts @@ -0,0 +1,41 @@ +import stream from 'stream'; +import help from '../help'; + +class MockTTY extends stream.PassThrough { + columns: number; + constructor() { + super(); + this.columns = 60; + } +} +describe('Version command', function() { + let stdout: MockTTY; + + beforeEach(function() { + stdout = new MockTTY(); + }); + + it('Should return the version from package.json', async function() { + await help(stdout); + expect(stdout.read().toString()).toMatchInlineSnapshot(` + "usage: nightcrawler [] + + --config= + Name of test file (default: nightcrawler.js). + --concurrency= + The number of requests to allow in-flight at once. + --json= + The name of the file to write JSON results to. + --junit= + The name of the file to write JUnit results to. + --silent + Silence all console output. + --help + Show the help text. + --version + Show the version number. + + " + `); + }); +}); diff --git a/src/cli/commands/__tests__/version.ts b/src/cli/commands/__tests__/version.ts new file mode 100644 index 0000000..7fdaf65 --- /dev/null +++ b/src/cli/commands/__tests__/version.ts @@ -0,0 +1,23 @@ +import stream from 'stream'; +import version from '../version'; + +class MockTTY extends stream.PassThrough { + columns: number; + constructor() { + super(); + this.columns = 60; + } +} +describe('Version command', function() { + let stdout: MockTTY; + + beforeEach(function() { + stdout = new MockTTY(); + }); + + it('Should return the version from package.json', async function() { + const currentVersion = require('../../../../package.json').version; + await version(stdout); + expect(stdout.read().toString()).toEqual(currentVersion + '\n'); + }); +}); diff --git a/src/cli/commands/crawl.ts b/src/cli/commands/crawl.ts index 3627416..36535c5 100644 --- a/src/cli/commands/crawl.ts +++ b/src/cli/commands/crawl.ts @@ -2,56 +2,23 @@ import { FailedAnalysisError } from '../errors'; import ConsoleReporter from '../formatters/ConsoleReporter'; import JUnitReporter from '../formatters/JUnitReporter'; import JSONReporter from '../formatters/JSONReporter'; -import { BuilderCallback } from 'yargs'; -import { ConfigArgs } from '../index'; import { hasFailure } from '../util'; import Reporter from '../formatters/Reporter'; +import TestContext from '../../testing/TestContext'; +import { StdoutShape } from '../index'; -export interface CrawlCommandArgs extends ConfigArgs { +type CommandArgv = { concurrency?: number; silent?: boolean; json?: string; junit?: string; - stdout?: NodeJS.WritableStream & { columns: number }; -} - -export const command = 'crawl [crawlerfile]'; -export const describe = - 'crawls a defined set of URLs and runs tests against the received responses.'; -export const builder: BuilderCallback = yargs => { - yargs.option('concurrency', { - alias: 'c', - describe: 'number of requests allowed in-flight at once', - type: 'number', - required: true, - default: 3 - }); - yargs.option('silent', { - alias: 'n', - describe: 'silence all output', - type: 'boolean', - default: false - }); - yargs.option('json', { - alias: 'j', - describe: 'filename to write JSON report to', - normalize: true, - type: 'string', - default: '' - }); - yargs.option('junit', { - alias: 'u', - describe: 'filename to write JUnit report to', - normalize: true, - type: 'string', - default: '' - }); + context: TestContext; }; -function getReporters(argv: CrawlCommandArgs): Reporter[] { +function getReporters(argv: CommandArgv, stdout: StdoutShape): Reporter[] { const reporters = []; if (!argv.silent) { - reporters.push(new ConsoleReporter(argv.stdout ?? process.stdout)); + reporters.push(new ConsoleReporter(stdout)); } if (argv.junit?.length) { reporters.push(new JUnitReporter(argv.junit)); @@ -61,11 +28,16 @@ function getReporters(argv: CrawlCommandArgs): Reporter[] { } return reporters; } -export const handler = async function(argv: CrawlCommandArgs): Promise { - const { context, concurrency = 3 } = argv; + +export default async function( + argv: CommandArgv, + stdout: StdoutShape +): Promise { + const { context, concurrency = 5 } = argv; + const reporters: Reporter[] = getReporters(argv, stdout); + let hasAnyFailure = false; - const reporters: Reporter[] = getReporters(argv); await Promise.all(reporters.map(reporter => reporter.start())); for await (const [url, result] of context.crawl(concurrency)) { @@ -78,4 +50,4 @@ export const handler = async function(argv: CrawlCommandArgs): Promise { if (hasAnyFailure) { throw new FailedAnalysisError('Testing reported an error.'); } -}; +} diff --git a/src/cli/commands/help.ts b/src/cli/commands/help.ts new file mode 100644 index 0000000..ff6645d --- /dev/null +++ b/src/cli/commands/help.ts @@ -0,0 +1,21 @@ +import { StdoutShape } from '../index'; + +export default async function(stdout: StdoutShape): Promise { + stdout.write(`usage: nightcrawler [] + + --config= + Name of test file (default: nightcrawler.js). + --concurrency= + The number of requests to allow in-flight at once. + --json= + The name of the file to write JSON results to. + --junit= + The name of the file to write JUnit results to. + --silent + Silence all console output. + --help + Show the help text. + --version + Show the version number. + \n`); +} diff --git a/src/cli/commands/version.ts b/src/cli/commands/version.ts new file mode 100644 index 0000000..0e14fc4 --- /dev/null +++ b/src/cli/commands/version.ts @@ -0,0 +1,7 @@ +import { StdoutShape } from '../index'; + +export default async function(stdout: StdoutShape): Promise { + // eslint-disable-next-line @typescript-eslint/no-var-requires + const pkg = require('../../../package.json'); + stdout.write(pkg.version + '\n'); +} diff --git a/src/cli/index.ts b/src/cli/index.ts index 2748e96..35fbc8d 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -1,33 +1,60 @@ -import yargs, { MiddlewareFunction } from 'yargs'; -import path from 'path'; -import TestContext from '../testing/TestContext'; +import parser from 'minimist'; +import crawl from './commands/crawl'; +import version from './commands/version'; +import help from './commands/help'; +import { loadContext } from './util'; -interface PreloadCrawlerArgs { +export type StdoutShape = NodeJS.WritableStream & { columns: number }; + +export type ArgVShape = { config: string; -} -export interface ConfigArgs { - config?: string; - context: TestContext; -} + concurrency?: string | number; + json?: string; + junit?: string; + silent?: boolean; + help?: boolean; + version?: boolean; +}; -const loadConfig: MiddlewareFunction = argv => { - // eslint-disable-next-line @typescript-eslint/no-var-requires - const context = require(path.resolve(process.cwd(), argv.config)); - if (context instanceof TestContext) { - return { - context - }; +function massageConcurrency( + concurrency: string | number | undefined, + defaultValue: number +): number { + if (concurrency === undefined || concurrency === '') { + return defaultValue; } - throw new Error( - `The configuration file at ${argv.config} does not export a valid test context.` - ); -}; + return typeof concurrency === 'string' ? parseInt(concurrency) : concurrency; +} + +export default async function( + input: string[], + stdout: StdoutShape, + cwd: string +): Promise { + const argv = parser(input, { + string: ['config', 'json', 'concurrency', 'junit'], + boolean: ['silent', 'help', 'version'], + default: { + config: './nightcrawler.js', + concurrency: 5, + silent: false, + help: false, + version: false + } + }); -yargs - .options({ - config: { type: 'string', default: './nightcrawler.js' } - }) - .commandDir('commands') - .middleware(loadConfig) - .demandCommand(1, '') - .help().argv; + if (argv.help) { + return help(stdout); + } else if (argv.version) { + return version(stdout); + } else { + return crawl( + { + ...argv, + concurrency: massageConcurrency(argv.concurrency, 5), + context: loadContext(argv.config, cwd) + }, + stdout + ); + } +} diff --git a/src/cli/util.ts b/src/cli/util.ts index ee33077..d146190 100644 --- a/src/cli/util.ts +++ b/src/cli/util.ts @@ -1,4 +1,4 @@ -import { TestResult, TestResultMap } from '../testing/TestContext'; +import TestContext, { TestResult, TestResultMap } from '../testing/TestContext'; export function makeResult(obj: { [k: string]: TestResult }): TestResultMap { return new Map(Object.entries(obj)); @@ -7,3 +7,21 @@ export function makeResult(obj: { [k: string]: TestResult }): TestResultMap { export function hasFailure(result: TestResultMap): boolean { return Array.from(result.values()).some(r => !r.pass); } + +export function loadContext(configFile: string, cwd: string): TestContext { + let context: TestContext | undefined; + try { + const resolved = require.resolve(configFile, { paths: [cwd] }); + // eslint-disable-next-line @typescript-eslint/no-var-requires + context = require(resolved); + } catch (e) { + throw new Error(`Unable to find configuration file at ${configFile}.`); + } + + if (context instanceof TestContext) { + return context; + } + throw new Error( + `The configuration file at ${configFile} does not export a valid test context.` + ); +} diff --git a/src/driver/__tests__/native.ts b/src/driver/__tests__/native.ts index 219def0..0481994 100644 --- a/src/driver/__tests__/native.ts +++ b/src/driver/__tests__/native.ts @@ -1,92 +1,53 @@ import nock from 'nock'; -import NativeDriver from '../native'; +import native from '../native'; describe('Native Driver', function() { it('Should throw an error for invalid schemes', async function() { - await expect( - new NativeDriver().fetch({ url: 'foo://bar' }) - ).rejects.toThrow('Unknown protocol: foo:'); + await expect(native('foo://bar')).rejects.toThrow('Unknown protocol: foo:'); }); - it('Should return a response for an HTTP crawlRequest', function() { + it('Should return a response for an HTTP crawlRequest', async function() { nock('http://www.example.com') .get('/') .reply(201); - return new NativeDriver() - .fetch({ url: 'http://www.example.com' }) - .then(function(res) { - expect(res.statusCode).toEqual(201); - }); + const response = await native('http://www.example.com'); + expect(response.statusCode).toEqual(201); }); - it('Should return a response for an HTTPS crawlRequest', function() { + it('Should return a response for an HTTPS crawlRequest', async function() { nock('https://www.example.com') .get('/') .reply(201); - return new NativeDriver() - .fetch({ url: 'https://www.example.com' }) - .then(function(res) { - expect(res.statusCode).toEqual(201); - }); + const response = await native('https://www.example.com'); + expect(response.statusCode).toEqual(201); }); - it('Should reply with the response even if it is a 500', function() { + it('Should reply with the response even if it is a 500', async function() { nock('http://www.example.com') .get('/') .reply(500); - return new NativeDriver() - .fetch({ url: 'http://www.example.com' }) - .then(function(res) { - expect(res.statusCode).toEqual(500); - }); + const response = await native('http://www.example.com'); + expect(response.statusCode).toEqual(500); }); - it('Should reply with the time the request took.', function() { + it('Should reply with the time the request took.', async function() { nock('http://www.example.com') .get('/') .reply(200); - return new NativeDriver() - .fetch({ url: 'http://www.example.com' }) - .then(function(res) { - expect(res.time).toBeGreaterThan(0); - }); + const response = await native('http://www.example.com'); + expect(response.time).toBeGreaterThan(0); }); - it('Should throw an error in the event of a network issue', function() { + it('Should throw an error in the event of a network issue', async function() { nock('http://www.example.com') .get('/') .replyWithError({ code: 'ETIMEDOUT' }); - - let called = 0; - const d = new NativeDriver(); - return d - .fetch({ url: 'http://www.example.com' }) - .catch(function(err) { - called++; - expect(err.code).toEqual('ETIMEDOUT'); - }) - .then(function() { - expect(called).toEqual(1); - }); - }); - - it('Should allow request configuration, including authentication', function() { - nock('http://www.example.com') - .get('/') - .basicAuth({ - user: 'john', - pass: 'doe' - }) - .reply(200); - - return new NativeDriver({ auth: 'john:doe' }) - .fetch({ url: 'http://www.example.com' }) - .then(function(res) { - expect(res.statusCode).toEqual(200); - }); + await expect(native('http://www.example.com')).rejects.toEqual({ + code: 'ETIMEDOUT' + }); }); it('Should reject when the socket times out.', async function() { @@ -94,10 +55,9 @@ describe('Native Driver', function() { .get('/') .socketDelay(500) .reply(200); - const driver = new NativeDriver({ timeout: 10 }); await expect( - driver.fetch({ url: 'https://www.example.com/' }) - ).rejects.toThrowError('socket hang up'); + native('https://www.example.com/', { timeout: 10 }) + ).rejects.toThrow('socket hang up'); }); it('Should reject when the request times out', async function() { @@ -105,10 +65,10 @@ describe('Native Driver', function() { .get('/') .delay(500) .reply(200); - const driver = new NativeDriver({ timeout: 10 }); + await expect( - driver.fetch({ url: 'https://www.example.com/' }) - ).rejects.toBeTruthy(); + native('https://www.example.com/', { timeout: 10 }) + ).rejects.toThrow('socket hang up'); }); it('Should require the headers to resolve before the timeout', async function() { @@ -116,10 +76,9 @@ describe('Native Driver', function() { .get('/') .delay({ head: 500 }) .reply(200); - const driver = new NativeDriver({ timeout: 10 }); await expect( - driver.fetch({ url: 'https://www.example.com/' }) - ).rejects.toBeTruthy(); + native('https://www.example.com/', { timeout: 10 }) + ).rejects.toThrow('socket hang up'); }); it('Should not require the body to resolve before the timeout', async function() { @@ -127,9 +86,23 @@ describe('Native Driver', function() { .get('/') .delay({ body: 500 }) .reply(200); - const driver = new NativeDriver({ timeout: 10 }); await expect( - driver.fetch({ url: 'https://www.example.com/' }) + native('https://www.example.com/', { timeout: 10 }) ).resolves.toBeTruthy(); }); + + it('Should allow options to be overridden using options.', async function() { + nock('http://www.example.com') + .get('/') + .basicAuth({ + user: 'john', + pass: 'doe' + }) + .reply(200); + + const response = await native('http://www.example.com/', { + auth: 'john:doe' + }); + expect(response.statusCode).toBe(200); + }); }); diff --git a/src/driver/dummy.ts b/src/driver/dummy.ts deleted file mode 100644 index f0a3803..0000000 --- a/src/driver/dummy.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { Driver, CrawlerRequest, CrawlerResponse } from '../types'; -import { performance } from 'perf_hooks'; - -type DummyResponse = { - statusCode: number; - time: number; -}; - -function delayResolve(value: T, timeout: number): Promise { - return new Promise(resolve => { - setTimeout(() => resolve(value), timeout); - }); -} -function delayReject(value: unknown, timeout: number): Promise { - return new Promise((resolve, reject) => { - setTimeout(() => reject(value), timeout); - }); -} - -/** - * Dummy driver for use in testing. - */ -export default class DummyDriver implements Driver { - fetch( - req: CrawlerRequest & { shouldFail?: string } - ): Promise { - const start = performance.now(); - const delay = 'delay' in req ? parseInt(req.delay as string) : 0; - if ('shouldFail' in req && req.shouldFail) { - return delayReject(req.shouldFail, delay); - } - return delayResolve( - { statusCode: 200, time: performance.now() - start }, - delay - ); - } -} diff --git a/src/driver/native.ts b/src/driver/native.ts index 4bd4503..f17ba9a 100644 --- a/src/driver/native.ts +++ b/src/driver/native.ts @@ -1,59 +1,54 @@ -import { Driver, CrawlerRequest } from '../types'; +import { Driver } from '../types'; import https from 'https'; import http from 'http'; import { URL } from 'url'; import { performance } from 'perf_hooks'; -type NativeDriverResponse = { +interface NativeDriverResponse extends http.IncomingMessage { statusCode: number; - statusMessage?: string; time: number; -}; +} -export default class NativeDriver implements Driver { - opts: https.RequestOptions; - constructor(opts: https.RequestOptions = {}) { - this.opts = opts; +function _getDriver(url: URL): typeof http | typeof https { + switch (url.protocol) { + case 'https:': + return https; + case 'http:': + return http; + default: + throw new Error('Unknown protocol: ' + url.protocol); } +} - fetch(crawlRequest: CrawlerRequest): Promise { - return new Promise((resolve, reject) => { - const parsed = new URL(crawlRequest.url); - const theseOptions = Object.assign( - { - protocol: parsed.protocol, - host: parsed.hostname, - port: parsed.port, - path: parsed.pathname, - method: 'GET', - timeout: 15000 - }, - this.opts +export type Options = http.RequestOptions | https.RequestOptions; + +const native: Driver = (url, options: {} = {}) => { + return new Promise((resolve, reject) => { + const parsed = new URL(url); + const theseOptions = { + protocol: parsed.protocol, + host: parsed.hostname, + port: parsed.port, + path: parsed.pathname, + method: 'GET', + timeout: 15000, + ...options + }; + const start = performance.now(); + const req = _getDriver(parsed).request(theseOptions, response => { + resolve( + Object.assign(response, { + time: performance.now() - start, + statusCode: response.statusCode ?? 0 + }) ); - const start = performance.now(); - const req = this._getDriver(parsed).request(theseOptions, res => { - resolve({ - statusCode: res.statusCode ?? 0, - statusMessage: res.statusMessage, - time: performance.now() - start - }); - }); - req.on('timeout', () => { - req.abort(); - }); - req.on('error', reject); - req.end(); }); - } + req.on('timeout', () => { + req.abort(); + }); + req.on('error', reject); + req.end(); + }); +}; - private _getDriver(url: URL): typeof http | typeof https { - switch (url.protocol) { - case 'https:': - return https; - case 'http:': - return http; - default: - throw new Error('Unknown protocol: ' + url.protocol); - } - } -} +export default native; diff --git a/src/index.ts b/src/index.ts index 02d0f2f..b408b70 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,4 @@ import Crawler from './Crawler'; -import TestContext from './testing/TestContext'; -export default Crawler; -export { Crawler, TestContext }; +export { Crawler }; export * from './testing/functions'; diff --git a/src/testing/TestContext.ts b/src/testing/TestContext.ts index 6ed0a7c..ca324d9 100644 --- a/src/testing/TestContext.ts +++ b/src/testing/TestContext.ts @@ -19,8 +19,8 @@ export type TestResultMap = Map; export default class TestContext { description: string; crawler?: Crawler; - testHandlers: OneHandler[]; - afterHandlers: AfterHandler[]; + private testHandlers: OneHandler[]; + private afterHandlers: AfterHandler[]; constructor(description: string) { this.description = description; diff --git a/src/testing/__mocks__/TestContext.ts b/src/testing/__mocks__/TestContext.ts new file mode 100644 index 0000000..0cd905a --- /dev/null +++ b/src/testing/__mocks__/TestContext.ts @@ -0,0 +1,12 @@ +const TestContext = jest.requireActual('../TestContext').default; + +export default jest.fn().mockImplementation(() => { + const context = new TestContext(); + context.iterator = []; + context.test = jest.fn(); + context.after = jest.fn(); + context.crawl = jest.fn(async function*() { + yield* context.iterator; + }); + return context; +}); diff --git a/src/testing/__tests__/TestContext.ts b/src/testing/__tests__/TestContext.ts index d1c3360..4cf9a88 100644 --- a/src/testing/__tests__/TestContext.ts +++ b/src/testing/__tests__/TestContext.ts @@ -2,20 +2,11 @@ import TestContext from '../TestContext'; import Crawler from '../../Crawler'; import { CrawlerUnit } from '../../types'; import { makeResult } from '../../cli/util'; +import { all } from '../../util'; import { mocked } from 'ts-jest/utils'; jest.mock('../../Crawler'); -async function all( - iterator: AsyncIterable -): Promise { - const collected = []; - for await (const i of iterator) { - collected.push(i); - } - return collected; -} - describe('TestContext', function() { const units = [ { request: { url: 'foo', groups: ['foo'] } }, diff --git a/src/testing/__tests__/functions.ts b/src/testing/__tests__/functions.ts index 7a83132..24a507c 100644 --- a/src/testing/__tests__/functions.ts +++ b/src/testing/__tests__/functions.ts @@ -12,8 +12,8 @@ const MockedCrawler = mocked(Crawler); describe('crawl function', function() { beforeEach(function() { - MockedContext.mockReset(); - MockedCrawler.mockReset(); + MockedContext.mockClear(); + MockedCrawler.mockClear(); }); it('crawl should create a crawler when an iterable is returned.', function() { diff --git a/src/types.ts b/src/types.ts index aece3db..921cbfc 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,35 +1,32 @@ export type CrawlerRequest = { + // The absolute URL to crawl. url: string; - driverOptions?: unknown; - groups?: string[]; + // Options to forward to the driver. + options?: DriverOptions; + // Any other properties the user wants to add. [key: string]: unknown; }; -export type RequestIterable = - | Iterable - | AsyncIterable; +export type RequestIterable = + | Iterable + | AsyncIterable; export type DriverResponse = { statusCode: number; time: number; - [key: string]: unknown; }; -export type CrawlerResponse = DriverResponse; export type CrawlerUnit = { error?: string | Error; request: CrawlerRequest; - response?: CrawlerResponse; + response?: DriverResponse; }; +type DriverOptions = Record; + export interface Driver< - ResponseType extends CrawlerResponse = CrawlerResponse + ResponseType extends DriverResponse = DriverResponse, + OptionsType extends DriverOptions = {} > { - /** - * Fetch a single URL. - * - * The driver should return a promise which is only rejected in the case - * where the response is a complete error. - */ - fetch(req: CrawlerRequest): Promise; + (url: string, options?: DriverOptions): Promise; } diff --git a/src/util.ts b/src/util.ts index dfc06e5..59d1d5d 100644 --- a/src/util.ts +++ b/src/util.ts @@ -1,4 +1,3 @@ -import { CrawlerRequest } from './types'; function isAsyncIterable(x: unknown): x is AsyncIterable { return ( @@ -30,10 +29,10 @@ export function toAsyncIterable( ); } -export function isCrawlerRequest(request: unknown): request is CrawlerRequest { - return ( - request !== null && - typeof request === 'object' && - typeof (request as CrawlerRequest).url === 'string' - ); +export async function all(iterator: AsyncIterable): Promise { + const collected = []; + for await (const i of iterator) { + collected.push(i); + } + return collected; } diff --git a/tsconfig.json b/tsconfig.json index ffb6961..94b1040 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,18 +1,20 @@ { "compilerOptions": { - // Target latest version of ECMAScript. - "target": "esnext", + // Target Node 10. + "target": "es2018", + "module": "commonjs", // Search under node_modules for non-relative imports. "moduleResolution": "node", // Process & infer types from .js files. "allowJs": true, - // Don't emit; allow Babel to transform files. - "noEmit": true, // Enable strictest settings like strictNullChecks & noImplicitAny. "strict": true, // Import non-ES modules as default imports. - "esModuleInterop": true + "esModuleInterop": true, + "outDir": "./dist", + "declaration": true }, + "exclude": ["**/__tests__/**"], "include": [ "src" ]