Skip to content

Commit

Permalink
test: add test for error codes doc and impl
Browse files Browse the repository at this point in the history
This adds several sanity checks for error codes.

It scans:
 * all natives (js sources),
 * doc/api/*.md documentation
 * src/node_errors.h (errors definition from the C++ side).

There is also a whitelist of manually created errors from JS side,
currently consisting of ERR_HTTP2_ERROR and ERR_UNKNOWN_BUILTIN_MODULE.

Alsom all ERR_NAPI_ codes are whitelisted, as those are created directly
on the cpp side, without declaring them first.

The performed checks:

  1. All errors used from JS should be defined in `internal/errors` and
     present in its .codes object. Whitelist (mentioned above) applies.

  2. All errors instantiated from JS without arguments should support
     0-arguments version.

  3. All errors mentioned in doc should defined either in JS, C++, or
     in the whitelist.

  4. All errors mentioned anywhere should be documented.

  5. Documentation of error codes should be sorted, have no repeats,
     and include exactly one entry for every error code mentioned in
     the documentation, formatted as `/\n### (ERR_[A-Z0-9_]+)\n`.

  6. All doc entries for error codes should have appropriate anchors.

There is also a --report flag, which prints all the current issues and
exits without asserting, for manual inspection.

Individual fixes for those issues are landed in separate commits.

Refs: nodejs#21421
Refs: nodejs#21440
Refs: nodejs#21483
Refs: nodejs#21484
Refs: nodejs#21485
Refs: nodejs#21487
PR-URL: nodejs#21470
  • Loading branch information
ChALkeR committed Sep 8, 2018
1 parent 9f06a05 commit 8e41fd7
Showing 1 changed file with 158 additions and 0 deletions.
158 changes: 158 additions & 0 deletions test/parallel/test-error-codes-impl-doc.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
'use strict';

// Flags: --expose-internals

// Note: this script reads Node.js sources and docs, and expects to be run on
// the very same Node.js version as the source and doc tree is.

require('../common');
const assert = require('assert');
const errors = require('internal/errors').codes;
const fs = require('fs');
const path = require('path');
const natives = process.binding('natives');

// --report prints only the list of failed checks and does not assert
const report = process.argv[2] === '--report';
const results = [];
function check(ok, type, name, reason) {
const label = `${type}: ${name} - ${reason}`;
if (report) {
if (!ok) results.push(label);
} else {
assert.ok(ok, label);
}
}

const set = {
all: new Set(),
js: new Set(), // Error codes found in natives.
doc: new Set(), // Error codes mentioned in `doc/api/*.md`.
cpp: new Set(), // Error codes found in `src/node_errors.h`.
noargs: new Set(), // Subset of `js` which is constructed without arguments.
// `set.manual` contains errors manually created from js and not defined in
// internals/errors. That is not scanned now, update this list if required.
manual: new Set(['ERR_HTTP2_ERROR', 'ERR_UNKNOWN_BUILTIN_MODULE']),
// `set.examples` contains errors mentioned in documentation purely for
// demonstration purposes. These errors are not required to be present.
examples: new Set(['ERR_ERROR_1'])
};

const root = path.join(__dirname, '..', '..');

// File containing error definitions
const jsdata = natives['internal/errors'];
// File containing error documentation
let docdata = fs.readFileSync(path.join(root, 'doc/api/errors.md'), 'utf-8')
docdata = docdata.replace(/## Legacy [\s\S]*/, '');
// File containing cpp-side errors
const cppdata = fs.readFileSync(path.join(root, 'src/node_errors.h'), 'utf-8');
// Directory containing documentation
const docdir = path.join(root, 'doc/api/');

function addSource(source, type) {
// eslint-disable-next-line node-core/no-unescaped-regexp-dot
const re = /(.)?(ERR_[A-Z0-9_]+)(..)?/g;
let match;
while (match = re.exec(source)) {
if (match[1] === '_') continue; // does not start with ERR_
const name = match[2];
if (type === 'doc' && set.examples.has(name)) continue; // is an example
set.all.add(name);
set[type].add(name);
if (type === 'js' && match[3] === '()') {
// Is directly called without arguments from js, we will try that
set.noargs.add(name);
}
}
}

// Add all errors from JS natives
for (const file of Object.keys(natives)) addSource(natives[file], 'js');
// Add all errors from src/node_errors.h
addSource(cppdata, 'cpp');
// Add all errors from doc/api/*.md files
for (const file of fs.readdirSync(docdir)) {
if (!file.endsWith('.md')) continue;
let data = fs.readFileSync(path.join(docdir, file), 'utf-8');
if (file === 'errors.md') data = data.replace(/## Legacy [\s\S]*/, '');
addSource(data, 'doc');
}

// Check that we have all js errors
for (const name of set.js) {
if (set.manual.has(name)) continue;
const defined = jsdata.includes(`E('${name}',`);
check(defined, 'js', name, 'missing JS implementation (source)');
if (defined) {
check(errors[name], 'js', name, 'missing JS implementation (runtime)');
}
}

// Check that we can initialize errors called without args
for (const name of set.noargs) {
if (!errors[name]) continue; // Already catched that above
let ok = true;
try {
new errors[name]();
} catch (e) {
ok = false;
}
check(ok, 'init', name, 'failed init without args, but is called with "()"');
}

// Check that we have implementation for all errors mentioned in docs.
// C++ does not need that, and JS is already checked above.
for (const name of set.doc) {
const ok = set.manual.has(name) ||
name.startsWith('ERR_NAPI_') || // napi errors are created directly
jsdata.includes(`E('${name}',`) ||
cppdata.includes(`V(${name}, `);
const reason = docdata.includes(`### ${name}\n`) ?
'documented' : 'mentioned in doc/api/';
check(ok, 'impl', name, `missing implementation, ${reason}`);
}

// Check that we have documentation for all errors
for (const name of set.all) {
const ok = docdata.includes(`### ${name}\n`);
check(ok, 'doc', name, 'missing documentation');
// Check that documentation references are correctly formatted
if (ok) {
const anchor = docdata.includes(`\n\n<a id="${name}"></a>\n### ${name}\n`);
check(anchor, 'doc', name, 'missing anchor or not properly formatted');
}
}

// Check that documentation is sorted, formatted, and does not contain dupes
{
const compare = (a, b) => {
// HTTP2_ should come after HTTP_
if (a.startsWith('ERR_HTTP_') && b.startsWith('ERR_HTTP2_')) return -1;
if (a.startsWith('ERR_HTTP2_') && b.startsWith('ERR_HTTP_')) return 1;
if (a < b) return -1;
if (a > b) return 1;
return 0;
};
const re = /\n### (ERR_[A-Z0-9_]+)\n/g;
let match;
let last;
const documented = new Set();
while (match = re.exec(docdata)) {
const name = match[1];
if (documented.has(name)) {
check(false, 'doc', name, 'duplicate documentation entry');
} else {
const sorted = !last || compare(name, last) === 1;
check(sorted, 'doc', name, 'is out of order');
documented.add(name);
}
last = name;
}
}

if (report) {
console.log('Report mode');
console.log(results.join('\n'));
console.log(`There were ${results.length} problems found`);
}

0 comments on commit 8e41fd7

Please sign in to comment.