tools: build all.json by combining generated JSON

Notes:

1) Removed a number of root properties that did not seem relevant:
   source, desc, and introduced_in.  There no longer is a source, and
   the other two are from the first include and do not reflect the
   entire API.

2) As with #20100, the current
   "desc" properties sometimes contained in-page links, other times
   referenced another page, and often did not match the links in the
   original HTML or JSON file. I chose to standardize on external links
   as "desc" values are isolated snippets as opposed to all.html which
   can be viewed as a standalone and self contained document.

3) Eliminated preprocessing for @include entirely, including the test
   case for this function.

4) _toc.md was renamed to index.md.

5) index comments no longer appear in embedded TOCs (left hand side
   column in the generated documentation.

PR-URL: #21637
Reviewed-By: Vse Mozhet Byt <vsemozhetbyt@gmail.com>
Reviewed-By: Rich Trott <rtrott@gmail.com>

Makefile

@@ -622,7 +622,7 @@ doc-only: $(apidoc_dirs) $(apiassets)  ## Builds the docs with the local or the
 	if [ ! -d doc/api/assets ]; then \
 		$(MAKE) tools/doc/node_modules/js-yaml/package.json; \
 	fi;
-	@$(MAKE) $(apidocs_html) $(apidocs_json)
+	@$(MAKE) out/doc/api/all.html out/doc/api/all.json
 
 .PHONY: doc
 doc: $(NODE_EXE) doc-only
@@ -672,10 +672,12 @@ out/doc/api/%.json: doc/api/%.md tools/doc/generate.js tools/doc/json.js
 out/doc/api/%.html: doc/api/%.md tools/doc/generate.js tools/doc/html.js
 	$(call available-node, $(gen-html))
 
-out/doc/api/all.html: $(filter-out out/doc/api/all.html, $(apidocs_html)) \
-	tools/doc/allhtml.js
+out/doc/api/all.html: $(apidocs_html) tools/doc/allhtml.js
 	$(call available-node, tools/doc/allhtml.js)
 
+out/doc/api/all.json: $(apidocs_json) tools/doc/alljson.js
+	$(call available-node, tools/doc/alljson.js)
+
 .PHONY: docopen
 docopen: $(apidocs_html)
 	@$(PYTHON) -mwebbrowser file://$(PWD)/out/doc/api/all.html

doc/api/index.md

@@ -1 +1,64 @@
-@include _toc.md
+<!--
+  NB(chrisdickinson): if you move this file, be sure to update
+  tools/doc/html.js to point at the new location.
+-->
+
+<!--introduced_in=v0.10.0-->
+
+* [About these Docs](documentation.html)
+* [Usage & Example](synopsis.html)
+
+<div class="line"></div>
+
+* [Assertion Testing](assert.html)
+* [Async Hooks](async_hooks.html)
+* [Buffer](buffer.html)
+* [C++ Addons](addons.html)
+* [C/C++ Addons - N-API](n-api.html)
+* [Child Processes](child_process.html)
+* [Cluster](cluster.html)
+* [Command Line Options](cli.html)
+* [Console](console.html)
+* [Crypto](crypto.html)
+* [Debugger](debugger.html)
+* [Deprecated APIs](deprecations.html)
+* [DNS](dns.html)
+* [Domain](domain.html)
+* [ECMAScript Modules](esm.html)
+* [Errors](errors.html)
+* [Events](events.html)
+* [File System](fs.html)
+* [Globals](globals.html)
+* [HTTP](http.html)
+* [HTTP/2](http2.html)
+* [HTTPS](https.html)
+* [Inspector](inspector.html)
+* [Internationalization](intl.html)
+* [Modules](modules.html)
+* [Net](net.html)
+* [OS](os.html)
+* [Path](path.html)
+* [Performance Hooks](perf_hooks.html)
+* [Process](process.html)
+* [Punycode](punycode.html)
+* [Query Strings](querystring.html)
+* [Readline](readline.html)
+* [REPL](repl.html)
+* [Stream](stream.html)
+* [String Decoder](string_decoder.html)
+* [Timers](timers.html)
+* [TLS/SSL](tls.html)
+* [Trace Events](tracing.html)
+* [TTY](tty.html)
+* [UDP/Datagram](dgram.html)
+* [URL](url.html)
+* [Utilities](util.html)
+* [V8](v8.html)
+* [VM](vm.html)
+* [Worker Threads](worker_threads.html)
+* [ZLIB](zlib.html)
+
+<div class="line"></div>
+
+* [GitHub Repo & Issue Tracker](https://github.com/nodejs/node)
+* [Mailing List](https://groups.google.com/group/nodejs)

test/doctool/test-doctool-html.js

@@ -11,7 +11,6 @@ try {
 const assert = require('assert');
 const { readFile } = require('fs');
 const fixtures = require('../common/fixtures');
-const processIncludes = require('../../tools/doc/preprocess.js');
 const toHTML = require('../../tools/doc/html.js');
 
 // Test data is a list of objects with two properties.
@@ -64,15 +63,6 @@ const testData = [
       '<!-- This is not a metadata comment --> ' +
       '<p>Describe <code>Something</code> in more detail here. </p>'
   },
-  {
-    file: fixtures.path('doc_with_includes.md'),
-    html: '<!-- [start-include:doc_inc_1.md] -->' +
-    '<p>Look <a href="doc_inc_2.html#doc_inc_2_foobar">here</a>!</p>' +
-    '<!-- [end-include:doc_inc_1.md] --><!-- [start-include:doc_inc_2.md] -->' +
-    '<h1>foobar<span><a class="mark" href="#doc_inc_2_foobar" ' +
-    'id="doc_inc_2_foobar">#</a></span></h1>' +
-    '<p>I exist and am being linked to.</p><!-- [end-include:doc_inc_2.md] -->'
-  },
   {
     file: fixtures.path('sample_document.md'),
     html: '<ol><li>fish</li><li><p>fish</p></li><li><p>Redfish</p></li>' +
@@ -90,36 +80,34 @@ testData.forEach(({ file, html, analyticsId }) => {
 
   readFile(file, 'utf8', common.mustCall((err, input) => {
     assert.ifError(err);
-    processIncludes(file, input, common.mustCall((err, preprocessed) => {
-      assert.ifError(err);
 
-      toHTML(
-        {
-          input: preprocessed,
-          filename: 'foo',
-          nodeVersion: process.version,
-          analytics: analyticsId,
-        },
-        common.mustCall((err, output) => {
-          assert.ifError(err);
+    toHTML(
+      {
+        input: input,
+        filename: 'foo',
+        nodeVersion: process.version,
+        analytics: analyticsId,
+      },
+      common.mustCall((err, output) => {
+        assert.ifError(err);
 
-          const actual = output.replace(spaces, '');
-          // Assert that the input stripped of all whitespace contains the
-          // expected markup.
-          assert(actual.includes(expected));
+        const actual = output.replace(spaces, '');
+        // Assert that the input stripped of all whitespace contains the
+        // expected markup.
+        assert(actual.includes(expected));
 
-          // Testing the insertion of Google Analytics script when
-          // an analytics id is provided. Should not be present by default.
-          const scriptDomain = 'google-analytics.com';
-          if (includeAnalytics) {
-            assert(actual.includes(scriptDomain),
-                   `Google Analytics script was not present in "${actual}"`);
-          } else {
-            assert.strictEqual(actual.includes(scriptDomain), false,
-                               'Google Analytics script was present in ' +
-                               `"${actual}"`);
-          }
-        }));
-    }));
+        // Testing the insertion of Google Analytics script when
+        // an analytics id is provided. Should not be present by default.
+        const scriptDomain = 'google-analytics.com';
+        if (includeAnalytics) {
+          assert(actual.includes(scriptDomain),
+                 `Google Analytics script was not present in "${actual}"`);
+        } else {
+          assert.strictEqual(actual.includes(scriptDomain), false,
+                             'Google Analytics script was present in ' +
+                             `"${actual}"`);
+        }
+      })
+    );
   }));
 });

test/doctool/test-make-doc.js

@@ -13,17 +13,16 @@ const path = require('path');
 
 const apiPath = path.resolve(__dirname, '..', '..', 'out', 'doc', 'api');
 const allDocs = fs.readdirSync(apiPath);
-assert.ok(allDocs.includes('_toc.html'));
+assert.ok(allDocs.includes('index.html'));
 
 const actualDocs = allDocs.filter(
   (name) => {
     const extension = path.extname(name);
-    return (extension === '.html' || extension === '.json') &&
-           name !== '_toc.html';
+    return extension === '.html' || extension === '.json';
   }
 );
 
-const toc = fs.readFileSync(path.resolve(apiPath, '_toc.html'), 'utf8');
+const toc = fs.readFileSync(path.resolve(apiPath, 'index.html'), 'utf8');
 const re = /href="([^/]+\.html)"/;
 const globalRe = new RegExp(re, 'g');
 const links = toc.match(globalRe);
@@ -32,8 +31,7 @@ assert.notStrictEqual(links, null);
 // Filter out duplicate links, leave just filenames, add expected JSON files.
 const linkedHtmls = [...new Set(links)].map((link) => link.match(re)[1]);
 const expectedJsons = linkedHtmls
-                       .map((name) => name.replace('.html', '.json'))
-                       .concat('_toc.json');
+                       .map((name) => name.replace('.html', '.json'));
 const expectedDocs = linkedHtmls.concat(expectedJsons);
 
 // Test that all the relative links in the TOC match to the actual documents.

tools/doc/allhtml.js

@@ -12,7 +12,7 @@ const htmlFiles = fs.readdirSync(source, 'utf8')
   .filter((name) => name.includes('.html') && name !== 'all.html');
 
 // Read the table of contents.
-const toc = fs.readFileSync(source + '/_toc.html', 'utf8');
+const toc = fs.readFileSync(source + '/index.html', 'utf8');
 
 // Extract (and concatenate) the toc and apicontent from each document.
 let contents = '';
@@ -47,11 +47,12 @@ for (const link of toc.match(/<a.*?>/g)) {
   seen[href] = true;
 }
 
-// Replace various mentions of _toc with all.
-let all = toc.replace(/_toc\.html/g, 'all.html')
-  .replace('_toc.json', 'all.json')
-  .replace('api-section-_toc', 'api-section-all')
-  .replace('data-id="_toc"', 'data-id="all"');
+// Replace various mentions of index with all.
+let all = toc.replace(/index\.html/g, 'all.html')
+  .replace('<a href="all.html" name="toc">', '<a href="index.html" name="toc">')
+  .replace('index.json', 'all.json')
+  .replace('api-section-index', 'api-section-all')
+  .replace('data-id="index"', 'data-id="all"');
 
 // Clean up the title.
 all = all.replace(/<title>.*?\| /, '<title>');