From aea99f81c98928a12fd8ade42326941db837cc18 Mon Sep 17 00:00:00 2001 From: Dominique Hazael-Massieux Date: Mon, 3 May 2021 08:41:35 +0200 Subject: [PATCH 1/4] Add IETF specs used in MDN close #280 --- specs.json | 13 +++++++++++++ src/build-index.js | 5 +++-- src/compute-repository.js | 7 ++++++- src/compute-shortname.js | 6 ++++++ src/determine-filename.js | 8 +++++++- src/fetch-groups.js | 25 ++++++++++++++++++++++++- src/parse-spec-url.js | 5 +++++ 7 files changed, 64 insertions(+), 5 deletions(-) diff --git a/specs.json b/specs.json index b42293d1..84f7eaec 100644 --- a/specs.json +++ b/specs.json @@ -6,6 +6,19 @@ } }, "https://console.spec.whatwg.org/", + "https://datatracker.ietf.org/doc/html/rfc6265", + "https://datatracker.ietf.org/doc/html/rfc6454", + "https://datatracker.ietf.org/doc/html/rfc7230", + "https://datatracker.ietf.org/doc/html/rfc7231", + "https://datatracker.ietf.org/doc/html/rfc7232", + "https://datatracker.ietf.org/doc/html/rfc7233", + "https://datatracker.ietf.org/doc/html/rfc7234", + "https://datatracker.ietf.org/doc/html/rfc7235", + "https://datatracker.ietf.org/doc/html/rfc7239", + "https://datatracker.ietf.org/doc/html/rfc7469", + "https://datatracker.ietf.org/doc/html/rfc7538", + "https://datatracker.ietf.org/doc/html/rfc7540", + "https://datatracker.ietf.org/doc/html/rfc8470", "https://dom.spec.whatwg.org/", "https://drafts.css-houdini.org/css-typed-om-2/ delta", "https://drafts.css-houdini.org/font-metrics-api-1/", diff --git a/src/build-index.js b/src/build-index.js index a61caa6b..a1c776e5 100644 --- a/src/build-index.js +++ b/src/build-index.js @@ -2,8 +2,9 @@ * Script that compiles and returns the final list of specs from the * "specs.json" input file. * - * The script will extract the W3C API key it needs from a "config.json" file - * in the root folder, which must exist and contain a "w3cApiKey" key. + * The script will extract the W3C API key and the github token it needs + * from a "config.json" file in the root folder + * which must exist and contain "w3cApiKey" and "githubToken" keys. */ const fs = require("fs").promises; diff --git a/src/compute-repository.js b/src/compute-repository.js index 3efb28b2..7161f86e 100644 --- a/src/compute-repository.js +++ b/src/compute-repository.js @@ -109,6 +109,9 @@ module.exports = async function (specs, options) { `specs/${spec.shortname.replace(/^svg-/, '')}/master/Overview.html`, `master/Overview.html`, + // HTTPWG specs + `specs/${spec.shortname}.xml`, + // Following patterns are used in a small number of cases, but could // perhaps appear again in the future, so worth handling here. "spec/index.bs", @@ -147,7 +150,9 @@ module.exports = async function (specs, options) { if (options.githubToken) { // Fetch the real name of repository owners (preserving case) for (const repo of repos) { - repo.owner = await fetchRealGitHubOwnerName(repo.owner); + if (repo) { + repo.owner = await fetchRealGitHubOwnerName(repo.owner); + } } } diff --git a/src/compute-shortname.js b/src/compute-shortname.js index 358ea8dd..11a69fe7 100644 --- a/src/compute-shortname.js +++ b/src/compute-shortname.js @@ -96,6 +96,12 @@ function computeShortname(url) { return "svg-" + svg[1]; } + // Handle IETF RFCs + const rfcs = url.match(/\/datatracker\.ietf\.org\/doc\/html\/(rfc[0-9]+)/); + if (rfcs) { + return rfcs[1]; + } + // Return name when one was given if (!url.match(/\//)) { return url; diff --git a/src/determine-filename.js b/src/determine-filename.js index ae444c6e..83a9282c 100644 --- a/src/determine-filename.js +++ b/src/determine-filename.js @@ -18,6 +18,12 @@ module.exports = async function (url) { return match[1]; } + // datatracker IETF RFC HTML rendering + const rfcMatch = url.match(/\/html\/(rfc[0-9]+)$/); + if (rfcMatch) { + return rfcMatch[1] + '.html'; + } + // Make sure that url ends with a "/" const urlWithSlash = url.endsWith("/") ? url : url + "/"; @@ -38,4 +44,4 @@ module.exports = async function (url) { const res = await fetch(url, { method: "HEAD" }); const filename = res.headers.get("Content-Location"); return filename; -} \ No newline at end of file +} diff --git a/src/fetch-groups.js b/src/fetch-groups.js index 96ef21b1..059950fc 100644 --- a/src/fetch-groups.js +++ b/src/fetch-groups.js @@ -50,6 +50,29 @@ module.exports = async function (specs, options) { for (const spec of specs) { const info = parseSpecUrl(spec.url); if (!info) { + // No general rule to identify repos and groups for IETF specs + // instead, fetching info on groups from https://www.rfc-editor.org/in-notes/rfcXXX.json + if (spec.url.match(/ietf\.org/)) { + spec.organization = spec.organization ?? "IETF"; + if (spec.groups) continue; + const rfcNumber = spec.url.slice(spec.url.lastIndexOf('/') + 1); + const rfcJSON = await fetchJSON(`https://www.rfc-editor.org/in-notes/${rfcNumber}.json`); + const wgName = rfcJSON.source; + // draft-ietf-websec-origin-06 → websec + // per https://www.ietf.org/standards/ids/guidelines/#7 + // not clear there is anything forbidding hyphens in WG acronyms though + // https://datatracker.ietf.org/doc/html/rfc2418#section-2.2 + const wgId = rfcJSON.draft.split('-')[2]; + if (wgName && wgId) { + spec.groups = [{ + name: `${wgName} Working Group`, + url: `https://datatracker.ietf.org/wg/${wgId}/` + }]; + continue; + } else { + throw new Error(`Could not identify IETF group producing ${spec.url}`); + } + } throw new Error(`Cannot extract any useful info from ${spec.url}`); } @@ -153,4 +176,4 @@ module.exports = async function (specs, options) { } return specs; -}; \ No newline at end of file +}; diff --git a/src/parse-spec-url.js b/src/parse-spec-url.js index e705b48e..5f7fd873 100644 --- a/src/parse-spec-url.js +++ b/src/parse-spec-url.js @@ -60,6 +60,11 @@ module.exports = function (url) { return { type: "custom", owner: "khronosgroup", name: "WebGL" }; } + const httpwg = url.match(/^https:\/\/httpwg\.org\/specs\/rfc[0-9]+\.html$/); + if (httpwg) { + return { type: "custom", owner: "httpwg", name: "httpwg.github.io" }; + } + const w3cTr = url.match(/^https?:\/\/(?:www\.)?w3\.org\/TR\/([^\/]+)\/$/); if (w3cTr) { return { type: "tr", owner: "w3c", name: w3cTr[1] }; From 0c93af3b2e857d62090a07669c51255568b6ec4c Mon Sep 17 00:00:00 2001 From: Dominique Hazael-Massieux Date: Thu, 1 Jul 2021 19:42:44 +0200 Subject: [PATCH 2/4] Update README re IETF specs --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a2d0e23b..8f6e4c54 100644 --- a/README.md +++ b/README.md @@ -462,7 +462,7 @@ Group, the WHATWG, or the Khronos Group. 3. Web browsers expressed some level of support for the spec, e.g. through a public intent to implement. 4. The spec sits at the application layer or is "close to it". For instance, -IETF specs are essentially out of scope, at least for now. +most IETF specs are likely out of scope, but some that are exposed to Web developers are in scope. 5. The spec defines normative content (terms, CSS, IDL), or it contains informative content that other specs often need to refer to (e.g. guidelines from horizontal activities such as accessibility, internationalization, privacy From d530fcbc26e63246d8457138b053a558ae75205c Mon Sep 17 00:00:00 2001 From: Dominique Hazael-Massieux Date: Fri, 2 Jul 2021 11:31:02 +0200 Subject: [PATCH 3/4] Update build with IETF specs --- index.json | 332 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 332 insertions(+) diff --git a/index.json b/index.json index a07f5aac..feadd91c 100644 --- a/index.json +++ b/index.json @@ -63,6 +63,338 @@ ] } }, + { + "url": "https://datatracker.ietf.org/doc/html/rfc6265", + "seriesComposition": "full", + "shortname": "rfc6265", + "series": { + "shortname": "rfc6265", + "currentSpecification": "rfc6265", + "nightlyUrl": "https://httpwg.org/specs/rfc6265.html" + }, + "organization": "IETF", + "groups": [ + { + "name": "HTTP State Management Mechanism Working Group", + "url": "https://datatracker.ietf.org/wg/httpstate/" + } + ], + "nightly": { + "url": "https://httpwg.org/specs/rfc6265.html", + "repository": "https://github.com/httpwg/httpwg.github.io", + "sourcePath": "specs/rfc6265.xml", + "filename": "rfc6265.html" + }, + "title": "HTTP State Management Mechanism", + "source": "specref", + "shortTitle": "HTTP State Management Mechanism" + }, + { + "url": "https://datatracker.ietf.org/doc/html/rfc6454", + "seriesComposition": "full", + "shortname": "rfc6454", + "series": { + "shortname": "rfc6454", + "currentSpecification": "rfc6454", + "nightlyUrl": "https://datatracker.ietf.org/doc/html/rfc6454" + }, + "organization": "IETF", + "groups": [ + { + "name": "Web Security Working Group", + "url": "https://datatracker.ietf.org/wg/websec/" + } + ], + "nightly": { + "url": "https://datatracker.ietf.org/doc/html/rfc6454", + "filename": "rfc6454.html" + }, + "title": "The Web Origin Concept", + "source": "specref", + "shortTitle": "The Web Origin Concept" + }, + { + "url": "https://datatracker.ietf.org/doc/html/rfc7230", + "seriesComposition": "full", + "shortname": "rfc7230", + "series": { + "shortname": "rfc7230", + "currentSpecification": "rfc7230", + "nightlyUrl": "https://httpwg.org/specs/rfc7230.html" + }, + "organization": "IETF", + "groups": [ + { + "name": "Hypertext Transfer Protocol Bis APP Working Group", + "url": "https://datatracker.ietf.org/wg/httpbis/" + } + ], + "nightly": { + "url": "https://httpwg.org/specs/rfc7230.html", + "repository": "https://github.com/httpwg/httpwg.github.io", + "sourcePath": "specs/rfc7230.xml", + "filename": "rfc7230.html" + }, + "title": "Hypertext Transfer Protocol (HTTP/1.1): Message Syntax and Routing", + "source": "specref", + "shortTitle": "HTTP/1.1" + }, + { + "url": "https://datatracker.ietf.org/doc/html/rfc7231", + "seriesComposition": "full", + "shortname": "rfc7231", + "series": { + "shortname": "rfc7231", + "currentSpecification": "rfc7231", + "nightlyUrl": "https://httpwg.org/specs/rfc7231.html" + }, + "organization": "IETF", + "groups": [ + { + "name": "Hypertext Transfer Protocol Bis APP Working Group", + "url": "https://datatracker.ietf.org/wg/httpbis/" + } + ], + "nightly": { + "url": "https://httpwg.org/specs/rfc7231.html", + "repository": "https://github.com/httpwg/httpwg.github.io", + "sourcePath": "specs/rfc7231.xml", + "filename": "rfc7231.html" + }, + "title": "Hypertext Transfer Protocol (HTTP/1.1): Semantics and Content", + "source": "specref", + "shortTitle": "HTTP/1.1" + }, + { + "url": "https://datatracker.ietf.org/doc/html/rfc7232", + "seriesComposition": "full", + "shortname": "rfc7232", + "series": { + "shortname": "rfc7232", + "currentSpecification": "rfc7232", + "nightlyUrl": "https://httpwg.org/specs/rfc7232.html" + }, + "organization": "IETF", + "groups": [ + { + "name": "Hypertext Transfer Protocol Bis APP Working Group", + "url": "https://datatracker.ietf.org/wg/httpbis/" + } + ], + "nightly": { + "url": "https://httpwg.org/specs/rfc7232.html", + "repository": "https://github.com/httpwg/httpwg.github.io", + "sourcePath": "specs/rfc7232.xml", + "filename": "rfc7232.html" + }, + "title": "Hypertext Transfer Protocol (HTTP/1.1): Conditional Requests", + "source": "specref", + "shortTitle": "HTTP/1.1" + }, + { + "url": "https://datatracker.ietf.org/doc/html/rfc7233", + "seriesComposition": "full", + "shortname": "rfc7233", + "series": { + "shortname": "rfc7233", + "currentSpecification": "rfc7233", + "nightlyUrl": "https://httpwg.org/specs/rfc7233.html" + }, + "organization": "IETF", + "groups": [ + { + "name": "Hypertext Transfer Protocol Bis APP Working Group", + "url": "https://datatracker.ietf.org/wg/httpbis/" + } + ], + "nightly": { + "url": "https://httpwg.org/specs/rfc7233.html", + "repository": "https://github.com/httpwg/httpwg.github.io", + "sourcePath": "specs/rfc7233.xml", + "filename": "rfc7233.html" + }, + "title": "Hypertext Transfer Protocol (HTTP/1.1): Range Requests", + "source": "specref", + "shortTitle": "HTTP/1.1" + }, + { + "url": "https://datatracker.ietf.org/doc/html/rfc7234", + "seriesComposition": "full", + "shortname": "rfc7234", + "series": { + "shortname": "rfc7234", + "currentSpecification": "rfc7234", + "nightlyUrl": "https://httpwg.org/specs/rfc7234.html" + }, + "organization": "IETF", + "groups": [ + { + "name": "Hypertext Transfer Protocol Bis APP Working Group", + "url": "https://datatracker.ietf.org/wg/httpbis/" + } + ], + "nightly": { + "url": "https://httpwg.org/specs/rfc7234.html", + "repository": "https://github.com/httpwg/httpwg.github.io", + "sourcePath": "specs/rfc7234.xml", + "filename": "rfc7234.html" + }, + "title": "Hypertext Transfer Protocol (HTTP/1.1): Caching", + "source": "specref", + "shortTitle": "HTTP/1.1" + }, + { + "url": "https://datatracker.ietf.org/doc/html/rfc7235", + "seriesComposition": "full", + "shortname": "rfc7235", + "series": { + "shortname": "rfc7235", + "currentSpecification": "rfc7235", + "nightlyUrl": "https://httpwg.org/specs/rfc7235.html" + }, + "organization": "IETF", + "groups": [ + { + "name": "Hypertext Transfer Protocol Bis APP Working Group", + "url": "https://datatracker.ietf.org/wg/httpbis/" + } + ], + "nightly": { + "url": "https://httpwg.org/specs/rfc7235.html", + "repository": "https://github.com/httpwg/httpwg.github.io", + "sourcePath": "specs/rfc7235.xml", + "filename": "rfc7235.html" + }, + "title": "Hypertext Transfer Protocol (HTTP/1.1): Authentication", + "source": "specref", + "shortTitle": "HTTP/1.1" + }, + { + "url": "https://datatracker.ietf.org/doc/html/rfc7239", + "seriesComposition": "full", + "shortname": "rfc7239", + "series": { + "shortname": "rfc7239", + "currentSpecification": "rfc7239", + "nightlyUrl": "https://datatracker.ietf.org/doc/html/rfc7239" + }, + "organization": "IETF", + "groups": [ + { + "name": "Applications Area Working Group APP Working Group", + "url": "https://datatracker.ietf.org/wg/appsawg/" + } + ], + "nightly": { + "url": "https://datatracker.ietf.org/doc/html/rfc7239", + "filename": "rfc7239.html" + }, + "title": "Forwarded HTTP Extension", + "source": "specref", + "shortTitle": "Forwarded HTTP Extension" + }, + { + "url": "https://datatracker.ietf.org/doc/html/rfc7469", + "seriesComposition": "full", + "shortname": "rfc7469", + "series": { + "shortname": "rfc7469", + "currentSpecification": "rfc7469", + "nightlyUrl": "https://datatracker.ietf.org/doc/html/rfc7469" + }, + "organization": "IETF", + "groups": [ + { + "name": "Web Security Working Group", + "url": "https://datatracker.ietf.org/wg/websec/" + } + ], + "nightly": { + "url": "https://datatracker.ietf.org/doc/html/rfc7469", + "filename": "rfc7469.html" + }, + "title": "Public Key Pinning Extension for HTTP", + "source": "specref", + "shortTitle": "Public Key Pinning Extension for HTTP" + }, + { + "url": "https://datatracker.ietf.org/doc/html/rfc7538", + "seriesComposition": "full", + "shortname": "rfc7538", + "series": { + "shortname": "rfc7538", + "currentSpecification": "rfc7538", + "nightlyUrl": "https://httpwg.org/specs/rfc7538.html" + }, + "organization": "IETF", + "groups": [ + { + "name": "Hypertext Transfer Protocol Bis APP Working Group", + "url": "https://datatracker.ietf.org/wg/httpbis/" + } + ], + "nightly": { + "url": "https://httpwg.org/specs/rfc7538.html", + "repository": "https://github.com/httpwg/httpwg.github.io", + "sourcePath": "specs/rfc7538.xml", + "filename": "rfc7538.html" + }, + "title": "The Hypertext Transfer Protocol Status Code 308 (Permanent Redirect)", + "source": "specref", + "shortTitle": "Permanent Redirect" + }, + { + "url": "https://datatracker.ietf.org/doc/html/rfc7540", + "seriesComposition": "full", + "shortname": "rfc7540", + "series": { + "shortname": "rfc7540", + "currentSpecification": "rfc7540", + "nightlyUrl": "https://httpwg.org/specs/rfc7540.html" + }, + "organization": "IETF", + "groups": [ + { + "name": "Hypertext Transfer Protocol Bis APP Working Group", + "url": "https://datatracker.ietf.org/wg/httpbis/" + } + ], + "nightly": { + "url": "https://httpwg.org/specs/rfc7540.html", + "repository": "https://github.com/httpwg/httpwg.github.io", + "sourcePath": "specs/rfc7540.xml", + "filename": "rfc7540.html" + }, + "title": "Hypertext Transfer Protocol Version 2 (HTTP/2)", + "source": "specref", + "shortTitle": "HTTP/2" + }, + { + "url": "https://datatracker.ietf.org/doc/html/rfc8470", + "seriesComposition": "full", + "shortname": "rfc8470", + "series": { + "shortname": "rfc8470", + "currentSpecification": "rfc8470", + "nightlyUrl": "https://httpwg.org/specs/rfc8470.html" + }, + "organization": "IETF", + "groups": [ + { + "name": "Hypertext Transfer Protocol Bis Working Group", + "url": "https://datatracker.ietf.org/wg/httpbis/" + } + ], + "nightly": { + "url": "https://httpwg.org/specs/rfc8470.html", + "repository": "https://github.com/httpwg/httpwg.github.io", + "sourcePath": "specs/rfc8470.xml", + "filename": "rfc8470.html" + }, + "title": "Using Early Data in HTTP", + "source": "specref", + "shortTitle": "Using Early Data in HTTP" + }, { "url": "https://dom.spec.whatwg.org/", "seriesComposition": "full", From 687e149c501167e2a189c28da28f648f80717fe6 Mon Sep 17 00:00:00 2001 From: Dominique Hazael-Massieux Date: Fri, 2 Jul 2021 12:42:51 +0200 Subject: [PATCH 4/4] Relax requirement on repo and sourcePath on IETF specs The information is not systematically available for those --- test/index.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/index.js b/test/index.js index c7d59e40..138fdbf4 100644 --- a/test/index.js +++ b/test/index.js @@ -102,13 +102,13 @@ describe("List of specs", () => { assert.deepStrictEqual(wrong, []); }); - it("contains repository URLs for all specs", () => { - const wrong = specs.filter(s => !s.nightly.repository); + it("contains repository URLs for all non IETF specs", () => { + const wrong = specs.filter(s => !s.nightly.repository && !s.nightly.url.match(/ietf\.org/)); assert.deepStrictEqual(wrong, []); }); - it("contains relative paths to source of nightly spec for all specs", () => { - const wrong = specs.filter(s => !s.nightly.sourcePath); + it("contains relative paths to source of nightly spec for all non IETF specs", () => { + const wrong = specs.filter(s => !s.nightly.sourcePath && !s.nightly.url.match(/ietf\.org/)); assert.deepStrictEqual(wrong, []); }); @@ -121,4 +121,4 @@ describe("List of specs", () => { const wrong = specs.filter(s => s.release && !s.release.filename); assert.deepStrictEqual(wrong, []); }); -}); \ No newline at end of file +});