diff --git a/munge_aggregates.js b/munge_aggregates.js
index 732e9cd42..26cbc52f4 100644
--- a/munge_aggregates.js
+++ b/munge_aggregates.js
@@ -18,6 +18,51 @@ if (!dbFile || !hugoOutput) {
process.exit(1);
}
+/**
+ * @param {string} u a spec URL like "specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header"
+ * @returns the spec's parent, or "null" if it's a top-level spec
+ */
+const computeParent = (u) => {
+ const url = new URL(u);
+ const segments = url.pathname.split('/').filter(Boolean);
+
+ // if there's a hash, consider it as a segment
+ if (url.hash) segments.push(url.hash.substring(1));
+
+ if (segments.length <= 1) {
+ return "null";
+ }
+
+ const parent = segments.slice(0, -1).join('/');
+ return `${url.protocol}//${url.host}/${parent}`
+};
+
+/**
+ * @param {string} u a spec URL like "specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header"
+ * @returns the spec's name, or the hash if it's a top-level spec and whether it was found in a hash
+ */
+const computeName = (u) => {
+ const url = new URL(u);
+
+ if (url.hash) {
+ return {
+ isHashed: true,
+ name: url.hash.substring(1),
+ };
+ }
+
+ const segments = url.pathname.split('/').filter(Boolean);
+
+ if (segments.length === 0) {
+ throw new Error(`Invalid spec URL: ${u}`);
+ }
+
+ return {
+ isHashed: false,
+ name: segments[segments.length - 1],
+ };
+};
+
const main = async () => {
let db = new sqlite3.Database(dbFile, (err) => {
if (err) {
@@ -58,6 +103,8 @@ const main = async () => {
`;
const testsRows = await all(testsQuery);
const groups = {};
+ const flatTestGroups = {}; // used for specs generation.
+
for (const row of testsRows) {
const { versions, full_name, name, parent_test_full_name } = row;
const slug = slugify(full_name);
@@ -66,10 +113,115 @@ const main = async () => {
groups[parent_test_full_name] = {};
}
- groups[parent_test_full_name][full_name] = { versions: versions?.split(',') || [], name, full_name, slug };
+ const g = { versions: versions?.split(',') || [], name, full_name, slug };
+
+ groups[parent_test_full_name][full_name] = g;
+ flatTestGroups[full_name] = g;
}
outputJSON("data/testgroups.json", groups);
+ // Query to fetch all test specs
+ const specsQuery = `
+ SELECT
+ spec_url as full_name,
+ GROUP_CONCAT(DISTINCT test_run_version) AS versions
+ FROM TestSpecs
+ GROUP BY full_name
+ ORDER BY full_name
+ `;
+ const specsRows = await all(specsQuery);
+ const specs = {};
+ const flatSpecs = {};
+
+ for (const row of specsRows) {
+ const { versions, full_name } = row;
+ let current = full_name;
+
+ while (current !== "null") {
+ const slug = slugify(current);
+ const parent = computeParent(current);
+ const { name, isHashed } = computeName(current)
+
+ if (!specs[parent]) {
+ specs[parent] = {};
+ }
+
+ flatSpecs[current] = true
+
+ specs[parent][current] = {
+ versions: versions?.split(',') || [],
+ spec_full_name: current,
+ slug,
+ name,
+ isHashed,
+ };
+
+ current = parent;
+ }
+ }
+ outputJSON("data/specs.json", specs);
+
+ const descendTheSpecsTree = (current, path) => {
+ Object.entries(specs[current] || {})
+ .forEach(([key, spec]) => {
+ const addSpecs = (current) => {
+ let hashes = [...(current.specs || []), spec.name];
+ hashes = [...new Set(hashes)]; // deduplicate
+ return { ...current, hashes }
+ };
+
+ // To reproduce the structure of URLs and hashes, we update existing specs pages
+ if (spec.isHashed) {
+ const p = path.join("/");
+ outputFrontmatter(
+ `content/specs/${p}/_index.md`,
+ addSpecs
+ );
+ // We assume there are no recursion / children for hashes
+ return
+ }
+
+ const newPath = [...path, spec.name];
+ const p = newPath.join("/");
+
+ outputFrontmatter(`content/specs/${p}/_index.md`, {
+ ...spec,
+ title: spec.name,
+ });
+
+ descendTheSpecsTree(key, newPath);
+ })
+ }
+
+ descendTheSpecsTree("null", [])
+
+ // Aggregate test results per specs
+ const specsTestGroups = {};
+
+ for (const fullName of Object.keys(flatSpecs)) {
+ // list all the test names for a given spec.
+ // we prefix search the database for spec_urls starting with the spec name
+ const specsQuery = `
+ SELECT
+ test_full_name
+ FROM TestSpecs
+ WHERE spec_url LIKE ?
+ ORDER BY test_full_name
+ `;
+ const tests = await all(specsQuery, [fullName + '%']);
+
+ const s = tests.map(x => x.test_full_name)
+ .reduce((acc, name) => {
+ return {
+ ...acc,
+ [name]: flatTestGroups[name]
+ }
+ }, {});
+ specsTestGroups[fullName] = s;
+ }
+
+ outputJSON("data/specsgroups.json", specsTestGroups);
+
// Query to fetch all stdouts
const logsQuery = `
SELECT
@@ -269,6 +421,7 @@ const slugify = (str) => {
.replace(/_+/g, '_') // remove consecutive underscores
.replace(/[\/]/g, "__")
.replace(/[^a-z0-9 -]/g, '-') // remove non-alphanumeric characters
+ .replace(/-+/g, '-') // remove consecutive dashes
}
const outputJSON = (p, data) => {
@@ -283,7 +436,7 @@ const outputJSON = (p, data) => {
fs.writeFileSync(fullPath, json);
}
-const outputFrontmatter = (p, data) => {
+const outputFrontmatter = (p, dataOrUpdate) => {
const fullPath = `${hugoOutput}/${p}`;
// TODO: implement update frontmatter
@@ -303,7 +456,12 @@ const outputFrontmatter = (p, data) => {
content.content = existing.content;
content.data = existing.data;
}
- content.data = { ...content.data, ...data };
+
+ if (typeof dataOrUpdate === "function") {
+ content.data = dataOrUpdate(content.data);
+ } else {
+ content.data = { ...content.data, ...dataOrUpdate };
+ }
const md = matter.stringify(content.content, content.data);
fs.writeFileSync(fullPath, md);
@@ -322,4 +480,4 @@ main()
.catch((e) => {
console.error(e);
process.exit(1);
- })
\ No newline at end of file
+ })
diff --git a/munge_sql.js b/munge_sql.js
index c45773f44..601cdefe7 100644
--- a/munge_sql.js
+++ b/munge_sql.js
@@ -100,6 +100,28 @@ const main = async () => {
);
`)
+ // Create the SPECS
+ await run(`
+ CREATE TABLE IF NOT EXISTS TestSpecs (
+ test_run_implementation_id TEXT,
+ test_run_version TEXT,
+ test_full_name TEXT,
+
+ spec_url TEXT,
+
+ PRIMARY KEY (test_run_implementation_id, test_run_version, test_full_name, spec_url),
+
+ -- test run
+ FOREIGN KEY (test_run_implementation_id, test_run_version)
+ REFERENCES TestRun (implementation_id, version),
+
+ -- test result
+ FOREIGN KEY (test_run_implementation_id, test_run_version, test_full_name)
+ REFERENCES TestResult (test_run_implementation_id, test_run_version, full_name)
+ );
+ `);
+
+
for (const file of files) {
const fileName = file.split("/").slice(-1)[0].split(".")[0];
const implemId = fileName;
@@ -146,6 +168,16 @@ const main = async () => {
VALUES (?, ?, ?, ?)
`, [implemId, version, fullName, test.output]);
+ const specsArray = test.meta?.specs || [];
+ for (const specUrl of specsArray) {
+ // add `https://` if the specs don't have it
+ const cleanSpecUrl = specUrl.startsWith("http") ? specUrl : `https://${specUrl}`;
+
+ await run(`
+ INSERT INTO TestSpecs (test_run_implementation_id, test_run_version, test_full_name, spec_url)
+ VALUES (?, ?, ?, ?)
+ `, [implemId, version, fullName, cleanSpecUrl]);
+ }
}
}
diff --git a/tests/path_gateway_dag_test.go b/tests/path_gateway_dag_test.go
index 9c4d5e42a..2c4c9828e 100644
--- a/tests/path_gateway_dag_test.go
+++ b/tests/path_gateway_dag_test.go
@@ -42,7 +42,7 @@ func TestGatewayJsonCbor(t *testing.T) {
},
{
Name: "GET UnixFS file with JSON bytes is returned with application/json Content-Type - with headers",
- Spec: "specs.ipfs.tech/http-gateways/path-gateway/#accept-request-header",
+ Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#accept-request-header",
Hint: `
## Quick regression check for JSON stored on UnixFS:
## it has nothing to do with DAG-JSON and JSON codecs,
@@ -479,7 +479,7 @@ func TestNativeDag(t *testing.T) {
Response: Expect().
Headers(
Header("Content-Type").Hint("expected Content-Type").Equals("application/vnd.ipld.dag-{{format}}", row.Format),
- Header("Content-Length").Spec("specs.ipfs.tech/http-gateways/path-gateway/#content-disposition-response-header").Hint("includes Content-Length").Equals("{{length}}", len(dagTraversal.RawData())),
+ Header("Content-Length").Spec("https://specs.ipfs.tech/http-gateways/path-gateway/#content-disposition-response-header").Hint("includes Content-Length").Equals("{{length}}", len(dagTraversal.RawData())),
Header("Content-Disposition").Hint("includes Content-Disposition").Contains(`{{disposition}}; filename="{{cid}}.{{format}}"`, row.Disposition, dagTraversalCID, row.Format),
Header("X-Content-Type-Options").Hint("includes nosniff hint").Contains("nosniff"),
),
@@ -553,7 +553,7 @@ func TestNativeDag(t *testing.T) {
},
{
Name: Fmt("HEAD {{name}} with only-if-cached for missing block returns HTTP 412 Precondition Failed", row.Name),
- Spec: "specs.ipfs.tech/http-gateways/path-gateway/#only-if-cached",
+ Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#only-if-cached",
Request: Request().
Path("/ipfs/{{cid}}", missingCID).
Header("Cache-Control", "only-if-cached").
diff --git a/tests/path_gateway_tar_test.go b/tests/path_gateway_tar_test.go
index d6131d0c7..15f5c3805 100644
--- a/tests/path_gateway_tar_test.go
+++ b/tests/path_gateway_tar_test.go
@@ -76,7 +76,7 @@ func TestTar(t *testing.T) {
},
{
Name: "GET TAR with explicit ?filename= succeeds with modified Content-Disposition header",
- Spec: "specs.ipfs.tech/http-gateways/path-gateway/#content-disposition-response-header",
+ Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#content-disposition-response-header",
Request: Request().
Path("/ipfs/{{cid}}", dirCID).
Query("filename", "testтест.tar").
diff --git a/tests/path_gateway_unixfs_test.go b/tests/path_gateway_unixfs_test.go
index c0e6fc500..ae3705d82 100644
--- a/tests/path_gateway_unixfs_test.go
+++ b/tests/path_gateway_unixfs_test.go
@@ -202,7 +202,7 @@ func TestGatewayCache(t *testing.T) {
// ==========
{
Name: "GET for /ipfs/ file with matching Etag in If-None-Match returns 304 Not Modified",
- Spec: "specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header",
+ Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header",
Request: Request().
Path("/ipfs/{{cid}}/root2/root3/root4/index.html", fixture.MustGetCid()).
Headers(
@@ -213,7 +213,7 @@ func TestGatewayCache(t *testing.T) {
},
{
Name: "GET for /ipfs/ dir with index.html file with matching Etag in If-None-Match returns 304 Not Modified",
- Spec: "specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header",
+ Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header",
Request: Request().
Path("/ipfs/{{cid}}/root2/root3/root4/", fixture.MustGetCid()).
Headers(
@@ -224,7 +224,7 @@ func TestGatewayCache(t *testing.T) {
},
{
Name: "GET for /ipfs/ file with matching third Etag in If-None-Match returns 304 Not Modified",
- Spec: "specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header",
+ Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header",
Request: Request().
Path("/ipfs/{{cid}}/root2/root3/root4/index.html", fixture.MustGetCid()).
Headers(
@@ -235,7 +235,7 @@ func TestGatewayCache(t *testing.T) {
},
{
Name: "GET for /ipfs/ file with matching weak Etag in If-None-Match returns 304 Not Modified",
- Spec: "specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header",
+ Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header",
Request: Request().
Path("/ipfs/{{cid}}/root2/root3/root4/index.html", fixture.MustGetCid()).
Headers(
@@ -246,7 +246,7 @@ func TestGatewayCache(t *testing.T) {
},
{
Name: "GET for /ipfs/ file with wildcard Etag in If-None-Match returns 304 Not Modified",
- Spec: "specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header",
+ Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header",
Request: Request().
Path("/ipfs/{{cid}}/root2/root3/root4/index.html", fixture.MustGetCid()).
Headers(
@@ -257,7 +257,7 @@ func TestGatewayCache(t *testing.T) {
},
{
Name: "GET for /ipfs/ dir listing with matching weak Etag in If-None-Match returns 304 Not Modified",
- Spec: "specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header",
+ Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header",
Request: Request().
Path("/ipfs/{{cid}}/root2/root3/", fixture.MustGetCid()).
Headers(
diff --git a/tests/redirects_file_test.go b/tests/redirects_file_test.go
index 9322c74ff..64e46d246 100644
--- a/tests/redirects_file_test.go
+++ b/tests/redirects_file_test.go
@@ -15,7 +15,7 @@ import (
)
func TestRedirectsFileSupport(t *testing.T) {
- tooling.LogSpecs(t, "specs.ipfs.tech/http-gateways/web-redirects-file/")
+ tooling.LogSpecs(t, "https://specs.ipfs.tech/http-gateways/web-redirects-file/")
fixture := car.MustOpenUnixfsCar("redirects_file/redirects.car")
redirectDir := fixture.MustGetNode("examples")
redirectDirCID := redirectDir.Base32Cid()
@@ -166,8 +166,8 @@ func TestRedirectsFileSupport(t *testing.T) {
Contains("could not parse _redirects:"),
Contains(`forced redirects (or "shadowing") are not supported`),
),
- ).Spec("specs.ipfs.tech/http-gateways/web-redirects-file/#no-forced-redirects"),
- Spec: "specs.ipfs.tech/http-gateways/web-redirects-file/#error-handling",
+ ).Spec("https://specs.ipfs.tech/http-gateways/web-redirects-file/#no-forced-redirects"),
+ Spec: "https://specs.ipfs.tech/http-gateways/web-redirects-file/#error-handling",
},
{
Name: "invalid file: request for $TOO_LARGE_REDIRECTS_DIR_HOSTNAME/not-found returns error about too large redirects file",
@@ -182,7 +182,7 @@ func TestRedirectsFileSupport(t *testing.T) {
Contains("redirects file size cannot exceed"),
),
),
- Spec: "specs.ipfs.tech/http-gateways/web-redirects-file/#max-file-size",
+ Spec: "https://specs.ipfs.tech/http-gateways/web-redirects-file/#max-file-size",
},
}...)
diff --git a/tests/trustless_gateway_car_test.go b/tests/trustless_gateway_car_test.go
index 5a5114c63..5f0aa7e3a 100644
--- a/tests/trustless_gateway_car_test.go
+++ b/tests/trustless_gateway_car_test.go
@@ -408,7 +408,7 @@ func TestTrustlessCarDagScopeAll(t *testing.T) {
func TestTrustlessCarEntityBytes(t *testing.T) {
tooling.LogTestGroup(t, GroupBlockCar)
- tooling.LogSpecs(t, "specs.ipfs.tech/http-gateways/trustless-gateway/#entity-bytes-request-query-parameter")
+ tooling.LogSpecs(t, "https://specs.ipfs.tech/http-gateways/trustless-gateway/#entity-bytes-request-query-parameter")
singleLayerHamtMultiBlockFilesFixture := car.MustOpenUnixfsCar("trustless_gateway_car/single-layer-hamt-with-multi-block-files.car")
subdirWithMixedBlockFiles := car.MustOpenUnixfsCar("trustless_gateway_car/subdir-with-mixed-block-files.car")
diff --git a/tests/trustless_gateway_raw_test.go b/tests/trustless_gateway_raw_test.go
index 8bef78ea4..2ba80f869 100644
--- a/tests/trustless_gateway_raw_test.go
+++ b/tests/trustless_gateway_raw_test.go
@@ -14,7 +14,7 @@ import (
func TestTrustlessRaw(t *testing.T) {
tooling.LogTestGroup(t, GroupBlockCar)
- tooling.LogSpecs(t, "specs.ipfs.tech/http-gateways/trustless-gateway/#block-responses-application-vnd-ipld-raw")
+ tooling.LogSpecs(t, "https://specs.ipfs.tech/http-gateways/trustless-gateway/#block-responses-application-vnd-ipld-raw")
fixture := car.MustOpenUnixfsCar("gateway-raw-block.car")
diff --git a/www/content/_index.md b/www/content/_index.md
index 631f9a21f..9f3d9c2a5 100644
--- a/www/content/_index.md
+++ b/www/content/_index.md
@@ -11,28 +11,26 @@ menu:
}}" class="w-32 h-32" />
- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed tincidunt sagittis arcu, in tempus nisi molestie at. Suspendisse imperdiet viverra fringilla. Sed eleifend elementum sem. Phasellus orci lectus, laoreet in sapien vulputate, bibendum cursus neque. Quisque luctus dictum ligula, sit amet sagittis lacus consectetur eget. Phasellus non diam sem. Duis pellentesque tellus quis dolor sodales, vitae faucibus nulla ornare. Proin eget odio eu orci tristique volutpat. Nunc non vehicula neque. Maecenas volutpat mollis sem eget vestibulum.
-
+
+IPFS Gateway Conformance - a vendor-agnostic gateway conformance test suite for users and implementers of IPFS Gateways. We ensure compliance with [specs.ipfs.tech](https://specs.ipfs.tech/http-gateways/). Find us on [Github](https://github.com/ipfs/gateway-conformance).
+
+