From 4a95ee35a87d8bdb04c964a439f99aed9dca2c2c Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Mon, 4 Dec 2023 01:05:27 -0500 Subject: [PATCH 01/26] ci: add gateway-conformance tests --- .github/workflows/gateway-conformance.yml | 120 ++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 .github/workflows/gateway-conformance.yml diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml new file mode 100644 index 0000000..ac8896a --- /dev/null +++ b/.github/workflows/gateway-conformance.yml @@ -0,0 +1,120 @@ +name: Gateway Conformance + +on: + push: + branches: + - main + pull_request: + +jobs: + gateway-conformance: + runs-on: ubuntu-latest + steps: + # 1. Start the Kubo gateway + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: 1.21.x + + - name: Install Kubo gateway from source + #uses: ipfs/download-ipfs-distribution-action@v1 + run: | + go install github.com/ipfs/kubo/cmd/ipfs@v0.24.0 + - name: Setup kubo config + run: | + ipfs init --profile=test + ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/8080" + + # 2. Download the gateway-conformance fixtures + - name: Download gateway-conformance fixtures + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.4.1 + with: + output: fixtures + + - name: Start Kubo gateway + uses: ipfs/start-ipfs-daemon-action@v1 + + # 3. Populate the Kubo gateway with the gateway-conformance fixtures + - name: Import fixtures + run: | + # Import car files + find ./fixtures -name '*.car' -exec ipfs dag import --pin-roots=false --offline {} \; + + # Import ipns records + records=$(find ./fixtures -name '*.ipns-record') + for record in $records + do + key=$(basename -s .ipns-record "$record" | cut -d'_' -f1) + ipfs routing put --allow-offline "/ipns/$key" "$record" + done + + # Import dnslink records + # the IPFS_NS_MAP env will be used by the daemon + export IPFS_NS_MAP=$(cat "./fixtures/dnslinks.json" | jq -r '.subdomains | to_entries | map("\(.key).example.com:\(.value)") | join(",")') + export IPFS_NS_MAP="$(cat "./fixtures/dnslinks.json" | jq -r '.domains | to_entries | map("\(.key):\(.value)") | join(",")'),${IPFS_NS_MAP}" + echo "IPFS_NS_MAP=${IPFS_NS_MAP}" >> $GITHUB_ENV + + # 4. Build helia-http-gateway + - name: Setup Node + uses: actions/setup-node@v3 + with: + node-version: 20 + + - name: Checkout helia-http-gateway + uses: actions/checkout@v3 + with: + path: helia-http-gateway + + - name: Install dependencies + run: npm ci + working-directory: helia-http-gateway + + - name: Build helia-http-gateway + run: npm run build + working-directory: helia-http-gateway + + # 5. Start helia-http-gateway + - name: Start helia-http-gateway + env: + GATEWAY_CONFORMANCE_TEST: true + TRUSTLESS_GATEWAY: "http://127.0.0.1:8080" + USE_LIBP2P: false + PORT: 8090 + run: | + # run gw + node dist/src/index.js & + working-directory: helia-http-gateway + + # 6. Run the gateway-conformance tests + - name: Run gateway-conformance tests + uses: ipfs/gateway-conformance/.github/actions/test@v0.4.1 + with: + gateway-url: http://127.0.0.1:8090 + json: output.json + xml: output.xml + html: output.html + markdown: output.md + specs: +trustless-gateway,+path-gateway,+subdomain-gateway,+dnslink-gateway,+redirects-file + # use below to skip specific test if needed + # args: -skip 'TestFooBr/GET_response_for_something' + # + # only-if-cached: helia-ht does not guarantee local cache, we will adjust upstream test (which was Kubo-specific) + # for now disabling these test cases + args: -skip 'TestGatewayCache/.*_for_/ipfs/_with_only-if-cached_succeeds_when_in_local_datastore' + + # 7. Upload the results + - name: Upload MD summary + if: failure() || success() + run: cat output.md >> $GITHUB_STEP_SUMMARY + - name: Upload HTML report + if: failure() || success() + uses: actions/upload-artifact@v3 + with: + name: gateway-conformance.html + path: output.html + - name: Upload JSON report + if: failure() || success() + uses: actions/upload-artifact@v3 + with: + name: gateway-conformance.json + path: output.json From db993faf1131ac9742008849db0f68173cfeaf54 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Mon, 4 Dec 2023 03:36:13 -0500 Subject: [PATCH 02/26] ci(gateway-conformance): use minimal specs for websites (no _redirects) --- .github/workflows/gateway-conformance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index ac8896a..562444e 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -94,7 +94,7 @@ jobs: xml: output.xml html: output.html markdown: output.md - specs: +trustless-gateway,+path-gateway,+subdomain-gateway,+dnslink-gateway,+redirects-file + specs: -trustless-gateway,+path-gateway,+subdomain-gateway,+dnslink-gateway,-path-dag-gateway,-path-tar-gateway # use below to skip specific test if needed # args: -skip 'TestFooBr/GET_response_for_something' # From 46c4ad9d92130f4812e8cb7246233bb9586c2f2e Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Mon, 4 Dec 2023 13:36:34 -0800 Subject: [PATCH 03/26] feat: enable gateway conformance testing locally --- .gitignore | 2 + package-lock.json | 658 ++++++++++++++++++++++++++++++++++++++----- package.json | 7 + scripts/kubo-init.js | 134 +++++++++ scripts/tmp/.gitkeep | 0 5 files changed, 730 insertions(+), 71 deletions(-) create mode 100644 scripts/kubo-init.js create mode 100644 scripts/tmp/.gitkeep diff --git a/.gitignore b/.gitignore index 149c6fe..45168ce 100644 --- a/.gitignore +++ b/.gitignore @@ -134,3 +134,5 @@ build /playwright/.cache/ screenshots .envrc +scripts/tmp/fixtures +scripts/tmp/kubo-path.txt diff --git a/package-lock.json b/package-lock.json index 2b623b4..afe3a42 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31,6 +31,8 @@ "concurrently": "^8.2.2", "cross-env": "^7.0.3", "debug": "4.3.4", + "execa": "^8.0.1", + "glob": "^10.3.10", "typescript": "5.x", "wait-on": "^7.1.0" }, @@ -1601,6 +1603,48 @@ "rimraf": "^3.0.2" } }, + "node_modules/@clinic/flame/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/@clinic/flame/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@clinic/flame/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/@clinic/flame/node_modules/rimraf": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", @@ -6311,6 +6355,16 @@ "inherits": "2.0.3" } }, + "node_modules/browserify/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, "node_modules/browserify/node_modules/buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.2.1.tgz", @@ -6336,12 +6390,44 @@ "typedarray": "^0.0.6" } }, + "node_modules/browserify/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/browserify/node_modules/inherits": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==", "dev": true }, + "node_modules/browserify/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/browserify/node_modules/punycode": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", @@ -6508,6 +6594,48 @@ "node": ">=12" } }, + "node_modules/c8/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/c8/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/c8/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/c8/node_modules/rimraf": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", @@ -7145,6 +7273,16 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/clinic/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, "node_modules/clinic/node_modules/buffer": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", @@ -7325,6 +7463,26 @@ "node": ">=6" } }, + "node_modules/clinic/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/clinic/node_modules/got": { "version": "9.6.0", "resolved": "https://registry.npmjs.org/got/-/got-9.6.0.tgz", @@ -7483,6 +7641,18 @@ "node": ">=6" } }, + "node_modules/clinic/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/clinic/node_modules/normalize-url": { "version": "4.5.1", "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.1.tgz", @@ -9414,6 +9584,36 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/del/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/del/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/del/node_modules/globby": { "version": "11.1.0", "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", @@ -9434,6 +9634,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/del/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/del/node_modules/p-map": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz", @@ -10095,6 +10307,38 @@ "node": ">=12" } }, + "node_modules/electron-mocha-main/node_modules/glob": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", + "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/electron-mocha-main/node_modules/glob/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/electron-mocha-main/node_modules/has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -12214,6 +12458,48 @@ "node": ">=12.0.0" } }, + "node_modules/flat-cache/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/flat-cache/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/flat-cache/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/flat-cache/node_modules/rimraf": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", @@ -12610,6 +12896,36 @@ "node": ">=0.10.0" } }, + "node_modules/gh-pages/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/gh-pages/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/gh-pages/node_modules/globby": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/globby/-/globby-6.1.0.tgz", @@ -12626,6 +12942,18 @@ "node": ">=0.10.0" } }, + "node_modules/gh-pages/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/gh-pages/node_modules/pify": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", @@ -12659,20 +12987,22 @@ } }, "node_modules/glob": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", - "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", + "version": "10.3.10", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.10.tgz", + "integrity": "sha512-fa46+tv1Ak0UPK1TOy/pZrIybNNt4HCv7SDzwyfiOZkvZLEbjsZkJBPtDHVshZjbecAoAGSC20MjLDG/qr679g==", "dev": true, "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" + "foreground-child": "^3.1.0", + "jackspeak": "^2.3.5", + "minimatch": "^9.0.1", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0", + "path-scurry": "^1.10.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" }, "engines": { - "node": "*" + "node": ">=16 || 14 >=14.17" }, "funding": { "url": "https://github.com/sponsors/isaacs" @@ -12690,26 +13020,29 @@ "node": ">=10.13.0" } }, - "node_modules/glob/node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "node_modules/glob/node_modules/foreground-child": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", + "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", "dev": true, "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" + "cross-spawn": "^7.0.0", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/glob/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "node_modules/glob/node_modules/minipass": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz", + "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==", "dev": true, - "dependencies": { - "brace-expansion": "^1.1.7" - }, "engines": { - "node": "*" + "node": ">=16 || 14 >=14.17" } }, "node_modules/global-agent": { @@ -14809,6 +15142,48 @@ "node": ">=8" } }, + "node_modules/istanbul-lib-processinfo/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/istanbul-lib-processinfo/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/istanbul-lib-processinfo/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/istanbul-lib-processinfo/node_modules/p-map": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/p-map/-/p-map-3.0.0.tgz", @@ -17728,6 +18103,48 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/mocha/node_modules/glob": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", + "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/mocha/node_modules/glob/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/mocha/node_modules/glob/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/mocha/node_modules/has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -22725,6 +23142,16 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/nyc/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, "node_modules/nyc/node_modules/camelcase": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", @@ -22776,6 +23203,26 @@ "node": ">=8" } }, + "node_modules/nyc/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/nyc/node_modules/is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", @@ -22797,6 +23244,18 @@ "node": ">=8" } }, + "node_modules/nyc/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/nyc/node_modules/p-limit": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", @@ -23918,6 +24377,16 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/patch-package/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, "node_modules/patch-package/node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -23983,6 +24452,26 @@ "node": ">=10" } }, + "node_modules/patch-package/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/patch-package/node_modules/has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -23992,6 +24481,18 @@ "node": ">=8" } }, + "node_modules/patch-package/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/patch-package/node_modules/path-key": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", @@ -26720,53 +27221,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/rimraf/node_modules/foreground-child": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", - "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", - "dev": true, - "dependencies": { - "cross-spawn": "^7.0.0", - "signal-exit": "^4.0.1" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/rimraf/node_modules/glob": { - "version": "10.3.10", - "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.10.tgz", - "integrity": "sha512-fa46+tv1Ak0UPK1TOy/pZrIybNNt4HCv7SDzwyfiOZkvZLEbjsZkJBPtDHVshZjbecAoAGSC20MjLDG/qr679g==", - "dev": true, - "dependencies": { - "foreground-child": "^3.1.0", - "jackspeak": "^2.3.5", - "minimatch": "^9.0.1", - "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0", - "path-scurry": "^1.10.1" - }, - "bin": { - "glob": "dist/esm/bin.mjs" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/rimraf/node_modules/minipass": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz", - "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==", - "dev": true, - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, "node_modules/ripemd160": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/ripemd160/-/ripemd160-2.0.2.tgz", @@ -28226,6 +28680,48 @@ "node": ">=8" } }, + "node_modules/spawn-wrap/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/spawn-wrap/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/spawn-wrap/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/spawn-wrap/node_modules/rimraf": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", @@ -28954,6 +29450,26 @@ "concat-map": "0.0.1" } }, + "node_modules/test-exclude/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/test-exclude/node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", diff --git a/package.json b/package.json index 8dd0837..6061c50 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,11 @@ "test:http-e2e": "cross-env USE_BITSWAP=false USE_LIBP2P=false playwright test", "test:e2e-flame": "concurrently -k -s all -n \"gateway,playwright\" -c \"magenta,blue\" \"npm run start:dev-flame\" \"wait-on 'tcp:$PORT' && npm run test:e2e\"", "test:e2e-doctor": "concurrently -k -s all -n \"gateway,playwright\" -c \"magenta,blue\" \"npm run start:dev-doctor\" \"wait-on 'tcp:$PORT' && npm run test:e2e\"", + "test:gwc-kubo": "node ./scripts/kubo-init.js", + "test:gwc-helia": "npm run build && USE_LIBP2P=false PORT=8090 TRUSTLESS_GATEWAY='http://127.0.0.1:8080' npm start", + "test:gwc-execute": "docker run --network host -v $PWD:/workspace -w /workspace ghcr.io/ipfs/gateway-conformance:v0.4.2 test --gateway-url http://localhost:8090 --json report.json --specs -path-ipns-gateway,-trustless-block-gateway,-trustless-car-gateway,-trustless-car-gateway-optional,-trustless-ipns-gateway,-trustless-gateway,-subdomain-gateway,-redirects-file -- -timeout 30m", + "test:gwc": "concurrently -k -s all -n \"kubo,helia,gateway-conformance\" -c \"magenta,blue,green\" \"npm run test:gwc-kubo\" \"wait-on 'tcp:8080' && npm run test:gwc-helia\" \"wait-on 'tcp:8090' && npm run test:gwc-execute\"", + "test:gateway-conformance": "TRUSTLESS_GATEWAY='http://localhost:8080' echo 'gateway conformance'", "healthcheck": "node dist/src/healthcheck.js", "debug:until-death": "./debugging/until-death.sh", "debug:test-gateways": "./debugging/test-gateways.sh" @@ -57,6 +62,8 @@ "concurrently": "^8.2.2", "cross-env": "^7.0.3", "debug": "4.3.4", + "execa": "^8.0.1", + "glob": "^10.3.10", "typescript": "5.x", "wait-on": "^7.1.0" }, diff --git a/scripts/kubo-init.js b/scripts/kubo-init.js new file mode 100644 index 0000000..b2678f0 --- /dev/null +++ b/scripts/kubo-init.js @@ -0,0 +1,134 @@ +/** + * This is part of the gateway conformance testing of helia-http-gateway. See ../DEVELOPER-NOTES.md for more details. + */ + +import { writeFile, readFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, basename, relative } from 'node:path' +import debug from 'debug' +import { $ } from 'execa' +import { glob } from 'glob' + +const log = debug('kubo-init') +const error = log.extend('error') +debug.enable('kubo-init*') + +const kuboFilePath = './scripts/tmp/kubo-path.txt' +const GWC_FIXTURES_PATH = `${dirname(kuboFilePath)}/fixtures` + +async function main () { + await $`mkdir -p ${dirname(kuboFilePath)}` + + const tmpDir = await writeKuboMetaData() + + await attemptKuboInit(tmpDir) + + await configureKubo(tmpDir) + + const ipfsNsMap = await loadFixtures(tmpDir) + // execute the daemon + const execaOptions = getExecaOptions({ tmpDir, ipfsNsMap }) + await $(execaOptions)`npx kubo daemon --offline` +} + +/** + * + * @param {Record} param0 + * @param {string} param0.tmpDir + * @param {string} [param0.cwd] + * @param {string} [param0.ipfsNsMap] + * + * @returns {import('execa').Options} + */ +function getExecaOptions ({ cwd, ipfsNsMap, tmpDir }) { + return { + cwd, + env: { + IPFS_PATH: tmpDir, + IPFS_NS_MAP: ipfsNsMap + } + } +} + +async function attemptKuboInit (tmpDir) { + const execaOptions = getExecaOptions({ tmpDir }) + try { + await $(execaOptions)`npx -y kubo init` + log('Kubo initialized at %s', tmpDir) + } catch (e) { + if (!e.stderr.includes('already exists!')) { + throw e + } + log('Kubo was already initialized at %s', tmpDir) + } +} + +async function writeKuboMetaData () { + let tmpDir + try { + const currentIpfsPath = await readFile('./scripts/tmp/kubo-path.txt', 'utf-8') + log('Existing kubo path found at %s', currentIpfsPath) + tmpDir = currentIpfsPath + } catch (e) { + error('Failed to read Kubo path from %s', kuboFilePath, e) + tmpDir = tmpdir() + '/kubo-tmp' + log('Using temporary Kubo path at %s', tmpDir) + } + try { + await writeFile(kuboFilePath, tmpDir) + } catch (e) { + error('Failed to save Kubo path to %s', kuboFilePath, e) + } + return tmpDir +} + +async function configureKubo (tmpDir) { + const execaOptions = getExecaOptions({ tmpDir }) + try { + await $(execaOptions)`npx -y kubo config Addresses.Gateway /ip4/127.0.0.1/tcp/8080` + log('Kubo configured') + } catch (e) { + error('Failed to configure Kubo', e) + } +} + +async function downloadFixtures () { + log('Downloading fixtures') + try { + await $`docker run -v ${process.cwd()}:/workspace -w /workspace ghcr.io/ipfs/gateway-conformance:v0.4.2 extract-fixtures --directory ${GWC_FIXTURES_PATH} --merged false` + } catch (e) { + error('Error downloading fixtures, assuming current or previous success', e) + } +} + +async function loadFixtures (tmpDir) { + await downloadFixtures() + const execaOptions = getExecaOptions({ tmpDir }) + + for (const carFile of await glob([`${GWC_FIXTURES_PATH}/**/*.car`])) { + log('Loading *.car fixture %s', carFile) + const { stdout } = await $(execaOptions)`npx kubo dag import --pin-roots=false --offline ${carFile}` + stdout.split('\n').forEach(log) + } + + for (const ipnsRecord of await glob([`${GWC_FIXTURES_PATH}/**/*.ipns-record`])) { + const key = basename(ipnsRecord, '.ipns-record') + const relativePath = relative(GWC_FIXTURES_PATH, ipnsRecord) + log('Loading *.ipns-record fixture %s', relativePath) + const { stdout } = await $(({ ...execaOptions }))`cd ${GWC_FIXTURES_PATH} && npx kubo routing put --allow-offline "/ipns/${key}" "${relativePath}"` + stdout.split('\n').forEach(log) + } + + const json = await readFile(`${GWC_FIXTURES_PATH}/dnslinks.json`, 'utf-8') + const { subdomains, domains } = JSON.parse(json) + const subdomainDnsLinks = Object.entries(subdomains).map(([key, value]) => `${key}.example.com:${value}`).join(',') + const domainDnsLinks = Object.entries(domains).map(([key, value]) => `${key}:${value}`).join(',') + const ipfsNsMap = `${domainDnsLinks},${subdomainDnsLinks}` + + return ipfsNsMap +} + +main().catch(e => { + error(e) + process.exit(1) +}) diff --git a/scripts/tmp/.gitkeep b/scripts/tmp/.gitkeep new file mode 100644 index 0000000..e69de29 From a57d76cceb3bbb279192de38d0cf31f2ac72021b Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Mon, 4 Dec 2023 13:42:13 -0800 Subject: [PATCH 04/26] docs: gwc info added to DEVELOPER-NOTES.md --- DEVELOPER-NOTES.md | 25 +++++++++++++++++++++++++ package.json | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 DEVELOPER-NOTES.md diff --git a/DEVELOPER-NOTES.md b/DEVELOPER-NOTES.md new file mode 100644 index 0000000..2fbc72e --- /dev/null +++ b/DEVELOPER-NOTES.md @@ -0,0 +1,25 @@ +# Developer notes + + + +## Gateway Conformance testing + +We have some code enabled that makes running gateway-conformance testing against helia-http-gateway easy. Follow the instructions in this section to run gateway-conformance tests locally + +### Prerequisites + +1. [Install docker](https://docs.docker.com/get-docker/) +2. [Install nodejs](https://nodejs.org/en/download/) + +### Run gateway-conformance tests locally + +```sh +$ npm run test:gwc +``` + +### Some callouts + +1. The file `./scripts/kubo-init.js` executes kubo using `execa` instead of `ipfsd-ctl` so there may be some gotchas, but it should be as cross-platform and stable as the `execa` library. +1. The IPFS_PATH used is a temporary directory. Your OS should handle removing it when vital, but you can also remove it manually. The path to this directory is printed out when the tests start, and saved in a file at `./scripts/tmp/kubo-path.txt`. +1. The tests save gateway-conformance fixtures to `./scripts/tmp/fixtures`. You can remove this directory manually if you want to re-run the tests with a fresh set of fixtures. +1. The results of the gateway-conformance tests are saved to `./gwc-report.json`. This file is overwritten every time the tests are run. diff --git a/package.json b/package.json index 6061c50..5ec89bd 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "test:e2e-doctor": "concurrently -k -s all -n \"gateway,playwright\" -c \"magenta,blue\" \"npm run start:dev-doctor\" \"wait-on 'tcp:$PORT' && npm run test:e2e\"", "test:gwc-kubo": "node ./scripts/kubo-init.js", "test:gwc-helia": "npm run build && USE_LIBP2P=false PORT=8090 TRUSTLESS_GATEWAY='http://127.0.0.1:8080' npm start", - "test:gwc-execute": "docker run --network host -v $PWD:/workspace -w /workspace ghcr.io/ipfs/gateway-conformance:v0.4.2 test --gateway-url http://localhost:8090 --json report.json --specs -path-ipns-gateway,-trustless-block-gateway,-trustless-car-gateway,-trustless-car-gateway-optional,-trustless-ipns-gateway,-trustless-gateway,-subdomain-gateway,-redirects-file -- -timeout 30m", + "test:gwc-execute": "docker run --network host -v $PWD:/workspace -w /workspace ghcr.io/ipfs/gateway-conformance:v0.4.2 test --gateway-url http://localhost:8090 --json gwc-report.json --specs -path-ipns-gateway,-trustless-block-gateway,-trustless-car-gateway,-trustless-car-gateway-optional,-trustless-ipns-gateway,-trustless-gateway,-subdomain-gateway,-redirects-file -- -timeout 30m", "test:gwc": "concurrently -k -s all -n \"kubo,helia,gateway-conformance\" -c \"magenta,blue,green\" \"npm run test:gwc-kubo\" \"wait-on 'tcp:8080' && npm run test:gwc-helia\" \"wait-on 'tcp:8090' && npm run test:gwc-execute\"", "test:gateway-conformance": "TRUSTLESS_GATEWAY='http://localhost:8080' echo 'gateway conformance'", "healthcheck": "node dist/src/healthcheck.js", From 98afa15b7fb0cd6b7f5ee31156ce4b136b3c66b5 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Mon, 4 Dec 2023 13:52:34 -0800 Subject: [PATCH 05/26] fix(test:gwc): test:gwc-setup split out from test:gwc --- package.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 5ec89bd..b3bde97 100644 --- a/package.json +++ b/package.json @@ -19,8 +19,9 @@ "test:e2e-doctor": "concurrently -k -s all -n \"gateway,playwright\" -c \"magenta,blue\" \"npm run start:dev-doctor\" \"wait-on 'tcp:$PORT' && npm run test:e2e\"", "test:gwc-kubo": "node ./scripts/kubo-init.js", "test:gwc-helia": "npm run build && USE_LIBP2P=false PORT=8090 TRUSTLESS_GATEWAY='http://127.0.0.1:8080' npm start", - "test:gwc-execute": "docker run --network host -v $PWD:/workspace -w /workspace ghcr.io/ipfs/gateway-conformance:v0.4.2 test --gateway-url http://localhost:8090 --json gwc-report.json --specs -path-ipns-gateway,-trustless-block-gateway,-trustless-car-gateway,-trustless-car-gateway-optional,-trustless-ipns-gateway,-trustless-gateway,-subdomain-gateway,-redirects-file -- -timeout 30m", - "test:gwc": "concurrently -k -s all -n \"kubo,helia,gateway-conformance\" -c \"magenta,blue,green\" \"npm run test:gwc-kubo\" \"wait-on 'tcp:8080' && npm run test:gwc-helia\" \"wait-on 'tcp:8090' && npm run test:gwc-execute\"", + "test:gwc-setup": "concurrently -k -s all -n \"kubo,helia\" -c \"magenta,blue\" \"npm run test:gwc-kubo\" \"wait-on 'tcp:8080' && npm run test:gwc-helia\"", + "test:gwc-execute": "docker run --network host -v $PWD:/workspace -w /workspace ghcr.io/ipfs/gateway-conformance:v0.4.2 test --gateway-url http://localhost:8090 --json gwc-report.json --specs subdomain-ipfs-gateway,subdomain-ipns-gateway -- -timeout 30m", + "test:gwc": "concurrently -k -s all -n \"kubo&helia,gateway-conformance\" -c \"magenta,blue\" \"npm run test:gwc-setup\" \"wait-on 'tcp:8090' && npm run test:gwc-execute\"", "test:gateway-conformance": "TRUSTLESS_GATEWAY='http://localhost:8080' echo 'gateway conformance'", "healthcheck": "node dist/src/healthcheck.js", "debug:until-death": "./debugging/until-death.sh", From 5549c417e6b6a6187f492c9c48d93d887e2ca2a4 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Mon, 4 Dec 2023 14:27:19 -0800 Subject: [PATCH 06/26] fix: use proper env var and cleanup scripts --- .github/workflows/gateway-conformance.yml | 8 ++++---- package.json | 5 ++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index 562444e..ddad9e4 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -59,16 +59,16 @@ jobs: uses: actions/setup-node@v3 with: node-version: 20 - + - name: Checkout helia-http-gateway uses: actions/checkout@v3 with: path: helia-http-gateway - + - name: Install dependencies run: npm ci working-directory: helia-http-gateway - + - name: Build helia-http-gateway run: npm run build working-directory: helia-http-gateway @@ -77,7 +77,7 @@ jobs: - name: Start helia-http-gateway env: GATEWAY_CONFORMANCE_TEST: true - TRUSTLESS_GATEWAY: "http://127.0.0.1:8080" + TRUSTLESS_GATEWAYS: "http://127.0.0.1:8080" USE_LIBP2P: false PORT: 8090 run: | diff --git a/package.json b/package.json index b3bde97..bf2dd55 100644 --- a/package.json +++ b/package.json @@ -18,11 +18,10 @@ "test:e2e-flame": "concurrently -k -s all -n \"gateway,playwright\" -c \"magenta,blue\" \"npm run start:dev-flame\" \"wait-on 'tcp:$PORT' && npm run test:e2e\"", "test:e2e-doctor": "concurrently -k -s all -n \"gateway,playwright\" -c \"magenta,blue\" \"npm run start:dev-doctor\" \"wait-on 'tcp:$PORT' && npm run test:e2e\"", "test:gwc-kubo": "node ./scripts/kubo-init.js", - "test:gwc-helia": "npm run build && USE_LIBP2P=false PORT=8090 TRUSTLESS_GATEWAY='http://127.0.0.1:8080' npm start", + "test:gwc-helia": "npm run build && USE_LIBP2P=false PORT=8090 TRUSTLESS_GATEWAYS='http://127.0.0.1:8080' npm start", "test:gwc-setup": "concurrently -k -s all -n \"kubo,helia\" -c \"magenta,blue\" \"npm run test:gwc-kubo\" \"wait-on 'tcp:8080' && npm run test:gwc-helia\"", - "test:gwc-execute": "docker run --network host -v $PWD:/workspace -w /workspace ghcr.io/ipfs/gateway-conformance:v0.4.2 test --gateway-url http://localhost:8090 --json gwc-report.json --specs subdomain-ipfs-gateway,subdomain-ipns-gateway -- -timeout 30m", + "test:gwc-execute": "docker run --network host -v $PWD:/workspace -w /workspace ghcr.io/ipfs/gateway-conformance:v0.4.2 test --gateway-url 'http://localhost:8090' --subdomain-url 'http://localhost:8090' --json gwc-report.json --specs subdomain-ipns-gateway -- -timeout 30m", "test:gwc": "concurrently -k -s all -n \"kubo&helia,gateway-conformance\" -c \"magenta,blue\" \"npm run test:gwc-setup\" \"wait-on 'tcp:8090' && npm run test:gwc-execute\"", - "test:gateway-conformance": "TRUSTLESS_GATEWAY='http://localhost:8080' echo 'gateway conformance'", "healthcheck": "node dist/src/healthcheck.js", "debug:until-death": "./debugging/until-death.sh", "debug:test-gateways": "./debugging/test-gateways.sh" From 0778d65eb5c1eb978ee0e743391945e4e4a8a37f Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Mon, 4 Dec 2023 14:38:36 -0800 Subject: [PATCH 07/26] chore(ci:gwc): enable only subdomain ipfs and ipns --- .github/workflows/gateway-conformance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index ddad9e4..345a181 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -94,7 +94,7 @@ jobs: xml: output.xml html: output.html markdown: output.md - specs: -trustless-gateway,+path-gateway,+subdomain-gateway,+dnslink-gateway,-path-dag-gateway,-path-tar-gateway + specs: subdomain-ipfs-gateway,subdomain-ipns-gateway # use below to skip specific test if needed # args: -skip 'TestFooBr/GET_response_for_something' # From daefd4f16a39ba18e5271ec3a1e3781007e7753b Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Mon, 4 Dec 2023 14:46:15 -0800 Subject: [PATCH 08/26] fix: enable kubo routing-v1 for ipns --- scripts/kubo-init.js | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/kubo-init.js b/scripts/kubo-init.js index b2678f0..50a7211 100644 --- a/scripts/kubo-init.js +++ b/scripts/kubo-init.js @@ -86,6 +86,7 @@ async function configureKubo (tmpDir) { const execaOptions = getExecaOptions({ tmpDir }) try { await $(execaOptions)`npx -y kubo config Addresses.Gateway /ip4/127.0.0.1/tcp/8080` + await $(execaOptions)`npx -y kubo config --json Gateway.ExposeRoutingAPI true` log('Kubo configured') } catch (e) { error('Failed to configure Kubo', e) From 8722856223c2ced89d87f8055b8da2fdb315f53d Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Mon, 4 Dec 2023 14:47:06 -0800 Subject: [PATCH 09/26] chore: kubo-init daemon starting line --- scripts/kubo-init.js | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/kubo-init.js b/scripts/kubo-init.js index 50a7211..8912fa8 100644 --- a/scripts/kubo-init.js +++ b/scripts/kubo-init.js @@ -28,6 +28,7 @@ async function main () { const ipfsNsMap = await loadFixtures(tmpDir) // execute the daemon const execaOptions = getExecaOptions({ tmpDir, ipfsNsMap }) + log('Starting Kubo daemon...') await $(execaOptions)`npx kubo daemon --offline` } From 13ed71ac1707ba8c3097c13389aa478481251d82 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Mon, 4 Dec 2023 15:23:19 -0800 Subject: [PATCH 10/26] fix(ci:gwc): set subdomain-url arg --- .github/workflows/gateway-conformance.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index 345a181..a686821 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -90,6 +90,7 @@ jobs: uses: ipfs/gateway-conformance/.github/actions/test@v0.4.1 with: gateway-url: http://127.0.0.1:8090 + subdomain-url: http://127.0.0.1:8090 json: output.json xml: output.xml html: output.html From 4b004675e8a485fcf1e8e0a7f68bda6cbc686060 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Mon, 4 Dec 2023 15:41:58 -0800 Subject: [PATCH 11/26] fix(ci:gwc): skip dir list and gateway cache tests --- .github/workflows/gateway-conformance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index a686821..fcb4cb7 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -101,7 +101,7 @@ jobs: # # only-if-cached: helia-ht does not guarantee local cache, we will adjust upstream test (which was Kubo-specific) # for now disabling these test cases - args: -skip 'TestGatewayCache/.*_for_/ipfs/_with_only-if-cached_succeeds_when_in_local_datastore' + args: -skip '^.*(DirectoryListing|TestGatewayCache).*$' # 7. Upload the results - name: Upload MD summary From d39a3dbe7a517314e6d1ebf5496b50d1b83e6f8e Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Mon, 4 Dec 2023 15:42:26 -0800 Subject: [PATCH 12/26] chore(ci:gcw): don't use bitswap --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index bf2dd55..277735c 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "test:e2e-flame": "concurrently -k -s all -n \"gateway,playwright\" -c \"magenta,blue\" \"npm run start:dev-flame\" \"wait-on 'tcp:$PORT' && npm run test:e2e\"", "test:e2e-doctor": "concurrently -k -s all -n \"gateway,playwright\" -c \"magenta,blue\" \"npm run start:dev-doctor\" \"wait-on 'tcp:$PORT' && npm run test:e2e\"", "test:gwc-kubo": "node ./scripts/kubo-init.js", - "test:gwc-helia": "npm run build && USE_LIBP2P=false PORT=8090 TRUSTLESS_GATEWAYS='http://127.0.0.1:8080' npm start", + "test:gwc-helia": "npm run build && USE_LIBP2P=false USE_BITSWAP=false PORT=8090 TRUSTLESS_GATEWAYS='http://127.0.0.1:8080' npm start", "test:gwc-setup": "concurrently -k -s all -n \"kubo,helia\" -c \"magenta,blue\" \"npm run test:gwc-kubo\" \"wait-on 'tcp:8080' && npm run test:gwc-helia\"", "test:gwc-execute": "docker run --network host -v $PWD:/workspace -w /workspace ghcr.io/ipfs/gateway-conformance:v0.4.2 test --gateway-url 'http://localhost:8090' --subdomain-url 'http://localhost:8090' --json gwc-report.json --specs subdomain-ipns-gateway -- -timeout 30m", "test:gwc": "concurrently -k -s all -n \"kubo&helia,gateway-conformance\" -c \"magenta,blue\" \"npm run test:gwc-setup\" \"wait-on 'tcp:8090' && npm run test:gwc-execute\"", From ad5be2e9ebdd326898bc6f1ac3e4559700e37ea1 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Mon, 4 Dec 2023 15:57:36 -0800 Subject: [PATCH 13/26] docs(ci:gwc): add more details about gwc local testing --- DEVELOPER-NOTES.md | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/DEVELOPER-NOTES.md b/DEVELOPER-NOTES.md index 2fbc72e..233526d 100644 --- a/DEVELOPER-NOTES.md +++ b/DEVELOPER-NOTES.md @@ -11,14 +11,35 @@ We have some code enabled that makes running gateway-conformance testing against 1. [Install docker](https://docs.docker.com/get-docker/) 2. [Install nodejs](https://nodejs.org/en/download/) -### Run gateway-conformance tests locally +### Run gateway-conformance tests locally (once) ```sh $ npm run test:gwc ``` +### Continuously develop while running gateway-conformance tests + +```sh +# terminal 1 +$ npm run test:gwc-kubo + +# terminal 2 +$ npm run test:gwc-helia # you will need to stop and start this one in-between code changes. It's not watching for changes + +# terminal 3 +$ npm run test:gwc + +# OR from the gateway-conformance repo directly with something like: +go run ./cmd/gateway-conformance/main.go test --gateway-url 'http://localhost:8090' --subdomain-url 'http://localhost:8090' --specs subdomain-ipfs-gateway,subdomain-ipns-gateway --json gwc-report.json -- -timeout 30m + +``` + + + ### Some callouts +1. You may want to run the gateway-conformance tests directly from the repo if you're on a macOS M1 due to some issues with docker and the proxying that the gateway-conformance testing tool uses. If you do this, you'll need to run `make gateway-conformance` in the `gateway-conformance` repo root, and then run the tests with something like `go run ./cmd/gateway-conformance/main.go test --gateway-url 'http://localhost:8090' --subdomain-url 'http://localhost:8090' --specs subdomain-ipfs-gateway,subdomain-ipns-gateway --json gwc-report.json -- -timeout 30m`. + - If you want to run a specific test, you can pass the `-run` gotest flag. e.g. `go run ./cmd/gateway-conformance/main.go test --gateway-url 'http://localhost:8090' --subdomain-url 'http://localhost:8090' --json gwc-report.json -- -timeout 30m -run 'TestGatewaySubdomains/request_for_example.com%2Fipfs%2F%7BCIDv1%7D_redirects_to_subdomain_%28HTTP_proxy_tunneling_via_CONNECT%29#01'` 1. The file `./scripts/kubo-init.js` executes kubo using `execa` instead of `ipfsd-ctl` so there may be some gotchas, but it should be as cross-platform and stable as the `execa` library. 1. The IPFS_PATH used is a temporary directory. Your OS should handle removing it when vital, but you can also remove it manually. The path to this directory is printed out when the tests start, and saved in a file at `./scripts/tmp/kubo-path.txt`. 1. The tests save gateway-conformance fixtures to `./scripts/tmp/fixtures`. You can remove this directory manually if you want to re-run the tests with a fresh set of fixtures. From b137aac780d35cc0ac55858197de22cc097d14af Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Mon, 4 Dec 2023 16:40:46 -0800 Subject: [PATCH 14/26] docs(ci:gwc): more documentation of things --- DEVELOPER-NOTES.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/DEVELOPER-NOTES.md b/DEVELOPER-NOTES.md index 233526d..e6c8dfd 100644 --- a/DEVELOPER-NOTES.md +++ b/DEVELOPER-NOTES.md @@ -24,7 +24,8 @@ $ npm run test:gwc $ npm run test:gwc-kubo # terminal 2 -$ npm run test:gwc-helia # you will need to stop and start this one in-between code changes. It's not watching for changes +# you will need to stop and start this one in-between code changes. It's not watching for changes +$ DEBUG="helia-http-gateway*" FILE_DATASTORE_PATH=./tmp/datastore npm run test:gwc-helia # terminal 3 $ npm run test:gwc @@ -44,3 +45,4 @@ go run ./cmd/gateway-conformance/main.go test --gateway-url 'http://localhost:80 1. The IPFS_PATH used is a temporary directory. Your OS should handle removing it when vital, but you can also remove it manually. The path to this directory is printed out when the tests start, and saved in a file at `./scripts/tmp/kubo-path.txt`. 1. The tests save gateway-conformance fixtures to `./scripts/tmp/fixtures`. You can remove this directory manually if you want to re-run the tests with a fresh set of fixtures. 1. The results of the gateway-conformance tests are saved to `./gwc-report.json`. This file is overwritten every time the tests are run. +1. The gateway-conformance tests are flaky and commands & documentation are not up to date. Running commands with CLI flags is supposed to work, and env vars aren't documented, but ENV_VARs are the only way to get the tests to run, but not when ran with docker. See [this issue](https://github.com/ipfs/gateway-conformance/issues/185#issuecomment-1839801366) From fc3e4f315a6ad37daecdfaeeaf3301aab3e0b16d Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Tue, 5 Dec 2023 10:18:05 -0800 Subject: [PATCH 15/26] chore: add ECHO_HEADERS & notes --- DEVELOPER-NOTES.md | 33 +++++++++++++++++++++++++++++++++ README.md | 1 + package.json | 2 +- src/constants.ts | 2 ++ src/heliaServer.ts | 8 +++++++- src/index.ts | 13 ++++++++++++- 6 files changed, 56 insertions(+), 3 deletions(-) diff --git a/DEVELOPER-NOTES.md b/DEVELOPER-NOTES.md index e6c8dfd..0792442 100644 --- a/DEVELOPER-NOTES.md +++ b/DEVELOPER-NOTES.md @@ -22,6 +22,7 @@ $ npm run test:gwc ```sh # terminal 1 $ npm run test:gwc-kubo +# You can also set up the kubo backing node with the instructions at https://github.com/ipfs/gateway-conformance/blob/main/docs/development.md#developing-against-kubo # terminal 2 # you will need to stop and start this one in-between code changes. It's not watching for changes @@ -46,3 +47,35 @@ go run ./cmd/gateway-conformance/main.go test --gateway-url 'http://localhost:80 1. The tests save gateway-conformance fixtures to `./scripts/tmp/fixtures`. You can remove this directory manually if you want to re-run the tests with a fresh set of fixtures. 1. The results of the gateway-conformance tests are saved to `./gwc-report.json`. This file is overwritten every time the tests are run. 1. The gateway-conformance tests are flaky and commands & documentation are not up to date. Running commands with CLI flags is supposed to work, and env vars aren't documented, but ENV_VARs are the only way to get the tests to run, but not when ran with docker. See [this issue](https://github.com/ipfs/gateway-conformance/issues/185#issuecomment-1839801366) + + +## Tiros info + +### Deploying to Tiros + +Go to https://github.com/plprobelab/probelab-infra/blob/main/aws/tf/tiros.tf + +update helia stuff + +run terraform apply (with AWS Config) + +### Kick off helia-http-gateway task manually: + +1. https://us-west-1.console.aws.amazon.com/ecs/v2/clusters/prod-usw1-ecs-cluster/run-task?region=us-west-1 +1. select launch type FARGATE +1. select deployment configuration application type to be task +1. select task-helia family +1. select vpc (non-default) +1. select security group for tiros +1. Click create + +### Todo + +- [ ] Fix helia_health_cmd (https://github.com/plprobelab/probelab-infra/blob/7ec47f16e84113545cdb06b07f865a3bc5787a0b/aws/tf/tiros.tf#L5C4-L5C4) to use the new helia-health command +- [ ] fix node max memory for helia-http-gateway docker container to support at least 4GB + +### Links + +graphs for runs are at https://probelab.grafana.net/d/GpwxraxVk/tiros-ops?orgId=1&from=now-7d&to=now&inspect=8&inspectTab=error +AWS logs for runs are at https://us-west-1.console.aws.amazon.com/cloudwatch/home?region=us-west-1#logsV2:log-groups/log-group/prod-usw1-cmi-tiros-ipfs (you need to be logged in to AWS) + diff --git a/README.md b/README.md index f058735..7733e9b 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,7 @@ $ docker run -it -p 8080:8080 -e DEBUG="helia-http-gateway*" helia-http-gateway: | `TRUSTLESS_GATEWAYS` | Comma separated list of trusted gateways to fetch content from | [Defined in Helia](https://github.com/ipfs/helia/blob/main/packages/helia/src/block-brokers/trustless-gateway/index.ts) | | `USE_LIBP2P` | Whether to use libp2p networking | `true` | | `RESOLVE_REDIRECTS` | Whether to resolve redirects before looking up dnslink entries | `true` | +| `ECHO_HEADERS` | A debug flag to indicate whether you want to output request and response headers | `false` |