From 23f467fc9a35a8f1d76291f1868639f5b365ec66 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Wed, 6 Dec 2023 08:57:14 +0100 Subject: [PATCH] feat: improvements & gateway-conformance support (light) (#57) Co-authored-by: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> --- .env-all | 18 + .env-delegated-routing | 16 + .env-gwc | 16 + .env-trustless-only | 14 + .github/workflows/gateway-conformance.yml | 121 ++ .github/workflows/playwright.yml | 19 + .gitignore | 3 + DEVELOPER-NOTES.md | 84 ++ README.md | 10 +- e2e-tests/smoketest.spec.ts | 4 +- package-lock.json | 1164 ++++++++++++++--- package.json | 18 +- patches/@fastify+cors+8.4.2.patch | 15 + ...ted-routing-v1-http-api-client+1.1.2.patch | 48 + playwright.config.ts | 7 +- scripts/kubo-init.js | 136 ++ scripts/tmp/.gitkeep | 0 src/constants.ts | 12 +- src/getCustomHelia.ts | 9 +- src/getCustomLibp2p.ts | 127 ++ src/heliaFetch.ts | 147 +-- src/heliaServer.ts | 95 +- src/index.ts | 45 +- src/ipns-address-utils.ts | 44 + 24 files changed, 1886 insertions(+), 286 deletions(-) create mode 100644 .env-all create mode 100644 .env-delegated-routing create mode 100644 .env-gwc create mode 100644 .env-trustless-only create mode 100644 .github/workflows/gateway-conformance.yml create mode 100644 DEVELOPER-NOTES.md create mode 100644 patches/@fastify+cors+8.4.2.patch create mode 100644 patches/@helia+delegated-routing-v1-http-api-client+1.1.2.patch create mode 100644 scripts/kubo-init.js create mode 100644 scripts/tmp/.gitkeep create mode 100644 src/getCustomLibp2p.ts create mode 100644 src/ipns-address-utils.ts diff --git a/.env-all b/.env-all new file mode 100644 index 0000000..4a3fe4f --- /dev/null +++ b/.env-all @@ -0,0 +1,18 @@ +# ENV vars recommended for running with everything enabled: +# * trustless gateways +# * delegated routing +# * libp2p +# * bitswap +USE_LIBP2P=true +USE_BITSWAP=true +DEBUG='helia-http-gateway*,*helia-fetch*' +USE_TRUSTLESS_GATEWAYS=true +USE_DELEGATED_ROUTING=true + +# Uncomment the two below to save blockstore and datastore to disk +# FILE_DATASTORE_PATH=./data/datastore +# FILE_BLOCKSTORE_PATH=./data/blockstore + +# If you want to run the gateway with a local trustless gateway, uncomment the below +# PORT=8090 # or whatever port you want to run helia-http-gateway on +# TRUSTLESS_GATEWAYS=http://127.0.0.1:8080 diff --git a/.env-delegated-routing b/.env-delegated-routing new file mode 100644 index 0000000..160b7dd --- /dev/null +++ b/.env-delegated-routing @@ -0,0 +1,16 @@ +# ENV vars recommended for running with only delegated-routing-v1-http +USE_LIBP2P=false +# needed to request blocks from peers we discover with delegated routing +USE_BITSWAP=true +# TRUSTLESS_GATEWAYS=http://127.0.0.1:8080 +DEBUG='helia-http-gateway*,*helia-fetch*,*delegated-routing-v1-http-api-client*' +USE_TRUSTLESS_GATEWAYS=false +USE_DELEGATED_ROUTING=true + +# IF you're delegating to kubo running locally you should uncomment the two below: +# PORT=8090 +# DELEGATED_ROUTING_V1_HOST=http://127.0.0.1:8080 + +# Uncomment the two below to save blockstore and datastore to disk +# FILE_DATASTORE_PATH=./data/datastore +# FILE_BLOCKSTORE_PATH=./data/blockstore diff --git a/.env-gwc b/.env-gwc new file mode 100644 index 0000000..f9a2c6b --- /dev/null +++ b/.env-gwc @@ -0,0 +1,16 @@ +# ENV vars recommended for running gateway-conformance tests +USE_LIBP2P=false +USE_BITSWAP=true +PORT=8090 # helia-http-gateway should be running here +TRUSTLESS_GATEWAYS=http://127.0.0.1:8080 # Kubo should be running here +DELEGATED_ROUTING_V1_HOST=http://127.0.0.1:8080 # Kubo should be running here +DEBUG='helia-http-gateway*,*helia-fetch*,*helia:trustless-gateway-block-broker*' +USE_TRUSTLESS_GATEWAYS=true +USE_DELEGATED_ROUTING=true + +# Uncomment the two below to save blockstore and datastore to disk +# FILE_DATASTORE_PATH=./data/datastore +# FILE_BLOCKSTORE_PATH=./data/blockstore + +# Uncomment the below to see request & response headers in the logs +# ECHO_HEADERS=true diff --git a/.env-trustless-only b/.env-trustless-only new file mode 100644 index 0000000..f8cac4f --- /dev/null +++ b/.env-trustless-only @@ -0,0 +1,14 @@ +# ENV vars recommended for running with trustless gateways only +USE_LIBP2P=false +USE_BITSWAP=false +DEBUG='helia-http-gateway*,*helia-fetch*,*helia:trustless-gateway-block-broker*' +USE_TRUSTLESS_GATEWAYS=true +USE_DELEGATED_ROUTING=false + +# Uncomment the two below to save blockstore and datastore to disk +# FILE_DATASTORE_PATH=./data/datastore +# FILE_BLOCKSTORE_PATH=./data/blockstore + +# If you want to run the gateway with a local trustless gateway, uncomment the below +# PORT=8090 # or whatever port you want to run helia-http-gateway on +# TRUSTLESS_GATEWAYS=http://127.0.0.1:8080 diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml new file mode 100644 index 0000000..615dfe9 --- /dev/null +++ b/.github/workflows/gateway-conformance.yml @@ -0,0 +1,121 @@ +name: Gateway Conformance + +on: + push: + branches: + - main + pull_request: + +jobs: + gateway-conformance: + runs-on: ubuntu-latest + steps: + # 1. Start the Kubo gateway + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: 1.21.x + + - name: Install Kubo gateway from source + #uses: ipfs/download-ipfs-distribution-action@v1 + run: | + go install github.com/ipfs/kubo/cmd/ipfs@v0.24.0 + - name: Setup kubo config + run: | + ipfs init --profile=test + ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/8080" + + # 2. Download the gateway-conformance fixtures + - name: Download gateway-conformance fixtures + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.4.1 + with: + output: fixtures + + - name: Start Kubo gateway + uses: ipfs/start-ipfs-daemon-action@v1 + + # 3. Populate the Kubo gateway with the gateway-conformance fixtures + - name: Import fixtures + run: | + # Import car files + find ./fixtures -name '*.car' -exec ipfs dag import --pin-roots=false --offline {} \; + + # Import ipns records + records=$(find ./fixtures -name '*.ipns-record') + for record in $records + do + key=$(basename -s .ipns-record "$record" | cut -d'_' -f1) + ipfs routing put --allow-offline "/ipns/$key" "$record" + done + + # Import dnslink records + # the IPFS_NS_MAP env will be used by the daemon + export IPFS_NS_MAP=$(cat "./fixtures/dnslinks.json" | jq -r '.subdomains | to_entries | map("\(.key).example.com:\(.value)") | join(",")') + export IPFS_NS_MAP="$(cat "./fixtures/dnslinks.json" | jq -r '.domains | to_entries | map("\(.key):\(.value)") | join(",")'),${IPFS_NS_MAP}" + echo "IPFS_NS_MAP=${IPFS_NS_MAP}" >> $GITHUB_ENV + + # 4. Build helia-http-gateway + - name: Setup Node + uses: actions/setup-node@v3 + with: + node-version: 20 + + - name: Checkout helia-http-gateway + uses: actions/checkout@v3 + with: + path: helia-http-gateway + + - name: Install dependencies + run: npm ci + working-directory: helia-http-gateway + + - name: Build helia-http-gateway + run: npm run build + working-directory: helia-http-gateway + + # 5. Start helia-http-gateway + - name: Start helia-http-gateway + env: + GATEWAY_CONFORMANCE_TEST: true + TRUSTLESS_GATEWAYS: "http://127.0.0.1:8080" + USE_LIBP2P: false + PORT: 8090 + run: | + # run gw + node dist/src/index.js & + working-directory: helia-http-gateway + + # 6. Run the gateway-conformance tests + - name: Run gateway-conformance tests + uses: ipfs/gateway-conformance/.github/actions/test@v0.4.1 + with: + gateway-url: http://127.0.0.1:8090 + subdomain-url: http://127.0.0.1:8090 + json: output.json + xml: output.xml + html: output.html + markdown: output.md + # specs: subdomain-ipfs-gateway,subdomain-ipns-gateway + # use below to skip specific test if needed + # args: -skip 'TestFooBr/GET_response_for_something' + # + # only-if-cached: helia-ht does not guarantee local cache, we will adjust upstream test (which was Kubo-specific) + # for now disabling these test cases + args: -skip '^.*(DirectoryListing|TestGatewayCache|TestSubdomainGatewayDNSLinkInlining|proxy|TestGatewaySubdomainAndIPNS|TestGatewaySubdomains|Trustless|TestGatewayIPNSRecord|RedirectsFile|TestGatewayUnixFSFileRanges|TestGatewayJSONCborAndIPNS|TestTar|Symlink|TestPathGatewayMiscellaneous|TestGatewayBlock|TestRedirectCanonicalIPNS|TestGatewayIPNSPath|TestNativeDag|TestPathing|TestPlainCodec|TestDagPbConversion|TestGatewayJsonCbor|TestCors).*$' + + # 7. Upload the results + - name: Upload MD summary + if: failure() || success() + run: cat output.md >> $GITHUB_STEP_SUMMARY + - name: Upload HTML report + if: failure() || success() + uses: actions/upload-artifact@v3 + with: + name: gateway-conformance.html + path: output.html + - name: Upload JSON report + if: failure() || success() + uses: actions/upload-artifact@v3 + with: + name: gateway-conformance.json + path: output.json diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index a3328cd..4c6bf16 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -19,6 +19,25 @@ jobs: run: npm ci - name: Install Playwright Browsers run: npx playwright install --with-deps + + # Cache playwright binaries + - uses: actions/cache@v3 + id: playwright-cache + with: + path: | + ~/.cache/ms-playwright + key: ${{ runner.os }}-playwright-${{ hashFiles('**/package-lock.json') }} + - run: npx playwright install --with-deps + if: steps.playwright-cache.outputs.cache-hit != 'true' + + # Cache datastores + - uses: actions/cache@v3 + id: e2e-datastore-and-blockstore + with: + path: | + ./test/fixtures/e2e + key: ${{ runner.os }}-e2e-stores-${{ hashFiles('**/package-lock.json') }} + - name: Run Playwright tests run: npm run test:e2e env: diff --git a/.gitignore b/.gitignore index 149c6fe..a4979be 100644 --- a/.gitignore +++ b/.gitignore @@ -134,3 +134,6 @@ build /playwright/.cache/ screenshots .envrc +scripts/tmp/fixtures +scripts/tmp/kubo-path.txt +test/fixtures/e2e diff --git a/DEVELOPER-NOTES.md b/DEVELOPER-NOTES.md new file mode 100644 index 0000000..cd644b3 --- /dev/null +++ b/DEVELOPER-NOTES.md @@ -0,0 +1,84 @@ +# Developer notes + + + +## Gateway Conformance testing + +We have some code enabled that makes running gateway-conformance testing against helia-http-gateway easy. Follow the instructions in this section to run gateway-conformance tests locally + +### Prerequisites + +1. [Install docker](https://docs.docker.com/get-docker/) +2. [Install nodejs](https://nodejs.org/en/download/) + +### Run gateway-conformance tests locally (once) + +```sh +$ npm run test:gwc +``` + +### Continuously develop while running gateway-conformance tests + +```sh +# terminal 1 +$ npm run test:gwc-kubo +# You can also set up the kubo backing node with the instructions at https://github.com/ipfs/gateway-conformance/blob/main/docs/development.md#developing-against-kubo + +# terminal 2 +# you will need to stop and start this one in-between code changes. It's not watching for changes +$ DEBUG="helia-http-gateway*" FILE_DATASTORE_PATH=./tmp/datastore npm run test:gwc-helia + +# terminal 3 +$ npm run test:gwc-execute +# To skip some tests, run something like: +# npm run test:gwc-execute -- -skip '^.*(DirectoryListing|TestGatewayCache|TestSubdomainGatewayDNSLinkInlining|TestGatewaySubdomainAndIPNS).*$' + +# OR from the gateway-conformance repo directly with something like: +go run ./cmd/gateway-conformance/main.go test --gateway-url 'http://localhost:8090' --subdomain-url 'http://localhost:8090' --specs subdomain-ipfs-gateway,subdomain-ipns-gateway --json gwc-report.json -- -timeout 30m + +``` + + + +### Some callouts + +1. You may want to set up an nginx proxy from `http://helia-http-gateway.localhost` to `http://localhost:8090`to help with the gateway-conformance tests. See [this issue](https://github.com/ipfs/gateway-conformance/issues/185) +1. You may want to run the gateway-conformance tests directly from the repo if you're on a macOS M1 due to some issues with docker and the proxying that the gateway-conformance testing tool uses. If you do this, you'll need to run `make gateway-conformance` in the `gateway-conformance` repo root, and then run the tests with something like `go run ./cmd/gateway-conformance/main.go test --gateway-url 'http://localhost:8090' --subdomain-url 'http://localhost:8090' --specs subdomain-ipfs-gateway,subdomain-ipns-gateway --json gwc-report.json -- -timeout 30m`. + - If you want to run a specific test, you can pass the `-run` gotest flag. e.g. `go run ./cmd/gateway-conformance/main.go test --gateway-url 'http://localhost:8090' --subdomain-url 'http://localhost:8090' --json gwc-report.json -- -timeout 30m -run 'TestGatewaySubdomains/request_for_example.com%2Fipfs%2F%7BCIDv1%7D_redirects_to_subdomain_%28HTTP_proxy_tunneling_via_CONNECT%29#01'` +1. The file `./scripts/kubo-init.js` executes kubo using `execa` instead of `ipfsd-ctl` so there may be some gotchas, but it should be as cross-platform and stable as the `execa` library. +1. The IPFS_PATH used is a temporary directory. Your OS should handle removing it when vital, but you can also remove it manually. The path to this directory is printed out when the tests start, and saved in a file at `./scripts/tmp/kubo-path.txt`. +1. The tests save gateway-conformance fixtures to `./scripts/tmp/fixtures`. You can remove this directory manually if you want to re-run the tests with a fresh set of fixtures. +1. The results of the gateway-conformance tests are saved to `./gwc-report.json`. This file is overwritten every time the tests are run. +1. The gateway-conformance tests are flaky and commands & documentation are not up to date. Running commands with CLI flags is supposed to work, and env vars aren't documented, but ENV_VARs are the only way to get the tests to run, but not when ran with docker. See [this issue](https://github.com/ipfs/gateway-conformance/issues/185#issuecomment-1839801366) + + +## Tiros info + +### Deploying to Tiros + +Go to https://github.com/plprobelab/probelab-infra/blob/main/aws/tf/tiros.tf + +update helia stuff + +run terraform apply (with AWS Config) + +### Kick off helia-http-gateway task manually: + +1. https://us-west-1.console.aws.amazon.com/ecs/v2/clusters/prod-usw1-ecs-cluster/run-task?region=us-west-1 +1. select launch type FARGATE +1. select deployment configuration application type to be task +1. select task-helia family +1. select vpc (non-default) +1. select security group for tiros +1. Click create + +### Todo + +- [ ] Fix helia_health_cmd (https://github.com/plprobelab/probelab-infra/blob/7ec47f16e84113545cdb06b07f865a3bc5787a0b/aws/tf/tiros.tf#L5C4-L5C4) to use the new helia-health command +- [ ] fix node max memory for helia-http-gateway docker container to support at least 4GB + +### Links + +graphs for runs are at https://probelab.grafana.net/d/GpwxraxVk/tiros-ops?orgId=1&from=now-7d&to=now&inspect=8&inspectTab=error +AWS logs for runs are at https://us-west-1.console.aws.amazon.com/cloudwatch/home?region=us-west-1#logsV2:log-groups/log-group/prod-usw1-cmi-tiros-ipfs (you need to be logged in to AWS) + diff --git a/README.md b/README.md index f058735..8ddfab0 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,8 @@ $ docker run -it -p 8080:8080 -e DEBUG="helia-http-gateway*" helia-http-gateway: | `USE_TRUSTLESS_GATEWAYS` | Whether to fetch content from trustless-gateways or not | `true` | | `TRUSTLESS_GATEWAYS` | Comma separated list of trusted gateways to fetch content from | [Defined in Helia](https://github.com/ipfs/helia/blob/main/packages/helia/src/block-brokers/trustless-gateway/index.ts) | | `USE_LIBP2P` | Whether to use libp2p networking | `true` | -| `RESOLVE_REDIRECTS` | Whether to resolve redirects before looking up dnslink entries | `true` | +| `ECHO_HEADERS` | A debug flag to indicate whether you want to output request and response headers | `false` | +| `DELEGATED_ROUTING_V1_HOST` | Hostname to use for delegated routing v1 | `https://delegated-ipfs.dev` |