diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml index fe9972128..2d5d088ab 100644 --- a/.github/actions/test/action.yml +++ b/.github/actions/test/action.yml @@ -4,6 +4,10 @@ inputs: gateway-url: description: "The URL of the IPFS Gateway implementation to be tested." required: true + subdomain-url: + description: "The Subdomain URL of the IPFS Gateway implementation to be tested." + default: "http://example.com" + required: false json: description: "The path where the JSON test report should be generated." required: true @@ -33,6 +37,7 @@ runs: uses: pl-strflt/docker-container-action@v1 env: URL: ${{ inputs.gateway-url }} + SUBDOMAIN: ${{ inputs.subdomain-url }} JSON: ${{ inputs.json }} SPECS: ${{ inputs.specs }} with: @@ -40,7 +45,7 @@ runs: ref: ${{ steps.github.outputs.action_ref }} dockerfile: Dockerfile opts: --network=host - args: test --url="$URL" --json="$JSON" --specs="$SPECS" -- ${{ inputs.args }} + args: test --url="$URL" --json="$JSON" --specs="$SPECS" --subdomain-url="$SUBDOMAIN" -- ${{ inputs.args }} - name: Create the XML if: (inputs.xml || inputs.html || inputs.markdown) && (failure() || success()) uses: pl-strflt/gotest-json-to-junit-xml@v1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e26e87e8f..2c0ca5d5a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,8 +11,6 @@ jobs: run: shell: bash steps: - - uses: ipfs/download-ipfs-distribution-action@v1 - - uses: ipfs/start-ipfs-daemon-action@v1 - name: Setup Go uses: actions/setup-go@v3 with: @@ -24,17 +22,24 @@ jobs: uses: ./gateway-conformance/.github/actions/extract-fixtures with: output: fixtures + - uses: ipfs/download-ipfs-distribution-action@v1 + - name: Configure Kubo Gateway + run: | + ipfs init; + ./gateway-conformance/kubo-config.example.sh; + - uses: ipfs/start-ipfs-daemon-action@v1 - name: Provision Kubo Gateway - run: find ./fixtures -name '*.car' -exec ipfs dag import {} \; + run: | + find ./fixtures -name '*.car' -exec ipfs dag import {} \; - name: Run the tests uses: ./gateway-conformance/.github/actions/test with: gateway-url: http://127.0.0.1:8080 + subdomain-url: http://example.com json: output.json xml: output.xml html: output.html markdown: output.md - specs: -subdomain-gateway args: -skip TestGatewayCar - name: Set summary if: (failure() || success()) diff --git a/Makefile b/Makefile index 5de08e5e9..fdec29df2 100644 --- a/Makefile +++ b/Makefile @@ -1,16 +1,22 @@ +all: test-kubo + +test-cargateway: provision-cargateway fixtures.car gateway-conformance + ./gateway-conformance test --json output.json --gateway-url http://127.0.0.1:8040 --specs -subdomain-gateway + +test-kubo-subdomains: provision-kubo gateway-conformance + ./kubo-config.example.sh + ./gateway-conformance test --json output.json --gateway-url http://127.0.0.1:8080 --subdomain-url http://example.com:8080 + +test-kubo: provision-kubo fixtures.car gateway-conformance + ./gateway-conformance test --json output.json --gateway-url http://127.0.0.1:8080 --specs -subdomain-gateway + provision-cargateway: ./fixtures.car # cd go-libipfs/examples/car && go install car -c ./fixtures.car & -test-cargateway: provision-cargateway - GATEWAY_URL=http://127.0.0.1:8040 make _test - provision-kubo: find ./fixtures -name '*.car' -exec ipfs dag import {} \; -test-kubo: provision-kubo - GATEWAY_URL=http://127.0.0.1:8080 make _test - # tools fixtures.car: gateway-conformance ./gateway-conformance extract-fixtures --merged=true --dir=. @@ -18,9 +24,6 @@ fixtures.car: gateway-conformance gateway-conformance: go build -o ./gateway-conformance ./cmd/gateway-conformance -_test: fixtures.car gateway-conformance - ./gateway-conformance test --json output.json --gateway-url ${GATEWAY_URL} - test-docker: fixtures.car gateway-conformance docker build -t gateway-conformance . docker run --rm -v "${PWD}:/workspace" -w "/workspace" --network=host gateway-conformance test diff --git a/README.md b/README.md index 3cad992da..85543d711 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ The `test` command is the main command of the tool. It is used to test a given I | Input | Availability | Description | Default | |---|---|---|---| | gateway-url | Both | The URL of the IPFS Gateway implementation to be tested. | http://localhost:8080 | +| subdomain-url | Both | The Subdomain URL of the IPFS Gateway implementation to be tested. | http://example.com | | json | Both | The path where the JSON test report should be generated. | `./report.json` | | xml | GitHub Action | The path where the JUnit XML test report should be generated. | `./report.xml` | | html | GitHub Action | The path where the one-page HTML test report should be generated. | `./report.html` | @@ -45,6 +46,19 @@ If you provide a list containing both prefixed and unprefixed specs, the prefixe This input should be used sparingly and with caution, as it involves interacting with the underlying internal processes, which may be subject to changes. It is recommended to use the `args` input only when you have a deep understanding of the tool's inner workings and need to fine-tune the testing process. Users should be mindful of the potential risks associated with using this input. +#### Subdomain Testing and `subdomain-url` + +The `subdomain-url` parameter is utilized when testing subdomain support in your IPFS gateway. It can be set to any domain that your gateway permits. +During testing, the suite keeps connecting to the `gateway-url` while employing HTTP techniques to simulate requests as if they were sent to the subdomain. +This approach enables testing of local gateways during development or continuous integration (CI) scenarios. + +A few examples: + +| Use Case | gateway-url | subdomain-url | +|----------|-------------|---------------| +| CI & Dev | http://127.0.0.1:8080 | http://example.com | +| Production | https://dweb.link | https://dweb.link | + #### Usage ##### GitHub Action @@ -148,4 +162,6 @@ Please let us know if you would like to see this feature implemented directly in ## In Development - How to deal with subdomains & configuration (t0114 for example)? - - Some test relies on querying URLs like `http://$CIDv1.ipfs.example.com/`. While `http://$CIDv1.ipfs.localhost/` works by default, do we need / want to test with `.example.com`? + - Some test relies on querying URLs like `http://$CIDv1.ipfs.example.com/`. While `http://$CIDv1.ipfs.localhost/` works by default, do we need / want to test with `.example.com`? +- Debug logging + - Set the environment variable `GOLOG_LOG_LEVEL="conformance=debug"` to toggle debug logging. \ No newline at end of file diff --git a/cmd/gateway-conformance/main.go b/cmd/gateway-conformance/main.go index c7fe22194..93cc3f1b3 100644 --- a/cmd/gateway-conformance/main.go +++ b/cmd/gateway-conformance/main.go @@ -57,6 +57,7 @@ func copyFiles(inputPaths []string, outputDirectoryPath string) error { func main() { var gatewayURL string + var subdomainGatewayURL string var jsonOutput string var specs string var directory string @@ -78,6 +79,12 @@ func main() { Value: "http://localhost:8080", Destination: &gatewayURL, }, + &cli.StringFlag{ + Name: "subdomain-url", + Usage: "The Subdomain URL of the IPFS Gateway implementation to be tested.", + Value: "http://example.com", + Destination: &subdomainGatewayURL, + }, &cli.StringFlag{ Name: "json-output", Aliases: []string{"json", "j"}, @@ -107,6 +114,11 @@ func main() { cmd := exec.Command("go", args...) cmd.Dir = tooling.Home() cmd.Env = append(os.Environ(), fmt.Sprintf("GATEWAY_URL=%s", gatewayURL)) + + if subdomainGatewayURL != "" { + cmd.Env = append(cmd.Env, fmt.Sprintf("SUBDOMAIN_GATEWAY_URL=%s", subdomainGatewayURL)) + } + cmd.Stdout = out{output} cmd.Stderr = os.Stderr testErr := cmd.Run() diff --git a/fixtures/t0114-gateway_subdomains.car b/fixtures/t0114-gateway_subdomains.car index bc7b913df..056ef7e1c 100644 Binary files a/fixtures/t0114-gateway_subdomains.car and b/fixtures/t0114-gateway_subdomains.car differ diff --git a/kubo-config.example.sh b/kubo-config.example.sh new file mode 100755 index 000000000..d1873534a --- /dev/null +++ b/kubo-config.example.sh @@ -0,0 +1,12 @@ +#! /usr/bin/env bash +ipfs config --json Gateway.PublicGateways '{ + "example.com": { + "UseSubdomains": true, + "Paths": ["/ipfs", "/ipns", "/api"] + }, + "localhost": { + "UseSubdomains": true, + "InlineDNSLink": true, + "Paths": ["/ipfs", "/ipns", "/api"] + } +}' \ No newline at end of file diff --git a/tests/t0113_gateway_symlink_test.go b/tests/t0113_gateway_symlink_test.go index 54ffb51f2..a7bf2921a 100644 --- a/tests/t0113_gateway_symlink_test.go +++ b/tests/t0113_gateway_symlink_test.go @@ -14,7 +14,7 @@ func TestGatewaySymlink(t *testing.T) { { Name: "Test the directory listing", Request: test.CRequest{ - Url: fmt.Sprintf("ipfs/%s?format=raw", fixture.MustGetCid()), + Path: fmt.Sprintf("ipfs/%s?format=raw", fixture.MustGetCid()), }, Response: test.CResponse{ StatusCode: 200, @@ -24,7 +24,7 @@ func TestGatewaySymlink(t *testing.T) { { Name: "Test the symlink", Request: test.CRequest{ - Url: fmt.Sprintf("ipfs/%s/bar", fixture.MustGetCid()), + Path: fmt.Sprintf("ipfs/%s/bar", fixture.MustGetCid()), }, Response: test.CResponse{ StatusCode: 200, diff --git a/tests/t0114_gateway_subdomains_test.go b/tests/t0114_gateway_subdomains_test.go index 86ac705b0..d2b89a73f 100644 --- a/tests/t0114_gateway_subdomains_test.go +++ b/tests/t0114_gateway_subdomains_test.go @@ -2,61 +2,471 @@ package tests import ( "fmt" + "net/url" "testing" + "github.com/ipfs/gateway-conformance/tooling/car" . "github.com/ipfs/gateway-conformance/tooling/check" . "github.com/ipfs/gateway-conformance/tooling/specs" . "github.com/ipfs/gateway-conformance/tooling/test" ) func TestGatewaySubdomains(t *testing.T) { - // fixture := car.MustOpenUnixfsCar("t0114-gateway_subdomains") - - CID_VAL := "hello" - CIDv1 := "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am" - // CIDv0 := "QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN" - // // CIDv0to1 is necessary because raw-leaves are enabled by default during - // // "ipfs add" with CIDv1 and disabled with CIDv0 - // CIDv0to1 := "bafybeiffndsajwhk3lwjewwdxqntmjm4b5wxaaanokonsggenkbw6slwk4" - // CIDv1_TOO_LONG := "bafkrgqhhyivzstcz3hhswshfjgy6ertgmnqeleynhwt4dlfsthi4hn7zgh4uvlsb5xncykzapi3ocd4lzogukir6ksdy6wzrnz6ohnv4aglcs" - DIR_CID := "bafybeiht6dtwk3les7vqm6ibpvz6qpohidvlshsfyr7l5mpysdw2vmbbhe" // ./testdirlisting - - tests := []CTest{ + fixture := car.MustOpenUnixfsCar("t0114-gateway_subdomains.car") + + CIDVal := string(fixture.MustGetRawData("hello-CIDv1")) // hello + DirCID := fixture.MustGetCid("testdirlisting") + CIDv1 := fixture.MustGetCid("hello-CIDv1") + CIDv0 := fixture.MustGetCid("hello-CIDv0") + CIDv0to1 := fixture.MustGetCid("hello-CIDv0to1") + CIDv1_TOO_LONG := fixture.MustGetCid("hello-CIDv1_TOO_LONG") + CIDWikipedia := "QmXoypizjW3WknFiJnKLwHCnL72vedxjQkDDP1mXWo6uco" + + tests := []CTest{} + + // sugar: readable way to add more tests + with := func(moreTests []CTest) { + tests = append(tests, moreTests...) + } + + // sugar: nicer looking sprintf call + URL := func(path string, args ...interface{}) string { + return fmt.Sprintf(path, args...) + } + + // We're going to run the same test against multiple gateways (localhost, and a subdomain gateway) + gatewayURLs := []string{ + SubdomainGatewayURL, + SubdomainLocalhostGatewayURL, + } + + for _, gatewayURL := range gatewayURLs { + u, err := url.Parse(gatewayURL) + if err != nil { + t.Fatal(err) + } + + with(testGatewayWithManyProtocols(t, + "request for example.com/ipfs/{CIDv1} redirects to subdomain", + ` + subdomains should not return payload directly, + but redirect to URL with proper origin isolation + `, + URL("%s/ipfs/%s/", gatewayURL, CIDv1), + Expect(). + Status(301). + Headers( + Header("Location"). + Hint("request for example.com/ipfs/{CIDv1} returns Location HTTP header for subdomain redirect in browsers"). + Contains("%s://%s.ipfs.%s/", u.Scheme, CIDv1, u.Host), + ). + BodyWithHint(` + We return body with HTTP 301 so existing cli scripts that use path-based + gateway do not break (curl doesn't auto-redirect without passing -L; wget + does not span across hostnames by default) + Context: https://github.com/ipfs/go-ipfs/issues/6975 + `, + IsEqual("hello\n"), + ). + Response(), + )) + + with(testGatewayWithManyProtocols(t, + "request for example.com/ipfs/{DirCID} redirects to subdomain", + ` + subdomains should not return payload directly, + but redirect to URL with proper origin isolation + `, + URL("%s/ipfs/%s/", gatewayURL, DirCID), + Expect(). + Status(301). + Headers( + Header("Location"). + Hint("request for example.com/ipfs/{DirCID} returns Location HTTP header for subdomain redirect in browsers"). + Contains("%s://%s.ipfs.%s/", u.Scheme, DirCID, u.Host), + ).Response(), + )) + + with(testGatewayWithManyProtocols(t, + "request for example.com/ipfs/{CIDv0} redirects to CIDv1 representation in subdomain", + "", + URL("%s/ipfs/%s/", gatewayURL, CIDv0), + Expect(). + Status(301). + Headers( + Header("Location"). + Hint("request for example.com/ipfs/{CIDv0to1} returns Location HTTP header for subdomain redirect in browsers"). + Contains("%s://%s.ipfs.%s/", u.Scheme, CIDv0to1, u.Host), + ).Response(), + )) + + // TODO: ipns + // TODO: dns link test + + // ============================================================================ + // Test subdomain-based requests to a local gateway with default config + // (origin per content root at http://*.example.com) + // ============================================================================ + + with(testGatewayWithManyProtocols(t, + "request for {CID}.ipfs.example.com should return expected payload", + "", + URL("%s://%s.ipfs.%s", u.Scheme, CIDv1, u.Host), + Expect(). + Status(200). + Body(Contains(CIDVal)). + Response(), + )) + + with(testGatewayWithManyProtocols(t, + "request for {CID}.ipfs.example.com/ipfs/{CID} should return HTTP 404", + "ensure /ipfs/ namespace is not mounted on subdomain", + URL("%s://%s.ipfs.%s/ipfs/%s", u.Scheme, CIDv1, u.Host, CIDv1), + Expect(). + Status(404). + Response(), + )) + + with(testGatewayWithManyProtocols(t, + "request for {CID}.ipfs.example.com/ipfs/file.txt should return data from a file in CID content root", + "ensure requests to /ipfs/* are not blocked, if content root has such subdirectory", + URL("%s://%s.ipfs.%s/ipfs/file.txt", u.Scheme, DirCID, u.Host), + Expect(). + Status(200). + Body(Contains("I am a txt file")). + Response(), + )) + + with(testGatewayWithManyProtocols(t, + "valid file and subdirectory paths in directory listing at {cid}.ipfs.example.com", + "{CID}.ipfs.example.com/sub/dir (Directory Listing)", + URL("%s://%s.ipfs.%s/", u.Scheme, DirCID, u.Host), + Expect(). + Status(200). + Body(And( + // TODO: implement html expectations + Contains("hello"), + Contains("ipfs"), + )). + Response(), + )) + + with(testGatewayWithManyProtocols(t, + "valid parent directory path in directory listing at {cid}.ipfs.example.com/sub/dir", + "", + URL("%s://%s.ipfs.%s/ipfs/ipns/", u.Scheme, DirCID, u.Host), + Expect(). + Status(200). + Body(And( + // TODO: implement html expectations + Contains(".."), + Contains("bar"), + )). + Response(), + )) + + with(testGatewayWithManyProtocols(t, + "request for deep path resource at {cid}.ipfs.localhost/sub/dir/file", + "", + URL("%s://%s.ipfs.%s/ipfs/ipns/bar", u.Scheme, DirCID, u.Host), + Expect(). + Status(200). + Body(Contains("text-file-content")). + Response(), + )) + + with(testGatewayWithManyProtocols(t, + "valid breadcrumb links in the header of directory listing at {cid}.ipfs.example.com/sub/dir", + ` + Note 1: we test for sneaky subdir names {cid}.ipfs.example.com/ipfs/ipns/ :^) + Note 2: example.com/ipfs/.. present in HTML will be redirected to subdomain, so this is expected behavior + `, + URL("%s://%s.ipfs.%s/ipfs/ipns/", u.Scheme, DirCID, u.Host), + Expect(). + Status(200). + Body( + And( + Contains("Index of"), + Contains("/ipfs/%s/ipfs/ipns", + u.Host, DirCID, DirCID, u.Host, DirCID, u.Host, DirCID), + ), + ). + Response(), + )) + + // TODO: # *.ipns.localhost + // TODO: # .ipns.localhost + // TODO: # .ipns.localhost + + // ## ============================================================================ + // ## Test DNSLink inlining on HTTP gateways + // ## ============================================================================ + + // TODO + + // ## ============================================================================ + // ## Test subdomain-based requests with a custom hostname config + // ## (origin per content root at http://*.example.com) + // ## ============================================================================ + + // # example.com/ip(f|n)s/* + // # ============================================================================= + + // # path requests to the root hostname should redirect + // # to a subdomain URL with proper origin isolation + + with(testGatewayWithManyProtocols(t, + "request for example.com/ipfs/{CIDv1} produces redirect to {CIDv1}.ipfs.example.com", + "path requests to the root hostname should redirect to a subdomain URL with proper origin isolation", + URL("%s://%s/ipfs/%s/", u.Scheme, u.Host, CIDv1), + Expect(). + Headers( + Header("Location").Equals("%s://%s.ipfs.%s/", u.Scheme, CIDv1, u.Host), + ). + Response(), + )) + + with(testGatewayWithManyProtocols(t, + "request for example.com/ipfs/{InvalidCID} produces useful error before redirect", + "error message should include original CID (and it should be case-sensitive, as we can't assume everyone uses base32)", + URL("%s://%s/ipfs/QmInvalidCID", u.Scheme, u.Host), + Expect(). + Body(Contains("invalid path \"/ipfs/QmInvalidCID\"")). + Response(), + )) + + with(testGatewayWithManyProtocols(t, + "request for example.com/ipfs/{CIDv0} produces redirect to {CIDv1}.ipfs.example.com", + "", + URL("%s://%s/ipfs/%s/", u.Scheme, u.Host, CIDv0), + Expect(). + Status(301). + Headers( + Header("Location").Equals("%s://%s.ipfs.%s/", u.Scheme, CIDv0to1, u.Host), + ). + Response(), + )) + + with(testGatewayWithManyProtocols(t, + "request for http://example.com/ipfs/{CID} with X-Forwarded-Proto: https produces redirect to HTTPS URL", + "Support X-Forwarded-Proto", + Request(). + URL("%s://%s/ipfs/%s/", u.Scheme, u.Host, CIDv1). + Header("X-Forwarded-Proto", "https"), + Expect(). + Status(301). + Headers( + Header("Location").Equals("https://%s.ipfs.%s/", CIDv1, u.Host), + ). + Response(), + )) + + with(testGatewayWithManyProtocols(t, + "request for example.com/ipfs/?uri=ipfs%3A%2F%2F.. produces redirect to /ipfs/.. content path", + "Support ipfs:// in https://developer.mozilla.org/en-US/docs/Web/API/Navigator/registerProtocolHandler", + Request(). + URL("%s://%s/ipfs/", u.Scheme, u.Host). + Query( + "uri", "ipfs://%s/wiki/Diego_Maradona.html", CIDWikipedia, + ), + Expect(). + Status(301). + Headers( + Header("Location").Equals("/ipfs/%s/wiki/Diego_Maradona.html", CIDWikipedia), + ). + Response(), + )) + + // # example.com/ipns/ + // TODO + + // # example.com/ipns/ + // TODO + + // # DNSLink on Public gateway with a single-level wildcard TLS cert + // # "Option C" from https://github.com/ipfs/in-web-browsers/issues/169 + // TODO + + // # Support ipns:// in https://developer.mozilla.org/en-US/docs/Web/API/Navigator/registerProtocolHandler + // TODO + + // # *.ipns.example.com + // # ============================================================================ + + // # .ipns.example.com + + // # API on subdomain gateway example.com + // # ============================================================================ + + // # DNSLink: .ipns.example.com + // # (not really useful outside of localhost, as setting TLS for more than one + // # level of wildcard is a pain, but we support it if someone really wants it) + // # ============================================================================ + // TODO + + // # DNSLink on Public gateway with a single-level wildcard TLS cert + // # "Option C" from https://github.com/ipfs/in-web-browsers/issues/169 + + // ## Test subdomain handling of CIDs that do not fit in a single DNS Label (>63chars) + // ## https://github.com/ipfs/go-ipfs/issues/7318 + // ## ============================================================================ + // TODO + + with(testGatewayWithManyProtocols(t, + "request for a too long CID at localhost/ipfs/{CIDv1} returns human readable error", + "router should not redirect to hostnames that could fail due to DNS limits", + URL("%s/ipfs/%s", gatewayURL, CIDv1_TOO_LONG), + Expect(). + Status(400). + Body(Contains("CID incompatible with DNS label length limit of 63")). + Response(), + )) + + with(testGatewayWithManyProtocols(t, + "request for a too long CID at {CIDv1}.ipfs.localhost returns expected payload", + "direct request should also fail (provides the same UX as router and avoids confusion)", + URL("%s://%s.ipfs.%s/", u.Scheme, CIDv1_TOO_LONG, u.Host), + Expect(). + Status(400). + Body(Contains("CID incompatible with DNS label length limit of 63")). + Response(), + )) + + // # public subdomain gateway: *.example.com + // TODO: IPNS + + // # Disable selected Paths for the subdomain gateway hostname + // # ============================================================================= + + // # disable /ipns for the hostname by not whitelisting it + + // # refuse requests to Paths that were not explicitly whitelisted for the hostname + + // MANY TODOs here + + // ## ============================================================================ + // ## Test support for X-Forwarded-Host + // ## ============================================================================ + + with(testGatewayWithManyProtocols(t, + "request for http://fake.domain.com/ipfs/{CID} doesn't match the example.com gateway", + "", + URL("%s://%s/ipfs/%s", u.Scheme, "fake.domain.com", CIDv1), + Expect(). + Status(200). + Response(), + )) + + with(testGatewayWithManyProtocols(t, + "request for http://fake.domain.com/ipfs/{CID} with X-Forwarded-Host: example.com match the example.com gateway", + "", + Request(). + URL("%s://%s/ipfs/%s", u.Scheme, "fake.domain.com", CIDv1). + Header("X-Forwarded-Host", u.Host), + Expect(). + Status(301). + Headers( + Header("Location").Equals("%s://%s.ipfs.%s/", u.Scheme, CIDv1, u.Host), + ). + Response(), + )) + + with(testGatewayWithManyProtocols(t, + "request for http://fake.domain.com/ipfs/{CID} with X-Forwarded-Host: example.com and X-Forwarded-Proto: https match the example.com gateway, redirect with https", + "", + Request(). + URL("%s://%s/ipfs/%s", u.Scheme, "fake.domain.com", CIDv1). + Header("X-Forwarded-Host", u.Host). + Header("X-Forwarded-Proto", "https"), + Expect(). + Status(301). + Headers( + Header("Location").Equals("https://%s.ipfs.%s/", CIDv1, u.Host), + ). + Response(), + )) + } + + if SubdomainGateway.IsEnabled() { + Run(t, tests) + } +} + +func testGatewayWithManyProtocols(t *testing.T, label string, hint string, reqURL interface{}, expected CResponse) []CTest { + t.Helper() + + baseURL := "" + baseReq := Request() + + switch req := reqURL.(type) { + case string: + baseURL = reqURL.(string) + case RequestBuilder: + baseReq = req + baseURL = req.GetURL() + default: + t.Fatalf("invalid type for reqURL: %T", reqURL) + } + + u, err := url.Parse(baseURL) + if err != nil { + t.Fatal(err) + } + // Because you might be testing an IPFS node in CI, or on your local machine, the test are designed + // to test the subdomain behavior (querying http://{CID}.my-subdomain-gateway.io/) even if the node is + // actually living on http://127.0.0.1:8080 or somewhere else. + // + // The test knows two addresses: + // - GatewayURL: the URL we connect to, it might be "dweb.link", "127.0.0.1:8080", etc. + // - SubdomainGatewayURL: the URL we test for subdomain requests, it might be "dweb.link", "localhost", "example.com", etc. + + // host is the hostname of the gateway we are testing, it might be `localhost` or `example.com` + host := u.Host + + // raw url is the url but we replace the host with our local url, it might be `http://127.0.0.1/ipfs/something` + u.Host = GatewayHost + rawURL := u.String() + + return []CTest{ { - Name: "request for 127.0.0.1/ipfs/{CID} stays on path", - Request: CRequest{ - Url: fmt.Sprintf("ipfs/%s", CIDv1), - }, - Response: CResponse{ - StatusCode: 200, - Body: Contains(CID_VAL), - }, + Name: fmt.Sprintf("%s (direct HTTP)", label), + Hint: fmt.Sprintf("%s\n%s", hint, "direct HTTP request (hostname in URL, raw IP in Host header)"), + Request: baseReq. + URL(rawURL). + DoNotFollowRedirects(). + Headers( + Header("Host", host), + ). + Request(), + Response: expected, }, { - Name: "request for localhost/ipfs/{CIDv1} returns HTTP 301 Moved Permanently", - Request: CRequest{ - DoNotFollowRedirects: true, - Url: fmt.Sprintf("/ipfs/%s", CIDv1), - }, - Response: CResponse{ - StatusCode: 301, - Headers: map[string]interface{}{ - "Location": Contains("http://%s.ipfs.localhost:8080", CIDv1), - }, - }, + Name: fmt.Sprintf("%s (HTTP proxy)", label), + Hint: fmt.Sprintf("%s\n%s", hint, "HTTP proxy (hostname is passed via URL)"), + Request: baseReq. + URL(baseURL). + Proxy(GatewayURL). + DoNotFollowRedirects(). + Request(), + Response: expected, }, { - Name: "request for {cid}.ipfs.localhost/api returns data if present on the content root", - Request: CRequest{ - RawURL: fmt.Sprintf("http://%s.ipfs.localhost:8080/api/file.txt", DIR_CID), - }, - Response: CResponse{ - Body: Contains("I am a txt file"), - }, + Name: fmt.Sprintf("%s (HTTP proxy tunneling via CONNECT)", label), + Hint: fmt.Sprintf("%s\n%s", hint, `HTTP proxy + In HTTP/1.x, the pseudo-method CONNECT, + can be used to convert an HTTP connection into a tunnel to a remote host + https://tools.ietf.org/html/rfc7231#section-4.3.6 + `), + Request: baseReq. + URL(baseURL). + Proxy(GatewayURL). + WithProxyTunnel(). + DoNotFollowRedirects(). + Headers( + Header("Host", host), + ). + Request(), + Response: expected, }, } - - if SubdomainGateway.IsEnabled() { - Run(t, tests) - } } diff --git a/tests/t0116_gateway_cache_test.go b/tests/t0116_gateway_cache_test.go index caf1d9f37..ac70fa7c4 100644 --- a/tests/t0116_gateway_cache_test.go +++ b/tests/t0116_gateway_cache_test.go @@ -20,7 +20,7 @@ func TestGatewayCache(t *testing.T) { { Name: "GET for /ipfs/ unixfs dir listing succeeds", Request: CRequest{ - Url: fmt.Sprintf("ipfs/%s/root2/root3/", fixture.MustGetCid()), + Path: fmt.Sprintf("ipfs/%s/root2/root3/", fixture.MustGetCid()), }, Response: CResponse{ StatusCode: 200, @@ -35,7 +35,7 @@ func TestGatewayCache(t *testing.T) { { Name: "GET for /ipfs/ unixfs dir with index.html succeeds", Request: CRequest{ - Url: fmt.Sprintf("ipfs/%s/root2/root3/root4/", fixture.MustGetCid()), + Path: fmt.Sprintf("ipfs/%s/root2/root3/root4/", fixture.MustGetCid()), }, Response: CResponse{ StatusCode: 200, @@ -50,7 +50,7 @@ func TestGatewayCache(t *testing.T) { { Name: "GET for /ipfs/ unixfs file succeeds", Request: CRequest{ - Url: fmt.Sprintf("ipfs/%s/root2/root3/root4/index.html", fixture.MustGetCid()), + Path: fmt.Sprintf("ipfs/%s/root2/root3/root4/index.html", fixture.MustGetCid()), }, Response: CResponse{ StatusCode: 200, @@ -65,7 +65,7 @@ func TestGatewayCache(t *testing.T) { { Name: "GET for /ipfs/ unixfs dir as DAG-JSON succeeds", Request: CRequest{ - Url: fmt.Sprintf("ipfs/%s/root2/root3/root4/?format=dag-json", fixture.MustGetCid()), + Path: fmt.Sprintf("ipfs/%s/root2/root3/root4/?format=dag-json", fixture.MustGetCid()), }, Response: CResponse{ StatusCode: 200, @@ -77,7 +77,7 @@ func TestGatewayCache(t *testing.T) { { Name: "GET for /ipfs/ unixfs dir as JSON succeeds", Request: CRequest{ - Url: fmt.Sprintf("ipfs/%s/root2/root3/root4/?format=json", fixture.MustGetCid()), + Path: fmt.Sprintf("ipfs/%s/root2/root3/root4/?format=json", fixture.MustGetCid()), }, Response: CResponse{ StatusCode: 200, @@ -89,7 +89,7 @@ func TestGatewayCache(t *testing.T) { { Name: "HEAD for /ipfs/ with only-if-cached succeeds when in local datastore", Request: CRequest{ - Url: fmt.Sprintf("ipfs/%s/root2/root3/root4/?format=json", fixture.MustGetCid()), + Path: fmt.Sprintf("ipfs/%s/root2/root3/root4/?format=json", fixture.MustGetCid()), Headers: map[string]string{ "Cache-Control": "only-if-cached", }, @@ -102,7 +102,7 @@ func TestGatewayCache(t *testing.T) { { Name: "HEAD for /ipfs/ with only-if-cached fails when not in local datastore", Request: CRequest{ - Url: "ipfs/QmYzfKSE55XCjD1MW128RfciAf2DViABhEiXfgVFMabSjN", + Path: "ipfs/QmYzfKSE55XCjD1MW128RfciAf2DViABhEiXfgVFMabSjN", Headers: map[string]string{ "Cache-Control": "only-if-cached", }, @@ -115,7 +115,7 @@ func TestGatewayCache(t *testing.T) { { Name: "GET for /ipfs/ with only-if-cached succeeds when in local datastore", Request: CRequest{ - Url: fmt.Sprintf("ipfs/%s/root2/root3/root4/?format=json", fixture.MustGetCid()), + Path: fmt.Sprintf("ipfs/%s/root2/root3/root4/?format=json", fixture.MustGetCid()), Headers: map[string]string{ "Cache-Control": "only-if-cached", }, @@ -127,7 +127,7 @@ func TestGatewayCache(t *testing.T) { { Name: "GET for /ipfs/ with only-if-cached fails when not in local datastore", Request: CRequest{ - Url: "ipfs/QmYzfKSE55XCjD1MW128RfciAf2DViABhEiXfgVFMabSjN", + Path: "ipfs/QmYzfKSE55XCjD1MW128RfciAf2DViABhEiXfgVFMabSjN", Headers: map[string]string{ "Cache-Control": "only-if-cached", }, @@ -139,7 +139,7 @@ func TestGatewayCache(t *testing.T) { { Name: "GET for /ipfs/ file with matching Etag in If-None-Match returns 304 Not Modified", Request: CRequest{ - Url: fmt.Sprintf("ipfs/%s/root2/root3/root4/index.html", fixture.MustGetCid()), + Path: fmt.Sprintf("ipfs/%s/root2/root3/root4/index.html", fixture.MustGetCid()), Headers: map[string]string{ "If-None-Match": fmt.Sprintf("\"%s\"", fixture.MustGetCid("root2", "root3", "root4", "index.html")), }, @@ -151,7 +151,7 @@ func TestGatewayCache(t *testing.T) { { Name: "GET for /ipfs/ dir with index.html file with matching Etag in If-None-Match returns 304 Not Modified", Request: CRequest{ - Url: fmt.Sprintf("ipfs/%s/root2/root3/root4/", fixture.MustGetCid()), + Path: fmt.Sprintf("ipfs/%s/root2/root3/root4/", fixture.MustGetCid()), Headers: map[string]string{ "If-None-Match": fmt.Sprintf("\"%s\"", fixture.MustGetCid("root2", "root3", "root4")), }, @@ -163,7 +163,7 @@ func TestGatewayCache(t *testing.T) { { Name: "GET for /ipfs/ file with matching third Etag in If-None-Match returns 304 Not Modified", Request: CRequest{ - Url: fmt.Sprintf("ipfs/%s/root2/root3/root4/index.html", fixture.MustGetCid()), + Path: fmt.Sprintf("ipfs/%s/root2/root3/root4/index.html", fixture.MustGetCid()), Headers: map[string]string{ "If-None-Match": fmt.Sprintf("\"fakeEtag1\", \"fakeEtag2\", \"%s\"", fixture.MustGetCid("root2", "root3", "root4", "index.html")), }, @@ -175,7 +175,7 @@ func TestGatewayCache(t *testing.T) { { Name: "GET for /ipfs/ file with matching weak Etag in If-None-Match returns 304 Not Modified", Request: CRequest{ - Url: fmt.Sprintf("ipfs/%s/root2/root3/root4/index.html", fixture.MustGetCid()), + Path: fmt.Sprintf("ipfs/%s/root2/root3/root4/index.html", fixture.MustGetCid()), Headers: map[string]string{ "If-None-Match": fmt.Sprintf("W/\"%s\"", fixture.MustGetCid("root2", "root3", "root4", "index.html")), }, @@ -186,7 +186,7 @@ func TestGatewayCache(t *testing.T) { }, { Name: "GET for /ipfs/ file with wildcard Etag in If-None-Match returns 304 Not Modified", Request: CRequest{ - Url: fmt.Sprintf("ipfs/%s/root2/root3/root4/index.html", fixture.MustGetCid()), + Path: fmt.Sprintf("ipfs/%s/root2/root3/root4/index.html", fixture.MustGetCid()), Headers: map[string]string{ "If-None-Match": "*", }, @@ -197,7 +197,7 @@ func TestGatewayCache(t *testing.T) { }, { Name: "GET for /ipfs/ dir listing with matching weak Etag in If-None-Match returns 304 Not Modified", Request: CRequest{ - Url: fmt.Sprintf("ipfs/%s/root2/root3/", fixture.MustGetCid()), + Path: fmt.Sprintf("ipfs/%s/root2/root3/", fixture.MustGetCid()), Headers: map[string]string{ "If-None-Match": fmt.Sprintf("W/\"%s\"", fixture.MustGetCid("root2", "root3")), }, diff --git a/tests/t0117_gateway_block_test.go b/tests/t0117_gateway_block_test.go index 34299496e..1baca7f2a 100644 --- a/tests/t0117_gateway_block_test.go +++ b/tests/t0117_gateway_block_test.go @@ -17,7 +17,7 @@ func TestGatewayBlock(t *testing.T) { { Name: "GET with format=raw param returns a raw block", Request: CRequest{ - Url: fmt.Sprintf("ipfs/%s/dir?format=raw", fixture.MustGetCid()), + Path: fmt.Sprintf("ipfs/%s/dir?format=raw", fixture.MustGetCid()), }, Response: CResponse{ StatusCode: 200, @@ -27,7 +27,7 @@ func TestGatewayBlock(t *testing.T) { { Name: "GET with application/vnd.ipld.raw header returns a raw block", Request: CRequest{ - Url: fmt.Sprintf("ipfs/%s/dir", fixture.MustGetCid()), + Path: fmt.Sprintf("ipfs/%s/dir", fixture.MustGetCid()), Headers: map[string]string{ "Accept": "application/vnd.ipld.raw", }, @@ -40,7 +40,7 @@ func TestGatewayBlock(t *testing.T) { { Name: "GET with application/vnd.ipld.raw header returns expected response headers", Request: CRequest{ - Url: fmt.Sprintf("ipfs/%s/dir/ascii.txt", fixture.MustGetCid()), + Path: fmt.Sprintf("ipfs/%s/dir/ascii.txt", fixture.MustGetCid()), Headers: map[string]string{ "Accept": "application/vnd.ipld.raw", }, @@ -59,7 +59,7 @@ func TestGatewayBlock(t *testing.T) { { Name: "GET with application/vnd.ipld.raw header and filename param returns expected Content-Disposition header", Request: CRequest{ - Url: fmt.Sprintf("ipfs/%s/dir/ascii.txt?filename=foobar.bin", fixture.MustGetCid()), + Path: fmt.Sprintf("ipfs/%s/dir/ascii.txt?filename=foobar.bin", fixture.MustGetCid()), Headers: map[string]string{ "Accept": "application/vnd.ipld.raw", }, @@ -74,7 +74,7 @@ func TestGatewayBlock(t *testing.T) { { Name: "GET with application/vnd.ipld.raw header returns expected caching headers", Request: CRequest{ - Url: fmt.Sprintf("ipfs/%s/dir/ascii.txt", fixture.MustGetCid()), + Path: fmt.Sprintf("ipfs/%s/dir/ascii.txt", fixture.MustGetCid()), Headers: map[string]string{ "Accept": "application/vnd.ipld.raw", }, diff --git a/tests/t0118_gateway_car_test.go b/tests/t0118_gateway_car_test.go index ef780548c..a1eb2f8a0 100644 --- a/tests/t0118_gateway_car_test.go +++ b/tests/t0118_gateway_car_test.go @@ -20,7 +20,7 @@ func TestGatewayCar(t *testing.T) { Name: "GET response for application/vnd.ipld.car", // Test between l85 and l112 Request: CRequest{ - Url: fmt.Sprintf("ipfs/%s/subdir/ascii.txt", fixture.MustGetCid()), + Path: fmt.Sprintf("ipfs/%s/subdir/ascii.txt", fixture.MustGetCid()), Headers: map[string]string{ "Accept": "application/vnd.ipld.car", }, @@ -49,7 +49,7 @@ func TestGatewayCar(t *testing.T) { // Test between l85 and l112 Name: "GET response for application/vnd.ipld.car2", Request: Request(). - Url("ipfs/%s/subdir/ascii.txt", fixture.MustGetCid()). + Path("ipfs/%s/subdir/ascii.txt", fixture.MustGetCid()). Headers( Header("Accept", "application/vnd.ipld.car"), ).Request(), diff --git a/tooling/check/check.go b/tooling/check/check.go index 1a00e1756..129325b43 100644 --- a/tooling/check/check.go +++ b/tooling/check/check.go @@ -64,6 +64,29 @@ func IsEmpty(hint ...string) interface{} { return CheckIsEmpty{} } +type CheckAnd[T any] struct { + Checks []Check[T] +} + +func And[T any](checks ...Check[T]) Check[T] { + return &CheckAnd[T]{ + Checks: checks, + } +} + +func (c *CheckAnd[T]) Check(v T) CheckOutput { + for _, check := range c.Checks { + output := check.Check(v) + if !output.Success { + return output + } + } + + return CheckOutput{ + Success: true, + } +} + type CheckIsEqual struct { Value string } diff --git a/tooling/test/config.go b/tooling/test/config.go new file mode 100644 index 000000000..8291eda01 --- /dev/null +++ b/tooling/test/config.go @@ -0,0 +1,56 @@ +package test + +import ( + "net/url" + "os" + "strings" + + logging "github.com/ipfs/go-log" +) + +var log = logging.Logger("conformance") + +func GetEnv(key string, fallback string) string { + if value, ok := os.LookupEnv(key); ok { + return value + } + return fallback +} + +var GatewayURL = strings.TrimRight( + GetEnv("GATEWAY_URL", "http://127.0.0.1:8080"), + "/") + +var SubdomainGatewayURL = strings.TrimRight( + GetEnv("SUBDOMAIN_GATEWAY_URL", "http://example.com"), + "/") + + +var GatewayHost = "" +var SubdomainGatewayHost = "" +var SubdomainGatewayScheme = "" + +var SubdomainLocalhostGatewayURL = "http://localhost" + +func init() { + parsed, err := url.Parse(GatewayURL) + if err != nil { + panic(err) + } + + GatewayHost = parsed.Host + + parsed, err = url.Parse(SubdomainGatewayURL) + if err != nil { + panic(err) + } + + SubdomainGatewayHost = parsed.Host + SubdomainGatewayScheme = parsed.Scheme + + log.Debugf("GatewayURL: %s", GatewayURL) + + log.Debugf("SubdomainGatewayURL: %s", SubdomainGatewayURL) + log.Debugf("SubdomainGatewayHost: %s", SubdomainGatewayHost) + log.Debugf("SubdomainGatewayScheme: %s", SubdomainGatewayScheme) +} \ No newline at end of file diff --git a/tooling/test/proxy.go b/tooling/test/proxy.go new file mode 100644 index 000000000..55e893e3e --- /dev/null +++ b/tooling/test/proxy.go @@ -0,0 +1,81 @@ +package test + +import ( + "bufio" + "context" + "crypto/tls" + "fmt" + "net" + "net/http" + "net/url" +) + +func NewProxyTunnelClient(proxyURL string) *http.Client { + proxy, err := url.Parse(proxyURL) + if err != nil { + panic(err) + } + + dialer := &net.Dialer{} + + transport := &http.Transport{ + Proxy: http.ProxyURL(proxy), + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + // Connect to the proxy server + conn, err := dialer.DialContext(ctx, "tcp", proxy.Host) + if err != nil { + return nil, err + } + + // Send the CONNECT request to establish a tunnel + connectReq := &http.Request{ + Method: "CONNECT", + URL: &url.URL{Opaque: addr}, + Host: addr, + Header: make(http.Header), + } + if err := connectReq.Write(conn); err != nil { + conn.Close() + return nil, err + } + + // Read the CONNECT response from the proxy + resp, err := http.ReadResponse(bufio.NewReader(conn), connectReq) + if err != nil { + conn.Close() + return nil, err + } + if resp.StatusCode != 200 { + conn.Close() + return nil, fmt.Errorf("proxy error: %v", resp.Status) + } + + return conn, nil + }, + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + + client := &http.Client{ + Transport: transport, + } + + return client +} + +func NewProxyClient(proxyURL string) *http.Client { + proxy, err := url.Parse(proxyURL) + if err != nil { + panic(err) + } + + transport := &http.Transport{ + Proxy: http.ProxyURL(proxy), + ForceAttemptHTTP2: false, + } + + client := &http.Client{ + Transport: transport, + } + + return client +} diff --git a/tooling/test/report.go b/tooling/test/report.go new file mode 100644 index 000000000..11b5df1d5 --- /dev/null +++ b/tooling/test/report.go @@ -0,0 +1,94 @@ +package test + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httputil" + "testing" + "text/template" +) + + +type ReportInput struct { + Req *http.Request + Res *http.Response + Err error + Test CTest +} + +const TEMPLATE = ` +Name: {{.Test.Name}} +Hint: {{.Test.Hint}} + +Error: {{.Err}} + +Request: +{{.Test.Request | json}} + +Expected Response: +{{.Test.Response | json}} + +Actual Request: +{{.Req | dump}} + +Actual Response: +{{.Res | dump}} +` + +func report(t *testing.T, test CTest, req *http.Request, res *http.Response, err error) { + input := ReportInput{ + Req: req, + Res: res, + Err: err, + Test: test, + } + + tmpl, err := template.New("report").Funcs(template.FuncMap{ + "json": func(v interface{}) string { + j, _ := json.MarshalIndent(v, "", " ") + return string(j) + }, + "dump": func(v interface{}) string { + if v == nil { + return "nil" + } + + var b []byte + var err error + switch v := v.(type) { + case *http.Request: + b, err = httputil.DumpRequestOut(v, true) + case *http.Response: + // TODO: we have to disable the body dump because + // it triggers an error: + // "http: ContentLength=6 with Body length 0" + b, err = httputil.DumpResponse(v, false) + default: + panic("unknown type") + } + + if err != nil { + panic(err) + } + + return string(b) + }, + }).Parse(TEMPLATE) + + if err != nil { + panic(err) + } + + var buf bytes.Buffer + err = tmpl.Execute(&buf, input) + if err != nil { + panic(err) + } + + if input.Err != nil { + t.Fatal(buf.String()) + } else { + t.Log(buf.String()) + } +} diff --git a/tooling/test/sugar.go b/tooling/test/sugar.go index 2b47100dd..9c1b353fe 100644 --- a/tooling/test/sugar.go +++ b/tooling/test/sugar.go @@ -2,58 +2,116 @@ package test import ( "fmt" + "net/url" "github.com/ipfs/gateway-conformance/tooling/check" ) type RequestBuilder struct { - Method_ string - Url_ string - Headers_ map[string]string + Method_ string + Path_ string + URL_ string + Proxy_ string + UseProxyTunnel bool + Headers_ map[string]string + DoNotFollowRedirects_ bool + Query_ url.Values } func Request() RequestBuilder { - return RequestBuilder{Method_: "GET"} + return RequestBuilder{Method_: "GET", + Query_: make(url.Values)} } -func (r RequestBuilder) Url(url string, args ...any) RequestBuilder { - r.Url_ = fmt.Sprintf(url, args...) +func (r RequestBuilder) Path(path string, args ...any) RequestBuilder { + r.Path_ = fmt.Sprintf(path, args...) return r } -func (r RequestBuilder) Header(h HeaderBuilder) RequestBuilder { +func (r RequestBuilder) URL(path string, args ...any) RequestBuilder { + r.URL_ = fmt.Sprintf(path, args...) + return r +} + +func (r RequestBuilder) Query(key, value string, args ...any) RequestBuilder { + r.Query_.Add(key, fmt.Sprintf(value, args...)) + return r +} + +func (r RequestBuilder) GetURL() string { + if r.Path_ != "" { + panic("not supported") + } + + return r.URL_ +} + +func (r RequestBuilder) Proxy(path string, args ...any) RequestBuilder { + r.Proxy_ = fmt.Sprintf(path, args...) + return r +} + +func (r RequestBuilder) WithProxyTunnel() RequestBuilder { + r.UseProxyTunnel = true + return r +} + +func (r RequestBuilder) DoNotFollowRedirects() RequestBuilder { + r.DoNotFollowRedirects_ = true + return r +} + +func (r RequestBuilder) Method(method string) RequestBuilder { + r.Method_ = method + return r +} + +func (r RequestBuilder) Header(k, v string) RequestBuilder { if r.Headers_ == nil { r.Headers_ = make(map[string]string) } - r.Headers_[h.Key] = h.Value + r.Headers_[k] = v return r } func (r RequestBuilder) Headers(hs ...HeaderBuilder) RequestBuilder { + if r.Headers_ == nil { + r.Headers_ = make(map[string]string) + } + for _, h := range hs { - r = r.Header(h) + r.Headers_[h.Key] = h.Value } return r } func (r RequestBuilder) Request() CRequest { + if r.URL_ != "" && r.Path_ != "" { + panic("Both 'URL' and 'Path' are set") + } + return CRequest{ - Method: r.Method_, - Url: r.Url_, - Headers: r.Headers_, + Method: r.Method_, + Path: r.Path_, + URL: r.URL_, + Query: r.Query_, + Proxy: r.Proxy_, + UseProxyTunnel: r.UseProxyTunnel, + Headers: r.Headers_, + DoNotFollowRedirects: r.DoNotFollowRedirects_, } } type ExpectBuilder struct { StatusCode int Headers_ []HeaderBuilder - Body []byte + Body_ interface{} } func Expect() ExpectBuilder { - return ExpectBuilder{} + return ExpectBuilder{Body_: nil} } func (e ExpectBuilder) Status(statusCode int) ExpectBuilder { @@ -71,6 +129,43 @@ func (e ExpectBuilder) Headers(hs ...HeaderBuilder) ExpectBuilder { return e } +func (e ExpectBuilder) Body(body interface{}) ExpectBuilder { + switch body := body.(type) { + case string: + e.Body_ = []byte(body) + case []byte: + e.Body_ = body + case check.Check[string]: + e.Body_ = body + case check.CheckWithHint[string]: + e.Body_ = body + default: + panic("body must be string, []byte, or a regular check") + } + + return e +} + +func (e ExpectBuilder) BodyWithHint(hint string, body interface{}) ExpectBuilder { + switch body := body.(type) { + case string: + e.Body_ = check.WithHint( + hint, + check.IsEqual(body), + ) + case []byte: + panic("body with hint for bytes is not implemented yet") + case check.Check[string]: + e.Body_ = check.WithHint(hint, body) + case check.CheckWithHint[string]: + panic("this check already has a hint") + default: + panic("body must be string, []byte, or a regular check") + } + + return e +} + func (e ExpectBuilder) Response() CResponse { headers := make(map[string]interface{}) @@ -86,7 +181,7 @@ func (e ExpectBuilder) Response() CResponse { return CResponse{ StatusCode: e.StatusCode, Headers: headers, - Body: e.Body, + Body: e.Body_, } } @@ -108,8 +203,8 @@ func Header(key string, opts ...string) HeaderBuilder { return HeaderBuilder{Key: key} } -func (h HeaderBuilder) Contains(value string) HeaderBuilder { - h.Check = check.Contains(value) +func (h HeaderBuilder) Contains(value string, rest ...any) HeaderBuilder { + h.Check = check.Contains(value, rest...) return h } diff --git a/tooling/test/test.go b/tooling/test/test.go index 8249af099..a2398ecc6 100644 --- a/tooling/test/test.go +++ b/tooling/test/test.go @@ -5,46 +5,41 @@ import ( "fmt" "io" "net/http" - "os" - "strings" + "net/url" "testing" "github.com/ipfs/gateway-conformance/tooling/check" ) -func GetEnv(key string, fallback string) string { - if value, ok := os.LookupEnv(key); ok { - return value - } - return fallback -} - -var GatewayUrl = strings.TrimRight( - GetEnv("GATEWAY_URL", "http://127.0.0.1:8080"), - "/") - type CRequest struct { - Method string - RawURL string - DoNotFollowRedirects bool - Url string - Headers map[string]string - Body []byte + Method string `json:"method,omitempty"` + URL string `json:"url,omitempty"` + Query url.Values `json:"query,omitempty"` + Proxy string `json:"proxy,omitempty"` + UseProxyTunnel bool `json:"useProxyTunnel,omitempty"` + DoNotFollowRedirects bool `json:"doNotFollowRedirects,omitempty"` + Path string `json:"path,omitempty"` + Subdomain string `json:"subdomain,omitempty"` + Headers map[string]string `json:"headers,omitempty"` + Body []byte `json:"body,omitempty"` } type CResponse struct { - StatusCode int - Headers map[string]interface{} - Body interface{} + StatusCode int `json:"statusCode,omitempty"` + Headers map[string]interface{} `json:"headers,omitempty"` + Body interface{} `json:"body,omitempty"` } type CTest struct { - Name string - Request CRequest - Response CResponse + Name string `json:"name,omitempty"` + Hint string `json:"hint,omitempty"` + Request CRequest `json:"request,omitempty"` + Response CResponse `json:"response,omitempty"` } func Run(t *testing.T, tests []CTest) { + // NewDialer() + for _, test := range tests { t.Run(test.Name, func(t *testing.T) { method := test.Request.Method @@ -52,21 +47,59 @@ func Run(t *testing.T, tests []CTest) { method = "GET" } + // Prepare a client, + // use proxy, deal with redirects, etc. client := &http.Client{} + if test.Request.UseProxyTunnel { + if test.Request.Proxy == "" { + t.Fatal("ProxyTunnel requires a proxy") + } + + client = NewProxyTunnelClient(test.Request.Proxy) + } else if test.Request.Proxy != "" { + client = NewProxyClient(test.Request.Proxy) + } + if test.Request.DoNotFollowRedirects { client.CheckRedirect = func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse } } - var url string - if test.Request.RawURL != "" { - if test.Request.Url != "" { - t.Fatalf("Both 'RawURL' and 'Url' are set") + var res *http.Response = nil + var req *http.Request = nil + + localReport := func(msg interface{}, rest ...interface{}) { + var err error + switch msg := msg.(type) { + case string: + err = fmt.Errorf(msg, rest...) + case error: + err = msg + default: + panic("msg must be string or error") } - url = test.Request.RawURL - } else { - url = fmt.Sprintf("%s/%s", GatewayUrl, test.Request.Url) + + report(t, test, req, res, err) + } + + var url string + if test.Request.URL != "" && test.Request.Path != "" { + localReport("Both 'URL' and 'Path' are set") + } + if test.Request.URL == "" && test.Request.Path == "" { + localReport("Neither 'URL' nor 'Path' are set") + } + if test.Request.URL != "" { + url = test.Request.URL + } + if test.Request.Path != "" { + url = fmt.Sprintf("%s/%s", GatewayURL, test.Request.Path) + } + + query := test.Request.Query.Encode() + if query != "" { + url = fmt.Sprintf("%s?%s", url, query) } var body io.Reader @@ -83,17 +116,23 @@ func Run(t *testing.T, tests []CTest) { // add headers for key, value := range test.Request.Headers { req.Header.Add(key, value) + + // https://github.com/golang/go/issues/7682 + if key == "Host" { + req.Host = value + } } // send request - res, err := client.Do(req) + log.Debugf("Querying %s", url) + res, err = client.Do(req) if err != nil { - t.Fatalf("Querying %s failed: %s", url, err) + localReport("Querying %s failed: %s", url, err) } if test.Response.StatusCode != 0 { if res.StatusCode != test.Response.StatusCode { - t.Fatalf("Status code is not %d. It is %d", test.Response.StatusCode, res.StatusCode) + localReport("Status code is not %d. It is %d", test.Response.StatusCode, res.StatusCode) } } @@ -112,14 +151,14 @@ func Run(t *testing.T, tests []CTest) { case string: output = check.IsEqual(v).Check(actual) default: - t.Fatalf("Header check '%s' has an invalid type: %T", key, value) + localReport("Header check '%s' has an invalid type: %T", key, value) } if !output.Success { if hint == "" { - t.Fatalf("Header '%s' %s", key, output.Reason) + localReport("Header '%s' %s", key, output.Reason) } else { - t.Fatalf("Header '%s' %s (%s)", key, output.Reason, hint) + localReport("Header '%s' %s (%s)", key, output.Reason, hint) } } } @@ -128,35 +167,34 @@ func Run(t *testing.T, tests []CTest) { defer res.Body.Close() resBody, err := io.ReadAll(res.Body) if err != nil { - t.Fatal(err) + localReport(err) } switch v := test.Response.Body.(type) { case check.Check[string]: output := v.Check(string(resBody)) if !output.Success { - t.Fatalf("Body %s", output.Reason) + localReport("Body %s", output.Reason) } case check.CheckWithHint[string]: output := v.Check.Check(string(resBody)) if !output.Success { - t.Fatalf("Body %s (%s)", output.Reason, v.Hint) + localReport("Body %s (%s)", output.Reason, v.Hint) } case string: if string(resBody) != v { - t.Fatalf("Body is not '%s'. It is: '%s'", v, resBody) + localReport("Body is not '%s'. It is: '%s'", v, resBody) } case []byte: if !bytes.Equal(resBody, v) { - if res.Header.Get("Content-Type") == "application/vnd.ipld.raw" { - t.Fatalf("Body is not '%+v'. It is: '%+v'", test.Response.Body, resBody) + localReport("Body is not '%+v'. It is: '%+v'", test.Response.Body, resBody) } else { - t.Fatalf("Body is not '%s'. It is: '%s'", test.Response.Body, resBody) + localReport("Body is not '%s'. It is: '%s'", test.Response.Body, resBody) } } default: - t.Fatalf("Body check has an invalid type: %T", test.Response.Body) + localReport("Body check has an invalid type: %T", test.Response.Body) } } })