From 1d8cb34e4dfa7bdd55c2a172d62cd36d73390661 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Thu, 23 May 2024 10:17:39 +0200 Subject: [PATCH] examples: verified CAR file fetcher --- examples/car-file-fetcher/.gitignore | 2 + examples/car-file-fetcher/README.md | 27 ++++++ examples/car-file-fetcher/hello.car | Bin 0 -> 107 bytes examples/car-file-fetcher/main.go | 121 +++++++++++++++++++++++++ examples/car-file-fetcher/main_test.go | 38 ++++++++ examples/go.mod | 2 +- 6 files changed, 189 insertions(+), 1 deletion(-) create mode 100644 examples/car-file-fetcher/.gitignore create mode 100644 examples/car-file-fetcher/README.md create mode 100644 examples/car-file-fetcher/hello.car create mode 100644 examples/car-file-fetcher/main.go create mode 100644 examples/car-file-fetcher/main_test.go diff --git a/examples/car-file-fetcher/.gitignore b/examples/car-file-fetcher/.gitignore new file mode 100644 index 000000000..6615a921e --- /dev/null +++ b/examples/car-file-fetcher/.gitignore @@ -0,0 +1,2 @@ +car-file-fetcher +hello.txt diff --git a/examples/car-file-fetcher/README.md b/examples/car-file-fetcher/README.md new file mode 100644 index 000000000..d44b9cb59 --- /dev/null +++ b/examples/car-file-fetcher/README.md @@ -0,0 +1,27 @@ +# CAR File Fetcher + +This example shows how to download a UnixFS file or directory from a gateway that implements +[application/vnd.ipld.car](https://www.iana.org/assignments/media-types/application/vnd.ipld.car) +responses of the [Trustles Gateway](https://specs.ipfs.tech/http-gateways/trustless-gateway/) +specification, in a trustless, verifiable manner. + +It relies on [IPIP-402](https://specs.ipfs.tech/ipips/ipip-0402/) to retrieve +the file entity via a single CAR request with all blocks required for end-to-end +verification. + +## Build + +```bash +> go build -o fetcher +``` + +## Usage + +First, you need a gateway that complies with the Trustless Gateway specification. +In our specific case, we need that the gateway supports CAR response type. + +As an example, you can verifiably fetch a `hello.txt` file from IPFS gateway at `https://trustless-gateway.link`: + +``` +./fetcher -g https://trustless-gateway.link -o hello.txt /ipfs/bafkreifzjut3te2nhyekklss27nh3k72ysco7y32koao5eei66wof36n5e +``` diff --git a/examples/car-file-fetcher/hello.car b/examples/car-file-fetcher/hello.car new file mode 100644 index 0000000000000000000000000000000000000000..b284068a8a5235053e38d5181d51033bef6a21a6 GIT binary patch literal 107 zcmcColv") + flag.PrintDefaults() + } + + gatewayUrlPtr := flag.String("g", "https://trustless-gateway.link", "trustless gateway to download the CAR file from") + userAgentPtr := flag.String("u", "", "user agent to use during the HTTP requests") + outputPtr := flag.String("o", "out", "output path to store the fetched path") + limitPtr := flag.Int64("l", 0, "file size limit for the gateway download") + flag.Parse() + + ipfsPath := flag.Arg(0) + if len(ipfsPath) == 0 { + flag.Usage() + os.Exit(1) + } + + if err := fetch(*gatewayUrlPtr, ipfsPath, *outputPtr, *userAgentPtr, *limitPtr); err != nil { + log.Fatal(err) + } +} + +func fetch(gatewayURL, ipfsPath, outputPath, userAgent string, limit int64) error { + // Parse the given IPFS path to make sure it is valid. + p, err := path.NewPath(ipfsPath) + if err != nil { + return err + } + + // Create a custom [http.Client] with the given user agent and the limit. + httpClient := &http.Client{ + Timeout: gateway.DefaultGetBlockTimeout, + Transport: &limitedTransport{ + RoundTripper: http.DefaultTransport, + limitBytes: limit, + userAgent: userAgent, + }, + } + + // Create the remote CAR gateway backend pointing to the given gateway URL and + // using our [http.Client]. A custom [http.Client] is not required and the called + // function would create a new one instead. + backend, err := gateway.NewRemoteCarBackend([]string{gatewayURL}, httpClient) + if err != nil { + return err + } + + // Resolve the given IPFS path to ensure that it is not mutable. This will + // resolve both DNSLink and regular IPNS links. For the latter, it is + // necessary that the given gateway supports [IPNS Record] response format. + // + // [IPNS Record]: https://www.iana.org/assignments/media-types/application/vnd.ipfs.ipns-record + imPath, _, _, err := backend.ResolveMutable(context.Background(), p) + if err != nil { + return err + } + + // Fetch the file or directory from the gateway. Since we're using a remote CAR + // backend gateway, this call will internally fetch a CAR file from the remote + // gateway and ensure that all blocks are present and verified. + _, file, err := backend.GetAll(context.Background(), imPath) + if err != nil { + return err + } + defer file.Close() + + // Write the returned UnixFS file or directory to the file system. + return files.WriteTo(file, outputPath) +} + +type limitedTransport struct { + http.RoundTripper + limitBytes int64 + userAgent string +} + +func (r *limitedTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if r.userAgent != "" { + req.Header.Set("User-Agent", r.userAgent) + } + resp, err := r.RoundTripper.RoundTrip(req) + if resp != nil && resp.Body != nil && r.limitBytes > 0 { + resp.Body = &limitReadCloser{ + limit: r.limitBytes, + ReadCloser: resp.Body, + } + } + return resp, err +} + +type limitReadCloser struct { + io.ReadCloser + limit int64 + bytesRead int64 +} + +func (l *limitReadCloser) Read(p []byte) (int, error) { + n, err := l.ReadCloser.Read(p) + l.bytesRead += int64(n) + if l.bytesRead > l.limit { + return 0, fmt.Errorf("reached read limit of %d bytes after reading %d bytes", l.limit, l.bytesRead) + } + return n, err +} diff --git a/examples/car-file-fetcher/main_test.go b/examples/car-file-fetcher/main_test.go new file mode 100644 index 000000000..59f1945fb --- /dev/null +++ b/examples/car-file-fetcher/main_test.go @@ -0,0 +1,38 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +const ( + HelloWorldCID = "bafkreifzjut3te2nhyekklss27nh3k72ysco7y32koao5eei66wof36n5e" +) + +func TestErrorOnInvalidContent(t *testing.T) { + rs := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("wrong data")) + })) + t.Cleanup(rs.Close) + + err := fetch(rs.URL, "/ipfs/"+HelloWorldCID, "hello.txt", "", 0) + require.Error(t, err) +} + +func TestSuccessOnValidContent(t *testing.T) { + data, err := os.ReadFile("./hello.car") + require.NoError(t, err) + + rs := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write(data) + })) + t.Cleanup(rs.Close) + + err = fetch(rs.URL, "/ipfs/"+HelloWorldCID, filepath.Join(t.TempDir(), "hello.txt"), "", 0) + require.NoError(t, err) +} diff --git a/examples/go.mod b/examples/go.mod index 4e935846c..0c3f025e6 100644 --- a/examples/go.mod +++ b/examples/go.mod @@ -3,7 +3,7 @@ module github.com/ipfs/boxo/examples go 1.21 require ( - github.com/ipfs/boxo v0.13.1 + github.com/ipfs/boxo v0.18.0 github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0