From a0f34b16ddc8151fd0ba8ab9674db25354232495 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Sat, 28 Oct 2023 05:34:14 +0200 Subject: [PATCH] feat: built-in content blocking based on IPIP-383 (#10161) Fixes #8492 This introduces "nopfs" as a preloaded plugin into Kubo with support for denylists from https://github.com/ipfs/specs/pull/383 It automatically makes Kubo watch *.deny files found in: - /etc/ipfs/denylists - $XDG_CONFIG_HOME/ipfs/denylists - $IPFS_PATH/denylists * test: Gateway.NoFetch and GatewayOverLibp2p adds missing tests for "no fetch" gateways one can expose, in both cases the offline mode is done by passing custom blockservice/exchange into path resolver, which means global path resolver that has nopfs intercept is not used, and the content blocking does not happen on these gateways. * fix: use offline path resolvers where appropriate this fixes the problem described in https://github.com/ipfs/kubo/pull/10161#issuecomment-1782175955 by adding explicit offline path resolvers that are backed by offline exchange, and using them in NoFetch gateways instead of the default online ones --------- Co-authored-by: Henrique Dias Co-authored-by: Marcin Rataj --- README.md | 1 + core/commands/dag/export.go | 16 +- core/core.go | 58 ++-- core/corehttp/gateway.go | 18 +- core/node/core.go | 51 ++-- docs/changelogs/v0.24.md | 9 + docs/content-blocking.md | 73 +++++ docs/environment-variables.md | 6 + docs/examples/kubo-as-a-library/go.mod | 6 +- docs/examples/kubo-as-a-library/go.sum | 10 +- go.mod | 4 +- go.sum | 8 +- plugin/loader/preload.go | 2 + plugin/loader/preload_list | 1 + plugin/plugins/nopfs/nopfs.go | 85 ++++++ test/cli/content_blocking_test.go | 303 +++++++++++++++++++ test/dependencies/go.mod | 2 +- test/dependencies/go.sum | 4 +- test/sharness/t0054-dag-car-import-export.sh | 4 +- 19 files changed, 596 insertions(+), 65 deletions(-) create mode 100644 docs/content-blocking.md create mode 100644 plugin/plugins/nopfs/nopfs.go create mode 100644 test/cli/content_blocking_test.go diff --git a/README.md b/README.md index c261ea699ce..74b53c3ad6e 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ Featureset - [HTTP Kubo RPC API](https://docs.ipfs.tech/reference/kubo/rpc/) (`/api/v0`) to access and control the daemon - [Command Line Interface](https://docs.ipfs.tech/reference/kubo/cli/) based on (`/api/v0`) RPC API - [WebUI](https://github.com/ipfs/ipfs-webui/#readme) to manage the Kubo node +- [Content blocking](/docs/content-blocking.md) support for operators of public nodes ### Other implementations diff --git a/core/commands/dag/export.go b/core/commands/dag/export.go index d46fa6e21bf..d97718d200e 100644 --- a/core/commands/dag/export.go +++ b/core/commands/dag/export.go @@ -14,6 +14,7 @@ import ( cid "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/kubo/core/commands/cmdenv" + "github.com/ipfs/kubo/core/commands/cmdutils" cmds "github.com/ipfs/go-ipfs-cmds" gocar "github.com/ipld/go-car" @@ -21,12 +22,10 @@ import ( ) func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { - c, err := cid.Decode(req.Arguments[0]) + // Accept CID or a content path + p, err := cmdutils.PathOrCidPath(req.Arguments[0]) if err != nil { - return fmt.Errorf( - "unable to parse root specification (currently only bare CIDs are supported): %s", - err, - ) + return err } api, err := cmdenv.GetApi(env, req) @@ -34,6 +33,13 @@ func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment return err } + // Resolve path and confirm the root block is available, fail fast if not + b, err := api.Block().Stat(req.Context, p) + if err != nil { + return err + } + c := b.Path().RootCid() + pipeR, pipeW := io.Pipe() errCh := make(chan error, 2) // we only report the 1st error diff --git a/core/core.go b/core/core.go index c35d3e4457f..c9bec6b8b00 100644 --- a/core/core.go +++ b/core/core.go @@ -76,35 +76,39 @@ type IpfsNode struct { PNetFingerprint libp2p.PNetFingerprint `optional:"true"` // fingerprint of private network // Services - Peerstore pstore.Peerstore `optional:"true"` // storage for other Peer instances - Blockstore bstore.GCBlockstore // the block store (lower level) - Filestore *filestore.Filestore `optional:"true"` // the filestore blockstore - BaseBlocks node.BaseBlocks // the raw blockstore, no filestore wrapping - GCLocker bstore.GCLocker // the locker used to protect the blockstore during gc - Blocks bserv.BlockService // the block service, get/add blocks. - DAG ipld.DAGService // the merkle dag service, get/add objects. - IPLDFetcherFactory fetcher.Factory `name:"ipldFetcher"` // fetcher that paths over the IPLD data model - UnixFSFetcherFactory fetcher.Factory `name:"unixfsFetcher"` // fetcher that interprets UnixFS data - Reporter *metrics.BandwidthCounter `optional:"true"` - Discovery mdns.Service `optional:"true"` - FilesRoot *mfs.Root - RecordValidator record.Validator + Peerstore pstore.Peerstore `optional:"true"` // storage for other Peer instances + Blockstore bstore.GCBlockstore // the block store (lower level) + Filestore *filestore.Filestore `optional:"true"` // the filestore blockstore + BaseBlocks node.BaseBlocks // the raw blockstore, no filestore wrapping + GCLocker bstore.GCLocker // the locker used to protect the blockstore during gc + Blocks bserv.BlockService // the block service, get/add blocks. + DAG ipld.DAGService // the merkle dag service, get/add objects. + IPLDFetcherFactory fetcher.Factory `name:"ipldFetcher"` // fetcher that paths over the IPLD data model + UnixFSFetcherFactory fetcher.Factory `name:"unixfsFetcher"` // fetcher that interprets UnixFS data + OfflineIPLDFetcherFactory fetcher.Factory `name:"offlineIpldFetcher"` // fetcher that paths over the IPLD data model without fetching new blocks + OfflineUnixFSFetcherFactory fetcher.Factory `name:"offlineUnixfsFetcher"` // fetcher that interprets UnixFS data without fetching new blocks + Reporter *metrics.BandwidthCounter `optional:"true"` + Discovery mdns.Service `optional:"true"` + FilesRoot *mfs.Root + RecordValidator record.Validator // Online - PeerHost p2phost.Host `optional:"true"` // the network host (server+client) - Peering *peering.PeeringService `optional:"true"` - Filters *ma.Filters `optional:"true"` - Bootstrapper io.Closer `optional:"true"` // the periodic bootstrapper - Routing irouting.ProvideManyRouter `optional:"true"` // the routing system. recommend ipfs-dht - DNSResolver *madns.Resolver // the DNS resolver - IPLDPathResolver pathresolver.Resolver `name:"ipldPathResolver"` // The IPLD path resolver - UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"` // The UnixFS path resolver - Exchange exchange.Interface // the block exchange + strategy (bitswap) - Namesys namesys.NameSystem // the name system, resolves paths to hashes - Provider provider.System // the value provider system - IpnsRepub *ipnsrp.Republisher `optional:"true"` - GraphExchange graphsync.GraphExchange `optional:"true"` - ResourceManager network.ResourceManager `optional:"true"` + PeerHost p2phost.Host `optional:"true"` // the network host (server+client) + Peering *peering.PeeringService `optional:"true"` + Filters *ma.Filters `optional:"true"` + Bootstrapper io.Closer `optional:"true"` // the periodic bootstrapper + Routing irouting.ProvideManyRouter `optional:"true"` // the routing system. recommend ipfs-dht + DNSResolver *madns.Resolver // the DNS resolver + IPLDPathResolver pathresolver.Resolver `name:"ipldPathResolver"` // The IPLD path resolver + UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"` // The UnixFS path resolver + OfflineIPLDPathResolver pathresolver.Resolver `name:"offlineIpldPathResolver"` // The IPLD path resolver that uses only locally available blocks + OfflineUnixFSPathResolver pathresolver.Resolver `name:"offlineUnixFSPathResolver"` // The UnixFS path resolver that uses only locally available blocks + Exchange exchange.Interface // the block exchange + strategy (bitswap) + Namesys namesys.NameSystem // the name system, resolves paths to hashes + Provider provider.System // the value provider system + IpnsRepub *ipnsrp.Republisher `optional:"true"` + GraphExchange graphsync.GraphExchange `optional:"true"` + ResourceManager network.ResourceManager `optional:"true"` PubSub *pubsub.PubSub `optional:"true"` PSRouter *psrouter.PubsubValueStore `optional:"true"` diff --git a/core/corehttp/gateway.go b/core/corehttp/gateway.go index ec9abef8e2a..3e0380d5a3c 100644 --- a/core/corehttp/gateway.go +++ b/core/corehttp/gateway.go @@ -81,7 +81,11 @@ func Libp2pGatewayOption() ServeOption { return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { bserv := blockservice.New(n.Blocks.Blockstore(), offline.Exchange(n.Blocks.Blockstore())) - backend, err := gateway.NewBlocksBackend(bserv) + backend, err := gateway.NewBlocksBackend(bserv, + // GatewayOverLibp2p only returns things that are in local blockstore + // (same as Gateway.NoFetch=true), we have to pass offline path resolver + gateway.WithResolver(n.OfflineUnixFSPathResolver), + ) if err != nil { return nil, err } @@ -111,6 +115,8 @@ func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) { bserv := n.Blocks var vsRouting routing.ValueStore = n.Routing nsys := n.Namesys + pathResolver := n.UnixFSPathResolver + if cfg.Gateway.NoFetch { bserv = blockservice.New(bserv.Blockstore(), offline.Exchange(bserv.Blockstore())) @@ -130,9 +136,17 @@ func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) { if err != nil { return nil, fmt.Errorf("error constructing namesys: %w", err) } + + // Gateway.NoFetch=true requires offline path resolver + // to avoid fetching missing blocks during path traversal + pathResolver = n.OfflineUnixFSPathResolver } - backend, err := gateway.NewBlocksBackend(bserv, gateway.WithValueStore(vsRouting), gateway.WithNameSystem(nsys)) + backend, err := gateway.NewBlocksBackend(bserv, + gateway.WithValueStore(vsRouting), + gateway.WithNameSystem(nsys), + gateway.WithResolver(pathResolver), + ) if err != nil { return nil, err } diff --git a/core/node/core.go b/core/node/core.go index d2d2c63d749..9a2035a4c8c 100644 --- a/core/node/core.go +++ b/core/node/core.go @@ -7,6 +7,7 @@ import ( "github.com/ipfs/boxo/blockservice" blockstore "github.com/ipfs/boxo/blockstore" exchange "github.com/ipfs/boxo/exchange" + offline "github.com/ipfs/boxo/exchange/offline" "github.com/ipfs/boxo/fetcher" bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice" "github.com/ipfs/boxo/filestore" @@ -21,9 +22,6 @@ import ( format "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-unixfsnode" dagpb "github.com/ipld/go-codec-dagpb" - "github.com/ipld/go-ipld-prime" - basicnode "github.com/ipld/go-ipld-prime/node/basic" - "github.com/ipld/go-ipld-prime/schema" "go.uber.org/fx" "github.com/ipfs/kubo/core/node/helpers" @@ -87,43 +85,58 @@ func (s *syncDagService) Session(ctx context.Context) format.NodeGetter { // FetchersOut allows injection of fetchers. type FetchersOut struct { fx.Out - IPLDFetcher fetcher.Factory `name:"ipldFetcher"` - UnixfsFetcher fetcher.Factory `name:"unixfsFetcher"` + IPLDFetcher fetcher.Factory `name:"ipldFetcher"` + UnixfsFetcher fetcher.Factory `name:"unixfsFetcher"` + OfflineIPLDFetcher fetcher.Factory `name:"offlineIpldFetcher"` + OfflineUnixfsFetcher fetcher.Factory `name:"offlineUnixfsFetcher"` } // FetchersIn allows using fetchers for other dependencies. type FetchersIn struct { fx.In - IPLDFetcher fetcher.Factory `name:"ipldFetcher"` - UnixfsFetcher fetcher.Factory `name:"unixfsFetcher"` + IPLDFetcher fetcher.Factory `name:"ipldFetcher"` + UnixfsFetcher fetcher.Factory `name:"unixfsFetcher"` + OfflineIPLDFetcher fetcher.Factory `name:"offlineIpldFetcher"` + OfflineUnixfsFetcher fetcher.Factory `name:"offlineUnixfsFetcher"` } // FetcherConfig returns a fetcher config that can build new fetcher instances func FetcherConfig(bs blockservice.BlockService) FetchersOut { ipldFetcher := bsfetcher.NewFetcherConfig(bs) - ipldFetcher.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { - if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { - return tlnkNd.LinkTargetNodePrototype(), nil - } - return basicnode.Prototype.Any, nil - }) - + ipldFetcher.PrototypeChooser = dagpb.AddSupportToChooser(bsfetcher.DefaultPrototypeChooser) unixFSFetcher := ipldFetcher.WithReifier(unixfsnode.Reify) - return FetchersOut{IPLDFetcher: ipldFetcher, UnixfsFetcher: unixFSFetcher} + + // Construct offline versions which we can safely use in contexts where + // path resolution should not fetch new blocks via exchange. + offlineBs := blockservice.New(bs.Blockstore(), offline.Exchange(bs.Blockstore())) + offlineIpldFetcher := bsfetcher.NewFetcherConfig(offlineBs) + offlineIpldFetcher.PrototypeChooser = dagpb.AddSupportToChooser(bsfetcher.DefaultPrototypeChooser) + offlineUnixFSFetcher := offlineIpldFetcher.WithReifier(unixfsnode.Reify) + + return FetchersOut{ + IPLDFetcher: ipldFetcher, + UnixfsFetcher: unixFSFetcher, + OfflineIPLDFetcher: offlineIpldFetcher, + OfflineUnixfsFetcher: offlineUnixFSFetcher, + } } // PathResolversOut allows injection of path resolvers type PathResolversOut struct { fx.Out - IPLDPathResolver pathresolver.Resolver `name:"ipldPathResolver"` - UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"` + IPLDPathResolver pathresolver.Resolver `name:"ipldPathResolver"` + UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"` + OfflineIPLDPathResolver pathresolver.Resolver `name:"offlineIpldPathResolver"` + OfflineUnixFSPathResolver pathresolver.Resolver `name:"offlineUnixFSPathResolver"` } // PathResolverConfig creates path resolvers with the given fetchers. func PathResolverConfig(fetchers FetchersIn) PathResolversOut { return PathResolversOut{ - IPLDPathResolver: pathresolver.NewBasicResolver(fetchers.IPLDFetcher), - UnixFSPathResolver: pathresolver.NewBasicResolver(fetchers.UnixfsFetcher), + IPLDPathResolver: pathresolver.NewBasicResolver(fetchers.IPLDFetcher), + UnixFSPathResolver: pathresolver.NewBasicResolver(fetchers.UnixfsFetcher), + OfflineIPLDPathResolver: pathresolver.NewBasicResolver(fetchers.OfflineIPLDFetcher), + OfflineUnixFSPathResolver: pathresolver.NewBasicResolver(fetchers.OfflineUnixfsFetcher), } } diff --git a/docs/changelogs/v0.24.md b/docs/changelogs/v0.24.md index ad337483820..29d88d9cbd0 100644 --- a/docs/changelogs/v0.24.md +++ b/docs/changelogs/v0.24.md @@ -6,6 +6,7 @@ - [Overview](#overview) - [🔦 Highlights](#-highlights) + - [Support for content blocking](#support-for-content-blocking) - [Gateway: the root of the CARs are no longer meaningful](#gateway-the-root-of-the-cars-are-no-longer-meaningful) - [IPNS: improved publishing defaults](#ipns-improved-publishing-defaults) - [IPNS: record TTL is used for caching](#ipns-record-ttl-is-used-for-caching) @@ -16,6 +17,14 @@ ### 🔦 Highlights +#### Support for content blocking + +This Kubo release ships with built-in content-blocking subsystem [announced earlier this year](https://blog.ipfs.tech/2023-content-blocking-for-the-ipfs-stack/). +Content blocking is an opt-in decision made by the operator of `ipfs daemon`. +The official build does not ship with any denylists. + +Learn more at [`/docs/content-blocking.md`](https://github.com/ipfs/kubo/blob/master/docs/content-blocking.md) + #### Gateway: the root of the CARs are no longer meaningful When requesting a CAR from the gateway, the root of the CAR might no longer be diff --git a/docs/content-blocking.md b/docs/content-blocking.md new file mode 100644 index 00000000000..ebc84bba3ea --- /dev/null +++ b/docs/content-blocking.md @@ -0,0 +1,73 @@ +

+
+ content blocking logo +
+ Content Blocking in Kubo +
+

+ +Kubo ships with built-in support for denylist format from [IPIP-383](https://github.com/ipfs/specs/pull/383). + +## Default behavior + +Official Kubo build does not ship with any denylists enabled by default. + +Content blocking is an opt-in decision made by the operator of `ipfs daemon`. + +## How to enable blocking + +Place a `*.deny` file in one of directories: + +- `$IPFS_PATH/denylists/` (`$HOME/.ipfs/denylists/` if `IPFS_PATH` is not set) +- `$XDG_CONFIG_HOME/ipfs/denylists/` (`$HOME/.config/ipfs/denylists/` if `XDG_CONFIG_HOME` is not set) +- `/etc/ipfs/denylists/` (global) + +Files need to be present before starting the `ipfs daemon` in order to be watched for updates. + +If a new denylist file is added, `ipfs daemon` needs to be restarted. + +CLI and Gateway users will receive errors in response to request impacted by a blocklist: + +``` +Error: /ipfs/QmQvjk82hPkSaZsyJ8vNER5cmzKW7HyGX5XVusK7EAenCN is blocked and cannot be provided +``` + +End user is not informed about the exact reason, see [How to +debug](#how-to-debug) if you need to find out which line of which denylist +caused the request to be blocked. + +## Denylist file format + +[NOpfs](https://github.com/ipfs-shipyard/nopfs) supports the format from [IPIP-383](https://github.com/ipfs/specs/pull/383). + +Clear-text rules are simple: just put content paths to block, one per line. +Paths with unicode and whitespace need to be percend-encoded: + +``` +/ipfs/QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR +/ipfs/bafybeihfg3d7rdltd43u3tfvncx7n5loqofbsobojcadtmokrljfthuc7y/927%20-%20Standards/927%20-%20Standards.png +``` + +Sensitive content paths can be double-hashed to block without revealing them. +Double-hashed list example: https://badbits.dwebops.pub/badbits.deny + +See [IPIP-383](https://github.com/ipfs/specs/pull/383) for detailed format specification and more examples. + +## How to suspend blocking without removing denylists + +Set `IPFS_CONTENT_BLOCKING_DISABLE` environment variable to `true` and restart the daemon. + + +## How to debug + +Debug logging of `nopfs` subsystem can be enabled with `GOLOG_LOG_LEVEL="nopfs=debug"` + +All block events are logged as warnings on a separate level named `nopfs-blocks`. + +To only log requests for blocked content set `GOLOG_LOG_LEVEL="nopfs-blocks=warn"`: + +``` +WARN (...) QmRFniDxwxoG2n4AcnGhRdjqDjCM5YeUcBE75K8WXmioH3: blocked (test.deny:9) +``` + + diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 4113be09b82..f0f6b3f183a 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -131,6 +131,12 @@ The above will replace implicit HTTP routers with single one, allowing for inspection/debug of HTTP requests sent by Kubo via `while true ; do nc -l 7423; done` or more advanced tools like [mitmproxy](https://docs.mitmproxy.org/stable/#mitmproxy). + +## `IPFS_CONTENT_BLOCKING_DISABLE` + +Disables the content-blocking subsystem. No denylists will be watched and no +content will be blocked. + ## `LIBP2P_TCP_REUSEPORT` Kubo tries to reuse the same source port for all connections to improve NAT diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 8c2edff6d89..dfb7848b036 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.20 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd + github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.31.0 github.com/multiformats/go-multiaddr v0.11.0 @@ -40,6 +40,7 @@ require ( github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 // indirect github.com/flynn/noise v1.0.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/gabriel-vasile/mimetype v1.4.1 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -60,6 +61,8 @@ require ( github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/huin/goupnp v1.2.0 // indirect + github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c // indirect + github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c // indirect github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.2.0 // indirect @@ -194,5 +197,6 @@ require ( google.golang.org/grpc v1.55.0 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/square/go-jose.v2 v2.5.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.2.1 // indirect ) diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 8fab68ddea7..49dbacfee11 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -163,6 +163,7 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gabriel-vasile/mimetype v1.4.1 h1:TRWk7se+TOjCYgRth7+1/OYLNiRNIotknkFtf/dnN7Q= github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -298,10 +299,14 @@ github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c h1:17FO7HnKiFhO7iadu3zCgII+EblpdRmJt5qg9FqQo8Y= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c h1:7UynTbtdlt+w08ggb1UGLGaGjp1mMaZhoTZSctpn5Ak= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd h1:CWz2mhz+cmkLRlKgQYlKXkDtx4oWYkCorSSF4ZWkH3o= -github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk= +github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b h1:0Qi1EhB82x3SZJOBieG51BtPvjU3kSy4H8OpMnxKvtk= +github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b/go.mod h1:pu8HsZvuyYeYJsqtLDCoYSvy8rHj6vI3dlh8P0f83Zs= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= @@ -1001,6 +1006,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= diff --git a/go.mod b/go.mod index 68db957cd22..c94a251cba4 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,9 @@ require ( github.com/fsnotify/fsnotify v1.6.0 github.com/google/uuid v1.3.1 github.com/hashicorp/go-multierror v1.1.1 - github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd + github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c + github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c + github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index 5b612c73aa3..928433eaaa2 100644 --- a/go.sum +++ b/go.sum @@ -333,10 +333,14 @@ github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c h1:17FO7HnKiFhO7iadu3zCgII+EblpdRmJt5qg9FqQo8Y= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c h1:7UynTbtdlt+w08ggb1UGLGaGjp1mMaZhoTZSctpn5Ak= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd h1:CWz2mhz+cmkLRlKgQYlKXkDtx4oWYkCorSSF4ZWkH3o= -github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk= +github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b h1:0Qi1EhB82x3SZJOBieG51BtPvjU3kSy4H8OpMnxKvtk= +github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b/go.mod h1:pu8HsZvuyYeYJsqtLDCoYSvy8rHj6vI3dlh8P0f83Zs= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= diff --git a/plugin/loader/preload.go b/plugin/loader/preload.go index 4304862119d..2ad84e59489 100644 --- a/plugin/loader/preload.go +++ b/plugin/loader/preload.go @@ -7,6 +7,7 @@ import ( pluginfxtest "github.com/ipfs/kubo/plugin/plugins/fxtest" pluginipldgit "github.com/ipfs/kubo/plugin/plugins/git" pluginlevelds "github.com/ipfs/kubo/plugin/plugins/levelds" + pluginnopfs "github.com/ipfs/kubo/plugin/plugins/nopfs" pluginpeerlog "github.com/ipfs/kubo/plugin/plugins/peerlog" ) @@ -22,4 +23,5 @@ func init() { Preload(pluginlevelds.Plugins...) Preload(pluginpeerlog.Plugins...) Preload(pluginfxtest.Plugins...) + Preload(pluginnopfs.Plugins...) } diff --git a/plugin/loader/preload_list b/plugin/loader/preload_list index c18ea80ccd5..462a3f39337 100644 --- a/plugin/loader/preload_list +++ b/plugin/loader/preload_list @@ -11,3 +11,4 @@ flatfs github.com/ipfs/kubo/plugin/plugins/flatfs * levelds github.com/ipfs/kubo/plugin/plugins/levelds * peerlog github.com/ipfs/kubo/plugin/plugins/peerlog * fxtest github.com/ipfs/kubo/plugin/plugins/fxtest * +nopfs github.com/ipfs/kubo/plugin/plugins/nopfs * \ No newline at end of file diff --git a/plugin/plugins/nopfs/nopfs.go b/plugin/plugins/nopfs/nopfs.go new file mode 100644 index 00000000000..64350830f94 --- /dev/null +++ b/plugin/plugins/nopfs/nopfs.go @@ -0,0 +1,85 @@ +package nopfs + +import ( + "os" + "path/filepath" + + "github.com/ipfs-shipyard/nopfs" + "github.com/ipfs-shipyard/nopfs/ipfs" + "github.com/ipfs/kubo/config" + "github.com/ipfs/kubo/core" + "github.com/ipfs/kubo/core/node" + "github.com/ipfs/kubo/plugin" + "go.uber.org/fx" +) + +// Plugins sets the list of plugins to be loaded. +var Plugins = []plugin.Plugin{ + &nopfsPlugin{}, +} + +// fxtestPlugin is used for testing the fx plugin. +// It merely adds an fx option that logs a debug statement, so we can verify that it works in tests. +type nopfsPlugin struct{} + +var _ plugin.PluginFx = (*nopfsPlugin)(nil) + +func (p *nopfsPlugin) Name() string { + return "nopfs" +} + +func (p *nopfsPlugin) Version() string { + return "0.0.10" +} + +func (p *nopfsPlugin) Init(env *plugin.Environment) error { + return nil +} + +// MakeBlocker is a factory for the blocker so that it can be provided with Fx. +func MakeBlocker() (*nopfs.Blocker, error) { + ipfsPath, err := config.PathRoot() + if err != nil { + return nil, err + } + + defaultFiles, err := nopfs.GetDenylistFiles() + if err != nil { + return nil, err + } + + kuboFiles, err := nopfs.GetDenylistFilesInDir(filepath.Join(ipfsPath, "denylists")) + if err != nil { + return nil, err + } + + files := append(defaultFiles, kuboFiles...) + + return nopfs.NewBlocker(files) +} + +// PathResolvers returns wrapped PathResolvers for Kubo. +func PathResolvers(fetchers node.FetchersIn, blocker *nopfs.Blocker) node.PathResolversOut { + res := node.PathResolverConfig(fetchers) + return node.PathResolversOut{ + IPLDPathResolver: ipfs.WrapResolver(res.IPLDPathResolver, blocker), + UnixFSPathResolver: ipfs.WrapResolver(res.UnixFSPathResolver, blocker), + OfflineIPLDPathResolver: ipfs.WrapResolver(res.OfflineIPLDPathResolver, blocker), + OfflineUnixFSPathResolver: ipfs.WrapResolver(res.OfflineUnixFSPathResolver, blocker), + } +} + +func (p *nopfsPlugin) Options(info core.FXNodeInfo) ([]fx.Option, error) { + if os.Getenv("IPFS_CONTENT_BLOCKING_DISABLE") != "" { + return info.FXOptions, nil + } + + opts := append( + info.FXOptions, + fx.Provide(MakeBlocker), + fx.Decorate(ipfs.WrapBlockService), + fx.Decorate(ipfs.WrapNameSystem), + fx.Decorate(PathResolvers), + ) + return opts, nil +} diff --git a/test/cli/content_blocking_test.go b/test/cli/content_blocking_test.go new file mode 100644 index 00000000000..ddb7c951e88 --- /dev/null +++ b/test/cli/content_blocking_test.go @@ -0,0 +1,303 @@ +package cli + +import ( + "context" + "fmt" + "io" + "log" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/ipfs/kubo/test/cli/harness" + "github.com/libp2p/go-libp2p" + "github.com/libp2p/go-libp2p/core/peer" + libp2phttp "github.com/libp2p/go-libp2p/p2p/http" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestContentBlocking(t *testing.T) { + // NOTE: we can't run this with t.Parallel() because we set IPFS_NS_MAP + // and running in parallel could impact other tests + + const blockedMsg = "blocked and cannot be provided" + const statusExpl = "specific HTTP error code is expected" + const bodyExpl = "Error message informing about content block is expected" + + h := harness.NewT(t) + + // Init IPFS_PATH + node := h.NewNode().Init("--empty-repo", "--profile=test") + + // Create CIDs we use in test + h.WriteFile("blocked-dir/subdir/indirectly-blocked-file.txt", "indirectly blocked file content") + parentDirCID := node.IPFS("add", "--raw-leaves", "-Q", "-r", filepath.Join(h.Dir, "blocked-dir")).Stdout.Trimmed() + + h.WriteFile("directly-blocked-file.txt", "directly blocked file content") + blockedCID := node.IPFS("add", "--raw-leaves", "-Q", filepath.Join(h.Dir, "directly-blocked-file.txt")).Stdout.Trimmed() + + h.WriteFile("not-blocked-file.txt", "not blocked file content") + allowedCID := node.IPFS("add", "--raw-leaves", "-Q", filepath.Join(h.Dir, "not-blocked-file.txt")).Stdout.Trimmed() + + // Create denylist at $IPFS_PATH/denylists/test.deny + denylistTmp := h.WriteToTemp("name: test list\n---\n" + + "//QmX9dhRcQcKUw3Ws8485T5a9dtjrSCQaUAHnG4iK9i4ceM\n" + // Double hash (sha256) CID block: base58btc(sha256-multihash(QmVTF1yEejXd9iMgoRTFDxBv7HAz9kuZcQNBzHrceuK9HR)) + "//gW813G35CnLsy7gRYYHuf63hrz71U1xoLFDVeV7actx6oX\n" + // Double hash (blake3) Path block under blake3 root CID: base58btc(blake3-multihash(gW7Nhu4HrfDtphEivm3Z9NNE7gpdh5Tga8g6JNZc1S8E47/path)) + "//8526ba05eec55e28f8db5974cc891d0d92c8af69d386fc6464f1e9f372caf549\n" + // Legacy CID double-hash block: sha256(bafkqahtcnrxwg23fmqqgi33vmjwgk2dbonuca3dfm5qwg6jamnuwicq/) + "//e5b7d2ce2594e2e09901596d8e1f29fa249b74c8c9e32ea01eda5111e4d33f07\n" + // Legacy Path double-hash block: sha256(bafyaagyscufaqalqaacauaqiaejao43vmjygc5didacauaqiae/subpath) + "/ipfs/" + blockedCID + "\n" + // block specific CID + "/ipfs/" + parentDirCID + "/subdir*\n" + // block only specific subpath + "/ipns/blocked-cid.example.com\n" + + "/ipns/blocked-dnslink.example.com\n") + + if err := os.MkdirAll(filepath.Join(node.Dir, "denylists"), 0o777); err != nil { + log.Panicf("failed to create denylists dir: %s", err.Error()) + } + if err := os.Rename(denylistTmp, filepath.Join(node.Dir, "denylists", "test.deny")); err != nil { + log.Panicf("failed to create test denylist: %s", err.Error()) + } + + // Add two entries to namesys resolution cache + // /ipns/blocked-cid.example.com point at a blocked CID (to confirm blocking impacts /ipns resolution) + // /ipns/blocked-dnslink.example.com with safe CID (to test blocking of /ipns/ paths) + os.Setenv("IPFS_NS_MAP", "blocked-cid.example.com:/ipfs/"+blockedCID+",blocked-dnslink.example.com/ipns/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") + defer os.Unsetenv("IPFS_NS_MAP") + + // Enable GatewayOverLibp2p as we want to test denylist there too + node.IPFS("config", "--json", "Experimental.GatewayOverLibp2p", "true") + + // Start daemon, it should pick up denylist from $IPFS_PATH/denylists/test.deny + node.StartDaemon() // we need online mode for GatewayOverLibp2p tests + client := node.GatewayClient() + + // First, confirm gateway works + t.Run("Gateway Allows CID that is not blocked", func(t *testing.T) { + t.Parallel() + resp := client.Get("/ipfs/" + allowedCID) + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, "not blocked file content", resp.Body) + }) + + // Then, does the most basic blocking case work? + t.Run("Gateway Denies directly blocked CID", func(t *testing.T) { + t.Parallel() + resp := client.Get("/ipfs/" + blockedCID) + assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) + assert.NotEqual(t, "directly blocked file content", resp.Body) + assert.Contains(t, resp.Body, blockedMsg, bodyExpl) + }) + + // Confirm parent of blocked subpath is not blocked + t.Run("Gateway Allows parent Path that is not blocked", func(t *testing.T) { + t.Parallel() + resp := client.Get("/ipfs/" + parentDirCID) + assert.Equal(t, http.StatusOK, resp.StatusCode) + }) + + // Ok, now the full list of test cases we want to cover in both CLI and Gateway + testCases := []struct { + name string + path string + }{ + { + name: "directly blocked CID", + path: "/ipfs/" + blockedCID, + }, + { + name: "indirectly blocked file (on a blocked subpath)", + path: "/ipfs/" + parentDirCID + "/subdir/indirectly-blocked-file.txt", + }, + { + name: "/ipns path that resolves to a blocked CID", + path: "/ipns/blocked-cid.example.com", + }, + { + name: "/ipns Path that is blocked by DNSLink name", + path: "/ipns/blocked-dnslink.example.com", + }, + { + name: "double-hash CID block (sha256-multihash)", + path: "/ipfs/QmVTF1yEejXd9iMgoRTFDxBv7HAz9kuZcQNBzHrceuK9HR", + }, + { + name: "double-hash Path block (blake3-multihash)", + path: "/ipfs/bafyb4ieqht3b2rssdmc7sjv2cy2gfdilxkfh7623nvndziyqnawkmo266a/path", + }, + { + name: "legacy CID double-hash block (sha256)", + path: "/ipfs/bafkqahtcnrxwg23fmqqgi33vmjwgk2dbonuca3dfm5qwg6jamnuwicq", + }, + + { + name: "legacy Path double-hash block (sha256)", + path: "/ipfs/bafyaagyscufaqalqaacauaqiaejao43vmjygc5didacauaqiae/subpath", + }, + } + + // Which specific cliCmds we test against testCases + cliCmds := [][]string{ + {"block", "get"}, + {"block", "stat"}, + {"dag", "get"}, + {"dag", "export"}, + {"dag", "stat"}, + {"cat"}, + {"ls"}, + {"get"}, + {"refs"}, + } + + expectedMsg := blockedMsg + for _, testCase := range testCases { + + // Confirm that denylist is active for every command in 'cliCmds' x 'testCases' + for _, cmd := range cliCmds { + cmd := cmd + cliTestName := fmt.Sprintf("CLI '%s' denies %s", strings.Join(cmd, " "), testCase.name) + t.Run(cliTestName, func(t *testing.T) { + t.Parallel() + args := append(cmd, testCase.path) + errMsg := node.RunIPFS(args...).Stderr.Trimmed() + if !strings.Contains(errMsg, expectedMsg) { + t.Errorf("Expected STDERR error message %q, but got: %q", expectedMsg, errMsg) + } + }) + } + + // Confirm that denylist is active for every content path in 'testCases' + gwTestName := fmt.Sprintf("Gateway denies %s", testCase.name) + t.Run(gwTestName, func(t *testing.T) { + resp := client.Get(testCase.path) + assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) + assert.Contains(t, resp.Body, blockedMsg, bodyExpl) + }) + + } + + // Extra edge cases on subdomain gateway + + t.Run("Gateway Denies /ipns Path that is blocked by DNSLink name (subdomain redirect)", func(t *testing.T) { + t.Parallel() + + gwURL, _ := url.Parse(node.GatewayURL()) + resp := client.Get("/ipns/blocked-dnslink.example.com", func(r *http.Request) { + r.Host = "localhost:" + gwURL.Port() + }) + + assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) + assert.Contains(t, resp.Body, blockedMsg, bodyExpl) + }) + + t.Run("Gateway Denies /ipns Path that is blocked by DNSLink name (subdomain, no TLS)", func(t *testing.T) { + t.Parallel() + + gwURL, _ := url.Parse(node.GatewayURL()) + resp := client.Get("/", func(r *http.Request) { + r.Host = "blocked-dnslink.example.com.ipns.localhost:" + gwURL.Port() + }) + + assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) + assert.Contains(t, resp.Body, blockedMsg, bodyExpl) + }) + + t.Run("Gateway Denies /ipns Path that is blocked by DNSLink name (subdomain, inlined for TLS)", func(t *testing.T) { + t.Parallel() + + gwURL, _ := url.Parse(node.GatewayURL()) + resp := client.Get("/", func(r *http.Request) { + // Inlined DNSLink to fit in single DNS label for TLS interop: + // https://specs.ipfs.tech/http-gateways/subdomain-gateway/#host-request-header + r.Host = "blocked--dnslink-example-com.ipns.localhost:" + gwURL.Port() + }) + + assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) + assert.Contains(t, resp.Body, blockedMsg, bodyExpl) + }) + + // We need to confirm denylist is active when gateway is run in NoFetch + // mode (which usually swaps blockservice to a read-only one, and that swap + // may cause denylists to not be applied, as it is a separate code path) + t.Run("GatewayNoFetch", func(t *testing.T) { + // NOTE: we don't run this in parallel, as it requires restart with different config + + // Switch gateway to NoFetch mode + node.StopDaemon() + node.IPFS("config", "--json", "Gateway.NoFetch", "true") + node.StartDaemon() + + // update client, as the port of test node might've changed after restart + client = node.GatewayClient() + + // First, confirm gateway works + t.Run("Allows CID that is not blocked", func(t *testing.T) { + resp := client.Get("/ipfs/" + allowedCID) + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, "not blocked file content", resp.Body) + }) + + // Then, does the most basic blocking case work? + t.Run("Denies directly blocked CID", func(t *testing.T) { + resp := client.Get("/ipfs/" + blockedCID) + assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) + assert.NotEqual(t, "directly blocked file content", resp.Body) + assert.Contains(t, resp.Body, blockedMsg, bodyExpl) + }) + + // Restore default + node.StopDaemon() + node.IPFS("config", "--json", "Gateway.NoFetch", "false") + node.StartDaemon() + client = node.GatewayClient() + }) + + // We need to confirm denylist is active on the + // trustless gateway exposed over libp2p + // when Experimental.GatewayOverLibp2p=true + // (https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#http-gateway-over-libp2p) + // NOTE: this type fo gateway is hardcoded to be NoFetch: it does not fetch + // data that is not in local store, so we only need to run it once: a + // simple smoke-test for allowed CID and blockedCID. + t.Run("GatewayOverLibp2p", func(t *testing.T) { + t.Parallel() + + // Create libp2p client that connects to our node over + // /http1.1 and then talks gateway semantics over the /ipfs/gateway sub-protocol + clientHost, err := libp2p.New(libp2p.NoListenAddrs) + require.NoError(t, err) + err = clientHost.Connect(context.Background(), peer.AddrInfo{ + ID: node.PeerID(), + Addrs: node.SwarmAddrs(), + }) + require.NoError(t, err) + + libp2pClient, err := (&libp2phttp.Host{StreamHost: clientHost}).NamespacedClient("/ipfs/gateway", peer.AddrInfo{ID: node.PeerID()}) + require.NoError(t, err) + + t.Run("Serves Allowed CID", func(t *testing.T) { + t.Parallel() + resp, err := libp2pClient.Get(fmt.Sprintf("/ipfs/%s?format=raw", allowedCID)) + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusOK, resp.StatusCode) + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, string(body), "not blocked file content", bodyExpl) + }) + + t.Run("Denies Blocked CID", func(t *testing.T) { + t.Parallel() + resp, err := libp2pClient.Get(fmt.Sprintf("/ipfs/%s?format=raw", blockedCID)) + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + assert.NotEqual(t, string(body), "directly blocked file content") + assert.Contains(t, string(body), blockedMsg, bodyExpl) + }) + }) +} diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index fd8c9458568..cbc577f9b40 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -7,7 +7,7 @@ replace github.com/ipfs/kubo => ../../ require ( github.com/Kubuxu/gocovmerge v0.0.0-20161216165753-7ecaa51963cd github.com/golangci/golangci-lint v1.54.1 - github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd + github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-cidutil v0.1.0 github.com/ipfs/go-datastore v0.6.0 diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index d37a47fe0c1..90a00aebcca 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -398,8 +398,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd h1:CWz2mhz+cmkLRlKgQYlKXkDtx4oWYkCorSSF4ZWkH3o= -github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk= +github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b h1:0Qi1EhB82x3SZJOBieG51BtPvjU3kSy4H8OpMnxKvtk= +github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b/go.mod h1:pu8HsZvuyYeYJsqtLDCoYSvy8rHj6vI3dlh8P0f83Zs= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs= diff --git a/test/sharness/t0054-dag-car-import-export.sh b/test/sharness/t0054-dag-car-import-export.sh index 0482f24d92d..e277cc46687 100755 --- a/test/sharness/t0054-dag-car-import-export.sh +++ b/test/sharness/t0054-dag-car-import-export.sh @@ -178,13 +178,11 @@ test_expect_success "basic offline export of 'getting started' dag works" ' ipfs dag export "$HASH_WELCOME_DOCS" >/dev/null ' - -echo "Error: block was not found locally (offline): ipld: could not find QmYwAPJXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX (currently offline, perhaps retry after attaching to the network)" > offline_fetch_error_expected test_expect_success "basic offline export of nonexistent cid" ' ! ipfs dag export QmYwAPJXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX 2> offline_fetch_error_actual >/dev/null ' test_expect_success "correct error" ' - test_cmp_sorted offline_fetch_error_expected offline_fetch_error_actual + test_should_contain "Error: block was not found locally (offline): ipld: could not find QmYwAPJXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" offline_fetch_error_actual ' cat >multiroot_import_json_stats_expected <