Skip to content

Commit

Permalink
refactor: switch gateway code to new API from go-libipfs
Browse files Browse the repository at this point in the history
Co-Authored-By: Marcin Rataj <lidel@lidel.org>
Co-Authored-By: Henrique Dias <hacdias@gmail.com>
  • Loading branch information
3 people committed Mar 21, 2023
1 parent e0b08ed commit cb2acb9
Show file tree
Hide file tree
Showing 13 changed files with 278 additions and 144 deletions.
154 changes: 98 additions & 56 deletions core/corehttp/gateway.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,20 +2,25 @@ package corehttp

import (
"context"
"errors"
"fmt"
"io"
"net"
"net/http"

cid "github.com/ipfs/go-cid"
"github.com/ipfs/go-libipfs/blocks"
"github.com/ipfs/go-blockservice"
"github.com/ipfs/go-cid"
offline "github.com/ipfs/go-ipfs-exchange-offline"
offlineroute "github.com/ipfs/go-ipfs-routing/offline"
"github.com/ipfs/go-libipfs/files"
"github.com/ipfs/go-libipfs/gateway"
"github.com/ipfs/go-namesys"
iface "github.com/ipfs/interface-go-ipfs-core"
options "github.com/ipfs/interface-go-ipfs-core/options"
nsopts "github.com/ipfs/interface-go-ipfs-core/options/namesys"
"github.com/ipfs/interface-go-ipfs-core/path"
"github.com/ipfs/kubo/core/node"
"github.com/libp2p/go-libp2p/core/routing"

"github.com/ipfs/go-libipfs/gateway"
options "github.com/ipfs/interface-go-ipfs-core/options"
version "github.com/ipfs/kubo"
config "github.com/ipfs/kubo/config"
core "github.com/ipfs/kubo/core"
Expand Down Expand Up @@ -47,7 +52,7 @@ func GatewayOption(writable bool, paths ...string) ServeOption {
Headers: headers,
}

gwAPI, err := newGatewayAPI(n)
gwAPI, err := newGatewayBackend(n)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -95,7 +100,7 @@ func HostnameOption() ServeOption {
return nil, err
}

gwAPI, err := newGatewayAPI(n)
gwAPI, err := newGatewayBackend(n)
if err != nil {
return nil, err
}
Expand All @@ -118,83 +123,120 @@ func VersionOption() ServeOption {
}
}

type gatewayAPI struct {
ns namesys.NameSystem
api iface.CoreAPI
offlineAPI iface.CoreAPI
}

func newGatewayAPI(n *core.IpfsNode) (*gatewayAPI, error) {
func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) {
cfg, err := n.Repo.Config()
if err != nil {
return nil, err
}

api, err := coreapi.NewCoreAPI(n, options.Api.FetchBlocks(!cfg.Gateway.NoFetch))
if err != nil {
return nil, err
bserv := n.Blocks
var vsRouting routing.ValueStore = n.Routing
nsys := n.Namesys
if cfg.Gateway.NoFetch {
bserv = blockservice.New(bserv.Blockstore(), offline.Exchange(bserv.Blockstore()))

cs := cfg.Ipns.ResolveCacheSize
if cs == 0 {
cs = node.DefaultIpnsCacheSize
}
if cs < 0 {
return nil, fmt.Errorf("cannot specify negative resolve cache size")
}

vsRouting = offlineroute.NewOfflineRouter(n.Repo.Datastore(), n.RecordValidator)
nsys, err = namesys.NewNameSystem(vsRouting,
namesys.WithDatastore(n.Repo.Datastore()),
namesys.WithDNSResolver(n.DNSResolver),
namesys.WithCache(cs))
if err != nil {
return nil, fmt.Errorf("error constructing namesys: %w", err)
}
}
offlineAPI, err := api.WithOptions(options.Api.Offline(true))

gw, err := gateway.NewBlocksGateway(bserv, gateway.WithValueStore(vsRouting), gateway.WithNameSystem(nsys))
if err != nil {
return nil, err
}
return &offlineGatewayErrWrapper{gwimpl: gw}, nil
}

type offlineGatewayErrWrapper struct {
gwimpl gateway.IPFSBackend
}

func offlineErrWrap(err error) error {
if errors.Is(err, iface.ErrOffline) {
return fmt.Errorf("%s : %w", err.Error(), gateway.ErrServiceUnavailable)
}
return err
}

return &gatewayAPI{
ns: n.Namesys,
api: api,
offlineAPI: offlineAPI,
}, nil
func (o *offlineGatewayErrWrapper) Get(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, *gateway.GetResponse, error) {
md, n, err := o.gwimpl.Get(ctx, path)
err = offlineErrWrap(err)
return md, n, err
}

func (gw *gatewayAPI) GetUnixFsNode(ctx context.Context, pth path.Resolved) (files.Node, error) {
return gw.api.Unixfs().Get(ctx, pth)
func (o *offlineGatewayErrWrapper) GetRange(ctx context.Context, path gateway.ImmutablePath, ranges ...gateway.GetRange) (gateway.ContentPathMetadata, files.File, error) {
md, n, err := o.gwimpl.GetRange(ctx, path, ranges...)
err = offlineErrWrap(err)
return md, n, err
}

func (gw *gatewayAPI) LsUnixFsDir(ctx context.Context, pth path.Resolved) (<-chan iface.DirEntry, error) {
// Optimization: use Unixfs.Ls without resolving children, but using the
// cumulative DAG size as the file size. This allows for a fast listing
// while keeping a good enough Size field.
return gw.api.Unixfs().Ls(ctx, pth,
options.Unixfs.ResolveChildren(false),
options.Unixfs.UseCumulativeSize(true),
)
func (o *offlineGatewayErrWrapper) GetAll(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) {
md, n, err := o.gwimpl.GetAll(ctx, path)
err = offlineErrWrap(err)
return md, n, err
}

func (gw *gatewayAPI) GetBlock(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
r, err := gw.api.Block().Get(ctx, path.IpfsPath(cid))
if err != nil {
return nil, err
}
func (o *offlineGatewayErrWrapper) GetBlock(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.File, error) {
md, n, err := o.gwimpl.GetBlock(ctx, path)
err = offlineErrWrap(err)
return md, n, err
}

data, err := io.ReadAll(r)
if err != nil {
return nil, err
}
func (o *offlineGatewayErrWrapper) Head(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) {
md, n, err := o.gwimpl.Head(ctx, path)
err = offlineErrWrap(err)
return md, n, err
}

return blocks.NewBlockWithCid(data, cid)
func (o *offlineGatewayErrWrapper) ResolvePath(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, error) {
md, err := o.gwimpl.ResolvePath(ctx, path)
err = offlineErrWrap(err)
return md, err
}

func (gw *gatewayAPI) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) {
return gw.api.Routing().Get(ctx, "/ipns/"+c.String())
func (o *offlineGatewayErrWrapper) GetCAR(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, io.ReadCloser, <-chan error, error) {
md, data, errCh, err := o.gwimpl.GetCAR(ctx, path)
err = offlineErrWrap(err)
return md, data, errCh, err
}

func (gw *gatewayAPI) GetDNSLinkRecord(ctx context.Context, hostname string) (path.Path, error) {
p, err := gw.ns.Resolve(ctx, "/ipns/"+hostname, nsopts.Depth(1))
if err == namesys.ErrResolveRecursion {
err = nil
}
return path.New(p.String()), err
func (o *offlineGatewayErrWrapper) IsCached(ctx context.Context, path path.Path) bool {
return o.gwimpl.IsCached(ctx, path)
}

func (o *offlineGatewayErrWrapper) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) {
rec, err := o.gwimpl.GetIPNSRecord(ctx, c)
err = offlineErrWrap(err)
return rec, err
}

func (gw *gatewayAPI) IsCached(ctx context.Context, pth path.Path) bool {
_, err := gw.offlineAPI.Block().Stat(ctx, pth)
return err == nil
func (o *offlineGatewayErrWrapper) ResolveMutable(ctx context.Context, path path.Path) (gateway.ImmutablePath, error) {
imPath, err := o.gwimpl.ResolveMutable(ctx, path)
err = offlineErrWrap(err)
return imPath, err
}

func (gw *gatewayAPI) ResolvePath(ctx context.Context, pth path.Path) (path.Resolved, error) {
return gw.api.ResolvePath(ctx, pth)
func (o *offlineGatewayErrWrapper) GetDNSLinkRecord(ctx context.Context, s string) (path.Path, error) {
p, err := o.gwimpl.GetDNSLinkRecord(ctx, s)
err = offlineErrWrap(err)
return p, err
}

var _ gateway.IPFSBackend = (*offlineGatewayErrWrapper)(nil)

var defaultPaths = []string{"/ipfs/", "/ipns/", "/api/", "/p2p/"}

var subdomainGatewaySpec = &gateway.Specification{
Expand Down
72 changes: 2 additions & 70 deletions core/node/dns.go
Original file line number Diff line number Diff line change
@@ -1,88 +1,20 @@
package node

import (
"fmt"
"math"
"strings"
"time"

"github.com/ipfs/go-libipfs/gateway"
config "github.com/ipfs/kubo/config"
doh "github.com/libp2p/go-doh-resolver"
madns "github.com/multiformats/go-multiaddr-dns"

"github.com/miekg/dns"
)

var defaultResolvers = map[string]string{
"eth.": "https://resolver.cloudflare-eth.com/dns-query",
"crypto.": "https://resolver.cloudflare-eth.com/dns-query",
}

func newResolver(url string, opts ...doh.Option) (madns.BasicResolver, error) {
if !strings.HasPrefix(url, "https://") {
return nil, fmt.Errorf("invalid resolver url: %s", url)
}

return doh.NewResolver(url, opts...)
}

func DNSResolver(cfg *config.Config) (*madns.Resolver, error) {
var opts []madns.Option
var err error

var dohOpts []doh.Option
if !cfg.DNS.MaxCacheTTL.IsDefault() {
dohOpts = append(dohOpts, doh.WithMaxCacheTTL(cfg.DNS.MaxCacheTTL.WithDefault(time.Duration(math.MaxUint32)*time.Second)))
}

domains := make(map[string]struct{}) // to track overridden default resolvers
rslvrs := make(map[string]madns.BasicResolver) // to reuse resolvers for the same URL

for domain, url := range cfg.DNS.Resolvers {
if domain != "." && !dns.IsFqdn(domain) {
return nil, fmt.Errorf("invalid domain %s; must be FQDN", domain)
}

domains[domain] = struct{}{}
if url == "" {
// allow overriding of implicit defaults with the default resolver
continue
}

rslv, ok := rslvrs[url]
if !ok {
rslv, err = newResolver(url, dohOpts...)
if err != nil {
return nil, fmt.Errorf("bad resolver for %s: %w", domain, err)
}
rslvrs[url] = rslv
}

if domain != "." {
opts = append(opts, madns.WithDomainResolver(domain, rslv))
} else {
opts = append(opts, madns.WithDefaultResolver(rslv))
}
}

// fill in defaults if not overridden by the user
for domain, url := range defaultResolvers {
_, ok := domains[domain]
if ok {
continue
}

rslv, ok := rslvrs[url]
if !ok {
rslv, err = newResolver(url)
if err != nil {
return nil, fmt.Errorf("bad resolver for %s: %w", domain, err)
}
rslvrs[url] = rslv
}

opts = append(opts, madns.WithDomainResolver(domain, rslv))
}

return madns.NewResolver(opts...)
return gateway.NewDNSResolver(cfg.DNS.Resolvers, dohOpts...)
}
8 changes: 6 additions & 2 deletions docs/examples/kubo-as-a-library/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ go 1.18
replace github.com/ipfs/kubo => ./../../..

require (
github.com/ipfs/go-libipfs v0.7.0
github.com/ipfs/go-libipfs v0.7.1-0.20230321095852-784eed6e63ff
github.com/ipfs/interface-go-ipfs-core v0.11.0
github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
github.com/libp2p/go-libp2p v0.26.3
Expand Down Expand Up @@ -42,6 +42,7 @@ require (
github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 // indirect
github.com/flynn/noise v1.0.0 // indirect
github.com/francoispqt/gojay v1.2.13 // indirect
github.com/gabriel-vasile/mimetype v1.4.1 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
Expand Down Expand Up @@ -88,6 +89,7 @@ require (
github.com/ipfs/go-ipfs-posinfo v0.0.1 // indirect
github.com/ipfs/go-ipfs-pq v0.0.3 // indirect
github.com/ipfs/go-ipfs-provider v0.8.1 // indirect
github.com/ipfs/go-ipfs-redirects-file v0.1.1 // indirect
github.com/ipfs/go-ipfs-routing v0.3.0 // indirect
github.com/ipfs/go-ipfs-util v0.0.2 // indirect
github.com/ipfs/go-ipld-cbor v0.0.6 // indirect
Expand All @@ -103,10 +105,11 @@ require (
github.com/ipfs/go-namesys v0.7.0 // indirect
github.com/ipfs/go-path v0.3.1 // indirect
github.com/ipfs/go-peertaskqueue v0.8.1 // indirect
github.com/ipfs/go-unixfs v0.4.4 // indirect
github.com/ipfs/go-unixfs v0.4.5-0.20230309101008-2acd046af3c6 // indirect
github.com/ipfs/go-unixfsnode v1.5.2 // indirect
github.com/ipfs/go-verifcid v0.0.2 // indirect
github.com/ipld/edelweiss v0.2.0 // indirect
github.com/ipld/go-car v0.5.0 // indirect
github.com/ipld/go-codec-dagpb v1.5.0 // indirect
github.com/ipld/go-ipld-prime v0.19.0 // indirect
github.com/jackpal/go-nat-pmp v1.0.2 // indirect
Expand Down Expand Up @@ -172,6 +175,7 @@ require (
github.com/samber/lo v1.36.0 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect
github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect
github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
Expand Down
Loading

0 comments on commit cb2acb9

Please sign in to comment.