From a8c7980721909f9cd16c35d699bc481574512c38 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Tue, 8 Dec 2020 17:32:33 -0800 Subject: [PATCH] add remote pinning to ipfs command (#7661) Added support for remote pinning services A pinning service is a service that accepts CIDs from a user in order to host the data associated with them. The spec for these services is defined at https://github.com/ipfs/pinning-services-api-spec Support is available via the `ipfs pin remote` CLI and the corresponding HTTP API Co-authored-by: Petar Maymounkov Co-authored-by: Marcin Rataj Co-authored-by: Adin Schmahmann --- .circleci/config.yml | 32 +- core/commands/commands_test.go | 10 +- core/commands/config.go | 80 +++- core/commands/{ => pin}/pin.go | 3 +- core/commands/pin/remotepin.go | 671 +++++++++++++++++++++++++++++++ core/commands/root.go | 3 +- docs/config.md | 54 +++ go.mod | 6 +- go.sum | 51 +-- test/sharness/t0700-remotepin.sh | 271 +++++++++++++ 10 files changed, 1127 insertions(+), 54 deletions(-) rename core/commands/{ => pin}/pin.go (99%) create mode 100644 core/commands/pin/remotepin.go create mode 100755 test/sharness/t0700-remotepin.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 644ba1846f5..d0d0c9114e6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -118,14 +118,42 @@ jobs: - store_artifacts: path: /tmp/circleci-test-results sharness: - executor: golang + machine: + image: ubuntu-2004:202010-01 + working_directory: ~/ipfs/go-ipfs + environment: + <<: *default_environment + GO111MODULE: "on" + TEST_NO_DOCKER: 1 + TEST_NO_FUSE: 1 + GOPATH: /home/circleci/go + TEST_VERBOSE: 1 steps: - run: sudo apt install socat - checkout + + - run: + mkdir rb-pinning-service-api && + cd rb-pinning-service-api && + git init && + git remote add origin https://github.com/ipfs-shipyard/rb-pinning-service-api.git && + git fetch --depth 1 origin 773c3adbb421c551d2d89288abac3e01e1f7c3a8 && + git checkout FETCH_HEAD + - run: + cd rb-pinning-service-api && + docker-compose pull && + docker-compose up -d + - *make_out_dirs - *restore_gomod - - run: make -O -j 10 coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1 + - run: + name: Setup Environment Variables + # we need the docker host IP; all ports exported by child containers can be accessed there. + command: echo "export DOCKER_HOST=$(ip -4 addr show docker0 | grep -Po 'inet \K[\d.]+')" >> $BASH_ENV + - run: + echo DOCKER_HOST=$DOCKER_HOST && + make -O -j 3 coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1 DOCKER_HOST=$DOCKER_HOST - run: when: always diff --git a/core/commands/commands_test.go b/core/commands/commands_test.go index 50545418b76..657ed2f4069 100644 --- a/core/commands/commands_test.go +++ b/core/commands/commands_test.go @@ -177,11 +177,19 @@ func TestCommands(t *testing.T) { "/p2p/stream/ls", "/pin", "/pin/add", - "/ping", "/pin/ls", + "/pin/remote", + "/pin/remote/add", + "/pin/remote/ls", + "/pin/remote/rm", + "/pin/remote/service", + "/pin/remote/service/add", + "/pin/remote/service/ls", + "/pin/remote/service/rm", "/pin/rm", "/pin/update", "/pin/verify", + "/ping", "/pubsub", "/pubsub/ls", "/pubsub/peers", diff --git a/core/commands/config.go b/core/commands/config.go index 50af4434357..3931bfc25dc 100644 --- a/core/commands/config.go +++ b/core/commands/config.go @@ -15,8 +15,8 @@ import ( "github.com/ipfs/go-ipfs/repo/fsrepo" "github.com/elgris/jsondiff" - "github.com/ipfs/go-ipfs-cmds" - "github.com/ipfs/go-ipfs-config" + cmds "github.com/ipfs/go-ipfs-cmds" + config "github.com/ipfs/go-ipfs-config" ) // ConfigUpdateOutput is config profile apply command's output @@ -36,6 +36,8 @@ const ( configDryRunOptionName = "dry-run" ) +var tryRemoteServiceApiErr = errors.New("cannot show or change pinning services through this API (try: ipfs pin remote service --help)") + var ConfigCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Get and set ipfs config values.", @@ -86,6 +88,12 @@ Set the value of the 'Datastore.Path' key: default: } + // Temporary fix until we move ApiKey secrets out of the config file + // (remote services are a map, so more advanced blocking is required) + if blocked := inBlockedScope(key, config.RemoteServicesSelector); blocked { + return tryRemoteServiceApiErr + } + cfgRoot, err := cmdenv.GetConfigRoot(env) if err != nil { return err @@ -140,11 +148,29 @@ Set the value of the 'Datastore.Path' key: Type: ConfigField{}, } +// Returns bool to indicate if tested key is in the blocked scope. +// (scope includes parent, direct, and child match) +func inBlockedScope(testKey string, blockedScope string) bool { + blockedScope = strings.ToLower(blockedScope) + roots := strings.Split(strings.ToLower(testKey), ".") + var scope []string + for _, name := range roots { + scope := append(scope, name) + impactedKey := strings.Join(scope, ".") + // blockedScope=foo.bar.BLOCKED should return true + // for parent and child impactedKeys: foo.bar and foo.bar.BLOCKED.subkey + if strings.HasPrefix(impactedKey, blockedScope) || strings.HasPrefix(blockedScope, impactedKey) { + return true + } + } + return false +} + var configShowCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Output config file contents.", ShortDescription: ` -NOTE: For security reasons, this command will omit your private key. If you would like to make a full backup of your config (private key included), you must copy the config file from your repo. +NOTE: For security reasons, this command will omit your private key and remote services. If you would like to make a full backup of your config (private key included), you must copy the config file from your repo. `, }, Type: map[string]interface{}{}, @@ -175,6 +201,11 @@ NOTE: For security reasons, this command will omit your private key. If you woul return err } + err = scrubOptionalValue(cfg, []string{config.PinningTag, config.RemoteServicesTag}) + if err != nil { + return err + } + return cmds.EmitOnce(res, &cfg) }, Encoders: cmds.EncoderMap{ @@ -190,7 +221,17 @@ NOTE: For security reasons, this command will omit your private key. If you woul }, } +// Scrubs value and returns error if missing func scrubValue(m map[string]interface{}, key []string) error { + return scrub(m, key, false) +} + +// Scrubs value and returns no error if missing +func scrubOptionalValue(m map[string]interface{}, key []string) error { + return scrub(m, key, true) +} + +func scrub(m map[string]interface{}, key []string, okIfMissing bool) error { find := func(m map[string]interface{}, k string) (string, interface{}, bool) { lckey := strings.ToLower(k) for mkey, val := range m { @@ -205,7 +246,7 @@ func scrubValue(m map[string]interface{}, key []string) error { cur := m for _, k := range key[:len(key)-1] { foundk, val, ok := find(cur, k) - if !ok { + if !ok && !okIfMissing { return errors.New("failed to find specified key") } @@ -223,7 +264,7 @@ func scrubValue(m map[string]interface{}, key []string) error { } todel, _, ok := find(cur, key[len(key)-1]) - if !ok { + if !ok && !okIfMissing { return fmt.Errorf("%s, not found", strings.Join(key, ".")) } @@ -466,6 +507,9 @@ func replaceConfig(r repo.Repo, file io.Reader) error { if err := json.NewDecoder(file).Decode(&cfg); err != nil { return errors.New("failed to decode file as config") } + + // Handle Identity.PrivKey (secret) + if len(cfg.Identity.PrivKey) != 0 { return errors.New("setting private key with API is not supported") } @@ -482,5 +526,31 @@ func replaceConfig(r repo.Repo, file io.Reader) error { cfg.Identity.PrivKey = pkstr + // Handle Pinning.RemoteServices (ApiKey of each service is secret) + // Note: these settings are opt-in and may be missing + + if len(cfg.Pinning.RemoteServices) != 0 { + return tryRemoteServiceApiErr + } + + // detect if existing config has any remote services defined.. + if remoteServicesTag, err := getConfig(r, config.RemoteServicesSelector); err == nil { + // seems that golang cannot type assert map[string]interface{} to map[string]config.RemotePinningService + // so we have to manually copy the data :-| + if val, ok := remoteServicesTag.Value.(map[string]interface{}); ok { + var services map[string]config.RemotePinningService + jsonString, err := json.Marshal(val) + if err != nil { + return fmt.Errorf("failed to replace config while preserving %s: %s", config.RemoteServicesSelector, err) + } + err = json.Unmarshal(jsonString, &services) + if err != nil { + return fmt.Errorf("failed to replace config while preserving %s: %s", config.RemoteServicesSelector, err) + } + // .. if so, apply them on top of the new config + cfg.Pinning.RemoteServices = services + } + } + return r.SetConfig(&cfg) } diff --git a/core/commands/pin.go b/core/commands/pin/pin.go similarity index 99% rename from core/commands/pin.go rename to core/commands/pin/pin.go index 9d209040a59..a951bac950d 100644 --- a/core/commands/pin.go +++ b/core/commands/pin/pin.go @@ -1,4 +1,4 @@ -package commands +package pin import ( "context" @@ -35,6 +35,7 @@ var PinCmd = &cmds.Command{ "ls": listPinCmd, "verify": verifyPinCmd, "update": updatePinCmd, + "remote": remotePinCmd, }, } diff --git a/core/commands/pin/remotepin.go b/core/commands/pin/remotepin.go new file mode 100644 index 00000000000..12a2639c998 --- /dev/null +++ b/core/commands/pin/remotepin.go @@ -0,0 +1,671 @@ +package pin + +import ( + "context" + "fmt" + "io" + "sort" + "strings" + "text/tabwriter" + "time" + + neturl "net/url" + + "golang.org/x/sync/errgroup" + + cid "github.com/ipfs/go-cid" + cmds "github.com/ipfs/go-ipfs-cmds" + config "github.com/ipfs/go-ipfs-config" + "github.com/ipfs/go-ipfs/core/commands/cmdenv" + fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo" + logging "github.com/ipfs/go-log" + pinclient "github.com/ipfs/go-pinning-service-http-client" + path "github.com/ipfs/interface-go-ipfs-core/path" + "github.com/libp2p/go-libp2p-core/host" + peer "github.com/libp2p/go-libp2p-core/peer" +) + +var log = logging.Logger("core/commands/cmdenv") + +var remotePinCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Pin (and unpin) objects to remote pinning service.", + }, + + Subcommands: map[string]*cmds.Command{ + "add": addRemotePinCmd, + "ls": listRemotePinCmd, + "rm": rmRemotePinCmd, + "service": remotePinServiceCmd, + }, +} + +var remotePinServiceCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Configure remote pinning services.", + }, + + Subcommands: map[string]*cmds.Command{ + "add": addRemotePinServiceCmd, + "ls": lsRemotePinServiceCmd, + "rm": rmRemotePinServiceCmd, + }, +} + +const pinNameOptionName = "name" +const pinCIDsOptionName = "cid" +const pinStatusOptionName = "status" +const pinServiceNameOptionName = "service" +const pinServiceURLOptionName = "url" +const pinServiceKeyOptionName = "key" +const pinServiceStatOptionName = "stat" +const pinBackgroundOptionName = "background" +const pinForceOptionName = "force" + +type RemotePinOutput struct { + Status string + Cid string + Name string +} + +func toRemotePinOutput(ps pinclient.PinStatusGetter) RemotePinOutput { + return RemotePinOutput{ + Name: ps.GetPin().GetName(), + Status: ps.GetStatus().String(), + Cid: ps.GetPin().GetCid().String(), + } +} + +func printRemotePinDetails(w io.Writer, out *RemotePinOutput) { + tw := tabwriter.NewWriter(w, 0, 0, 1, ' ', 0) + defer tw.Flush() + fw := func(k string, v string) { + fmt.Fprintf(tw, "%s:\t%s\n", k, v) + } + fw("CID", out.Cid) + fw("Name", out.Name) + fw("Status", out.Status) +} + +// remote pin commands + +var pinServiceNameOption = cmds.StringOption(pinServiceNameOptionName, "Name of the remote pinning service to use.") + +var addRemotePinCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Pin object to remote pinning service.", + ShortDescription: "Stores an IPFS object from a given path to a remote pinning service.", + }, + + Arguments: []cmds.Argument{ + cmds.StringArg("ipfs-path", true, false, "Path to object(s) to be pinned."), + }, + Options: []cmds.Option{ + cmds.StringOption(pinNameOptionName, "An optional name for the pin."), + pinServiceNameOption, + cmds.BoolOption(pinBackgroundOptionName, "Add to the queue on the remote service and return immediately (does not wait for pinned status).").WithDefault(false), + }, + Type: RemotePinOutput{}, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + ctx, cancel := context.WithCancel(req.Context) + defer cancel() + + // Get remote service + c, err := getRemotePinServiceFromRequest(req, env) + if err != nil { + return err + } + + // Prepare value for Pin.cid + if len(req.Arguments) != 1 { + return fmt.Errorf("expecting one CID argument") + } + api, err := cmdenv.GetApi(env, req) + if err != nil { + return err + } + rp, err := api.ResolvePath(ctx, path.New(req.Arguments[0])) + if err != nil { + return err + } + + // Prepare Pin.name + opts := []pinclient.AddOption{} + if name, nameFound := req.Options[pinNameOptionName]; nameFound { + nameStr := name.(string) + opts = append(opts, pinclient.PinOpts.WithName(nameStr)) + } + + // Prepare Pin.origins + // Add own multiaddrs to the 'origins' array, so Pinning Service can + // use that as a hint and connect back to us (if possible) + node, err := cmdenv.GetNode(env) + if err != nil { + return err + } + if node.PeerHost != nil { + addrs, err := peer.AddrInfoToP2pAddrs(host.InfoFromHost(node.PeerHost)) + if err != nil { + return err + } + opts = append(opts, pinclient.PinOpts.WithOrigins(addrs...)) + } + + // Execute remote pin request + // TODO: fix panic when pinning service is down + ps, err := c.Add(ctx, rp.Cid(), opts...) + if err != nil { + return err + } + + // Act on PinStatus.delegates + // If Pinning Service returned any delegates, proactively try to + // connect to them to facilitate data exchange without waiting for DHT + // lookup + for _, d := range ps.GetDelegates() { + // TODO: confirm this works as expected + p, err := peer.AddrInfoFromP2pAddr(d) + if err != nil { + return err + } + if err := api.Swarm().Connect(ctx, *p); err != nil { + log.Infof("error connecting to remote pin delegate %v : %w", d, err) + } + } + + // Block unless --background=true is passed + if !req.Options[pinBackgroundOptionName].(bool) { + requestId := ps.GetRequestId() + for { + ps, err = c.GetStatusByID(ctx, requestId) + if err != nil { + return fmt.Errorf("failed to check pin status for requestid=%q due to error: %v", requestId, err) + } + if ps.GetRequestId() != requestId { + return fmt.Errorf("failed to check pin status for requestid=%q, remote service sent unexpected requestid=%q", requestId, ps.GetRequestId()) + } + s := ps.GetStatus() + if s == pinclient.StatusPinned { + break + } + if s == pinclient.StatusFailed { + return fmt.Errorf("remote service failed to pin requestid=%q", requestId) + } + tmr := time.NewTimer(time.Second / 2) + select { + case <-tmr.C: + case <-ctx.Done(): + return fmt.Errorf("waiting for pin interrupted, requestid=%q remains on remote service", requestId) + } + } + } + + return res.Emit(toRemotePinOutput(ps)) + }, + Encoders: cmds.EncoderMap{ + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *RemotePinOutput) error { + printRemotePinDetails(w, out) + return nil + }), + }, +} + +var listRemotePinCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "List objects pinned to remote pinning service.", + ShortDescription: ` +Returns a list of objects that are pinned to a remote pinning service. +`, + LongDescription: ` +Returns a list of objects that are pinned to a remote pinning service. +`, + }, + + Arguments: []cmds.Argument{}, + Options: []cmds.Option{ + cmds.StringOption(pinNameOptionName, "Return pins objects with names that contain provided value (case-sensitive, exact match)."), + cmds.StringsOption(pinCIDsOptionName, "Return only pin objects for the specified CID(s); optional, comma separated."), + cmds.StringsOption(pinStatusOptionName, "Return only pin objects with the specified statuses (queued,pinning,pinned,failed)").WithDefault([]string{"pinned"}), + pinServiceNameOption, + }, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + ctx, cancel := context.WithCancel(req.Context) + defer cancel() + + c, err := getRemotePinServiceFromRequest(req, env) + if err != nil { + return err + } + + psCh, errCh, err := lsRemote(ctx, req, c) + if err != nil { + return err + } + + for ps := range psCh { + if err := res.Emit(toRemotePinOutput(ps)); err != nil { + return err + } + } + + return <-errCh + }, + Type: RemotePinOutput{}, + Encoders: cmds.EncoderMap{ + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *RemotePinOutput) error { + // pin remote ls produces a flat output similar to legacy pin ls + fmt.Fprintf(w, "%s\t%s\t%s\n", out.Cid, out.Status, out.Name) + return nil + }), + }, +} + +// Executes GET /pins/?query-with-filters +func lsRemote(ctx context.Context, req *cmds.Request, c *pinclient.Client) (chan pinclient.PinStatusGetter, chan error, error) { + opts := []pinclient.LsOption{} + if name, nameFound := req.Options[pinNameOptionName]; nameFound { + nameStr := name.(string) + opts = append(opts, pinclient.PinOpts.FilterName(nameStr)) + } + + if cidsRaw, cidsFound := req.Options[pinCIDsOptionName]; cidsFound { + cidsRawArr := cidsRaw.([]string) + parsedCIDs := []cid.Cid{} + for _, rawCID := range flattenCommaList(cidsRawArr) { + parsedCID, err := cid.Decode(rawCID) + if err != nil { + return nil, nil, fmt.Errorf("CID %q cannot be parsed: %v", rawCID, err) + } + parsedCIDs = append(parsedCIDs, parsedCID) + } + opts = append(opts, pinclient.PinOpts.FilterCIDs(parsedCIDs...)) + } + if statusRaw, statusFound := req.Options[pinStatusOptionName]; statusFound { + statusRawArr := statusRaw.([]string) + parsedStatuses := []pinclient.Status{} + for _, rawStatus := range flattenCommaList(statusRawArr) { + s := pinclient.Status(rawStatus) + if s.String() == string(pinclient.StatusUnknown) { + return nil, nil, fmt.Errorf("status %q is not valid", rawStatus) + } + parsedStatuses = append(parsedStatuses, s) + } + opts = append(opts, pinclient.PinOpts.FilterStatus(parsedStatuses...)) + } + + psCh, errCh := c.Ls(ctx, opts...) + + return psCh, errCh, nil +} + +func flattenCommaList(list []string) []string { + flatList := list[:0] + for _, s := range list { + flatList = append(flatList, strings.Split(s, ",")...) + } + return flatList +} + +var rmRemotePinCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Remove pinned objects from remote pinning service.", + ShortDescription: ` +Removes the pin from the given object allowing it to be garbage +collected if needed. +`, + }, + + Arguments: []cmds.Argument{}, + Options: []cmds.Option{ + pinServiceNameOption, + cmds.StringOption(pinNameOptionName, "Remove pin objects with names that contain provided value (case-sensitive, exact match)."), + cmds.StringsOption(pinCIDsOptionName, "Remove only pin objects for the specified CID(s)."), + cmds.StringsOption(pinStatusOptionName, "Remove only pin objects with the specified statuses (queued,pinning,pinned,failed).").WithDefault([]string{"pinned"}), + cmds.BoolOption(pinForceOptionName, "Remove multiple pins without confirmation.").WithDefault(false), + }, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + ctx, cancel := context.WithCancel(req.Context) + defer cancel() + + c, err := getRemotePinServiceFromRequest(req, env) + if err != nil { + return err + } + + rmIDs := []string{} + if len(req.Arguments) == 0 { + psCh, errCh, err := lsRemote(ctx, req, c) + if err != nil { + return err + } + for ps := range psCh { + rmIDs = append(rmIDs, ps.GetRequestId()) + } + if err = <-errCh; err != nil { + return fmt.Errorf("error while listing remote pins: %v", err) + } + + if len(rmIDs) > 1 && !req.Options[pinForceOptionName].(bool) { + return fmt.Errorf("multiple remote pins are matching this query, add --force to confirm the bulk removal") + } + } else { + return fmt.Errorf("unexpected argument %q", req.Arguments[0]) + } + + for _, rmID := range rmIDs { + if err := c.DeleteByID(ctx, rmID); err != nil { + return fmt.Errorf("removing pin identified by requestid=%q failed: %v", rmID, err) + } + } + return nil + }, +} + +// remote service commands + +var addRemotePinServiceCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Add remote pinning service.", + ShortDescription: "Add a credentials for access to a remote pinning service.", + }, + Arguments: []cmds.Argument{ + cmds.StringArg(pinServiceNameOptionName, true, false, "Service name."), + cmds.StringArg(pinServiceURLOptionName, true, false, "Service URL."), + cmds.StringArg(pinServiceKeyOptionName, true, false, "Service key."), + }, + Type: nil, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + cfgRoot, err := cmdenv.GetConfigRoot(env) + if err != nil { + return err + } + repo, err := fsrepo.Open(cfgRoot) + if err != nil { + return err + } + defer repo.Close() + + if len(req.Arguments) < 3 { + return fmt.Errorf("expecting three arguments: service name, url and key") + } + + name := req.Arguments[0] + url := strings.TrimSuffix(req.Arguments[1], "/pins") // fix /pins/pins :-) + key := req.Arguments[2] + + u, err := neturl.ParseRequestURI(url) + if err != nil || !strings.HasPrefix(u.Scheme, "http") { + return fmt.Errorf("service url must be a valid HTTP URL") + } + + cfg, err := repo.Config() + if err != nil { + return err + } + if cfg.Pinning.RemoteServices != nil { + if _, present := cfg.Pinning.RemoteServices[name]; present { + return fmt.Errorf("service already present") + } + } else { + cfg.Pinning.RemoteServices = map[string]config.RemotePinningService{} + } + + cfg.Pinning.RemoteServices[name] = config.RemotePinningService{ + Api: config.RemotePinningServiceApi{ + Endpoint: url, + Key: key, + }, + } + + return repo.SetConfig(cfg) + }, +} + +var rmRemotePinServiceCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Remove remote pinning service.", + ShortDescription: "Remove credentials for access to a remote pinning service.", + }, + Arguments: []cmds.Argument{ + cmds.StringArg("remote-pin-service", true, false, "Name of remote pinning service to remove."), + }, + Options: []cmds.Option{}, + Type: nil, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + cfgRoot, err := cmdenv.GetConfigRoot(env) + if err != nil { + return err + } + repo, err := fsrepo.Open(cfgRoot) + if err != nil { + return err + } + defer repo.Close() + + if len(req.Arguments) != 1 { + return fmt.Errorf("expecting one argument: name") + } + name := req.Arguments[0] + + cfg, err := repo.Config() + if err != nil { + return err + } + if cfg.Pinning.RemoteServices != nil { + delete(cfg.Pinning.RemoteServices, name) + } + return repo.SetConfig(cfg) + }, +} + +var lsRemotePinServiceCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "List remote pinning services.", + ShortDescription: "List remote pinning services.", + }, + Arguments: []cmds.Argument{}, + Options: []cmds.Option{ + cmds.BoolOption(pinServiceStatOptionName, "Try to fetch and display current pin count on remote service (queued/pinning/pinned/failed).").WithDefault(false), + }, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + ctx, cancel := context.WithCancel(req.Context) + defer cancel() + + cfgRoot, err := cmdenv.GetConfigRoot(env) + if err != nil { + return err + } + repo, err := fsrepo.Open(cfgRoot) + if err != nil { + return err + } + defer repo.Close() + + cfg, err := repo.Config() + if err != nil { + return err + } + if cfg.Pinning.RemoteServices == nil { + return nil // no pinning services added yet + } + services := cfg.Pinning.RemoteServices + result := PinServicesList{make([]ServiceDetails, 0, len(services))} + for svcName, svcConfig := range services { + svcDetails := ServiceDetails{svcName, svcConfig.Api.Endpoint, nil} + + // if --pin-count is passed, we try to fetch pin numbers from remote service + if req.Options[pinServiceStatOptionName].(bool) { + lsRemotePinCount := func(ctx context.Context, env cmds.Environment, svcName string) (*PinCount, error) { + c, err := getRemotePinService(env, svcName) + if err != nil { + return nil, err + } + // we only care about total count, so requesting smallest batch + batch := pinclient.PinOpts.Limit(1) + fs := pinclient.PinOpts.FilterStatus + + statuses := []pinclient.Status{ + pinclient.StatusQueued, + pinclient.StatusPinning, + pinclient.StatusPinned, + pinclient.StatusFailed, + } + + g, ctx := errgroup.WithContext(ctx) + pc := &PinCount{} + + for _, s := range statuses { + status := s // lol https://golang.org/doc/faq#closures_and_goroutines + g.Go(func() error { + _, n, err := c.LsBatchSync(ctx, batch, fs(status)) + if err != nil { + return err + } + switch status { + case pinclient.StatusQueued: + pc.Queued = n + case pinclient.StatusPinning: + pc.Pinning = n + case pinclient.StatusPinned: + pc.Pinned = n + case pinclient.StatusFailed: + pc.Failed = n + } + return nil + }) + } + if err := g.Wait(); err != nil { + return nil, err + } + + return pc, nil + } + + pinCount, err := lsRemotePinCount(ctx, env, svcName) + + // PinCount is present only if we were able to fetch counts. + // We don't want to break listing of services so this is best-effort. + // (verbose err is returned by 'pin remote ls', if needed) + svcDetails.Stat = &Stat{} + if err == nil { + svcDetails.Stat.Status = "valid" + svcDetails.Stat.PinCount = pinCount + } else { + svcDetails.Stat.Status = "invalid" + } + } + result.RemoteServices = append(result.RemoteServices, svcDetails) + } + sort.Sort(result) + return cmds.EmitOnce(res, &result) + }, + Type: PinServicesList{}, + Encoders: cmds.EncoderMap{ + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, list *PinServicesList) error { + tw := tabwriter.NewWriter(w, 1, 2, 1, ' ', 0) + withStat := req.Options[pinServiceStatOptionName].(bool) + for _, s := range list.RemoteServices { + if withStat { + stat := s.Stat.Status + pc := s.Stat.PinCount + if s.Stat.PinCount != nil { + stat = fmt.Sprintf("%d/%d/%d/%d", pc.Queued, pc.Pinning, pc.Pinned, pc.Failed) + } + fmt.Fprintf(tw, "%s\t%s\t%s\n", s.Service, s.ApiEndpoint, stat) + } else { + fmt.Fprintf(tw, "%s\t%s\n", s.Service, s.ApiEndpoint) + } + } + tw.Flush() + return nil + }), + }, +} + +type ServiceDetails struct { + Service string + ApiEndpoint string + Stat *Stat `json:",omitempty"` // present only when --stat not passed +} + +type Stat struct { + Status string + PinCount *PinCount `json:",omitempty"` // missing when --stat is passed but the service is offline +} + +type PinCount struct { + Queued int + Pinning int + Pinned int + Failed int +} + +// Struct returned by ipfs pin remote service ls --enc=json | jq +type PinServicesList struct { + RemoteServices []ServiceDetails +} + +func (l PinServicesList) Len() int { + return len(l.RemoteServices) +} + +func (l PinServicesList) Swap(i, j int) { + s := l.RemoteServices + s[i], s[j] = s[j], s[i] +} + +func (l PinServicesList) Less(i, j int) bool { + s := l.RemoteServices + return s[i].Service < s[j].Service +} + +func getRemotePinServiceFromRequest(req *cmds.Request, env cmds.Environment) (*pinclient.Client, error) { + service, serviceFound := req.Options[pinServiceNameOptionName] + if !serviceFound { + return nil, fmt.Errorf("a service name must be passed") + } + + serviceStr := service.(string) + var err error + c, err := getRemotePinService(env, serviceStr) + if err != nil { + return nil, err + } + + return c, nil +} + +func getRemotePinService(env cmds.Environment, name string) (*pinclient.Client, error) { + if name == "" { + return nil, fmt.Errorf("remote pinning service name not specified") + } + url, key, err := getRemotePinServiceInfo(env, name) + if err != nil { + return nil, err + } + return pinclient.NewClient(url, key), nil +} + +func getRemotePinServiceInfo(env cmds.Environment, name string) (url, key string, err error) { + cfgRoot, err := cmdenv.GetConfigRoot(env) + if err != nil { + return "", "", err + } + repo, err := fsrepo.Open(cfgRoot) + if err != nil { + return "", "", err + } + defer repo.Close() + cfg, err := repo.Config() + if err != nil { + return "", "", err + } + if cfg.Pinning.RemoteServices == nil { + return "", "", fmt.Errorf("service not known") + } + service, present := cfg.Pinning.RemoteServices[name] + if !present { + return "", "", fmt.Errorf("service not known") + } + return service.Api.Endpoint, service.Api.Key, nil +} diff --git a/core/commands/root.go b/core/commands/root.go index b9a8dc40909..1440c31b7eb 100644 --- a/core/commands/root.go +++ b/core/commands/root.go @@ -7,6 +7,7 @@ import ( dag "github.com/ipfs/go-ipfs/core/commands/dag" name "github.com/ipfs/go-ipfs/core/commands/name" ocmd "github.com/ipfs/go-ipfs/core/commands/object" + "github.com/ipfs/go-ipfs/core/commands/pin" unixfs "github.com/ipfs/go-ipfs/core/commands/unixfs" cmds "github.com/ipfs/go-ipfs-cmds" @@ -136,7 +137,7 @@ var rootSubcommands = map[string]*cmds.Command{ "mount": MountCmd, "name": name.NameCmd, "object": ocmd.ObjectCmd, - "pin": PinCmd, + "pin": pin.PinCmd, "ping": PingCmd, "p2p": P2PCmd, "refs": RefsCmd, diff --git a/docs/config.md b/docs/config.md index 938477b719d..2ea82bca39a 100644 --- a/docs/config.md +++ b/docs/config.md @@ -176,6 +176,11 @@ does (e.g, `"1d2h4m40.01s"`). - [`Mounts.IPFS`](#mountsipfs) - [`Mounts.IPNS`](#mountsipns) - [`Mounts.FuseAllowOther`](#mountsfuseallowother) +- [`Pinning`](#pinning) + - [`Pinning.RemoteServices`](#pinningremoteservices) + - [`Pinning.RemoteServices.API`](#pinningremoteservices-api) + - [`Pinning.RemoteServices.API.Endpoint`](#pinningremoteservices-apiendpoint) + - [`Pinning.RemoteServices.API.Key`](#pinningremoteservices-apikey) - [`Pubsub`](#pubsub) - [`Pubsub.Router`](#pubsubrouter) - [`Pubsub.DisableSigning`](#pubsubdisablesigning) @@ -813,6 +818,55 @@ Type: `string` (filesystem path) Sets the FUSE allow other option on the mountpoint. +## `Pinning` + +Pinning configures the options available for pinning content +(i.e. keeping content longer term instead of as temporarily cached storage). + +### `Pinning.RemoteServices` + +`RemoteServices` maps a name for a remote pinning service to its configuration. + +A remote pinning service is a remote service that exposes an API for managing +that service's interest in longer term data storage. + +The exposed API conforms to the specification defined at +https://ipfs.github.io/pinning-services-api-spec/ + +#### `Pinning.RemoteServices: API` + +Contains information relevant to utilizing the remote pinning service + +Example: +```json +{ + "Pinning": { + "RemoteServices": { + "myPinningService": { + "API" : { + "Endpoint" : "https://pinningservice.tld:1234/my/api/path", + "Key" : "someOpaqueKey" + } + } + } + } +} +``` + +##### `Pinning.RemoteServices: API.Endpoint` + +The HTTP(S) endpoint through which to access the pinning service + +Example: "https://pinningservice.tld:1234/my/api/path" + +Type: `string` + +##### `Pinning.RemoteServices: API.Key` + +The key through which access to the pinning service is granted + +Type: `string` + ## `Pubsub` Pubsub configures the `ipfs pubsub` subsystem. To use, it must be enabled by diff --git a/go.mod b/go.mod index 3419cb7d47e..208ec025780 100644 --- a/go.mod +++ b/go.mod @@ -30,8 +30,8 @@ require ( github.com/ipfs/go-graphsync v0.5.1 github.com/ipfs/go-ipfs-blockstore v0.1.4 github.com/ipfs/go-ipfs-chunker v0.0.5 - github.com/ipfs/go-ipfs-cmds v0.4.0 - github.com/ipfs/go-ipfs-config v0.9.0 + github.com/ipfs/go-ipfs-cmds v0.5.0 + github.com/ipfs/go-ipfs-config v0.11.0 github.com/ipfs/go-ipfs-ds-help v0.1.1 github.com/ipfs/go-ipfs-exchange-interface v0.0.1 github.com/ipfs/go-ipfs-exchange-offline v0.0.1 @@ -51,6 +51,7 @@ require ( github.com/ipfs/go-metrics-prometheus v0.0.2 github.com/ipfs/go-mfs v0.1.2 github.com/ipfs/go-path v0.0.8 + github.com/ipfs/go-pinning-service-http-client v0.1.0 github.com/ipfs/go-unixfs v0.2.4 github.com/ipfs/go-verifcid v0.0.1 github.com/ipfs/interface-go-ipfs-core v0.4.0 @@ -104,6 +105,7 @@ require ( go.uber.org/zap v1.16.0 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect + golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 ) diff --git a/go.sum b/go.sum index a425b5e0bd4..03a445461ce 100644 --- a/go.sum +++ b/go.sum @@ -35,7 +35,6 @@ github.com/Kubuxu/go-os-helper v0.0.1 h1:EJiD2VUQyh5A9hWJLmc6iWg6yIcJ7jpBcwC8GMG github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/Stebalien/go-bitfield v0.0.0-20180330043415-076a62f9ce6e/go.mod h1:3oM7gXIttpYDAJXpVNnSCiUMYBLIZ6cb1t+Ip982MRo= github.com/Stebalien/go-bitfield v0.0.1 h1:X3kbSSPUaJK60wV2hjOPZwmpljr6VGCqdq4cBLhbQBo= github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI+XWrX9Wf2XH0s= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= @@ -142,7 +141,6 @@ github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 h1:BBso6MBKW github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5/go.mod h1:JpoxHjuQauoxiFMl1ie8Xc/7TfLuMZ5eOCONd1sUBHg= github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= -github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= @@ -253,7 +251,6 @@ github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:Fecb github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= -github.com/gxed/pubsub v0.0.0-20180201040156-26ebdf44f824/go.mod h1:OiEWyHgK+CWrmOlVquHaIK1vhpUJydC9m0Je6mhaiNE= github.com/hannahhoward/cbor-gen-for v0.0.0-20200817222906-ea96cece81f1/go.mod h1:jvfsLIxk0fY/2BKSQ1xf2406AKA5dwMmKKv0ADcOfN8= github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e h1:3YKHER4nmd7b5qy5t0GWDTwSn4OyRgfAXSmo6VnryBY= github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e/go.mod h1:I8h3MITA53gN9OnWGCgaMa0JWVRdXthWw4M3CPM54OY= @@ -268,7 +265,6 @@ github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uG github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huin/goupnp v0.0.0-20180415215157-1395d1447324/go.mod h1:MZ2ZmwcBpvOoJ22IJsc7va19ZwoheaBk43rKg12SKag= github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= @@ -277,7 +273,6 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/go-bitswap v0.0.3/go.mod h1:jadAZYsP/tcRMl47ZhFxhaNuDQoXawT8iHMg+iFoQbg= github.com/ipfs/go-bitswap v0.0.9/go.mod h1:kAPf5qgn2W2DrgAcscZ3HrM9qh4pH+X8Fkk3UPrwvis= github.com/ipfs/go-bitswap v0.1.0/go.mod h1:FFJEf18E9izuCqUtHxbWEvq+reg7o4CW5wSAE1wsxj0= github.com/ipfs/go-bitswap v0.1.2/go.mod h1:qxSWS4NXGs7jQ6zQvoPY3+NmOfHHG47mhkiLzBpJQIs= @@ -288,7 +283,6 @@ github.com/ipfs/go-bitswap v0.3.3/go.mod h1:AyWWfN3moBzQX0banEtfKOfbXb3ZeoOeXnZG github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/baNLgayt4pFc= github.com/ipfs/go-block-format v0.0.2 h1:qPDvcP19izTjU8rgo6p7gTXZlkMkF5bz5G3fqIsSCPE= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= -github.com/ipfs/go-blockservice v0.0.3/go.mod h1:/NNihwTi6V2Yr6g8wBI+BSwPuURpBRMtYNGrlxZ8KuI= github.com/ipfs/go-blockservice v0.0.7/go.mod h1:EOfb9k/Y878ZTRY/CH0x5+ATtaipfbRhbvNSdgc/7So= github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M= github.com/ipfs/go-blockservice v0.1.1/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I= @@ -368,10 +362,10 @@ github.com/ipfs/go-ipfs-chunker v0.0.1 h1:cHUUxKFQ99pozdahi+uSC/3Y6HeRpi9oTeUHbE github.com/ipfs/go-ipfs-chunker v0.0.1/go.mod h1:tWewYK0we3+rMbOh7pPFGDyypCtvGcBFymgY4rSDLAw= github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8= github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8= -github.com/ipfs/go-ipfs-cmds v0.4.0 h1:xUavIxA9Ts8U6PAHmQBvDGMlGfUrQ13Rymd+5t8LIF4= -github.com/ipfs/go-ipfs-cmds v0.4.0/go.mod h1:ZgYiWVnCk43ChwoH8hAmI1IRbuVtq3GSTHwtRB/Kqhk= -github.com/ipfs/go-ipfs-config v0.9.0 h1:qTXJ9CyOyQv1LFJUMysxz8fi6RxxnP9QqcmiobuANvw= -github.com/ipfs/go-ipfs-config v0.9.0/go.mod h1:GQUxqb0NfkZmEU92PxqqqLVVFTLpoGGUlBaTyDaAqrE= +github.com/ipfs/go-ipfs-cmds v0.5.0 h1:T1ZT6Qu3IUCp6FgU2IzVtvGLaexEWo9q13+S5ic+Q5Y= +github.com/ipfs/go-ipfs-cmds v0.5.0/go.mod h1:ZgYiWVnCk43ChwoH8hAmI1IRbuVtq3GSTHwtRB/Kqhk= +github.com/ipfs/go-ipfs-config v0.11.0 h1:w4t2pz415Gtg6MTUKAq06C7ezC59/Us+k3+n1Tje+wg= +github.com/ipfs/go-ipfs-config v0.11.0/go.mod h1:Ei/FLgHGTdPyqCPK0oPCwGTe8VSnsjJjx7HZqUb6Ry0= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= @@ -383,11 +377,9 @@ github.com/ipfs/go-ipfs-exchange-interface v0.0.1 h1:LJXIo9W7CAmugqI+uofioIpRb6r github.com/ipfs/go-ipfs-exchange-interface v0.0.1/go.mod h1:c8MwfHjtQjPoDyiy9cFquVtVHkO9b9Ob3FG91qJnWCM= github.com/ipfs/go-ipfs-exchange-offline v0.0.1 h1:P56jYKZF7lDDOLx5SotVh5KFxoY6C81I1NSHW1FxGew= github.com/ipfs/go-ipfs-exchange-offline v0.0.1/go.mod h1:WhHSFCVYX36H/anEKQboAzpUws3x7UeEGkzQc3iNkM0= -github.com/ipfs/go-ipfs-files v0.0.2/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4= github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4= github.com/ipfs/go-ipfs-files v0.0.8 h1:8o0oFJkJ8UkO/ABl8T6ac6tKF3+NIpj67aAB6ZpusRg= github.com/ipfs/go-ipfs-files v0.0.8/go.mod h1:wiN/jSG8FKyk7N0WyctKSvq3ljIa2NNTiZB55kpTdOs= -github.com/ipfs/go-ipfs-flags v0.0.1/go.mod h1:RnXBb9WV53GSfTrSDVK61NLTFKvWc60n+K9EgCDh+rA= github.com/ipfs/go-ipfs-pinner v0.1.0 h1:rjSrbUDYd1YYHZ5dOgu+QEOuLcU0m/2a/brcxC/ReeU= github.com/ipfs/go-ipfs-pinner v0.1.0/go.mod h1:EzyyaWCWeZJ/he9cDBH6QrEkSuRqTRWMmCoyNkylTTg= github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs= @@ -404,7 +396,6 @@ github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8= github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= -github.com/ipfs/go-ipld-cbor v0.0.1/go.mod h1:RXHr8s4k0NE0TKhnrxqZC9M888QfsBN9rhS5NjfKzY8= github.com/ipfs/go-ipld-cbor v0.0.2/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= github.com/ipfs/go-ipld-cbor v0.0.3 h1:ENsxvybwkmke7Z/QJOmeJfoguj6GH3Y0YOaGrfy9Q0I= github.com/ipfs/go-ipld-cbor v0.0.3/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= @@ -441,7 +432,6 @@ github.com/ipfs/go-log/v2 v2.0.5 h1:fL4YI+1g5V/b1Yxr1qAiXTMg1H8z9vx/VmJxBuQMHvU= github.com/ipfs/go-log/v2 v2.0.5/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscwkWG+dw= github.com/ipfs/go-log/v2 v2.1.1 h1:G4TtqN+V9y9HY9TA6BwbCVyyBZ2B9MbCjR2MtGx8FR0= github.com/ipfs/go-log/v2 v2.1.1/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM= -github.com/ipfs/go-merkledag v0.0.3/go.mod h1:Oc5kIXLHokkE1hWGMBHw+oxehkAaTOqtEb7Zbh6BhLA= github.com/ipfs/go-merkledag v0.0.6/go.mod h1:QYPdnlvkOg7GnQRofu9XZimC5ZW5Wi3bKys/4GQQfto= github.com/ipfs/go-merkledag v0.1.0/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= github.com/ipfs/go-merkledag v0.2.3 h1:aMdkK9G1hEeNvn3VXfiEMLY0iJnbiQQUHnM0HFJREsE= @@ -458,7 +448,6 @@ github.com/ipfs/go-metrics-prometheus v0.0.2 h1:9i2iljLg12S78OhC6UAiXi176xvQGiZa github.com/ipfs/go-metrics-prometheus v0.0.2/go.mod h1:ELLU99AQQNi+zX6GCGm2lAgnzdSH3u5UVlCdqSXnEks= github.com/ipfs/go-mfs v0.1.2 h1:DlelNSmH+yz/Riy0RjPKlooPg0KML4lXGdLw7uZkfAg= github.com/ipfs/go-mfs v0.1.2/go.mod h1:T1QBiZPEpkPLzDqEJLNnbK55BVKVlNi2a+gVm4diFo0= -github.com/ipfs/go-path v0.0.3/go.mod h1:zIRQUez3LuQIU25zFjC2hpBTHimWx7VK5bjZgRLbbdo= github.com/ipfs/go-path v0.0.7 h1:H06hKMquQ0aYtHiHryOMLpQC1qC3QwXwkahcEVD51Ho= github.com/ipfs/go-path v0.0.7/go.mod h1:6KTKmeRnBXgqrTvzFrPV3CamxcgvXX/4z79tfAd2Sno= github.com/ipfs/go-path v0.0.8 h1:R0k6t9x/pa+g8qzl5apQIPurJFozXhopks3iw3MX+jU= @@ -468,14 +457,13 @@ github.com/ipfs/go-peertaskqueue v0.1.0/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3 github.com/ipfs/go-peertaskqueue v0.1.1/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= github.com/ipfs/go-peertaskqueue v0.2.0 h1:2cSr7exUGKYyDeUyQ7P/nHPs9P7Ht/B+ROrpN1EJOjc= github.com/ipfs/go-peertaskqueue v0.2.0/go.mod h1:5/eNrBEbtSKWCG+kQK8K8fGNixoYUnr+P7jivavs9lY= -github.com/ipfs/go-unixfs v0.0.4/go.mod h1:eIo/p9ADu/MFOuyxzwU+Th8D6xoxU//r590vUpWyfz8= +github.com/ipfs/go-pinning-service-http-client v0.1.0 h1:Au0P4NglL5JfzhNSZHlZ1qra+IcJyO3RWMd9EYCwqSY= +github.com/ipfs/go-pinning-service-http-client v0.1.0/go.mod h1:tcCKmlkWWH9JUUkKs8CrOZBanacNc1dmKLfjlyXAMu4= github.com/ipfs/go-unixfs v0.1.0/go.mod h1:lysk5ELhOso8+Fed9U1QTGey2ocsfaZ18h0NCO2Fj9s= github.com/ipfs/go-unixfs v0.2.4 h1:6NwppOXefWIyysZ4LR/qUBPvXd5//8J3jiMdvpbw6Lo= github.com/ipfs/go-unixfs v0.2.4/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw= github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E= github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= -github.com/ipfs/interface-go-ipfs-core v0.3.0 h1:oZdLLfh256gPGcYPURjivj/lv296GIcr8mUqZUnXOEI= -github.com/ipfs/interface-go-ipfs-core v0.3.0/go.mod h1:Tihp8zxGpUeE3Tokr94L6zWZZdkRQvG5TL6i9MuNE+s= github.com/ipfs/interface-go-ipfs-core v0.4.0 h1:+mUiamyHIwedqP8ZgbCIwpy40oX7QcXUbo4CZOeJVJg= github.com/ipfs/interface-go-ipfs-core v0.4.0/go.mod h1:UJBcU6iNennuI05amq3FQ7g0JHUkibHFAfhfUIy927o= github.com/ipld/go-car v0.1.1-0.20201015032735-ff6ccdc46acc h1:BdI33Q56hLWG9Ef0WbQ7z+dwmbRYhTb45SMjw0RudbQ= @@ -486,7 +474,6 @@ github.com/ipld/go-ipld-prime v0.5.1-0.20201021195245-109253e8a018/go.mod h1:0xE github.com/ipld/go-ipld-prime-proto v0.0.0-20200922192210-9a2bfd4440a6/go.mod h1:3pHYooM9Ea65jewRwrb2u5uHZCNkNTe9ABsVB+SrkH0= github.com/ipld/go-ipld-prime-proto v0.1.0 h1:j7gjqrfwbT4+gXpHwEx5iMssma3mnctC7YaCimsFP70= github.com/ipld/go-ipld-prime-proto v0.1.0/go.mod h1:11zp8f3sHVgIqtb/c9Kr5ZGqpnCLF1IVTNOez9TopzE= -github.com/jackpal/gateway v1.0.4/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/gateway v1.0.5 h1:qzXWUJfuMdlLMtt0a3Dgt+xkWQiA5itDEITVJtuSwMc= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1 h1:i0LektDkO1QlrTm/cSuP+PyBCDnYvjPLGl4LdWEMiaA= @@ -551,7 +538,6 @@ github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoR github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c= github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= github.com/libp2p/go-conn-security v0.0.1/go.mod h1:bGmu51N0KU9IEjX7kl2PQjgZa40JQWnayTvNMgD/vyk= -github.com/libp2p/go-conn-security-multistream v0.0.1/go.mod h1:nc9vud7inQ+d6SO0I/6dSWrdMnHnzZNHeyUQqrAJulE= github.com/libp2p/go-conn-security-multistream v0.0.2/go.mod h1:nc9vud7inQ+d6SO0I/6dSWrdMnHnzZNHeyUQqrAJulE= github.com/libp2p/go-conn-security-multistream v0.1.0 h1:aqGmto+ttL/uJgX0JtQI0tD21CIEy5eYd1Hlp0juHY0= github.com/libp2p/go-conn-security-multistream v0.1.0/go.mod h1:aw6eD7LOsHEX7+2hJkDxw1MteijaVcI+/eP2/x3J1xc= @@ -567,7 +553,6 @@ github.com/libp2p/go-flow-metrics v0.0.2 h1:U5TvqfoyR6GVRM+bC15Ux1ltar1kbj6Zw6xO github.com/libp2p/go-flow-metrics v0.0.2/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= github.com/libp2p/go-flow-metrics v0.0.3 h1:8tAs/hSdNvUiLgtlSy3mxwxWP4I9y/jlkPFT7epKdeM= github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= -github.com/libp2p/go-libp2p v0.0.2/go.mod h1:Qu8bWqFXiocPloabFGUcVG4kk94fLvfC8mWTDdFC9wE= github.com/libp2p/go-libp2p v0.0.30/go.mod h1:XWT8FGHlhptAv1+3V/+J5mEpzyui/5bvFsNuWYs611A= github.com/libp2p/go-libp2p v0.1.0/go.mod h1:6D/2OBauqLUoqcADOJpn9WbKqvaM07tDw68qHM0BxUM= github.com/libp2p/go-libp2p v0.1.1/go.mod h1:I00BRo1UuUSdpuc8Q2mN7yDF/oTUTRAX6JWpTiK9Rp8= @@ -582,7 +567,6 @@ github.com/libp2p/go-libp2p v0.12.0 h1:+xai9RQnQ9l5elFOKvp5wRyjyWisSwEx+6nU2+onp github.com/libp2p/go-libp2p v0.12.0/go.mod h1:FpHZrfC1q7nA8jitvdjKBDF31hguaC676g/nT9PgQM0= github.com/libp2p/go-libp2p-asn-util v0.0.0-20200825225859-85005c6cf052 h1:BM7aaOF7RpmNn9+9g6uTjGJ0cTzWr5j9i9IKeun2M8U= github.com/libp2p/go-libp2p-asn-util v0.0.0-20200825225859-85005c6cf052/go.mod h1:nRMRTab+kZuk0LnKZpxhOVH/ndsdr2Nr//Zltc/vwgo= -github.com/libp2p/go-libp2p-autonat v0.0.2/go.mod h1:fs71q5Xk+pdnKU014o2iq1RhMs9/PMaG5zXRFNnIIT4= github.com/libp2p/go-libp2p-autonat v0.0.6/go.mod h1:uZneLdOkZHro35xIhpbtTzLlgYturpu4J5+0cZK3MqE= github.com/libp2p/go-libp2p-autonat v0.1.0 h1:aCWAu43Ri4nU0ZPO7NyLzUvvfqd0nE3dX0R/ZGYVgOU= github.com/libp2p/go-libp2p-autonat v0.1.0/go.mod h1:1tLf2yXxiE/oKGtDwPYWTSYG3PtvYlJmg7NeVtPRqH8= @@ -602,7 +586,6 @@ github.com/libp2p/go-libp2p-blankhost v0.1.4 h1:I96SWjR4rK9irDHcHq3XHN6hawCRTPUA github.com/libp2p/go-libp2p-blankhost v0.1.4/go.mod h1:oJF0saYsAXQCSfDq254GMNmLNz6ZTHTOvtF4ZydUvwU= github.com/libp2p/go-libp2p-blankhost v0.2.0 h1:3EsGAi0CBGcZ33GwRuXEYJLLPoVWyXJ1bcJzAJjINkk= github.com/libp2p/go-libp2p-blankhost v0.2.0/go.mod h1:eduNKXGTioTuQAUcZ5epXi9vMl+t4d8ugUBRQ4SqaNQ= -github.com/libp2p/go-libp2p-circuit v0.0.1/go.mod h1:Dqm0s/BiV63j8EEAs8hr1H5HudqvCAeXxDyic59lCwE= github.com/libp2p/go-libp2p-circuit v0.0.9/go.mod h1:uU+IBvEQzCu953/ps7bYzC/D/R0Ho2A9LfKVVCatlqU= github.com/libp2p/go-libp2p-circuit v0.1.0/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFkZeIRmfunbA7pmFh8= github.com/libp2p/go-libp2p-circuit v0.1.4 h1:Phzbmrg3BkVzbqd4ZZ149JxCuUWu2wZcXf/Kr6hZJj8= @@ -650,7 +633,6 @@ github.com/libp2p/go-libp2p-crypto v0.0.1/go.mod h1:yJkNyDmO341d5wwXxDUGO0LykUVT github.com/libp2p/go-libp2p-crypto v0.0.2/go.mod h1:eETI5OUfBnvARGOHrJz2eWNyTUxEGZnBxMcbUjfIj4I= github.com/libp2p/go-libp2p-crypto v0.1.0 h1:k9MFy+o2zGDNGsaoZl0MA3iZ75qXxr9OOoAZF+sD5OQ= github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI= -github.com/libp2p/go-libp2p-discovery v0.0.1/go.mod h1:ZkkF9xIFRLA1xCc7bstYFkd80gBGK8Fc1JqGoU2i+zI= github.com/libp2p/go-libp2p-discovery v0.0.5/go.mod h1:YtF20GUxjgoKZ4zmXj8j3Nb2TUSBHFlOCetzYdbZL5I= github.com/libp2p/go-libp2p-discovery v0.1.0 h1:j+R6cokKcGbnZLf4kcNwpx6mDEUPF3N6SrqMymQhmvs= github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFTGElt8HnoDzwkFZm29g= @@ -688,7 +670,6 @@ github.com/libp2p/go-libp2p-mplex v0.2.3 h1:2zijwaJvpdesST2MXpI5w9wWFRgYtMcpRX7r github.com/libp2p/go-libp2p-mplex v0.2.3/go.mod h1:CK3p2+9qH9x+7ER/gWWDYJ3QW5ZxWDkm+dVvjfuG3ek= github.com/libp2p/go-libp2p-mplex v0.3.0 h1:CZyqqKP0BSGQyPLvpRQougbfXaaaJZdGgzhCpJNuNSk= github.com/libp2p/go-libp2p-mplex v0.3.0/go.mod h1:l9QWxRbbb5/hQMECEb908GbS9Sm2UAR2KFZKUJEynEs= -github.com/libp2p/go-libp2p-nat v0.0.2/go.mod h1:QrjXQSD5Dj4IJOdEcjHRkWTSomyxRo6HnUkf/TfQpLQ= github.com/libp2p/go-libp2p-nat v0.0.4 h1:+KXK324yaY701On8a0aGjTnw8467kW3ExKcqW2wwmyw= github.com/libp2p/go-libp2p-nat v0.0.4/go.mod h1:N9Js/zVtAXqaeT99cXgTV9e75KpnWCvVOiGzlcHmBbY= github.com/libp2p/go-libp2p-nat v0.0.5 h1:/mH8pXFVKleflDL1YwqMg27W9GD8kjEx7NY0P6eGc98= @@ -743,7 +724,6 @@ github.com/libp2p/go-libp2p-record v0.1.3/go.mod h1:yNUff/adKIfPnYQXgp6FQmNu3gLJ github.com/libp2p/go-libp2p-routing v0.0.1/go.mod h1:N51q3yTr4Zdr7V8Jt2JIktVU+3xBBylx1MZeVA6t1Ys= github.com/libp2p/go-libp2p-routing-helpers v0.2.3 h1:xY61alxJ6PurSi+MXbywZpelvuU4U4p/gPTxjqCqTzY= github.com/libp2p/go-libp2p-routing-helpers v0.2.3/go.mod h1:795bh+9YeoFl99rMASoiVgHdi5bjack0N1+AFAdbvBw= -github.com/libp2p/go-libp2p-secio v0.0.1/go.mod h1:IdG6iQybdcYmbTzxp4J5dwtUEDTOvZrT0opIDVNPrJs= github.com/libp2p/go-libp2p-secio v0.0.3/go.mod h1:hS7HQ00MgLhRO/Wyu1bTX6ctJKhVpm+j2/S2A5UqYb0= github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= github.com/libp2p/go-libp2p-secio v0.2.0 h1:ywzZBsWEEz2KNTn5RtzauEDq5RFEefPsttXYwAWqHng= @@ -752,7 +732,6 @@ github.com/libp2p/go-libp2p-secio v0.2.1 h1:eNWbJTdyPA7NxhP7J3c5lT97DC5d+u+Ildkg github.com/libp2p/go-libp2p-secio v0.2.1/go.mod h1:cWtZpILJqkqrSkiYcDBh5lA3wbT2Q+hz3rJQq3iftD8= github.com/libp2p/go-libp2p-secio v0.2.2 h1:rLLPvShPQAcY6eNurKNZq3eZjPWfU9kXF2eI9jIYdrg= github.com/libp2p/go-libp2p-secio v0.2.2/go.mod h1:wP3bS+m5AUnFA+OFO7Er03uO1mncHG0uVwGrwvjYlNY= -github.com/libp2p/go-libp2p-swarm v0.0.1/go.mod h1:mh+KZxkbd3lQnveQ3j2q60BM1Cw2mX36XXQqwfPOShs= github.com/libp2p/go-libp2p-swarm v0.0.6/go.mod h1:s5GZvzg9xXe8sbeESuFpjt8CJPTCa8mhEusweJqyFy8= github.com/libp2p/go-libp2p-swarm v0.1.0/go.mod h1:wQVsCdjsuZoc730CgOvh5ox6K8evllckjebkdiY5ta4= github.com/libp2p/go-libp2p-swarm v0.2.2 h1:T4hUpgEs2r371PweU3DuH7EOmBIdTBCwWs+FLcgx3bQ= @@ -778,9 +757,7 @@ github.com/libp2p/go-libp2p-testing v0.3.0/go.mod h1:efZkql4UZ7OVsEfaxNHZPzIehts github.com/libp2p/go-libp2p-tls v0.1.3 h1:twKMhMu44jQO+HgQK9X8NHO5HkeJu2QbhLzLJpa8oNM= github.com/libp2p/go-libp2p-tls v0.1.3/go.mod h1:wZfuewxOndz5RTnCAxFliGjvYSDA40sKitV4c50uI1M= github.com/libp2p/go-libp2p-transport v0.0.1/go.mod h1:UzbUs9X+PHOSw7S3ZmeOxfnwaQY5vGDzZmKPod3N3tk= -github.com/libp2p/go-libp2p-transport v0.0.4/go.mod h1:StoY3sx6IqsP6XKoabsPnHCwqKXWUMWU7Rfcsubee/A= github.com/libp2p/go-libp2p-transport v0.0.5/go.mod h1:StoY3sx6IqsP6XKoabsPnHCwqKXWUMWU7Rfcsubee/A= -github.com/libp2p/go-libp2p-transport-upgrader v0.0.1/go.mod h1:NJpUAgQab/8K6K0m+JmZCe5RUXG10UMEx4kWe9Ipj5c= github.com/libp2p/go-libp2p-transport-upgrader v0.0.4/go.mod h1:RGq+tupk+oj7PzL2kn/m1w6YXxcIAYJYeI90h6BGgUc= github.com/libp2p/go-libp2p-transport-upgrader v0.1.1 h1:PZMS9lhjK9VytzMCW3tWHAXtKXmlURSc3ZdvwEcKCzw= github.com/libp2p/go-libp2p-transport-upgrader v0.1.1/go.mod h1:IEtA6or8JUbsV07qPW4r01GnTenLW4oi3lOPbUMGJJA= @@ -808,7 +785,6 @@ github.com/libp2p/go-maddr-filter v0.0.4/go.mod h1:6eT12kSQMA9x2pvFQa+xesMKUBlj9 github.com/libp2p/go-maddr-filter v0.0.5 h1:CW3AgbMO6vUvT4kf87y4N+0P8KUl2aqLYhrGyDUbLSg= github.com/libp2p/go-maddr-filter v0.0.5/go.mod h1:Jk+36PMfIqCJhAnaASRH83bdAvfDRp/w6ENFaC9bG+M= github.com/libp2p/go-maddr-filter v0.1.0/go.mod h1:VzZhTXkMucEGGEOSKddrwGiOv0tUhgnKqNEmIAz/bPU= -github.com/libp2p/go-mplex v0.0.1/go.mod h1:pK5yMLmOoBR1pNCqDlA2GQrdAVTMkqFalaTWe7l4Yd0= github.com/libp2p/go-mplex v0.0.3/go.mod h1:pK5yMLmOoBR1pNCqDlA2GQrdAVTMkqFalaTWe7l4Yd0= github.com/libp2p/go-mplex v0.0.4/go.mod h1:pK5yMLmOoBR1pNCqDlA2GQrdAVTMkqFalaTWe7l4Yd0= github.com/libp2p/go-mplex v0.1.0 h1:/nBTy5+1yRyY82YaO6HXQRnO5IAGsXTjEJaR3LdTPc0= @@ -819,7 +795,6 @@ github.com/libp2p/go-mplex v0.1.2 h1:qOg1s+WdGLlpkrczDqmhYzyk3vCfsQ8+RxRTQjOZWwI github.com/libp2p/go-mplex v0.1.2/go.mod h1:Xgz2RDCi3co0LeZfgjm4OgUF15+sVR8SRcu3SFXI1lk= github.com/libp2p/go-mplex v0.2.0 h1:Ov/D+8oBlbRkjBs1R1Iua8hJ8cUfbdiW8EOdZuxcgaI= github.com/libp2p/go-mplex v0.2.0/go.mod h1:0Oy/A9PQlwBytDRp4wSkFnzHYDKcpLot35JQ6msjvYQ= -github.com/libp2p/go-msgio v0.0.1/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.2/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.3/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.4 h1:agEFehY3zWJFUHK6SEMR7UYmk2z6kC3oeCM7ybLhguA= @@ -850,7 +825,6 @@ github.com/libp2p/go-reuseport v0.0.1 h1:7PhkfH73VXfPJYKQ6JwS5I/eVcoyYi9IMNGc6FW github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA= github.com/libp2p/go-reuseport v0.0.2 h1:XSG94b1FJfGA01BUrT82imejHQyTxO4jEWqheyCXYvU= github.com/libp2p/go-reuseport v0.0.2/go.mod h1:SPD+5RwGC7rcnzngoYC86GjPzjSywuQyMVAheVBD9nQ= -github.com/libp2p/go-reuseport-transport v0.0.1/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs= github.com/libp2p/go-reuseport-transport v0.0.2 h1:WglMwyXyBu61CMkjCCtnmqNqnjib0GIEjMiHTwR/KN4= github.com/libp2p/go-reuseport-transport v0.0.2/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs= github.com/libp2p/go-reuseport-transport v0.0.3 h1:zzOeXnTooCkRvoH+bSXEfXhn76+LAiwoneM0gnXjF2M= @@ -868,7 +842,6 @@ github.com/libp2p/go-stream-muxer-multistream v0.2.0 h1:714bRJ4Zy9mdhyTLJ+ZKiROm github.com/libp2p/go-stream-muxer-multistream v0.2.0/go.mod h1:j9eyPol/LLRqT+GPLSxvimPhNph4sfYfMoDPd7HkzIc= github.com/libp2p/go-stream-muxer-multistream v0.3.0 h1:TqnSHPJEIqDEO7h1wZZ0p3DXdvDSiLHQidKKUGZtiOY= github.com/libp2p/go-stream-muxer-multistream v0.3.0/go.mod h1:yDh8abSIzmZtqtOt64gFJUXEryejzNb0lisTt+fAMJA= -github.com/libp2p/go-tcp-transport v0.0.1/go.mod h1:mnjg0o0O5TmXUaUIanYPUqkW4+u6mK0en8rlpA6BBTs= github.com/libp2p/go-tcp-transport v0.0.4/go.mod h1:+E8HvC8ezEVOxIo3V5vCK9l1y/19K427vCzQ+xHKH/o= github.com/libp2p/go-tcp-transport v0.1.0/go.mod h1:oJ8I5VXryj493DEJ7OsBieu8fcg2nHGctwtInJVpipc= github.com/libp2p/go-tcp-transport v0.1.1 h1:yGlqURmqgNA2fvzjSgZNlHcsd/IulAnKM8Ncu+vlqnw= @@ -879,7 +852,6 @@ github.com/libp2p/go-tcp-transport v0.2.1 h1:ExZiVQV+h+qL16fzCWtd1HSzPsqWottJ8KX github.com/libp2p/go-tcp-transport v0.2.1/go.mod h1:zskiJ70MEfWz2MKxvFB/Pv+tPIB1PpPUrHIWQ8aFw7M= github.com/libp2p/go-testutil v0.0.1/go.mod h1:iAcJc/DKJQanJ5ws2V+u5ywdL2n12X1WbbEG+Jjy69I= github.com/libp2p/go-testutil v0.1.0/go.mod h1:81b2n5HypcVyrCg/MJx4Wgfp/VHojytjVe/gLzZ2Ehc= -github.com/libp2p/go-ws-transport v0.0.1/go.mod h1:p3bKjDWHEgtuKKj+2OdPYs5dAPIjtpQGHF2tJfGz7Ww= github.com/libp2p/go-ws-transport v0.0.5/go.mod h1:Qbl4BxPfXXhhd/o0wcrgoaItHqA9tnZjoFZnxykuaXU= github.com/libp2p/go-ws-transport v0.1.0/go.mod h1:rjw1MG1LU9YDC6gzmwObkPd/Sqwhw7yT74kj3raBFuo= github.com/libp2p/go-ws-transport v0.2.0 h1:MJCw2OrPA9+76YNRvdo1wMnSOxb9Bivj6sVFY1Xrj6w= @@ -933,7 +905,6 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= -github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.12 h1:WMhc1ik4LNkTg8U9l3hI1LvxKmIL+f1+WV/SZtCbDDA= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.28 h1:gQhy5bsJa8zTlVI8lywCTZp1lguor+xevFoYlzeCTQY= @@ -950,6 +921,7 @@ github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKU github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -1011,7 +983,6 @@ github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77 github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= -github.com/multiformats/go-multihash v0.0.7/go.mod h1:XuKXPp8VHcTygube3OWZC+aZrA+H1IhmjoCDtJc7PXM= github.com/multiformats/go-multihash v0.0.8 h1:wrYcW5yxSi3dU07n5jnuS5PrNwyHy0zRHGVoUugWvXg= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.9 h1:aoijQXYYl7Xtb2pUUP68R+ys1TlnlR3eX6wmozr0Hp4= @@ -1205,10 +1176,6 @@ github.com/whyrusleeping/go-logging v0.0.1 h1:fwpzlmT0kRC/Fmd0MdmGgJG/CXIZ6gFq46 github.com/whyrusleeping/go-logging v0.0.1/go.mod h1:lDPYj54zutzG1XYfHAhcc7oNXEburHQBn+Iqd4yS4vE= github.com/whyrusleeping/go-notifier v0.0.0-20170827234753-097c5d47330f h1:M/lL30eFZTKnomXY6huvM6G0+gVquFNf6mxghaWlFUg= github.com/whyrusleeping/go-notifier v0.0.0-20170827234753-097c5d47330f/go.mod h1:cZNvX9cFybI01GriPRMXDtczuvUhgbcYr9iCGaNlRv8= -github.com/whyrusleeping/go-smux-multiplex v3.0.16+incompatible/go.mod h1:34LEDbeKFZInPUrAG+bjuJmUXONGdEFW7XL0SpTY1y4= -github.com/whyrusleeping/go-smux-multistream v2.0.2+incompatible/go.mod h1:dRWHHvc4HDQSHh9gbKEBbUZ+f2Q8iZTPG3UOGYODxSQ= -github.com/whyrusleeping/go-smux-yamux v2.0.8+incompatible/go.mod h1:6qHUzBXUbB9MXmw3AUdB52L8sEb/hScCqOdW2kj/wuI= -github.com/whyrusleeping/go-smux-yamux v2.0.9+incompatible/go.mod h1:6qHUzBXUbB9MXmw3AUdB52L8sEb/hScCqOdW2kj/wuI= github.com/whyrusleeping/go-sysinfo v0.0.0-20190219211824-4a357d4b90b1 h1:ctS9Anw/KozviCCtK6VWMz5kPL9nbQzbQY4yfqlIV4M= github.com/whyrusleeping/go-sysinfo v0.0.0-20190219211824-4a357d4b90b1/go.mod h1:tKH72zYNt/exx6/5IQO6L9LoQ0rEjd5SbbWaDTs9Zso= github.com/whyrusleeping/mafmt v1.2.8 h1:TCghSl5kkwEE0j+sU/gudyhVMRlpBin8fMBBHg59EbA= @@ -1222,7 +1189,6 @@ github.com/whyrusleeping/tar-utils v0.0.0-20201201191210-20a61371de5b h1:wA3QeTs github.com/whyrusleeping/tar-utils v0.0.0-20201201191210-20a61371de5b/go.mod h1:xT1Y5p2JR2PfSZihE0s4mjdJaRGp1waCTf5JzhQLBck= github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow= github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:m2aV4LZI4Aez7dP5PMyVKEHhUyEJ/RjmPEDOpDvudHg= -github.com/whyrusleeping/yamux v1.1.5/go.mod h1:E8LnQQ8HKx5KD29HZFUwM1PxCOdPRzGwur1mcYhXcD8= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1332,7 +1298,6 @@ golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180524181706-dfa909b99c79/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1372,6 +1337,7 @@ golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAG golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1503,6 +1469,7 @@ google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh new file mode 100755 index 00000000000..a033f87457f --- /dev/null +++ b/test/sharness/t0700-remotepin.sh @@ -0,0 +1,271 @@ +#!/usr/bin/env bash + +test_description="Test ipfs remote pinning operations" + +. lib/test-lib.sh + +if [ -z ${DOCKER_HOST+x} ]; then + # TODO: set up instead of skipping? + skip_all='Skipping pinning service integration tests: missing DOCKER_HOST, remote pinning service not available' + test_done +fi + +# daemon running in online mode to ensure Pin.origins/PinStatus.delegates work +test_init_ipfs +test_launch_ipfs_daemon + +# create user on pinning service +TEST_PIN_SVC="http://${DOCKER_HOST}:5000/api/v1" +TEST_PIN_SVC_KEY=$(curl -s -X POST "$TEST_PIN_SVC/users" -d email="go-ipfs-sharness@ipfs.example.com" | jq --raw-output .access_token) + +# pin remote service add|ls|rm + +# add valid and invalid services +test_expect_success "creating test user on remote pinning service" ' + echo CI host IP address ${TEST_PIN_SVC} && + ipfs pin remote service add test_pin_svc ${TEST_PIN_SVC} ${TEST_PIN_SVC_KEY} && + ipfs pin remote service add test_invalid_key_svc ${TEST_PIN_SVC} fake_api_key && + ipfs pin remote service add test_invalid_url_path_svc ${TEST_PIN_SVC}/invalid-path fake_api_key && + ipfs pin remote service add test_invalid_url_dns_svc https://invalid-service.example.com fake_api_key +' + +test_expect_success "test 'ipfs pin remote service ls'" ' + ipfs pin remote service ls | tee ls_out && + grep -q test_pin_svc ls_out && + grep -q test_invalid_key_svc ls_out && + grep -q test_invalid_url_path_svc ls_out && + grep -q test_invalid_url_dns_svc ls_out +' + +# SECURITY of access tokens in Api.Key fields: +# Pinning.RemoteServices includes Api.Key, and we give it the same treatment +# as Identity.PrivKey to prevent exposing it on the network + +test_expect_success "'ipfs config Pinning' fails" ' + test_expect_code 1 ipfs config Pinning 2>&1 > config_out +' +test_expect_success "output does not include Api.Key" ' + test_expect_code 1 grep -q Key config_out +' + +test_expect_success "'ipfs config Pinning.RemoteServices.test_pin_svc.Api.Key' fails" ' + test_expect_code 1 ipfs config Pinning.RemoteServices.test_pin_svc.Api.Key 2> config_out +' + +test_expect_success "output includes meaningful error" ' + echo "Error: cannot show or change pinning services through this API (try: ipfs pin remote service --help)" > config_exp && + test_cmp config_exp config_out +' + +test_expect_success "'ipfs config Pinning.RemoteServices.test_pin_svc' fails" ' + test_expect_code 1 ipfs config Pinning.RemoteServices.test_pin_svc 2> config_out +' +test_expect_success "output includes meaningful error" ' + test_cmp config_exp config_out +' + +test_expect_success "'ipfs config show' doesn't include RemoteServices" ' + ipfs config show > show_config && + test_expect_code 1 grep RemoteServices show_config +' + +test_expect_success "'ipfs config replace' injects remote services back" ' + test_expect_code 1 grep -q -E "test_.+_svc" show_config && + ipfs config replace show_config && + test_expect_code 0 grep -q test_pin_svc "$IPFS_PATH/config" && + test_expect_code 0 grep -q test_invalid_key_svc "$IPFS_PATH/config" && + test_expect_code 0 grep -q test_invalid_url_path_svc "$IPFS_PATH/config" && + test_expect_code 0 grep -q test_invalid_url_dns_svc "$IPFS_PATH/config" +' + +# note: we remove Identity.PrivKey to ensure error is triggered by Pinning.RemoteServices +test_expect_success "'ipfs config replace' with remote services errors out" ' + jq -M "del(.Identity.PrivKey)" "$IPFS_PATH/config" | jq ".Pinning += { RemoteServices: {\"foo\": {} }}" > new_config && + test_expect_code 1 ipfs config replace - < new_config 2> replace_out +' +test_expect_success "output includes meaningful error" ' + echo "Error: cannot show or change pinning services through this API (try: ipfs pin remote service --help)" > replace_expected && + test_cmp replace_out replace_expected +' + +# /SECURITY + +test_expect_success "pin remote service ls --stat' returns numbers for a valid service" ' + ipfs pin remote service ls --stat | grep -E "^test_pin_svc.+[0-9]+/[0-9]+/[0-9]+/[0-9]+$" +' + +test_expect_success "pin remote service ls --enc=json --stat' returns valid status" " + ipfs pin remote service ls --stat --enc=json | jq --raw-output '.RemoteServices[] | select(.Service == \"test_pin_svc\") | .Stat.Status' | tee stat_out && + echo valid > stat_expected && + test_cmp stat_out stat_expected +" + +test_expect_success "pin remote service ls --stat' returns invalid status for invalid service" ' + ipfs pin remote service ls --stat | grep -E "^test_invalid_url_path_svc.+invalid$" +' + +test_expect_success "pin remote service ls --enc=json --stat' returns invalid status" " + ipfs pin remote service ls --stat --enc=json | jq --raw-output '.RemoteServices[] | select(.Service == \"test_invalid_url_path_svc\") | .Stat.Status' | tee stat_out && + echo invalid > stat_expected && + test_cmp stat_out stat_expected +" + +test_expect_success "pin remote service ls --enc=json' (without --stat) returns no Stat object" " + ipfs pin remote service ls --enc=json | jq --raw-output '.RemoteServices[] | select(.Service == \"test_invalid_url_path_svc\") | .Stat' | tee stat_out && + echo null > stat_expected && + test_cmp stat_out stat_expected +" + +test_expect_success "check connection to the test pinning service" ' + ipfs pin remote ls --service=test_pin_svc --enc=json +' + +test_expect_success "unauthorized pinning service calls fail" ' + test_expect_code 1 ipfs pin remote ls --service=test_invalid_key_svc +' + +test_expect_success "misconfigured pinning service calls fail (wrong path)" ' + test_expect_code 1 ipfs pin remote ls --service=test_invalid_url_path_svc +' + +test_expect_success "misconfigured pinning service calls fail (dns error)" ' + test_expect_code 1 ipfs pin remote ls --service=test_invalid_url_dns_svc +' + +# pin remote service rm + +test_expect_success "remove pinning service" ' + ipfs pin remote service rm test_invalid_key_svc && + ipfs pin remote service rm test_invalid_url_path_svc && + ipfs pin remote service rm test_invalid_url_dns_svc +' + +test_expect_success "verify pinning service removal works" ' + ipfs pin remote service ls | tee ls_out && + test_expect_code 1 grep test_invalid_key_svc ls_out && + test_expect_code 1 grep test_invalid_url_path_svc ls_out && + test_expect_code 1 grep test_invalid_url_dns_svc ls_out +' + +# pin remote add + +# we leverage the fact that inlined CID can be pinned instantly on the remote service +# (https://github.com/ipfs-shipyard/rb-pinning-service-api/issues/8) +# below test ensures that assumption is correct (before we proceed to actual tests) +test_expect_success "verify that default add (implicit --background=false) works with data inlined in CID" ' + ipfs pin remote add --service=test_pin_svc --name=inlined_null bafkqaaa && + ipfs pin remote ls --service=test_pin_svc --enc=json --name=inlined_null --status=pinned | jq --raw-output .Status | tee ls_out && + grep -q "pinned" ls_out +' + +test_remote_pins() { + BASE=$1 + if [ -n "$BASE" ]; then + BASE_ARGS="--cid-base=$BASE" + fi + + # note: HAS_MISSING is not inlined nor imported to IPFS on purpose, to reliably test 'queued' state + test_expect_success "create some hashes using base $BASE" ' + export HASH_A=$(echo -n "A @ $(date +%s.%N)" | ipfs add $BASE_ARGS -q --inline --inline-limit 1000 --pin=false) && + export HASH_B=$(echo -n "B @ $(date +%s.%N)" | ipfs add $BASE_ARGS -q --inline --inline-limit 1000 --pin=false) && + export HASH_C=$(echo -n "C @ $(date +%s.%N)" | ipfs add $BASE_ARGS -q --inline --inline-limit 1000 --pin=false) && + export HASH_MISSING=$(echo "MISSING FROM IPFS @ $(date +%s.%N)" | ipfs add $BASE_ARGS -q --only-hash) && + echo "A: $HASH_A" && + echo "B: $HASH_B" && + echo "C: $HASH_C" && + echo "M: $HASH_MISSING" + ' + + test_expect_success "'ipfs pin remote add --background=true'" ' + ipfs pin remote add --background=true --service=test_pin_svc --enc=json $BASE_ARGS --name=name_a $HASH_A + ' + + test_expect_success "verify background add worked (instantly pinned variant)" ' + ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_a | tee ls_out && + test_expect_code 0 grep -q name_a ls_out && + test_expect_code 0 grep -q $HASH_A ls_out + ' + + test_expect_success "'ipfs pin remote add --background=true' with CID that is not available" ' + test_expect_code 0 ipfs pin remote add --background=true --service=test_pin_svc --enc=json $BASE_ARGS --name=name_m $HASH_MISSING + ' + + test_expect_success "verify background add worked (queued variant)" ' + ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_m --status=queued,pinning | tee ls_out && + test_expect_code 0 grep -q name_m ls_out && + test_expect_code 0 grep -q $HASH_MISSING ls_out + ' + + test_expect_success "'ipfs pin remote add --background=false'" ' + test_expect_code 0 ipfs pin remote add --background=false --service=test_pin_svc --enc=json $BASE_ARGS --name=name_b $HASH_B + ' + + test_expect_success "verify foreground add worked" ' + ipfs pin remote ls --service=test_pin_svc --enc=json $ID_B | tee ls_out && + test_expect_code 0 grep -q name_b ls_out && + test_expect_code 0 grep -q pinned ls_out && + test_expect_code 0 grep -q $HASH_B ls_out + ' + + test_expect_success "'ipfs pin remote ls' for existing pins by multiple statuses" ' + ipfs pin remote ls --service=test_pin_svc --enc=json --status=queued,pinning,pinned,failed | tee ls_out && + test_expect_code 0 grep -q $HASH_A ls_out && + test_expect_code 0 grep -q $HASH_B ls_out && + test_expect_code 0 grep -q $HASH_MISSING ls_out + ' + + test_expect_success "'ipfs pin remote ls' for existing pins by CID" ' + ipfs pin remote ls --service=test_pin_svc --enc=json --cid=$HASH_B | tee ls_out && + test_expect_code 0 grep -q $HASH_B ls_out + ' + + test_expect_success "'ipfs pin remote ls' for existing pins by name" ' + ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_a | tee ls_out && + test_expect_code 0 grep -q $HASH_A ls_out + ' + + test_expect_success "'ipfs pin remote ls' for ongoing pins by status" ' + ipfs pin remote ls --service=test_pin_svc --status=queued,pinning | tee ls_out && + test_expect_code 0 grep -q $HASH_MISSING ls_out + ' + + # --force is required only when more than a single match is found, + # so we add second pin with the same name (but different CID) to simulate that scenario + test_expect_success "'ipfs pin remote rm --name' fails without --force when matching multiple pins" ' + test_expect_code 0 ipfs pin remote add --service=test_pin_svc --enc=json $BASE_ARGS --name=name_b $HASH_C && + test_expect_code 1 ipfs pin remote rm --service=test_pin_svc --name=name_b 2> rm_out && + echo "Error: multiple remote pins are matching this query, add --force to confirm the bulk removal" > rm_expected && + test_cmp rm_out rm_expected + ' + + test_expect_success "'ipfs pin remote rm --name' without --force did not remove matching pins" ' + ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_b | jq --raw-output .Cid | tee ls_out && + test_expect_code 0 grep -q $HASH_B ls_out && + test_expect_code 0 grep -q $HASH_C ls_out + ' + + test_expect_success "'ipfs pin remote rm --name' with --force removes all matching pins" ' + test_expect_code 0 ipfs pin remote rm --service=test_pin_svc --name=name_b --force && + ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_b | jq --raw-output .Cid | tee ls_out && + test_expect_code 1 grep -q $HASH_B ls_out && + test_expect_code 1 grep -q $HASH_C ls_out + ' + + test_expect_success "'ipfs pin remote rm --force' removes all pinned items" ' + ipfs pin remote ls --service=test_pin_svc --enc=json --status=queued,pinning,pinned,failed | jq --raw-output .Cid | tee ls_out && + test_expect_code 0 grep -q $HASH_A ls_out && + test_expect_code 0 grep -q $HASH_MISSING ls_out && + ipfs pin remote rm --service=test_pin_svc --status=queued,pinning,pinned,failed --force && + ipfs pin remote ls --service=test_pin_svc --enc=json --status=queued,pinning,pinned,failed | jq --raw-output .Cid | tee ls_out && + test_expect_code 1 grep -q $HASH_A ls_out && + test_expect_code 1 grep -q $HASH_MISSING ls_out + ' + +} + +test_remote_pins "" + +test_kill_ipfs_daemon +test_done + +# vim: ts=2 sw=2 sts=2 et: