diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a3b80f1ea..fb163d053 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -27,7 +27,7 @@ updates: - package-ecosystem: gomod directory: "/services/dropbox" schedule: - interval: daily + interval: daily - package-ecosystem: gomod directory: "/services/ftp" schedule: @@ -35,12 +35,16 @@ updates: - package-ecosystem: gomod directory: "/services/gcs" schedule: - interval: daily + interval: daily - package-ecosystem: gomod directory: "/services/gdrive" schedule: - interval: daily + interval: daily - package-ecosystem: gomod directory: "/services/hdfs" schedule: - interval: daily + interval: daily + - package-ecosystem: gomod + directory: "/services/ipfs" + schedule: + interval: daily diff --git a/.github/workflows/services-test-ipfs.yml b/.github/workflows/services-test-ipfs.yml new file mode 100644 index 000000000..290a7ef46 --- /dev/null +++ b/.github/workflows/services-test-ipfs.yml @@ -0,0 +1,43 @@ +name: "Services Test Ipfs" + +on: + push: + paths: + - 'services/ipfs/**' + pull_request: + paths: + - 'services/ipfs/**' + +jobs: + integration_test: + name: "Integration Test" + runs-on: ubuntu-latest + + strategy: + matrix: + go: [ "1.15", "1.16" ] + ipfs: ["0.8", "0.9"] + + steps: + - name: Set up Go 1.x + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go }} + + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Set up IPFS ${{ matrix.ipfs }} + uses: ibnesayeed/setup-ipfs@8f99e662ba25be9a1158fffdeea57e9024468bfd + id: ipfs_setup + with: + ipfs_version: ${{ matrix.ipfs }} + run_daemon: true + + - name: Test + env: + STORAGE_IPFS_INTEGRATION_TEST: "on" + STORAGE_IPFS_ENDPOINT: "http:127.0.0.1:5001" + STORAGE_IPFS_GATEWAY: "http:127.0.0.1:8080" + working-directory: services/ipfs + run: make integration_test diff --git a/services/ipfs/.gitignore b/services/ipfs/.gitignore new file mode 100644 index 000000000..b22085920 --- /dev/null +++ b/services/ipfs/.gitignore @@ -0,0 +1,7 @@ +coverage.* +bin/ +Makefile.env + +# IDE +.idea +.vscode \ No newline at end of file diff --git a/services/ipfs/CHANGELOG.md b/services/ipfs/CHANGELOG.md new file mode 100644 index 000000000..2f8174401 --- /dev/null +++ b/services/ipfs/CHANGELOG.md @@ -0,0 +1,34 @@ +# Change Log + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/) +and this project adheres to [Semantic Versioning](https://semver.org/). + +## [v0.2.0] - 2021-09-13 + +### Added + +- docs: RFC-13 Add Gateway Pair (#13) + +### Changed + +- ci: Enable auto merge for Dependabot +- docs: Update README (#22) + +### Fixed + +- fix: Code not generated + +### Upgraded + +- ci: Upgrade fetch-metadata +- build(deps): Bump github.com/beyondstorage/go-endpoint to 1.1.0 (#19) + +## v0.1.0 - 2021-07-15 + +### Added + +- Implement IPFS Storager, Copier, Mover, Direr. + +[v0.2.0]: https://github.com/beyondstorage/go-service-ipfs/compare/v0.1.0...v0.2.0 diff --git a/services/ipfs/Makefile b/services/ipfs/Makefile new file mode 100644 index 000000000..f92dccd49 --- /dev/null +++ b/services/ipfs/Makefile @@ -0,0 +1,43 @@ +SHELL := /bin/bash + +-include Makefile.env + +.PHONY: all check format lint build test generate tidy + +help: + @echo "Please use \`make \` where is one of" + @echo " check to do static check" + @echo " build to create bin directory and build" + @echo " generate to generate code" + @echo " test to run test" + +check: vet + +format: + go fmt ./... + +vet: + go vet ./... + +generate: + @echo "generate code" + go generate ./... + go fmt ./... + +build: tidy generate check + go build ./... + +test: + go test -race -coverprofile=coverage.txt -covermode=atomic -v . + go tool cover -html="coverage.txt" -o "coverage.html" + +integration_test: + go test -race -count=1 -covermode=atomic -v ./tests + +tidy: + go mod tidy + go mod verify + +clean: + @echo "clean generated files" + find . -type f -name 'generated.go' -delete diff --git a/services/ipfs/Makefile.env.example b/services/ipfs/Makefile.env.example new file mode 100644 index 000000000..8748001a9 --- /dev/null +++ b/services/ipfs/Makefile.env.example @@ -0,0 +1,3 @@ +export STORAGE_IPFS_INTEGRATION_TEST=on +export STORAGE_IPFS_ENDPOINT=http:127.0.0.1:5001 +export STORAGE_IPFS_GATEWAY=http:127.0.0.1:8080 diff --git a/services/ipfs/README.md b/services/ipfs/README.md new file mode 100644 index 000000000..4f6ca5091 --- /dev/null +++ b/services/ipfs/README.md @@ -0,0 +1,35 @@ +[![Services Test Ipfs](https://github.com/beyondstorage/go-storage/actions/workflows/services-test-ipfs.yml/badge.svg)](https://github.com/beyondstorage/go-storage/actions/workflows/services-test-ipfs.yml) + +# ipfs + +[InterPlanetary File System(IPFS)](https://ipfs.io/) support for [go-storage](https://github.com/beyondstorage/go-storage). + +## Install + +```go +go get go.beyondstorage.io/services/ipfs +``` + +## Usage + +```go +import ( + "log" + + _ "go.beyondstorage.io/services/ipfs" + "go.beyondstorage.io/v5/services" +) + +func main() { + store, err := services.NewStoragerFromString("ipfs:///path/to/workdir?endpoint=&gateway=") + if err != nil { + log.Fatal(err) + } + + // Write data from io.Reader into hello.txt + n, err := store.Write("hello.txt", r, length) +} +``` + +- See more examples in [go-storage-example](https://github.com/beyondstorage/go-storage-example). +- Read [more docs](https://beyondstorage.io/docs/go-storage/services/ipfs) about go-service-ipfs. diff --git a/services/ipfs/doc.go b/services/ipfs/doc.go new file mode 100644 index 000000000..1c30651ce --- /dev/null +++ b/services/ipfs/doc.go @@ -0,0 +1,6 @@ +/* +Package ipfs provided support for the InterPlanetary File System (IPFS). +*/ +package ipfs + +//go:generate go run -tags tools go.beyondstorage.io/v5/cmd/definitions service.toml diff --git a/services/ipfs/docs/rfcs/0-example.md b/services/ipfs/docs/rfcs/0-example.md new file mode 100644 index 000000000..0461f1387 --- /dev/null +++ b/services/ipfs/docs/rfcs/0-example.md @@ -0,0 +1,41 @@ +- Author: (fill me in with `name `, e.g., zu1k ) +- Start Date: (fill me in with today's date, YYYY-MM-DD) +- RFC PR: [beyondstorage/go-service-ipfs#0](https://github.com/beyondstorage/go-service-ipfs/issues/0) +- Tracking Issue: [beyondstorage/go-service-ipfs#0](https://github.com/beyondstorage/go-service-ipfs/issues/0) + +# RFC-0: + +- Updates: (delete this part if not applicable) + - [GSP-20](./20-abc): Deletes something +- Updated By: (delete this part if not applicable) + - [GSP-10](./10-do-be-do-be-do): Adds something + - [GSP-1000](./1000-lalala): Deprecates this RFC + +## Background + +Explain why we are doing this. + +Related issues and early discussions can be linked, but the RFC should try to be self-contained if possible. + +## Proposal + + + +## Rationale + + + +Possible content: + +- Design Principles +- Drawbacks +- Alternative implementations and comparison +- Possible Q&As + +## Compatibility + + + +## Implementation + +Explain what steps should be done to implement this proposal. diff --git a/services/ipfs/docs/rfcs/13-add-gateway-pair.md b/services/ipfs/docs/rfcs/13-add-gateway-pair.md new file mode 100644 index 000000000..18a1c7ce0 --- /dev/null +++ b/services/ipfs/docs/rfcs/13-add-gateway-pair.md @@ -0,0 +1,69 @@ +- Author: zu1k +- Start Date: 2021-07-15 +- RFC PR: [beyondstorage/go-service-ipfs#13](https://github.com/beyondstorage/go-service-ipfs/pull/13) +- Tracking Issue: [beyondstorage/go-service-ipfs#14](https://github.com/beyondstorage/go-service-ipfs/issues/14) + +# RFC-13: Add Gateway Pair + +Releated issue: [beyondstorage/go-service-ipfs#5](https://github.com/beyondstorage/go-service-ipfs/issues/5) + +## Background + +With IPFS, we can get the `CID` of an object, then we can use it with a public or internal IPFS Gateway to splice out the access link. + +When using internal gateway like `127.0.0.1:8080`, the access link are inaccessible from external network. + +When using a public gateway, e.g. `ipfs.io`, `cf-ipfs.com`, if the service is an intranet IPFS cluster, it is possible that the files are not accessible from these public gateway as well. + +### How to get the `CID` of an object? + +We can [stat](https://docs.ipfs.io/reference/http/api/#api-v0-files-stat) the object, the `hash` would be its `CID`. + +``` +$ ipfs files stat /part1 +QmSvxBhBHn2gu9kAdjCTYfUgDNnyBcNoXXVojRyKdBjZUA # CID +Size: 262144 +CumulativeSize: 262158 +ChildBlocks: 0 +Type: file +``` + +### How to splice out the access link? + +- https://{gateway URL}/ipfs/{CID}/{optional path to resource} +- https://{CID}.ipfs.{gatewayURL}/{optional path to resource} + +## Proposal + +I propose to add a pair to let the user specify the `gateway`. + +- The `type` of `gateway` should be `String` +- The `format` of `gateway` should follow [go-endpoint](https://github.com/beyondstorage/go-endpoint/blob/master/README.md) +- The `value` of `gateway` should be parsed into `HTTP` or `HTTPS` +- Now we use `gateway` only in `Reach` operation + +## Rationale + +### Why not infer the `gateway` from the `endpoint`? + +If the `endpoint` used is not the loopback address, it is possible to infer the `gateway` address. + +For example, if a user tells IPFS to listen to `0.0.0.0` and use the public IP to access, the `endpoint` may be `:5001` or `:5001`, we can infer the `gateway` as `:8080` or `:8080`. + +The problem with this approach is that it is very unstable and the IPFS api does not come with a forensic mechanism, so it is almost impossible for users to access directly through the public IP, so the inferred gateway is most likely invalid. + +### Why not use the `gateway` in the configuration? + +We can get the `gateway` of IPFS through the [config-show api](https://docs.ipfs.io/reference/http/api/#api-v0-config-show). + +This method works if the user has modified the default gateway. + +However, we cannot guarantee that the user has modified this configuration item, and it may be more common for users to use reverse proxy to make IPFS public. + +## Compatibility + +No compatibility issues at this time. + +## Implementation + +First we define a pair of `String` type with the name `gateway` in `service.toml`. Then we `generate` code and implement the `Reacher` interface. diff --git a/services/ipfs/generated.go b/services/ipfs/generated.go new file mode 100644 index 000000000..ebff2c16f --- /dev/null +++ b/services/ipfs/generated.go @@ -0,0 +1,798 @@ +// Code generated by go generate via cmd/definitions; DO NOT EDIT. +package ipfs + +import ( + "context" + "io" + "net/http" + "strings" + "time" + + . "go.beyondstorage.io/v5/pairs" + "go.beyondstorage.io/v5/pkg/httpclient" + "go.beyondstorage.io/v5/services" + . "go.beyondstorage.io/v5/types" +) + +var ( + _ Storager + _ services.ServiceError + _ httpclient.Options + _ time.Duration + _ http.Request + _ Error +) + +// Type is the type for ipfs +const Type = "ipfs" + +// ObjectSystemMetadata stores system metadata for object. +type ObjectSystemMetadata struct { + Blocks int + CumulativeSize uint64 + Hash string + Local bool + SizeLocal uint64 + WithLocality bool +} + +// GetObjectSystemMetadata will get ObjectSystemMetadata from Object. +// +// - This function should not be called by service implementer. +// - The returning ObjectServiceMetadata is read only and should not be modified. +func GetObjectSystemMetadata(o *Object) ObjectSystemMetadata { + sm, ok := o.GetSystemMetadata() + if ok { + return sm.(ObjectSystemMetadata) + } + return ObjectSystemMetadata{} +} + +// setObjectSystemMetadata will set ObjectSystemMetadata into Object. +// +// - This function should only be called once, please make sure all data has been written before set. +func setObjectSystemMetadata(o *Object, sm ObjectSystemMetadata) { + o.SetSystemMetadata(sm) +} + +// StorageSystemMetadata stores system metadata for object. +type StorageSystemMetadata struct { + Blocks int + CumulativeSize uint64 + Hash string + Local bool + SizeLocal uint64 + WithLocality bool +} + +// GetStorageSystemMetadata will get StorageSystemMetadata from Storage. +// +// - This function should not be called by service implementer. +// - The returning StorageServiceMetadata is read only and should not be modified. +func GetStorageSystemMetadata(s *StorageMeta) StorageSystemMetadata { + sm, ok := s.GetSystemMetadata() + if ok { + return sm.(StorageSystemMetadata) + } + return StorageSystemMetadata{} +} + +// setStorageSystemMetadata will set StorageSystemMetadata into Storage. +// +// - This function should only be called once, please make sure all data has been written before set. +func setStorageSystemMetadata(s *StorageMeta, sm StorageSystemMetadata) { + s.SetSystemMetadata(sm) +} + +// WithDefaultStoragePairs will apply default_storage_pairs value to Options. +// +// set default pairs for storager actions +func WithDefaultStoragePairs(v DefaultStoragePairs) Pair { + return Pair{Key: "default_storage_pairs", Value: v} +} + +// WithGateway will apply gateway value to Options. +// +// set storage gateway, for http(s) request purpose +func WithGateway(v string) Pair { + return Pair{Key: "gateway", Value: v} +} + +// WithStorageFeatures will apply storage_features value to Options. +// +// set storage features +func WithStorageFeatures(v StorageFeatures) Pair { + return Pair{Key: "storage_features", Value: v} +} + +var pairMap = map[string]string{"content_md5": "string", "content_type": "string", "context": "context.Context", "continuation_token": "string", "credential": "string", "default_content_type": "string", "default_io_callback": "func([]byte)", "default_storage_pairs": "DefaultStoragePairs", "endpoint": "string", "expire": "time.Duration", "gateway": "string", "http_client_options": "*httpclient.Options", "interceptor": "Interceptor", "io_callback": "func([]byte)", "list_mode": "ListMode", "location": "string", "multipart_id": "string", "name": "string", "object_mode": "ObjectMode", "offset": "int64", "size": "int64", "storage_features": "StorageFeatures", "work_dir": "string"} +var ( + _ Copier = &Storage{} + _ Direr = &Storage{} + _ Mover = &Storage{} + _ StorageHTTPSigner = &Storage{} + _ Storager = &Storage{} +) + +type StorageFeatures struct { +} + +// pairStorageNew is the parsed struct +type pairStorageNew struct { + pairs []Pair + + // Required pairs + HasEndpoint bool + Endpoint string + HasGateway bool + Gateway string + // Optional pairs + HasDefaultContentType bool + DefaultContentType string + HasDefaultIoCallback bool + DefaultIoCallback func([]byte) + HasDefaultStoragePairs bool + DefaultStoragePairs DefaultStoragePairs + HasStorageFeatures bool + StorageFeatures StorageFeatures + HasWorkDir bool + WorkDir string + // Enable features +} + +// parsePairStorageNew will parse Pair slice into *pairStorageNew +func parsePairStorageNew(opts []Pair) (pairStorageNew, error) { + result := + pairStorageNew{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "endpoint": + if result.HasEndpoint { + continue + } + result.HasEndpoint = true + result.Endpoint = v.Value.(string) + case "gateway": + if result.HasGateway { + continue + } + result.HasGateway = true + result.Gateway = v.Value.(string) + case "default_content_type": + if result.HasDefaultContentType { + continue + } + result.HasDefaultContentType = true + result.DefaultContentType = v.Value.(string) + case "default_io_callback": + if result.HasDefaultIoCallback { + continue + } + result.HasDefaultIoCallback = true + result.DefaultIoCallback = v.Value.(func([]byte)) + case "default_storage_pairs": + if result.HasDefaultStoragePairs { + continue + } + result.HasDefaultStoragePairs = true + result.DefaultStoragePairs = v.Value.(DefaultStoragePairs) + case "storage_features": + if result.HasStorageFeatures { + continue + } + result.HasStorageFeatures = true + result.StorageFeatures = v.Value.(StorageFeatures) + case "work_dir": + if result.HasWorkDir { + continue + } + result.HasWorkDir = true + result.WorkDir = v.Value.(string) + } + } + // Enable features + + // Default pairs + if result.HasDefaultContentType { + result.HasDefaultStoragePairs = true + result.DefaultStoragePairs.Write = append(result.DefaultStoragePairs.Write, WithContentType(result.DefaultContentType)) + } + if result.HasDefaultIoCallback { + result.HasDefaultStoragePairs = true + result.DefaultStoragePairs.Read = append(result.DefaultStoragePairs.Read, WithIoCallback(result.DefaultIoCallback)) + result.DefaultStoragePairs.Write = append(result.DefaultStoragePairs.Write, WithIoCallback(result.DefaultIoCallback)) + } + if !result.HasEndpoint { + return pairStorageNew{}, services.PairRequiredError{Keys: []string{"endpoint"}} + } + if !result.HasGateway { + return pairStorageNew{}, services.PairRequiredError{Keys: []string{"gateway"}} + } + return result, nil +} + +// DefaultStoragePairs is default pairs for specific action +type DefaultStoragePairs struct { + Copy []Pair + Create []Pair + CreateDir []Pair + Delete []Pair + List []Pair + Metadata []Pair + Move []Pair + QuerySignHTTPDelete []Pair + QuerySignHTTPRead []Pair + QuerySignHTTPWrite []Pair + Read []Pair + Stat []Pair + Write []Pair +} +type pairStorageCopy struct { + pairs []Pair + // Required pairs + // Optional pairs +} + +func (s *Storage) parsePairStorageCopy(opts []Pair) (pairStorageCopy, error) { + result := + pairStorageCopy{pairs: opts} + + for _, v := range opts { + switch v.Key { + default: + return pairStorageCopy{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageCreate struct { + pairs []Pair + // Required pairs + // Optional pairs + HasObjectMode bool + ObjectMode ObjectMode +} + +func (s *Storage) parsePairStorageCreate(opts []Pair) (pairStorageCreate, error) { + result := + pairStorageCreate{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "object_mode": + if result.HasObjectMode { + continue + } + result.HasObjectMode = true + result.ObjectMode = v.Value.(ObjectMode) + default: + return pairStorageCreate{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageCreateDir struct { + pairs []Pair + // Required pairs + // Optional pairs +} + +func (s *Storage) parsePairStorageCreateDir(opts []Pair) (pairStorageCreateDir, error) { + result := + pairStorageCreateDir{pairs: opts} + + for _, v := range opts { + switch v.Key { + default: + return pairStorageCreateDir{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageDelete struct { + pairs []Pair + // Required pairs + // Optional pairs + HasObjectMode bool + ObjectMode ObjectMode +} + +func (s *Storage) parsePairStorageDelete(opts []Pair) (pairStorageDelete, error) { + result := + pairStorageDelete{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "object_mode": + if result.HasObjectMode { + continue + } + result.HasObjectMode = true + result.ObjectMode = v.Value.(ObjectMode) + default: + return pairStorageDelete{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageList struct { + pairs []Pair + // Required pairs + // Optional pairs + HasListMode bool + ListMode ListMode +} + +func (s *Storage) parsePairStorageList(opts []Pair) (pairStorageList, error) { + result := + pairStorageList{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "list_mode": + if result.HasListMode { + continue + } + result.HasListMode = true + result.ListMode = v.Value.(ListMode) + default: + return pairStorageList{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageMetadata struct { + pairs []Pair + // Required pairs + // Optional pairs +} + +func (s *Storage) parsePairStorageMetadata(opts []Pair) (pairStorageMetadata, error) { + result := + pairStorageMetadata{pairs: opts} + + for _, v := range opts { + switch v.Key { + default: + return pairStorageMetadata{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageMove struct { + pairs []Pair + // Required pairs + // Optional pairs +} + +func (s *Storage) parsePairStorageMove(opts []Pair) (pairStorageMove, error) { + result := + pairStorageMove{pairs: opts} + + for _, v := range opts { + switch v.Key { + default: + return pairStorageMove{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageQuerySignHTTPDelete struct { + pairs []Pair + // Required pairs + // Optional pairs +} + +func (s *Storage) parsePairStorageQuerySignHTTPDelete(opts []Pair) (pairStorageQuerySignHTTPDelete, error) { + result := + pairStorageQuerySignHTTPDelete{pairs: opts} + + for _, v := range opts { + switch v.Key { + default: + return pairStorageQuerySignHTTPDelete{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageQuerySignHTTPRead struct { + pairs []Pair + // Required pairs + // Optional pairs +} + +func (s *Storage) parsePairStorageQuerySignHTTPRead(opts []Pair) (pairStorageQuerySignHTTPRead, error) { + result := + pairStorageQuerySignHTTPRead{pairs: opts} + + for _, v := range opts { + switch v.Key { + default: + return pairStorageQuerySignHTTPRead{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageQuerySignHTTPWrite struct { + pairs []Pair + // Required pairs + // Optional pairs +} + +func (s *Storage) parsePairStorageQuerySignHTTPWrite(opts []Pair) (pairStorageQuerySignHTTPWrite, error) { + result := + pairStorageQuerySignHTTPWrite{pairs: opts} + + for _, v := range opts { + switch v.Key { + default: + return pairStorageQuerySignHTTPWrite{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageRead struct { + pairs []Pair + // Required pairs + // Optional pairs + HasIoCallback bool + IoCallback func([]byte) + HasOffset bool + Offset int64 + HasSize bool + Size int64 +} + +func (s *Storage) parsePairStorageRead(opts []Pair) (pairStorageRead, error) { + result := + pairStorageRead{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "io_callback": + if result.HasIoCallback { + continue + } + result.HasIoCallback = true + result.IoCallback = v.Value.(func([]byte)) + case "offset": + if result.HasOffset { + continue + } + result.HasOffset = true + result.Offset = v.Value.(int64) + case "size": + if result.HasSize { + continue + } + result.HasSize = true + result.Size = v.Value.(int64) + default: + return pairStorageRead{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageStat struct { + pairs []Pair + // Required pairs + // Optional pairs + HasObjectMode bool + ObjectMode ObjectMode +} + +func (s *Storage) parsePairStorageStat(opts []Pair) (pairStorageStat, error) { + result := + pairStorageStat{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "object_mode": + if result.HasObjectMode { + continue + } + result.HasObjectMode = true + result.ObjectMode = v.Value.(ObjectMode) + default: + return pairStorageStat{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} + +type pairStorageWrite struct { + pairs []Pair + // Required pairs + // Optional pairs + HasContentMd5 bool + ContentMd5 string + HasContentType bool + ContentType string + HasIoCallback bool + IoCallback func([]byte) +} + +func (s *Storage) parsePairStorageWrite(opts []Pair) (pairStorageWrite, error) { + result := + pairStorageWrite{pairs: opts} + + for _, v := range opts { + switch v.Key { + case "content_md5": + if result.HasContentMd5 { + continue + } + result.HasContentMd5 = true + result.ContentMd5 = v.Value.(string) + case "content_type": + if result.HasContentType { + continue + } + result.HasContentType = true + result.ContentType = v.Value.(string) + case "io_callback": + if result.HasIoCallback { + continue + } + result.HasIoCallback = true + result.IoCallback = v.Value.(func([]byte)) + default: + return pairStorageWrite{}, services.PairUnsupportedError{Pair: v} + } + } + + return result, nil +} +func (s *Storage) Copy(src string, dst string, pairs ...Pair) (err error) { + ctx := context.Background() + return s.CopyWithContext(ctx, src, dst, pairs...) +} +func (s *Storage) CopyWithContext(ctx context.Context, src string, dst string, pairs ...Pair) (err error) { + defer func() { + err = + s.formatError("copy", err, src, dst) + }() + + pairs = append(pairs, s.defaultPairs.Copy...) + var opt pairStorageCopy + + opt, err = s.parsePairStorageCopy(pairs) + if err != nil { + return + } + return s.copy(ctx, strings.ReplaceAll(src, "\\", "/"), strings.ReplaceAll(dst, "\\", "/"), opt) +} +func (s *Storage) Create(path string, pairs ...Pair) (o *Object) { + pairs = append(pairs, s.defaultPairs.Create...) + var opt pairStorageCreate + + // Ignore error while handling local functions. + opt, _ = s.parsePairStorageCreate(pairs) + return s.create(path, opt) +} +func (s *Storage) CreateDir(path string, pairs ...Pair) (o *Object, err error) { + ctx := context.Background() + return s.CreateDirWithContext(ctx, path, pairs...) +} +func (s *Storage) CreateDirWithContext(ctx context.Context, path string, pairs ...Pair) (o *Object, err error) { + defer func() { + err = + s.formatError("create_dir", err, path) + }() + + pairs = append(pairs, s.defaultPairs.CreateDir...) + var opt pairStorageCreateDir + + opt, err = s.parsePairStorageCreateDir(pairs) + if err != nil { + return + } + return s.createDir(ctx, strings.ReplaceAll(path, "\\", "/"), opt) +} +func (s *Storage) Delete(path string, pairs ...Pair) (err error) { + ctx := context.Background() + return s.DeleteWithContext(ctx, path, pairs...) +} +func (s *Storage) DeleteWithContext(ctx context.Context, path string, pairs ...Pair) (err error) { + defer func() { + err = + s.formatError("delete", err, path) + }() + + pairs = append(pairs, s.defaultPairs.Delete...) + var opt pairStorageDelete + + opt, err = s.parsePairStorageDelete(pairs) + if err != nil { + return + } + return s.delete(ctx, strings.ReplaceAll(path, "\\", "/"), opt) +} +func (s *Storage) List(path string, pairs ...Pair) (oi *ObjectIterator, err error) { + ctx := context.Background() + return s.ListWithContext(ctx, path, pairs...) +} +func (s *Storage) ListWithContext(ctx context.Context, path string, pairs ...Pair) (oi *ObjectIterator, err error) { + defer func() { + err = + s.formatError("list", err, path) + }() + + pairs = append(pairs, s.defaultPairs.List...) + var opt pairStorageList + + opt, err = s.parsePairStorageList(pairs) + if err != nil { + return + } + return s.list(ctx, strings.ReplaceAll(path, "\\", "/"), opt) +} +func (s *Storage) Metadata(pairs ...Pair) (meta *StorageMeta) { + pairs = append(pairs, s.defaultPairs.Metadata...) + var opt pairStorageMetadata + + // Ignore error while handling local functions. + opt, _ = s.parsePairStorageMetadata(pairs) + return s.metadata(opt) +} +func (s *Storage) Move(src string, dst string, pairs ...Pair) (err error) { + ctx := context.Background() + return s.MoveWithContext(ctx, src, dst, pairs...) +} +func (s *Storage) MoveWithContext(ctx context.Context, src string, dst string, pairs ...Pair) (err error) { + defer func() { + err = + s.formatError("move", err, src, dst) + }() + + pairs = append(pairs, s.defaultPairs.Move...) + var opt pairStorageMove + + opt, err = s.parsePairStorageMove(pairs) + if err != nil { + return + } + return s.move(ctx, strings.ReplaceAll(src, "\\", "/"), strings.ReplaceAll(dst, "\\", "/"), opt) +} +func (s *Storage) QuerySignHTTPDelete(path string, expire time.Duration, pairs ...Pair) (req *http.Request, err error) { + ctx := context.Background() + return s.QuerySignHTTPDeleteWithContext(ctx, path, expire, pairs...) +} +func (s *Storage) QuerySignHTTPDeleteWithContext(ctx context.Context, path string, expire time.Duration, pairs ...Pair) (req *http.Request, err error) { + defer func() { + err = + s.formatError("query_sign_http_delete", err, path) + }() + + pairs = append(pairs, s.defaultPairs.QuerySignHTTPDelete...) + var opt pairStorageQuerySignHTTPDelete + + opt, err = s.parsePairStorageQuerySignHTTPDelete(pairs) + if err != nil { + return + } + return s.querySignHTTPDelete(ctx, strings.ReplaceAll(path, "\\", "/"), expire, opt) +} +func (s *Storage) QuerySignHTTPRead(path string, expire time.Duration, pairs ...Pair) (req *http.Request, err error) { + ctx := context.Background() + return s.QuerySignHTTPReadWithContext(ctx, path, expire, pairs...) +} +func (s *Storage) QuerySignHTTPReadWithContext(ctx context.Context, path string, expire time.Duration, pairs ...Pair) (req *http.Request, err error) { + defer func() { + err = + s.formatError("query_sign_http_read", err, path) + }() + + pairs = append(pairs, s.defaultPairs.QuerySignHTTPRead...) + var opt pairStorageQuerySignHTTPRead + + opt, err = s.parsePairStorageQuerySignHTTPRead(pairs) + if err != nil { + return + } + return s.querySignHTTPRead(ctx, strings.ReplaceAll(path, "\\", "/"), expire, opt) +} +func (s *Storage) QuerySignHTTPWrite(path string, size int64, expire time.Duration, pairs ...Pair) (req *http.Request, err error) { + ctx := context.Background() + return s.QuerySignHTTPWriteWithContext(ctx, path, size, expire, pairs...) +} +func (s *Storage) QuerySignHTTPWriteWithContext(ctx context.Context, path string, size int64, expire time.Duration, pairs ...Pair) (req *http.Request, err error) { + defer func() { + err = + s.formatError("query_sign_http_write", err, path) + }() + + pairs = append(pairs, s.defaultPairs.QuerySignHTTPWrite...) + var opt pairStorageQuerySignHTTPWrite + + opt, err = s.parsePairStorageQuerySignHTTPWrite(pairs) + if err != nil { + return + } + return s.querySignHTTPWrite(ctx, strings.ReplaceAll(path, "\\", "/"), size, expire, opt) +} +func (s *Storage) Read(path string, w io.Writer, pairs ...Pair) (n int64, err error) { + ctx := context.Background() + return s.ReadWithContext(ctx, path, w, pairs...) +} +func (s *Storage) ReadWithContext(ctx context.Context, path string, w io.Writer, pairs ...Pair) (n int64, err error) { + defer func() { + err = + s.formatError("read", err, path) + }() + + pairs = append(pairs, s.defaultPairs.Read...) + var opt pairStorageRead + + opt, err = s.parsePairStorageRead(pairs) + if err != nil { + return + } + return s.read(ctx, strings.ReplaceAll(path, "\\", "/"), w, opt) +} +func (s *Storage) Stat(path string, pairs ...Pair) (o *Object, err error) { + ctx := context.Background() + return s.StatWithContext(ctx, path, pairs...) +} +func (s *Storage) StatWithContext(ctx context.Context, path string, pairs ...Pair) (o *Object, err error) { + defer func() { + err = + s.formatError("stat", err, path) + }() + + pairs = append(pairs, s.defaultPairs.Stat...) + var opt pairStorageStat + + opt, err = s.parsePairStorageStat(pairs) + if err != nil { + return + } + return s.stat(ctx, strings.ReplaceAll(path, "\\", "/"), opt) +} +func (s *Storage) Write(path string, r io.Reader, size int64, pairs ...Pair) (n int64, err error) { + ctx := context.Background() + return s.WriteWithContext(ctx, path, r, size, pairs...) +} +func (s *Storage) WriteWithContext(ctx context.Context, path string, r io.Reader, size int64, pairs ...Pair) (n int64, err error) { + defer func() { + err = + s.formatError("write", err, path) + }() + + pairs = append(pairs, s.defaultPairs.Write...) + var opt pairStorageWrite + + opt, err = s.parsePairStorageWrite(pairs) + if err != nil { + return + } + return s.write(ctx, strings.ReplaceAll(path, "\\", "/"), r, size, opt) +} +func init() { + services.RegisterStorager(Type, NewStorager) + services.RegisterSchema(Type, pairMap) +} diff --git a/services/ipfs/go.mod b/services/ipfs/go.mod new file mode 100644 index 000000000..1f773d388 --- /dev/null +++ b/services/ipfs/go.mod @@ -0,0 +1,11 @@ +module go.beyondstorage.io/services/ipfs + +go 1.15 + +require ( + github.com/google/uuid v1.3.0 + github.com/ipfs/go-ipfs-api v0.2.0 + github.com/ipfs/go-ipfs-cmds v0.6.0 + go.beyondstorage.io/endpoint v1.2.0 + go.beyondstorage.io/v5 v5.0.0 +) diff --git a/services/ipfs/go.sum b/services/ipfs/go.sum new file mode 100644 index 000000000..45689ad58 --- /dev/null +++ b/services/ipfs/go.sum @@ -0,0 +1,272 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= +github.com/Xuanwo/gg v0.2.0 h1:axbZmA0qmidh3s9PA86GqvBXVQ3o7Bbpf0aImGtlimA= +github.com/Xuanwo/gg v0.2.0/go.mod h1:0fLiiSxR87u2UA0ZNZiKZXuz3jnJdbDHWtU2xpdcH3s= +github.com/Xuanwo/go-bufferpool v0.2.0 h1:DXzqJD9lJufXbT/03GrcEvYOs4gXYUj9/g5yi6Q9rUw= +github.com/Xuanwo/go-bufferpool v0.2.0/go.mod h1:Mle++9GGouhOwGj52i9PJLNAPmW2nb8PWBP7JJzNCzk= +github.com/Xuanwo/templateutils v0.1.0 h1:WpkWOqQtIQ2vAIpJLa727DdN8WtxhUkkbDGa6UhntJY= +github.com/Xuanwo/templateutils v0.1.0/go.mod h1:OdE0DJ+CJxDBq6psX5DPV+gOZi8bhuHuVUpPCG++Wb8= +github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= +github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= +github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= +github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= +github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= +github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= +github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= +github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= +github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= +github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= +github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= +github.com/dave/dst v0.26.2 h1:lnxLAKI3tx7MgLNVDirFCsDTlTG9nKTk7GcptKcWSwY= +github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU= +github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ= +github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= +github.com/dave/kerr v0.0.0-20170318121727-bc25dd6abe8e/go.mod h1:qZqlPyPvfsDJt+3wHJ1EvSXDuVjFTK0j2p/ca+gtsb8= +github.com/dave/rebecca v0.9.1/go.mod h1:N6XYdMD/OKw3lkF3ywh8Z6wPGuwNFDNtWYEMFWEmXBA= +github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/pprof v0.0.0-20181127221834-b4f47329b966/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ipfs/go-cid v0.0.7 h1:ysQJVJA3fNDF1qigJbsSQOdjhVLsOEoPdh0+R97k3jY= +github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= +github.com/ipfs/go-ipfs-api v0.2.0 h1:BXRctUU8YOUOQT/jW1s56d9wLa85ntOqK6bptvCKb8c= +github.com/ipfs/go-ipfs-api v0.2.0/go.mod h1:zCTyTl+BuyvUqoSmVb8vjezCJLVTW7G/HBZbCXpTgeM= +github.com/ipfs/go-ipfs-cmds v0.6.0 h1:yAxdowQZzoFKjcLI08sXVNnqVj3jnABbf9smrPQmBsw= +github.com/ipfs/go-ipfs-cmds v0.6.0/go.mod h1:ZgYiWVnCk43ChwoH8hAmI1IRbuVtq3GSTHwtRB/Kqhk= +github.com/ipfs/go-ipfs-files v0.0.8 h1:8o0oFJkJ8UkO/ABl8T6ac6tKF3+NIpj67aAB6ZpusRg= +github.com/ipfs/go-ipfs-files v0.0.8/go.mod h1:wiN/jSG8FKyk7N0WyctKSvq3ljIa2NNTiZB55kpTdOs= +github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= +github.com/ipfs/go-log v1.0.4 h1:6nLQdX4W8P9yZZFH7mO+X/PzjN8Laozm/lMJ6esdgzY= +github.com/ipfs/go-log v1.0.4/go.mod h1:oDCg2FkjogeFOhqqb+N39l2RpTNPL6F/StPkB3kPgcs= +github.com/ipfs/go-log/v2 v2.0.5 h1:fL4YI+1g5V/b1Yxr1qAiXTMg1H8z9vx/VmJxBuQMHvU= +github.com/ipfs/go-log/v2 v2.0.5/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscwkWG+dw= +github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= +github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= +github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/kevinburke/go-bindata v3.22.0+incompatible h1:/JmqEhIWQ7GRScV0WjX/0tqBrC5D21ALg0H0U/KZ/ts= +github.com/kevinburke/go-bindata v3.22.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/libp2p/go-buffer-pool v0.0.2 h1:QNK2iAFa8gjAe1SPz6mHSMuCcjs+X1wlHzeOSqcmlfs= +github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= +github.com/libp2p/go-flow-metrics v0.0.3 h1:8tAs/hSdNvUiLgtlSy3mxwxWP4I9y/jlkPFT7epKdeM= +github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= +github.com/libp2p/go-libp2p-core v0.6.1 h1:XS+Goh+QegCDojUZp00CaPMfiEADCrLjNZskWE7pvqs= +github.com/libp2p/go-libp2p-core v0.6.1/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= +github.com/libp2p/go-msgio v0.0.6/go.mod h1:4ecVB6d9f4BDSL5fqvPiC4A3KivjWn+Venn/1ALLMWA= +github.com/libp2p/go-openssl v0.0.7 h1:eCAzdLejcNVBzP/iZM9vqHnQm+XyCEbSSIheIPRGNsw= +github.com/libp2p/go-openssl v0.0.7/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= +github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= +github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI= +github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= +github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4= +github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= +github.com/multiformats/go-multiaddr v0.2.2/go.mod h1:NtfXiOtHvghW9KojvtySjH5y0u0xW5UouOmQQrn6a3Y= +github.com/multiformats/go-multiaddr v0.3.0 h1:z1Old9IYcUyMEtSbvwCOJ1jcrmJdU0LYH8aFBvZKzcQ= +github.com/multiformats/go-multiaddr v0.3.0/go.mod h1:dF9kph9wfJ+3VLAaeBqo9Of8x4fJxp6ggJGteB8HQTI= +github.com/multiformats/go-multiaddr-net v0.2.0 h1:MSXRGN0mFymt6B1yo/6BPnIRpLPEnKgQNvVfCX5VDJk= +github.com/multiformats/go-multiaddr-net v0.2.0/go.mod h1:gGdH3UXny6U3cKKYCvpXI5rnK7YaOIEOPVDI9tsJbEA= +github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= +github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= +github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= +github.com/multiformats/go-multihash v0.0.14 h1:QoBceQYQQtNUuf6s7wHxnE2c8bhbMqhfGzNI032se/I= +github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= +github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY= +github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= +github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.6 h1:lH+Snxmzl92r1jww8/jYPqKkhs3C9AF4LunzU56ZZr4= +github.com/smartystreets/goconvey v1.6.6/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= +github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e/go.mod h1:XDKHRm5ThF8YJjx001LtgelzsoaEcvnA7lVWz9EeX3g= +github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c h1:GGsyl0dZ2jJgVT+VvWBf/cNijrHRhkrTjkmp5wg7li0= +github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c/go.mod h1:xxcJeBb7SIUl/Wzkz1eVKJE/CB34YNrqX2TQI6jY9zs= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.beyondstorage.io/endpoint v1.2.0 h1:/7mgKquTykeqJ9op82hso2+WQfECeywGd/Lda1N3tF4= +go.beyondstorage.io/endpoint v1.2.0/go.mod h1:oZ7Z7HZ7mAo337JBLjuCF/DM66HVEUu6+hw68c3UcLs= +go.beyondstorage.io/v5 v5.0.0 h1:k9Axfgbt+oZXoDwSBVCl1XANHSL4rkNTGP2Lz9YdJe0= +go.beyondstorage.io/v5 v5.0.0/go.mod h1:3wV9gCQnqu7tD/3LMeo2yimUKIeTSHpTc6wHSb0yY20= +go.opencensus.io v0.22.4 h1:LYy1Hy3MJdrCdMwwzxA/dRok4ejH+RwNGbuoD9fCjto= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.14.1 h1:nYDKopTbvAPq/NrUVZwT15y2lpROBiLLyoRTbXOYWOo= +go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= +golang.org/x/arch v0.0.0-20180920145803-b19384d3c130/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8= +golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d h1:2+ZP7EfsZV7Vvmx3TIqSlSzATMkTAKqM14YGFPoSKjI= +golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.1.1 h1:wGiQel/hW0NnEkJUk8lbzkX2gFJU6PFxf1v5OlCfuOs= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/src-d/go-billy.v4 v4.3.0 h1:KtlZ4c1OWbIs4jCv5ZXrTqG8EQocr0g/d4DjNg70aek= +gopkg.in/src-d/go-billy.v4 v4.3.0/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= diff --git a/services/ipfs/service.toml b/services/ipfs/service.toml new file mode 100644 index 000000000..ada755ec6 --- /dev/null +++ b/services/ipfs/service.toml @@ -0,0 +1,62 @@ +name = "ipfs" + +[namespace.storage] +implement = ["copier", "mover", "direr", "storage_http_signer"] + +[namespace.storage.new] +required = ["endpoint", "gateway"] +optional = ["storage_features", "default_storage_pairs", "work_dir"] + +[namespace.storage.op.create] +optional = ["object_mode"] + +[namespace.storage.op.list] +optional = ["list_mode"] + +[namespace.storage.op.read] +optional = ["offset", "io_callback", "size"] + +[namespace.storage.op.write] +optional = ["content_md5", "content_type", "io_callback"] + +[namespace.storage.op.delete] +optional = ["object_mode"] + +[namespace.storage.op.stat] +optional = ["object_mode"] + +[pairs.gateway] +type = "string" +description = "set storage gateway, for http(s) request purpose" + +[pairs.storage_features] +type = "StorageFeatures" +description = "set storage features" + +[pairs.default_storage_pairs] +type = "DefaultStoragePairs" +description = "set default pairs for storager actions" + +[infos.object.meta.hash] +type = "string" +description = "the CID of the file or directory" + +[infos.object.meta.blocks] +type = "int" +description = "the number of files in the directory or the number of blocks that make up the file" + +[infos.object.meta.cumulative_size] +type = "uint64" +description = "the size of the DAGNodes making up the file in Bytes, or the sum of the sizes of all files in the directory" + +[infos.object.meta.local] +type = "bool" +description = "whether the file`s dags is fully present locally" + +[infos.object.meta.with_locality] +type = "bool" +description = "whether the locality information is present" + +[infos.object.meta.size_local] +type = "uint64" +description = "the cumulative size of the data present locally" diff --git a/services/ipfs/storage.go b/services/ipfs/storage.go new file mode 100644 index 000000000..afb22713e --- /dev/null +++ b/services/ipfs/storage.go @@ -0,0 +1,213 @@ +package ipfs + +import ( + "bytes" + "context" + "errors" + "io" + "net/http" + "time" + + ipfs "github.com/ipfs/go-ipfs-api" + + "go.beyondstorage.io/v5/pkg/iowrap" + "go.beyondstorage.io/v5/services" + . "go.beyondstorage.io/v5/types" +) + +// The src of `ipfs files cp` supports both `IPFS-path` and `MFS-path` +// After `s.getAbsPath(src)`, if the absolute path matches `IPFS-path`, it will take precedence +// This means that if the `workDir` is `/ipfs/`, there is a high probability that an error will be returned +// See https://github.com/beyondstorage/specs/pull/134#discussion_r663594807 for more details +func (s *Storage) copy(ctx context.Context, src string, dst string, opt pairStorageCopy) (err error) { + dst = s.getAbsPath(dst) + stat, err := s.ipfs.FilesStat(ctx, dst) + if err == nil { + if stat.Type == "directory" { + return services.ErrObjectModeInvalid + } else { + err = s.ipfs.FilesRm(ctx, dst, true) + if err != nil { + return err + } + } + } else if !errors.Is(formatError(err), services.ErrObjectNotExist) { + return err + } + return s.ipfs.FilesCp(ctx, s.getAbsPath(src), dst) +} + +func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) { + if opt.HasObjectMode && opt.ObjectMode.IsDir() { + path += "/" + o = NewObject(s, true) + o.Mode = ModeDir + } else { + o = NewObject(s, false) + o.Mode = ModeRead + } + o.ID = s.getAbsPath(path) + o.Path = path + return o +} + +func (s *Storage) createDir(ctx context.Context, path string, opt pairStorageCreateDir) (o *Object, err error) { + path = s.getAbsPath(path) + err = s.ipfs.FilesMkdir(ctx, path, ipfs.FilesMkdir.Parents(true)) + if err != nil { + return nil, err + } + o = NewObject(s, true) + o.ID = path + o.Path = path + o.Mode = ModeDir + return +} + +// GSP-46: Idempotent Storager Delete Operation +// ref: https://github.com/beyondstorage/specs/blob/master/rfcs/46-idempotent-delete.md +func (s *Storage) delete(ctx context.Context, path string, opt pairStorageDelete) (err error) { + err = s.ipfs.FilesRm(ctx, s.getAbsPath(path), true) + return +} + +func (s *Storage) list(ctx context.Context, path string, opt pairStorageList) (oi *ObjectIterator, err error) { + rp := s.getAbsPath(path) + if !opt.HasListMode || opt.ListMode.IsDir() { + nextFn := func(ctx context.Context, page *ObjectPage) error { + dir, err := s.ipfs.FilesLs(ctx, rp, ipfs.FilesLs.Stat(true)) + if err != nil { + return err + } + for _, f := range dir { + o := NewObject(s, true) + o.ID = f.Hash + o.Path = f.Name + switch f.Type { + case ipfs.TFile: + o.Mode |= ModeRead + case ipfs.TDirectory: + o.Mode |= ModeDir + } + o.SetContentLength(int64(f.Size)) + page.Data = append(page.Data, o) + } + return IterateDone + } + oi = NewObjectIterator(ctx, nextFn, nil) + return + } else { + return nil, services.ListModeInvalidError{Actual: opt.ListMode} + } +} + +func (s *Storage) metadata(opt pairStorageMetadata) (meta *StorageMeta) { + meta = NewStorageMeta() + meta.WorkDir = s.workDir + return meta +} + +func (s *Storage) move(ctx context.Context, src string, dst string, opt pairStorageMove) (err error) { + dst = s.getAbsPath(dst) + stat, err := s.ipfs.FilesStat(ctx, dst) + if err == nil { + if stat.Type == "directory" { + return services.ErrObjectModeInvalid + } + } else if !errors.Is(formatError(err), services.ErrObjectNotExist) { + return err + } + return s.ipfs.FilesMv(ctx, s.getAbsPath(src), s.getAbsPath(dst)) +} + +func (s *Storage) querySignHTTPDelete(ctx context.Context, path string, expire time.Duration, opt pairStorageQuerySignHTTPDelete) (req *http.Request, err error) { + panic("not implemented") +} + +func (s *Storage) querySignHTTPRead(ctx context.Context, path string, expire time.Duration, opt pairStorageQuerySignHTTPRead) (req *http.Request, err error) { + rp := s.getAbsPath(path) + stat, err := s.ipfs.FilesStat(ctx, rp, ipfs.FilesStat.WithLocal(true)) + if err != nil { + return nil, err + } + if stat.Type != "file" { + return nil, errors.New("path not a file") + } + + return http.NewRequest(http.MethodGet, s.gateway+"/ipfs/"+stat.Hash, nil) +} + +func (s *Storage) querySignHTTPWrite(ctx context.Context, path string, size int64, expire time.Duration, opt pairStorageQuerySignHTTPWrite) (req *http.Request, err error) { + panic("not implemented") +} + +func (s *Storage) read(ctx context.Context, path string, w io.Writer, opt pairStorageRead) (n int64, err error) { + fileOpts := make([]ipfs.FilesOpt, 0) + if opt.HasOffset { + fileOpts = append(fileOpts, ipfs.FilesRead.Offset(opt.Offset)) + } + if opt.HasSize { + fileOpts = append(fileOpts, ipfs.FilesRead.Count(opt.Size)) + } + f, err := s.ipfs.FilesRead(ctx, s.getAbsPath(path), fileOpts...) + if err != nil { + return 0, err + } + if opt.HasIoCallback { + f = iowrap.CallbackReadCloser(f, opt.IoCallback) + } + return io.Copy(w, f) +} + +func (s *Storage) stat(ctx context.Context, path string, opt pairStorageStat) (o *Object, err error) { + rp := s.getAbsPath(path) + stat, err := s.ipfs.FilesStat(ctx, rp, ipfs.FilesStat.WithLocal(true)) + if err != nil { + return nil, err + } + o = NewObject(s, true) + o.ID = stat.Hash + o.Path = path + if opt.HasObjectMode && opt.ObjectMode.IsDir() { + o.Mode |= ModeDir + } else { + o.Mode |= ModeRead + } + o.SetContentType(stat.Type) + o.SetContentLength(int64(stat.Size)) + var sm ObjectSystemMetadata + sm.Hash = stat.Hash + sm.Blocks = stat.Blocks + sm.Local = stat.Local + sm.WithLocality = stat.WithLocality + sm.CumulativeSize = stat.CumulativeSize + sm.SizeLocal = stat.SizeLocal + o.SetSystemMetadata(sm) + return +} + +func (s *Storage) write(ctx context.Context, path string, r io.Reader, size int64, opt pairStorageWrite) (n int64, err error) { + if r == nil { + if size > 0 { + return 0, errors.New("size is not 0 when io.Reader is nil") + } + r = bytes.NewReader([]byte{}) + } + r = io.LimitReader(r, size) + + if opt.HasIoCallback { + r = iowrap.CallbackReader(r, opt.IoCallback) + } + + err = s.ipfs.FilesWrite( + ctx, s.getAbsPath(path), r, + ipfs.FilesWrite.Create(true), + ipfs.FilesWrite.Parents(true), + ipfs.FilesWrite.Truncate(true), + ipfs.FilesWrite.Count(size), + ) + if err != nil { + return 0, err + } + return size, nil +} diff --git a/services/ipfs/tests/README.md b/services/ipfs/tests/README.md new file mode 100644 index 000000000..c8dfbac5f --- /dev/null +++ b/services/ipfs/tests/README.md @@ -0,0 +1,31 @@ +## How run integration tests + +### Run tests locally + +Copy example files and update corresponding values. + +```shell +cp Makefile.env.example Makefile.env +``` + +Run tests + +```shell +make integration_test +``` + +### Run tests in CI + +Set following environment variables: + +```shell +export STORAGE_IPFS_INTEGRATION_TEST=on +export STORAGE_IPFS_ENDPOINT=http:127.0.0.1:5001 +export STORAGE_IPFS_GATEWAY=http:127.0.0.1:8080 +``` + +Run tests + +```shell +make integration_test +``` diff --git a/services/ipfs/tests/storage_test.go b/services/ipfs/tests/storage_test.go new file mode 100644 index 000000000..7668b990e --- /dev/null +++ b/services/ipfs/tests/storage_test.go @@ -0,0 +1,36 @@ +package tests + +import ( + "os" + "testing" + + "go.beyondstorage.io/v5/tests" +) + +func TestStorager(t *testing.T) { + if os.Getenv("STORAGE_IPFS_INTEGRATION_TEST") != "on" { + t.Skipf("STORAGE_IPFS_INTEGRATION_TEST is not 'on', skipped") + } + tests.TestStorager(t, setupTest(t)) +} + +func TestCopier(t *testing.T) { + if os.Getenv("STORAGE_IPFS_INTEGRATION_TEST") != "on" { + t.Skipf("STORAGE_IPFS_INTEGRATION_TEST is not 'on', skipped") + } + tests.TestCopier(t, setupTest(t)) +} + +func TestMover(t *testing.T) { + if os.Getenv("STORAGE_IPFS_INTEGRATION_TEST") != "on" { + t.Skipf("STORAGE_IPFS_INTEGRATION_TEST is not 'on', skipped") + } + tests.TestMover(t, setupTest(t)) +} + +func TestStorageHttpSignerRead(t *testing.T) { + if os.Getenv("STORAGE_IPFS_INTEGRATION_TEST") != "on" { + t.Skipf("STORAGE_IPFS_INTEGRATION_TEST is not 'on', skipped") + } + tests.TestStorageHTTPSignerRead(t, setupTest(t)) +} diff --git a/services/ipfs/tests/utils_test.go b/services/ipfs/tests/utils_test.go new file mode 100644 index 000000000..81b80228a --- /dev/null +++ b/services/ipfs/tests/utils_test.go @@ -0,0 +1,26 @@ +package tests + +import ( + "os" + "testing" + + "github.com/google/uuid" + ipfs "go.beyondstorage.io/services/ipfs" + + "go.beyondstorage.io/v5/pairs" + "go.beyondstorage.io/v5/types" +) + +func setupTest(t *testing.T) types.Storager { + t.Log("Setup test for IPFS") + + store, err := ipfs.NewStorager( + pairs.WithEndpoint(os.Getenv("STORAGE_IPFS_ENDPOINT")), + ipfs.WithGateway(os.Getenv("STORAGE_IPFS_GATEWAY")), + pairs.WithWorkDir("/"+uuid.New().String()+"/"), + ) + if err != nil { + t.Errorf("new storager: %v", err) + } + return store +} diff --git a/services/ipfs/tools.go b/services/ipfs/tools.go new file mode 100644 index 000000000..8e9225166 --- /dev/null +++ b/services/ipfs/tools.go @@ -0,0 +1,8 @@ +//go:build tools +// +build tools + +package ipfs + +import ( + _ "go.beyondstorage.io/v5/cmd/definitions" +) diff --git a/services/ipfs/utils.go b/services/ipfs/utils.go new file mode 100644 index 000000000..5db80ad04 --- /dev/null +++ b/services/ipfs/utils.go @@ -0,0 +1,142 @@ +package ipfs + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + ipfs "github.com/ipfs/go-ipfs-api" + cmds "github.com/ipfs/go-ipfs-cmds" + + "go.beyondstorage.io/endpoint" + ps "go.beyondstorage.io/v5/pairs" + "go.beyondstorage.io/v5/services" + "go.beyondstorage.io/v5/types" +) + +// Storage is the example client. +type Storage struct { + ipfs *ipfs.Shell + + defaultPairs DefaultStoragePairs + features StorageFeatures + + workDir string + gateway string + + types.UnimplementedStorager + types.UnimplementedCopier + types.UnimplementedMover + types.UnimplementedDirer + types.UnimplementedStorageHTTPSigner +} + +// String implements Storager.String +func (s *Storage) String() string { + return fmt.Sprintf("Storager IPFS {WorkDir: %s}", s.workDir) +} + +// NewStorager will create Storager only. +func NewStorager(pairs ...types.Pair) (types.Storager, error) { + opt, err := parsePairStorageNew(pairs) + if err != nil { + return nil, err + } + + st := &Storage{ + workDir: "/", + } + if opt.HasWorkDir { + if !strings.HasSuffix(opt.WorkDir, "/") { + opt.WorkDir += "/" + } + st.workDir = opt.WorkDir + } + + ep, err := endpoint.Parse(opt.Endpoint) + if err != nil { + return nil, err + } + var e string + switch ep.Protocol() { + case endpoint.ProtocolHTTP: + e, _, _ = ep.HTTP() + case endpoint.ProtocolHTTPS: + e, _, _ = ep.HTTPS() + default: + return nil, services.PairUnsupportedError{Pair: ps.WithEndpoint(opt.Endpoint)} + } + + gate, err := endpoint.Parse(opt.Gateway) + if err != nil { + return nil, err + } + switch gate.Protocol() { + case endpoint.ProtocolHTTP: + st.gateway, _, _ = gate.HTTP() + case endpoint.ProtocolHTTPS: + st.gateway, _, _ = gate.HTTPS() + default: + return nil, services.PairUnsupportedError{Pair: WithGateway(opt.Gateway)} + } + + sh := ipfs.NewShell(e) + if !sh.IsUp() { + return nil, errors.New("ipfs not online") + } + st.ipfs = sh + + return st, nil +} + +func formatError(err error) error { + if _, ok := err.(services.InternalError); ok { + return err + } + + e, ok := err.(*ipfs.Error) + if !ok { + return fmt.Errorf("%w: %v", services.ErrUnexpected, err) + } + + switch e.Message { + case os.ErrNotExist.Error(): + return fmt.Errorf("%w: %v", services.ErrObjectNotExist, err) + } + + // ref: https://github.com/ipfs/go-ipfs-cmds/blob/4ade007405e5d3befb14184290576c63cc43a6a3/error.go#L31 + switch e.Code { + case int(cmds.ErrRateLimited): + return fmt.Errorf("%w: %v", services.ErrRequestThrottled, err) + case int(cmds.ErrImplementation): + return fmt.Errorf("%w: %v", services.ErrServiceInternal, err) + } + + return fmt.Errorf("%w: %v", services.ErrUnexpected, err) +} + +func (s *Storage) formatError(op string, err error, path ...string) error { + if err == nil { + return nil + } + + return services.StorageError{ + Op: op, + Err: formatError(err), + Storager: s, + Path: path, + } +} + +// getAbsPath will calculate object storage's abs path +func (s *Storage) getAbsPath(path string) string { + path = strings.ReplaceAll(path, "\\", "/") + + if filepath.IsAbs(path) { + return path + } + + return s.workDir + path +}