Skip to content

Commit

Permalink
feat(ipld): bindnode support for all voucher types (#713)
Browse files Browse the repository at this point in the history
* feat(ipld): new data-transfer ipld vouchers + bindnode

* feat(ipld): simplify ipldutils API

* feat(ipld): use new bindnode registry in go-ipld-prime

Ref: ipld/go-ipld-prime#437
  • Loading branch information
rvagg authored Jul 28, 2022
1 parent b93af0a commit 727a2b1
Show file tree
Hide file tree
Showing 40 changed files with 1,457 additions and 722 deletions.
26 changes: 11 additions & 15 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,16 +1,15 @@
module github.com/filecoin-project/go-fil-markets

go 1.13
go 1.16

require (
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/filecoin-project/dagstore v0.5.2
github.com/filecoin-project/go-address v0.0.6
github.com/filecoin-project/go-bitfield v0.2.4 // indirect
github.com/filecoin-project/go-cbor-util v0.0.1
github.com/filecoin-project/go-commp-utils v0.1.3
github.com/filecoin-project/go-crypto v0.0.1 // indirect
github.com/filecoin-project/go-data-transfer/v2 v2.0.0-20220511223325-5253bfe075cd
github.com/filecoin-project/go-data-transfer/v2 v2.0.0-20220603004528-681bfedccef1
github.com/filecoin-project/go-ds-versioning v0.1.1
github.com/filecoin-project/go-fil-commcid v0.1.0
github.com/filecoin-project/go-fil-commp-hashhash v0.1.0
Expand All @@ -28,8 +27,8 @@ require (
github.com/hashicorp/go-multierror v1.1.1
github.com/ipfs/go-block-format v0.0.3
github.com/ipfs/go-blockservice v0.2.1
github.com/ipfs/go-cid v0.1.0
github.com/ipfs/go-cidutil v0.0.2
github.com/ipfs/go-cid v0.2.0
github.com/ipfs/go-cidutil v0.1.0
github.com/ipfs/go-datastore v0.5.1
github.com/ipfs/go-filestore v1.1.0
github.com/ipfs/go-graphsync v0.13.1
Expand All @@ -41,32 +40,29 @@ require (
github.com/ipfs/go-ipfs-files v0.0.9
github.com/ipfs/go-ipld-cbor v0.0.6-0.20211211231443-5d9b9e1f6fa8
github.com/ipfs/go-ipld-format v0.2.0
github.com/ipfs/go-log/v2 v2.5.0
github.com/ipfs/go-log/v2 v2.5.1
github.com/ipfs/go-merkledag v0.5.1
github.com/ipfs/go-unixfs v0.3.1
github.com/ipld/go-car v0.3.3
github.com/ipld/go-car/v2 v2.1.1
github.com/ipld/go-ipld-prime v0.16.0
github.com/ipld/go-ipld-prime v0.17.1-0.20220624062450-534ccf82237d
github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c
github.com/jpillora/backoff v1.0.0
github.com/libp2p/go-libp2p v0.18.0
github.com/libp2p/go-libp2p-core v0.14.0
github.com/libp2p/go-libp2p v0.19.4
github.com/libp2p/go-libp2p-core v0.15.1
github.com/multiformats/go-multiaddr v0.5.0
github.com/multiformats/go-multibase v0.0.3
github.com/multiformats/go-multicodec v0.4.1
github.com/multiformats/go-multicodec v0.5.0
github.com/multiformats/go-multihash v0.1.0
github.com/multiformats/go-varint v0.0.6
github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9
github.com/stretchr/testify v1.7.0
github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11
github.com/whyrusleeping/cbor-gen v0.0.0-20220302191723-37c43cae8e14
github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 // indirect
golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b // indirect
golang.org/x/exp v0.0.0-20210715201039-d37aa40e8013
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2
golang.org/x/sys v0.0.0-20211209171907-798191bca915 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
lukechampine.com/blake3 v1.1.7 // indirect
golang.org/x/net v0.0.0-20220418201149-a630d4f3e7a2
golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f
)

replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi
182 changes: 120 additions & 62 deletions go.sum

Large diffs are not rendered by default.

165 changes: 165 additions & 0 deletions retrievalmarket/bindnodeoptions.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
package retrievalmarket

import (
"bytes"
"fmt"
"io"

"github.com/ipld/go-ipld-prime/codec/dagcbor"
"github.com/ipld/go-ipld-prime/datamodel"
"github.com/ipld/go-ipld-prime/node/basicnode"
"github.com/ipld/go-ipld-prime/node/bindnode"
"github.com/ipld/go-ipld-prime/schema"
cbg "github.com/whyrusleeping/cbor-gen"

"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/go-state-types/crypto"
)

// go type converter functions for bindnode for common Filecoin data types

// CborGenCompatibleNodeBindnodeOption converts a CborGenCompatibleNode type to
// and from an Any field in a schema
var CborGenCompatibleNodeBindnodeOption = bindnode.TypedAnyConverter(&CborGenCompatibleNode{}, cborGenCompatibleNodeFromAny, cborGenCompatibleNodeToAny)

// BigIntBindnodeOption converts a big.Int type to and from a Bytes field in a
// schema
var BigIntBindnodeOption = bindnode.TypedBytesConverter(&big.Int{}, bigIntFromBytes, bigIntToBytes)

// TokenAmountBindnodeOption converts a filecoin abi.TokenAmount type to and
// from a Bytes field in a schema
var TokenAmountBindnodeOption = bindnode.TypedBytesConverter(&abi.TokenAmount{}, tokenAmountFromBytes, tokenAmountToBytes)

// AddressBindnodeOption converts a filecoin Address type to and from a Bytes
// field in a schema
var AddressBindnodeOption = bindnode.TypedBytesConverter(&address.Address{}, addressFromBytes, addressToBytes)

// SignatureBindnodeOption converts a filecoin Signature type to and from a
// Bytes field in a schema
var SignatureBindnodeOption = bindnode.TypedBytesConverter(&crypto.Signature{}, signatureFromBytes, signatureToBytes)

// CborGenCompatibleNode is for cbor-gen / go-ipld-prime compatibility, to
// replace Deferred types that are used to represent datamodel.Nodes.
// This shouldn't be used as a pointer (nullable/optional) as it can consume
// "Null" tokens and therefore be a Null. Instead, use
// CborGenCompatibleNode#IsNull to check for null status.
type CborGenCompatibleNode struct {
Node datamodel.Node
}

func (sn CborGenCompatibleNode) IsNull() bool {
return sn.Node == nil || sn.Node == datamodel.Null
}

// UnmarshalCBOR is for cbor-gen compatibility
func (sn *CborGenCompatibleNode) UnmarshalCBOR(r io.Reader) error {
// use cbg.Deferred.UnmarshalCBOR to figure out how much to pull
def := cbg.Deferred{}
if err := def.UnmarshalCBOR(r); err != nil {
return err
}
// convert it to a Node
na := basicnode.Prototype.Any.NewBuilder()
if err := dagcbor.Decode(na, bytes.NewReader(def.Raw)); err != nil {
return err
}
sn.Node = na.Build()
return nil
}

// MarshalCBOR is for cbor-gen compatibility
func (sn *CborGenCompatibleNode) MarshalCBOR(w io.Writer) error {
node := datamodel.Null
if sn != nil && sn.Node != nil {
node = sn.Node
if tn, ok := node.(schema.TypedNode); ok {
node = tn.Representation()
}
}
return dagcbor.Encode(node, w)
}

func cborGenCompatibleNodeFromAny(node datamodel.Node) (interface{}, error) {
return &CborGenCompatibleNode{Node: node}, nil
}

func cborGenCompatibleNodeToAny(iface interface{}) (datamodel.Node, error) {
sn, ok := iface.(*CborGenCompatibleNode)
if !ok {
return nil, fmt.Errorf("expected *CborGenCompatibleNode value")
}
if sn.Node == nil {
return datamodel.Null, nil
}
return sn.Node, nil
}

func tokenAmountFromBytes(b []byte) (interface{}, error) {
return bigIntFromBytes(b)
}

func bigIntFromBytes(b []byte) (interface{}, error) {
if len(b) == 0 {
return big.NewInt(0), nil
}
return big.FromBytes(b)
}

func tokenAmountToBytes(iface interface{}) ([]byte, error) {
return bigIntToBytes(iface)
}

func bigIntToBytes(iface interface{}) ([]byte, error) {
bi, ok := iface.(*big.Int)
if !ok {
return nil, fmt.Errorf("expected *big.Int value")
}
if bi == nil || bi.Int == nil {
*bi = big.Zero()
}
return bi.Bytes()
}

func addressFromBytes(b []byte) (interface{}, error) {
return address.NewFromBytes(b)
}

func addressToBytes(iface interface{}) ([]byte, error) {
addr, ok := iface.(*address.Address)
if !ok {
return nil, fmt.Errorf("expected *Address value")
}
return addr.Bytes(), nil
}

// Signature is a byteprefix union
func signatureFromBytes(b []byte) (interface{}, error) {
if len(b) > crypto.SignatureMaxLength {
return nil, fmt.Errorf("string too long")
}
if len(b) == 0 {
return nil, fmt.Errorf("string empty")
}
var s crypto.Signature
switch crypto.SigType(b[0]) {
default:
return nil, fmt.Errorf("invalid signature type in cbor input: %d", b[0])
case crypto.SigTypeSecp256k1:
s.Type = crypto.SigTypeSecp256k1
case crypto.SigTypeBLS:
s.Type = crypto.SigTypeBLS
}
s.Data = b[1:]
return &s, nil
}

func signatureToBytes(iface interface{}) ([]byte, error) {
s, ok := iface.(*crypto.Signature)
if !ok {
return nil, fmt.Errorf("expected *Signature value")
}
ba := append([]byte{byte(s.Type)}, s.Data...)
return ba, nil
}
22 changes: 0 additions & 22 deletions retrievalmarket/common.go

This file was deleted.

25 changes: 8 additions & 17 deletions retrievalmarket/impl/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -108,21 +108,17 @@ func NewClient(
if err != nil {
return nil, err
}
err = dataTransfer.RegisterVoucherResultType(&retrievalmarket.DealResponse{})
err = dataTransfer.RegisterVoucherType(retrievalmarket.DealProposalType, nil)
if err != nil {
return nil, err
}
err = dataTransfer.RegisterVoucherType(&retrievalmarket.DealProposal{}, nil)
if err != nil {
return nil, err
}
err = dataTransfer.RegisterVoucherType(&retrievalmarket.DealPayment{}, nil)
err = dataTransfer.RegisterVoucherType(retrievalmarket.DealPaymentType, nil)
if err != nil {
return nil, err
}
dataTransfer.SubscribeToEvents(dtutils.ClientDataTransferSubscriber(c.stateMachines))
transportConfigurer := dtutils.TransportConfigurer(network.ID(), &clientStoreGetter{c})
err = dataTransfer.RegisterTransportConfigurer(&retrievalmarket.DealProposal{}, transportConfigurer)
err = dataTransfer.RegisterTransportConfigurer(retrievalmarket.DealProposalType, transportConfigurer)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -406,20 +402,15 @@ func (c *clientDealEnvironment) Node() retrievalmarket.RetrievalClientNode {
func (c *clientDealEnvironment) OpenDataTransfer(ctx context.Context, to peer.ID, proposal *retrievalmarket.DealProposal) (datatransfer.ChannelID, error) {
sel := selectorparse.CommonSelector_ExploreAllRecursively
if proposal.SelectorSpecified() {
var err error
sel, err = retrievalmarket.DecodeNode(proposal.Selector)
if err != nil {
return datatransfer.ChannelID{}, xerrors.Errorf("selector is invalid: %w", err)
}
sel = proposal.Selector.Node
}

var vouch datatransfer.Voucher = proposal
return c.c.dataTransfer.OpenPullDataChannel(ctx, to, vouch, proposal.PayloadCID, sel)
vouch := retrievalmarket.BindnodeRegistry.TypeToNode(proposal)
return c.c.dataTransfer.OpenPullDataChannel(ctx, to, datatransfer.TypedVoucher{Voucher: vouch, Type: retrievalmarket.DealProposalType}, proposal.PayloadCID, sel)
}

func (c *clientDealEnvironment) SendDataTransferVoucher(ctx context.Context, channelID datatransfer.ChannelID, payment *retrievalmarket.DealPayment) error {
var vouch datatransfer.Voucher = payment
return c.c.dataTransfer.SendVoucher(ctx, channelID, vouch)
vouch := retrievalmarket.BindnodeRegistry.TypeToNode(payment)
return c.c.dataTransfer.SendVoucher(ctx, channelID, datatransfer.TypedVoucher{Voucher: vouch, Type: retrievalmarket.DealPaymentType})
}

func (c *clientDealEnvironment) CloseDataTransfer(ctx context.Context, channelID datatransfer.ChannelID) error {
Expand Down
28 changes: 8 additions & 20 deletions retrievalmarket/impl/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,10 @@ import (
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/namespace"
dss "github.com/ipfs/go-datastore/sync"
"github.com/ipld/go-ipld-prime/codec/dagcbor"
selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
cbg "github.com/whyrusleeping/cbor-gen"

"github.com/filecoin-project/go-address"
datatransfer "github.com/filecoin-project/go-data-transfer/v2"
Expand Down Expand Up @@ -47,17 +45,11 @@ func TestClient_Construction(t *testing.T) {
require.NoError(t, err)

require.Len(t, dt.Subscribers, 1)
require.Len(t, dt.RegisteredVoucherResultTypes, 1)
_, ok := dt.RegisteredVoucherResultTypes[0].(*retrievalmarket.DealResponse)
require.True(t, ok)
require.Len(t, dt.RegisteredVoucherTypes, 2)
_, ok = dt.RegisteredVoucherTypes[0].VoucherType.(*retrievalmarket.DealProposal)
require.True(t, ok)
_, ok = dt.RegisteredVoucherTypes[1].VoucherType.(*retrievalmarket.DealPayment)
require.True(t, ok)
require.Equal(t, dt.RegisteredVoucherTypes[0].VoucherType, retrievalmarket.DealProposalType)
require.Equal(t, dt.RegisteredVoucherTypes[1].VoucherType, retrievalmarket.DealPaymentType)
require.Len(t, dt.RegisteredTransportConfigurers, 1)
_, ok = dt.RegisteredTransportConfigurers[0].VoucherType.(*retrievalmarket.DealProposal)
require.True(t, ok)
require.Equal(t, dt.RegisteredTransportConfigurers[0].VoucherType, retrievalmarket.DealProposalType)
}

func TestClient_Query(t *testing.T) {
Expand Down Expand Up @@ -304,7 +296,7 @@ func TestClient_DuplicateRetrieve(t *testing.T) {

// Retrieve first payload CID from first peer
params := retrievalmarket.Params{
Selector: nil,
Selector: retrievalmarket.CborGenCompatibleNode{},
PieceCID: &tut.GenerateCids(1)[0],
PricePerByte: abi.NewTokenAmount(1),
PaymentInterval: 1,
Expand Down Expand Up @@ -382,10 +374,6 @@ func TestMigrations(t *testing.T) {
voucherShortfalls := make([]abi.TokenAmount, numDeals)
selfPeer := tut.GeneratePeers(1)[0]

allSelectorBuf := new(bytes.Buffer)
err := dagcbor.Encode(selectorparse.CommonSelector_ExploreAllRecursively, allSelectorBuf)
require.NoError(t, err)
allSelectorBytes := allSelectorBuf.Bytes()
emptyList, err := versioned.BuilderList{}.Build()
require.NoError(t, err)
oldDs, migrate := versionedds.NewVersionedDatastore(retrievalDs, emptyList, "1")
Expand Down Expand Up @@ -425,8 +413,8 @@ func TestMigrations(t *testing.T) {
PayloadCID: payloadCIDs[i],
ID: iDs[i],
Params: retrievalmarket.Params{
Selector: &cbg.Deferred{
Raw: allSelectorBytes,
Selector: retrievalmarket.CborGenCompatibleNode{
Node: selectorparse.CommonSelector_ExploreAllRecursively,
},
PieceCID: pieceCIDs[i],
PricePerByte: pricePerBytes[i],
Expand Down Expand Up @@ -477,8 +465,8 @@ func TestMigrations(t *testing.T) {
PayloadCID: payloadCIDs[i],
ID: iDs[i],
Params: retrievalmarket.Params{
Selector: &cbg.Deferred{
Raw: allSelectorBytes,
Selector: retrievalmarket.CborGenCompatibleNode{
Node: selectorparse.CommonSelector_ExploreAllRecursively,
},
PieceCID: pieceCIDs[i],
PricePerByte: pricePerBytes[i],
Expand Down
Loading

0 comments on commit 727a2b1

Please sign in to comment.