packet-forward-middleware packet memo, retry on timeout, and atomic f…

…orwards (#306)

* Make examples its own module

* Make cosmos specific module and run in CI in separate task

* Update e2e test for memo refactor

* Update test to chain-level params

* Use gaia with pfm with configurable timeout and retries

* Update SendIBCTransfer uses

* fix interchain test

* Add GetClients method to Relayer and helper for getting transfer channel between chains

* Update packet forward test for multi-hop, add multi-hop refund test

* Update tests for atomic forwards

* Wait for a block after ack before checking balances

* reduce wait to 1 block

* Add multi-hop flow with refund through chain with native denom. Add assertions for escrow accounts

* Remove stale comment

* handle feedback

* Add TransferOptions

chain/cosmos/chain_node.go

@@ -584,18 +584,27 @@ type CosmosTx struct {
 	RawLog string `json:"raw_log"`
 }
 
-func (tn *ChainNode) SendIBCTransfer(ctx context.Context, channelID string, keyName string, amount ibc.WalletAmount, timeout *ibc.IBCTimeout) (string, error) {
+func (tn *ChainNode) SendIBCTransfer(
+	ctx context.Context,
+	channelID string,
+	keyName string,
+	amount ibc.WalletAmount,
+	options ibc.TransferOptions,
+) (string, error) {
 	command := []string{
 		"ibc-transfer", "transfer", "transfer", channelID,
 		amount.Address, fmt.Sprintf("%d%s", amount.Amount, amount.Denom),
 	}
-	if timeout != nil {
-		if timeout.NanoSeconds > 0 {
-			command = append(command, "--packet-timeout-timestamp", fmt.Sprint(timeout.NanoSeconds))
-		} else if timeout.Height > 0 {
-			command = append(command, "--packet-timeout-height", fmt.Sprintf("0-%d", timeout.Height))
+	if options.Timeout != nil {
+		if options.Timeout.NanoSeconds > 0 {
+			command = append(command, "--packet-timeout-timestamp", fmt.Sprint(options.Timeout.NanoSeconds))
+		} else if options.Timeout.Height > 0 {
+			command = append(command, "--packet-timeout-height", fmt.Sprintf("0-%d", options.Timeout.Height))
 		}
 	}
+	if options.Memo != "" {
+		command = append(command, "--memo", options.Memo)
+	}
 	return tn.ExecTx(ctx, keyName, command...)
 }
 

chain/cosmos/cosmos_chain.go

@@ -228,8 +228,14 @@ func (c *CosmosChain) SendFunds(ctx context.Context, keyName string, amount ibc.
 }
 
 // Implements Chain interface
-func (c *CosmosChain) SendIBCTransfer(ctx context.Context, channelID, keyName string, amount ibc.WalletAmount, timeout *ibc.IBCTimeout) (tx ibc.Tx, _ error) {
-	txHash, err := c.getFullNode().SendIBCTransfer(ctx, channelID, keyName, amount, timeout)
+func (c *CosmosChain) SendIBCTransfer(
+	ctx context.Context,
+	channelID string,
+	keyName string,
+	amount ibc.WalletAmount,
+	options ibc.TransferOptions,
+) (tx ibc.Tx, _ error) {
+	txHash, err := c.getFullNode().SendIBCTransfer(ctx, channelID, keyName, amount, options)
 	if err != nil {
 		return tx, fmt.Errorf("send ibc transfer: %w", err)
 	}

chain/cosmos/poll.go

@@ -2,15 +2,18 @@ package cosmos
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
-	"github.com/strangelove-ventures/ibctest/v3/test"
+	codectypes "github.com/cosmos/cosmos-sdk/codec/types"
+	"github.com/strangelove-ventures/ibctest/v3/ibc"
+	"github.com/strangelove-ventures/ibctest/v3/testutil"
 )
 
 // PollForProposalStatus attempts to find a proposal with matching ID and status.
 func PollForProposalStatus(ctx context.Context, chain *CosmosChain, startHeight, maxHeight uint64, proposalID string, status string) (ProposalResponse, error) {
 	var zero ProposalResponse
-	doPoll := func(ctx context.Context, height uint64) (any, error) {
+	doPoll := func(ctx context.Context, height uint64) (ProposalResponse, error) {
 		p, err := chain.QueryProposal(ctx, proposalID)
 		if err != nil {
 			return zero, err
@@ -20,10 +23,60 @@ func PollForProposalStatus(ctx context.Context, chain *CosmosChain, startHeight,
 		}
 		return *p, nil
 	}
-	bp := test.BlockPoller{CurrentHeight: chain.Height, PollFunc: doPoll}
-	p, err := bp.DoPoll(ctx, startHeight, maxHeight)
+	bp := testutil.BlockPoller[ProposalResponse]{CurrentHeight: chain.Height, PollFunc: doPoll}
+	return bp.DoPoll(ctx, startHeight, maxHeight)
+}
+
+// PollForMessage searches every transaction for a message. Must pass a coded registry capable of decoding the cosmos transaction.
+// fn is optional. Return true from the fn to stop polling and return the found message. If fn is nil, returns the first message to match type T.
+func PollForMessage[T any](ctx context.Context, chain *CosmosChain, registry codectypes.InterfaceRegistry, startHeight, maxHeight uint64, fn func(found T) bool) (T, error) {
+	var zero T
+	if fn == nil {
+		fn = func(T) bool { return true }
+	}
+	doPoll := func(ctx context.Context, height uint64) (T, error) {
+		h := int64(height)
+		block, err := chain.getFullNode().Client.Block(ctx, &h)
+		if err != nil {
+			return zero, err
+		}
+		for _, tx := range block.Block.Txs {
+			sdkTx, err := decodeTX(registry, tx)
+			if err != nil {
+				return zero, err
+			}
+			for _, msg := range sdkTx.GetMsgs() {
+				if found, ok := msg.(T); ok {
+					if fn(found) {
+						return found, nil
+					}
+				}
+			}
+		}
+		return zero, errors.New("not found")
+	}
+
+	bp := testutil.BlockPoller[T]{CurrentHeight: chain.Height, PollFunc: doPoll}
+	return bp.DoPoll(ctx, startHeight, maxHeight)
+}
+
+// PollForBalance polls until the balance matches
+func PollForBalance(ctx context.Context, chain *CosmosChain, deltaBlocks uint64, balance ibc.WalletAmount) error {
+	h, err := chain.Height(ctx)
 	if err != nil {
-		return zero, err
+		return fmt.Errorf("failed to get height: %w", err)
+	}
+	doPoll := func(ctx context.Context, height uint64) (any, error) {
+		bal, err := chain.GetBalance(ctx, balance.Address, balance.Denom)
+		if err != nil {
+			return nil, err
+		}
+		if bal != balance.Amount {
+			return nil, fmt.Errorf("balance (%d) does not match expected: (%d)", bal, balance.Amount)
+		}
+		return nil, nil
 	}
-	return p.(ProposalResponse), nil
+	bp := testutil.BlockPoller[any]{CurrentHeight: chain.Height, PollFunc: doPoll}
+	_, err = bp.DoPoll(ctx, h, h+deltaBlocks)
+	return err
 }

chain/penumbra/penumbra_app_node.go

@@ -187,7 +187,13 @@ func (p *PenumbraAppNode) SendFunds(ctx context.Context, keyName string, amount
 	return errors.New("not yet implemented")
 }
 
-func (p *PenumbraAppNode) SendIBCTransfer(ctx context.Context, channelID, keyName string, amount ibc.WalletAmount, timeout *ibc.IBCTimeout) (ibc.Tx, error) {
+func (p *PenumbraAppNode) SendIBCTransfer(
+	ctx context.Context,
+	channelID string,
+	keyName string,
+	amount ibc.WalletAmount,
+	options ibc.TransferOptions,
+) (ibc.Tx, error) {
 	return ibc.Tx{}, errors.New("not yet implemented")
 }
 

chain/penumbra/penumbra_chain.go

@@ -145,8 +145,14 @@ func (c *PenumbraChain) SendFunds(ctx context.Context, keyName string, amount ib
 }
 
 // Implements Chain interface
-func (c *PenumbraChain) SendIBCTransfer(ctx context.Context, channelID, keyName string, amount ibc.WalletAmount, timeout *ibc.IBCTimeout) (ibc.Tx, error) {
-	return c.getRelayerNode().PenumbraAppNode.SendIBCTransfer(ctx, channelID, keyName, amount, timeout)
+func (c *PenumbraChain) SendIBCTransfer(
+	ctx context.Context,
+	channelID string,
+	keyName string,
+	amount ibc.WalletAmount,
+	options ibc.TransferOptions,
+) (ibc.Tx, error) {
+	return c.getRelayerNode().PenumbraAppNode.SendIBCTransfer(ctx, channelID, keyName, amount, options)
 }
 
 // Implements Chain interface

chain/polkadot/polkadot_chain.go

@@ -538,7 +538,13 @@ func (c *PolkadotChain) SendFunds(ctx context.Context, keyName string, amount ib
 
 // SendIBCTransfer sends an IBC transfer returning a transaction or an error if the transfer failed.
 // Implements Chain interface.
-func (c *PolkadotChain) SendIBCTransfer(ctx context.Context, channelID, keyName string, amount ibc.WalletAmount, timeout *ibc.IBCTimeout) (ibc.Tx, error) {
+func (c *PolkadotChain) SendIBCTransfer(
+	ctx context.Context,
+	channelID string,
+	keyName string,
+	amount ibc.WalletAmount,
+	options ibc.TransferOptions,
+) (ibc.Tx, error) {
 	panic("not implemented yet")
 }
 

conformance/flush.go

@@ -79,7 +79,7 @@ func TestRelayerFlushing(t *testing.T, ctx context.Context, cf ibctest.ChainFact
 		Address: c1FaucetAddr,
 		Denom:   c0.Config().Denom,
 		Amount:  txAmount,
-	}, nil)
+	}, ibc.TransferOptions{})
 	req.NoError(err)
 	req.NoError(tx.Validate())
 

conformance/test.go

@@ -166,7 +166,7 @@ func sendIBCTransfersFromBothChainsWithTimeout(
 	eg.Go(func() (err error) {
 		for i, channel := range channels {
 			srcChannelID := channel.ChannelID
-			srcTxs[i], err = srcChain.SendIBCTransfer(ctx, srcChannelID, srcUser.KeyName, testCoinSrcToDst, timeout)
+			srcTxs[i], err = srcChain.SendIBCTransfer(ctx, srcChannelID, srcUser.KeyName, testCoinSrcToDst, ibc.TransferOptions{Timeout: timeout})
 			if err != nil {
 				return fmt.Errorf("failed to send ibc transfer from source: %w", err)
 			}
@@ -180,7 +180,7 @@ func sendIBCTransfersFromBothChainsWithTimeout(
 	eg.Go(func() (err error) {
 		for i, channel := range channels {
 			dstChannelID := channel.Counterparty.ChannelID
-			dstTxs[i], err = dstChain.SendIBCTransfer(ctx, dstChannelID, dstUser.KeyName, testCoinDstToSrc, timeout)
+			dstTxs[i], err = dstChain.SendIBCTransfer(ctx, dstChannelID, dstUser.KeyName, testCoinDstToSrc, ibc.TransferOptions{Timeout: timeout})
 			if err != nil {
 				return fmt.Errorf("failed to send ibc transfer from destination: %w", err)
 			}

docs/writeCustomTests.md

@@ -216,13 +216,12 @@ osmosisRPC := osmosis.GetGRPCAddress()
 Here we send an IBC Transaction:
 ```go
 amountToSend := int64(1_000_000)
-tx, err := gaia.SendIBCTransfer(ctx, gaiaChannelID, gaiaUser.KeyName, ibc.WalletAmount{
+transfer := ibc.WalletAmount{
     Address: osmosisUser.Bech32Address(osmosis.Config().Bech32Prefix),
     Denom:   gaia.Config().Denom,
     Amount:  amountToSend,
-},
-    nil,
-)
+}
+tx, err := gaia.SendIBCTransfer(ctx, gaiaChannelID, gaiaUser.KeyName, transfer, ibc.TransferOptions{})
 ```
 
 The `Exec` method allows any arbitrary command to be passed into a chain binary or relayer binary. 

examples/cosmos/light_client_test.go

@@ -0,0 +1,100 @@
+package cosmos_test
+
+import (
+	"context"
+	"testing"
+
+	clienttypes "github.com/cosmos/ibc-go/v3/modules/core/02-client/types"
+	ibctest "github.com/strangelove-ventures/ibctest/v3"
+	"github.com/strangelove-ventures/ibctest/v3/chain/cosmos"
+	"github.com/strangelove-ventures/ibctest/v3/ibc"
+	"github.com/strangelove-ventures/ibctest/v3/testreporter"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/zap/zaptest"
+)
+
+func TestUpdateLightClients(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping in short mode")
+	}
+
+	t.Parallel()
+
+	ctx := context.Background()
+
+	// Chains
+	cf := ibctest.NewBuiltinChainFactory(zaptest.NewLogger(t), []*ibctest.ChainSpec{
+		{Name: "gaia", Version: gaiaVersion},
+		{Name: "osmosis", Version: osmosisVersion},
+	})
+
+	chains, err := cf.Chains(t.Name())
+	require.NoError(t, err)
+	gaia, osmosis := chains[0], chains[1]
+
+	// Relayer
+	client, network := ibctest.DockerSetup(t)
+	r := ibctest.NewBuiltinRelayerFactory(ibc.CosmosRly, zaptest.NewLogger(t)).Build(
+		t, client, network)
+
+	ic := ibctest.NewInterchain().
+		AddChain(gaia).
+		AddChain(osmosis).
+		AddRelayer(r, "relayer").
+		AddLink(ibctest.InterchainLink{
+			Chain1:  gaia,
+			Chain2:  osmosis,
+			Relayer: r,
+			Path:    "client-test-path",
+		})
+
+	// Build interchain
+	rep := testreporter.NewNopReporter()
+	eRep := rep.RelayerExecReporter(t)
+	require.NoError(t, ic.Build(ctx, eRep, ibctest.InterchainBuildOptions{
+		TestName:  t.Name(),
+		Client:    client,
+		NetworkID: network,
+	}))
+	t.Cleanup(func() {
+		_ = ic.Close()
+	})
+
+	require.NoError(t, r.StartRelayer(ctx, eRep))
+	t.Cleanup(func() {
+		_ = r.StopRelayer(ctx, eRep)
+	})
+
+	// Create and Fund User Wallets
+	fundAmount := int64(10_000_000)
+	users := ibctest.GetAndFundTestUsers(t, ctx, "default", fundAmount, gaia, osmosis)
+	gaiaUser, osmoUser := users[0], users[1]
+
+	// Get Channel ID
+	gaiaChannelInfo, err := r.GetChannels(ctx, eRep, gaia.Config().ChainID)
+	require.NoError(t, err)
+	chanID := gaiaChannelInfo[0].ChannelID
+
+	height, err := osmosis.Height(ctx)
+	require.NoError(t, err)
+
+	amountToSend := int64(553255) // Unique amount to make log searching easier.
+	dstAddress := osmoUser.Bech32Address(osmosis.Config().Bech32Prefix)
+	transfer := ibc.WalletAmount{
+		Address: dstAddress,
+		Denom:   gaia.Config().Denom,
+		Amount:  amountToSend,
+	}
+	tx, err := gaia.SendIBCTransfer(ctx, chanID, gaiaUser.KeyName, transfer, ibc.TransferOptions{})
+	require.NoError(t, err)
+	require.NoError(t, tx.Validate())
+
+	chain := osmosis.(*cosmos.CosmosChain)
+	reg := chain.Config().EncodingConfig.InterfaceRegistry
+	msg, err := cosmos.PollForMessage[*clienttypes.MsgUpdateClient](ctx, chain, reg, height, height+10, nil)
+	require.NoError(t, err)
+
+	require.Equal(t, "07-tendermint-0", msg.ClientId)
+	require.NotEmpty(t, msg.Signer)
+	// TODO: Assert header information
+}

examples/ibc/learn_ibc_test.go

@@ -89,13 +89,12 @@ func TestLearn(t *testing.T) {
 	// Send Transaction
 	amountToSend := int64(1_000_000)
 	dstAddress := osmosisUser.Bech32Address(osmosis.Config().Bech32Prefix)
-	tx, err := gaia.SendIBCTransfer(ctx, gaiaChannelID, gaiaUser.KeyName, ibc.WalletAmount{
+	transfer := ibc.WalletAmount{
 		Address: dstAddress,
 		Denom:   gaia.Config().Denom,
 		Amount:  amountToSend,
-	},
-		nil,
-	)
+	}
+	tx, err := gaia.SendIBCTransfer(ctx, gaiaChannelID, gaiaUser.KeyName, transfer, ibc.TransferOptions{})
 	require.NoError(t, err)
 	require.NoError(t, tx.Validate())