One config file to rule them all and in cryptography bind them

Makefile

@@ -18,7 +18,7 @@ build-linux:
 	@GOOS=linux GOARCH=amd64 go build --mod readonly $(BUILD_FLAGS) -o ./build/horcrux ./cmd/horcrux
 
 test:
-	@go test -race -timeout 20m -mod readonly -v ./...
+	@go test -race -timeout 30m -mod readonly -v ./...
 
 test-short:
 	@go test -mod readonly -run TestDownedSigners2of3 -v ./... 

cmd/horcrux/cmd/config.go

@@ -1,10 +1,7 @@
 package cmd
 
 import (
-	"errors"
 	"fmt"
-	"net"
-	"net/url"
 	"os"
 
 	"github.com/spf13/cobra"
@@ -68,28 +65,10 @@ func initCmd() *cobra.Command {
 					return err
 				}
 
-				listen, _ := cmdFlags.GetString("listen")
-				if listen == "" {
-					return errors.New("must input at least one node")
-				}
-				url, err := url.Parse(listen)
-				if err != nil {
-					return fmt.Errorf("error parsing listen address: %s, %v", listen, err)
-				}
-				host, _, err := net.SplitHostPort(url.Host)
-				if err != nil {
-					return err
-				}
-				if host == "0.0.0.0" {
-					return errors.New("host cannot be 0.0.0.0, must be reachable from other peers")
-				}
-
 				cfg = signer.Config{
 					PrivValKeyDir: keyDir,
 					CosignerConfig: &signer.CosignerConfig{
 						Threshold: threshold,
-						Shares:    len(peers) + 1,
-						P2PListen: listen,
 						Peers:     peers,
 						Timeout:   timeout,
 					},
@@ -133,7 +112,6 @@ func initCmd() *cobra.Command {
 		"cosigner peer addresses in format tcp://{addr}:{port}|{share-id} \n"+
 			"(i.e. \"tcp://node-1:2222|2,tcp://node-2:2222|3\")")
 	cmd.Flags().IntP("threshold", "t", 0, "indicate number of signatures required for threshold signature")
-	cmd.Flags().StringP("listen", "l", "", "listen address of the signer")
 	cmd.Flags().StringP("debug-addr", "d", "", "listen address for Debug and Prometheus metrics in format localhost:8543")
 	cmd.Flags().StringP("key-dir", "k", "", "priv val key directory")
 	cmd.Flags().String("timeout", "1500ms", "configure cosigner rpc server timeout value, \n"+

cmd/horcrux/cmd/config_test.go

@@ -23,9 +23,8 @@ func TestConfigInitCmd(t *testing.T) {
 			args: []string{
 				"tcp://10.168.0.1:1234",
 				"-c",
-				"-p", "tcp://10.168.1.2:2222|2,tcp://10.168.1.3:2222|3",
+				"-p", "tcp://10.168.1.1:2222|1,tcp://10.168.1.2:2222|2,tcp://10.168.1.3:2222|3",
 				"-t", "2",
-				"-l", "tcp://10.168.1.1:2222",
 				"--timeout", "1500ms",
 			},
 			expectErr: false,
@@ -36,9 +35,8 @@ func TestConfigInitCmd(t *testing.T) {
 			args: []string{
 				"://10.168.0.1:1234", // Missing/malformed protocol scheme
 				"-c",
-				"-p", "tcp://10.168.1.2:2222|2,tcp://10.168.1.3:2222|3",
+				"-p", "tcp://10.168.1.1:2222|1,tcp://10.168.1.2:2222|2,tcp://10.168.1.3:2222|3",
 				"-t", "2",
-				"-l", "tcp://10.168.1.1:2222",
 				"--timeout", "1500ms",
 			},
 			expectErr: true,
@@ -49,9 +47,8 @@ func TestConfigInitCmd(t *testing.T) {
 			args: []string{
 				"tcp://10.168.0.1:1234",
 				"-c",
-				"-p", "tcp://10.168.1.2:2222,tcp://10.168.1.3:2222", // Missing share IDs
+				"-p", "tcp://10.168.1.1:2222|1,tcp://10.168.1.2:2222,tcp://10.168.1.3:2222", // Missing share IDs
 				"-t", "2",
-				"-l", "tcp://10.168.1.1:2222",
 				"--timeout", "1500ms",
 			},
 			expectErr: true,

cmd/horcrux/cmd/cosigner.go

@@ -143,36 +143,39 @@ func startCosignerCmd() *cobra.Command {
 				return fmt.Errorf("error reading cosigner key (%s): %w", keyFile, err)
 			}
 
-			cosigners := []signer.Cosigner{}
+			cosignerConfig := config.Config.CosignerConfig
 
-			// add ourselves as a peer so localcosigner can handle GetEphSecPart requests
-			peers := []signer.CosignerPeer{{
-				ID:        key.ID,
-				PublicKey: key.RSAKey.PublicKey,
-			}}
+			remoteCosigners := make([]signer.Cosigner, 0, len(cosignerConfig.Peers)-1)
+			peers := make([]signer.CosignerPeer, len(cosignerConfig.Peers))
 
-			cosignerConfig := config.Config.CosignerConfig
+			var p2pListen string
 
-			for _, cosignerParams := range cosignerConfig.Peers {
-				cosigner := signer.NewRemoteCosigner(cosignerParams.ShareID, cosignerParams.P2PAddr)
-				cosigners = append(cosigners, cosigner)
+			for i, cosignerParams := range cosignerConfig.Peers {
+				if cosignerParams.ShareID != key.ID {
+					remoteCosigners = append(
+						remoteCosigners,
+						signer.NewRemoteCosigner(cosignerParams.ShareID, cosignerParams.P2PAddr),
+					)
+				} else {
+					p2pListen = cosignerParams.P2PAddr
+				}
 
 				pubKey := key.CosignerKeys[cosignerParams.ShareID-1]
-				peers = append(peers, signer.CosignerPeer{
-					ID:        cosigner.GetID(),
+				peers[i] = signer.CosignerPeer{
+					ID:        cosignerParams.ShareID,
 					PublicKey: *pubKey,
-				})
+				}
 			}
 
-			total := len(cosignerConfig.Peers) + 1
+			if p2pListen == "" {
+				return fmt.Errorf("peer config does not exist for our share ID %d", key.ID)
+			}
 
 			localCosigner := signer.NewLocalCosigner(
 				&config,
-				key.ID,
-				key.RSAKey,
+				key,
 				peers,
-				cosignerConfig.P2PListen,
-				uint8(total),
+				p2pListen,
 				uint8(cosignerConfig.Threshold),
 			)
 
@@ -191,7 +194,7 @@ func startCosignerCmd() *cobra.Command {
 
 			// Start RAFT store listener
 			raftStore := signer.NewRaftStore(nodeID,
-				raftDir, cosignerConfig.P2PListen, timeout, logger, localCosigner, cosigners)
+				raftDir, p2pListen, timeout, logger, localCosigner, remoteCosigners)
 			if err := raftStore.Start(); err != nil {
 				return fmt.Errorf("error starting raft store: %w", err)
 			}
@@ -202,7 +205,7 @@ func startCosignerCmd() *cobra.Command {
 				&config,
 				cosignerConfig.Threshold,
 				localCosigner,
-				cosigners,
+				remoteCosigners,
 				raftStore,
 			)
 

cmd/horcrux/cmd/leader_election.go

@@ -9,6 +9,7 @@ import (
 	grpcretry "github.com/grpc-ecosystem/go-grpc-middleware/retry"
 	"github.com/spf13/cobra"
 	"github.com/strangelove-ventures/horcrux/client"
+	"github.com/strangelove-ventures/horcrux/signer"
 	"github.com/strangelove-ventures/horcrux/signer/proto"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
@@ -93,20 +94,43 @@ func getLeaderCmd() *cobra.Command {
 		Example:      `horcrux leader`,
 		SilenceUsage: true,
 		RunE: func(cmd *cobra.Command, args []string) (err error) {
-			if config.Config.CosignerConfig == nil {
+			cosignerConfig := config.Config.CosignerConfig
+			if cosignerConfig == nil {
 				return fmt.Errorf("cosigner configuration is not present in config file")
 			}
 
-			if len(config.Config.CosignerConfig.Peers) == 0 {
+			if len(cosignerConfig.Peers) == 0 {
 				return fmt.Errorf("cosigner configuration has no peers")
 			}
 
+			keyFile, err := config.KeyFileExistsCosignerRSA()
+			if err != nil {
+				return err
+			}
+
+			key, err := signer.LoadCosignerKeyRSA(keyFile)
+			if err != nil {
+				return fmt.Errorf("error reading cosigner key (%s): %w", keyFile, err)
+			}
+
+			var p2pListen string
+
+			for _, peer := range cosignerConfig.Peers {
+				if peer.ShareID == key.ID {
+					p2pListen = peer.P2PAddr
+				}
+			}
+
+			if p2pListen == "" {
+				return fmt.Errorf("peer config does not exist for our share ID %d", key.ID)
+			}
+
 			retryOpts := []grpcretry.CallOption{
 				grpcretry.WithBackoff(grpcretry.BackoffExponential(100 * time.Millisecond)),
 				grpcretry.WithMax(5),
 			}
 
-			grpcAddress, err := client.SanitizeAddress(config.Config.CosignerConfig.P2PListen)
+			grpcAddress, err := client.SanitizeAddress(p2pListen)
 			if err != nil {
 				return err
 			}

cmd/horcrux/cmd/migrate.go

@@ -23,6 +23,9 @@ type (
 	v2Config struct {
 		ChainID        string  `json:"chain-id" yaml:"chain-id"`
 		PrivValKeyFile *string `json:"key-file,omitempty" yaml:"key-file,omitempty"`
+		Cosigner       struct {
+			P2PListen string `json:"p2p-listen"  yaml:"p2p-listen"`
+		} `json:"cosigner"  yaml:"cosigner"`
 	}
 
 	v2CosignerKey struct {
@@ -118,10 +121,10 @@ func (cosignerKey *v2CosignerKey) validate() error {
 
 func migrateCmd() *cobra.Command {
 	return &cobra.Command{
-		Use:          "migrate",
+		Use:          "migrate [chain-id]",
 		Short:        "Migrate config and key files from v2 to v3",
 		SilenceUsage: true,
-		Args:         cobra.NoArgs,
+		Args:         cobra.RangeArgs(0, 1),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			cmd.SilenceUsage = true
 
@@ -130,17 +133,23 @@ func migrateCmd() *cobra.Command {
 				return err
 			}
 
+			var chainID string
+
 			var legacyConfig v2Config
 
 			if err := yaml.Unmarshal(configFile, &legacyConfig); err != nil {
 				return fmt.Errorf("failed to read config file as legacy: %w", err)
 			}
 
-			if legacyConfig.ChainID == "" {
-				return fmt.Errorf("unable to migrate v2 config without chain-id")
-			}
+			if len(args) == 1 {
+				chainID = args[0]
+			} else {
+				if legacyConfig.ChainID == "" {
+					return fmt.Errorf("unable to migrate v2 config without chain-id. please provide [chain-id] argument")
+				}
 
-			chainID := legacyConfig.ChainID
+				chainID = legacyConfig.ChainID
+			}
 
 			var legacyCosignerKeyFile string
 
@@ -203,6 +212,13 @@ func migrateCmd() *cobra.Command {
 				return fmt.Errorf("failed to write new RSA key to %s: %w", newRSAPath, err)
 			}
 
+			if legacyConfig.Cosigner.P2PListen != "" {
+				config.Config.CosignerConfig.Peers = append(config.Config.CosignerConfig.Peers, signer.CosignerPeerConfig{
+					ShareID: legacyCosignerKey.ID,
+					P2PAddr: legacyConfig.Cosigner.P2PListen,
+				})
+			}
+
 			if err := config.WriteConfigFile(); err != nil {
 				return err
 			}

cmd/horcrux/cmd/state_test.go

@@ -26,8 +26,7 @@ func TestStateSetCmd(t *testing.T) {
 		"tcp://10.168.0.1:1234",
 		"-c",
 		"-t", "2",
-		"-p", "tcp://10.168.1.2:2222|2,tcp://10.168.1.3:2222|3",
-		"-l", "tcp://10.168.1.1:2222",
+		"-p", "tcp://10.168.1.1:2222|1,tcp://10.168.1.2:2222|2,tcp://10.168.1.3:2222|3",
 		"--timeout", "1500ms",
 	})
 	err := cmd.Execute()

cmd/horcrux/cmd/testdata/config-migrated.yaml

@@ -1,8 +1,8 @@
 cosigner:
   threshold: 2
-  shares: 3
-  p2p-listen: tcp://127.0.0.1:2222
   peers:
+  - share-id: 1
+    p2p-addr: tcp://127.0.0.1:2222
   - share-id: 2
     p2p-addr: tcp://127.0.0.1:2223
   - share-id: 3

cmd/horcrux/cmd/testdata/config-v2.yaml

@@ -2,12 +2,12 @@ chain-id: test
 cosigner:
   threshold: 2
   shares: 3
-  p2p-listen: tcp://127.0.0.1:2222
+  p2p-listen: tcp://127.0.0.1:2224
   peers:
+  - share-id: 1
+    p2p-addr: tcp://127.0.0.1:2222
   - share-id: 2
     p2p-addr: tcp://127.0.0.1:2223
-  - share-id: 3
-    p2p-addr: tcp://127.0.0.1:2224
   rpc-timeout: 1000ms
 chain-nodes:
 - priv-val-addr: tcp://127.0.0.1:1234

docs/migrating.md

@@ -63,20 +63,11 @@ $ sudo nano /etc/systemd/system/horcrux.service
 $ sudo systemctl daemon-reload
 ```
 
-After that is done, initialize the configuration for each node using the `horcrux` cli. Each node will require a slightly different command. Below are the commands for each of the 3 signer nodes given the private IPs above. Input your own data here:
+After that is done, initialize the configuration for the cosigners using the `horcrux` cli. If you would like different cosigners to connect to different sentry node(s), modify the first argument to generate different config for each cosigner.
 
 ```bash
-# Run this command on the signer-1 VM
-# signer-1 connects to sentry-1
-$ horcrux config init "tcp://10.168.0.1:1234" -c -p "tcp://10.168.1.2:2222|2,tcp://10.168.1.3:2222|3" -l "tcp://10.168.1.1:2222" -t 2
-
-# Run this command on the signer-2 VM
-# signer-2 connects to sentry-2
-$ horcrux config init "tcp://10.168.0.2:1234" -c -p "tcp://10.168.1.1:2222|1,tcp://10.168.1.3:2222|3" -l "tcp://10.168.1.2:2222" -t 2
-
-# Run this command on the signer-3 VM
-# signer-3 connects to sentry-3
-$ horcrux config init "tcp://10.168.0.3:1234" -c -p "tcp://10.168.1.1:2222|1,tcp://10.168.1.2:2222|2" -l "tcp://10.168.1.3:2222" -t 2
+# Run this command to generate a config file that can be used on all config nodes.
+$ horcrux config init "tcp://10.168.0.1:1234" -c -p "tcp://10.168.1.1:2222|1,tcp://10.168.1.2:2222|2,tcp://10.168.1.3:2222|3" -t 2 --timeout 1000ms
 ```
 
 > **Note** 
@@ -219,6 +210,6 @@ You now can sleep much better at night because you are much less likely to have
 
 ### 8. Administration Commands
 
-`horcrux elect` - Elect a new cluster leader. Pass an optional argument with the intended leader ID to elect that cosigner as the new leader, e.g. `horcrux elect 3` to elect cosigner with `ID: 3` as leader
+`horcrux elect` - Elect a new cluster leader. Pass an optional argument with the intended leader ID to elect that cosigner as the new leader, e.g. `horcrux elect 3` to elect cosigner with `ID: 3` as leader. This is an optimistic leader election, it is not guaranteed that the exact requested leader will be elected.
 
 `horcrux cosigner address` - Get the public key address as both hex and optionally the validator consensus bech32 address. To retrieve the valcons bech32 address, pass an optional argument with the chain's bech32 valcons prefix, e.g. `horcrux cosigner address cosmosvalcons`