Support vote extensions

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/Jille/raft-grpc-transport v1.4.0
 	github.com/Jille/raftadmin v1.2.1
 	github.com/armon/go-metrics v0.4.1
-	github.com/cometbft/cometbft v0.38.0
+	github.com/cometbft/cometbft v0.38.2
 	github.com/cosmos/cosmos-sdk v0.50.1
 	github.com/cosmos/gogoproto v1.4.11
 	github.com/ethereum/go-ethereum v1.13.5

go.sum

@@ -132,8 +132,8 @@ github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwP
 github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
 github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
 github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ=
-github.com/cometbft/cometbft v0.38.0 h1:ogKnpiPX7gxCvqTEF4ly25/wAxUqf181t30P3vqdpdc=
-github.com/cometbft/cometbft v0.38.0/go.mod h1:5Jz0Z8YsHSf0ZaAqGvi/ifioSdVFPtEGrm8Y9T/993k=
+github.com/cometbft/cometbft v0.38.2 h1:io0JCh5EPxINKN5ZMI5hCdpW3QVZRy+o8qWe3mlJa/8=
+github.com/cometbft/cometbft v0.38.2/go.mod h1:PIi48BpzwlHqtV3mzwPyQgOyOnU94BNBimLS2ebBHOg=
 github.com/cometbft/cometbft-db v0.7.0 h1:uBjbrBx4QzU0zOEnU8KxoDl18dMNgDh+zZRUE0ucsbo=
 github.com/cometbft/cometbft-db v0.7.0/go.mod h1:yiKJIm2WKrt6x8Cyxtq9YTEcIMPcEe4XPxhgX59Fzf0=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=

proto/strangelove/horcrux/cosigner.proto

@@ -17,7 +17,8 @@ message Block {
 	int64 round = 2;
 	int32 step = 3;
 	bytes signBytes = 4;
-	int64 timestamp = 5;
+	bytes voteExtSignBytes = 5;
+	int64 timestamp = 6;
 }
 
 message SignBlockRequest {
@@ -27,7 +28,8 @@ message SignBlockRequest {
 
 message SignBlockResponse {
 	bytes signature = 1;
-	int64 timestamp = 2;
+	bytes vote_ext_signature = 2;
+	int64 timestamp = 3;
 }
 
 message Nonce {
@@ -59,9 +61,11 @@ message SetNoncesAndSignRequest {
 }
 
 message SetNoncesAndSignResponse {
-	bytes noncePublic = 1;
-	int64 timestamp = 2;
+	int64 timestamp = 1;
+	bytes noncePublic = 2;
 	bytes signature = 3;
+	bytes voteExtNoncePublic = 4;
+	bytes voteExtSignature = 5;
 }
 
 message GetNoncesRequest {

scripts/protocgen.sh

@@ -11,5 +11,5 @@ for dir in $proto_dirs; do
   done
 done
 
-cp -r github.com/strangelove-ventures/horcrux/signer ./
+cp -r github.com/strangelove-ventures/horcrux/v3/signer ./
 rm -rf github.com

signer/cosigner.go

@@ -45,15 +45,19 @@ func (cosigners Cosigners) GetByID(id int) Cosigner {
 // CosignerSignRequest is sent to a co-signer to obtain their signature for the SignBytes
 // The SignBytes should be a serialized block
 type CosignerSignRequest struct {
-	ChainID   string
-	SignBytes []byte
-	UUID      uuid.UUID
+	ChainID                string
+	SignBytes              []byte
+	UUID                   uuid.UUID
+	VoteExtensionSignBytes []byte
+	VoteExtUUID            uuid.UUID
 }
 
 type CosignerSignResponse struct {
-	NoncePublic []byte
-	Timestamp   time.Time
-	Signature   []byte
+	Timestamp                time.Time
+	NoncePublic              []byte
+	Signature                []byte
+	VoteExtensionNoncePublic []byte
+	VoteExtensionSignature   []byte
 }
 
 type CosignerNonce struct {
@@ -107,7 +111,8 @@ type CosignerSignBlockRequest struct {
 }
 
 type CosignerSignBlockResponse struct {
-	Signature []byte
+	Signature              []byte
+	VoteExtensionSignature []byte
 }
 type CosignerUUIDNonces struct {
 	UUID   uuid.UUID
@@ -138,8 +143,10 @@ func (n CosignerUUIDNoncesMultiple) toProto() []*proto.UUIDNonce {
 }
 
 type CosignerSetNoncesAndSignRequest struct {
-	ChainID   string
-	Nonces    *CosignerUUIDNonces
-	HRST      HRSTKey
-	SignBytes []byte
+	ChainID                string
+	Nonces                 *CosignerUUIDNonces
+	VoteExtensionNonces    *CosignerUUIDNonces
+	HRST                   HRSTKey
+	SignBytes              []byte
+	VoteExtensionSignBytes []byte
 }

signer/cosigner_grpc_server.go

@@ -34,12 +34,13 @@ func (rpc *CosignerGRPCServer) SignBlock(
 	ctx context.Context,
 	req *proto.SignBlockRequest,
 ) (*proto.SignBlockResponse, error) {
-	res, _, err := rpc.thresholdValidator.Sign(ctx, req.ChainID, BlockFromProto(req.Block))
+	sig, voteExtSig, _, err := rpc.thresholdValidator.Sign(ctx, req.ChainID, BlockFromProto(req.Block))
 	if err != nil {
 		return nil, err
 	}
 	return &proto.SignBlockResponse{
-		Signature: res,
+		Signature:        sig,
+		VoteExtSignature: voteExtSig,
 	}, nil
 }
 
@@ -75,9 +76,11 @@ func (rpc *CosignerGRPCServer) SetNoncesAndSign(
 		"step", req.Hrst.Step,
 	)
 	return &proto.SetNoncesAndSignResponse{
-		NoncePublic: res.NoncePublic,
-		Timestamp:   res.Timestamp.UnixNano(),
-		Signature:   res.Signature,
+		NoncePublic:        res.NoncePublic,
+		Timestamp:          res.Timestamp.UnixNano(),
+		Signature:          res.Signature,
+		VoteExtNoncePublic: res.VoteExtensionNoncePublic,
+		VoteExtSignature:   res.VoteExtensionSignature,
 	}, nil
 }
 

signer/file.go

@@ -200,14 +200,27 @@ func (pv *FilePV) GetPubKey() (crypto.PubKey, error) {
 	return pv.Key.PubKey, nil
 }
 
-func (pv *FilePV) Sign(block Block) ([]byte, time.Time, error) {
-	height, round, step, signBytes := block.Height, int32(block.Round), block.Step, block.SignBytes
+func (pv *FilePV) Sign(block Block) ([]byte, []byte, time.Time, error) {
+	height, round, step, signBytes, voteExtensionSignBytes := block.Height, int32(block.Round), block.Step, block.SignBytes, block.VoteExtensionSignBytes
 
 	lss := pv.LastSignState
 
 	sameHRS, err := lss.CheckHRS(height, round, step)
 	if err != nil {
-		return nil, block.Timestamp, err
+		return nil, nil, block.Timestamp, err
+	}
+
+	// Vote extensions are non-deterministic, so it is possible that an
+	// application may have created a different extension. We therefore always
+	// re-sign the vote extensions of precommits. For prevotes and nil
+	// precommits, the extension signature will always be empty.
+	// Even if the signed over data is empty, we still add the signature
+	var extSig []byte
+	if block.Step == stepPrecommit && len(voteExtensionSignBytes) > 0 {
+		extSig, err = pv.Key.PrivKey.Sign(voteExtensionSignBytes)
+		if err != nil {
+			return nil, nil, block.Timestamp, err
+		}
 	}
 
 	// We might crash before writing to the wal,
@@ -218,28 +231,28 @@ func (pv *FilePV) Sign(block Block) ([]byte, time.Time, error) {
 	if sameHRS {
 		switch {
 		case bytes.Equal(signBytes, lss.SignBytes):
-			return lss.Signature, block.Timestamp, nil
+			return lss.Signature, nil, block.Timestamp, nil
 		case block.Step == stepPropose:
 			if timestamp, ok := checkProposalsOnlyDifferByTimestamp(lss.SignBytes, signBytes); ok {
-				return lss.Signature, timestamp, nil
+				return lss.Signature, nil, timestamp, nil
 			}
 		case block.Step == stepPrevote || block.Step == stepPrecommit:
 			if timestamp, ok := checkVotesOnlyDifferByTimestamp(lss.SignBytes, signBytes); ok {
-				return lss.Signature, timestamp, nil
+				return lss.Signature, extSig, timestamp, nil
 			}
 		}
 
-		return nil, block.Timestamp, fmt.Errorf("conflicting data")
+		return nil, extSig, block.Timestamp, fmt.Errorf("conflicting data")
 	}
 
 	// It passed the checks. Sign the vote
 	sig, err := pv.Key.PrivKey.Sign(signBytes)
 	if err != nil {
-		return nil, block.Timestamp, err
+		return nil, nil, block.Timestamp, err
 	}
 	pv.saveSigned(height, round, step, signBytes, sig)
 
-	return sig, block.Timestamp, nil
+	return sig, extSig, block.Timestamp, nil
 }
 
 // Save persists the FilePV to disk.

signer/local_cosigner.go

@@ -232,6 +232,13 @@ func (cosigner *LocalCosigner) sign(req CosignerSignRequest) (CosignerSignRespon
 		return res, nil
 	}
 
+	defer func() {
+		cosigner.noncesMu.Lock()
+		delete(cosigner.nonces, req.UUID)
+		delete(cosigner.nonces, req.VoteExtUUID)
+		cosigner.noncesMu.Unlock()
+	}()
+
 	nonces, err := cosigner.combinedNonces(
 		cosigner.GetID(),
 		uint8(cosigner.config.Config.ThresholdModeConfig.Threshold),
@@ -241,17 +248,45 @@ func (cosigner *LocalCosigner) sign(req CosignerSignRequest) (CosignerSignRespon
 		return res, err
 	}
 
-	sig, err := ccs.signer.Sign(nonces, req.SignBytes)
-	if err != nil {
+	var voteExtNonces []Nonce
+	if len(req.VoteExtensionSignBytes) > 0 {
+		voteExtNonces, err = cosigner.combinedNonces(
+			cosigner.GetID(),
+			uint8(cosigner.config.Config.ThresholdModeConfig.Threshold),
+			req.VoteExtUUID,
+		)
+		if err != nil {
+			return res, err
+		}
+	}
+
+	var eg errgroup.Group
+
+	var sig, voteExtSig []byte
+	eg.Go(func() error {
+		var err error
+		sig, err = ccs.signer.Sign(nonces, req.SignBytes)
+		return err
+	})
+	if len(req.VoteExtensionSignBytes) > 0 {
+		eg.Go(func() error {
+			var err error
+			voteExtSig, err = ccs.signer.Sign(voteExtNonces, req.VoteExtensionSignBytes)
+			return err
+		})
+	}
+
+	if err := eg.Wait(); err != nil {
 		return res, err
 	}
 
 	err = ccs.lastSignState.Save(SignStateConsensus{
-		Height:    hrst.Height,
-		Round:     hrst.Round,
-		Step:      hrst.Step,
-		Signature: sig,
-		SignBytes: req.SignBytes,
+		Height:                 hrst.Height,
+		Round:                  hrst.Round,
+		Step:                   hrst.Step,
+		Signature:              sig,
+		SignBytes:              req.SignBytes,
+		VoteExtensionSignature: res.VoteExtensionSignature,
 	}, &cosigner.pendingDiskWG)
 
 	if err != nil {
@@ -260,11 +295,8 @@ func (cosigner *LocalCosigner) sign(req CosignerSignRequest) (CosignerSignRespon
 		}
 	}
 
-	cosigner.noncesMu.Lock()
-	delete(cosigner.nonces, req.UUID)
-	cosigner.noncesMu.Unlock()
-
 	res.Signature = sig
+	res.VoteExtensionSignature = voteExtSig
 
 	// Note - Function may return before this line so elapsed time for Finish may be multiple block times
 	metricsTimeKeeper.SetPreviousLocalSignFinish(time.Now())
@@ -496,14 +528,31 @@ func (cosigner *LocalCosigner) SetNoncesAndSign(
 		})
 	}
 
+	if req.VoteExtensionNonces != nil {
+		for _, secretPart := range req.VoteExtensionNonces.Nonces {
+			secretPart := secretPart
+
+			eg.Go(func() error {
+				return cosigner.setNonce(req.VoteExtensionNonces.UUID, secretPart)
+			})
+		}
+	}
+
 	if err := eg.Wait(); err != nil {
 		return nil, err
 	}
 
-	res, err := cosigner.sign(CosignerSignRequest{
+	cosignerReq := CosignerSignRequest{
 		UUID:      req.Nonces.UUID,
 		ChainID:   chainID,
 		SignBytes: req.SignBytes,
-	})
+	}
+
+	if len(req.VoteExtensionSignBytes) > 0 {
+		cosignerReq.VoteExtensionSignBytes = req.VoteExtensionSignBytes
+		cosignerReq.VoteExtUUID = req.VoteExtensionNonces.UUID
+	}
+
+	res, err := cosigner.sign(cosignerReq)
 	return &res, err
 }