Add compatibility with old signature format (#192)

* Add compatibility with old signature format

There are still many advertisements that are using the deprecated signature format.  Since these ads are still valid, allow the indexer to process them.  Log when an ad with a deprecated signature format is seen.

Other changes:
- Removed some unwanted/unneeded logging from finder and admin client
- Update version

* Update to version 0.3.0 because of API changes

api/v0/admin/client/http/client.go

@@ -13,13 +13,10 @@ import (
 	"path"
 
 	"github.com/filecoin-project/storetheindex/api/v0/httpclient"
-	logging "github.com/ipfs/go-log/v2"
 	"github.com/libp2p/go-libp2p-core/peer"
 	"github.com/multiformats/go-multiaddr"
 )
 
-var log = logging.Logger("adminhttpclient")
-
 const (
 	adminPort = 3002
 
@@ -68,7 +65,6 @@ func (c *Client) ImportFromManifest(ctx context.Context, fileName string, provID
 		}
 		return fmt.Errorf("importing from manifest failed: %v%s", http.StatusText(resp.StatusCode), errMsg)
 	}
-	log.Infow("Success")
 	return nil
 }
 
@@ -95,7 +91,6 @@ func (c *Client) ImportFromCidList(ctx context.Context, fileName string, provID
 		}
 		return fmt.Errorf("importing from cidlist failed: %v%s", http.StatusText(resp.StatusCode), errMsg)
 	}
-	log.Infow("Success")
 	return nil
 }
 

api/v0/finder/client/http/client.go

@@ -10,13 +10,10 @@ import (
 
 	"github.com/filecoin-project/storetheindex/api/v0/finder/model"
 	"github.com/filecoin-project/storetheindex/api/v0/httpclient"
-	logging "github.com/ipfs/go-log/v2"
 	"github.com/libp2p/go-libp2p-core/peer"
 	"github.com/multiformats/go-multihash"
 )
 
-var log = logging.Logger("finderhttpclient")
-
 const (
 	finderPort    = 3000
 	finderPath    = "/multihash"
@@ -142,7 +139,6 @@ func (c *Client) sendRequest(req *http.Request) (*model.FindResponse, error) {
 	// Handle failed requests
 	if resp.StatusCode != http.StatusOK {
 		if resp.StatusCode == http.StatusNotFound {
-			log.Info("Entry not found in indexer")
 			return &model.FindResponse{}, nil
 		}
 		return nil, fmt.Errorf("batch find query failed: %v", http.StatusText(resp.StatusCode))

api/v0/ingest/schema/envelope.go

@@ -6,13 +6,16 @@ import (
 	"fmt"
 
 	"github.com/ipfs/go-cid"
+	logging "github.com/ipfs/go-log/v2"
 	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
 	crypto "github.com/libp2p/go-libp2p-core/crypto"
 	"github.com/libp2p/go-libp2p-core/peer"
 	"github.com/libp2p/go-libp2p-core/record"
-	mh "github.com/multiformats/go-multihash"
+	"github.com/multiformats/go-multihash"
 )
 
+var log = logging.Logger("indexer/schema")
+
 const (
 	adSignatureCodec  = "/indexer/ingest/adSignature"
 	adSignatureDomain = "indexer"
@@ -48,7 +51,7 @@ func (r *advSignatureRecord) UnmarshalRecord(buf []byte) error {
 }
 
 // Generates the data payload used for signature.
-func signaturePayload(previousID Link_Advertisement, provider string, addrs []string, entries Link, metadata []byte, isRm bool) ([]byte, error) {
+func signaturePayload(previousID Link_Advertisement, provider string, addrs []string, entries Link, metadata []byte, isRm, oldFormat bool) ([]byte, error) {
 	bindex := cid.Undef.Bytes()
 	lindex, err := previousID.AsLink()
 	if err != nil {
@@ -84,7 +87,14 @@ func signaturePayload(previousID Link_Advertisement, provider string, addrs []st
 		sigBuf.WriteByte(0)
 	}
 
-	return mh.Sum(sigBuf.Bytes(), mhCode, -1)
+	// Generates the old (incorrect) data payload used for signature.  This is
+	// only for compatability with existing advertisements that have the old
+	// signatures, and should be removed when no longer needed.
+	if oldFormat {
+		return multihash.Encode(sigBuf.Bytes(), mhCode)
+	}
+
+	return multihash.Sum(sigBuf.Bytes(), mhCode, -1)
 }
 
 // Signs advertisements using libp2p envelope
@@ -99,7 +109,7 @@ func signAdvertisement(privkey crypto.PrivKey, ad Advertisement) ([]byte, error)
 	entries := ad.FieldEntries()
 	metadata := ad.FieldMetadata().x
 
-	advID, err := signaturePayload(&previousID, provider, addrs, entries, metadata, isRm)
+	advID, err := signaturePayload(&previousID, provider, addrs, entries, metadata, isRm, false)
 	if err != nil {
 		return nil, err
 	}
@@ -113,6 +123,10 @@ func signAdvertisement(privkey crypto.PrivKey, ad Advertisement) ([]byte, error)
 // VerifyAdvertisement verifies that the advertisement has been signed and
 // generated correctly.  Returns the peer ID of the signer.
 func VerifyAdvertisement(ad Advertisement) (peer.ID, error) {
+	// sigSize is the size of the current signature.  Any signature that is not
+	// this size is the old signature format.
+	const sigSize = 34
+
 	previousID := ad.FieldPreviousID().v
 	provider := ad.FieldProvider().x
 	addrs, err := IpldToGoStrings(ad.FieldAddresses())
@@ -124,25 +138,31 @@ func VerifyAdvertisement(ad Advertisement) (peer.ID, error) {
 	metadata := ad.FieldMetadata().x
 	sig := ad.FieldSignature().x
 
-	genID, err := signaturePayload(&previousID, provider, addrs, entries, metadata, isRm)
+	// Consume envelope
+	rec := &advSignatureRecord{}
+	envelope, err := record.ConsumeTypedEnvelope(sig, rec)
 	if err != nil {
 		return peer.ID(""), err
 	}
 
-	// Consume envelope
-	rec := &advSignatureRecord{}
-	envelope, err := record.ConsumeTypedEnvelope(sig, rec)
+	oldFormat := len(rec.advID) != sigSize
+	genID, err := signaturePayload(&previousID, provider, addrs, entries, metadata, isRm, oldFormat)
 	if err != nil {
 		return peer.ID(""), err
 	}
+
 	if !bytes.Equal(genID, rec.advID) {
-		return peer.ID(""), errors.New("envelope signed with the wrong ID")
+		return peer.ID(""), errors.New("invalid signature")
 	}
 
 	signerID, err := peer.IDFromPublicKey(envelope.PublicKey)
 	if err != nil {
 		return peer.ID(""), fmt.Errorf("cannot convert public key to peer ID: %s", err)
 	}
 
+	if oldFormat {
+		log.Warnw("advertisement has deprecated signature format", "signer", signerID)
+	}
+
 	return signerID, nil
 }

internal/ingest/ingest_test.go

@@ -47,7 +47,7 @@ const (
 	testRetryTimeout  = 10 * time.Second
 
 	testEntriesChunkCount = 3
-	testEntriesChunkSize  = 10
+	testEntriesChunkSize  = 15
 )
 
 var (
@@ -249,7 +249,6 @@ func TestRmWithNoEntries(t *testing.T) {
 	allMhs := typehelpers.AllMultihashesFromAd(t, prevAdNode.(schema.Advertisement), te.publisherLinkSys)
 	// Remove the mhs from the first ad (since the last add removed this from the indexer)
 	allMhs = allMhs[1:]
-	fmt.Println("!!!!", allMhs)
 	checkMhsIndexedEventually(t, te.ingester.indexer, te.pubHost.ID(), allMhs)
 }
 func TestSync(t *testing.T) {

version.json

@@ -1,3 +1,3 @@
 {
-  "version": "v0.2.5"
+  "version": "v0.3.0"
 }