From 67c2738155a8ad1d996134f9747d180bc86e185b Mon Sep 17 00:00:00 2001 From: gammazero Date: Sun, 30 Jan 2022 02:43:36 -0800 Subject: [PATCH 1/3] Add compatibility with old signature format There are still many advertisements that are using the depricated signature format. Since these ads are still valid, allow the indexer to process them. Log when an ad with a depricated signature format is seen. Other changes: - Removed some unwanted/unneeded logging from finder and admin client - Update version --- api/v0/admin/client/http/client.go | 5 ---- api/v0/finder/client/http/client.go | 4 --- api/v0/ingest/schema/envelope.go | 38 ++++++++++++++++++++++------- version.json | 2 +- 4 files changed, 30 insertions(+), 19 deletions(-) diff --git a/api/v0/admin/client/http/client.go b/api/v0/admin/client/http/client.go index db14cee7d..f7ba3102c 100644 --- a/api/v0/admin/client/http/client.go +++ b/api/v0/admin/client/http/client.go @@ -13,13 +13,10 @@ import ( "path" "github.com/filecoin-project/storetheindex/api/v0/httpclient" - logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p-core/peer" "github.com/multiformats/go-multiaddr" ) -var log = logging.Logger("adminhttpclient") - const ( adminPort = 3002 @@ -68,7 +65,6 @@ func (c *Client) ImportFromManifest(ctx context.Context, fileName string, provID } return fmt.Errorf("importing from manifest failed: %v%s", http.StatusText(resp.StatusCode), errMsg) } - log.Infow("Success") return nil } @@ -95,7 +91,6 @@ func (c *Client) ImportFromCidList(ctx context.Context, fileName string, provID } return fmt.Errorf("importing from cidlist failed: %v%s", http.StatusText(resp.StatusCode), errMsg) } - log.Infow("Success") return nil } diff --git a/api/v0/finder/client/http/client.go b/api/v0/finder/client/http/client.go index 1b81cffa1..9984023c3 100644 --- a/api/v0/finder/client/http/client.go +++ b/api/v0/finder/client/http/client.go @@ -10,13 +10,10 @@ import ( "github.com/filecoin-project/storetheindex/api/v0/finder/model" "github.com/filecoin-project/storetheindex/api/v0/httpclient" - logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p-core/peer" "github.com/multiformats/go-multihash" ) -var log = logging.Logger("finderhttpclient") - const ( finderPort = 3000 finderPath = "/multihash" @@ -142,7 +139,6 @@ func (c *Client) sendRequest(req *http.Request) (*model.FindResponse, error) { // Handle failed requests if resp.StatusCode != http.StatusOK { if resp.StatusCode == http.StatusNotFound { - log.Info("Entry not found in indexer") return &model.FindResponse{}, nil } return nil, fmt.Errorf("batch find query failed: %v", http.StatusText(resp.StatusCode)) diff --git a/api/v0/ingest/schema/envelope.go b/api/v0/ingest/schema/envelope.go index 122520c04..ffd2154ca 100644 --- a/api/v0/ingest/schema/envelope.go +++ b/api/v0/ingest/schema/envelope.go @@ -6,13 +6,16 @@ import ( "fmt" "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" cidlink "github.com/ipld/go-ipld-prime/linking/cid" crypto "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/record" - mh "github.com/multiformats/go-multihash" + "github.com/multiformats/go-multihash" ) +var log = logging.Logger("indexer/schema") + const ( adSignatureCodec = "/indexer/ingest/adSignature" adSignatureDomain = "indexer" @@ -48,7 +51,7 @@ func (r *advSignatureRecord) UnmarshalRecord(buf []byte) error { } // Generates the data payload used for signature. -func signaturePayload(previousID Link_Advertisement, provider string, addrs []string, entries Link, metadata []byte, isRm bool) ([]byte, error) { +func signaturePayload(previousID Link_Advertisement, provider string, addrs []string, entries Link, metadata []byte, isRm, oldFormat bool) ([]byte, error) { bindex := cid.Undef.Bytes() lindex, err := previousID.AsLink() if err != nil { @@ -84,7 +87,14 @@ func signaturePayload(previousID Link_Advertisement, provider string, addrs []st sigBuf.WriteByte(0) } - return mh.Sum(sigBuf.Bytes(), mhCode, -1) + // Generates the old (incorrect) data payload used for signature. This is + // only for compatability with existing advertisements that have the old + // signatures, and should be removed when no longer needed. + if oldFormat { + return multihash.Encode(sigBuf.Bytes(), mhCode) + } + + return multihash.Sum(sigBuf.Bytes(), mhCode, -1) } // Signs advertisements using libp2p envelope @@ -99,7 +109,7 @@ func signAdvertisement(privkey crypto.PrivKey, ad Advertisement) ([]byte, error) entries := ad.FieldEntries() metadata := ad.FieldMetadata().x - advID, err := signaturePayload(&previousID, provider, addrs, entries, metadata, isRm) + advID, err := signaturePayload(&previousID, provider, addrs, entries, metadata, isRm, false) if err != nil { return nil, err } @@ -113,6 +123,10 @@ func signAdvertisement(privkey crypto.PrivKey, ad Advertisement) ([]byte, error) // VerifyAdvertisement verifies that the advertisement has been signed and // generated correctly. Returns the peer ID of the signer. func VerifyAdvertisement(ad Advertisement) (peer.ID, error) { + // sigSize is the size of the current signature. Any signature that is not + // this size is the old signature format. + const sigSize = 34 + previousID := ad.FieldPreviousID().v provider := ad.FieldProvider().x addrs, err := IpldToGoStrings(ad.FieldAddresses()) @@ -124,19 +138,21 @@ func VerifyAdvertisement(ad Advertisement) (peer.ID, error) { metadata := ad.FieldMetadata().x sig := ad.FieldSignature().x - genID, err := signaturePayload(&previousID, provider, addrs, entries, metadata, isRm) + // Consume envelope + rec := &advSignatureRecord{} + envelope, err := record.ConsumeTypedEnvelope(sig, rec) if err != nil { return peer.ID(""), err } - // Consume envelope - rec := &advSignatureRecord{} - envelope, err := record.ConsumeTypedEnvelope(sig, rec) + oldFormat := len(rec.advID) != sigSize + genID, err := signaturePayload(&previousID, provider, addrs, entries, metadata, isRm, oldFormat) if err != nil { return peer.ID(""), err } + if !bytes.Equal(genID, rec.advID) { - return peer.ID(""), errors.New("envelope signed with the wrong ID") + return peer.ID(""), errors.New("invalid signature") } signerID, err := peer.IDFromPublicKey(envelope.PublicKey) @@ -144,5 +160,9 @@ func VerifyAdvertisement(ad Advertisement) (peer.ID, error) { return peer.ID(""), fmt.Errorf("cannot convert public key to peer ID: %s", err) } + if oldFormat { + log.Warnw("advertisement has deprecated signature format", "signer", signerID) + } + return signerID, nil } diff --git a/version.json b/version.json index 7af5dc487..d53fb2eeb 100644 --- a/version.json +++ b/version.json @@ -1,3 +1,3 @@ { - "version": "v0.2.5" + "version": "v0.2.6" } From 6735032b796ad5c01974cce7dab76758b612aac3 Mon Sep 17 00:00:00 2001 From: gammazero Date: Sun, 30 Jan 2022 02:56:03 -0800 Subject: [PATCH 2/3] Update to version 0.3.0 because of API changes --- version.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.json b/version.json index d53fb2eeb..a654d65ab 100644 --- a/version.json +++ b/version.json @@ -1,3 +1,3 @@ { - "version": "v0.2.6" + "version": "v0.3.0" } From 67560ecff8ebe3bb81447c8a1f07f10eafef66ad Mon Sep 17 00:00:00 2001 From: gammazero Date: Sun, 30 Jan 2022 03:09:52 -0800 Subject: [PATCH 3/3] Remove debug print --- internal/ingest/ingest_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go index a1716d7c4..db2a02d9f 100644 --- a/internal/ingest/ingest_test.go +++ b/internal/ingest/ingest_test.go @@ -47,7 +47,7 @@ const ( testRetryTimeout = 10 * time.Second testEntriesChunkCount = 3 - testEntriesChunkSize = 10 + testEntriesChunkSize = 15 ) var ( @@ -249,7 +249,6 @@ func TestRmWithNoEntries(t *testing.T) { allMhs := typehelpers.AllMultihashesFromAd(t, prevAdNode.(schema.Advertisement), te.publisherLinkSys) // Remove the mhs from the first ad (since the last add removed this from the indexer) allMhs = allMhs[1:] - fmt.Println("!!!!", allMhs) checkMhsIndexedEventually(t, te.ingester.indexer, te.pubHost.ID(), allMhs) } func TestSync(t *testing.T) {