Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update libipni and use dagsync general block hook #2477

Merged
merged 1 commit into from
Jan 10, 2024
Merged

Update libipni and use dagsync general block hook #2477

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,11 @@ require (
github.com/ipld/go-ipld-prime v0.21.0
github.com/ipld/go-ipld-prime/storage/dsadapter v0.0.0-20230102063945-1a409dc236dd
github.com/ipni/go-indexer-core v0.8.7
github.com/ipni/go-libipni v0.5.8
github.com/ipni/go-libipni v0.5.9
github.com/libp2p/go-libp2p v0.32.2
github.com/libp2p/go-msgio v0.3.0
github.com/mitchellh/go-homedir v1.1.0
github.com/multiformats/go-multiaddr v0.12.0
github.com/multiformats/go-multiaddr v0.12.1
github.com/multiformats/go-multicodec v0.9.0
github.com/multiformats/go-multihash v0.2.3
github.com/multiformats/go-varint v0.0.7
Expand Down
8 changes: 4 additions & 4 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -593,8 +593,8 @@ github.com/ipld/go-ipld-prime/storage/dsadapter v0.0.0-20230102063945-1a409dc236
github.com/ipld/go-ipld-prime/storage/dsadapter v0.0.0-20230102063945-1a409dc236dd/go.mod h1:9DD/GM0JNPoisgR09F62kbBi7kHa4eDIea4XshXYOVc=
github.com/ipni/go-indexer-core v0.8.7 h1:IaEBoVe1RiTBDTj8MiUlbsYk/L32AOjtCBhQQmXuxIo=
github.com/ipni/go-indexer-core v0.8.7/go.mod h1:lLWTrQ7dhKwCak1qn6AQBNoSrGYBAXJJy2OGc02BO2Q=
github.com/ipni/go-libipni v0.5.8 h1:iSob6WRjdRuKpDZmwYw5aBD13utosnEGlRRun/21ZCs=
github.com/ipni/go-libipni v0.5.8/go.mod h1:QvWhyJAY6CZ6IoQNw0DvZQW2nDJWxt/diwxv8udoAIw=
github.com/ipni/go-libipni v0.5.9 h1:AlYlqZScX2jusGXXWkW5j6OMUtMKgQKNcl1Mi8g3glA=
github.com/ipni/go-libipni v0.5.9/go.mod h1:c8mHa6J9iFREpDB29GlPIsbvztRq6bnhg5zJKrnvdUg=
github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI=
github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0=
github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI=
Expand Down Expand Up @@ -958,8 +958,8 @@ github.com/multiformats/go-multiaddr v0.2.1/go.mod h1:s/Apk6IyxfvMjDafnhJgJ3/46z
github.com/multiformats/go-multiaddr v0.2.2/go.mod h1:NtfXiOtHvghW9KojvtySjH5y0u0xW5UouOmQQrn6a3Y=
github.com/multiformats/go-multiaddr v0.3.0/go.mod h1:dF9kph9wfJ+3VLAaeBqo9Of8x4fJxp6ggJGteB8HQTI=
github.com/multiformats/go-multiaddr v0.3.1/go.mod h1:uPbspcUPd5AfaP6ql3ujFY+QWzmBD8uLLL4bXW0XfGc=
github.com/multiformats/go-multiaddr v0.12.0 h1:1QlibTFkoXJuDjjYsMHhE73TnzJQl8FSWatk/0gxGzE=
github.com/multiformats/go-multiaddr v0.12.0/go.mod h1:WmZXgObOQOYp9r3cslLlppkrz1FYSHmE834dfz/lWu8=
github.com/multiformats/go-multiaddr v0.12.1 h1:vm+BA/WZA8QZDp1pF1FWhi5CT3g1tbi5GJmqpb6wnlk=
github.com/multiformats/go-multiaddr v0.12.1/go.mod h1:7mPkiBMmLeFipt+nNSq9pHZUeJSt8lHBgH6yhj0YQzE=
github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q=
github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q=
github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0=
Expand Down
14 changes: 9 additions & 5 deletions internal/ingest/linksystem.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -489,7 +489,7 @@ func (ing *Ingester) ingestEntriesFromPublisher(ctx context.Context, ad schema.A
return 0, adIngestError{adIngestEntryChunkErr, fmt.Errorf("failed to load first entry chunk: %w", err)}
}

err = ing.ingestEntryChunk(ctx, ad, providerID, entsCid, *chunk, log)
err = ing.ingestEntryChunk(ctx, ad, providerID, entsCid, chunk, log)
if err != nil {
// There was an error storing the multihashes.
return 0, adIngestError{adIngestIndexerErr, fmt.Errorf("failed to ingest first entry chunk: %w", err)}
Expand All @@ -506,7 +506,7 @@ func (ing *Ingester) ingestEntriesFromPublisher(ctx context.Context, ad schema.A
actions.FailSync(adIngestError{adIngestIndexerErr, fmt.Errorf("failed to load entry chunk: %w", err)})
return
}
err = ing.ingestEntryChunk(ctx, ad, providerID, c, *chunk, log)
err = ing.ingestEntryChunk(ctx, ad, providerID, c, chunk, log)
if err != nil {
actions.FailSync(adIngestError{adIngestIndexerErr, fmt.Errorf("failed to ingest entry chunk: %w", err)})
return
Expand Down Expand Up @@ -688,12 +688,16 @@ func (ing *Ingester) loadAd(c cid.Cid) (schema.Advertisement, error) {
return *ad, nil
}

func (ing *Ingester) loadEntryChunk(c cid.Cid) (*schema.EntryChunk, error) {
func (ing *Ingester) loadEntryChunk(c cid.Cid) (schema.EntryChunk, error) {
node, err := ing.loadNode(c, schema.EntryChunkPrototype)
if err != nil {
return nil, err
return schema.EntryChunk{}, err
}
entChunk, err := schema.UnwrapEntryChunk(node)
if err != nil {
return schema.EntryChunk{}, fmt.Errorf("cannot decode entry chunk: %w", err)
}
return schema.UnwrapEntryChunk(node)
return *entChunk, nil
}

func (ing *Ingester) loadHamt(c cid.Cid) (*hamt.Node, error) {
Expand Down
60 changes: 9 additions & 51 deletions ipni-gc/reaper/reaper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import (
logging "github.com/ipfs/go-log/v2"
"github.com/ipld/go-ipld-prime"
cidlink "github.com/ipld/go-ipld-prime/linking/cid"
"github.com/ipld/go-ipld-prime/multicodec"
indexer "github.com/ipni/go-indexer-core"
"github.com/ipni/go-libipni/dagsync"
"github.com/ipni/go-libipni/ingest/schema"
Expand Down Expand Up @@ -622,27 +621,12 @@ func (r *Reaper) makeSubscriber(dstoreTmp datastore.Batching) (*dagsync.Subscrib
dagsync.SegmentDepthLimit(int64(r.syncSegSize)))
}

func (s *scythe) generalDagsyncBlockHook(_ peer.ID, adCid cid.Cid, actions dagsync.SegmentSyncActions) {
// The only kind of block we should get by loading CIDs here should be
// Advertisement.
//
// Because:
// - the default subscription selector only selects advertisements.
// - explicit Ingester.Sync only selects advertisement.
// - entries are synced with an explicit selector separate from
// advertisement syncs and should use dagsync.ScopedBlockHook to
// override this hook and decode chunks instead.
//
// Therefore, we only attempt to load advertisements here and signal
// failure if the load fails.
func (s *scythe) prevAdCid(adCid cid.Cid) (cid.Cid, error) {
ad, err := s.loadAd(adCid)
if err != nil {
actions.FailSync(err)
} else if ad.PreviousID != nil {
actions.SetNextSyncCid(ad.PreviousID.(cidlink.Link).Cid)
} else {
actions.SetNextSyncCid(cid.Undef)
return cid.Undef, err
}
return ad.PreviousCid(), nil
}

func (s *scythe) reap(ctx context.Context, latestAdCid cid.Cid) error {
Expand All @@ -666,7 +650,7 @@ func (s *scythe) reap(ctx context.Context, latestAdCid cid.Cid) error {
_, err = s.sub.SyncAdChain(ctx, s.publisher,
dagsync.WithHeadAdCid(latestAdCid),
dagsync.WithStopAdCid(gcState.LastProcessedAdCid),
dagsync.ScopedBlockHook(s.generalDagsyncBlockHook),
dagsync.ScopedBlockHook(dagsync.MakeGeneralBlockHook(s.prevAdCid)),
)
if err != nil {
return fmt.Errorf("failed to sync advertisement chain: %w", err)
Expand Down Expand Up @@ -1200,43 +1184,17 @@ func (s *scythe) archiveDatastore(ctx context.Context) error {
}

func (s *scythe) loadAd(c cid.Cid) (schema.Advertisement, error) {
adn, err := s.loadNode(c, schema.AdvertisementPrototype)
val, err := s.dstoreTmp.Get(context.Background(), datastore.NewKey(c.String()))
if err != nil {
return schema.Advertisement{}, err
}
ad, err := schema.UnwrapAdvertisement(adn)
if err != nil {
return schema.Advertisement{}, fmt.Errorf("cannot decode advertisement: %w", err)
}

return *ad, nil
}

func (s *scythe) loadEntryChunk(c cid.Cid) (*schema.EntryChunk, error) {
node, err := s.loadNode(c, schema.EntryChunkPrototype)
if err != nil {
return nil, err
}
return schema.UnwrapEntryChunk(node)
return schema.BytesToAdvertisement(c, val)
}

func (s *scythe) loadNode(c cid.Cid, prototype ipld.NodePrototype) (ipld.Node, error) {
func (s *scythe) loadEntryChunk(c cid.Cid) (schema.EntryChunk, error) {
val, err := s.dstoreTmp.Get(context.Background(), datastore.NewKey(c.String()))
if err != nil {
return nil, err
}
return decodeIPLDNode(c.Prefix().Codec, bytes.NewBuffer(val), prototype)
}

// decodeIPLDNode decodes an ipld.Node from bytes read from an io.Reader.
func decodeIPLDNode(codec uint64, r io.Reader, prototype ipld.NodePrototype) (ipld.Node, error) {
nb := prototype.NewBuilder()
decoder, err := multicodec.LookupDecoder(codec)
if err != nil {
return nil, err
}
if err = decoder(nb, r); err != nil {
return nil, err
return schema.EntryChunk{}, err
}
return nb.Build(), nil
return schema.BytesToEntryChunk(c, val)
}