generated from ipfs/ipfs-repository-template
-
Notifications
You must be signed in to change notification settings - Fork 97
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
pinner: change the interface to have async pin listing
The rational is that if the pin list get big, a synchronous call to get the complete list can delay handling unnecessarily. For example, when listing indirect pins, you can start walking the DAGs immediately with the first recursive pin instead of waiting for the full list. This matters even more on low power device, of if the pin list is stored remotely. * coreiface: allow to return an error not linked to a specific Cid * merkledag/test: add a DAG generator Rationale is that generating a test DAG is quite difficult, and anything that helps writing better tests is helpful.
- Loading branch information
1 parent
4c5c98b
commit e2fc7f2
Showing
9 changed files
with
312 additions
and
97 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,86 @@ | ||
| package mdutils | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
|
|
||
| blocks "github.com/ipfs/go-block-format" | ||
| "github.com/ipfs/go-cid" | ||
| format "github.com/ipfs/go-ipld-format" | ||
|
|
||
| "github.com/ipfs/boxo/ipld/merkledag" | ||
| ) | ||
|
|
||
| // NewDAGGenerator returns an object capable of | ||
| // producing IPLD DAGs. | ||
| func NewDAGGenerator() *DAGGenerator { | ||
| return &DAGGenerator{} | ||
| } | ||
|
|
||
| // DAGGenerator generates BasicBlocks on demand. | ||
| // For each instance of DAGGenerator, each new DAG is different from the | ||
| // previous, although two different instances will produce the same, given the | ||
| // same parameters. | ||
| type DAGGenerator struct { | ||
| seq int | ||
| } | ||
|
|
||
| // MakeDagBlock generate a balanced DAG with the given fanout and depth, and add the blocks to the adder. | ||
| // This adder can be for example a blockstore.Put or a blockservice.AddBlock. | ||
| func (dg *DAGGenerator) MakeDagBlock(adder func(ctx context.Context, block blocks.Block) error, fanout uint, depth uint) (c cid.Cid, allCids []cid.Cid, err error) { | ||
| return dg.MakeDagNode(func(ctx context.Context, node format.Node) error { | ||
| return adder(ctx, node.(blocks.Block)) | ||
| }, fanout, depth) | ||
| } | ||
|
|
||
| // MakeDagNode generate a balanced DAG with the given fanout and depth, and add the blocks to the adder. | ||
| // This adder can be for example a DAGService.Add. | ||
| func (dg *DAGGenerator) MakeDagNode(adder func(ctx context.Context, node format.Node) error, fanout uint, depth uint) (c cid.Cid, allCids []cid.Cid, err error) { | ||
| c, _, allCids, err = dg.generate(adder, fanout, depth) | ||
| return c, allCids, err | ||
| } | ||
|
|
||
| func (dg *DAGGenerator) generate(adder func(ctx context.Context, node format.Node) error, fanout uint, depth uint) (c cid.Cid, size uint64, allCids []cid.Cid, err error) { | ||
| if depth == 0 { | ||
| panic("depth should be at least 1") | ||
| } | ||
| if depth == 1 { | ||
| c, size, err = dg.encodeBlock(adder) | ||
| if err != nil { | ||
| return cid.Undef, 0, nil, err | ||
| } | ||
| return c, size, []cid.Cid{c}, nil | ||
| } | ||
| links := make([]*format.Link, fanout) | ||
| for i := uint(0); i < fanout; i++ { | ||
| root, size, children, err := dg.generate(adder, fanout, depth-1) | ||
| if err != nil { | ||
| return cid.Undef, 0, nil, err | ||
| } | ||
| links[i] = &format.Link{Cid: root, Size: size} | ||
| allCids = append(allCids, children...) | ||
| } | ||
| c, size, err = dg.encodeBlock(adder, links...) | ||
| if err != nil { | ||
| return cid.Undef, 0, nil, err | ||
| } | ||
| return c, size, append([]cid.Cid{c}, allCids...), nil | ||
| } | ||
|
|
||
| func (dg *DAGGenerator) encodeBlock(adder func(ctx context.Context, node format.Node) error, links ...*format.Link) (cid.Cid, uint64, error) { | ||
| dg.seq++ | ||
| nd := &merkledag.ProtoNode{} | ||
| nd.SetData([]byte(fmt.Sprint(dg.seq))) | ||
| for i, link := range links { | ||
| err := nd.AddRawLink(fmt.Sprintf("link-%d", i), link) | ||
| if err != nil { | ||
| return cid.Undef, 0, err | ||
| } | ||
| } | ||
| err := adder(context.Background(), nd) | ||
| if err != nil { | ||
| return cid.Undef, 0, err | ||
| } | ||
| size, err := nd.Size() | ||
| return nd.Cid(), size, err | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,92 @@ | ||
| package mdutils | ||
|
|
||
| import ( | ||
| "context" | ||
| "sync" | ||
| "testing" | ||
|
|
||
| "github.com/ipfs/go-cid" | ||
| format "github.com/ipfs/go-ipld-format" | ||
| ) | ||
|
|
||
| type testDagServ struct { | ||
| mu sync.Mutex | ||
| nodes map[string]format.Node | ||
| } | ||
|
|
||
| func newTestDagServ() *testDagServ { | ||
| return &testDagServ{nodes: make(map[string]format.Node)} | ||
| } | ||
|
|
||
| func (d *testDagServ) Get(_ context.Context, cid cid.Cid) (format.Node, error) { | ||
| d.mu.Lock() | ||
| defer d.mu.Unlock() | ||
| if n, ok := d.nodes[cid.KeyString()]; ok { | ||
| return n, nil | ||
| } | ||
| return nil, format.ErrNotFound{Cid: cid} | ||
| } | ||
|
|
||
| func (d *testDagServ) Add(_ context.Context, node format.Node) error { | ||
| d.mu.Lock() | ||
| defer d.mu.Unlock() | ||
| d.nodes[node.Cid().KeyString()] = node | ||
| return nil | ||
| } | ||
|
|
||
| func TestNodesAreDifferent(t *testing.T) { | ||
| dserv := newTestDagServ() | ||
| gen := NewDAGGenerator() | ||
|
|
||
| var allCids []cid.Cid | ||
| var allNodes []format.Node | ||
|
|
||
| const nbDag = 5 | ||
|
|
||
| for i := 0; i < nbDag; i++ { | ||
| c, cids, err := gen.MakeDagNode(dserv.Add, 5, 3) | ||
| if err != nil { | ||
| t.Fatal(err) | ||
| } | ||
|
|
||
| allCids = append(allCids, cids...) | ||
|
|
||
| // collect all nodes | ||
| var getChildren func(n format.Node) | ||
| getChildren = func(n format.Node) { | ||
| for _, link := range n.Links() { | ||
| n, err = dserv.Get(context.Background(), link.Cid) | ||
| if err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| allNodes = append(allNodes, n) | ||
| getChildren(n) | ||
| } | ||
| } | ||
| n, err := dserv.Get(context.Background(), c) | ||
| if err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| allNodes = append(allNodes, n) | ||
| getChildren(n) | ||
|
|
||
| // make sure they are all different | ||
| for i, node1 := range allNodes { | ||
| for j, node2 := range allNodes { | ||
| if i != j { | ||
| if node1.Cid().String() == node2.Cid().String() { | ||
| t.Error("Found duplicate node") | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // expected count | ||
| if len(allNodes) != nbDag*31 { | ||
| t.Error("expected nbDag*31 nodes (1+5+5*5)") | ||
| } | ||
| if len(allCids) != nbDag*31 { | ||
| t.Error("expected nbDag*31 cids (1+5+5*5)") | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.