From de84da91b1d3c7698cee999f6ab4b3e9997e95e9 Mon Sep 17 00:00:00 2001 From: Siva Chandran Date: Sat, 5 Mar 2016 17:50:19 +0530 Subject: [PATCH 1/4] Support for listing only cached links in refs command Added "--cached" option to "refs" command that lists only cached links. Also added "linksize" token in format to print the size of the link. By listing only cached refs and their sizes we can easily find the cache/download progress of an object. License: MIT Signed-off-by: Sivachandran --- core/commands/refs.go | 101 ++++++++++++++++++++++++++++++------------ 1 file changed, 73 insertions(+), 28 deletions(-) diff --git a/core/commands/refs.go b/core/commands/refs.go index 4c637fd22dc..f8eec2962e2 100644 --- a/core/commands/refs.go +++ b/core/commands/refs.go @@ -5,8 +5,10 @@ import ( "errors" "fmt" "io" + "strconv" "strings" + blockstore "github.com/ipfs/go-ipfs/blocks/blockstore" key "github.com/ipfs/go-ipfs/blocks/key" cmds "github.com/ipfs/go-ipfs/commands" "github.com/ipfs/go-ipfs/core" @@ -49,10 +51,11 @@ Note: List all references recursively by using the flag '-r'. cmds.StringArg("ipfs-path", true, true, "Path to the object(s) to list refs from.").EnableStdin(), }, Options: []cmds.Option{ - cmds.StringOption("format", "Emit edges with given format. Available tokens: ."), + cmds.StringOption("format", "Emit edges with given format. Available tokens: "), cmds.BoolOption("edges", "e", "Emit edge format: ` -> `."), cmds.BoolOption("unique", "u", "Omit duplicate refs from output."), cmds.BoolOption("recursive", "r", "Recursively list links of child nodes."), + cmds.BoolOption("cached", "List only links of child nodes cached locally"), }, Run: func(req cmds.Request, res cmds.Response) { ctx := req.Context() @@ -86,6 +89,12 @@ Note: List all references recursively by using the flag '-r'. return } + cached, _, err := req.Option("cached").Bool() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + objs, err := objectsForPaths(ctx, n, req.Arguments()) if err != nil { res.SetError(err, cmds.ErrNormal) @@ -99,13 +108,15 @@ Note: List all references recursively by using the flag '-r'. defer close(out) rw := RefWriter{ - out: out, - DAG: n.DAG, - Ctx: ctx, - Unique: unique, - PrintEdge: edges, - PrintFmt: format, - Recursive: recursive, + out: out, + DAG: n.DAG, + Ctx: ctx, + Blockstore: n.Blockstore, + Unique: unique, + PrintEdge: edges, + PrintFmt: format, + Recursive: recursive, + Cached: cached, } for _, o := range objs { @@ -206,12 +217,14 @@ type RefWrapper struct { } type RefWriter struct { - out chan interface{} - DAG dag.DAGService - Ctx context.Context + out chan interface{} + DAG dag.DAGService + Ctx context.Context + Blockstore blockstore.Blockstore Unique bool Recursive bool + Cached bool PrintEdge bool PrintFmt string @@ -233,27 +246,57 @@ func (rw *RefWriter) writeRefsRecursive(n *dag.Node) (int, error) { } var count int - for i, ng := range dag.GetDAG(rw.Ctx, rw.DAG, n) { - lk := key.Key(n.Links[i].Hash) - if rw.skip(lk) { - continue - } + if rw.Cached { + for _, link := range n.Links { + lk := key.Key(link.Hash) + if rw.skip(lk) { + continue + } - if err := rw.WriteEdge(nkey, lk, n.Links[i].Name); err != nil { - return count, err - } + block, _ := rw.Blockstore.Get(lk) + cached := (block != nil) - nd, err := ng.Get(rw.Ctx) - if err != nil { - return count, err + if err := rw.WriteEdge(nkey, lk, link.Name, link.Size, cached); err != nil { + return count, err + } + + if cached { + nd, err := dag.Decoded(block.Data) + if err != nil { + return count, err + } + + c, err := rw.writeRefsRecursive(nd) + count += c + if err != nil { + return count, err + } + } } + } else { + for i, ng := range dag.GetDAG(rw.Ctx, rw.DAG, n) { + lk := key.Key(n.Links[i].Hash) + if rw.skip(lk) { + continue + } - c, err := rw.writeRefsRecursive(nd) - count += c - if err != nil { - return count, err + if err := rw.WriteEdge(nkey, lk, n.Links[i].Name, n.Links[i].Size, true); err != nil { + return count, err + } + + nd, err := ng.Get(rw.Ctx) + if err != nil { + return count, err + } + + c, err := rw.writeRefsRecursive(nd) + count += c + if err != nil { + return count, err + } } } + return count, nil } @@ -275,7 +318,7 @@ func (rw *RefWriter) writeRefsSingle(n *dag.Node) (int, error) { continue } - if err := rw.WriteEdge(nkey, lk, l.Name); err != nil { + if err := rw.WriteEdge(nkey, lk, l.Name, l.Size, true); err != nil { return count, err } count++ @@ -301,7 +344,7 @@ func (rw *RefWriter) skip(k key.Key) bool { } // Write one edge -func (rw *RefWriter) WriteEdge(from, to key.Key, linkname string) error { +func (rw *RefWriter) WriteEdge(from, to key.Key, linkname string, linksize uint64, cached bool) error { if rw.Ctx != nil { select { case <-rw.Ctx.Done(): // just in case. @@ -317,6 +360,8 @@ func (rw *RefWriter) WriteEdge(from, to key.Key, linkname string) error { s = strings.Replace(s, "", from.B58String(), -1) s = strings.Replace(s, "", to.B58String(), -1) s = strings.Replace(s, "", linkname, -1) + s = strings.Replace(s, "", strconv.FormatUint(linksize, 10), -1) + s = strings.Replace(s, "", strconv.FormatBool(cached), -1) case rw.PrintEdge: s = from.B58String() + " -> " + to.B58String() default: From affc15e2fbfd116e6b59cb8e75b27ba8e34e3a40 Mon Sep 17 00:00:00 2001 From: Siva Chandran Date: Tue, 8 Mar 2016 07:41:06 +0530 Subject: [PATCH 2/4] Added final period in usage instruction. License: MIT Signed-off-by: Sivachandran --- core/commands/refs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/commands/refs.go b/core/commands/refs.go index f8eec2962e2..51cfcd00926 100644 --- a/core/commands/refs.go +++ b/core/commands/refs.go @@ -55,7 +55,7 @@ Note: List all references recursively by using the flag '-r'. cmds.BoolOption("edges", "e", "Emit edge format: ` -> `."), cmds.BoolOption("unique", "u", "Omit duplicate refs from output."), cmds.BoolOption("recursive", "r", "Recursively list links of child nodes."), - cmds.BoolOption("cached", "List only links of child nodes cached locally"), + cmds.BoolOption("cached", "List only links of child nodes cached locally."), }, Run: func(req cmds.Request, res cmds.Response) { ctx := req.Context() From eee954931b93365b8befc5912ce705a3ecf74fba Mon Sep 17 00:00:00 2001 From: Siva Chandran Date: Fri, 11 Mar 2016 15:20:58 +0530 Subject: [PATCH 3/4] Now offline BlockService is used if listing only cached refs. License: MIT Signed-off-by: Sivachandran --- core/commands/refs.go | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/core/commands/refs.go b/core/commands/refs.go index 51cfcd00926..b6613181613 100644 --- a/core/commands/refs.go +++ b/core/commands/refs.go @@ -10,8 +10,10 @@ import ( blockstore "github.com/ipfs/go-ipfs/blocks/blockstore" key "github.com/ipfs/go-ipfs/blocks/key" + bserv "github.com/ipfs/go-ipfs/blockservice" cmds "github.com/ipfs/go-ipfs/commands" "github.com/ipfs/go-ipfs/core" + offline "github.com/ipfs/go-ipfs/exchange/offline" dag "github.com/ipfs/go-ipfs/merkledag" path "github.com/ipfs/go-ipfs/path" u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util" @@ -95,6 +97,11 @@ Note: List all references recursively by using the flag '-r'. return } + ds := n.DAG + if cached { + ds = dag.NewDAGService(bserv.New(n.Blockstore, offline.Exchange(n.Blockstore))) + } + objs, err := objectsForPaths(ctx, n, req.Arguments()) if err != nil { res.SetError(err, cmds.ErrNormal) @@ -108,15 +115,15 @@ Note: List all references recursively by using the flag '-r'. defer close(out) rw := RefWriter{ - out: out, - DAG: n.DAG, - Ctx: ctx, - Blockstore: n.Blockstore, - Unique: unique, - PrintEdge: edges, - PrintFmt: format, - Recursive: recursive, - Cached: cached, + out: out, + DAG: ds, + Ctx: ctx, + Bstore: n.Blockstore, + Unique: unique, + PrintEdge: edges, + PrintFmt: format, + Recursive: recursive, + Cached: cached, } for _, o := range objs { @@ -217,10 +224,10 @@ type RefWrapper struct { } type RefWriter struct { - out chan interface{} - DAG dag.DAGService - Ctx context.Context - Blockstore blockstore.Blockstore + out chan interface{} + DAG dag.DAGService + Ctx context.Context + Bstore blockstore.Blockstore Unique bool Recursive bool @@ -253,7 +260,7 @@ func (rw *RefWriter) writeRefsRecursive(n *dag.Node) (int, error) { continue } - block, _ := rw.Blockstore.Get(lk) + block, _ := rw.Bstore.Get(lk) cached := (block != nil) if err := rw.WriteEdge(nkey, lk, link.Name, link.Size, cached); err != nil { @@ -261,7 +268,7 @@ func (rw *RefWriter) writeRefsRecursive(n *dag.Node) (int, error) { } if cached { - nd, err := dag.Decoded(block.Data) + nd, err := dag.DecodeProtobuf(block.Data) if err != nil { return count, err } From 60bc015653cdc9b4f5dfda397032d248682b9a66 Mon Sep 17 00:00:00 2001 From: Siva Chandran Date: Fri, 15 Apr 2016 16:39:52 +0530 Subject: [PATCH 4/4] Uses DAG node get result to determine link cached status Link cached status was determined by checking the presence of link hash in blockstore. Now link cached status is determined by successfully getting node from DAG. License: MIT Signed-off-by: Sivachandran --- core/commands/refs.go | 74 ++++++++++++++----------------------------- 1 file changed, 24 insertions(+), 50 deletions(-) diff --git a/core/commands/refs.go b/core/commands/refs.go index b6613181613..b84eaae3ff7 100644 --- a/core/commands/refs.go +++ b/core/commands/refs.go @@ -8,7 +8,6 @@ import ( "strconv" "strings" - blockstore "github.com/ipfs/go-ipfs/blocks/blockstore" key "github.com/ipfs/go-ipfs/blocks/key" bserv "github.com/ipfs/go-ipfs/blockservice" cmds "github.com/ipfs/go-ipfs/commands" @@ -53,7 +52,7 @@ Note: List all references recursively by using the flag '-r'. cmds.StringArg("ipfs-path", true, true, "Path to the object(s) to list refs from.").EnableStdin(), }, Options: []cmds.Option{ - cmds.StringOption("format", "Emit edges with given format. Available tokens: "), + cmds.StringOption("format", "Emit edges with given format. Available tokens: "), cmds.BoolOption("edges", "e", "Emit edge format: ` -> `."), cmds.BoolOption("unique", "u", "Omit duplicate refs from output."), cmds.BoolOption("recursive", "r", "Recursively list links of child nodes."), @@ -118,7 +117,6 @@ Note: List all references recursively by using the flag '-r'. out: out, DAG: ds, Ctx: ctx, - Bstore: n.Blockstore, Unique: unique, PrintEdge: edges, PrintFmt: format, @@ -224,10 +222,9 @@ type RefWrapper struct { } type RefWriter struct { - out chan interface{} - DAG dag.DAGService - Ctx context.Context - Bstore blockstore.Blockstore + out chan interface{} + DAG dag.DAGService + Ctx context.Context Unique bool Recursive bool @@ -253,54 +250,31 @@ func (rw *RefWriter) writeRefsRecursive(n *dag.Node) (int, error) { } var count int - if rw.Cached { - for _, link := range n.Links { - lk := key.Key(link.Hash) - if rw.skip(lk) { - continue - } - - block, _ := rw.Bstore.Get(lk) - cached := (block != nil) - - if err := rw.WriteEdge(nkey, lk, link.Name, link.Size, cached); err != nil { - return count, err - } + for i, ng := range dag.GetDAG(rw.Ctx, rw.DAG, n) { + lk := key.Key(n.Links[i].Hash) + if rw.skip(lk) { + continue + } - if cached { - nd, err := dag.DecodeProtobuf(block.Data) - if err != nil { - return count, err - } + nd, err := ng.Get(rw.Ctx) - c, err := rw.writeRefsRecursive(nd) - count += c - if err != nil { - return count, err - } - } + linkCached := (nd != nil) + if werr := rw.WriteEdge(nkey, lk, n.Links[i].Name, n.Links[i].Size, linkCached); werr != nil { + return count, werr } - } else { - for i, ng := range dag.GetDAG(rw.Ctx, rw.DAG, n) { - lk := key.Key(n.Links[i].Hash) - if rw.skip(lk) { - continue - } - - if err := rw.WriteEdge(nkey, lk, n.Links[i].Name, n.Links[i].Size, true); err != nil { - return count, err - } - nd, err := ng.Get(rw.Ctx) - if err != nil { + if err != nil { + if rw.Cached { + continue + } else { return count, err } + } - c, err := rw.writeRefsRecursive(nd) - count += c - if err != nil { - return count, err - } + c, err := rw.writeRefsRecursive(nd) + count += c + if err != nil { + return count, err } } @@ -351,7 +325,7 @@ func (rw *RefWriter) skip(k key.Key) bool { } // Write one edge -func (rw *RefWriter) WriteEdge(from, to key.Key, linkname string, linksize uint64, cached bool) error { +func (rw *RefWriter) WriteEdge(from, to key.Key, linkname string, linksize uint64, linkcached bool) error { if rw.Ctx != nil { select { case <-rw.Ctx.Done(): // just in case. @@ -368,7 +342,7 @@ func (rw *RefWriter) WriteEdge(from, to key.Key, linkname string, linksize uint6 s = strings.Replace(s, "", to.B58String(), -1) s = strings.Replace(s, "", linkname, -1) s = strings.Replace(s, "", strconv.FormatUint(linksize, 10), -1) - s = strings.Replace(s, "", strconv.FormatBool(cached), -1) + s = strings.Replace(s, "", strconv.FormatBool(linkcached), -1) case rw.PrintEdge: s = from.B58String() + " -> " + to.B58String() default: