From ef6e9cf2839b25bd13f2bcf2aff9530ee4e0161b Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Thu, 18 Oct 2018 10:49:22 +0100 Subject: [PATCH 1/7] feat(commands): --stream option for ls Convert LS Command to use current cmds lib Update LS Command to support streaming Rebase fixes License: MIT Signed-off-by: hannahhoward --- core/commands/ls.go | 287 ++++++++++++++++++++++++++---------------- core/commands/root.go | 5 +- 2 files changed, 184 insertions(+), 108 deletions(-) diff --git a/core/commands/ls.go b/core/commands/ls.go index 9ee64e400af..42eabc4ad6a 100644 --- a/core/commands/ls.go +++ b/core/commands/ls.go @@ -1,12 +1,11 @@ package commands import ( - "bytes" "fmt" "io" "text/tabwriter" - cmds "github.com/ipfs/go-ipfs/commands" + cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv" e "github.com/ipfs/go-ipfs/core/commands/e" iface "github.com/ipfs/go-ipfs/core/coreapi/interface" @@ -16,29 +15,40 @@ import ( unixfspb "gx/ipfs/QmUnHNqhSB1JgzVCxL1Kz3yb4bdyB4q1Z9AD5AUBVmt3fZ/go-unixfs/pb" blockservice "gx/ipfs/QmVDTbzzTwnuBwNbJdhW3u7LoBQp46bezm9yp4z1RoEepM/go-blockservice" offline "gx/ipfs/QmYZwey1thDTynSrvd6qQkX24UpTka6TFhQ2v569UpoqxD/go-ipfs-exchange-offline" + cmds "gx/ipfs/Qma6uuSyjkecGhMFFLfzyJDPyoDtNJSHJNweDccZhaWkgU/go-ipfs-cmds" merkledag "gx/ipfs/QmcGt25mrjuB2kKW2zhPbXVZNHc4yoTDQ65NA8m6auP2f1/go-merkledag" ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format" "gx/ipfs/Qmde5VP1qUkyQXKCfmEUA7bP64V2HAptbJ7phuPp7jXWwg/go-ipfs-cmdkit" ) +// LsLink contains printable data for a single ipld link in ls output type LsLink struct { Name, Hash string Size uint64 Type unixfspb.Data_DataType } +// LsObject is an element of LsOutput +// It can represent a whole directory, a directory header, one or more links, +// Or a the end of a directory type LsObject struct { - Hash string - Links []LsLink + Hash string + Links []LsLink + HasHeader bool + HasLinks bool + HasFooter bool } +// LsOutput is a set of printable data for directories type LsOutput struct { - Objects []LsObject + MultipleFolders bool + Objects []LsObject } const ( lsHeadersOptionNameTime = "headers" lsResolveTypeOptionName = "resolve-type" + lsStreamOptionName = "stream" ) var LsCmd = &cmds.Command{ @@ -60,32 +70,20 @@ The JSON output contains type information. Options: []cmdkit.Option{ cmdkit.BoolOption(lsHeadersOptionNameTime, "v", "Print table headers (Hash, Size, Name)."), cmdkit.BoolOption(lsResolveTypeOptionName, "Resolve linked objects to find out their types.").WithDefault(true), + cmdkit.BoolOption(lsStreamOptionName, "s", "Stream directory entries as they are found."), }, - Run: func(req cmds.Request, res cmds.Response) { - nd, err := req.InvocContext().GetNode() + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + nd, err := cmdenv.GetNode(env) if err != nil { - res.SetError(err, cmdkit.ErrNormal) - return + return err } - api, err := req.InvocContext().GetApi() + api, err := cmdenv.GetApi(env) if err != nil { - res.SetError(err, cmdkit.ErrNormal) - return - } - - // get options early -> exit early in case of error - if _, _, err := req.Option(lsHeadersOptionNameTime).Bool(); err != nil { - res.SetError(err, cmdkit.ErrNormal) - return - } - - resolve, _, err := req.Option(lsResolveTypeOptionName).Bool() - if err != nil { - res.SetError(err, cmdkit.ErrNormal) - return + return err } + resolve, _ := req.Options[lsResolveTypeOptionName].(bool) dserv := nd.DAG if !resolve { offlineexch := offline.Exchange(nd.Blockstore) @@ -93,125 +91,204 @@ The JSON output contains type information. dserv = merkledag.NewDAGService(bserv) } - paths := req.Arguments() + err = req.ParseBodyArgs() + if err != nil { + return err + } + + paths := req.Arguments var dagnodes []ipld.Node for _, fpath := range paths { p, err := iface.ParsePath(fpath) if err != nil { - res.SetError(err, cmdkit.ErrNormal) - return + return err } - dagnode, err := api.ResolveNode(req.Context(), p) + dagnode, err := api.ResolveNode(req.Context, p) if err != nil { - res.SetError(err, cmdkit.ErrNormal) - return + return err } dagnodes = append(dagnodes, dagnode) } - - output := make([]LsObject, len(req.Arguments())) - ng := merkledag.NewSession(req.Context(), nd.DAG) + ng := merkledag.NewSession(req.Context, nd.DAG) ro := merkledag.NewReadOnlyDagService(ng) + stream, _ := req.Options[lsStreamOptionName].(bool) + multipleFolders := len(req.Arguments) > 1 + if !stream { + output := make([]LsObject, len(req.Arguments)) + + for i, dagnode := range dagnodes { + dir, err := uio.NewDirectoryFromNode(ro, dagnode) + if err != nil && err != uio.ErrNotADir { + return fmt.Errorf("the data in %s (at %q) is not a UnixFS directory: %s", dagnode.Cid(), paths[i], err) + } + + var links []*ipld.Link + if dir == nil { + links = dagnode.Links() + } else { + links, err = dir.Links(req.Context) + if err != nil { + return err + } + } + outputLinks := make([]LsLink, len(links)) + for j, link := range links { + lsLink, err := makeLsLink(req, dserv, resolve, link) + if err != nil { + return err + } + outputLinks[j] = *lsLink + } + output[i] = newFullDirectoryLsObject(paths[i], outputLinks) + } + + return cmds.EmitOnce(res, &LsOutput{multipleFolders, output}) + } + for i, dagnode := range dagnodes { dir, err := uio.NewDirectoryFromNode(ro, dagnode) if err != nil && err != uio.ErrNotADir { - res.SetError(fmt.Errorf("the data in %s (at %q) is not a UnixFS directory: %s", dagnode.Cid(), paths[i], err), cmdkit.ErrNormal) - return + return fmt.Errorf("the data in %s (at %q) is not a UnixFS directory: %s", dagnode.Cid(), paths[i], err) } - var links []*ipld.Link + var linkResults <-chan unixfs.LinkResult if dir == nil { - links = dagnode.Links() + linkResults = makeDagNodeLinkResults(req, dagnode) } else { - links, err = dir.Links(req.Context()) - if err != nil { - res.SetError(err, cmdkit.ErrNormal) - return - } + linkResults = dir.EnumLinksAsync(req.Context) } - output[i] = LsObject{ - Hash: paths[i], - Links: make([]LsLink, len(links)), - } - - for j, link := range links { - t := unixfspb.Data_DataType(-1) - - switch link.Cid.Type() { - case cid.Raw: - // No need to check with raw leaves - t = unixfs.TFile - case cid.DagProtobuf: - linkNode, err := link.GetNode(req.Context(), dserv) - if err == ipld.ErrNotFound && !resolve { - // not an error - linkNode = nil - } else if err != nil { - res.SetError(err, cmdkit.ErrNormal) - return - } + output := make([]LsObject, 1) + outputLinks := make([]LsLink, 1) - if pn, ok := linkNode.(*merkledag.ProtoNode); ok { - d, err := unixfs.FSNodeFromBytes(pn.Data()) - if err != nil { - res.SetError(err, cmdkit.ErrNormal) - return - } - t = d.Type() - } + output[0] = newDirectoryHeaderLsObject(paths[i]) + if err = res.Emit(&LsOutput{multipleFolders, output}); err != nil { + return nil + } + for linkResult := range linkResults { + if linkResult.Err != nil { + return linkResult.Err + } + link := linkResult.Link + lsLink, err := makeLsLink(req, dserv, resolve, link) + if err != nil { + return err } - output[i].Links[j] = LsLink{ - Name: link.Name, - Hash: link.Cid.String(), - Size: link.Size, - Type: t, + outputLinks[0] = *lsLink + output[0] = newDirectoryLinksLsObject(outputLinks) + if err = res.Emit(&LsOutput{multipleFolders, output}); err != nil { + return err } } + output[0] = newDirectoryFooterLsObject() + if err = res.Emit(&LsOutput{multipleFolders, output}); err != nil { + return err + } } - - res.SetOutput(&LsOutput{output}) + return nil }, - Marshalers: cmds.MarshalerMap{ - cmds.Text: func(res cmds.Response) (io.Reader, error) { - - v, err := unwrapOutput(res.Output()) - if err != nil { - return nil, err - } - - headers, _, _ := res.Request().Option(lsHeadersOptionNameTime).Bool() + Encoders: cmds.EncoderMap{ + cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, v interface{}) error { + headers, _ := req.Options[lsHeadersOptionNameTime].(bool) output, ok := v.(*LsOutput) if !ok { - return nil, e.TypeErr(output, v) + return e.TypeErr(output, v) } - buf := new(bytes.Buffer) - w := tabwriter.NewWriter(buf, 1, 2, 1, ' ', 0) + tw := tabwriter.NewWriter(w, 1, 2, 1, ' ', 0) for _, object := range output.Objects { - if len(output.Objects) > 1 { - fmt.Fprintf(w, "%s:\n", object.Hash) - } - if headers { - fmt.Fprintln(w, "Hash\tSize\tName") + if object.HasHeader { + if output.MultipleFolders { + fmt.Fprintf(tw, "%s:\n", object.Hash) + } + if headers { + fmt.Fprintln(tw, "Hash\tSize\tName") + } } - for _, link := range object.Links { - if link.Type == unixfs.TDirectory { - link.Name += "/" + if object.HasLinks { + for _, link := range object.Links { + if link.Type == unixfs.TDirectory { + link.Name += "/" + } + + fmt.Fprintf(tw, "%s\t%v\t%s\n", link.Hash, link.Size, link.Name) } - fmt.Fprintf(w, "%s\t%v\t%s\n", link.Hash, link.Size, link.Name) } - if len(output.Objects) > 1 { - fmt.Fprintln(w) + if object.HasFooter { + if output.MultipleFolders { + fmt.Fprintln(tw) + } } } - w.Flush() - - return buf, nil - }, + tw.Flush() + return nil + }), }, Type: LsOutput{}, } + +func makeDagNodeLinkResults(req *cmds.Request, dagnode ipld.Node) <-chan unixfs.LinkResult { + linkResults := make(chan unixfs.LinkResult) + go func() { + defer close(linkResults) + for _, l := range dagnode.Links() { + select { + case linkResults <- unixfs.LinkResult{ + Link: l, + Err: nil, + }: + case <-req.Context.Done(): + return + } + } + }() + return linkResults +} + +func newFullDirectoryLsObject(hash string, links []LsLink) LsObject { + return LsObject{hash, links, true, true, true} +} +func newDirectoryHeaderLsObject(hash string) LsObject { + return LsObject{hash, nil, true, false, false} +} +func newDirectoryLinksLsObject(links []LsLink) LsObject { + return LsObject{"", links, false, true, false} +} +func newDirectoryFooterLsObject() LsObject { + return LsObject{"", nil, false, false, true} +} + +func makeLsLink(req *cmds.Request, dserv ipld.DAGService, resolve bool, link *ipld.Link) (*LsLink, error) { + t := unixfspb.Data_DataType(-1) + + switch link.Cid.Type() { + case cid.Raw: + // No need to check with raw leaves + t = unixfs.TFile + case cid.DagProtobuf: + linkNode, err := link.GetNode(req.Context, dserv) + if err == ipld.ErrNotFound && !resolve { + // not an error + linkNode = nil + } else if err != nil { + return nil, err + } + + if pn, ok := linkNode.(*merkledag.ProtoNode); ok { + d, err := unixfs.FSNodeFromBytes(pn.Data()) + if err != nil { + return nil, err + } + t = d.Type() + } + } + return &LsLink{ + Name: link.Name, + Hash: link.Cid.String(), + Size: link.Size, + Type: t, + }, nil +} diff --git a/core/commands/root.go b/core/commands/root.go index 7a28525115f..1ce7721e904 100644 --- a/core/commands/root.go +++ b/core/commands/root.go @@ -3,7 +3,6 @@ package commands import ( "errors" - lgc "github.com/ipfs/go-ipfs/commands/legacy" dag "github.com/ipfs/go-ipfs/core/commands/dag" name "github.com/ipfs/go-ipfs/core/commands/name" ocmd "github.com/ipfs/go-ipfs/core/commands/object" @@ -127,7 +126,7 @@ var rootSubcommands = map[string]*cmds.Command{ "id": IDCmd, "key": KeyCmd, "log": LogCmd, - "ls": lgc.NewCommand(LsCmd), + "ls": LsCmd, "mount": MountCmd, "name": name.NameCmd, "object": ocmd.ObjectCmd, @@ -165,7 +164,7 @@ var rootROSubcommands = map[string]*cmds.Command{ }, "get": GetCmd, "dns": DNSCmd, - "ls": lgc.NewCommand(LsCmd), + "ls": LsCmd, "name": { Subcommands: map[string]*cmds.Command{ "resolve": name.IpnsCmd, From 5672ae6bbb32382b65b6e4a1ee9c49a7659c3646 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Thu, 8 Nov 2018 14:07:42 -0800 Subject: [PATCH 2/7] fix(commands): Refactor to use a PostRun Removes flags on LSOutput License: MIT Signed-off-by: hannahhoward --- core/commands/ls.go | 128 +++++++++++++++++++++++--------------------- 1 file changed, 66 insertions(+), 62 deletions(-) diff --git a/core/commands/ls.go b/core/commands/ls.go index 42eabc4ad6a..7308d25c357 100644 --- a/core/commands/ls.go +++ b/core/commands/ls.go @@ -3,6 +3,7 @@ package commands import ( "fmt" "io" + "os" "text/tabwriter" cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv" @@ -29,20 +30,16 @@ type LsLink struct { } // LsObject is an element of LsOutput -// It can represent a whole directory, a directory header, one or more links, -// Or a the end of a directory +// It can represent all or part of a directory type LsObject struct { - Hash string - Links []LsLink - HasHeader bool - HasLinks bool - HasFooter bool + Hash string + Links []LsLink } -// LsOutput is a set of printable data for directories +// LsOutput is a set of printable data for directories, +// it can be complete or partial type LsOutput struct { - MultipleFolders bool - Objects []LsObject + Objects []LsObject } const ( @@ -114,10 +111,10 @@ The JSON output contains type information. ng := merkledag.NewSession(req.Context, nd.DAG) ro := merkledag.NewReadOnlyDagService(ng) + output := make([]LsObject, len(req.Arguments)) + stream, _ := req.Options[lsStreamOptionName].(bool) - multipleFolders := len(req.Arguments) > 1 if !stream { - output := make([]LsObject, len(req.Arguments)) for i, dagnode := range dagnodes { dir, err := uio.NewDirectoryFromNode(ro, dagnode) @@ -142,10 +139,21 @@ The JSON output contains type information. } outputLinks[j] = *lsLink } - output[i] = newFullDirectoryLsObject(paths[i], outputLinks) + output[i] = LsObject{ + Hash: paths[i], + Links: outputLinks, + } } - return cmds.EmitOnce(res, &LsOutput{multipleFolders, output}) + return cmds.EmitOnce(res, &LsOutput{output}) + } + + outputLinks := make([]LsLink, 1) + for i, path := range paths { + output[i] = LsObject{ + Hash: path, + Links: nil, + } } for i, dagnode := range dagnodes { @@ -161,13 +169,8 @@ The JSON output contains type information. linkResults = dir.EnumLinksAsync(req.Context) } - output := make([]LsObject, 1) - outputLinks := make([]LsLink, 1) + output[i].Links = outputLinks - output[0] = newDirectoryHeaderLsObject(paths[i]) - if err = res.Emit(&LsOutput{multipleFolders, output}); err != nil { - return nil - } for linkResult := range linkResults { if linkResult.Err != nil { return linkResult.Err @@ -178,37 +181,57 @@ The JSON output contains type information. return err } outputLinks[0] = *lsLink - output[0] = newDirectoryLinksLsObject(outputLinks) - if err = res.Emit(&LsOutput{multipleFolders, output}); err != nil { + if err = res.Emit(&LsOutput{output}); err != nil { return err } } - output[0] = newDirectoryFooterLsObject() - if err = res.Emit(&LsOutput{multipleFolders, output}); err != nil { - return err - } + output[i].Links = nil } return nil }, - Encoders: cmds.EncoderMap{ - cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, v interface{}) error { + PostRun: cmds.PostRunMap{ + cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error { + req := res.Request() headers, _ := req.Options[lsHeadersOptionNameTime].(bool) - output, ok := v.(*LsOutput) - if !ok { - return e.TypeErr(output, v) - } + multipleFolders := len(req.Arguments) > 1 + lastDirectoryWritten := -1 + tw := tabwriter.NewWriter(os.Stdout, 1, 2, 1, ' ', 0) + for { + v, err := res.Next() + if err != nil { + if err == io.EOF { + if multipleFolders { + fmt.Fprintln(os.Stdout) + } + return nil + } + + return err + } + + output, ok := v.(*LsOutput) + if !ok { + return e.TypeErr(output, v) + } - tw := tabwriter.NewWriter(w, 1, 2, 1, ' ', 0) - for _, object := range output.Objects { - if object.HasHeader { - if output.MultipleFolders { - fmt.Fprintf(tw, "%s:\n", object.Hash) + for i, object := range output.Objects { + if len(object.Links) == 0 { + continue } - if headers { - fmt.Fprintln(tw, "Hash\tSize\tName") + if i > lastDirectoryWritten { + if i > 0 { + if multipleFolders { + fmt.Fprintln(tw) + } + } + if multipleFolders { + fmt.Fprintf(tw, "%s:\n", object.Hash) + } + if headers { + fmt.Fprintln(tw, "Hash\tSize\tName") + } + lastDirectoryWritten = i } - } - if object.HasLinks { for _, link := range object.Links { if link.Type == unixfs.TDirectory { link.Name += "/" @@ -217,15 +240,9 @@ The JSON output contains type information. fmt.Fprintf(tw, "%s\t%v\t%s\n", link.Hash, link.Size, link.Name) } } - if object.HasFooter { - if output.MultipleFolders { - fmt.Fprintln(tw) - } - } + tw.Flush() } - tw.Flush() - return nil - }), + }, }, Type: LsOutput{}, } @@ -248,19 +265,6 @@ func makeDagNodeLinkResults(req *cmds.Request, dagnode ipld.Node) <-chan unixfs. return linkResults } -func newFullDirectoryLsObject(hash string, links []LsLink) LsObject { - return LsObject{hash, links, true, true, true} -} -func newDirectoryHeaderLsObject(hash string) LsObject { - return LsObject{hash, nil, true, false, false} -} -func newDirectoryLinksLsObject(links []LsLink) LsObject { - return LsObject{"", links, false, true, false} -} -func newDirectoryFooterLsObject() LsObject { - return LsObject{"", nil, false, false, true} -} - func makeLsLink(req *cmds.Request, dserv ipld.DAGService, resolve bool, link *ipld.Link) (*LsLink, error) { t := unixfspb.Data_DataType(-1) From 20b928094c990af969e71beadaa47b48c5079c9f Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Thu, 8 Nov 2018 16:20:10 -0800 Subject: [PATCH 3/7] fix(commands): Add sharness tests for streaming Also fixes issues discovered with sharness tests License: MIT Signed-off-by: hannahhoward --- core/commands/ls.go | 39 +++++++++++------- test/sharness/t0045-ls.sh | 83 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 14 deletions(-) diff --git a/core/commands/ls.go b/core/commands/ls.go index 7308d25c357..a6cca891456 100644 --- a/core/commands/ls.go +++ b/core/commands/ls.go @@ -111,10 +111,9 @@ The JSON output contains type information. ng := merkledag.NewSession(req.Context, nd.DAG) ro := merkledag.NewReadOnlyDagService(ng) - output := make([]LsObject, len(req.Arguments)) - stream, _ := req.Options[lsStreamOptionName].(bool) if !stream { + output := make([]LsObject, len(req.Arguments)) for i, dagnode := range dagnodes { dir, err := uio.NewDirectoryFromNode(ro, dagnode) @@ -148,14 +147,6 @@ The JSON output contains type information. return cmds.EmitOnce(res, &LsOutput{output}) } - outputLinks := make([]LsLink, 1) - for i, path := range paths { - output[i] = LsObject{ - Hash: path, - Links: nil, - } - } - for i, dagnode := range dagnodes { dir, err := uio.NewDirectoryFromNode(ro, dagnode) if err != nil && err != uio.ErrNotADir { @@ -169,9 +160,17 @@ The JSON output contains type information. linkResults = dir.EnumLinksAsync(req.Context) } - output[i].Links = outputLinks - for linkResult := range linkResults { + output := make([]LsObject, len(req.Arguments)) + + for i, path := range paths { + output[i] = LsObject{ + Hash: path, + Links: nil, + } + } + outputLinks := make([]LsLink, 1) + if linkResult.Err != nil { return linkResult.Err } @@ -181,11 +180,11 @@ The JSON output contains type information. return err } outputLinks[0] = *lsLink + output[i].Links = outputLinks if err = res.Emit(&LsOutput{output}); err != nil { return err } } - output[i].Links = nil } return nil }, @@ -193,9 +192,21 @@ The JSON output contains type information. cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error { req := res.Request() headers, _ := req.Options[lsHeadersOptionNameTime].(bool) + stream, _ := req.Options[lsStreamOptionName].(bool) + + // in streaming mode we can't automatically align the tabs + // so we take a best guess + var minTabWidth int + if stream { + minTabWidth = 10 + } else { + minTabWidth = 1 + } + multipleFolders := len(req.Arguments) > 1 lastDirectoryWritten := -1 - tw := tabwriter.NewWriter(os.Stdout, 1, 2, 1, ' ', 0) + + tw := tabwriter.NewWriter(os.Stdout, minTabWidth, 2, 1, ' ', 0) for { v, err := res.Next() if err != nil { diff --git a/test/sharness/t0045-ls.sh b/test/sharness/t0045-ls.sh index 2cb46a7dde9..869f9a57fe9 100755 --- a/test/sharness/t0045-ls.sh +++ b/test/sharness/t0045-ls.sh @@ -90,6 +90,87 @@ EOF ' } + +test_ls_cmd_streaming() { + + test_expect_success "'ipfs add -r testData' succeeds" ' + mkdir -p testData testData/d1 testData/d2 && + echo "test" >testData/f1 && + echo "data" >testData/f2 && + echo "hello" >testData/d1/a && + random 128 42 >testData/d1/128 && + echo "world" >testData/d2/a && + random 1024 42 >testData/d2/1024 && + ipfs add -r testData >actual_add + ' + + test_expect_success "'ipfs add' output looks good" ' + cat <<-\EOF >expected_add && +added QmQNd6ubRXaNG6Prov8o6vk3bn6eWsj9FxLGrAVDUAGkGe testData/d1/128 +added QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN testData/d1/a +added QmbQBUSRL9raZtNXfpTDeaxQapibJEG6qEY8WqAN22aUzd testData/d2/1024 +added QmaRGe7bVmVaLmxbrMiVNXqW4pRNNp3xq7hFtyRKA3mtJL testData/d2/a +added QmeomffUNfmQy76CQGy9NdmqEnnHU9soCexBnGU3ezPHVH testData/f1 +added QmNtocSs7MoDkJMc1RkyisCSKvLadujPsfJfSdJ3e1eA1M testData/f2 +added QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss testData/d1 +added QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy testData/d2 +added QmfNy183bXiRVyrhyWtq3TwHn79yHEkiAGFr18P7YNzESj testData +EOF + test_cmp expected_add actual_add + ' + + test_expect_success "'ipfs ls --stream ' succeeds" ' + ipfs ls --stream QmfNy183bXiRVyrhyWtq3TwHn79yHEkiAGFr18P7YNzESj QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss >actual_ls_stream + ' + + test_expect_success "'ipfs ls --stream ' output looks good" ' + cat <<-\EOF >expected_ls_stream && +QmfNy183bXiRVyrhyWtq3TwHn79yHEkiAGFr18P7YNzESj: +QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss 246 d1/ +QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy 1143 d2/ +QmeomffUNfmQy76CQGy9NdmqEnnHU9soCexBnGU3ezPHVH 13 f1 +QmNtocSs7MoDkJMc1RkyisCSKvLadujPsfJfSdJ3e1eA1M 13 f2 + +QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy: +QmbQBUSRL9raZtNXfpTDeaxQapibJEG6qEY8WqAN22aUzd 1035 1024 +QmaRGe7bVmVaLmxbrMiVNXqW4pRNNp3xq7hFtyRKA3mtJL 14 a + +QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss: +QmQNd6ubRXaNG6Prov8o6vk3bn6eWsj9FxLGrAVDUAGkGe 139 128 +QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN 14 a + +EOF + test_cmp expected_ls_stream actual_ls_stream + ' + + test_expect_success "'ipfs ls --stream --headers ' succeeds" ' + ipfs ls --stream --headers QmfNy183bXiRVyrhyWtq3TwHn79yHEkiAGFr18P7YNzESj QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss >actual_ls_stream_headers + ' + + test_expect_success "'ipfs ls --stream --headers ' output looks good" ' + cat <<-\EOF >expected_ls_stream_headers && +QmfNy183bXiRVyrhyWtq3TwHn79yHEkiAGFr18P7YNzESj: +Hash Size Name +QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss 246 d1/ +QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy 1143 d2/ +QmeomffUNfmQy76CQGy9NdmqEnnHU9soCexBnGU3ezPHVH 13 f1 +QmNtocSs7MoDkJMc1RkyisCSKvLadujPsfJfSdJ3e1eA1M 13 f2 + +QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy: +Hash Size Name +QmbQBUSRL9raZtNXfpTDeaxQapibJEG6qEY8WqAN22aUzd 1035 1024 +QmaRGe7bVmVaLmxbrMiVNXqW4pRNNp3xq7hFtyRKA3mtJL 14 a + +QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss: +Hash Size Name +QmQNd6ubRXaNG6Prov8o6vk3bn6eWsj9FxLGrAVDUAGkGe 139 128 +QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN 14 a + +EOF + test_cmp expected_ls_stream_headers actual_ls_stream_headers + ' +} + test_ls_cmd_raw_leaves() { test_expect_success "'ipfs add -r --raw-leaves' then 'ipfs ls' works as expected" ' mkdir -p somedir && @@ -114,12 +195,14 @@ test_ls_object() { # should work offline test_ls_cmd +test_ls_cmd_streaming test_ls_cmd_raw_leaves test_ls_object # should work online test_launch_ipfs_daemon test_ls_cmd +test_ls_cmd_streaming test_ls_cmd_raw_leaves test_kill_ipfs_daemon test_ls_object From 7391cfd42146015859cbd13b12d96d39c9f24615 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 9 Nov 2018 15:09:25 -0800 Subject: [PATCH 4/7] fix(commands): reduce complexity w/ typed encoder uses single flag to support state needed by PostRun supports encoding=text License: MIT Signed-off-by: hannahhoward --- core/commands/ls.go | 97 +++++++++++++++------------------------ test/sharness/t0045-ls.sh | 4 -- 2 files changed, 36 insertions(+), 65 deletions(-) diff --git a/core/commands/ls.go b/core/commands/ls.go index a6cca891456..ce53d49ed8f 100644 --- a/core/commands/ls.go +++ b/core/commands/ls.go @@ -3,11 +3,9 @@ package commands import ( "fmt" "io" - "os" "text/tabwriter" cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv" - e "github.com/ipfs/go-ipfs/core/commands/e" iface "github.com/ipfs/go-ipfs/core/coreapi/interface" cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" @@ -39,7 +37,8 @@ type LsObject struct { // LsOutput is a set of printable data for directories, // it can be complete or partial type LsOutput struct { - Objects []LsObject + Objects []LsObject + LastObjectHash string } const ( @@ -112,6 +111,8 @@ The JSON output contains type information. ro := merkledag.NewReadOnlyDagService(ng) stream, _ := req.Options[lsStreamOptionName].(bool) + lastObjectHash := "" + if !stream { output := make([]LsObject, len(req.Arguments)) @@ -144,7 +145,7 @@ The JSON output contains type information. } } - return cmds.EmitOnce(res, &LsOutput{output}) + return cmds.EmitOnce(res, &LsOutput{output, lastObjectHash}) } for i, dagnode := range dagnodes { @@ -161,15 +162,6 @@ The JSON output contains type information. } for linkResult := range linkResults { - output := make([]LsObject, len(req.Arguments)) - - for i, path := range paths { - output[i] = LsObject{ - Hash: path, - Links: nil, - } - } - outputLinks := make([]LsLink, 1) if linkResult.Err != nil { return linkResult.Err @@ -179,21 +171,24 @@ The JSON output contains type information. if err != nil { return err } - outputLinks[0] = *lsLink - output[i].Links = outputLinks - if err = res.Emit(&LsOutput{output}); err != nil { + output := []LsObject{ + { + Hash: paths[i], + Links: []LsLink{*lsLink}, + }, + } + if err = res.Emit(&LsOutput{output, lastObjectHash}); err != nil { return err } + lastObjectHash = paths[i] } } return nil }, - PostRun: cmds.PostRunMap{ - cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error { - req := res.Request() + Encoders: cmds.EncoderMap{ + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *LsOutput) error { headers, _ := req.Options[lsHeadersOptionNameTime].(bool) stream, _ := req.Options[lsStreamOptionName].(bool) - // in streaming mode we can't automatically align the tabs // so we take a best guess var minTabWidth int @@ -204,56 +199,36 @@ The JSON output contains type information. } multipleFolders := len(req.Arguments) > 1 - lastDirectoryWritten := -1 - - tw := tabwriter.NewWriter(os.Stdout, minTabWidth, 2, 1, ' ', 0) - for { - v, err := res.Next() - if err != nil { - if err == io.EOF { - if multipleFolders { - fmt.Fprintln(os.Stdout) - } - return nil - } + lastObjectHash := out.LastObjectHash - return err - } + tw := tabwriter.NewWriter(w, minTabWidth, 2, 1, ' ', 0) - output, ok := v.(*LsOutput) - if !ok { - return e.TypeErr(output, v) - } + for _, object := range out.Objects { - for i, object := range output.Objects { - if len(object.Links) == 0 { - continue - } - if i > lastDirectoryWritten { - if i > 0 { - if multipleFolders { - fmt.Fprintln(tw) - } - } - if multipleFolders { - fmt.Fprintf(tw, "%s:\n", object.Hash) + if object.Hash != lastObjectHash { + if multipleFolders { + if lastObjectHash != "" { + fmt.Fprintln(tw) } - if headers { - fmt.Fprintln(tw, "Hash\tSize\tName") - } - lastDirectoryWritten = i + fmt.Fprintf(tw, "%s:\n", object.Hash) } - for _, link := range object.Links { - if link.Type == unixfs.TDirectory { - link.Name += "/" - } + if headers { + fmt.Fprintln(tw, "Hash\tSize\tName") + } + lastObjectHash = object.Hash + } - fmt.Fprintf(tw, "%s\t%v\t%s\n", link.Hash, link.Size, link.Name) + for _, link := range object.Links { + if link.Type == unixfs.TDirectory { + link.Name += "/" } + + fmt.Fprintf(tw, "%s\t%v\t%s\n", link.Hash, link.Size, link.Name) } - tw.Flush() } - }, + tw.Flush() + return nil + }), }, Type: LsOutput{}, } diff --git a/test/sharness/t0045-ls.sh b/test/sharness/t0045-ls.sh index 869f9a57fe9..4d3fb296c97 100755 --- a/test/sharness/t0045-ls.sh +++ b/test/sharness/t0045-ls.sh @@ -57,7 +57,6 @@ QmaRGe7bVmVaLmxbrMiVNXqW4pRNNp3xq7hFtyRKA3mtJL 14 a QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss: QmQNd6ubRXaNG6Prov8o6vk3bn6eWsj9FxLGrAVDUAGkGe 139 128 QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN 14 a - EOF test_cmp expected_ls actual_ls ' @@ -84,7 +83,6 @@ QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss: Hash Size Name QmQNd6ubRXaNG6Prov8o6vk3bn6eWsj9FxLGrAVDUAGkGe 139 128 QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN 14 a - EOF test_cmp expected_ls_headers actual_ls_headers ' @@ -138,7 +136,6 @@ QmaRGe7bVmVaLmxbrMiVNXqW4pRNNp3xq7hFtyRKA3mtJL 14 a QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss: QmQNd6ubRXaNG6Prov8o6vk3bn6eWsj9FxLGrAVDUAGkGe 139 128 QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN 14 a - EOF test_cmp expected_ls_stream actual_ls_stream ' @@ -165,7 +162,6 @@ QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss: Hash Size Name QmQNd6ubRXaNG6Prov8o6vk3bn6eWsj9FxLGrAVDUAGkGe 139 128 QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN 14 a - EOF test_cmp expected_ls_stream_headers actual_ls_stream_headers ' From be4c4abcc2da604b774096f2ee22068f23d79de1 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Wed, 14 Nov 2018 13:26:43 -0800 Subject: [PATCH 5/7] chore(commands): Mark feature as experimental License: MIT Signed-off-by: hannahhoward --- core/commands/ls.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/commands/ls.go b/core/commands/ls.go index ce53d49ed8f..e5329af032a 100644 --- a/core/commands/ls.go +++ b/core/commands/ls.go @@ -37,7 +37,9 @@ type LsObject struct { // LsOutput is a set of printable data for directories, // it can be complete or partial type LsOutput struct { - Objects []LsObject + Objects []LsObject + // temporary flag to help us figure out where we are in the process of ls-ing + // the directory when we are streaming LastObjectHash string } @@ -66,7 +68,7 @@ The JSON output contains type information. Options: []cmdkit.Option{ cmdkit.BoolOption(lsHeadersOptionNameTime, "v", "Print table headers (Hash, Size, Name)."), cmdkit.BoolOption(lsResolveTypeOptionName, "Resolve linked objects to find out their types.").WithDefault(true), - cmdkit.BoolOption(lsStreamOptionName, "s", "Stream directory entries as they are found."), + cmdkit.BoolOption(lsStreamOptionName, "s", "Enable exprimental streaming of directory entries as they are traversed."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { nd, err := cmdenv.GetNode(env) From b0dc73c45fc1814990a843f8ae64819797375037 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Wed, 14 Nov 2018 17:00:26 -0800 Subject: [PATCH 6/7] fix(commands): Use post-run to remove flag License: MIT Signed-off-by: hannahhoward --- core/commands/ls.go | 123 +++++++++++++++++++++++++------------------- 1 file changed, 71 insertions(+), 52 deletions(-) diff --git a/core/commands/ls.go b/core/commands/ls.go index e5329af032a..cf4d7b9666b 100644 --- a/core/commands/ls.go +++ b/core/commands/ls.go @@ -3,6 +3,7 @@ package commands import ( "fmt" "io" + "os" "text/tabwriter" cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv" @@ -38,9 +39,6 @@ type LsObject struct { // it can be complete or partial type LsOutput struct { Objects []LsObject - // temporary flag to help us figure out where we are in the process of ls-ing - // the directory when we are streaming - LastObjectHash string } const ( @@ -102,7 +100,6 @@ The JSON output contains type information. if err != nil { return err } - dagnode, err := api.ResolveNode(req.Context, p) if err != nil { return err @@ -113,7 +110,6 @@ The JSON output contains type information. ro := merkledag.NewReadOnlyDagService(ng) stream, _ := req.Options[lsStreamOptionName].(bool) - lastObjectHash := "" if !stream { output := make([]LsObject, len(req.Arguments)) @@ -147,7 +143,7 @@ The JSON output contains type information. } } - return cmds.EmitOnce(res, &LsOutput{output, lastObjectHash}) + return cmds.EmitOnce(res, &LsOutput{output}) } for i, dagnode := range dagnodes { @@ -173,62 +169,42 @@ The JSON output contains type information. if err != nil { return err } - output := []LsObject{ - { - Hash: paths[i], - Links: []LsLink{*lsLink}, - }, - } - if err = res.Emit(&LsOutput{output, lastObjectHash}); err != nil { + output := []LsObject{{ + Hash: paths[i], + Links: []LsLink{*lsLink}, + }} + if err = res.Emit(&LsOutput{output}); err != nil { return err } - lastObjectHash = paths[i] } } return nil }, - Encoders: cmds.EncoderMap{ - cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *LsOutput) error { - headers, _ := req.Options[lsHeadersOptionNameTime].(bool) - stream, _ := req.Options[lsStreamOptionName].(bool) - // in streaming mode we can't automatically align the tabs - // so we take a best guess - var minTabWidth int - if stream { - minTabWidth = 10 - } else { - minTabWidth = 1 - } - - multipleFolders := len(req.Arguments) > 1 - lastObjectHash := out.LastObjectHash - - tw := tabwriter.NewWriter(w, minTabWidth, 2, 1, ' ', 0) + PostRun: cmds.PostRunMap{ + cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error { + req := res.Request() + lastObjectHash := "" - for _, object := range out.Objects { - - if object.Hash != lastObjectHash { - if multipleFolders { - if lastObjectHash != "" { - fmt.Fprintln(tw) - } - fmt.Fprintf(tw, "%s:\n", object.Hash) - } - if headers { - fmt.Fprintln(tw, "Hash\tSize\tName") - } - lastObjectHash = object.Hash - } - - for _, link := range object.Links { - if link.Type == unixfs.TDirectory { - link.Name += "/" + for { + v, err := res.Next() + if err != nil { + if err == io.EOF { + return nil } - - fmt.Fprintf(tw, "%s\t%v\t%s\n", link.Hash, link.Size, link.Name) + return err } + out := v.(*LsOutput) + lastObjectHash = tabularOutput(req, os.Stdout, out, lastObjectHash, false) } - tw.Flush() + }, + }, + Encoders: cmds.EncoderMap{ + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *LsOutput) error { + // when streaming over HTTP using a text encoder, we cannot render breaks + // between directories because we don't know the hash of the last + // directory encoder + ignoreBreaks, _ := req.Options[lsStreamOptionName].(bool) + tabularOutput(req, w, out, "", ignoreBreaks) return nil }), }, @@ -284,3 +260,46 @@ func makeLsLink(req *cmds.Request, dserv ipld.DAGService, resolve bool, link *ip Type: t, }, nil } + +func tabularOutput(req *cmds.Request, w io.Writer, out *LsOutput, lastObjectHash string, ignoreBreaks bool) string { + headers, _ := req.Options[lsHeadersOptionNameTime].(bool) + stream, _ := req.Options[lsStreamOptionName].(bool) + // in streaming mode we can't automatically align the tabs + // so we take a best guess + var minTabWidth int + if stream { + minTabWidth = 10 + } else { + minTabWidth = 1 + } + + multipleFolders := len(req.Arguments) > 1 + + tw := tabwriter.NewWriter(w, minTabWidth, 2, 1, ' ', 0) + + for _, object := range out.Objects { + + if !ignoreBreaks && object.Hash != lastObjectHash { + if multipleFolders { + if lastObjectHash != "" { + fmt.Fprintln(tw) + } + fmt.Fprintf(tw, "%s:\n", object.Hash) + } + if headers { + fmt.Fprintln(tw, "Hash\tSize\tName") + } + lastObjectHash = object.Hash + } + + for _, link := range object.Links { + if link.Type == unixfs.TDirectory { + link.Name += "/" + } + + fmt.Fprintf(tw, "%s\t%v\t%s\n", link.Hash, link.Size, link.Name) + } + } + tw.Flush() + return lastObjectHash +} From f5ab6a3f5ecbb7129f2fea8ccaecc8dbbadbc261 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Mon, 19 Nov 2018 10:46:09 -0800 Subject: [PATCH 7/7] refactor(commands): cleanup makeDagNodeLinkResults License: MIT Signed-off-by: hannahhoward --- core/commands/ls.go | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/core/commands/ls.go b/core/commands/ls.go index cf4d7b9666b..422f554d1a7 100644 --- a/core/commands/ls.go +++ b/core/commands/ls.go @@ -212,20 +212,15 @@ The JSON output contains type information. } func makeDagNodeLinkResults(req *cmds.Request, dagnode ipld.Node) <-chan unixfs.LinkResult { - linkResults := make(chan unixfs.LinkResult) - go func() { - defer close(linkResults) - for _, l := range dagnode.Links() { - select { - case linkResults <- unixfs.LinkResult{ - Link: l, - Err: nil, - }: - case <-req.Context.Done(): - return - } + links := dagnode.Links() + linkResults := make(chan unixfs.LinkResult, len(links)) + defer close(linkResults) + for _, l := range links { + linkResults <- unixfs.LinkResult{ + Link: l, + Err: nil, } - }() + } return linkResults }