From f1604daca4c61b37111992ab34406aa5907bdae4 Mon Sep 17 00:00:00 2001 From: arthurgavazza Date: Fri, 31 Mar 2023 18:23:53 -0300 Subject: [PATCH 01/24] checkpoint1 --- core/commands/dag/dag.go | 55 ++++++++++++++++++-- core/commands/dag/stat.go | 104 ++++++++++++++++++++------------------ go.mod | 2 + go.sum | 1 + test.json | 1 + test/cli/dag_test.go | 28 ++++++++++ test2.json | 1 + 7 files changed, 140 insertions(+), 52 deletions(-) create mode 100644 test.json create mode 100644 test/cli/dag_test.go create mode 100644 test2.json diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index 75588b8ed5f..42a924395b7 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -275,13 +275,56 @@ CAR file follows the CARv1 format: https://ipld.io/specs/transport/car/carv1/ } // DagStat is a dag stat command response + type DagStat struct { + Cid cid.Cid Size uint64 NumBlocks int64 } func (s *DagStat) String() string { - return fmt.Sprintf("Size: %d, NumBlocks: %d", s.Size, s.NumBlocks) + return fmt.Sprintf("%s %d %d", s.Cid.String(), s.Size, s.NumBlocks) +} + +type DagStatSummary struct { + redundantSize uint64 + UniqueBlocks int + TotalSize uint64 + SharedSize uint64 + Ratio float32 +} + +func (s *DagStatSummary) String() string { + return fmt.Sprintf("Total Size: %d\n Unique Blocks: %d\n Shared Size: %d\n Ratio: %f", s.TotalSize, s.UniqueBlocks, s.SharedSize, s.Ratio) +} + +func (s *DagStatSummary) incrementTotalSize(size uint64) { + s.TotalSize += size +} +func (s *DagStatSummary) incrementRedundantSize(size uint64) { + s.redundantSize += size +} + +type DagStatCalculator struct { + Summary *DagStatSummary + CurrentDag *DagStat +} + +func (m *DagStatCalculator) calculateRatio() { + if m.Summary.TotalSize > 0 { + m.Summary.Ratio = float32(m.Summary.redundantSize) / float32(m.Summary.TotalSize) + } +} + +func (m *DagStatCalculator) calculateSharedSize() { + m.Summary.SharedSize = m.Summary.redundantSize - m.Summary.TotalSize +} + +func (m *DagStatCalculator) String() string { + if m.CurrentDag == nil { + return m.Summary.String() + } + return m.CurrentDag.String() } // DagStatCmd is a command for getting size information about an ipfs-stored dag @@ -296,18 +339,22 @@ Note: This command skips duplicate blocks in reporting both size and the number `, }, Arguments: []cmds.Argument{ - cmds.StringArg("root", true, false, "CID of a DAG root to get statistics for").EnableStdin(), + + cmds.StringArg("root", true, true, "CID of a DAG root to get statistics for").EnableStdin(), }, Options: []cmds.Option{ cmds.BoolOption(progressOptionName, "p", "Return progressive data while reading through the DAG").WithDefault(true), }, Run: dagStat, - Type: DagStat{}, + Type: DagStatCalculator{}, PostRun: cmds.PostRunMap{ cmds.CLI: finishCLIStat, }, Encoders: cmds.EncoderMap{ - cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, event *DagStat) error { + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, event *DagStatCalculator) error { + + fmt.Println("Summary") + _, err := fmt.Fprintf( w, "%v\n", diff --git a/core/commands/dag/stat.go b/core/commands/dag/stat.go index b4617a5b559..56de43994a4 100644 --- a/core/commands/dag/stat.go +++ b/core/commands/dag/stat.go @@ -6,70 +6,75 @@ import ( "os" "github.com/ipfs/boxo/coreiface/path" - "github.com/ipfs/boxo/ipld/merkledag/traverse" - "github.com/ipfs/kubo/core/commands/cmdenv" - "github.com/ipfs/kubo/core/commands/e" - mdag "github.com/ipfs/boxo/ipld/merkledag" + cid "github.com/ipfs/go-cid" cmds "github.com/ipfs/go-ipfs-cmds" + "github.com/ipfs/go-merkledag/traverse" + "github.com/ipfs/kubo/core/commands/cmdenv" + "github.com/ipfs/kubo/core/commands/e" ) func dagStat(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { - progressive := req.Options[progressOptionName].(bool) - api, err := cmdenv.GetApi(env, req) if err != nil { return err } - - rp, err := api.ResolvePath(req.Context, path.New(req.Arguments[0])) - if err != nil { - return err - } - - if len(rp.Remainder()) > 0 { - return fmt.Errorf("cannot return size for anything other than a DAG with a root CID") - } - nodeGetter := mdag.NewSession(req.Context, api.Dag()) - obj, err := nodeGetter.Get(req.Context, rp.Cid()) - if err != nil { - return err - } - dagstats := &DagStat{} - err = traverse.Traverse(obj, traverse.Options{ - DAG: nodeGetter, - Order: traverse.DFSPre, - Func: func(current traverse.State) error { - dagstats.Size += uint64(len(current.Node.RawData())) - dagstats.NumBlocks++ + cidSet := cid.NewSet() + dagStatCalculator := &DagStatCalculator{Summary: &DagStatSummary{}} + for _, a := range req.Arguments { + rp, err := api.ResolvePath(req.Context, path.New(a)) + if err != nil { + return err + } + if len(rp.Remainder()) > 0 { + return fmt.Errorf("cannot return size for anything other than a DAG with a root CID") + } - if progressive { - if err := res.Emit(dagstats); err != nil { - return err + obj, err := nodeGetter.Get(req.Context, rp.Cid()) + if err != nil { + return err + } + dagstats := &DagStat{Cid: rp.Cid()} + err = traverse.Traverse(obj, traverse.Options{ + DAG: nodeGetter, + Order: traverse.DFSPre, + Func: func(current traverse.State) error { + dagstats.Size += uint64(len(current.Node.RawData())) + dagstats.NumBlocks++ + if !cidSet.Has(current.Node.Cid()) { + dagStatCalculator.Summary.incrementTotalSize(dagstats.Size) } - } - return nil - }, - ErrFunc: nil, - SkipDuplicates: true, - }) - if err != nil { - return fmt.Errorf("error traversing DAG: %w", err) - } - - if !progressive { - if err := res.Emit(dagstats); err != nil { + dagStatCalculator.Summary.incrementRedundantSize(dagstats.Size) + cidSet.Add(current.Node.Cid()) + return nil + }, + ErrFunc: nil, + SkipDuplicates: true, + }) + if err != nil { + return fmt.Errorf("error traversing DAG: %w", err) + } + dagStatCalculator.CurrentDag = dagstats + if err := res.Emit(dagStatCalculator); err != nil { return err } } + dagStatCalculator.Summary.UniqueBlocks = cidSet.Len() + dagStatCalculator.calculateSharedSize() + dagStatCalculator.calculateRatio() + dagStatCalculator.CurrentDag = nil + if err := res.Emit(dagStatCalculator); err != nil { + return err + } return nil } func finishCLIStat(res cmds.Response, re cmds.ResponseEmitter) error { - var dagStats *DagStat + + var dagStats *DagStatCalculator for { v, err := res.Next() if err != nil { @@ -78,13 +83,16 @@ func finishCLIStat(res cmds.Response, re cmds.ResponseEmitter) error { } return err } - - out, ok := v.(*DagStat) - if !ok { + switch out := v.(type) { + case *DagStatCalculator: + dagStats = out + if out.CurrentDag != nil { + fmt.Fprint(os.Stderr, *out) + fmt.Fprint(os.Stderr, out) + } + default: return e.TypeErr(out, v) } - dagStats = out - fmt.Fprintf(os.Stderr, "%v\r", out) } return re.Emit(dagStats) } diff --git a/go.mod b/go.mod index c3785381c6b..5037ac7e86b 100644 --- a/go.mod +++ b/go.mod @@ -93,6 +93,8 @@ require ( golang.org/x/sys v0.6.0 ) +require github.com/ipfs/go-merkledag v0.10.0 + require ( github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect github.com/Kubuxu/go-os-helper v0.0.1 // indirect diff --git a/go.sum b/go.sum index 5822ea8983c..49b4a738062 100644 --- a/go.sum +++ b/go.sum @@ -444,6 +444,7 @@ github.com/ipfs/go-log/v2 v2.3.0/go.mod h1:QqGoj30OTpnKaG/LKTGTxoP2mmQtjVMEnK72g github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= github.com/ipfs/go-merkledag v0.10.0 h1:IUQhj/kzTZfam4e+LnaEpoiZ9vZF6ldimVlby+6OXL4= +github.com/ipfs/go-merkledag v0.10.0/go.mod h1:zkVav8KiYlmbzUzNM6kENzkdP5+qR7+2mCwxkQ6GIj8= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= github.com/ipfs/go-metrics-prometheus v0.0.2 h1:9i2iljLg12S78OhC6UAiXi176xvQGiZaGVF1CUVdE+s= diff --git a/test.json b/test.json new file mode 100644 index 00000000000..210eaef4912 --- /dev/null +++ b/test.json @@ -0,0 +1 @@ +{"hello1": "world"} \ No newline at end of file diff --git a/test/cli/dag_test.go b/test/cli/dag_test.go new file mode 100644 index 00000000000..ac7fd7e819d --- /dev/null +++ b/test/cli/dag_test.go @@ -0,0 +1,28 @@ +package cli + +import ( + "testing" + + "github.com/ipfs/kubo/test/cli/harness" + "github.com/stretchr/testify/assert" +) + +func TestDag(t *testing.T) { + t.Parallel() + t.Run("ipfs add, adds file", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + + output := node.IPFSAddStr("hello world") + output2 := node.IPFSAddStr("hello world 2") + output3 := node.IPFSAddStr("hello world 3") + assert.NotEqual(t, "", output) + + stat := node.RunIPFS("dag", "stat", output, output2, output3) + str := stat.Stdout.String() + err := stat.Stderr.String() + assert.NotEqual(t, "", str) + assert.Nil(t, err) + + }) +} diff --git a/test2.json b/test2.json new file mode 100644 index 00000000000..d309f7fa707 --- /dev/null +++ b/test2.json @@ -0,0 +1 @@ +{"hello3": "world"} \ No newline at end of file From ac3e6b02b8d45c1bab4ac59edf48c36cf0039e50 Mon Sep 17 00:00:00 2001 From: arthurgavazza Date: Fri, 31 Mar 2023 19:32:56 -0300 Subject: [PATCH 02/24] feat: achieve correct minimal output --- core/commands/dag/dag.go | 54 ++++++++++++++++++++++----------------- core/commands/dag/stat.go | 29 ++++++++------------- 2 files changed, 40 insertions(+), 43 deletions(-) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index 42a924395b7..9f015f4aba2 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -1,6 +1,7 @@ package dagcmd import ( + "encoding/csv" "fmt" "io" @@ -283,7 +284,7 @@ type DagStat struct { } func (s *DagStat) String() string { - return fmt.Sprintf("%s %d %d", s.Cid.String(), s.Size, s.NumBlocks) + return fmt.Sprintf("%s %d %d", s.Cid.String()[:20], s.Size, s.NumBlocks) } type DagStatSummary struct { @@ -292,10 +293,11 @@ type DagStatSummary struct { TotalSize uint64 SharedSize uint64 Ratio float32 + DagStatsArray []*DagStat } func (s *DagStatSummary) String() string { - return fmt.Sprintf("Total Size: %d\n Unique Blocks: %d\n Shared Size: %d\n Ratio: %f", s.TotalSize, s.UniqueBlocks, s.SharedSize, s.Ratio) + return fmt.Sprintf("Total Size: %d\nUnique Blocks: %d\nShared Size: %d\nRatio: %f", s.TotalSize, s.UniqueBlocks, s.SharedSize, s.Ratio) } func (s *DagStatSummary) incrementTotalSize(size uint64) { @@ -305,26 +307,14 @@ func (s *DagStatSummary) incrementRedundantSize(size uint64) { s.redundantSize += size } -type DagStatCalculator struct { - Summary *DagStatSummary - CurrentDag *DagStat -} - -func (m *DagStatCalculator) calculateRatio() { - if m.Summary.TotalSize > 0 { - m.Summary.Ratio = float32(m.Summary.redundantSize) / float32(m.Summary.TotalSize) +func (s *DagStatSummary) calculateRatio() { + if s.TotalSize > 0 { + s.Ratio = float32(s.redundantSize) / float32(s.TotalSize) } } -func (m *DagStatCalculator) calculateSharedSize() { - m.Summary.SharedSize = m.Summary.redundantSize - m.Summary.TotalSize -} - -func (m *DagStatCalculator) String() string { - if m.CurrentDag == nil { - return m.Summary.String() - } - return m.CurrentDag.String() +func (s *DagStatSummary) calculateSharedSize() { + s.SharedSize = s.redundantSize - s.TotalSize } // DagStatCmd is a command for getting size information about an ipfs-stored dag @@ -346,20 +336,36 @@ Note: This command skips duplicate blocks in reporting both size and the number cmds.BoolOption(progressOptionName, "p", "Return progressive data while reading through the DAG").WithDefault(true), }, Run: dagStat, - Type: DagStatCalculator{}, + Type: DagStatSummary{}, PostRun: cmds.PostRunMap{ cmds.CLI: finishCLIStat, }, Encoders: cmds.EncoderMap{ - cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, event *DagStatCalculator) error { - - fmt.Println("Summary") - + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, event *DagStatSummary) error { + csvWriter := csv.NewWriter(w) + csvWriter.Comma = '\t' + header := []string{"CID","Blocks","Size"} + fmt.Println() + csvWriter.Write(header) + for _,dagStat := range event.DagStatsArray { + err := csvWriter.Write([]string{ + dagStat.Cid.String()[:12], + fmt.Sprint(dagStat.NumBlocks), + fmt.Sprint(dagStat.Size), + }) + if err != nil { + return err + } + } + csvWriter.Flush() + fmt.Print("\nSummary\n\n") + _, err := fmt.Fprintf( w, "%v\n", event, ) + fmt.Print("\n\n") return err }), }, diff --git a/core/commands/dag/stat.go b/core/commands/dag/stat.go index 56de43994a4..e266adecb6b 100644 --- a/core/commands/dag/stat.go +++ b/core/commands/dag/stat.go @@ -3,7 +3,6 @@ package dagcmd import ( "fmt" "io" - "os" "github.com/ipfs/boxo/coreiface/path" mdag "github.com/ipfs/boxo/ipld/merkledag" @@ -22,7 +21,7 @@ func dagStat(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) nodeGetter := mdag.NewSession(req.Context, api.Dag()) cidSet := cid.NewSet() - dagStatCalculator := &DagStatCalculator{Summary: &DagStatSummary{}} + dagStatSummary := &DagStatSummary{DagStatsArray: []*DagStat{}} for _, a := range req.Arguments { rp, err := api.ResolvePath(req.Context, path.New(a)) if err != nil { @@ -44,9 +43,9 @@ func dagStat(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) dagstats.Size += uint64(len(current.Node.RawData())) dagstats.NumBlocks++ if !cidSet.Has(current.Node.Cid()) { - dagStatCalculator.Summary.incrementTotalSize(dagstats.Size) + dagStatSummary.incrementTotalSize(dagstats.Size) } - dagStatCalculator.Summary.incrementRedundantSize(dagstats.Size) + dagStatSummary.incrementRedundantSize(dagstats.Size) cidSet.Add(current.Node.Cid()) return nil }, @@ -56,17 +55,13 @@ func dagStat(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) if err != nil { return fmt.Errorf("error traversing DAG: %w", err) } - dagStatCalculator.CurrentDag = dagstats - if err := res.Emit(dagStatCalculator); err != nil { - return err - } + dagStatSummary.DagStatsArray = append(dagStatSummary.DagStatsArray,dagstats) } - dagStatCalculator.Summary.UniqueBlocks = cidSet.Len() - dagStatCalculator.calculateSharedSize() - dagStatCalculator.calculateRatio() - dagStatCalculator.CurrentDag = nil - if err := res.Emit(dagStatCalculator); err != nil { + dagStatSummary.UniqueBlocks = cidSet.Len() + dagStatSummary.calculateSharedSize() + dagStatSummary.calculateRatio() + if err := res.Emit(dagStatSummary); err != nil { return err } return nil @@ -74,7 +69,7 @@ func dagStat(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) func finishCLIStat(res cmds.Response, re cmds.ResponseEmitter) error { - var dagStats *DagStatCalculator + var dagStats *DagStatSummary for { v, err := res.Next() if err != nil { @@ -84,12 +79,8 @@ func finishCLIStat(res cmds.Response, re cmds.ResponseEmitter) error { return err } switch out := v.(type) { - case *DagStatCalculator: + case *DagStatSummary: dagStats = out - if out.CurrentDag != nil { - fmt.Fprint(os.Stderr, *out) - fmt.Fprint(os.Stderr, out) - } default: return e.TypeErr(out, v) } From ce81fd063417464eda4e72e67ab90f0169e72ef6 Mon Sep 17 00:00:00 2001 From: arthurgavazza Date: Fri, 31 Mar 2023 19:50:16 -0300 Subject: [PATCH 03/24] feat: start adding json encoding --- core/commands/dag/dag.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index 9f015f4aba2..1420046ba17 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -2,6 +2,7 @@ package dagcmd import ( "encoding/csv" + "encoding/json" "fmt" "io" @@ -278,8 +279,8 @@ CAR file follows the CARv1 format: https://ipld.io/specs/transport/car/carv1/ // DagStat is a dag stat command response type DagStat struct { - Cid cid.Cid - Size uint64 + Cid cid.Cid + Size uint64 NumBlocks int64 } @@ -288,8 +289,8 @@ func (s *DagStat) String() string { } type DagStatSummary struct { - redundantSize uint64 - UniqueBlocks int + redundantSize uint64 `json:"-"` + UniqueBlocks int TotalSize uint64 SharedSize uint64 Ratio float32 @@ -368,5 +369,11 @@ Note: This command skips duplicate blocks in reporting both size and the number fmt.Print("\n\n") return err }), + cmds.JSON: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer,event *DagStatSummary) error { + return json.NewEncoder(w).Encode(event) + }, + + ), }, + } From 6dffb587de2889155040a5373c628213d90e2c39 Mon Sep 17 00:00:00 2001 From: arthurgavazza Date: Fri, 31 Mar 2023 19:52:41 -0300 Subject: [PATCH 04/24] fix: fmt --- core/commands/dag/dag.go | 22 ++++++++++------------ core/commands/dag/stat.go | 2 +- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index 1420046ba17..c5991631e0a 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -279,8 +279,8 @@ CAR file follows the CARv1 format: https://ipld.io/specs/transport/car/carv1/ // DagStat is a dag stat command response type DagStat struct { - Cid cid.Cid - Size uint64 + Cid cid.Cid + Size uint64 NumBlocks int64 } @@ -289,8 +289,8 @@ func (s *DagStat) String() string { } type DagStatSummary struct { - redundantSize uint64 `json:"-"` - UniqueBlocks int + redundantSize uint64 `json:"-"` + UniqueBlocks int TotalSize uint64 SharedSize uint64 Ratio float32 @@ -343,12 +343,12 @@ Note: This command skips duplicate blocks in reporting both size and the number }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, event *DagStatSummary) error { - csvWriter := csv.NewWriter(w) + csvWriter := csv.NewWriter(w) csvWriter.Comma = '\t' - header := []string{"CID","Blocks","Size"} + header := []string{"CID", "Blocks", "Size"} fmt.Println() csvWriter.Write(header) - for _,dagStat := range event.DagStatsArray { + for _, dagStat := range event.DagStatsArray { err := csvWriter.Write([]string{ dagStat.Cid.String()[:12], fmt.Sprint(dagStat.NumBlocks), @@ -360,7 +360,7 @@ Note: This command skips duplicate blocks in reporting both size and the number } csvWriter.Flush() fmt.Print("\nSummary\n\n") - + _, err := fmt.Fprintf( w, "%v\n", @@ -369,11 +369,9 @@ Note: This command skips duplicate blocks in reporting both size and the number fmt.Print("\n\n") return err }), - cmds.JSON: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer,event *DagStatSummary) error { + cmds.JSON: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, event *DagStatSummary) error { return json.NewEncoder(w).Encode(event) }, - - ), + ), }, - } diff --git a/core/commands/dag/stat.go b/core/commands/dag/stat.go index e266adecb6b..676608ebff8 100644 --- a/core/commands/dag/stat.go +++ b/core/commands/dag/stat.go @@ -55,7 +55,7 @@ func dagStat(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) if err != nil { return fmt.Errorf("error traversing DAG: %w", err) } - dagStatSummary.DagStatsArray = append(dagStatSummary.DagStatsArray,dagstats) + dagStatSummary.DagStatsArray = append(dagStatSummary.DagStatsArray, dagstats) } dagStatSummary.UniqueBlocks = cidSet.Len() From 946fdd6cc9e1fdff27c9ab8cbf21d5ec1b4528ff Mon Sep 17 00:00:00 2001 From: arthurgavazza Date: Sat, 1 Apr 2023 11:35:41 -0300 Subject: [PATCH 05/24] feat: add custom json encoder and format text encoder --- core/commands/dag/dag.go | 59 ++++++++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 15 deletions(-) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index c5991631e0a..60c1c4cd1d8 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -279,22 +279,52 @@ CAR file follows the CARv1 format: https://ipld.io/specs/transport/car/carv1/ // DagStat is a dag stat command response type DagStat struct { - Cid cid.Cid - Size uint64 - NumBlocks int64 + Cid cid.Cid `json:",omitempty"` + Size uint64 `json:",omitempty"` + NumBlocks int64 `json:",omitempty"` } func (s *DagStat) String() string { return fmt.Sprintf("%s %d %d", s.Cid.String()[:20], s.Size, s.NumBlocks) } +func (s *DagStat) MarshalJSON() ([]byte, error) { + type Alias DagStat + return json.Marshal(&struct { + Cid string `json:"Cid"` + *Alias + }{ + Cid: s.Cid.String(), + Alias: (*Alias)(s), + }) +} + +func (s *DagStat) UnmarshalJSON(data []byte) error { + type Alias DagStat + aux := &struct { + Cid string `json:"Cid"` + *Alias + }{ + Alias: (*Alias)(s), + } + if err := json.Unmarshal(data, &aux); err != nil { + return err + } + Cid, err := cid.Parse(aux.Cid) + if err != nil { + return err + } + s.Cid = Cid + return nil +} + type DagStatSummary struct { - redundantSize uint64 `json:"-"` - UniqueBlocks int - TotalSize uint64 - SharedSize uint64 - Ratio float32 - DagStatsArray []*DagStat + redundantSize uint64 `json:"-"` + UniqueBlocks int `json:",omitempty"` + TotalSize uint64 `json:",omitempty"` + SharedSize uint64 `json:",omitempty"` + Ratio float32 `json:",omitempty"` + DagStatsArray []*DagStat `json:"DagStats,omitempty"` } func (s *DagStatSummary) String() string { @@ -330,7 +360,6 @@ Note: This command skips duplicate blocks in reporting both size and the number `, }, Arguments: []cmds.Argument{ - cmds.StringArg("root", true, true, "CID of a DAG root to get statistics for").EnableStdin(), }, Options: []cmds.Option{ @@ -343,15 +372,16 @@ Note: This command skips duplicate blocks in reporting both size and the number }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, event *DagStatSummary) error { + fmt.Println() csvWriter := csv.NewWriter(w) csvWriter.Comma = '\t' - header := []string{"CID", "Blocks", "Size"} - fmt.Println() + header := []string{fmt.Sprintf("%-*s", 46, "CID"), fmt.Sprintf("%-15s", "Blocks"), "Size"} csvWriter.Write(header) for _, dagStat := range event.DagStatsArray { + numBlocksStr := fmt.Sprint(dagStat.NumBlocks) err := csvWriter.Write([]string{ - dagStat.Cid.String()[:12], - fmt.Sprint(dagStat.NumBlocks), + dagStat.Cid.String(), + fmt.Sprintf("%-15s", numBlocksStr), fmt.Sprint(dagStat.Size), }) if err != nil { @@ -360,7 +390,6 @@ Note: This command skips duplicate blocks in reporting both size and the number } csvWriter.Flush() fmt.Print("\nSummary\n\n") - _, err := fmt.Fprintf( w, "%v\n", From 63d271b0fcc300febe38cfbe01b481221cf9f003 Mon Sep 17 00:00:00 2001 From: arthurgavazza Date: Sat, 1 Apr 2023 11:37:22 -0300 Subject: [PATCH 06/24] fix: remove test json files --- test.json | 1 - test2.json | 1 - 2 files changed, 2 deletions(-) delete mode 100644 test.json delete mode 100644 test2.json diff --git a/test.json b/test.json deleted file mode 100644 index 210eaef4912..00000000000 --- a/test.json +++ /dev/null @@ -1 +0,0 @@ -{"hello1": "world"} \ No newline at end of file diff --git a/test2.json b/test2.json deleted file mode 100644 index d309f7fa707..00000000000 --- a/test2.json +++ /dev/null @@ -1 +0,0 @@ -{"hello3": "world"} \ No newline at end of file From fbc1c9c5299662dd13fd96af2b7a9c8637277b73 Mon Sep 17 00:00:00 2001 From: Arthur Gavazza <32915690+arthurgavazza@users.noreply.github.com> Date: Tue, 2 May 2023 21:06:08 -0300 Subject: [PATCH 07/24] Update core/commands/dag/dag.go Co-authored-by: Jorropo --- core/commands/dag/dag.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index 60c1c4cd1d8..d7b0fbea121 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -395,7 +395,7 @@ Note: This command skips duplicate blocks in reporting both size and the number "%v\n", event, ) - fmt.Print("\n\n") + fmt.Fprint(w, "\n\n") return err }), cmds.JSON: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, event *DagStatSummary) error { From eac10dfbd63163e9a2fe1a33eee37c7efb162f4f Mon Sep 17 00:00:00 2001 From: Arthur Gavazza <32915690+arthurgavazza@users.noreply.github.com> Date: Tue, 2 May 2023 21:21:15 -0300 Subject: [PATCH 08/24] fix: check error on cdv write Co-authored-by: Jorropo --- core/commands/dag/dag.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index d7b0fbea121..a672dd83e0d 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -376,7 +376,9 @@ Note: This command skips duplicate blocks in reporting both size and the number csvWriter := csv.NewWriter(w) csvWriter.Comma = '\t' header := []string{fmt.Sprintf("%-*s", 46, "CID"), fmt.Sprintf("%-15s", "Blocks"), "Size"} - csvWriter.Write(header) + if err := csvWriter.Write(header); err != nil { + return err + } for _, dagStat := range event.DagStatsArray { numBlocksStr := fmt.Sprint(dagStat.NumBlocks) err := csvWriter.Write([]string{ From 27547d34b7695aefc140915f4f6f308884b87ecb Mon Sep 17 00:00:00 2001 From: Arthur Gavazza <32915690+arthurgavazza@users.noreply.github.com> Date: Tue, 2 May 2023 21:22:21 -0300 Subject: [PATCH 09/24] fix: tsv title format Co-authored-by: Jorropo --- core/commands/dag/dag.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index a672dd83e0d..d0ec88f7c27 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -391,7 +391,7 @@ Note: This command skips duplicate blocks in reporting both size and the number } } csvWriter.Flush() - fmt.Print("\nSummary\n\n") + fmt.Fprint(w, "\nSummary\n") _, err := fmt.Fprintf( w, "%v\n", From 7f38e7e83496b0290989f2232de3cbb9d98fde73 Mon Sep 17 00:00:00 2001 From: Arthur Gavazza <32915690+arthurgavazza@users.noreply.github.com> Date: Tue, 2 May 2023 21:24:15 -0300 Subject: [PATCH 10/24] fix: print line with Fprintln Co-authored-by: Jorropo --- core/commands/dag/dag.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index d0ec88f7c27..af3ff4d2451 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -372,7 +372,7 @@ Note: This command skips duplicate blocks in reporting both size and the number }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, event *DagStatSummary) error { - fmt.Println() + fmt.Fprintln(w) csvWriter := csv.NewWriter(w) csvWriter.Comma = '\t' header := []string{fmt.Sprintf("%-*s", 46, "CID"), fmt.Sprintf("%-15s", "Blocks"), "Size"} From 76e5f200b36f0f1061d3badc0b89f63d6af8a410 Mon Sep 17 00:00:00 2001 From: Arthur Gavazza <32915690+arthurgavazza@users.noreply.github.com> Date: Tue, 2 May 2023 21:25:03 -0300 Subject: [PATCH 11/24] fix: use boxo traverse Co-authored-by: Jorropo --- core/commands/dag/stat.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/commands/dag/stat.go b/core/commands/dag/stat.go index 676608ebff8..d594ea8ce92 100644 --- a/core/commands/dag/stat.go +++ b/core/commands/dag/stat.go @@ -8,7 +8,7 @@ import ( mdag "github.com/ipfs/boxo/ipld/merkledag" cid "github.com/ipfs/go-cid" cmds "github.com/ipfs/go-ipfs-cmds" - "github.com/ipfs/go-merkledag/traverse" + "github.com/ipfs/boxo/ipld/merkledag/traverse" "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/commands/e" ) From b48dcb0f333d23d797503bdffadd5bd997c84ab5 Mon Sep 17 00:00:00 2001 From: arthurgavazza Date: Tue, 2 May 2023 21:37:57 -0300 Subject: [PATCH 12/24] refactor: calculate summary with only one method --- core/commands/dag/dag.go | 9 ++------- core/commands/dag/stat.go | 6 +++--- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index af3ff4d2451..80e2acfc0ec 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -338,13 +338,8 @@ func (s *DagStatSummary) incrementRedundantSize(size uint64) { s.redundantSize += size } -func (s *DagStatSummary) calculateRatio() { - if s.TotalSize > 0 { - s.Ratio = float32(s.redundantSize) / float32(s.TotalSize) - } -} - -func (s *DagStatSummary) calculateSharedSize() { +func (s *DagStatSummary) calculateSummary() { + s.Ratio = float32(s.redundantSize) / float32(s.TotalSize) s.SharedSize = s.redundantSize - s.TotalSize } diff --git a/core/commands/dag/stat.go b/core/commands/dag/stat.go index d594ea8ce92..cab3f50d68f 100644 --- a/core/commands/dag/stat.go +++ b/core/commands/dag/stat.go @@ -6,9 +6,9 @@ import ( "github.com/ipfs/boxo/coreiface/path" mdag "github.com/ipfs/boxo/ipld/merkledag" + "github.com/ipfs/boxo/ipld/merkledag/traverse" cid "github.com/ipfs/go-cid" cmds "github.com/ipfs/go-ipfs-cmds" - "github.com/ipfs/boxo/ipld/merkledag/traverse" "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/commands/e" ) @@ -59,8 +59,8 @@ func dagStat(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) } dagStatSummary.UniqueBlocks = cidSet.Len() - dagStatSummary.calculateSharedSize() - dagStatSummary.calculateRatio() + dagStatSummary.calculateSummary() + if err := res.Emit(dagStatSummary); err != nil { return err } From 9bbef7fb257d5b2961414ad8db740d8e5d8e0d31 Mon Sep 17 00:00:00 2001 From: arthurgavazza Date: Tue, 2 May 2023 21:57:03 -0300 Subject: [PATCH 13/24] refactor: make simplifications --- core/commands/dag/dag.go | 7 +++++-- core/commands/dag/stat.go | 6 +++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index 80e2acfc0ec..02df1a0cebf 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -290,7 +290,7 @@ func (s *DagStat) String() string { func (s *DagStat) MarshalJSON() ([]byte, error) { type Alias DagStat - return json.Marshal(&struct { + return json.Marshal(struct { Cid string `json:"Cid"` *Alias }{ @@ -301,7 +301,7 @@ func (s *DagStat) MarshalJSON() ([]byte, error) { func (s *DagStat) UnmarshalJSON(data []byte) error { type Alias DagStat - aux := &struct { + aux := struct { Cid string `json:"Cid"` *Alias }{ @@ -337,6 +337,9 @@ func (s *DagStatSummary) incrementTotalSize(size uint64) { func (s *DagStatSummary) incrementRedundantSize(size uint64) { s.redundantSize += size } +func (s *DagStatSummary) appendStats(stats *DagStat) { + s.DagStatsArray = append(s.DagStatsArray, stats) +} func (s *DagStatSummary) calculateSummary() { s.Ratio = float32(s.redundantSize) / float32(s.TotalSize) diff --git a/core/commands/dag/stat.go b/core/commands/dag/stat.go index cab3f50d68f..2199097694b 100644 --- a/core/commands/dag/stat.go +++ b/core/commands/dag/stat.go @@ -13,6 +13,10 @@ import ( "github.com/ipfs/kubo/core/commands/e" ) +// TODO cache every cid traversal in a dp cache +// if the cid exists in the cache, don't traverse it, and use the cached result +// to compute the new state + func dagStat(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) if err != nil { @@ -55,7 +59,7 @@ func dagStat(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) if err != nil { return fmt.Errorf("error traversing DAG: %w", err) } - dagStatSummary.DagStatsArray = append(dagStatSummary.DagStatsArray, dagstats) + dagStatSummary.appendStats(dagstats) } dagStatSummary.UniqueBlocks = cidSet.Len() From 620f73c5471f4ecbacb7f79116197be92d43cdb7 Mon Sep 17 00:00:00 2001 From: arthurgavazza Date: Fri, 12 May 2023 13:24:20 -0300 Subject: [PATCH 14/24] fix: add support to progressive information --- core/commands/dag/dag.go | 13 +++++++++++++ core/commands/dag/stat.go | 20 +++++++++++++++++--- test/cli/dag_test.go | 2 +- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index 02df1a0cebf..f4ceeec0ae7 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -290,6 +290,11 @@ func (s *DagStat) String() string { func (s *DagStat) MarshalJSON() ([]byte, error) { type Alias DagStat + /* + We can't rely on cid.Cid.MarshalJSON since it uses the {"/": "..."} + format. To make the output consistent and follow the Kubo API patterns + we use the Cid.String method + */ return json.Marshal(struct { Cid string `json:"Cid"` *Alias @@ -300,6 +305,11 @@ func (s *DagStat) MarshalJSON() ([]byte, error) { } func (s *DagStat) UnmarshalJSON(data []byte) error { + /* + We can't rely on cid.Cid.UnmarshalJSON since it uses the {"/": "..."} + format. To make the output consistent and follow the Kubo API patterns + we use the Cid.Parse method + */ type Alias DagStat aux := struct { Cid string `json:"Cid"` @@ -325,12 +335,15 @@ type DagStatSummary struct { SharedSize uint64 `json:",omitempty"` Ratio float32 `json:",omitempty"` DagStatsArray []*DagStat `json:"DagStats,omitempty"` + Done bool } func (s *DagStatSummary) String() string { return fmt.Sprintf("Total Size: %d\nUnique Blocks: %d\nShared Size: %d\nRatio: %f", s.TotalSize, s.UniqueBlocks, s.SharedSize, s.Ratio) } + + func (s *DagStatSummary) incrementTotalSize(size uint64) { s.TotalSize += size } diff --git a/core/commands/dag/stat.go b/core/commands/dag/stat.go index 2199097694b..9c3daba4291 100644 --- a/core/commands/dag/stat.go +++ b/core/commands/dag/stat.go @@ -3,6 +3,7 @@ package dagcmd import ( "fmt" "io" + "os" "github.com/ipfs/boxo/coreiface/path" mdag "github.com/ipfs/boxo/ipld/merkledag" @@ -18,6 +19,7 @@ import ( // to compute the new state func dagStat(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + progressive := req.Options[progressOptionName].(bool) api, err := cmdenv.GetApi(env, req) if err != nil { return err @@ -40,6 +42,7 @@ func dagStat(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) return err } dagstats := &DagStat{Cid: rp.Cid()} + dagStatSummary.appendStats(dagstats) err = traverse.Traverse(obj, traverse.Options{ DAG: nodeGetter, Order: traverse.DFSPre, @@ -51,6 +54,11 @@ func dagStat(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) } dagStatSummary.incrementRedundantSize(dagstats.Size) cidSet.Add(current.Node.Cid()) + if progressive { + if err := res.Emit(dagStatSummary); err != nil { + return err + } + } return nil }, ErrFunc: nil, @@ -59,12 +67,11 @@ func dagStat(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) if err != nil { return fmt.Errorf("error traversing DAG: %w", err) } - dagStatSummary.appendStats(dagstats) } dagStatSummary.UniqueBlocks = cidSet.Len() dagStatSummary.calculateSummary() - + dagStatSummary.Done = true if err := res.Emit(dagStatSummary); err != nil { return err } @@ -72,7 +79,6 @@ func dagStat(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) } func finishCLIStat(res cmds.Response, re cmds.ResponseEmitter) error { - var dagStats *DagStatSummary for { v, err := res.Next() @@ -85,8 +91,16 @@ func finishCLIStat(res cmds.Response, re cmds.ResponseEmitter) error { switch out := v.(type) { case *DagStatSummary: dagStats = out + if !dagStats.Done { + length := len(dagStats.DagStatsArray) + if length > 0 { + currentStat := dagStats.DagStatsArray[length-1] + fmt.Fprintf(os.Stderr,"CID: %s, Size: %d, NumBlocks: %d\n", currentStat.Cid,currentStat.Size,currentStat.NumBlocks) + } + } default: return e.TypeErr(out, v) + } } return re.Emit(dagStats) diff --git a/test/cli/dag_test.go b/test/cli/dag_test.go index ac7fd7e819d..2b5aaba60ac 100644 --- a/test/cli/dag_test.go +++ b/test/cli/dag_test.go @@ -18,7 +18,7 @@ func TestDag(t *testing.T) { output3 := node.IPFSAddStr("hello world 3") assert.NotEqual(t, "", output) - stat := node.RunIPFS("dag", "stat", output, output2, output3) + stat := node.RunIPFS("dag", "stat","-p",output, output2, output3) str := stat.Stdout.String() err := stat.Stderr.String() assert.NotEqual(t, "", str) From 9146d9586bfe578f283b91b8ce5bef1bbc966d2d Mon Sep 17 00:00:00 2001 From: arthurgavazza Date: Fri, 12 May 2023 14:28:11 -0300 Subject: [PATCH 15/24] fix: fix ci --- core/commands/dag/dag.go | 24 +++++++++++------------- core/commands/dag/stat.go | 4 ++-- test/cli/dag_test.go | 2 +- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index f4ceeec0ae7..7f11b13eaa1 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -290,11 +290,11 @@ func (s *DagStat) String() string { func (s *DagStat) MarshalJSON() ([]byte, error) { type Alias DagStat - /* - We can't rely on cid.Cid.MarshalJSON since it uses the {"/": "..."} - format. To make the output consistent and follow the Kubo API patterns - we use the Cid.String method - */ + /* + We can't rely on cid.Cid.MarshalJSON since it uses the {"/": "..."} + format. To make the output consistent and follow the Kubo API patterns + we use the Cid.String method + */ return json.Marshal(struct { Cid string `json:"Cid"` *Alias @@ -305,11 +305,11 @@ func (s *DagStat) MarshalJSON() ([]byte, error) { } func (s *DagStat) UnmarshalJSON(data []byte) error { - /* - We can't rely on cid.Cid.UnmarshalJSON since it uses the {"/": "..."} - format. To make the output consistent and follow the Kubo API patterns - we use the Cid.Parse method - */ + /* + We can't rely on cid.Cid.UnmarshalJSON since it uses the {"/": "..."} + format. To make the output consistent and follow the Kubo API patterns + we use the Cid.Parse method + */ type Alias DagStat aux := struct { Cid string `json:"Cid"` @@ -342,8 +342,6 @@ func (s *DagStatSummary) String() string { return fmt.Sprintf("Total Size: %d\nUnique Blocks: %d\nShared Size: %d\nRatio: %f", s.TotalSize, s.UniqueBlocks, s.SharedSize, s.Ratio) } - - func (s *DagStatSummary) incrementTotalSize(size uint64) { s.TotalSize += size } @@ -388,7 +386,7 @@ Note: This command skips duplicate blocks in reporting both size and the number csvWriter.Comma = '\t' header := []string{fmt.Sprintf("%-*s", 46, "CID"), fmt.Sprintf("%-15s", "Blocks"), "Size"} if err := csvWriter.Write(header); err != nil { - return err + return err } for _, dagStat := range event.DagStatsArray { numBlocksStr := fmt.Sprint(dagStat.NumBlocks) diff --git a/core/commands/dag/stat.go b/core/commands/dag/stat.go index 9c3daba4291..0cad737b6fd 100644 --- a/core/commands/dag/stat.go +++ b/core/commands/dag/stat.go @@ -95,12 +95,12 @@ func finishCLIStat(res cmds.Response, re cmds.ResponseEmitter) error { length := len(dagStats.DagStatsArray) if length > 0 { currentStat := dagStats.DagStatsArray[length-1] - fmt.Fprintf(os.Stderr,"CID: %s, Size: %d, NumBlocks: %d\n", currentStat.Cid,currentStat.Size,currentStat.NumBlocks) + fmt.Fprintf(os.Stderr, "CID: %s, Size: %d, NumBlocks: %d\n", currentStat.Cid, currentStat.Size, currentStat.NumBlocks) } } default: return e.TypeErr(out, v) - + } } return re.Emit(dagStats) diff --git a/test/cli/dag_test.go b/test/cli/dag_test.go index 2b5aaba60ac..9bb80639141 100644 --- a/test/cli/dag_test.go +++ b/test/cli/dag_test.go @@ -18,7 +18,7 @@ func TestDag(t *testing.T) { output3 := node.IPFSAddStr("hello world 3") assert.NotEqual(t, "", output) - stat := node.RunIPFS("dag", "stat","-p",output, output2, output3) + stat := node.RunIPFS("dag", "stat", "-p", output, output2, output3) str := stat.Stdout.String() err := stat.Stderr.String() assert.NotEqual(t, "", str) From ee82e973a48b68398e714310be79d390bac17b7d Mon Sep 17 00:00:00 2001 From: arthurgavazza Date: Mon, 15 May 2023 23:39:08 -0300 Subject: [PATCH 16/24] test: add a winner test using a fixture --- core/commands/dag/dag.go | 1 - core/commands/dag/stat.go | 4 ++-- test/cli/dag_test.go | 35 ++++++++++++++++++++---------- test/cli/fixtures/TestDagStat.car | Bin 0 -> 402 bytes 4 files changed, 25 insertions(+), 15 deletions(-) create mode 100644 test/cli/fixtures/TestDagStat.car diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index 7f11b13eaa1..f45ba688e22 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -335,7 +335,6 @@ type DagStatSummary struct { SharedSize uint64 `json:",omitempty"` Ratio float32 `json:",omitempty"` DagStatsArray []*DagStat `json:"DagStats,omitempty"` - Done bool } func (s *DagStatSummary) String() string { diff --git a/core/commands/dag/stat.go b/core/commands/dag/stat.go index 0cad737b6fd..afc9f1c83e8 100644 --- a/core/commands/dag/stat.go +++ b/core/commands/dag/stat.go @@ -71,7 +71,7 @@ func dagStat(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) dagStatSummary.UniqueBlocks = cidSet.Len() dagStatSummary.calculateSummary() - dagStatSummary.Done = true + if err := res.Emit(dagStatSummary); err != nil { return err } @@ -91,7 +91,7 @@ func finishCLIStat(res cmds.Response, re cmds.ResponseEmitter) error { switch out := v.(type) { case *DagStatSummary: dagStats = out - if !dagStats.Done { + if dagStats.Ratio == 0 { length := len(dagStats.DagStatsArray) if length > 0 { currentStat := dagStats.DagStatsArray[length-1] diff --git a/test/cli/dag_test.go b/test/cli/dag_test.go index 9bb80639141..9a189ca528d 100644 --- a/test/cli/dag_test.go +++ b/test/cli/dag_test.go @@ -1,28 +1,39 @@ package cli import ( + "os" "testing" "github.com/ipfs/kubo/test/cli/harness" "github.com/stretchr/testify/assert" ) +const ( + fixtureFile = "./fixtures/TestDagStat.car" + node1Cid = "bafyreibmdfd7c5db4kls4ty57zljfhqv36gi43l6txl44pi423wwmeskwy" + node2Cid = "bafyreie3njilzdi4ixumru4nzgecsnjtu7fzfcwhg7e6s4s5i7cnbslvn4" + fixtureCid = "bafyreifrm6uf5o4dsaacuszf35zhibyojlqclabzrms7iak67pf62jygaq" +) + +// The Fixture file represents a dag where 2 nodes of size = 46B each, have a common child of 53B +// when traversing the DAG from the root's children (node1 and node2) we count (46 + 53)x2 bytes (counting redundant bytes) = 198 +// since both nodes share a common child of 53 bytes we actually had to read (46)x2 + 53 = 145 bytes +// we should get a dedup ratio of 198/145 that results in approximatelly 1.3655173 + func TestDag(t *testing.T) { t.Parallel() - t.Run("ipfs add, adds file", func(t *testing.T) { + t.Run("ipfs dag stat", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() - - output := node.IPFSAddStr("hello world") - output2 := node.IPFSAddStr("hello world 2") - output3 := node.IPFSAddStr("hello world 3") - assert.NotEqual(t, "", output) - - stat := node.RunIPFS("dag", "stat", "-p", output, output2, output3) - str := stat.Stdout.String() - err := stat.Stderr.String() - assert.NotEqual(t, "", str) + // Import fixture + r, err := os.Open(fixtureFile) assert.Nil(t, err) - + defer r.Close() + err = node.IPFSDagImport(r, fixtureCid) + assert.Nil(t, err) + stat := node.RunIPFS("dag", "stat", "--progress=false", node1Cid, node2Cid) + str := stat.Stdout.String() + expected := "\nCID \tBlocks \tSize\nbafyreibmdfd7c5db4kls4ty57zljfhqv36gi43l6txl44pi423wwmeskwy\t2 \t53\nbafyreie3njilzdi4ixumru4nzgecsnjtu7fzfcwhg7e6s4s5i7cnbslvn4\t2 \t53\n\nSummary\nTotal Size: 145\nUnique Blocks: 3\nShared Size: 53\nRatio: 1.365517\n\n\n" + assert.Equal(t, expected, str) }) } diff --git a/test/cli/fixtures/TestDagStat.car b/test/cli/fixtures/TestDagStat.car new file mode 100644 index 0000000000000000000000000000000000000000..d8af41b4da086eb89986648843199905ee0fcb79 GIT binary patch literal 402 zcmcColvX9eu(m2rJ>;&SiF4oy?z<) Date: Tue, 16 May 2023 08:31:04 -0300 Subject: [PATCH 17/24] Update test/cli/dag_test.go Co-authored-by: Jorropo --- test/cli/dag_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cli/dag_test.go b/test/cli/dag_test.go index 9a189ca528d..a0b2b7c80df 100644 --- a/test/cli/dag_test.go +++ b/test/cli/dag_test.go @@ -30,7 +30,7 @@ func TestDag(t *testing.T) { assert.Nil(t, err) defer r.Close() err = node.IPFSDagImport(r, fixtureCid) - assert.Nil(t, err) + assert.NoError(t, err) stat := node.RunIPFS("dag", "stat", "--progress=false", node1Cid, node2Cid) str := stat.Stdout.String() expected := "\nCID \tBlocks \tSize\nbafyreibmdfd7c5db4kls4ty57zljfhqv36gi43l6txl44pi423wwmeskwy\t2 \t53\nbafyreie3njilzdi4ixumru4nzgecsnjtu7fzfcwhg7e6s4s5i7cnbslvn4\t2 \t53\n\nSummary\nTotal Size: 145\nUnique Blocks: 3\nShared Size: 53\nRatio: 1.365517\n\n\n" From f1840ca8781603da82b8d17e4e79ce4304066cd2 Mon Sep 17 00:00:00 2001 From: arthurgavazza Date: Sun, 21 May 2023 13:17:01 -0300 Subject: [PATCH 18/24] fix: deduplication calculations and tests --- core/commands/dag/dag.go | 3 +- core/commands/dag/stat.go | 8 +-- test/cli/dag_test.go | 102 ++++++++++++++++++++++++++++++++--- test/cli/testutils/floats.go | 10 ++++ test/sharness/t0053-dag.sh | 35 +----------- 5 files changed, 112 insertions(+), 46 deletions(-) create mode 100644 test/cli/testutils/floats.go diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index f45ba688e22..89aa53f8693 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -383,7 +383,8 @@ Note: This command skips duplicate blocks in reporting both size and the number fmt.Fprintln(w) csvWriter := csv.NewWriter(w) csvWriter.Comma = '\t' - header := []string{fmt.Sprintf("%-*s", 46, "CID"), fmt.Sprintf("%-15s", "Blocks"), "Size"} + cidSpacing := len(event.DagStatsArray[0].Cid.String()) + header := []string{fmt.Sprintf("%-*s", cidSpacing, "CID"), fmt.Sprintf("%-15s", "Blocks"), "Size"} if err := csvWriter.Write(header); err != nil { return err } diff --git a/core/commands/dag/stat.go b/core/commands/dag/stat.go index afc9f1c83e8..a8897c79c80 100644 --- a/core/commands/dag/stat.go +++ b/core/commands/dag/stat.go @@ -47,12 +47,14 @@ func dagStat(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) DAG: nodeGetter, Order: traverse.DFSPre, Func: func(current traverse.State) error { - dagstats.Size += uint64(len(current.Node.RawData())) + fmt.Println("previousDagStatSize:", dagstats.Size) + currentNodeSize := uint64(len(current.Node.RawData())) + dagstats.Size += currentNodeSize dagstats.NumBlocks++ if !cidSet.Has(current.Node.Cid()) { - dagStatSummary.incrementTotalSize(dagstats.Size) + dagStatSummary.incrementTotalSize(currentNodeSize) } - dagStatSummary.incrementRedundantSize(dagstats.Size) + dagStatSummary.incrementRedundantSize(currentNodeSize) cidSet.Add(current.Node.Cid()) if progressive { if err := res.Emit(dagStatSummary); err != nil { diff --git a/test/cli/dag_test.go b/test/cli/dag_test.go index a0b2b7c80df..a85ee8fc99e 100644 --- a/test/cli/dag_test.go +++ b/test/cli/dag_test.go @@ -1,10 +1,13 @@ package cli import ( + "encoding/json" "os" + "strings" "testing" "github.com/ipfs/kubo/test/cli/harness" + "github.com/ipfs/kubo/test/cli/testutils" "github.com/stretchr/testify/assert" ) @@ -15,13 +18,81 @@ const ( fixtureCid = "bafyreifrm6uf5o4dsaacuszf35zhibyojlqclabzrms7iak67pf62jygaq" ) -// The Fixture file represents a dag where 2 nodes of size = 46B each, have a common child of 53B -// when traversing the DAG from the root's children (node1 and node2) we count (46 + 53)x2 bytes (counting redundant bytes) = 198 -// since both nodes share a common child of 53 bytes we actually had to read (46)x2 + 53 = 145 bytes -// we should get a dedup ratio of 198/145 that results in approximatelly 1.3655173 +type DagStat struct { + Cid string `json:"Cid"` + Size int `json:"Size"` + NumBlocks int `json:"NumBlocks"` +} + +type Data struct { + UniqueBlocks int `json:"UniqueBlocks"` + TotalSize int `json:"TotalSize"` + SharedSize int `json:"SharedSize"` + Ratio float64 `json:"Ratio"` + DagStats []DagStat `json:"DagStats"` +} + +func unescapeString(escapedString string) string { + unescapedString := strings.NewReplacer( + "\n", "\\n", + "\t", "\\t", + ).Replace(escapedString) + + return unescapedString +} + + +// The Fixture file represents a dag where 2 nodes of size = 46B each, have a common child of 7B +// when traversing the DAG from the root's children (node1 and node2) we count (46 + 7)x2 bytes (counting redundant bytes) = 106 +// since both nodes share a common child of 7 bytes we actually had to read (46)x2 + 7 = 99 bytes +// we should get a dedup ratio of 106/99 that results in approximatelly 1.0707071 func TestDag(t *testing.T) { t.Parallel() + + t.Run("ipfs dag stat --enc=json", func (t *testing.T){ + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + // Import fixture + r, err := os.Open(fixtureFile) + assert.Nil(t, err) + defer r.Close() + err = node.IPFSDagImport(r, fixtureCid) + assert.NoError(t, err) + stat := node.RunIPFS("dag", "stat", "--progress=false","--enc=json",node1Cid, node2Cid) + var data Data + err = json.Unmarshal(stat.Stdout.Bytes(), &data) + assert.NoError(t,err) + + expectedUniqueBlocks := 3 + expectedSharedSize := 7 + expectedTotalSize := 99 + expectedRatio := float64(expectedSharedSize + expectedTotalSize)/float64(expectedTotalSize) + expectedDagStatsLength := 2 + // Validate UniqueBlocks + assert.Equal(t,expectedUniqueBlocks,data.UniqueBlocks) + assert.Equal(t,expectedSharedSize,data.SharedSize) + assert.Equal(t,expectedTotalSize,data.TotalSize) + assert.Equal(t,testutils.FloatTruncate(expectedRatio,4),testutils.FloatTruncate(data.Ratio,4)) + + // Validate DagStats + assert.Equal(t,expectedDagStatsLength,len(data.DagStats)) + node1Output := data.DagStats[0] + node2Output := data.DagStats[1] + + assert.Equal(t,node1Output.Cid,node1Cid) + assert.Equal(t,node2Output.Cid,node2Cid) + + expectedNode1Size := (expectedTotalSize + expectedSharedSize)/2 + expectedNode2Size := (expectedTotalSize + expectedSharedSize)/2 + assert.Equal(t,expectedNode1Size,node1Output.Size) + assert.Equal(t,expectedNode2Size,node2Output.Size) + + expectedNode1Blocks := 2 + expectedNode2Blocks := 2 + assert.Equal(t,expectedNode1Blocks,node1Output.NumBlocks) + assert.Equal(t,expectedNode2Blocks,node2Output.NumBlocks) + }) t.Run("ipfs dag stat", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() @@ -31,9 +102,24 @@ func TestDag(t *testing.T) { defer r.Close() err = node.IPFSDagImport(r, fixtureCid) assert.NoError(t, err) - stat := node.RunIPFS("dag", "stat", "--progress=false", node1Cid, node2Cid) - str := stat.Stdout.String() - expected := "\nCID \tBlocks \tSize\nbafyreibmdfd7c5db4kls4ty57zljfhqv36gi43l6txl44pi423wwmeskwy\t2 \t53\nbafyreie3njilzdi4ixumru4nzgecsnjtu7fzfcwhg7e6s4s5i7cnbslvn4\t2 \t53\n\nSummary\nTotal Size: 145\nUnique Blocks: 3\nShared Size: 53\nRatio: 1.365517\n\n\n" - assert.Equal(t, expected, str) + stat := node.RunIPFS("dag", "stat", node1Cid, node2Cid) + str := unescapeString(stat.Stdout.String()) + expectedOutput := `CID Blocks Size + bafyreibmdfd7c5db4kls4ty57zljfhqv36gi43l6txl44pi423wwmeskwy 2 53 + bafyreie3njilzdi4ixumru4nzgecsnjtu7fzfcwhg7e6s4s5i7cnbslvn4 2 53 + + Summary + Total Size: 99 + Unique Blocks: 3 + Shared Size: 7 + Ratio: 1.070707 + ` + assert.Equal(t,strings.TrimSpace(expectedOutput),strings.TrimSpace(str)) + + + + }) } + + diff --git a/test/cli/testutils/floats.go b/test/cli/testutils/floats.go new file mode 100644 index 00000000000..8673fa58744 --- /dev/null +++ b/test/cli/testutils/floats.go @@ -0,0 +1,10 @@ +package testutils + + +func FloatTruncate(value float64, decimalPlaces int) float64 { + pow := 1.0 + for i := 0; i < decimalPlaces; i++ { + pow *= 10.0 + } + return float64(int(value*pow)) / pow +} \ No newline at end of file diff --git a/test/sharness/t0053-dag.sh b/test/sharness/t0053-dag.sh index 7514476be6e..21fd2c04f1d 100755 --- a/test/sharness/t0053-dag.sh +++ b/test/sharness/t0053-dag.sh @@ -428,40 +428,7 @@ test_expect_success "'ipfs dag put' check block size" ' test_cmp resolve_data_exp resolve_data ' - test_expect_success "dag stat of simple IPLD object" ' - ipfs dag stat $NESTED_HASH > actual_stat_inner_ipld_obj && - echo "Size: 8, NumBlocks: 1" > exp_stat_inner_ipld_obj && - test_cmp exp_stat_inner_ipld_obj actual_stat_inner_ipld_obj && - ipfs dag stat $HASH > actual_stat_ipld_obj && - echo "Size: 54, NumBlocks: 2" > exp_stat_ipld_obj && - test_cmp exp_stat_ipld_obj actual_stat_ipld_obj - ' - - test_expect_success "dag stat of simple UnixFS object" ' - BASIC_UNIXFS=$(echo "1234" | ipfs add --pin=false -q) && - ipfs dag stat $BASIC_UNIXFS > actual_stat_basic_unixfs && - echo "Size: 13, NumBlocks: 1" > exp_stat_basic_unixfs && - test_cmp exp_stat_basic_unixfs actual_stat_basic_unixfs - ' - - # The multiblock file is just 10000000 copies of the number 1 - # As most of its data is replicated it should have a small number of blocks - test_expect_success "dag stat of multiblock UnixFS object" ' - MULTIBLOCK_UNIXFS=$(printf "1%.0s" {1..10000000} | ipfs add --pin=false -q) && - ipfs dag stat $MULTIBLOCK_UNIXFS > actual_stat_multiblock_unixfs && - echo "Size: 302582, NumBlocks: 3" > exp_stat_multiblock_unixfs && - test_cmp exp_stat_multiblock_unixfs actual_stat_multiblock_unixfs - ' - - test_expect_success "dag stat of directory of UnixFS objects" ' - mkdir -p unixfsdir && - echo "1234" > unixfsdir/small.txt - printf "1%.0s" {1..10000000} > unixfsdir/many1s.txt && - DIRECTORY_UNIXFS=$(ipfs add -r --pin=false -Q unixfsdir) && - ipfs dag stat $DIRECTORY_UNIXFS > actual_stat_directory_unixfs && - echo "Size: 302705, NumBlocks: 5" > exp_stat_directory_unixfs && - test_cmp exp_stat_directory_unixfs actual_stat_directory_unixfs - ' + } # should work offline From 864e91b4ff3c119c67bad91f782dd6bdfc728670 Mon Sep 17 00:00:00 2001 From: arthurgavazza Date: Sun, 21 May 2023 13:19:56 -0300 Subject: [PATCH 19/24] fix: lint --- test/cli/dag_test.go | 70 +++++++++++++++++------------------- test/cli/testutils/floats.go | 3 +- 2 files changed, 33 insertions(+), 40 deletions(-) diff --git a/test/cli/dag_test.go b/test/cli/dag_test.go index a85ee8fc99e..938089e96db 100644 --- a/test/cli/dag_test.go +++ b/test/cli/dag_test.go @@ -19,17 +19,17 @@ const ( ) type DagStat struct { - Cid string `json:"Cid"` - Size int `json:"Size"` - NumBlocks int `json:"NumBlocks"` + Cid string `json:"Cid"` + Size int `json:"Size"` + NumBlocks int `json:"NumBlocks"` } type Data struct { - UniqueBlocks int `json:"UniqueBlocks"` - TotalSize int `json:"TotalSize"` - SharedSize int `json:"SharedSize"` - Ratio float64 `json:"Ratio"` - DagStats []DagStat `json:"DagStats"` + UniqueBlocks int `json:"UniqueBlocks"` + TotalSize int `json:"TotalSize"` + SharedSize int `json:"SharedSize"` + Ratio float64 `json:"Ratio"` + DagStats []DagStat `json:"DagStats"` } func unescapeString(escapedString string) string { @@ -41,7 +41,6 @@ func unescapeString(escapedString string) string { return unescapedString } - // The Fixture file represents a dag where 2 nodes of size = 46B each, have a common child of 7B // when traversing the DAG from the root's children (node1 and node2) we count (46 + 7)x2 bytes (counting redundant bytes) = 106 // since both nodes share a common child of 7 bytes we actually had to read (46)x2 + 7 = 99 bytes @@ -50,7 +49,7 @@ func unescapeString(escapedString string) string { func TestDag(t *testing.T) { t.Parallel() - t.Run("ipfs dag stat --enc=json", func (t *testing.T){ + t.Run("ipfs dag stat --enc=json", func(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() // Import fixture @@ -59,39 +58,39 @@ func TestDag(t *testing.T) { defer r.Close() err = node.IPFSDagImport(r, fixtureCid) assert.NoError(t, err) - stat := node.RunIPFS("dag", "stat", "--progress=false","--enc=json",node1Cid, node2Cid) + stat := node.RunIPFS("dag", "stat", "--progress=false", "--enc=json", node1Cid, node2Cid) var data Data err = json.Unmarshal(stat.Stdout.Bytes(), &data) - assert.NoError(t,err) + assert.NoError(t, err) expectedUniqueBlocks := 3 expectedSharedSize := 7 expectedTotalSize := 99 - expectedRatio := float64(expectedSharedSize + expectedTotalSize)/float64(expectedTotalSize) - expectedDagStatsLength := 2 + expectedRatio := float64(expectedSharedSize+expectedTotalSize) / float64(expectedTotalSize) + expectedDagStatsLength := 2 // Validate UniqueBlocks - assert.Equal(t,expectedUniqueBlocks,data.UniqueBlocks) - assert.Equal(t,expectedSharedSize,data.SharedSize) - assert.Equal(t,expectedTotalSize,data.TotalSize) - assert.Equal(t,testutils.FloatTruncate(expectedRatio,4),testutils.FloatTruncate(data.Ratio,4)) - + assert.Equal(t, expectedUniqueBlocks, data.UniqueBlocks) + assert.Equal(t, expectedSharedSize, data.SharedSize) + assert.Equal(t, expectedTotalSize, data.TotalSize) + assert.Equal(t, testutils.FloatTruncate(expectedRatio, 4), testutils.FloatTruncate(data.Ratio, 4)) + // Validate DagStats - assert.Equal(t,expectedDagStatsLength,len(data.DagStats)) - node1Output := data.DagStats[0] + assert.Equal(t, expectedDagStatsLength, len(data.DagStats)) + node1Output := data.DagStats[0] node2Output := data.DagStats[1] - - assert.Equal(t,node1Output.Cid,node1Cid) - assert.Equal(t,node2Output.Cid,node2Cid) - - expectedNode1Size := (expectedTotalSize + expectedSharedSize)/2 - expectedNode2Size := (expectedTotalSize + expectedSharedSize)/2 - assert.Equal(t,expectedNode1Size,node1Output.Size) - assert.Equal(t,expectedNode2Size,node2Output.Size) - + + assert.Equal(t, node1Output.Cid, node1Cid) + assert.Equal(t, node2Output.Cid, node2Cid) + + expectedNode1Size := (expectedTotalSize + expectedSharedSize) / 2 + expectedNode2Size := (expectedTotalSize + expectedSharedSize) / 2 + assert.Equal(t, expectedNode1Size, node1Output.Size) + assert.Equal(t, expectedNode2Size, node2Output.Size) + expectedNode1Blocks := 2 expectedNode2Blocks := 2 - assert.Equal(t,expectedNode1Blocks,node1Output.NumBlocks) - assert.Equal(t,expectedNode2Blocks,node2Output.NumBlocks) + assert.Equal(t, expectedNode1Blocks, node1Output.NumBlocks) + assert.Equal(t, expectedNode2Blocks, node2Output.NumBlocks) }) t.Run("ipfs dag stat", func(t *testing.T) { t.Parallel() @@ -114,12 +113,7 @@ func TestDag(t *testing.T) { Shared Size: 7 Ratio: 1.070707 ` - assert.Equal(t,strings.TrimSpace(expectedOutput),strings.TrimSpace(str)) - - + assert.Equal(t, strings.TrimSpace(expectedOutput), strings.TrimSpace(str)) - }) } - - diff --git a/test/cli/testutils/floats.go b/test/cli/testutils/floats.go index 8673fa58744..cecb7d93498 100644 --- a/test/cli/testutils/floats.go +++ b/test/cli/testutils/floats.go @@ -1,10 +1,9 @@ package testutils - func FloatTruncate(value float64, decimalPlaces int) float64 { pow := 1.0 for i := 0; i < decimalPlaces; i++ { pow *= 10.0 } return float64(int(value*pow)) / pow -} \ No newline at end of file +} From b689c10a0c8f9338841ae00430932aca2bc835ad Mon Sep 17 00:00:00 2001 From: arthurgavazza Date: Sun, 21 May 2023 13:31:31 -0300 Subject: [PATCH 20/24] fix: CI --- go.mod | 2 -- go.sum | 1 - test/cli/dag_test.go | 33 --------------------------------- 3 files changed, 36 deletions(-) diff --git a/go.mod b/go.mod index 68683acb484..a756f0f3206 100644 --- a/go.mod +++ b/go.mod @@ -86,8 +86,6 @@ require ( golang.org/x/sys v0.8.0 ) -require github.com/ipfs/go-merkledag v0.10.0 - require ( github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect github.com/Kubuxu/go-os-helper v0.0.1 // indirect diff --git a/go.sum b/go.sum index 7dceae47652..af7ec2fc01a 100644 --- a/go.sum +++ b/go.sum @@ -444,7 +444,6 @@ github.com/ipfs/go-log/v2 v2.3.0/go.mod h1:QqGoj30OTpnKaG/LKTGTxoP2mmQtjVMEnK72g github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= github.com/ipfs/go-merkledag v0.10.0 h1:IUQhj/kzTZfam4e+LnaEpoiZ9vZF6ldimVlby+6OXL4= -github.com/ipfs/go-merkledag v0.10.0/go.mod h1:zkVav8KiYlmbzUzNM6kENzkdP5+qR7+2mCwxkQ6GIj8= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= github.com/ipfs/go-metrics-prometheus v0.0.2 h1:9i2iljLg12S78OhC6UAiXi176xvQGiZaGVF1CUVdE+s= diff --git a/test/cli/dag_test.go b/test/cli/dag_test.go index 938089e96db..92391f31c18 100644 --- a/test/cli/dag_test.go +++ b/test/cli/dag_test.go @@ -3,7 +3,6 @@ package cli import ( "encoding/json" "os" - "strings" "testing" "github.com/ipfs/kubo/test/cli/harness" @@ -32,15 +31,6 @@ type Data struct { DagStats []DagStat `json:"DagStats"` } -func unescapeString(escapedString string) string { - unescapedString := strings.NewReplacer( - "\n", "\\n", - "\t", "\\t", - ).Replace(escapedString) - - return unescapedString -} - // The Fixture file represents a dag where 2 nodes of size = 46B each, have a common child of 7B // when traversing the DAG from the root's children (node1 and node2) we count (46 + 7)x2 bytes (counting redundant bytes) = 106 // since both nodes share a common child of 7 bytes we actually had to read (46)x2 + 7 = 99 bytes @@ -92,28 +82,5 @@ func TestDag(t *testing.T) { assert.Equal(t, expectedNode1Blocks, node1Output.NumBlocks) assert.Equal(t, expectedNode2Blocks, node2Output.NumBlocks) }) - t.Run("ipfs dag stat", func(t *testing.T) { - t.Parallel() - node := harness.NewT(t).NewNode().Init().StartDaemon() - // Import fixture - r, err := os.Open(fixtureFile) - assert.Nil(t, err) - defer r.Close() - err = node.IPFSDagImport(r, fixtureCid) - assert.NoError(t, err) - stat := node.RunIPFS("dag", "stat", node1Cid, node2Cid) - str := unescapeString(stat.Stdout.String()) - expectedOutput := `CID Blocks Size - bafyreibmdfd7c5db4kls4ty57zljfhqv36gi43l6txl44pi423wwmeskwy 2 53 - bafyreie3njilzdi4ixumru4nzgecsnjtu7fzfcwhg7e6s4s5i7cnbslvn4 2 53 - - Summary - Total Size: 99 - Unique Blocks: 3 - Shared Size: 7 - Ratio: 1.070707 - ` - assert.Equal(t, strings.TrimSpace(expectedOutput), strings.TrimSpace(str)) - }) } From f2051db127fdc6fcfdb85bdeb8593fba2172f239 Mon Sep 17 00:00:00 2001 From: arthurgavazza Date: Sun, 4 Jun 2023 16:20:57 -0300 Subject: [PATCH 21/24] test: add text output tests --- test/cli/dag_test.go | 30 ++++++++++++++++--- .../fixtures/TestDagStatExpectedOutput.txt | 12 ++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) create mode 100644 test/cli/fixtures/TestDagStatExpectedOutput.txt diff --git a/test/cli/dag_test.go b/test/cli/dag_test.go index 92391f31c18..7aa8c55a6ee 100644 --- a/test/cli/dag_test.go +++ b/test/cli/dag_test.go @@ -11,10 +11,11 @@ import ( ) const ( - fixtureFile = "./fixtures/TestDagStat.car" - node1Cid = "bafyreibmdfd7c5db4kls4ty57zljfhqv36gi43l6txl44pi423wwmeskwy" - node2Cid = "bafyreie3njilzdi4ixumru4nzgecsnjtu7fzfcwhg7e6s4s5i7cnbslvn4" - fixtureCid = "bafyreifrm6uf5o4dsaacuszf35zhibyojlqclabzrms7iak67pf62jygaq" + fixtureFile = "./fixtures/TestDagStat.car" + textOutputPath = "./fixtures/TestDagStatExpectedOutput.txt" + node1Cid = "bafyreibmdfd7c5db4kls4ty57zljfhqv36gi43l6txl44pi423wwmeskwy" + node2Cid = "bafyreie3njilzdi4ixumru4nzgecsnjtu7fzfcwhg7e6s4s5i7cnbslvn4" + fixtureCid = "bafyreifrm6uf5o4dsaacuszf35zhibyojlqclabzrms7iak67pf62jygaq" ) type DagStat struct { @@ -83,4 +84,25 @@ func TestDag(t *testing.T) { assert.Equal(t, expectedNode2Blocks, node2Output.NumBlocks) }) + t.Run("ipfs dag stat", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + r, err := os.Open(fixtureFile) + assert.NoError(t, err) + defer r.Close() + f, err := os.Open(textOutputPath) + assert.NoError(t, err) + defer f.Close() + fileInfo, err := f.Stat() + assert.NoError(t, err) + fileSize := fileInfo.Size() + content := make([]byte, fileSize) + _, err = f.Read(content) + assert.NoError(t, err) + err = node.IPFSDagImport(r, fixtureCid) + assert.NoError(t, err) + stat := node.RunIPFS("dag", "stat", "--progress=false", node1Cid, node2Cid) + assert.Equal(t, content, stat.Stdout.Bytes()) + }) + } diff --git a/test/cli/fixtures/TestDagStatExpectedOutput.txt b/test/cli/fixtures/TestDagStatExpectedOutput.txt new file mode 100644 index 00000000000..9e709f4a215 --- /dev/null +++ b/test/cli/fixtures/TestDagStatExpectedOutput.txt @@ -0,0 +1,12 @@ + +CID Blocks Size +bafyreibmdfd7c5db4kls4ty57zljfhqv36gi43l6txl44pi423wwmeskwy 2 53 +bafyreie3njilzdi4ixumru4nzgecsnjtu7fzfcwhg7e6s4s5i7cnbslvn4 2 53 + +Summary +Total Size: 99 +Unique Blocks: 3 +Shared Size: 7 +Ratio: 1.070707 + + From 08f837cf8812305bce37bec0e16414e7336a031d Mon Sep 17 00:00:00 2001 From: Jorropo Date: Mon, 5 Jun 2023 12:34:08 +0200 Subject: [PATCH 22/24] Update core/commands/dag/dag.go Co-authored-by: Henrique Dias --- core/commands/dag/dag.go | 1 - 1 file changed, 1 deletion(-) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index 89aa53f8693..7d21eb0710f 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -277,7 +277,6 @@ CAR file follows the CARv1 format: https://ipld.io/specs/transport/car/carv1/ } // DagStat is a dag stat command response - type DagStat struct { Cid cid.Cid `json:",omitempty"` Size uint64 `json:",omitempty"` From 538f5bbbcaf55bb0a20c36159519042a8ae75734 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Mon, 5 Jun 2023 12:47:57 +0200 Subject: [PATCH 23/24] Update test/cli/dag_test.go --- test/cli/dag_test.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/test/cli/dag_test.go b/test/cli/dag_test.go index 7aa8c55a6ee..98b9234c08e 100644 --- a/test/cli/dag_test.go +++ b/test/cli/dag_test.go @@ -93,11 +93,7 @@ func TestDag(t *testing.T) { f, err := os.Open(textOutputPath) assert.NoError(t, err) defer f.Close() - fileInfo, err := f.Stat() - assert.NoError(t, err) - fileSize := fileInfo.Size() - content := make([]byte, fileSize) - _, err = f.Read(content) + content, err := io.ReadAll(f) assert.NoError(t, err) err = node.IPFSDagImport(r, fixtureCid) assert.NoError(t, err) From 336a98ce844a5aa4c9f42eddd4d7f5cd20710ad0 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Mon, 5 Jun 2023 14:05:23 +0200 Subject: [PATCH 24/24] FIXUP --- test/cli/dag_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test/cli/dag_test.go b/test/cli/dag_test.go index 98b9234c08e..edcacffaefa 100644 --- a/test/cli/dag_test.go +++ b/test/cli/dag_test.go @@ -2,6 +2,7 @@ package cli import ( "encoding/json" + "io" "os" "testing"