test: utility command for random dense car files

cmd/boostx/main.go

@@ -33,6 +33,7 @@ func main() {
 		Commands: []*cli.Command{
 			commpCmd,
 			generatecarCmd,
+			generateRandCar,
 			marketAddCmd,
 			marketWithdrawCmd,
 			statsCmd,

cmd/boostx/utils_cmd.go

@@ -5,12 +5,15 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path"
 	"path/filepath"
+	"time"
 
 	clinode "github.com/filecoin-project/boost/cli/node"
 	"github.com/filecoin-project/boost/cmd"
 	"github.com/filecoin-project/boost/node"
 	"github.com/filecoin-project/boost/node/config"
+	"github.com/filecoin-project/boost/testutil"
 	"github.com/filecoin-project/go-commp-utils/writer"
 	"github.com/filecoin-project/go-fil-markets/stores"
 	"github.com/filecoin-project/go-state-types/abi"
@@ -237,6 +240,72 @@ var commpCmd = &cli.Command{
 	},
 }
 
+var generateRandCar = &cli.Command{
+	Name:      "generate-rand-car",
+	Usage:     "creates a randomly generated dense car",
+	ArgsUsage: "<outputPath>",
+	Before:    before,
+	Flags: []cli.Flag{
+		&cli.IntFlag{
+			Name:    "size",
+			Aliases: []string{"s"},
+			Usage:   "The size of the data to turn into a car",
+			Value:   8000000,
+		},
+		&cli.IntFlag{
+			Name:    "chunksize",
+			Aliases: []string{"c"},
+			Value:   512,
+			Usage:   "Size of chunking that should occur",
+		},
+		&cli.IntFlag{
+			Name:    "maxlinks",
+			Aliases: []string{"l"},
+			Value:   8,
+			Usage:   "Max number of leaves per level",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if cctx.Args().Len() < 1 {
+			return fmt.Errorf("usage: generate-car <outputPath> -size -chunksize -maxleaves")
+		}
+
+		outPath := cctx.Args().Get(0)
+		size := cctx.Int("size")
+		cs := cctx.Int64("chunksize")
+		ml := cctx.Int("maxlinks")
+
+		rf, err := testutil.CreateRandomFile(outPath, int(time.Now().Unix()), size)
+		if err != nil {
+			return err
+		}
+
+		root, cn, err := testutil.CreateDenseCARv2With(outPath, rf, cs, ml)
+		if err != nil {
+			return err
+		}
+
+		err = os.Remove(rf)
+		if err != nil {
+			return err
+		}
+
+		encoder := cidenc.Encoder{Base: multibase.MustNewEncoder(multibase.Base32)}
+		rn := encoder.Encode(root)
+		base := path.Dir(cn)
+		np := path.Join(base, rn+".car")
+
+		err = os.Rename(cn, np)
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("Payload CID: %s, written to: %s\n", rn, np)
+
+		return nil
+	},
+}
+
 var generatecarCmd = &cli.Command{
 	Name:      "generate-car",
 	Usage:     "",

testutil/car.go

@@ -61,13 +61,19 @@ func CreateRandomFile(dir string, rseed, size int) (string, error) {
 	return file.Name(), nil
 }
 
+func CreateDenseCARv2(dir, src string) (cid.Cid, string, error) {
+	cs := int64(unixfsChunkSize)
+	maxlinks := unixfsLinksPerLevel
+	return CreateDenseCARv2With(dir, src, cs, maxlinks)
+}
+
 // CreateDenseCARv2 generates a "dense" UnixFS CARv2 from the supplied ordinary file.
 // A dense UnixFS CARv2 is one storing leaf data. Contrast to CreateRefCARv2.
-func CreateDenseCARv2(dir, src string) (cid.Cid, string, error) {
+func CreateDenseCARv2With(dir, src string, chunksize int64, maxlinks int) (cid.Cid, string, error) {
 	bs := bstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore()))
 	dagSvc := merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs)))
 
-	root, err := WriteUnixfsDAGTo(src, dagSvc)
+	root, err := WriteUnixfsDAGTo(src, dagSvc, chunksize, maxlinks)
 	if err != nil {
 		return cid.Undef, "", err
 	}
@@ -90,7 +96,7 @@ func CreateDenseCARv2(dir, src string) (cid.Cid, string, error) {
 
 	dagSvc = merkledag.NewDAGService(blockservice.New(rw, offline.Exchange(rw)))
 
-	root2, err := WriteUnixfsDAGTo(src, dagSvc)
+	root2, err := WriteUnixfsDAGTo(src, dagSvc, chunksize, maxlinks)
 	if err != nil {
 		return cid.Undef, "", err
 	}
@@ -107,7 +113,7 @@ func CreateDenseCARv2(dir, src string) (cid.Cid, string, error) {
 	return root, out.Name(), nil
 }
 
-func WriteUnixfsDAGTo(path string, into ipldformat.DAGService) (cid.Cid, error) {
+func WriteUnixfsDAGTo(path string, into ipldformat.DAGService, chunksize int64, maxlinks int) (cid.Cid, error) {
 	file, err := os.Open(path)
 	if err != nil {
 		return cid.Undef, err
@@ -137,7 +143,7 @@ func WriteUnixfsDAGTo(path string, into ipldformat.DAGService) (cid.Cid, error)
 
 	bufferedDS := ipldformat.NewBufferedDAG(context.Background(), into)
 	params := ihelper.DagBuilderParams{
-		Maxlinks:  unixfsLinksPerLevel,
+		Maxlinks:  maxlinks,
 		RawLeaves: true,
 		// NOTE: InlineBuilder not recommended, we are using this to test identity CIDs
 		CidBuilder: cidutil.InlineBuilder{
@@ -148,7 +154,7 @@ func WriteUnixfsDAGTo(path string, into ipldformat.DAGService) (cid.Cid, error)
 		NoCopy:  true,
 	}
 
-	db, err := params.New(chunk.NewSizeSplitter(rpf, int64(unixfsChunkSize)))
+	db, err := params.New(chunk.NewSizeSplitter(rpf, chunksize))
 	if err != nil {
 		return cid.Undef, err
 	}