From 6b45c927693473a8de41a7afd348e6b74927f2af Mon Sep 17 00:00:00 2001 From: Lucas Molas Date: Mon, 2 May 2022 13:09:28 -0300 Subject: [PATCH] feat(cmds/add): --to-files option as files cp --- core/commands/add.go | 70 ++++++++++++++++++++++++++++++++++++++++-- core/commands/files.go | 29 +++++++++++++---- 2 files changed, 90 insertions(+), 9 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index c2d73f8deee..04ed5fe3506 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -13,6 +13,8 @@ import ( "github.com/cheggaaa/pb" cmds "github.com/ipfs/go-ipfs-cmds" files "github.com/ipfs/go-ipfs-files" + ipld "github.com/ipfs/go-ipld-format" + mfs "github.com/ipfs/go-mfs" coreiface "github.com/ipfs/interface-go-ipfs-core" "github.com/ipfs/interface-go-ipfs-core/options" mh "github.com/multiformats/go-multihash" @@ -45,6 +47,7 @@ const ( hashOptionName = "hash" inlineOptionName = "inline" inlineLimitOptionName = "inline-limit" + toFilesOptionName = "to-files" ) const adderOutChanSize = 8 @@ -79,6 +82,13 @@ You can now refer to the added file in a gateway, like so: /ipfs/QmaG4FuMqEBnQNn3C8XJ5bpW8kLs7zq2ZXgHptJHbKDDVx/example.jpg +Files imported with 'ipfs add' are protected from GC (implicit '--pin=true'), +but it is up to you to remember the returned CID to get the data back later. + +Passing '--to-files' creates a reference in Files API (MFS), making it easier +to find it in the future. See 'ipfs files --help' to learn more about using MFS +for keeping track of added files and directories. + The chunker option, '-s', specifies the chunking strategy that dictates how to break files into blocks. Blocks with same content can be deduplicated. Different chunking strategies will produce different @@ -132,7 +142,6 @@ only-hash, and progress/status related flags) will change the final hash. cmds.BoolOption(onlyHashOptionName, "n", "Only chunk and hash - do not write to disk."), cmds.BoolOption(wrapOptionName, "w", "Wrap files with a directory object."), cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm, size-[bytes], rabin-[min]-[avg]-[max] or buzhash").WithDefault("size-262144"), - cmds.BoolOption(pinOptionName, "Pin this object when adding.").WithDefault(true), cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes."), cmds.BoolOption(noCopyOptionName, "Add the file using filestore. Implies raw-leaves. (experimental)"), cmds.BoolOption(fstoreCacheOptionName, "Check the filestore for pre-existing blocks. (experimental)"), @@ -140,6 +149,8 @@ only-hash, and progress/status related flags) will change the final hash. cmds.StringOption(hashOptionName, "Hash function to use. Implies CIDv1 if not sha2-256. (experimental)").WithDefault("sha2-256"), cmds.BoolOption(inlineOptionName, "Inline small blocks into CIDs. (experimental)"), cmds.IntOption(inlineLimitOptionName, "Maximum block size to inline. (experimental)").WithDefault(32), + cmds.BoolOption(pinOptionName, "Pin locally to protect added files from garbage collection.").WithDefault(true), + cmds.StringOption(toFilesOptionName, "Add reference to Files API (MFS) at the provided path."), }, PreRun: func(req *cmds.Request, env cmds.Environment) error { quiet, _ := req.Options[quietOptionName].(bool) @@ -180,10 +191,11 @@ only-hash, and progress/status related flags) will change the final hash. hashFunStr, _ := req.Options[hashOptionName].(string) inline, _ := req.Options[inlineOptionName].(bool) inlineLimit, _ := req.Options[inlineLimitOptionName].(int) + toFilesStr, toFilesSet := req.Options[toFilesOptionName].(string) hashFunCode, ok := mh.Names[strings.ToLower(hashFunStr)] if !ok { - return fmt.Errorf("unrecognized hash function: %s", strings.ToLower(hashFunStr)) + return fmt.Errorf("unrecognized hash function: %q", strings.ToLower(hashFunStr)) } enc, err := cmdenv.GetCidEncoder(req) @@ -198,6 +210,42 @@ only-hash, and progress/status related flags) will change the final hash. }) } + ipfsNode, err := cmdenv.GetNode(env) + if err != nil { + return err + } + toFilesDst := "" + if toFilesSet { + dirIterator := toadd.Entries() + if !dirIterator.Next() { + if dirIterator.Err() != nil { + return dirIterator.Err() + } + return fmt.Errorf("%s: to-no file entry to copy to MFS path %q", toFilesOptionName, toFilesStr) + } + srcName := dirIterator.Name() + if dirIterator.Next() { + return fmt.Errorf("%s: more than one entry to copy to MFS path %q: %q and %q", toFilesOptionName, toFilesStr, srcName, dirIterator.Name()) + } + + if toFilesStr == "" { + toFilesStr = "/" + } + toFilesDst, err = checkPath(toFilesStr) + if err != nil { + return fmt.Errorf("%s: %w", toFilesOptionName, err) + } + + if toFilesDst[len(toFilesDst)-1] == '/' { + toFilesDst += path.Base(srcName) + } + + _, err = mfs.Lookup(ipfsNode.FilesRoot, path.Dir(toFilesDst)) + if err != nil { + return fmt.Errorf("%s: MFS destination directory %q does not exist: %w", toFilesOptionName, path.Dir(toFilesDst), err) + } + } + opts := []options.UnixfsAddOption{ options.Unixfs.Hash(hashFunCode), @@ -240,7 +288,23 @@ only-hash, and progress/status related flags) will change the final hash. go func() { var err error defer close(events) - _, err = api.Unixfs().Add(req.Context, addit.Node(), opts...) + pathAdded, err := api.Unixfs().Add(req.Context, addit.Node(), opts...) + if err != nil { + errCh <- err + return + } + if toFilesSet { + var nodeAdded ipld.Node + nodeAdded, err = api.Dag().Get(req.Context, pathAdded.Cid()) + if err != nil { + errCh <- err + return + } + err = mfs.PutNode(ipfsNode.FilesRoot, toFilesDst, nodeAdded) + if err != nil { + err = fmt.Errorf("%s: cannot put node in path %s: %s", toFilesOptionName, toFilesDst, err) + } + } errCh <- err }() diff --git a/core/commands/files.go b/core/commands/files.go index 95432ef6152..594d4f3936f 100644 --- a/core/commands/files.go +++ b/core/commands/files.go @@ -581,7 +581,7 @@ const ( var filesReadCmd = &cmds.Command{ Helptext: cmds.HelpText{ - Tagline: "Read a file in a given MFS.", + Tagline: "Read a file from MFS.", ShortDescription: ` Read a specified number of bytes from a file at a given offset. By default, it will read the entire file similar to the Unix cat. @@ -724,11 +724,16 @@ const ( var filesWriteCmd = &cmds.Command{ Helptext: cmds.HelpText{ - Tagline: "Write to a mutable file in a given filesystem.", + Tagline: "Append to (modify) a file in MFS.", ShortDescription: ` -Write data to a file in a given filesystem. This command allows you to specify -a beginning offset to write to. The entire length of the input will be -written. +A low-level MFS command that allows you to append data to a file. If you want +to add a file without modifying an existing one, use 'ipfs add --to-files' +instead. +`, + LongDescription: ` +A low-level MFS command that allows you to append data at the end of a file, or +specify a beginning offset within a file to write to. The entire length of the +input will be written. If the '--create' option is specified, the file will be created if it does not exist. Nonexistent intermediate directories will not be created unless the @@ -755,6 +760,18 @@ WARNING: Usage of the '--flush=false' option does not guarantee data durability until the tree has been flushed. This can be accomplished by running 'ipfs files stat' on the file or any of its ancestors. + +WARNING: + +The CID produced by 'files write' will be different from 'ipfs add' because +'ipfs file write' creates a trickle-dag optimized for append-only operations +See '--trickle' in 'ipfs add --help' for more information. + +If you want to add a file without modifying an existing one, +use 'ipfs add --to-files' instead. Below will add to MFS at /myfs/file.txt: + + ipfs add --to-files /myfs/ file.txt + `, }, Arguments: []cmds.Argument{ @@ -1019,7 +1036,7 @@ func updatePath(rt *mfs.Root, pth string, builder cid.Builder) error { var filesRmCmd = &cmds.Command{ Helptext: cmds.HelpText{ - Tagline: "Remove a file.", + Tagline: "Remove a file from MFS.", ShortDescription: ` Remove files or directories.