From 04e7e9502e09959171c31bb880255ddda17ed848 Mon Sep 17 00:00:00 2001 From: Lucas Molas Date: Fri, 11 Mar 2022 19:23:11 -0300 Subject: [PATCH] feat(cmds): extend block size check for dag|block put (#8751) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(cmds): extend block size check for dag|block put * feat(cmds): block size check for dag import * style: dag-pb → UnixFS, 1MB → 1MiB Co-authored-by: Marcin Rataj --- core/commands/block.go | 6 +++ core/commands/cmdutils/utils.go | 51 ++++++++++++++++++++ core/commands/dag/dag.go | 3 ++ core/commands/dag/import.go | 4 ++ core/commands/dag/put.go | 5 ++ core/commands/get.go | 2 +- core/commands/object/patch.go | 43 +++-------------- test/sharness/t0050-block.sh | 14 ++++++ test/sharness/t0051-object.sh | 4 +- test/sharness/t0053-dag.sh | 14 ++++++ test/sharness/t0054-dag-car-import-export.sh | 13 +++++ 11 files changed, 120 insertions(+), 39 deletions(-) create mode 100644 core/commands/cmdutils/utils.go diff --git a/core/commands/block.go b/core/commands/block.go index a06bfe0683e..4cfe1fc6ef9 100644 --- a/core/commands/block.go +++ b/core/commands/block.go @@ -10,6 +10,7 @@ import ( util "github.com/ipfs/go-ipfs/blocks/blockstoreutil" cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv" + "github.com/ipfs/go-ipfs/core/commands/cmdutils" cmds "github.com/ipfs/go-ipfs-cmds" options "github.com/ipfs/interface-go-ipfs-core/options" @@ -138,6 +139,7 @@ other than 'sha2-256' or format to anything other than 'v0' will result in CIDv1 cmds.StringOption(mhtypeOptionName, "multihash hash function").WithDefault("sha2-256"), cmds.IntOption(mhlenOptionName, "multihash hash length").WithDefault(-1), cmds.BoolOption(pinOptionName, "pin added blocks recursively").WithDefault(false), + cmdutils.AllowBigBlockOption, }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) @@ -182,6 +184,10 @@ other than 'sha2-256' or format to anything other than 'v0' will result in CIDv1 return err } + if err := cmdutils.CheckBlockSize(req, uint64(p.Size())); err != nil { + return err + } + err = res.Emit(&BlockStat{ Key: p.Path().Cid().String(), Size: p.Size(), diff --git a/core/commands/cmdutils/utils.go b/core/commands/cmdutils/utils.go new file mode 100644 index 00000000000..ebbbca64e46 --- /dev/null +++ b/core/commands/cmdutils/utils.go @@ -0,0 +1,51 @@ +package cmdutils + +import ( + "fmt" + + cmds "github.com/ipfs/go-ipfs-cmds" + + "github.com/ipfs/go-cid" + coreiface "github.com/ipfs/interface-go-ipfs-core" +) + +const ( + AllowBigBlockOptionName = "allow-big-block" + SoftBlockLimit = 1024 * 1024 // https://github.com/ipfs/go-ipfs/issues/7421#issuecomment-910833499 +) + +var AllowBigBlockOption cmds.Option + +func init() { + AllowBigBlockOption = cmds.BoolOption(AllowBigBlockOptionName, "Disable block size check and allow creation of blocks bigger than 1MiB. WARNING: such blocks won't be transferable over the standard bitswap.").WithDefault(false) +} + +func CheckCIDSize(req *cmds.Request, c cid.Cid, dagAPI coreiface.APIDagService) error { + n, err := dagAPI.Get(req.Context, c) + if err != nil { + return fmt.Errorf("CheckCIDSize: getting dag: %w", err) + } + + nodeSize, err := n.Size() + if err != nil { + return fmt.Errorf("CheckCIDSize: getting node size: %w", err) + } + + return CheckBlockSize(req, nodeSize) +} + +func CheckBlockSize(req *cmds.Request, size uint64) error { + allowAnyBlockSize, _ := req.Options[AllowBigBlockOptionName].(bool) + if allowAnyBlockSize { + return nil + } + + // We do not allow producing blocks bigger than 1 MiB to avoid errors + // when transmitting them over BitSwap. The 1 MiB constant is an + // unenforced and undeclared rule of thumb hard-coded here. + if size > SoftBlockLimit { + return fmt.Errorf("produced block is over 1MiB: big blocks can't be exchanged with other peers. consider using UnixFS for automatic chunking of bigger files, or pass --allow-big-block to override") + } + return nil + +} diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index cbaf7f93fe1..6d2e480b284 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -5,6 +5,7 @@ import ( "io" "github.com/ipfs/go-ipfs/core/commands/cmdenv" + "github.com/ipfs/go-ipfs/core/commands/cmdutils" cid "github.com/ipfs/go-cid" cidenc "github.com/ipfs/go-cidutil/cidenc" @@ -88,6 +89,7 @@ into an object of the specified format. cmds.StringOption("input-codec", "Codec that the input object is encoded in").WithDefault("dag-json"), cmds.BoolOption("pin", "Pin this object when adding."), cmds.StringOption("hash", "Hash function to use").WithDefault("sha2-256"), + cmdutils.AllowBigBlockOption, }, Run: dagPut, Type: OutputObject{}, @@ -205,6 +207,7 @@ Maximum supported CAR version: 1 cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing.").WithDefault(true), cmds.BoolOption(silentOptionName, "No output."), cmds.BoolOption(statsOptionName, "Output stats."), + cmdutils.AllowBigBlockOption, }, Type: CarImportOutput{}, Run: dagImport, diff --git a/core/commands/dag/import.go b/core/commands/dag/import.go index e206652ce36..476b986c825 100644 --- a/core/commands/dag/import.go +++ b/core/commands/dag/import.go @@ -8,6 +8,7 @@ import ( cid "github.com/ipfs/go-cid" files "github.com/ipfs/go-ipfs-files" "github.com/ipfs/go-ipfs/core/commands/cmdenv" + "github.com/ipfs/go-ipfs/core/commands/cmdutils" ipld "github.com/ipfs/go-ipld-format" iface "github.com/ipfs/interface-go-ipfs-core" "github.com/ipfs/interface-go-ipfs-core/options" @@ -180,6 +181,9 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI, } else if block == nil { break } + if err := cmdutils.CheckBlockSize(req, uint64(len(block.RawData()))); err != nil { + return err + } // the double-decode is suboptimal, but we need it for batching nd, err := ipld.Decode(block) diff --git a/core/commands/dag/put.go b/core/commands/dag/put.go index 0bb7fd2ae50..e741f11124d 100644 --- a/core/commands/dag/put.go +++ b/core/commands/dag/put.go @@ -7,6 +7,7 @@ import ( blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "github.com/ipfs/go-ipfs/core/commands/cmdenv" + "github.com/ipfs/go-ipfs/core/commands/cmdutils" ipldlegacy "github.com/ipfs/go-ipld-legacy" "github.com/ipld/go-ipld-prime/multicodec" basicnode "github.com/ipld/go-ipld-prime/node/basic" @@ -102,6 +103,10 @@ func dagPut(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e Node: n, } + if err := cmdutils.CheckBlockSize(req, uint64(bd.Len())); err != nil { + return err + } + if err := b.Add(req.Context, &ln); err != nil { return err } diff --git a/core/commands/get.go b/core/commands/get.go index b7826cf30fb..65ab46aa57c 100644 --- a/core/commands/get.go +++ b/core/commands/get.go @@ -257,7 +257,7 @@ func getCompressOptions(req *cmds.Request) (int, error) { return cmplvl, nil } -// DefaultBufSize is the buffer size for gets. for now, 1MB, which is ~4 blocks. +// DefaultBufSize is the buffer size for gets. for now, 1MiB, which is ~4 blocks. // TODO: does this need to be configurable? var DefaultBufSize = 1048576 diff --git a/core/commands/object/patch.go b/core/commands/object/patch.go index 78f92703254..ab76ebb23f4 100644 --- a/core/commands/object/patch.go +++ b/core/commands/object/patch.go @@ -4,20 +4,14 @@ import ( "fmt" "io" - "github.com/ipfs/go-cid" cmds "github.com/ipfs/go-ipfs-cmds" "github.com/ipfs/go-ipfs/core/commands/cmdenv" - coreiface "github.com/ipfs/interface-go-ipfs-core" + "github.com/ipfs/go-ipfs/core/commands/cmdutils" "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" ) -const ( - softBlockLimit = 1024 * 1024 // https://github.com/ipfs/go-ipfs/issues/7421#issuecomment-910833499 - allowBigBlock = "allow-big-block" -) - var ObjectPatchCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Deprecated way to create a new merkledag object based on an existing one. Use MFS with 'files cp|rm' instead.", @@ -49,7 +43,7 @@ For modern use cases, use MFS with 'files' commands: 'ipfs files --help'. "set-data": patchSetDataCmd, }, Options: []cmds.Option{ - cmds.BoolOption(allowBigBlock, "Disable block size check and allow creation of blocks bigger than 1MB. WARNING: such blocks won't be transferable over the standard bitswap.").WithDefault(false), + cmdutils.AllowBigBlockOption, }, } @@ -64,7 +58,7 @@ Example: $ echo "hello" | ipfs object patch $HASH append-data NOTE: This does not append data to a file - it modifies the actual raw -data within a dag-pb object. Blocks have a max size of 1MB and objects larger than +data within a dag-pb object. Blocks have a max size of 1MiB and objects larger than the limit will not be respected by the network. DEPRECATED and provided for legacy reasons. Use 'ipfs add' or 'ipfs files' instead. @@ -92,7 +86,7 @@ DEPRECATED and provided for legacy reasons. Use 'ipfs add' or 'ipfs files' inste return err } - if err := checkBlockSize(req, p.Cid(), api.Dag()); err != nil { + if err := cmdutils.CheckCIDSize(req, p.Cid(), api.Dag()); err != nil { return err } @@ -142,7 +136,7 @@ DEPRECATED and provided for legacy reasons. Use 'files cp' and 'dag put' instead return err } - if err := checkBlockSize(req, p.Cid(), api.Dag()); err != nil { + if err := cmdutils.CheckCIDSize(req, p.Cid(), api.Dag()); err != nil { return err } @@ -184,7 +178,7 @@ DEPRECATED and provided for legacy reasons. Use 'files rm' instead. return err } - if err := checkBlockSize(req, p.Cid(), api.Dag()); err != nil { + if err := cmdutils.CheckCIDSize(req, p.Cid(), api.Dag()); err != nil { return err } @@ -254,7 +248,7 @@ Use MFS and 'files' commands instead: return err } - if err := checkBlockSize(req, p.Cid(), api.Dag()); err != nil { + if err := cmdutils.CheckCIDSize(req, p.Cid(), api.Dag()); err != nil { return err } @@ -268,26 +262,3 @@ Use MFS and 'files' commands instead: }), }, } - -func checkBlockSize(req *cmds.Request, c cid.Cid, dagAPI coreiface.APIDagService) error { - allowAnyBlockSize, _ := req.Options[allowBigBlock].(bool) - if allowAnyBlockSize { - return nil - } - - // We do not allow producing blocks bigger than 1 MiB to avoid errors - // when transmitting them over BitSwap. The 1 MiB constant is an - // unenforced and undeclared rule of thumb hard-coded here. - modifiedNode, err := dagAPI.Get(req.Context, c) - if err != nil { - return err - } - modifiedNodeSize, err := modifiedNode.Size() - if err != nil { - return err - } - if modifiedNodeSize > softBlockLimit { - return fmt.Errorf("produced block is over 1MB, object API is deprecated and does not support HAMT-sharding: to create big directories, please use the files API (MFS)") - } - return nil -} diff --git a/test/sharness/t0050-block.sh b/test/sharness/t0050-block.sh index 70639f623e9..9f0a533dbb7 100755 --- a/test/sharness/t0050-block.sh +++ b/test/sharness/t0050-block.sh @@ -248,4 +248,18 @@ test_expect_success "put with sha3 and cidv0 fails" ' echo "foooo" | test_must_fail ipfs block put --mhtype=sha3 --mhlen=20 --format=v0 ' +test_expect_success "'ipfs block put' check block size" ' + dd if=/dev/zero bs=2MB count=1 > 2-MB-file && + test_expect_code 1 ipfs block put 2-MB-file >block_put_out 2>&1 + ' + + test_expect_success "ipfs block put output has the correct error" ' + grep "produced block is over 1MiB" block_put_out + ' + + test_expect_success "ipfs block put --allow-big-block=true works" ' + test_expect_code 0 ipfs block put 2-MB-file --allow-big-block=true && + rm 2-MB-file + ' + test_done diff --git a/test/sharness/t0051-object.sh b/test/sharness/t0051-object.sh index 875548411b4..316c220abd5 100755 --- a/test/sharness/t0051-object.sh +++ b/test/sharness/t0051-object.sh @@ -229,12 +229,12 @@ test_object_cmd() { do DIR=$(ipfs object patch "$DIR" add-link "$DIR.jpg" "$DIR") done - # Fail when new block goes over the BS limit of 1MB, but allow manual override + # Fail when new block goes over the BS limit of 1MiB, but allow manual override test_expect_code 1 ipfs object patch "$DIR" add-link "$DIR.jpg" "$DIR" >patch_out 2>&1 ' test_expect_success "ipfs object patch add-link output has the correct error" ' - grep "produced block is over 1MB, object API is deprecated and does not support HAMT-sharding: to create big directories, please use the files API (MFS)" patch_out + grep "produced block is over 1MiB" patch_out ' test_expect_success "ipfs object patch --allow-big-block=true add-link works" ' diff --git a/test/sharness/t0053-dag.sh b/test/sharness/t0053-dag.sh index c46dcc8c19a..7514476be6e 100755 --- a/test/sharness/t0053-dag.sh +++ b/test/sharness/t0053-dag.sh @@ -44,6 +44,20 @@ test_dag_cmd() { test $EXPHASH = $IPLDHASH ' +test_expect_success "'ipfs dag put' check block size" ' + dd if=/dev/zero bs=2MB count=1 > 2-MB-file && + test_expect_code 1 ipfs dag put --input-codec=raw --store-codec=raw 2-MB-file >dag_put_out 2>&1 + ' + + test_expect_success "ipfs dag put output has the correct error" ' + grep "produced block is over 1MiB" dag_put_out + ' + + test_expect_success "ipfs dag put --allow-big-block=true works" ' + test_expect_code 0 ipfs dag put --input-codec=raw --store-codec=raw 2-MB-file --allow-big-block=true && + rm 2-MB-file + ' + test_expect_success "can add an ipld object using dag-json to dag-json" ' IPLDHASH=$(cat ipld_object | ipfs dag put --input-codec dag-json --store-codec dag-json) ' diff --git a/test/sharness/t0054-dag-car-import-export.sh b/test/sharness/t0054-dag-car-import-export.sh index 311833748f0..13c2100279a 100755 --- a/test/sharness/t0054-dag-car-import-export.sh +++ b/test/sharness/t0054-dag-car-import-export.sh @@ -233,4 +233,17 @@ test_expect_success "naked root import expected output" ' test_cmp_sorted naked_root_import_json_expected naked_root_import_json_actual ' +test_expect_success "'ipfs dag import' check block size" ' + BIG_CID=$(dd if=/dev/zero bs=2MB count=1 | ipfs dag put --input-codec=raw --store-codec=raw --allow-big-block) && + ipfs dag export $BIG_CID > 2-MB-block.car && + test_expect_code 1 ipfs dag import 2-MB-block.car >dag_import_out 2>&1 +' +test_expect_success "ipfs dag import output has the correct error" ' + grep "block is over 1MiB" dag_import_out +' + +test_expect_success "ipfs dag import --allow-big-block works" ' + test_expect_code 0 ipfs dag import --allow-big-block 2-MB-block.car +' + test_done