From f9073c6705ef3864344a3a1dce90e95d306d5ca5 Mon Sep 17 00:00:00 2001 From: Lucas Molas Date: Thu, 24 Feb 2022 13:56:02 -0300 Subject: [PATCH 1/5] feat(cmds): extend block size check for dag|block put --- core/commands/block.go | 6 ++++ core/commands/cmdutils/utils.go | 51 +++++++++++++++++++++++++++++++++ core/commands/dag/dag.go | 2 ++ core/commands/dag/put.go | 5 ++++ core/commands/object/patch.go | 41 ++++---------------------- 5 files changed, 70 insertions(+), 35 deletions(-) create mode 100644 core/commands/cmdutils/utils.go diff --git a/core/commands/block.go b/core/commands/block.go index a06bfe0683e..4cfe1fc6ef9 100644 --- a/core/commands/block.go +++ b/core/commands/block.go @@ -10,6 +10,7 @@ import ( util "github.com/ipfs/go-ipfs/blocks/blockstoreutil" cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv" + "github.com/ipfs/go-ipfs/core/commands/cmdutils" cmds "github.com/ipfs/go-ipfs-cmds" options "github.com/ipfs/interface-go-ipfs-core/options" @@ -138,6 +139,7 @@ other than 'sha2-256' or format to anything other than 'v0' will result in CIDv1 cmds.StringOption(mhtypeOptionName, "multihash hash function").WithDefault("sha2-256"), cmds.IntOption(mhlenOptionName, "multihash hash length").WithDefault(-1), cmds.BoolOption(pinOptionName, "pin added blocks recursively").WithDefault(false), + cmdutils.AllowBigBlockOption, }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { api, err := cmdenv.GetApi(env, req) @@ -182,6 +184,10 @@ other than 'sha2-256' or format to anything other than 'v0' will result in CIDv1 return err } + if err := cmdutils.CheckBlockSize(req, uint64(p.Size())); err != nil { + return err + } + err = res.Emit(&BlockStat{ Key: p.Path().Cid().String(), Size: p.Size(), diff --git a/core/commands/cmdutils/utils.go b/core/commands/cmdutils/utils.go new file mode 100644 index 00000000000..3e848939b59 --- /dev/null +++ b/core/commands/cmdutils/utils.go @@ -0,0 +1,51 @@ +package cmdutils + +import ( + "fmt" + + cmds "github.com/ipfs/go-ipfs-cmds" + + "github.com/ipfs/go-cid" + coreiface "github.com/ipfs/interface-go-ipfs-core" +) + +const ( + AllowBigBlockOptionName = "allow-big-block" + SoftBlockLimit = 1024 * 1024 // https://github.com/ipfs/go-ipfs/issues/7421#issuecomment-910833499 +) + +var AllowBigBlockOption cmds.Option + +func init() { + AllowBigBlockOption = cmds.BoolOption(AllowBigBlockOptionName, "Disable block size check and allow creation of blocks bigger than 1MB. WARNING: such blocks won't be transferable over the standard bitswap.").WithDefault(false) +} + +func CheckCIDSize(req *cmds.Request, c cid.Cid, dagAPI coreiface.APIDagService) error { + n, err := dagAPI.Get(req.Context, c) + if err != nil { + return fmt.Errorf("CheckCIDSize: getting dag: %w", err) + } + + nodeSize, err := n.Size() + if err != nil { + return fmt.Errorf("CheckCIDSize: getting node size: %w", err) + } + + return CheckBlockSize(req, nodeSize) +} + +func CheckBlockSize(req *cmds.Request, size uint64) error { + allowAnyBlockSize, _ := req.Options[AllowBigBlockOptionName].(bool) + if allowAnyBlockSize { + return nil + } + + // We do not allow producing blocks bigger than 1 MiB to avoid errors + // when transmitting them over BitSwap. The 1 MiB constant is an + // unenforced and undeclared rule of thumb hard-coded here. + if size > SoftBlockLimit { + return fmt.Errorf("produced block is over 1MB: big blocks can't be exchanged with other peers. consider using dag-pb for automatic chunking of bigger files, or pass --allow-big-block to override") + } + return nil + +} diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index cbaf7f93fe1..166a8cc0f66 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -5,6 +5,7 @@ import ( "io" "github.com/ipfs/go-ipfs/core/commands/cmdenv" + "github.com/ipfs/go-ipfs/core/commands/cmdutils" cid "github.com/ipfs/go-cid" cidenc "github.com/ipfs/go-cidutil/cidenc" @@ -88,6 +89,7 @@ into an object of the specified format. cmds.StringOption("input-codec", "Codec that the input object is encoded in").WithDefault("dag-json"), cmds.BoolOption("pin", "Pin this object when adding."), cmds.StringOption("hash", "Hash function to use").WithDefault("sha2-256"), + cmdutils.AllowBigBlockOption, }, Run: dagPut, Type: OutputObject{}, diff --git a/core/commands/dag/put.go b/core/commands/dag/put.go index 0bb7fd2ae50..e741f11124d 100644 --- a/core/commands/dag/put.go +++ b/core/commands/dag/put.go @@ -7,6 +7,7 @@ import ( blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "github.com/ipfs/go-ipfs/core/commands/cmdenv" + "github.com/ipfs/go-ipfs/core/commands/cmdutils" ipldlegacy "github.com/ipfs/go-ipld-legacy" "github.com/ipld/go-ipld-prime/multicodec" basicnode "github.com/ipld/go-ipld-prime/node/basic" @@ -102,6 +103,10 @@ func dagPut(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e Node: n, } + if err := cmdutils.CheckBlockSize(req, uint64(bd.Len())); err != nil { + return err + } + if err := b.Add(req.Context, &ln); err != nil { return err } diff --git a/core/commands/object/patch.go b/core/commands/object/patch.go index 78f92703254..69125c74123 100644 --- a/core/commands/object/patch.go +++ b/core/commands/object/patch.go @@ -4,20 +4,14 @@ import ( "fmt" "io" - "github.com/ipfs/go-cid" cmds "github.com/ipfs/go-ipfs-cmds" "github.com/ipfs/go-ipfs/core/commands/cmdenv" - coreiface "github.com/ipfs/interface-go-ipfs-core" + "github.com/ipfs/go-ipfs/core/commands/cmdutils" "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" ) -const ( - softBlockLimit = 1024 * 1024 // https://github.com/ipfs/go-ipfs/issues/7421#issuecomment-910833499 - allowBigBlock = "allow-big-block" -) - var ObjectPatchCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Deprecated way to create a new merkledag object based on an existing one. Use MFS with 'files cp|rm' instead.", @@ -49,7 +43,7 @@ For modern use cases, use MFS with 'files' commands: 'ipfs files --help'. "set-data": patchSetDataCmd, }, Options: []cmds.Option{ - cmds.BoolOption(allowBigBlock, "Disable block size check and allow creation of blocks bigger than 1MB. WARNING: such blocks won't be transferable over the standard bitswap.").WithDefault(false), + cmdutils.AllowBigBlockOption, }, } @@ -92,7 +86,7 @@ DEPRECATED and provided for legacy reasons. Use 'ipfs add' or 'ipfs files' inste return err } - if err := checkBlockSize(req, p.Cid(), api.Dag()); err != nil { + if err := cmdutils.CheckCIDSize(req, p.Cid(), api.Dag()); err != nil { return err } @@ -142,7 +136,7 @@ DEPRECATED and provided for legacy reasons. Use 'files cp' and 'dag put' instead return err } - if err := checkBlockSize(req, p.Cid(), api.Dag()); err != nil { + if err := cmdutils.CheckCIDSize(req, p.Cid(), api.Dag()); err != nil { return err } @@ -184,7 +178,7 @@ DEPRECATED and provided for legacy reasons. Use 'files rm' instead. return err } - if err := checkBlockSize(req, p.Cid(), api.Dag()); err != nil { + if err := cmdutils.CheckCIDSize(req, p.Cid(), api.Dag()); err != nil { return err } @@ -254,7 +248,7 @@ Use MFS and 'files' commands instead: return err } - if err := checkBlockSize(req, p.Cid(), api.Dag()); err != nil { + if err := cmdutils.CheckCIDSize(req, p.Cid(), api.Dag()); err != nil { return err } @@ -268,26 +262,3 @@ Use MFS and 'files' commands instead: }), }, } - -func checkBlockSize(req *cmds.Request, c cid.Cid, dagAPI coreiface.APIDagService) error { - allowAnyBlockSize, _ := req.Options[allowBigBlock].(bool) - if allowAnyBlockSize { - return nil - } - - // We do not allow producing blocks bigger than 1 MiB to avoid errors - // when transmitting them over BitSwap. The 1 MiB constant is an - // unenforced and undeclared rule of thumb hard-coded here. - modifiedNode, err := dagAPI.Get(req.Context, c) - if err != nil { - return err - } - modifiedNodeSize, err := modifiedNode.Size() - if err != nil { - return err - } - if modifiedNodeSize > softBlockLimit { - return fmt.Errorf("produced block is over 1MB, object API is deprecated and does not support HAMT-sharding: to create big directories, please use the files API (MFS)") - } - return nil -} From 0c26f93169746e0db19586c4baeae0d24fcab736 Mon Sep 17 00:00:00 2001 From: Lucas Molas Date: Fri, 25 Feb 2022 11:20:12 -0300 Subject: [PATCH 2/5] sharness --- test/sharness/t0050-block.sh | 14 ++++++++++++++ test/sharness/t0051-object.sh | 2 +- test/sharness/t0053-dag.sh | 14 ++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/test/sharness/t0050-block.sh b/test/sharness/t0050-block.sh index 70639f623e9..0574c58fa4d 100755 --- a/test/sharness/t0050-block.sh +++ b/test/sharness/t0050-block.sh @@ -248,4 +248,18 @@ test_expect_success "put with sha3 and cidv0 fails" ' echo "foooo" | test_must_fail ipfs block put --mhtype=sha3 --mhlen=20 --format=v0 ' +test_expect_success "'ipfs block put' check block size" ' + dd if=/dev/zero bs=2MB count=1 > 2-MB-file && + test_expect_code 1 ipfs block put 2-MB-file >block_put_out 2>&1 + ' + + test_expect_success "ipfs block put output has the correct error" ' + grep "produced block is over 1MB" block_put_out + ' + + test_expect_success "ipfs block put --allow-big-block=true works" ' + test_expect_code 0 ipfs block put 2-MB-file --allow-big-block=true && + rm 2-MB-file + ' + test_done diff --git a/test/sharness/t0051-object.sh b/test/sharness/t0051-object.sh index 875548411b4..9d48863a435 100755 --- a/test/sharness/t0051-object.sh +++ b/test/sharness/t0051-object.sh @@ -234,7 +234,7 @@ test_object_cmd() { ' test_expect_success "ipfs object patch add-link output has the correct error" ' - grep "produced block is over 1MB, object API is deprecated and does not support HAMT-sharding: to create big directories, please use the files API (MFS)" patch_out + grep "produced block is over 1MB" patch_out ' test_expect_success "ipfs object patch --allow-big-block=true add-link works" ' diff --git a/test/sharness/t0053-dag.sh b/test/sharness/t0053-dag.sh index c46dcc8c19a..af95f4043d3 100755 --- a/test/sharness/t0053-dag.sh +++ b/test/sharness/t0053-dag.sh @@ -44,6 +44,20 @@ test_dag_cmd() { test $EXPHASH = $IPLDHASH ' +test_expect_success "'ipfs dag put' check block size" ' + dd if=/dev/zero bs=2MB count=1 > 2-MB-file && + test_expect_code 1 ipfs dag put --input-codec=raw --store-codec=raw 2-MB-file >dag_put_out 2>&1 + ' + + test_expect_success "ipfs dag put output has the correct error" ' + grep "produced block is over 1MB" dag_put_out + ' + + test_expect_success "ipfs dag put --allow-big-block=true works" ' + test_expect_code 0 ipfs dag put --input-codec=raw --store-codec=raw 2-MB-file --allow-big-block=true && + rm 2-MB-file + ' + test_expect_success "can add an ipld object using dag-json to dag-json" ' IPLDHASH=$(cat ipld_object | ipfs dag put --input-codec dag-json --store-codec dag-json) ' From 26dc0e6ef3114eb923997f9030b1161595ba49c1 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 11 Mar 2022 21:41:48 +0100 Subject: [PATCH 3/5] feat(cmds): block size check for dag import --- core/commands/dag/dag.go | 1 + core/commands/dag/import.go | 4 ++++ test/sharness/t0054-dag-car-import-export.sh | 13 +++++++++++++ 3 files changed, 18 insertions(+) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index 166a8cc0f66..6d2e480b284 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -207,6 +207,7 @@ Maximum supported CAR version: 1 cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing.").WithDefault(true), cmds.BoolOption(silentOptionName, "No output."), cmds.BoolOption(statsOptionName, "Output stats."), + cmdutils.AllowBigBlockOption, }, Type: CarImportOutput{}, Run: dagImport, diff --git a/core/commands/dag/import.go b/core/commands/dag/import.go index e206652ce36..476b986c825 100644 --- a/core/commands/dag/import.go +++ b/core/commands/dag/import.go @@ -8,6 +8,7 @@ import ( cid "github.com/ipfs/go-cid" files "github.com/ipfs/go-ipfs-files" "github.com/ipfs/go-ipfs/core/commands/cmdenv" + "github.com/ipfs/go-ipfs/core/commands/cmdutils" ipld "github.com/ipfs/go-ipld-format" iface "github.com/ipfs/interface-go-ipfs-core" "github.com/ipfs/interface-go-ipfs-core/options" @@ -180,6 +181,9 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI, } else if block == nil { break } + if err := cmdutils.CheckBlockSize(req, uint64(len(block.RawData()))); err != nil { + return err + } // the double-decode is suboptimal, but we need it for batching nd, err := ipld.Decode(block) diff --git a/test/sharness/t0054-dag-car-import-export.sh b/test/sharness/t0054-dag-car-import-export.sh index 311833748f0..ce05ea467c9 100755 --- a/test/sharness/t0054-dag-car-import-export.sh +++ b/test/sharness/t0054-dag-car-import-export.sh @@ -233,4 +233,17 @@ test_expect_success "naked root import expected output" ' test_cmp_sorted naked_root_import_json_expected naked_root_import_json_actual ' +test_expect_success "'ipfs dag import' check block size" ' + BIG_CID=$(dd if=/dev/zero bs=2MB count=1 | ipfs dag put --input-codec=raw --store-codec=raw --allow-big-block) && + ipfs dag export $BIG_CID > 2-MB-block.car && + test_expect_code 1 ipfs dag import 2-MB-block.car >dag_import_out 2>&1 +' +test_expect_success "ipfs dag import output has the correct error" ' + grep "block is over 1MB" dag_import_out +' + +test_expect_success "ipfs dag import --allow-big-block works" ' + test_expect_code 0 ipfs dag import --allow-big-block 2-MB-block.car +' + test_done From 3dbfaa202f158ad4c8f1c419340684f03385550c Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 11 Mar 2022 21:49:18 +0100 Subject: [PATCH 4/5] =?UTF-8?q?chore:=20dag-pb=20=E2=86=92=20UnixFS?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/commands/cmdutils/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/commands/cmdutils/utils.go b/core/commands/cmdutils/utils.go index 3e848939b59..f94f100d36e 100644 --- a/core/commands/cmdutils/utils.go +++ b/core/commands/cmdutils/utils.go @@ -44,7 +44,7 @@ func CheckBlockSize(req *cmds.Request, size uint64) error { // when transmitting them over BitSwap. The 1 MiB constant is an // unenforced and undeclared rule of thumb hard-coded here. if size > SoftBlockLimit { - return fmt.Errorf("produced block is over 1MB: big blocks can't be exchanged with other peers. consider using dag-pb for automatic chunking of bigger files, or pass --allow-big-block to override") + return fmt.Errorf("produced block is over 1MB: big blocks can't be exchanged with other peers. consider using UnixFS for automatic chunking of bigger files, or pass --allow-big-block to override") } return nil From 5704e5485c718e10a87f69dcaa72b1e2f9053f46 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 11 Mar 2022 22:02:10 +0100 Subject: [PATCH 5/5] =?UTF-8?q?fix:=201MB=20=E2=86=92=201MiB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/commands/cmdutils/utils.go | 4 ++-- core/commands/get.go | 2 +- core/commands/object/patch.go | 2 +- test/sharness/t0050-block.sh | 2 +- test/sharness/t0051-object.sh | 4 ++-- test/sharness/t0053-dag.sh | 2 +- test/sharness/t0054-dag-car-import-export.sh | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/core/commands/cmdutils/utils.go b/core/commands/cmdutils/utils.go index f94f100d36e..ebbbca64e46 100644 --- a/core/commands/cmdutils/utils.go +++ b/core/commands/cmdutils/utils.go @@ -17,7 +17,7 @@ const ( var AllowBigBlockOption cmds.Option func init() { - AllowBigBlockOption = cmds.BoolOption(AllowBigBlockOptionName, "Disable block size check and allow creation of blocks bigger than 1MB. WARNING: such blocks won't be transferable over the standard bitswap.").WithDefault(false) + AllowBigBlockOption = cmds.BoolOption(AllowBigBlockOptionName, "Disable block size check and allow creation of blocks bigger than 1MiB. WARNING: such blocks won't be transferable over the standard bitswap.").WithDefault(false) } func CheckCIDSize(req *cmds.Request, c cid.Cid, dagAPI coreiface.APIDagService) error { @@ -44,7 +44,7 @@ func CheckBlockSize(req *cmds.Request, size uint64) error { // when transmitting them over BitSwap. The 1 MiB constant is an // unenforced and undeclared rule of thumb hard-coded here. if size > SoftBlockLimit { - return fmt.Errorf("produced block is over 1MB: big blocks can't be exchanged with other peers. consider using UnixFS for automatic chunking of bigger files, or pass --allow-big-block to override") + return fmt.Errorf("produced block is over 1MiB: big blocks can't be exchanged with other peers. consider using UnixFS for automatic chunking of bigger files, or pass --allow-big-block to override") } return nil diff --git a/core/commands/get.go b/core/commands/get.go index 7fb53467f5d..011fc1d6b9b 100644 --- a/core/commands/get.go +++ b/core/commands/get.go @@ -246,7 +246,7 @@ func getCompressOptions(req *cmds.Request) (int, error) { return cmplvl, nil } -// DefaultBufSize is the buffer size for gets. for now, 1MB, which is ~4 blocks. +// DefaultBufSize is the buffer size for gets. for now, 1MiB, which is ~4 blocks. // TODO: does this need to be configurable? var DefaultBufSize = 1048576 diff --git a/core/commands/object/patch.go b/core/commands/object/patch.go index 69125c74123..ab76ebb23f4 100644 --- a/core/commands/object/patch.go +++ b/core/commands/object/patch.go @@ -58,7 +58,7 @@ Example: $ echo "hello" | ipfs object patch $HASH append-data NOTE: This does not append data to a file - it modifies the actual raw -data within a dag-pb object. Blocks have a max size of 1MB and objects larger than +data within a dag-pb object. Blocks have a max size of 1MiB and objects larger than the limit will not be respected by the network. DEPRECATED and provided for legacy reasons. Use 'ipfs add' or 'ipfs files' instead. diff --git a/test/sharness/t0050-block.sh b/test/sharness/t0050-block.sh index 0574c58fa4d..9f0a533dbb7 100755 --- a/test/sharness/t0050-block.sh +++ b/test/sharness/t0050-block.sh @@ -254,7 +254,7 @@ test_expect_success "'ipfs block put' check block size" ' ' test_expect_success "ipfs block put output has the correct error" ' - grep "produced block is over 1MB" block_put_out + grep "produced block is over 1MiB" block_put_out ' test_expect_success "ipfs block put --allow-big-block=true works" ' diff --git a/test/sharness/t0051-object.sh b/test/sharness/t0051-object.sh index 9d48863a435..316c220abd5 100755 --- a/test/sharness/t0051-object.sh +++ b/test/sharness/t0051-object.sh @@ -229,12 +229,12 @@ test_object_cmd() { do DIR=$(ipfs object patch "$DIR" add-link "$DIR.jpg" "$DIR") done - # Fail when new block goes over the BS limit of 1MB, but allow manual override + # Fail when new block goes over the BS limit of 1MiB, but allow manual override test_expect_code 1 ipfs object patch "$DIR" add-link "$DIR.jpg" "$DIR" >patch_out 2>&1 ' test_expect_success "ipfs object patch add-link output has the correct error" ' - grep "produced block is over 1MB" patch_out + grep "produced block is over 1MiB" patch_out ' test_expect_success "ipfs object patch --allow-big-block=true add-link works" ' diff --git a/test/sharness/t0053-dag.sh b/test/sharness/t0053-dag.sh index af95f4043d3..7514476be6e 100755 --- a/test/sharness/t0053-dag.sh +++ b/test/sharness/t0053-dag.sh @@ -50,7 +50,7 @@ test_expect_success "'ipfs dag put' check block size" ' ' test_expect_success "ipfs dag put output has the correct error" ' - grep "produced block is over 1MB" dag_put_out + grep "produced block is over 1MiB" dag_put_out ' test_expect_success "ipfs dag put --allow-big-block=true works" ' diff --git a/test/sharness/t0054-dag-car-import-export.sh b/test/sharness/t0054-dag-car-import-export.sh index ce05ea467c9..13c2100279a 100755 --- a/test/sharness/t0054-dag-car-import-export.sh +++ b/test/sharness/t0054-dag-car-import-export.sh @@ -239,7 +239,7 @@ test_expect_success "'ipfs dag import' check block size" ' test_expect_code 1 ipfs dag import 2-MB-block.car >dag_import_out 2>&1 ' test_expect_success "ipfs dag import output has the correct error" ' - grep "block is over 1MB" dag_import_out + grep "block is over 1MiB" dag_import_out ' test_expect_success "ipfs dag import --allow-big-block works" '