From aab549368b04c58859c25953d53983292804ad2c Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Mon, 15 Nov 2021 19:22:21 -0500 Subject: [PATCH 1/4] test: t0260-sharding.sh rename largedir to to dir --- test/sharness/t0260-sharding.sh | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/test/sharness/t0260-sharding.sh b/test/sharness/t0260-sharding.sh index 623c4b0a940..4717b1d3143 100755 --- a/test/sharness/t0260-sharding.sh +++ b/test/sharness/t0260-sharding.sh @@ -16,14 +16,14 @@ test_expect_success "set up test data" ' done ' -test_add_large_dir() { +test_add_dir() { exphash="$1" - test_expect_success "ipfs add on very large directory succeeds" ' + test_expect_success "ipfs add on directory succeeds" ' ipfs add -r -Q testdata > sharddir_out && echo "$exphash" > sharddir_exp && test_cmp sharddir_exp sharddir_out ' - test_expect_success "ipfs get on very large directory succeeds" ' + test_expect_success "ipfs get on directory succeeds" ' ipfs get -o testdata-out "$exphash" && test_cmp testdata testdata-out ' @@ -32,11 +32,11 @@ test_add_large_dir() { test_init_ipfs UNSHARDED="QmavrTrQG4VhoJmantURAYuw3bowq3E2WcvP36NRQDAC1N" -test_add_large_dir "$UNSHARDED" +test_add_dir "$UNSHARDED" test_launch_ipfs_daemon -test_add_large_dir "$UNSHARDED" +test_add_dir "$UNSHARDED" test_kill_ipfs_daemon @@ -45,11 +45,11 @@ test_expect_success "enable sharding" ' ' SHARDED="QmSCJD1KYLhVVHqBK3YyXuoEqHt7vggyJhzoFYbT8v1XYL" -test_add_large_dir "$SHARDED" +test_add_dir "$SHARDED" test_launch_ipfs_daemon -test_add_large_dir "$SHARDED" +test_add_dir "$SHARDED" test_kill_ipfs_daemon @@ -93,9 +93,9 @@ test_expect_success "'ipfs resolve' can resolve sharded dirs" ' test_kill_ipfs_daemon -test_add_large_dir_v1() { +test_add_dir_v1() { exphash="$1" - test_expect_success "ipfs add (CIDv1) on very large directory succeeds" ' + test_expect_success "ipfs add (CIDv1) on directory succeeds" ' ipfs add -r -Q --cid-version=1 testdata > sharddir_out && echo "$exphash" > sharddir_exp && test_cmp sharddir_exp sharddir_out @@ -109,11 +109,11 @@ test_add_large_dir_v1() { # this hash implies the directory is CIDv1 and leaf entries are CIDv1 and raw SHARDEDV1="bafybeibiemewfzzdyhq2l74wrd6qj2oz42usjlktgnlqv4yfawgouaqn4u" -test_add_large_dir_v1 "$SHARDEDV1" +test_add_dir_v1 "$SHARDEDV1" test_launch_ipfs_daemon -test_add_large_dir_v1 "$SHARDEDV1" +test_add_dir_v1 "$SHARDEDV1" test_kill_ipfs_daemon From 927fad8aa404de0d6fa4a13aae0444e97e420ccf Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Mon, 15 Nov 2021 19:38:07 -0500 Subject: [PATCH 2/4] test: add tests for automatic sharding and unsharding --- test/sharness/t0260-sharding.sh | 67 ++++++++++++++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/test/sharness/t0260-sharding.sh b/test/sharness/t0260-sharding.sh index 4717b1d3143..c98712f095e 100755 --- a/test/sharness/t0260-sharding.sh +++ b/test/sharness/t0260-sharding.sh @@ -8,12 +8,25 @@ test_description="Test directory sharding" . lib/test-lib.sh +# We shard based on size with a threshold of 256 KiB (see config file docs) +# above which directories are sharded. +# +# The directory size is estimated as the size of each link. Links are roughly +# the entry name + the CID byte length (e.g. 34 bytes for a CIDv0). So for +# entries of length 10 we need 256 KiB / (34 + 10) ~ 6000 entries in the +# directory to trigger sharding. test_expect_success "set up test data" ' mkdir testdata for i in `seq 2000` do echo $i > testdata/file$i done + + mkdir big_dir + for i in `seq 5960` # Just above the number of entries that trigger sharding for 256KiB + do + echo $i > big_dir/`printf "file%06d" $i` # fixed length of 10 chars + done ' test_add_dir() { @@ -32,6 +45,11 @@ test_add_dir() { test_init_ipfs UNSHARDED="QmavrTrQG4VhoJmantURAYuw3bowq3E2WcvP36NRQDAC1N" + +test_expect_success "force sharding off" ' +ipfs config --json Internal.UnixFSShardingSizeThreshold "\"1G\"" +' + test_add_dir "$UNSHARDED" test_launch_ipfs_daemon @@ -40,7 +58,7 @@ test_add_dir "$UNSHARDED" test_kill_ipfs_daemon -test_expect_success "enable sharding" ' +test_expect_success "force sharding on" ' ipfs config --json Internal.UnixFSShardingSizeThreshold "\"1B\"" ' @@ -137,4 +155,51 @@ test_list_incomplete_dir() { test_list_incomplete_dir +# Test automatic sharding and unsharding + +# TODO: This does not need to report an error https://github.com/ipfs/go-ipfs/issues/8088 +test_expect_failure "reset automatic sharding" ' + ipfs config --json Internal.UnixFSShardingSizeThreshold null +' + +LARGE_SHARDED="QmWfjnRWRvdvYezQWnfbvrvY7JjrpevsE9cato1x76UqGr" +LARGE_MINUS_5_UNSHAREDED="QmbVxi5zDdzytrjdufUejM92JsWj8wGVmukk6tiPce3p1m" + +test_add_large_sharded_dir() { + exphash="$1" + test_expect_success "ipfs add on directory succeeds" ' + ipfs add -r -Q big_dir > shardbigdir_out && + echo "$exphash" > shardbigdir_exp && + test_cmp shardbigdir_exp shardbigdir_out + ' + + test_expect_success "can access a path under the dir" ' + ipfs cat "$exphash/file000030" > file30_out && + test_cmp big_dir/file000030 file30_out + ' +} + +test_add_large_sharded_dir "$LARGE_SHARDED" + +test_expect_success "remove a few entries from big_dir/ to trigger unsharding" ' + ipfs files cp /ipfs/"$LARGE_SHARDED" /big_dir && + for i in `seq 5` + do + ipfs files rm /big_dir/`printf "file%06d" $i` + done && + ipfs files stat --hash /big_dir > unshard_dir_hash && + echo "$LARGE_MINUS_5_UNSHAREDED" > unshard_exp && + test_cmp unshard_exp unshard_dir_hash +' + +test_expect_success "add a few entries to big_dir/ to retrigger sharding" ' + for i in `seq 5` + do + ipfs files cp /ipfs/"$LARGE_SHARDED"/`printf "file%06d" $i` /big_dir/`printf "file%06d" $i` + done && + ipfs files stat --hash /big_dir > shard_dir_hash && + echo "$LARGE_SHARDED" > shard_exp && + test_cmp shard_exp shard_dir_hash +' + test_done From 16622c8f1db16fcee180fb1e08ab893134fe2357 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Tue, 16 Nov 2021 00:48:34 -0500 Subject: [PATCH 3/4] move automatic sharding/unsharding tests from t0260 to t0250 --- test/sharness/t0250-files-api.sh | 66 ++++++++++++++++++++++++++++++++ test/sharness/t0260-sharding.sh | 60 ----------------------------- 2 files changed, 66 insertions(+), 60 deletions(-) diff --git a/test/sharness/t0250-files-api.sh b/test/sharness/t0250-files-api.sh index abed7f9b814..2063f6f5427 100755 --- a/test/sharness/t0250-files-api.sh +++ b/test/sharness/t0250-files-api.sh @@ -858,4 +858,70 @@ test_sharding "(cidv1 root)" "--cid-version=1" test_kill_ipfs_daemon +# Test automatic sharding and unsharding + +# We shard based on size with a threshold of 256 KiB (see config file docs) +# above which directories are sharded. +# +# The directory size is estimated as the size of each link. Links are roughly +# the entry name + the CID byte length (e.g. 34 bytes for a CIDv0). So for +# entries of length 10 we need 256 KiB / (34 + 10) ~ 6000 entries in the +# directory to trigger sharding. +test_expect_success "set up automatic sharding/unsharding data" ' + mkdir big_dir + for i in `seq 5960` # Just above the number of entries that trigger sharding for 256KiB + do + echo $i > big_dir/`printf "file%06d" $i` # fixed length of 10 chars + done +' + +# TODO: This does not need to report an error https://github.com/ipfs/go-ipfs/issues/8088 +test_expect_failure "reset automatic sharding" ' + ipfs config --json Internal.UnixFSShardingSizeThreshold null +' + +test_launch_ipfs_daemon_without_network + +LARGE_SHARDED="QmWfjnRWRvdvYezQWnfbvrvY7JjrpevsE9cato1x76UqGr" +LARGE_MINUS_5_UNSHAREDED="QmbVxi5zDdzytrjdufUejM92JsWj8wGVmukk6tiPce3p1m" + +test_add_large_sharded_dir() { + exphash="$1" + test_expect_success "ipfs add on directory succeeds" ' + ipfs add -r -Q big_dir > shardbigdir_out && + echo "$exphash" > shardbigdir_exp && + test_cmp shardbigdir_exp shardbigdir_out + ' + + test_expect_success "can access a path under the dir" ' + ipfs cat "$exphash/file000030" > file30_out && + test_cmp big_dir/file000030 file30_out + ' +} + +test_add_large_sharded_dir "$LARGE_SHARDED" + +test_expect_success "remove a few entries from big_dir/ to trigger unsharding" ' + ipfs files cp /ipfs/"$LARGE_SHARDED" /big_dir && + for i in `seq 5` + do + ipfs files rm /big_dir/`printf "file%06d" $i` + done && + ipfs files stat --hash /big_dir > unshard_dir_hash && + echo "$LARGE_MINUS_5_UNSHAREDED" > unshard_exp && + test_cmp unshard_exp unshard_dir_hash +' + +test_expect_success "add a few entries to big_dir/ to retrigger sharding" ' + for i in `seq 5` + do + ipfs files cp /ipfs/"$LARGE_SHARDED"/`printf "file%06d" $i` /big_dir/`printf "file%06d" $i` + done && + ipfs files stat --hash /big_dir > shard_dir_hash && + echo "$LARGE_SHARDED" > shard_exp && + test_cmp shard_exp shard_dir_hash +' + +test_kill_ipfs_daemon + test_done diff --git a/test/sharness/t0260-sharding.sh b/test/sharness/t0260-sharding.sh index c98712f095e..85e4a7ca708 100755 --- a/test/sharness/t0260-sharding.sh +++ b/test/sharness/t0260-sharding.sh @@ -8,25 +8,12 @@ test_description="Test directory sharding" . lib/test-lib.sh -# We shard based on size with a threshold of 256 KiB (see config file docs) -# above which directories are sharded. -# -# The directory size is estimated as the size of each link. Links are roughly -# the entry name + the CID byte length (e.g. 34 bytes for a CIDv0). So for -# entries of length 10 we need 256 KiB / (34 + 10) ~ 6000 entries in the -# directory to trigger sharding. test_expect_success "set up test data" ' mkdir testdata for i in `seq 2000` do echo $i > testdata/file$i done - - mkdir big_dir - for i in `seq 5960` # Just above the number of entries that trigger sharding for 256KiB - do - echo $i > big_dir/`printf "file%06d" $i` # fixed length of 10 chars - done ' test_add_dir() { @@ -155,51 +142,4 @@ test_list_incomplete_dir() { test_list_incomplete_dir -# Test automatic sharding and unsharding - -# TODO: This does not need to report an error https://github.com/ipfs/go-ipfs/issues/8088 -test_expect_failure "reset automatic sharding" ' - ipfs config --json Internal.UnixFSShardingSizeThreshold null -' - -LARGE_SHARDED="QmWfjnRWRvdvYezQWnfbvrvY7JjrpevsE9cato1x76UqGr" -LARGE_MINUS_5_UNSHAREDED="QmbVxi5zDdzytrjdufUejM92JsWj8wGVmukk6tiPce3p1m" - -test_add_large_sharded_dir() { - exphash="$1" - test_expect_success "ipfs add on directory succeeds" ' - ipfs add -r -Q big_dir > shardbigdir_out && - echo "$exphash" > shardbigdir_exp && - test_cmp shardbigdir_exp shardbigdir_out - ' - - test_expect_success "can access a path under the dir" ' - ipfs cat "$exphash/file000030" > file30_out && - test_cmp big_dir/file000030 file30_out - ' -} - -test_add_large_sharded_dir "$LARGE_SHARDED" - -test_expect_success "remove a few entries from big_dir/ to trigger unsharding" ' - ipfs files cp /ipfs/"$LARGE_SHARDED" /big_dir && - for i in `seq 5` - do - ipfs files rm /big_dir/`printf "file%06d" $i` - done && - ipfs files stat --hash /big_dir > unshard_dir_hash && - echo "$LARGE_MINUS_5_UNSHAREDED" > unshard_exp && - test_cmp unshard_exp unshard_dir_hash -' - -test_expect_success "add a few entries to big_dir/ to retrigger sharding" ' - for i in `seq 5` - do - ipfs files cp /ipfs/"$LARGE_SHARDED"/`printf "file%06d" $i` /big_dir/`printf "file%06d" $i` - done && - ipfs files stat --hash /big_dir > shard_dir_hash && - echo "$LARGE_SHARDED" > shard_exp && - test_cmp shard_exp shard_dir_hash -' - test_done From c393aeca7345b16172a51fd49e93b2ebaa637501 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Tue, 16 Nov 2021 00:49:22 -0500 Subject: [PATCH 4/4] fix typo --- test/sharness/t0250-files-api.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/sharness/t0250-files-api.sh b/test/sharness/t0250-files-api.sh index 2063f6f5427..d94b43380a6 100755 --- a/test/sharness/t0250-files-api.sh +++ b/test/sharness/t0250-files-api.sh @@ -883,7 +883,7 @@ test_expect_failure "reset automatic sharding" ' test_launch_ipfs_daemon_without_network LARGE_SHARDED="QmWfjnRWRvdvYezQWnfbvrvY7JjrpevsE9cato1x76UqGr" -LARGE_MINUS_5_UNSHAREDED="QmbVxi5zDdzytrjdufUejM92JsWj8wGVmukk6tiPce3p1m" +LARGE_MINUS_5_UNSHARDED="QmbVxi5zDdzytrjdufUejM92JsWj8wGVmukk6tiPce3p1m" test_add_large_sharded_dir() { exphash="$1" @@ -908,7 +908,7 @@ test_expect_success "remove a few entries from big_dir/ to trigger unsharding" ' ipfs files rm /big_dir/`printf "file%06d" $i` done && ipfs files stat --hash /big_dir > unshard_dir_hash && - echo "$LARGE_MINUS_5_UNSHAREDED" > unshard_exp && + echo "$LARGE_MINUS_5_UNSHARDED" > unshard_exp && test_cmp unshard_exp unshard_dir_hash '