Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add automatic sharding/unsharding tests #8547

Merged
merged 4 commits into from
Nov 16, 2021
Merged

Add automatic sharding/unsharding tests #8547

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
aab5493
test: t0260-sharding.sh rename largedir to to dir
aschmahmann Nov 16, 2021
927fad8
test: add tests for automatic sharding and unsharding
aschmahmann Nov 16, 2021
16622c8
move automatic sharding/unsharding tests from t0260 to t0250
aschmahmann Nov 16, 2021
c393aec
fix typo
aschmahmann Nov 16, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 66 additions & 0 deletions test/sharness/t0250-files-api.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -858,4 +858,70 @@ test_sharding "(cidv1 root)" "--cid-version=1"

test_kill_ipfs_daemon

# Test automatic sharding and unsharding

# We shard based on size with a threshold of 256 KiB (see config file docs)
# above which directories are sharded.
#
# The directory size is estimated as the size of each link. Links are roughly
# the entry name + the CID byte length (e.g. 34 bytes for a CIDv0). So for
# entries of length 10 we need 256 KiB / (34 + 10) ~ 6000 entries in the
# directory to trigger sharding.
test_expect_success "set up automatic sharding/unsharding data" '
mkdir big_dir
for i in `seq 5960` # Just above the number of entries that trigger sharding for 256KiB
do
echo $i > big_dir/`printf "file%06d" $i` # fixed length of 10 chars
done
'

# TODO: This does not need to report an error https://github.com/ipfs/go-ipfs/issues/8088
test_expect_failure "reset automatic sharding" '
ipfs config --json Internal.UnixFSShardingSizeThreshold null
'

test_launch_ipfs_daemon_without_network

LARGE_SHARDED="QmWfjnRWRvdvYezQWnfbvrvY7JjrpevsE9cato1x76UqGr"
LARGE_MINUS_5_UNSHARDED="QmbVxi5zDdzytrjdufUejM92JsWj8wGVmukk6tiPce3p1m"

test_add_large_sharded_dir() {
exphash="$1"
test_expect_success "ipfs add on directory succeeds" '
ipfs add -r -Q big_dir > shardbigdir_out &&
echo "$exphash" > shardbigdir_exp &&
test_cmp shardbigdir_exp shardbigdir_out
'

test_expect_success "can access a path under the dir" '
ipfs cat "$exphash/file000030" > file30_out &&
test_cmp big_dir/file000030 file30_out
'
}

test_add_large_sharded_dir "$LARGE_SHARDED"

test_expect_success "remove a few entries from big_dir/ to trigger unsharding" '
ipfs files cp /ipfs/"$LARGE_SHARDED" /big_dir &&
for i in `seq 5`
do
ipfs files rm /big_dir/`printf "file%06d" $i`
done &&
ipfs files stat --hash /big_dir > unshard_dir_hash &&
echo "$LARGE_MINUS_5_UNSHARDED" > unshard_exp &&
test_cmp unshard_exp unshard_dir_hash
'

test_expect_success "add a few entries to big_dir/ to retrigger sharding" '
for i in `seq 5`
do
ipfs files cp /ipfs/"$LARGE_SHARDED"/`printf "file%06d" $i` /big_dir/`printf "file%06d" $i`
done &&
ipfs files stat --hash /big_dir > shard_dir_hash &&
echo "$LARGE_SHARDED" > shard_exp &&
test_cmp shard_exp shard_dir_hash
'

test_kill_ipfs_daemon

test_done
29 changes: 17 additions & 12 deletions test/sharness/t0260-sharding.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,14 @@ test_expect_success "set up test data" '
done
'

test_add_large_dir() {
test_add_dir() {
exphash="$1"
test_expect_success "ipfs add on very large directory succeeds" '
test_expect_success "ipfs add on directory succeeds" '
ipfs add -r -Q testdata > sharddir_out &&
echo "$exphash" > sharddir_exp &&
test_cmp sharddir_exp sharddir_out
'
test_expect_success "ipfs get on very large directory succeeds" '
test_expect_success "ipfs get on directory succeeds" '
ipfs get -o testdata-out "$exphash" &&
test_cmp testdata testdata-out
'
Expand All @@ -32,24 +32,29 @@ test_add_large_dir() {
test_init_ipfs

UNSHARDED="QmavrTrQG4VhoJmantURAYuw3bowq3E2WcvP36NRQDAC1N"
test_add_large_dir "$UNSHARDED"

test_expect_success "force sharding off" '
ipfs config --json Internal.UnixFSShardingSizeThreshold "\"1G\""
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The 0 value disables the options.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ya that's fair although, I'm not sure how I feel about 0 here. I'm not even sure if when go-ipfs processes the config file it should accept 0 as a value. It restores behavior to the go-ipfs <v0.10.0 defaults, but I'm not sure it's obvious to reason about (i.e. the feature is turned off which means data is add as a basic directory and no automatic sharding or unsharding of existing MFS directories).

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

no problem, just flagging that theoretically at least we expose a "disabled" option in UnixFS, but 1G is more than enough to consider it disabled for all intended purposes

'

test_add_dir "$UNSHARDED"

test_launch_ipfs_daemon

test_add_large_dir "$UNSHARDED"
test_add_dir "$UNSHARDED"

test_kill_ipfs_daemon

test_expect_success "enable sharding" '
test_expect_success "force sharding on" '
ipfs config --json Internal.UnixFSShardingSizeThreshold "\"1B\""
'

SHARDED="QmSCJD1KYLhVVHqBK3YyXuoEqHt7vggyJhzoFYbT8v1XYL"
test_add_large_dir "$SHARDED"
test_add_dir "$SHARDED"

test_launch_ipfs_daemon

test_add_large_dir "$SHARDED"
test_add_dir "$SHARDED"

test_kill_ipfs_daemon

Expand Down Expand Up @@ -93,9 +98,9 @@ test_expect_success "'ipfs resolve' can resolve sharded dirs" '

test_kill_ipfs_daemon

test_add_large_dir_v1() {
test_add_dir_v1() {
exphash="$1"
test_expect_success "ipfs add (CIDv1) on very large directory succeeds" '
test_expect_success "ipfs add (CIDv1) on directory succeeds" '
ipfs add -r -Q --cid-version=1 testdata > sharddir_out &&
echo "$exphash" > sharddir_exp &&
test_cmp sharddir_exp sharddir_out
Expand All @@ -109,11 +114,11 @@ test_add_large_dir_v1() {

# this hash implies the directory is CIDv1 and leaf entries are CIDv1 and raw
SHARDEDV1="bafybeibiemewfzzdyhq2l74wrd6qj2oz42usjlktgnlqv4yfawgouaqn4u"
test_add_large_dir_v1 "$SHARDEDV1"
test_add_dir_v1 "$SHARDEDV1"

test_launch_ipfs_daemon

test_add_large_dir_v1 "$SHARDEDV1"
test_add_dir_v1 "$SHARDEDV1"

test_kill_ipfs_daemon

Expand Down