From 6bd9be9c81a1fbc125f3764dbe93733cc3fb0e87 Mon Sep 17 00:00:00 2001 From: Lucas Molas Date: Tue, 18 May 2021 19:52:05 -0300 Subject: [PATCH] fix sharness --- test/sharness/t0260-sharding.sh | 88 +++++++++++++-------------------- 1 file changed, 35 insertions(+), 53 deletions(-) diff --git a/test/sharness/t0260-sharding.sh b/test/sharness/t0260-sharding.sh index e958cd8fb09..dd0de0eaf20 100755 --- a/test/sharness/t0260-sharding.sh +++ b/test/sharness/t0260-sharding.sh @@ -8,32 +8,35 @@ test_description="Test directory sharding" . lib/test-lib.sh -# FIXME: We now have dropped the global binary option of either sharding nothing -# or sharding everything. We now have a threshold of 256 KiB (see `HAMTShardingSize` -# in core/node/groups.go) above which directories are sharded. The directory size -# is estimated as the size of each link (roughly entry name and CID byte length, -# normally 34 bytes). So we need 256 KiB / (34 + 10) ~ 6000 entries in the directory -# (estimating a fixed 10 char for each name) to trigger sharding. -# We also need to update the SHARDED/UNSHARDED CIDs. - +# We shard based on size with a threshold of 256 KiB (see `HAMTShardingSize` +# in core/node/groups.go) above which directories are sharded. +# The directory size is estimated as the size of each link (roughly entry name, +# here of always 10 chars, and CID byte length, normally 34 bytes). So we need +# 256 KiB / (34 + 10) ~ 6000 entries in the directory to trigger sharding. +# We create then two directories: one above the threshold (big_dir) and one +# below (small_dir), and hard-code the CIDs of their sharded and unsharded +# codifications as IPFS directories. test_expect_success "set up test data" ' mkdir big_dir for i in `seq 6500` # just to be sure do - echo $i > big_dir/`printf "file%06d" $i` + echo $i > big_dir/`printf "file%06d" $i` # fixed length of 10 chars done mkdir small_dir for i in `seq 100` do - echo $i > small_dir/file$i + echo $i > small_dir/`printf "file%06d" $i` done ' +# CID of big_dir/ which will be sharded. +SHARDED="Qmbgq1VC2KSWWhdFLvH6mapN1wuY67FZVPYbCGyR5TnYSv" +# CID of small_dir/ which will *not* be sharded. +UNSHARDED="QmdBXmm4HRpUhyzzctbFvi2tLai3XFL1YjmE1qfpJe61NX" test_add_large_dir() { exphash="$1" - input_dir="$2" # FIXME: added a new entry to switch between small and big_dir - # we need to update the calls. + input_dir="$2" test_expect_success "ipfs add on very large directory succeeds" ' ipfs add -r -q $input_dir | tail -n1 > sharddir_out && echo "$exphash" > sharddir_exp && @@ -42,59 +45,37 @@ test_add_large_dir() { test_expect_success "ipfs get on very large directory succeeds" ' ipfs get -o output_dir "$exphash" && test_cmp $input_dir output_dir - rm output_dir -r # FIXME: Cleaning the output directory because we now have - # two different directories and it seems `ipfs get` doesnt - # overwrite just adds files to the output dir. + rm output_dir -r ' } test_init_ipfs -# FIXME: Updating CID based just on the expected output of a failed test run. These -# need to be confirmed. -UNSHARDED_SMALL="QmZedLGyvgWiyGQzAw2GpuLZeqzxmVcUQUbyRkVWx5DkxK" -test_add_large_dir "$UNSHARDED_SMALL" small_dir +test_add_large_dir "$SHARDED" big_dir +test_add_large_dir "$UNSHARDED" small_dir test_launch_ipfs_daemon -test_add_large_dir "$UNSHARDED_SMALL" small_dir +test_add_large_dir "$SHARDED" big_dir +test_add_large_dir "$UNSHARDED" small_dir test_kill_ipfs_daemon -SHARDED_BIG="Qmbgq1VC2KSWWhdFLvH6mapN1wuY67FZVPYbCGyR5TnYSv" -test_add_large_dir "$SHARDED_BIG" big_dir - -test_launch_ipfs_daemon - -test_add_large_dir "$SHARDED_BIG" big_dir - -test_kill_ipfs_daemon - -# FIXME: This test no longer works, we have different directories for sharded -# and unsharded. We might have to duplicate the UNSHARDED/SHARDED for small -# and big dirs. -test_expect_success "sharded and unsharded output look the same" ' - ipfs ls "$SHARDED" | sort > sharded_out && - ipfs ls "$UNSHARDED" | sort > unsharded_out && - test_cmp sharded_out unsharded_out -' - test_expect_success "ipfs cat error output the same" ' test_expect_code 1 ipfs cat "$SHARDED" 2> sharded_err && test_expect_code 1 ipfs cat "$UNSHARDED" 2> unsharded_err && test_cmp sharded_err unsharded_err ' -# FIXME: Should we replicate this test for the small (non sharded) directory as well? test_expect_success "'ipfs ls --resolve-type=false --size=false' admits missing block" ' - ipfs ls "$SHARDED_BIG" | head -1 > first_file && - ipfs ls --size=false "$SHARDED_BIG" | sort > sharded_out_nosize && + ipfs ls "$SHARDED" | head -1 > first_file && + ipfs ls --size=false "$SHARDED" | sort > sharded_out_nosize && read -r HASH _ NAME missing_out && + test_expect_code 1 ipfs cat "$SHARDED/$NAME" && + test_expect_code 1 ipfs ls "$SHARDED" && + ipfs ls --resolve-type=false --size=false "$SHARDED" | sort > missing_out && test_cmp sharded_out_nosize missing_out ' @@ -102,13 +83,13 @@ test_launch_ipfs_daemon test_expect_success "gateway can resolve sharded dirs" ' echo 100 > expected && - curl -sfo actual "http://127.0.0.1:$GWAY_PORT/ipfs/$SHARDED_BIG/file100" && + curl -sfo actual "http://127.0.0.1:$GWAY_PORT/ipfs/$SHARDED/file000100" && test_cmp expected actual ' test_expect_success "'ipfs resolve' can resolve sharded dirs" ' echo /ipfs/QmZ3RfWk1u5LEGYLHA633B5TNJy3Du27K6Fny9wcxpowGS > expected && - ipfs resolve "/ipfs/$SHARDED_BIG/file100" > actual && + ipfs resolve "/ipfs/$SHARDED/file000100" > actual && test_cmp expected actual ' @@ -116,25 +97,26 @@ test_kill_ipfs_daemon test_add_large_dir_v1() { exphash="$1" + input_dir="$2" test_expect_success "ipfs add (CIDv1) on very large directory succeeds" ' - ipfs add -r -q --cid-version=1 testdata | tail -n1 > sharddir_out && + ipfs add -r -q --cid-version=1 "$input_dir" | tail -n1 > sharddir_out && echo "$exphash" > sharddir_exp && test_cmp sharddir_exp sharddir_out ' test_expect_success "can access a path under the dir" ' - ipfs cat "$exphash/file20" > file20_out && - test_cmp testdata/file20 file20_out + ipfs cat "$exphash/file000020" > file20_out && + test_cmp "$input_dir/file000020" file20_out ' } # this hash implies the directory is CIDv1 and leaf entries are CIDv1 and raw -SHARDEDV1="bafybeibiemewfzzdyhq2l74wrd6qj2oz42usjlktgnlqv4yfawgouaqn4u" -test_add_large_dir_v1 "$SHARDEDV1" +SHARDEDV1="bafybeif5mvotyb33vzncgkcorenaj6s7gwg7pgq3jmzuhosup3nrq2x7fi" +test_add_large_dir_v1 "$SHARDEDV1" big_dir test_launch_ipfs_daemon -test_add_large_dir_v1 "$SHARDEDV1" +test_add_large_dir_v1 "$SHARDEDV1" big_dir test_kill_ipfs_daemon