Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: check paths completed for all retrieval types #386

Closed
wants to merge 21 commits into from
Closed

feat: check paths completed for all retrieval types #386

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
d6902ca
feat: support verifying byte range CARs
rvagg Jun 9, 2023
71501f2
feat: remove unixfs-preload use
rvagg Jun 16, 2023
4b36024
chore: test fixtures for various unixfs queries
rvagg Jun 27, 2023
145cf04
feat: make byte ranges work, lots of tests
rvagg Jun 28, 2023
33a31e7
fix: make work on windows
rvagg Jun 28, 2023
0a9c368
feat: support _or_ matching when entity-bytes query reaches non-bytes
rvagg Jun 29, 2023
4d921b2
feat: support entity-bytes via daemon, add unixfs fixture itests
rvagg Jun 30, 2023
7f21fd6
feat: add entity-bytes to fetch cli
rvagg Jun 30, 2023
287ffe2
chore: tests for etag w/ byte range
rvagg Jun 30, 2023
c3e621d
feat: add bitswap and graphsync tests to trustless fixture suite
rvagg Jun 30, 2023
a34f177
fix: address feedback
rvagg Jul 3, 2023
3652817
chore: extract test fixture data to ipld/ipld/specs repo
rvagg Jul 4, 2023
cd7d7a7
fix: restore old car-scope parsing
rvagg Jul 5, 2023
448f431
fix: update dependencies
rvagg Jul 7, 2023
3cd75d6
chore(mod): update deps to tagged
hannahhoward Jul 7, 2023
b362305
fix: correct error messages for http handler
rvagg Aug 4, 2023
cb82950
chore: minor cleanups around byte ranges
rvagg Jul 12, 2023
365f73e
fix(bitswap): update boxo dependency to deal with bitswap race
rvagg Aug 16, 2023
3328409
feat: negative byte ranges
rvagg Aug 16, 2023
c40fe59
chore: update go-car/v2 dep to tagged version
rvagg Aug 22, 2023
88ccb73
feat: check paths completed for all retrieval types
rvagg Aug 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 23 additions & 1 deletion cmd/lassie/fetch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,19 @@ var fetchFlags = []cli.Flag{
return nil
},
},
&cli.StringFlag{
Name: "entity-bytes",
Usage: "describes the byte range to consider when selecting the blocks from a sharded file." +
" Valid values should be of the form from:to, where from and to are byte offsets and to may be '*'",
DefaultText: "defaults to the entire file, 0:*",
Action: func(cctx *cli.Context, v string) error {
if _, err := types.ParseByteRange(v); err != nil {
return fmt.Errorf("invalid entity-bytes parameter, must be of the form from:to," +
" where from and to are byte offsets and to may be '*'")
}
return nil
},
},
FlagIPNIEndpoint,
FlagEventRecorderAuth,
FlagEventRecorderInstanceId,
Expand Down Expand Up @@ -80,6 +93,7 @@ func fetchAction(cctx *cli.Context) error {
dataWriter := cctx.App.Writer

dagScope := cctx.String("dag-scope")
entityBytes := cctx.String("entity-bytes")
tempDir := cctx.String("tempdir")
progress := cctx.Bool("progress")

Expand Down Expand Up @@ -113,6 +127,7 @@ func fetchAction(cctx *cli.Context) error {
rootCid,
path,
dagScope,
entityBytes,
tempDir,
progress,
outfile,
Expand Down Expand Up @@ -190,6 +205,7 @@ type fetchRunFunc func(
rootCid cid.Cid,
path string,
dagScope string,
entityBytes string,
tempDir string,
progress bool,
outfile string,
Expand All @@ -209,6 +225,7 @@ func defaultFetchRun(
rootCid cid.Cid,
path string,
dagScope string,
entityBytes string,
tempDir string,
progress bool,
outfile string,
Expand Down Expand Up @@ -260,7 +277,12 @@ func defaultFetchRun(
}
}, false)

request, err := types.NewRequestForPath(carStore, rootCid, path, types.DagScope(dagScope))
byteRange, _ := types.ParseByteRange(entityBytes)
var br *types.ByteRange
if !byteRange.IsDefault() {
br = &byteRange
}
request, err := types.NewRequestForPath(carStore, rootCid, path, types.DagScope(dagScope), br)
if err != nil {
return err
}
Expand Down
76 changes: 58 additions & 18 deletions cmd/lassie/fetch_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,12 @@ func TestFetchCommandFlags(t *testing.T) {
{
name: "with default args",
args: []string{"fetch", "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4"},
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
// fetch specific params
require.Equal(t, "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4", rootCid.String())
require.Equal(t, "", path)
require.Equal(t, string(types.DagScopeAll), dagScope)
require.Empty(t, entityBytes)
require.Equal(t, false, progress)
require.Equal(t, "bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4.car", outfile)

Expand Down Expand Up @@ -70,7 +71,7 @@ func TestFetchCommandFlags(t *testing.T) {
"fetch",
"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4/birb.mp4",
},
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
require.Equal(t, "/birb.mp4", path)
return nil
},
Expand All @@ -83,7 +84,7 @@ func TestFetchCommandFlags(t *testing.T) {
"entity",
"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
},
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
require.Equal(t, string(types.DagScopeEntity), dagScope)
return nil
},
Expand All @@ -96,19 +97,58 @@ func TestFetchCommandFlags(t *testing.T) {
"block",
"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
},
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
require.Equal(t, string(types.DagScopeBlock), dagScope)
return nil
},
},
{
name: "with entity-bytes 0:*",
args: []string{
"fetch",
"--entity-bytes",
"0:*",
"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
},
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
require.Equal(t, "0:*", entityBytes)
return nil
},
},
{
name: "with entity-bytes 0:10",
args: []string{
"fetch",
"--entity-bytes",
"0:10",
"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
},
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
require.Equal(t, "0:10", entityBytes)
return nil
},
},
{
name: "with entity-bytes 1000:20000",
args: []string{
"fetch",
"--entity-bytes",
"1000:20000",
"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
},
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
require.Equal(t, "1000:20000", entityBytes)
return nil
},
},
{
name: "with progress",
args: []string{
"fetch",
"--progress",
"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
},
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
require.Equal(t, true, progress)
return nil
},
Expand All @@ -121,7 +161,7 @@ func TestFetchCommandFlags(t *testing.T) {
"myfile",
"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
},
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
require.Equal(t, "myfile", outfile)
return nil
},
Expand All @@ -134,7 +174,7 @@ func TestFetchCommandFlags(t *testing.T) {
"/ip4/127.0.0.1/tcp/5000/p2p/12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4",
"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
},
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
require.IsType(t, &retriever.DirectCandidateFinder{}, lCfg.Finder, "finder should be a DirectCandidateFinder when providers are specified")
return nil
},
Expand All @@ -147,7 +187,7 @@ func TestFetchCommandFlags(t *testing.T) {
"https://cid.contact",
"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
},
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
require.IsType(t, &indexerlookup.IndexerCandidateFinder{}, lCfg.Finder, "finder should be an IndexerCandidateFinder when providing an ipni endpoint")
return nil
},
Expand All @@ -170,7 +210,7 @@ func TestFetchCommandFlags(t *testing.T) {
"/mytmpdir",
"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
},
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
require.Equal(t, "/mytmpdir", tempDir)
return nil
},
Expand All @@ -183,7 +223,7 @@ func TestFetchCommandFlags(t *testing.T) {
"30s",
"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
},
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
require.Equal(t, 30*time.Second, lCfg.ProviderTimeout)
return nil
},
Expand All @@ -196,7 +236,7 @@ func TestFetchCommandFlags(t *testing.T) {
"30s",
"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
},
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
require.Equal(t, 30*time.Second, lCfg.GlobalTimeout)
return nil
},
Expand All @@ -209,7 +249,7 @@ func TestFetchCommandFlags(t *testing.T) {
"bitswap,graphsync",
"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
},
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
require.Equal(t, []multicodec.Code{multicodec.TransportBitswap, multicodec.TransportGraphsyncFilecoinv1}, lCfg.Protocols)
return nil
},
Expand All @@ -222,7 +262,7 @@ func TestFetchCommandFlags(t *testing.T) {
"12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4,12D3KooWPNbkEgjdBNeaCGpsgCrPRETe4uBZf1ShFXStobdN18ys",
"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
},
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
p1, err := peer.Decode("12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4")
require.NoError(t, err)
p2, err := peer.Decode("12D3KooWPNbkEgjdBNeaCGpsgCrPRETe4uBZf1ShFXStobdN18ys")
Expand All @@ -241,7 +281,7 @@ func TestFetchCommandFlags(t *testing.T) {
"10",
"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
},
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
require.Equal(t, 10, lCfg.BitswapConcurrency)
return nil
},
Expand All @@ -254,7 +294,7 @@ func TestFetchCommandFlags(t *testing.T) {
"https://myeventrecorder.com/v1/retrieval-events",
"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
},
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
require.Equal(t, "https://myeventrecorder.com/v1/retrieval-events", erCfg.EndpointURL)
return nil
},
Expand All @@ -267,7 +307,7 @@ func TestFetchCommandFlags(t *testing.T) {
"secret",
"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
},
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
require.Equal(t, "secret", erCfg.EndpointAuthorization)
return nil
},
Expand All @@ -280,7 +320,7 @@ func TestFetchCommandFlags(t *testing.T) {
"myinstanceid",
"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4",
},
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
assertRun: func(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
require.Equal(t, "myinstanceid", erCfg.InstanceID)
return nil
},
Expand Down Expand Up @@ -312,6 +352,6 @@ func TestFetchCommandFlags(t *testing.T) {
}
}

func noopRun(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, tempDir string, progress bool, outfile string) error {
func noopRun(ctx context.Context, lCfg *l.LassieConfig, erCfg *a.EventRecorderConfig, msgWriter io.Writer, dataWriter io.Writer, rootCid cid.Cid, path string, dagScope string, entityBytes string, tempDir string, progress bool, outfile string) error {
return nil
}
Loading
Loading