-
-
Notifications
You must be signed in to change notification settings - Fork 3k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
improve gateway symlink handling #6680
Merged
Merged
Changes from all commits
Commits
Show all changes
9 commits
Select commit
Hold shift + click to select a range
36db5bf
dep: update go-ipfs-files/go-unixfs
Stebalien 6245103
fix(gateway): serve the index with serveFile
Stebalien e8a6c0c
fix(gateway): gracefully handle files with unknown sizes in directory…
Stebalien 3859f08
fix(gateway): better seeking/sized
Stebalien 1a06fb6
fix(gateway): correct symlink content type
Stebalien 9a9ec02
test(sharness): add gateway symlink test
Stebalien 453b789
chore(gateway): remove dead code
Stebalien 6cb03d4
fix(gateway): fix seek read length typo
Stebalien c64eb11
test(gateway): test the lazy seeker
Stebalien File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| package corehttp | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "io" | ||
| ) | ||
|
|
||
| // The HTTP server uses seek to determine the file size. Actually _seeking_ can | ||
| // be slow so we wrap the seeker in a _lazy_ seeker. | ||
| type lazySeeker struct { | ||
| reader io.ReadSeeker | ||
|
|
||
| size int64 | ||
| offset int64 | ||
| realOffset int64 | ||
| } | ||
|
|
||
| func (s *lazySeeker) Seek(offset int64, whence int) (int64, error) { | ||
| switch whence { | ||
| case io.SeekEnd: | ||
| return s.Seek(s.size+offset, io.SeekStart) | ||
| case io.SeekCurrent: | ||
| return s.Seek(s.offset+offset, io.SeekStart) | ||
| case io.SeekStart: | ||
| if offset < 0 { | ||
| return s.offset, fmt.Errorf("invalid seek offset") | ||
| } | ||
| s.offset = offset | ||
| return s.offset, nil | ||
| default: | ||
| return s.offset, fmt.Errorf("invalid whence: %d", whence) | ||
| } | ||
| } | ||
|
|
||
| func (s *lazySeeker) Read(b []byte) (int, error) { | ||
| // If we're past the end, EOF. | ||
| if s.offset >= s.size { | ||
| return 0, io.EOF | ||
| } | ||
|
|
||
| // actually seek | ||
| for s.offset != s.realOffset { | ||
| off, err := s.reader.Seek(s.offset, io.SeekStart) | ||
| if err != nil { | ||
| return 0, err | ||
| } | ||
| s.realOffset = off | ||
| } | ||
| off, err := s.reader.Read(b) | ||
| s.realOffset += int64(off) | ||
| s.offset += int64(off) | ||
| return off, err | ||
| } | ||
|
|
||
| func (s *lazySeeker) Close() error { | ||
| if closer, ok := s.reader.(io.Closer); ok { | ||
| return closer.Close() | ||
| } | ||
| return nil | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,137 @@ | ||
| package corehttp | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "io" | ||
| "io/ioutil" | ||
| "strings" | ||
| "testing" | ||
| ) | ||
|
|
||
| type badSeeker struct { | ||
| io.ReadSeeker | ||
| } | ||
|
|
||
| var badSeekErr = fmt.Errorf("I'm a bad seeker") | ||
|
|
||
| func (bs badSeeker) Seek(offset int64, whence int) (int64, error) { | ||
| off, err := bs.ReadSeeker.Seek(0, io.SeekCurrent) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
| return off, badSeekErr | ||
| } | ||
|
|
||
| func TestLazySeekerError(t *testing.T) { | ||
| underlyingBuffer := strings.NewReader("fubar") | ||
| s := &lazySeeker{ | ||
| reader: badSeeker{underlyingBuffer}, | ||
| size: underlyingBuffer.Size(), | ||
| } | ||
| off, err := s.Seek(0, io.SeekEnd) | ||
| if err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| if off != s.size { | ||
| t.Fatal("expected to seek to the end") | ||
| } | ||
|
|
||
| // shouldn't have actually seeked. | ||
| b, err := ioutil.ReadAll(s) | ||
| if err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| if len(b) != 0 { | ||
| t.Fatal("expected to read nothing") | ||
| } | ||
|
|
||
| // shouldn't need to actually seek. | ||
| off, err = s.Seek(0, io.SeekStart) | ||
| if err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| if off != 0 { | ||
| t.Fatal("expected to seek to the start") | ||
| } | ||
| b, err = ioutil.ReadAll(s) | ||
| if err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| if string(b) != "fubar" { | ||
| t.Fatal("expected to read string") | ||
| } | ||
|
|
||
| // should fail the second time. | ||
| off, err = s.Seek(0, io.SeekStart) | ||
| if err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| if off != 0 { | ||
| t.Fatal("expected to seek to the start") | ||
| } | ||
| // right here... | ||
| b, err = ioutil.ReadAll(s) | ||
| if err == nil { | ||
| t.Fatalf("expected an error, got output %s", string(b)) | ||
| } | ||
| if err != badSeekErr { | ||
| t.Fatalf("expected a bad seek error, got %s", err) | ||
| } | ||
| if len(b) != 0 { | ||
| t.Fatalf("expected to read nothing") | ||
| } | ||
| } | ||
|
|
||
| func TestLazySeeker(t *testing.T) { | ||
| underlyingBuffer := strings.NewReader("fubar") | ||
| s := &lazySeeker{ | ||
| reader: underlyingBuffer, | ||
| size: underlyingBuffer.Size(), | ||
| } | ||
| expectByte := func(b byte) { | ||
| t.Helper() | ||
| var buf [1]byte | ||
| n, err := io.ReadFull(s, buf[:]) | ||
| if err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| if n != 1 { | ||
| t.Fatalf("expected to read one byte, read %d", n) | ||
| } | ||
| if buf[0] != b { | ||
| t.Fatalf("expected %b, got %b", b, buf[0]) | ||
| } | ||
| } | ||
| expectSeek := func(whence int, off, expOff int64, expErr string) { | ||
| t.Helper() | ||
| n, err := s.Seek(off, whence) | ||
| if expErr == "" { | ||
| if err != nil { | ||
| t.Fatal("unexpected seek error: ", err) | ||
| } | ||
| } else { | ||
| if err == nil || err.Error() != expErr { | ||
| t.Fatalf("expected %s, got %s", err, expErr) | ||
| } | ||
| } | ||
| if n != expOff { | ||
| t.Fatalf("expected offset %d, got, %d", expOff, n) | ||
| } | ||
| } | ||
|
|
||
| expectSeek(io.SeekEnd, 0, s.size, "") | ||
| b, err := ioutil.ReadAll(s) | ||
| if err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| if len(b) != 0 { | ||
| t.Fatal("expected to read nothing") | ||
| } | ||
| expectSeek(io.SeekEnd, -1, s.size-1, "") | ||
| expectByte('r') | ||
| expectSeek(io.SeekStart, 0, 0, "") | ||
| expectByte('f') | ||
| expectSeek(io.SeekCurrent, 1, 2, "") | ||
| expectByte('b') | ||
| expectSeek(io.SeekCurrent, -100, 3, "invalid seek offset") | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| #!/usr/bin/env bash | ||
| # | ||
| # Copyright (c) Protocol Labs | ||
|
|
||
| test_description="Test symlink support on the HTTP gateway" | ||
|
|
||
| . lib/test-lib.sh | ||
|
|
||
| test_init_ipfs | ||
| test_launch_ipfs_daemon | ||
|
|
||
|
|
||
| test_expect_success "Create a test directory with symlinks" ' | ||
| mkdir testfiles && | ||
| echo "content" > testfiles/foo && | ||
| ln -s foo testfiles/bar && | ||
| test_cmp testfiles/foo testfiles/bar | ||
| ' | ||
|
|
||
| test_expect_success "Add the test directory" ' | ||
| HASH=$(ipfs add -Qr testfiles) | ||
| ' | ||
|
|
||
| test_expect_success "Test the directory listing" ' | ||
| curl "$GWAY_ADDR/ipfs/$HASH" > list_response && | ||
| test_should_contain ">foo<" list_response && | ||
| test_should_contain ">bar<" list_response | ||
| ' | ||
|
|
||
| test_expect_success "Test the symlink" ' | ||
| curl "$GWAY_ADDR/ipfs/$HASH/bar" > bar_actual && | ||
| echo -n "foo" > bar_expected && | ||
| test_cmp bar_expected bar_actual | ||
| ' | ||
|
|
||
| test_kill_ipfs_daemon | ||
|
|
||
| test_done |
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
isn't it
s.size-offset?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
If we're seeking from the end, the offset will already be negative.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Actually, this was wrong. It should have beenNvm. Seeking to the end is correctly done with as.size-1+offset.Seek(-1, SeekEnd).