diff --git a/cmd/send/send.go b/cmd/send/send.go index 721e39c4..4c1c939e 100644 --- a/cmd/send/send.go +++ b/cmd/send/send.go @@ -18,7 +18,11 @@ func main() { ctx := context.Background() s := util.NewProtoStream(ctx, os.Stdin, os.Stdout) - if err := fsutil.Send(ctx, s, fsutil.NewFS(flag.Args()[0], nil), nil); err != nil { + fs, err := fsutil.NewFS(flag.Args()[0]) + if err != nil { + panic(err) + } + if err := fsutil.Send(ctx, s, fs, nil); err != nil { panic(err) } } diff --git a/cmd/walk/walk.go b/cmd/walk/walk.go index 2b4311ab..d558279b 100644 --- a/cmd/walk/walk.go +++ b/cmd/walk/walk.go @@ -25,7 +25,7 @@ func main() { excludes = strings.Split(string(dt), "\n") } - if err := fsutil.Walk(context.Background(), flag.Args()[0], &fsutil.WalkOpt{ + if err := fsutil.Walk(context.Background(), flag.Args()[0], &fsutil.FilterOpt{ ExcludePatterns: excludes, }, func(path string, fi os.FileInfo, err error) error { if err != nil { diff --git a/walker.go b/filter.go similarity index 54% rename from walker.go rename to filter.go index 545f5e90..05f4a466 100644 --- a/walker.go +++ b/filter.go @@ -2,25 +2,35 @@ package fsutil import ( "context" + "io" gofs "io/fs" "os" "path/filepath" "strings" "syscall" - "time" "github.com/moby/patternmatcher" "github.com/pkg/errors" "github.com/tonistiigi/fsutil/types" ) -type WalkOpt struct { +type FilterOpt struct { + // IncludePatterns requires that the path matches at least one of the + // specified patterns. IncludePatterns []string + + // ExcludePatterns requires that the path does not match any of the + // specified patterns. ExcludePatterns []string - // FollowPaths contains symlinks that are resolved into include patterns - // before performing the fs walk + + // FollowPaths contains symlinks that are resolved into IncludePatterns + // at the time of the call to NewFilterFS. FollowPaths []string - Map MapFunc + + // Map is called for each path that is included in the result. + // The function can modify the stat info for each element, while the result + // of the function controls both how Walk continues. + Map MapFunc } type MapFunc func(string, *types.Stat) MapResult @@ -43,33 +53,41 @@ const ( MapResultSkipDir ) -func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) error { - root, err := filepath.EvalSymlinks(p) - if err != nil { - return errors.WithStack(&os.PathError{Op: "resolve", Path: root, Err: err}) - } - rootFI, err := os.Stat(root) - if err != nil { - return errors.WithStack(err) - } - if !rootFI.IsDir() { - return errors.WithStack(&os.PathError{Op: "walk", Path: root, Err: syscall.ENOTDIR}) - } +type filterFS struct { + fs FS - var ( - includePatterns []string - includeMatcher *patternmatcher.PatternMatcher - excludeMatcher *patternmatcher.PatternMatcher - ) + includeMatcher *patternmatcher.PatternMatcher + excludeMatcher *patternmatcher.PatternMatcher + onlyPrefixIncludes bool + onlyPrefixExcludeExceptions bool + + mapFn MapFunc +} + +// NewFilterFS creates a new FS that filters the given FS using the given +// FilterOpt. + +// The returned FS will not contain any paths that do not match the provided +// include and exclude patterns, or that are are exlcluded using the mapping +// function. +// +// The FS is assumed to be a snapshot of the filesystem at the time of the +// call to NewFilterFS. If the underlying filesystem changes, calls to the +// underlying FS may be inconsistent. +func NewFilterFS(fs FS, opt *FilterOpt) (FS, error) { + if opt == nil { + return fs, nil + } - if opt != nil && opt.IncludePatterns != nil { + var includePatterns []string + if opt.IncludePatterns != nil { includePatterns = make([]string, len(opt.IncludePatterns)) copy(includePatterns, opt.IncludePatterns) } - if opt != nil && opt.FollowPaths != nil { - targets, err := FollowLinks(p, opt.FollowPaths) + if opt.FollowPaths != nil { + targets, err := FollowLinks(fs, opt.FollowPaths) if err != nil { - return err + return nil, err } if targets != nil { includePatterns = append(includePatterns, targets...) @@ -82,11 +100,18 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err patternChars += `\` } - onlyPrefixIncludes := true - if len(includePatterns) != 0 { + var ( + includeMatcher *patternmatcher.PatternMatcher + excludeMatcher *patternmatcher.PatternMatcher + err error + onlyPrefixIncludes = true + onlyPrefixExcludeExceptions = true + ) + + if len(includePatterns) > 0 { includeMatcher, err = patternmatcher.New(includePatterns) if err != nil { - return errors.Wrapf(err, "invalid includepatterns: %s", opt.IncludePatterns) + return nil, errors.Wrapf(err, "invalid includepatterns: %s", opt.IncludePatterns) } for _, p := range includeMatcher.Patterns() { @@ -98,11 +123,10 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err } - onlyPrefixExcludeExceptions := true - if opt != nil && opt.ExcludePatterns != nil { + if len(opt.ExcludePatterns) > 0 { excludeMatcher, err = patternmatcher.New(opt.ExcludePatterns) if err != nil { - return errors.Wrapf(err, "invalid excludepatterns: %s", opt.ExcludePatterns) + return nil, errors.Wrapf(err, "invalid excludepatterns: %s", opt.ExcludePatterns) } for _, p := range excludeMatcher.Patterns() { @@ -113,10 +137,41 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err } } + return &filterFS{ + fs: fs, + includeMatcher: includeMatcher, + excludeMatcher: excludeMatcher, + onlyPrefixIncludes: onlyPrefixIncludes, + onlyPrefixExcludeExceptions: onlyPrefixExcludeExceptions, + mapFn: opt.Map, + }, nil +} + +func (fs *filterFS) Open(p string) (io.ReadCloser, error) { + if fs.includeMatcher != nil { + m, err := fs.includeMatcher.MatchesOrParentMatches(p) + if err != nil { + return nil, err + } + if !m { + return nil, errors.Wrapf(os.ErrNotExist, "open %s", p) + } + } + if fs.excludeMatcher != nil { + m, err := fs.excludeMatcher.MatchesOrParentMatches(p) + if err != nil { + return nil, err + } + if m { + return nil, errors.Wrapf(os.ErrNotExist, "open %s", p) + } + } + return fs.fs.Open(p) +} + +func (fs *filterFS) Walk(ctx context.Context, target string, fn gofs.WalkDirFunc) error { type visitedDir struct { - fi os.FileInfo - path string - origpath string + entry gofs.DirEntry pathWithSep string includeMatchInfo patternmatcher.MatchInfo excludeMatchInfo patternmatcher.MatchInfo @@ -126,34 +181,22 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err // used only for include/exclude handling var parentDirs []visitedDir - seenFiles := make(map[uint64]string) - return filepath.WalkDir(root, func(path string, dirEntry gofs.DirEntry, walkErr error) (retErr error) { + return fs.fs.Walk(ctx, target, func(path string, dirEntry gofs.DirEntry, walkErr error) (retErr error) { defer func() { if retErr != nil && isNotExist(retErr) { retErr = filepath.SkipDir } }() - origpath := path - path, err = filepath.Rel(root, path) - if err != nil { - return err - } - // Skip root - if path == "." { - return nil - } - var ( dir visitedDir isDir bool - fi gofs.FileInfo ) if dirEntry != nil { isDir = dirEntry.IsDir() } - if includeMatcher != nil || excludeMatcher != nil { + if fs.includeMatcher != nil || fs.excludeMatcher != nil { for len(parentDirs) != 0 { lastParentDir := parentDirs[len(parentDirs)-1].pathWithSep if strings.HasPrefix(path, lastParentDir) { @@ -163,15 +206,8 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err } if isDir { - fi, err = dirEntry.Info() - if err != nil { - return err - } - dir = visitedDir{ - fi: fi, - path: path, - origpath: origpath, + entry: dirEntry, pathWithSep: path + string(filepath.Separator), } } @@ -179,12 +215,12 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err skip := false - if includeMatcher != nil { + if fs.includeMatcher != nil { var parentIncludeMatchInfo patternmatcher.MatchInfo if len(parentDirs) != 0 { parentIncludeMatchInfo = parentDirs[len(parentDirs)-1].includeMatchInfo } - m, matchInfo, err := includeMatcher.MatchesUsingParentResults(path, parentIncludeMatchInfo) + m, matchInfo, err := fs.includeMatcher.MatchesUsingParentResults(path, parentIncludeMatchInfo) if err != nil { return errors.Wrap(err, "failed to match includepatterns") } @@ -194,11 +230,11 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err } if !m { - if isDir && onlyPrefixIncludes { + if isDir && fs.onlyPrefixIncludes { // Optimization: we can skip walking this dir if no include // patterns could match anything inside it. dirSlash := path + string(filepath.Separator) - for _, pat := range includeMatcher.Patterns() { + for _, pat := range fs.includeMatcher.Patterns() { if pat.Exclusion() { continue } @@ -214,12 +250,12 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err } } - if excludeMatcher != nil { + if fs.excludeMatcher != nil { var parentExcludeMatchInfo patternmatcher.MatchInfo if len(parentDirs) != 0 { parentExcludeMatchInfo = parentDirs[len(parentDirs)-1].excludeMatchInfo } - m, matchInfo, err := excludeMatcher.MatchesUsingParentResults(path, parentExcludeMatchInfo) + m, matchInfo, err := fs.excludeMatcher.MatchesUsingParentResults(path, parentExcludeMatchInfo) if err != nil { return errors.Wrap(err, "failed to match excludepatterns") } @@ -229,16 +265,16 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err } if m { - if isDir && onlyPrefixExcludeExceptions { + if isDir && fs.onlyPrefixExcludeExceptions { // Optimization: we can skip walking this dir if no // exceptions to exclude patterns could match anything // inside it. - if !excludeMatcher.Exclusions() { + if !fs.excludeMatcher.Exclusions() { return filepath.SkipDir } dirSlash := path + string(filepath.Separator) - for _, pat := range excludeMatcher.Patterns() { + for _, pat := range fs.excludeMatcher.Patterns() { if !pat.Exclusion() { continue } @@ -261,7 +297,7 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err return walkErr } - if includeMatcher != nil || excludeMatcher != nil { + if fs.includeMatcher != nil || fs.excludeMatcher != nil { defer func() { if isDir { parentDirs = append(parentDirs, dir) @@ -275,25 +311,21 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err dir.calledFn = true - // The FileInfo might have already been read further up. - if fi == nil { - fi, err = dirEntry.Info() - if err != nil { - return err - } - } - - stat, err := mkstat(origpath, path, fi, seenFiles) + fi, err := dirEntry.Info() if err != nil { return err } + stat, ok := fi.Sys().(*types.Stat) + if !ok { + return errors.WithStack(&os.PathError{Path: path, Err: syscall.EBADMSG, Op: "fileinfo without stat info"}) + } select { case <-ctx.Done(): return ctx.Err() default: - if opt != nil && opt.Map != nil { - result := opt.Map(stat.Path, stat) + if fs.mapFn != nil { + result := fs.mapFn(stat.Path, stat) if result == MapResultSkipDir { return filepath.SkipDir } else if result == MapResultExclude { @@ -304,29 +336,33 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err if parentDir.calledFn { continue } - parentStat, err := mkstat(parentDir.origpath, parentDir.path, parentDir.fi, seenFiles) + parentFi, err := parentDir.entry.Info() if err != nil { return err } + parentStat, ok := parentFi.Sys().(*types.Stat) + if !ok { + return errors.WithStack(&os.PathError{Path: path, Err: syscall.EBADMSG, Op: "fileinfo without stat info"}) + } select { case <-ctx.Done(): return ctx.Err() default: } - if opt != nil && opt.Map != nil { - result := opt.Map(parentStat.Path, parentStat) + if fs.mapFn != nil { + result := fs.mapFn(parentStat.Path, parentStat) if result == MapResultSkipDir || result == MapResultExclude { continue } } - if err := fn(parentStat.Path, &StatInfo{parentStat}, nil); err != nil { + if err := fn(parentStat.Path, &DirEntryInfo{Stat: parentStat}, nil); err != nil { return err } parentDirs[i].calledFn = true } - if err := fn(stat.Path, &StatInfo{stat}, nil); err != nil { + if err := fn(stat.Path, &DirEntryInfo{Stat: stat}, nil); err != nil { return err } } @@ -334,6 +370,40 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err }) } +func Walk(ctx context.Context, p string, opt *FilterOpt, fn filepath.WalkFunc) error { + f, err := NewFS(p) + if err != nil { + return err + } + f, err = NewFilterFS(f, opt) + if err != nil { + return err + } + return f.Walk(ctx, "/", func(path string, d gofs.DirEntry, err error) error { + var info gofs.FileInfo + if d != nil { + var err2 error + info, err2 = d.Info() + if err == nil { + err = err2 + } + } + return fn(path, info, err) + }) +} + +func WalkDir(ctx context.Context, p string, opt *FilterOpt, fn gofs.WalkDirFunc) error { + f, err := NewFS(p) + if err != nil { + return err + } + f, err = NewFilterFS(f, opt) + if err != nil { + return err + } + return f.Walk(ctx, "/", fn) +} + func patternWithoutTrailingGlob(p *patternmatcher.Pattern) string { patStr := p.String() // We use filepath.Separator here because patternmatcher.Pattern patterns @@ -344,29 +414,6 @@ func patternWithoutTrailingGlob(p *patternmatcher.Pattern) string { return patStr } -type StatInfo struct { - *types.Stat -} - -func (s *StatInfo) Name() string { - return filepath.Base(s.Stat.Path) -} -func (s *StatInfo) Size() int64 { - return s.Stat.Size_ -} -func (s *StatInfo) Mode() os.FileMode { - return os.FileMode(s.Stat.Mode) -} -func (s *StatInfo) ModTime() time.Time { - return time.Unix(s.Stat.ModTime/1e9, s.Stat.ModTime%1e9) -} -func (s *StatInfo) IsDir() bool { - return s.Mode().IsDir() -} -func (s *StatInfo) Sys() interface{} { - return s.Stat -} - func isNotExist(err error) bool { return errors.Is(err, os.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) } diff --git a/walker_test.go b/filter_test.go similarity index 69% rename from walker_test.go rename to filter_test.go index cb4ea87b..5d25ffdd 100644 --- a/walker_test.go +++ b/filter_test.go @@ -4,6 +4,8 @@ import ( "bytes" "context" "fmt" + "io" + gofs "io/fs" "net" "os" "path/filepath" @@ -34,6 +36,18 @@ file foo2 } +func TestInvalidExcludePatterns(t *testing.T) { + d, err := tmpDir(changeStream([]string{ + "ADD foo file data1", + })) + assert.NoError(t, err) + defer os.RemoveAll(d) + fs, err := NewFS(d) + assert.NoError(t, err) + _, err = NewFilterFS(fs, &FilterOpt{ExcludePatterns: []string{"!"}}) + assert.Error(t, err) +} + func TestWalkerInclude(t *testing.T) { d, err := tmpDir(changeStream([]string{ "ADD bar dir", @@ -43,7 +57,7 @@ func TestWalkerInclude(t *testing.T) { assert.NoError(t, err) defer os.RemoveAll(d) b := &bytes.Buffer{} - err = Walk(context.Background(), d, &WalkOpt{ + err = Walk(context.Background(), d, &FilterOpt{ IncludePatterns: []string{"bar"}, }, bufWalk(b)) assert.NoError(t, err) @@ -53,7 +67,7 @@ file bar/foo `, string(b.Bytes())) b.Reset() - err = Walk(context.Background(), d, &WalkOpt{ + err = Walk(context.Background(), d, &FilterOpt{ IncludePatterns: []string{"bar/foo"}, }, bufWalk(b)) assert.NoError(t, err) @@ -63,7 +77,7 @@ file bar/foo `, string(b.Bytes())) b.Reset() - err = Walk(context.Background(), d, &WalkOpt{ + err = Walk(context.Background(), d, &FilterOpt{ IncludePatterns: []string{"b*"}, }, bufWalk(b)) assert.NoError(t, err) @@ -73,7 +87,7 @@ file bar/foo `, string(b.Bytes())) b.Reset() - err = Walk(context.Background(), d, &WalkOpt{ + err = Walk(context.Background(), d, &FilterOpt{ IncludePatterns: []string{"bar/f*"}, }, bufWalk(b)) assert.NoError(t, err) @@ -83,7 +97,7 @@ file bar/foo `, string(b.Bytes())) b.Reset() - err = Walk(context.Background(), d, &WalkOpt{ + err = Walk(context.Background(), d, &FilterOpt{ IncludePatterns: []string{"bar/g*"}, }, bufWalk(b)) assert.NoError(t, err) @@ -91,7 +105,7 @@ file bar/foo assert.Empty(t, b.Bytes()) b.Reset() - err = Walk(context.Background(), d, &WalkOpt{ + err = Walk(context.Background(), d, &FilterOpt{ IncludePatterns: []string{"f*"}, }, bufWalk(b)) assert.NoError(t, err) @@ -100,7 +114,7 @@ file bar/foo `, string(b.Bytes())) b.Reset() - err = Walk(context.Background(), d, &WalkOpt{ + err = Walk(context.Background(), d, &FilterOpt{ IncludePatterns: []string{"b*/f*"}, }, bufWalk(b)) assert.NoError(t, err) @@ -110,7 +124,7 @@ file bar/foo `, string(b.Bytes())) b.Reset() - err = Walk(context.Background(), d, &WalkOpt{ + err = Walk(context.Background(), d, &FilterOpt{ IncludePatterns: []string{"b*/foo"}, }, bufWalk(b)) assert.NoError(t, err) @@ -120,7 +134,7 @@ file bar/foo `, string(b.Bytes())) b.Reset() - err = Walk(context.Background(), d, &WalkOpt{ + err = Walk(context.Background(), d, &FilterOpt{ IncludePatterns: []string{"b*/"}, }, bufWalk(b)) assert.NoError(t, err) @@ -140,7 +154,7 @@ func TestWalkerExclude(t *testing.T) { assert.NoError(t, err) defer os.RemoveAll(d) b := &bytes.Buffer{} - err = Walk(context.Background(), d, &WalkOpt{ + err = Walk(context.Background(), d, &FilterOpt{ ExcludePatterns: []string{"foo*", "!foo/bar2"}, }, bufWalk(b)) assert.NoError(t, err) @@ -166,7 +180,7 @@ func TestWalkerFollowLinks(t *testing.T) { assert.NoError(t, err) defer os.RemoveAll(d) b := &bytes.Buffer{} - err = Walk(context.Background(), d, &WalkOpt{ + err = Walk(context.Background(), d, &FilterOpt{ FollowPaths: []string{"foo/l*", "bar"}, }, bufWalk(b)) assert.NoError(t, err) @@ -193,7 +207,7 @@ func TestWalkerFollowLinksToRoot(t *testing.T) { assert.NoError(t, err) defer os.RemoveAll(d) b := &bytes.Buffer{} - err = Walk(context.Background(), d, &WalkOpt{ + err = Walk(context.Background(), d, &FilterOpt{ FollowPaths: []string{"foo"}, }, bufWalk(b)) assert.NoError(t, err) @@ -216,7 +230,7 @@ func TestWalkerMap(t *testing.T) { assert.NoError(t, err) defer os.RemoveAll(d) b := &bytes.Buffer{} - err = Walk(context.Background(), d, &WalkOpt{ + err = Walk(context.Background(), d, &FilterOpt{ Map: func(_ string, s *types.Stat) MapResult { if strings.HasPrefix(s.Path, "foo") { s.Path = "_" + s.Path @@ -247,7 +261,7 @@ func TestWalkerMapSkipDir(t *testing.T) { // bother walking directories we don't care about. walked := []string{} b := &bytes.Buffer{} - err = Walk(context.Background(), d, &WalkOpt{ + err = Walk(context.Background(), d, &FilterOpt{ Map: func(_ string, s *types.Stat) MapResult { walked = append(walked, s.Path) if strings.HasPrefix(s.Path, "excludeDir") { @@ -288,13 +302,13 @@ func TestWalkerPermissionDenied(t *testing.T) { }() b := &bytes.Buffer{} - err = Walk(context.Background(), d, &WalkOpt{}, bufWalk(b)) + err = Walk(context.Background(), d, &FilterOpt{}, bufWalk(b)) if assert.Error(t, err) { assert.Contains(t, err.Error(), "permission denied") } b.Reset() - err = Walk(context.Background(), d, &WalkOpt{ + err = Walk(context.Background(), d, &FilterOpt{ ExcludePatterns: []string{"**/bar"}, }, bufWalk(b)) assert.NoError(t, err) @@ -302,7 +316,7 @@ func TestWalkerPermissionDenied(t *testing.T) { `, string(b.Bytes())) b.Reset() - err = Walk(context.Background(), d, &WalkOpt{ + err = Walk(context.Background(), d, &FilterOpt{ ExcludePatterns: []string{"**/bar", "!foo/bar/baz"}, }, bufWalk(b)) assert.NoError(t, err) @@ -310,7 +324,7 @@ func TestWalkerPermissionDenied(t *testing.T) { `, string(b.Bytes())) b.Reset() - err = Walk(context.Background(), d, &WalkOpt{ + err = Walk(context.Background(), d, &FilterOpt{ ExcludePatterns: []string{"**/bar", "!foo/bar"}, }, bufWalk(b)) if assert.Error(t, err) { @@ -318,7 +332,7 @@ func TestWalkerPermissionDenied(t *testing.T) { } b.Reset() - err = Walk(context.Background(), d, &WalkOpt{ + err = Walk(context.Background(), d, &FilterOpt{ IncludePatterns: []string{"foo", "!**/bar"}, }, bufWalk(b)) assert.NoError(t, err) @@ -327,7 +341,39 @@ func TestWalkerPermissionDenied(t *testing.T) { } func bufWalk(buf *bytes.Buffer) filepath.WalkFunc { - return func(path string, fi os.FileInfo, err error) error { + return func(path string, fi gofs.FileInfo, err error) error { + if err != nil { + return err + } + stat, ok := fi.Sys().(*types.Stat) + if !ok { + return errors.Errorf("invalid symlink %s", path) + } + t := "file" + if fi.IsDir() { + t = "dir" + } + if fi.Mode()&os.ModeSymlink != 0 { + t = "symlink:" + stat.Linkname + } + fmt.Fprintf(buf, "%s %s", t, path) + if fi.Mode()&os.ModeSymlink == 0 && stat.Linkname != "" { + fmt.Fprintf(buf, " >%s", stat.Linkname) + } + fmt.Fprintln(buf) + return nil + } +} + +func bufWalkDir(buf *bytes.Buffer) gofs.WalkDirFunc { + return func(path string, entry gofs.DirEntry, err error) error { + if err != nil { + return err + } + fi, err := entry.Info() + if err != nil { + return err + } stat, ok := fi.Sys().(*types.Stat) if !ok { return errors.Errorf("invalid symlink %s", path) @@ -488,14 +534,14 @@ func BenchmarkWalker(b *testing.B) { for i := 0; i < b.N; i++ { count := 0 - walkOpt := &WalkOpt{ + walkOpt := &FilterOpt{ IncludePatterns: []string{scenario.pattern}, } if scenario.exclude != "" { walkOpt.ExcludePatterns = []string{scenario.exclude} } err = Walk(context.Background(), tmpdir, walkOpt, - func(path string, fi os.FileInfo, err error) error { + func(path string, fi gofs.FileInfo, err error) error { count++ return nil }) @@ -531,7 +577,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { assert.NoError(t, err) defer os.RemoveAll(d) b := &bytes.Buffer{} - err = Walk(context.Background(), d, &WalkOpt{ + err = Walk(context.Background(), d, &FilterOpt{ IncludePatterns: []string{"**"}, }, bufWalk(b)) assert.NoError(t, err) @@ -553,7 +599,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { `, string(b.Bytes())) b.Reset() - err = Walk(context.Background(), d, &WalkOpt{ + err = Walk(context.Background(), d, &FilterOpt{ IncludePatterns: []string{"**/bar"}, }, bufWalk(b)) assert.NoError(t, err) @@ -572,7 +618,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { `, string(b.Bytes())) b.Reset() - err = Walk(context.Background(), d, &WalkOpt{ + err = Walk(context.Background(), d, &FilterOpt{ IncludePatterns: []string{"**/bar/foo"}, }, bufWalk(b)) assert.NoError(t, err) @@ -587,7 +633,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { `, string(b.Bytes())) b.Reset() - err = Walk(context.Background(), d, &WalkOpt{ + err = Walk(context.Background(), d, &FilterOpt{ IncludePatterns: []string{"**/b*"}, }, bufWalk(b)) assert.NoError(t, err) @@ -608,7 +654,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { `, string(b.Bytes())) b.Reset() - err = Walk(context.Background(), d, &WalkOpt{ + err = Walk(context.Background(), d, &FilterOpt{ IncludePatterns: []string{"**/bar/f*"}, }, bufWalk(b)) assert.NoError(t, err) @@ -624,7 +670,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { `, string(b.Bytes())) b.Reset() - err = Walk(context.Background(), d, &WalkOpt{ + err = Walk(context.Background(), d, &FilterOpt{ IncludePatterns: []string{"**/bar/g*"}, }, bufWalk(b)) assert.NoError(t, err) @@ -632,7 +678,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { trimEqual(t, ``, string(b.Bytes())) b.Reset() - err = Walk(context.Background(), d, &WalkOpt{ + err = Walk(context.Background(), d, &FilterOpt{ IncludePatterns: []string{"**/f*"}, }, bufWalk(b)) assert.NoError(t, err) @@ -652,7 +698,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { `, string(b.Bytes())) b.Reset() - err = Walk(context.Background(), d, &WalkOpt{ + err = Walk(context.Background(), d, &FilterOpt{ IncludePatterns: []string{"**/b*/f*"}, }, bufWalk(b)) assert.NoError(t, err) @@ -668,7 +714,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { `, string(b.Bytes())) b.Reset() - err = Walk(context.Background(), d, &WalkOpt{ + err = Walk(context.Background(), d, &FilterOpt{ IncludePatterns: []string{"**/b*/foo"}, }, bufWalk(b)) assert.NoError(t, err) @@ -683,7 +729,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { `, string(b.Bytes())) b.Reset() - err = Walk(context.Background(), d, &WalkOpt{ + err = Walk(context.Background(), d, &FilterOpt{ IncludePatterns: []string{"**/foo/**"}, }, bufWalk(b)) assert.NoError(t, err) @@ -695,7 +741,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { `, string(b.Bytes())) b.Reset() - err = Walk(context.Background(), d, &WalkOpt{ + err = Walk(context.Background(), d, &FilterOpt{ IncludePatterns: []string{"**/baz"}, }, bufWalk(b)) assert.NoError(t, err) @@ -708,6 +754,172 @@ func TestWalkerDoublestarInclude(t *testing.T) { `, string(b.Bytes())) } +func TestFSWalk(t *testing.T) { + d, err := tmpDir(changeStream([]string{ + "ADD foo file", + "ADD bar dir", + "ADD bar/foo2 file", + })) + assert.NoError(t, err) + defer os.RemoveAll(d) + f, err := NewFS(d) + assert.NoError(t, err) + + b := &bytes.Buffer{} + err = f.Walk(context.Background(), "", bufWalkDir(b)) + assert.NoError(t, err) + assert.Equal(t, string(b.Bytes()), `dir bar +file bar/foo2 +file foo +`) + + b = &bytes.Buffer{} + err = f.Walk(context.Background(), "foo", bufWalkDir(b)) + assert.NoError(t, err) + assert.Equal(t, string(b.Bytes()), `file foo +`) + + b = &bytes.Buffer{} + err = f.Walk(context.Background(), "bar", bufWalkDir(b)) + assert.NoError(t, err) + assert.Equal(t, string(b.Bytes()), `dir bar +file bar/foo2 +`) +} + +func TestFSWalkNested(t *testing.T) { + d, err := tmpDir(changeStream([]string{ + "ADD foo dir", + "ADD foo/bar file", + })) + assert.NoError(t, err) + defer os.RemoveAll(d) + f, err := NewFS(d) + assert.NoError(t, err) + + f2, err := NewFilterFS(f, &FilterOpt{ + ExcludePatterns: []string{"foo", "!foo/bar"}, + }) + assert.NoError(t, err) + b := &bytes.Buffer{} + err = f2.Walk(context.Background(), "", bufWalkDir(b)) + assert.NoError(t, err) + assert.Equal(t, string(b.Bytes()), `dir foo +file foo/bar +`) + + f2, err = NewFilterFS(f, &FilterOpt{ + ExcludePatterns: []string{"!foo/bar"}, + }) + assert.NoError(t, err) + f2, err = NewFilterFS(f2, &FilterOpt{ + ExcludePatterns: []string{"foo"}, + }) + assert.NoError(t, err) + b = &bytes.Buffer{} + err = f2.Walk(context.Background(), "", bufWalkDir(b)) + assert.NoError(t, err) + assert.Equal(t, string(b.Bytes()), ``) + + f2, err = NewFilterFS(f, &FilterOpt{ + ExcludePatterns: []string{"foo"}, + }) + assert.NoError(t, err) + f2, err = NewFilterFS(f2, &FilterOpt{ + ExcludePatterns: []string{"!foo/bar"}, + }) + assert.NoError(t, err) + b = &bytes.Buffer{} + err = f2.Walk(context.Background(), "", bufWalkDir(b)) + assert.NoError(t, err) + assert.Equal(t, string(b.Bytes()), ``) +} + +func TestFilteredOpen(t *testing.T) { + d, err := tmpDir(changeStream([]string{ + "ADD foo file", + "ADD bar file", + })) + assert.NoError(t, err) + defer os.RemoveAll(d) + f, err := NewFS(d) + assert.NoError(t, err) + + f, err = NewFilterFS(f, &FilterOpt{ + ExcludePatterns: []string{"bar"}, + }) + assert.NoError(t, err) + + b := &bytes.Buffer{} + err = f.Walk(context.Background(), "", bufWalkDir(b)) + assert.NoError(t, err) + assert.Equal(t, string(b.Bytes()), `file foo +`) + + r, err := f.Open("foo") + assert.NoError(t, err) + defer r.Close() + dt, err := io.ReadAll(r) + assert.NoError(t, err) + assert.Equal(t, "", string(dt)) + + _, err = f.Open("bar") + assert.ErrorIs(t, err, os.ErrNotExist) +} + +func TestFilteredOpenWildcard(t *testing.T) { + d, err := tmpDir(changeStream([]string{ + "ADD baz file", + "ADD bar dir", + "ADD bar2 file", + "ADD bar/foo file", + })) + assert.NoError(t, err) + defer os.RemoveAll(d) + f, err := NewFS(d) + assert.NoError(t, err) + f, err = NewFilterFS(f, &FilterOpt{ + IncludePatterns: []string{"bar*"}, + }) + assert.NoError(t, err) + + _, err = f.Open("baz") + assert.ErrorIs(t, err, os.ErrNotExist) + + r, err := f.Open("bar2") + assert.NoError(t, err) + assert.NoError(t, r.Close()) + + r, err = f.Open("bar/foo") + assert.NoError(t, err) + assert.NoError(t, r.Close()) +} + +func TestFilteredOpenInvert(t *testing.T) { + d, err := tmpDir(changeStream([]string{ + "ADD foo dir", + "ADD foo/bar dir", + "ADD foo/bar/baz dir", + "ADD foo/bar/baz/x file", + "ADD foo/bar/baz/y file", + })) + assert.NoError(t, err) + defer os.RemoveAll(d) + f, err := NewFS(d) + assert.NoError(t, err) + f, err = NewFilterFS(f, &FilterOpt{ + ExcludePatterns: []string{"foo", "!foo/bar", "foo/bar/baz", "!foo/bar/baz/x"}, + }) + assert.NoError(t, err) + + r, err := f.Open("foo/bar/baz/x") + assert.NoError(t, err) + assert.NoError(t, r.Close()) + + _, err = f.Open("foo/bar/baz/y") + assert.ErrorIs(t, err, os.ErrNotExist) +} + func trimEqual(t assert.TestingT, expected, actual string, msgAndArgs ...interface{}) bool { lines := []string{} for _, line := range strings.Split(expected, "\n") { diff --git a/followlinks.go b/followlinks.go index f03a9cf3..3a0bd2bb 100644 --- a/followlinks.go +++ b/followlinks.go @@ -1,17 +1,21 @@ package fsutil import ( + "context" + gofs "io/fs" "os" "path/filepath" "runtime" "sort" strings "strings" + "syscall" "github.com/pkg/errors" + "github.com/tonistiigi/fsutil/types" ) -func FollowLinks(root string, paths []string) ([]string, error) { - r := &symlinkResolver{root: root, resolved: map[string]struct{}{}} +func FollowLinks(fs FS, paths []string) ([]string, error) { + r := &symlinkResolver{fs: fs, resolved: map[string]struct{}{}} for _, p := range paths { if err := r.append(p); err != nil { return nil, err @@ -26,7 +30,7 @@ func FollowLinks(root string, paths []string) ([]string, error) { } type symlinkResolver struct { - root string + fs FS resolved map[string]struct{} } @@ -76,10 +80,9 @@ func (r *symlinkResolver) append(p string) error { } func (r *symlinkResolver) readSymlink(p string, allowWildcard bool) ([]string, error) { - realPath := filepath.Join(r.root, p) base := filepath.Base(p) if allowWildcard && containsWildcards(base) { - fis, err := os.ReadDir(filepath.Dir(realPath)) + fis, err := readDir(r.fs, filepath.Dir(p)) if err != nil { if isNotFound(err) { return nil, nil @@ -99,21 +102,30 @@ func (r *symlinkResolver) readSymlink(p string, allowWildcard bool) ([]string, e return out, nil } - fi, err := os.Lstat(realPath) + entry, err := statFile(r.fs, p) if err != nil { if isNotFound(err) { return nil, nil } return nil, errors.WithStack(err) } - if fi.Mode()&os.ModeSymlink == 0 { + if entry == nil { return nil, nil } - link, err := os.Readlink(realPath) + if entry.Type()&os.ModeSymlink == 0 { + return nil, nil + } + + fi, err := entry.Info() if err != nil { return nil, errors.WithStack(err) } - link = filepath.Clean(link) + stat, ok := fi.Sys().(*types.Stat) + if !ok { + return nil, errors.WithStack(&os.PathError{Path: p, Err: syscall.EBADMSG, Op: "fileinfo without stat info"}) + } + + link := filepath.Clean(stat.Linkname) if filepath.IsAbs(link) { return []string{link}, nil } @@ -122,6 +134,76 @@ func (r *symlinkResolver) readSymlink(p string, allowWildcard bool) ([]string, e }, nil } +func statFile(fs FS, root string) (os.DirEntry, error) { + var out os.DirEntry + + root = filepath.Clean(root) + if root == "/" || root == "." { + return nil, nil + } + + err := fs.Walk(context.TODO(), root, func(p string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if p != root { + return errors.Errorf("expected single entry %q but got %q", root, p) + } + out = entry + if entry.IsDir() { + return filepath.SkipDir + } + return nil + }) + if err != nil { + return nil, err + } + + if out == nil { + return nil, errors.Wrapf(os.ErrNotExist, "readFile %s", root) + } + return out, nil +} + +func readDir(fs FS, root string) ([]os.DirEntry, error) { + var out []os.DirEntry + + root = filepath.Clean(root) + if root == "/" || root == "." { + root = "." + out = make([]gofs.DirEntry, 0) + } + + err := fs.Walk(context.TODO(), root, func(p string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if p == root { + if !entry.IsDir() { + return errors.WithStack(&os.PathError{Op: "walk", Path: root, Err: syscall.ENOTDIR}) + } + out = make([]gofs.DirEntry, 0) + return nil + } + if out == nil { + return errors.Errorf("expected to read parent entry %q before child %q", root, p) + } + out = append(out, entry) + if entry.IsDir() { + return filepath.SkipDir + } + return nil + }) + if err != nil { + return nil, err + } + + if out == nil && root != "." { + return nil, errors.Wrapf(os.ErrNotExist, "readDir %s", root) + } + return out, nil +} + func containsWildcards(name string) bool { isWindows := runtime.GOOS == "windows" for i := 0; i < len(name); i++ { diff --git a/followlinks_test.go b/followlinks_test.go index fd4e31a8..cc9e425f 100644 --- a/followlinks_test.go +++ b/followlinks_test.go @@ -1,6 +1,7 @@ package fsutil import ( + "os" "path/filepath" "runtime" "testing" @@ -11,7 +12,6 @@ import ( func TestFollowLinks(t *testing.T) { tmpDir := t.TempDir() - apply := fstest.Apply( fstest.CreateDir("dir", 0700), fstest.CreateFile("dir/foo", []byte("contents"), 0600), @@ -20,10 +20,11 @@ func TestFollowLinks(t *testing.T) { fstest.CreateFile("bar", nil, 0600), fstest.CreateFile("baz", nil, 0600), ) - require.NoError(t, apply.Apply(tmpDir)) + tmpfs, err := NewFS(tmpDir) + require.NoError(t, err) - out, err := FollowLinks(tmpDir, []string{"l2", "bar"}) + out, err := FollowLinks(tmpfs, []string{"l2", "bar"}) require.NoError(t, err) require.Equal(t, out, []string{"bar", "dir/foo", "dir/l1", "l2"}) @@ -31,28 +32,28 @@ func TestFollowLinks(t *testing.T) { func TestFollowLinksLoop(t *testing.T) { tmpDir := t.TempDir() - apply := fstest.Apply( fstest.Symlink("l1", "l1"), fstest.Symlink("l2", "l3"), fstest.Symlink("l3", "l2"), ) require.NoError(t, apply.Apply(tmpDir)) + tmpfs, err := NewFS(tmpDir) + require.NoError(t, err) - out, err := FollowLinks(tmpDir, []string{"l1", "l3"}) + out, err := FollowLinks(tmpfs, []string{"l1", "l3"}) require.NoError(t, err) require.Equal(t, out, []string{"l1", "l2", "l3"}) } func TestFollowLinksAbsolute(t *testing.T) { - tmpDir := t.TempDir() - abslutePathForBaz := "/foo/bar/baz" if runtime.GOOS == "windows" { abslutePathForBaz = "C:/foo/bar/baz" } + tmpDir := t.TempDir() apply := fstest.Apply( fstest.CreateDir("dir", 0700), fstest.Symlink(abslutePathForBaz, "dir/l1"), @@ -61,15 +62,16 @@ func TestFollowLinksAbsolute(t *testing.T) { fstest.CreateFile("baz", nil, 0600), ) require.NoError(t, apply.Apply(tmpDir)) + tmpfs, err := NewFS(tmpDir) + require.NoError(t, err) - out, err := FollowLinks(tmpDir, []string{"dir/l1"}) + out, err := FollowLinks(tmpfs, []string{"dir/l1"}) require.NoError(t, err) require.Equal(t, []string{"baz", "dir/l1", "foo/bar"}, out) // same but a link outside root tmpDir = t.TempDir() - apply = fstest.Apply( fstest.CreateDir("dir", 0700), fstest.Symlink(abslutePathForBaz, "dir/l1"), @@ -78,8 +80,10 @@ func TestFollowLinksAbsolute(t *testing.T) { fstest.CreateFile("baz", nil, 0600), ) require.NoError(t, apply.Apply(tmpDir)) + tmpfs, err = NewFS(tmpDir) + require.NoError(t, err) - out, err = FollowLinks(tmpDir, []string{"dir/l1"}) + out, err = FollowLinks(tmpfs, []string{"dir/l1"}) require.NoError(t, err) require.Equal(t, []string{"baz", "dir/l1", "foo/bar"}, out) @@ -87,19 +91,21 @@ func TestFollowLinksAbsolute(t *testing.T) { func TestFollowLinksNotExists(t *testing.T) { tmpDir := t.TempDir() + tmpfs, err := NewFS(tmpDir) + require.NoError(t, err) - out, err := FollowLinks(tmpDir, []string{"foo/bar/baz", "bar/baz"}) + out, err := FollowLinks(tmpfs, []string{"foo/bar/baz", "bar/baz"}) require.NoError(t, err) require.Equal(t, out, []string{"bar/baz", "foo/bar/baz"}) // root works fine with empty directory - out, err = FollowLinks(tmpDir, []string{"."}) + out, err = FollowLinks(tmpfs, []string{"."}) require.NoError(t, err) require.Equal(t, out, []string(nil)) - out, err = FollowLinks(tmpDir, []string{"f*/foo/t*"}) + out, err = FollowLinks(tmpfs, []string{"f*/foo/t*"}) require.NoError(t, err) require.Equal(t, []string{"f*/foo/t*"}, out) @@ -107,8 +113,10 @@ func TestFollowLinksNotExists(t *testing.T) { func TestFollowLinksNormalized(t *testing.T) { tmpDir := t.TempDir() + tmpfs, err := NewFS(tmpDir) + require.NoError(t, err) - out, err := FollowLinks(tmpDir, []string{"foo/bar/baz", "foo/bar"}) + out, err := FollowLinks(tmpfs, []string{"foo/bar/baz", "foo/bar"}) require.NoError(t, err) require.Equal(t, out, []string{"foo/bar"}) @@ -127,25 +135,24 @@ func TestFollowLinksNormalized(t *testing.T) { ) require.NoError(t, apply.Apply(tmpDir)) - out, err = FollowLinks(tmpDir, []string{"dir/l1", "foo/bar"}) + out, err = FollowLinks(tmpfs, []string{"dir/l1", "foo/bar"}) require.NoError(t, err) require.Equal(t, out, []string{"dir/l1", "foo"}) - out, err = FollowLinks(tmpDir, []string{"dir/l2", "foo", "foo/bar"}) + out, err = FollowLinks(tmpfs, []string{"dir/l2", "foo", "foo/bar"}) require.NoError(t, err) require.Equal(t, out, []string(nil)) } func TestFollowLinksWildcard(t *testing.T) { - tmpDir := t.TempDir() - absolutePathForFoo := "/foo" if runtime.GOOS == "windows" { absolutePathForFoo = "C:/foo" } + tmpDir := t.TempDir() apply := fstest.Apply( fstest.CreateDir("dir", 0700), fstest.CreateDir("foo", 0700), @@ -158,19 +165,89 @@ func TestFollowLinksWildcard(t *testing.T) { fstest.CreateFile("baz", nil, 0600), ) require.NoError(t, apply.Apply(tmpDir)) + tmpfs, err := NewFS(tmpDir) + require.NoError(t, err) - out, err := FollowLinks(tmpDir, []string{"dir/l*"}) + out, err := FollowLinks(tmpfs, []string{"dir/l*"}) require.NoError(t, err) require.Equal(t, []string{"baz", "dir/l*", "foo/bar1", "foo/bar2"}, out) - out, err = FollowLinks(tmpDir, []string{"dir"}) + out, err = FollowLinks(tmpfs, []string{"dir"}) require.NoError(t, err) require.Equal(t, out, []string{"dir"}) - out, err = FollowLinks(tmpDir, []string{"dir", "dir/*link"}) + out, err = FollowLinks(tmpfs, []string{"dir", "dir/*link"}) require.NoError(t, err) require.Equal(t, out, []string{"dir", "foo/bar3"}) } + +func TestInternalReadFile(t *testing.T) { + tmpDir := t.TempDir() + apply := fstest.Apply( + fstest.CreateDir("dir", 0700), + fstest.CreateFile("dir/foo1", nil, 0600), + ) + require.NoError(t, apply.Apply(tmpDir)) + tmpfs, err := NewFS(tmpDir) + require.NoError(t, err) + + entry, err := statFile(tmpfs, "/") + require.NoError(t, err) + require.Nil(t, entry) + entry, err = statFile(tmpfs, "") + require.NoError(t, err) + require.Nil(t, entry) + + entry, err = statFile(tmpfs, "dir") + require.NoError(t, err) + require.Equal(t, "dir", entry.Name()) + require.True(t, entry.Type().IsDir()) + entry, err = statFile(tmpfs, "dir/foo1") + require.NoError(t, err) + require.Equal(t, "foo1", entry.Name()) + require.False(t, entry.Type().IsDir()) + + entry, err = statFile(tmpfs, "dir/foo2") + require.Error(t, err) + require.ErrorIs(t, err, os.ErrNotExist) + require.Nil(t, entry) + entry, err = statFile(tmpfs, "dir/x/y/z") + require.Error(t, err) + require.ErrorIs(t, err, os.ErrNotExist) + require.Nil(t, entry) +} + +func TestInternalReadDir(t *testing.T) { + tmpDir := t.TempDir() + apply := fstest.Apply( + fstest.CreateDir("dir", 0700), + fstest.CreateFile("dir/foo1", nil, 0600), + ) + require.NoError(t, apply.Apply(tmpDir)) + tmpfs, err := NewFS(tmpDir) + require.NoError(t, err) + + entries, err := readDir(tmpfs, "/") + require.NoError(t, err) + require.Len(t, entries, 1) + require.Equal(t, "dir", entries[0].Name()) + require.True(t, entries[0].IsDir()) + + entries, err = readDir(tmpfs, "dir") + require.NoError(t, err) + require.Len(t, entries, 1) + require.Equal(t, "foo1", entries[0].Name()) + + entries, err = readDir(tmpfs, "dir2") + require.Error(t, err) + require.ErrorIs(t, err, os.ErrNotExist) + require.Empty(t, entries) + + entries, err = readDir(tmpfs, "dir/foo1") + require.Error(t, err) + require.ErrorIs(t, err, os.ErrNotExist) + require.Empty(t, entries) +} diff --git a/fs.go b/fs.go index db587b77..fe370194 100644 --- a/fs.go +++ b/fs.go @@ -3,36 +3,86 @@ package fsutil import ( "context" "io" + gofs "io/fs" "os" "path" "path/filepath" "sort" "strings" "syscall" + "time" "github.com/pkg/errors" "github.com/tonistiigi/fsutil/types" ) type FS interface { - Walk(context.Context, filepath.WalkFunc) error + Walk(context.Context, string, gofs.WalkDirFunc) error Open(string) (io.ReadCloser, error) } -func NewFS(root string, opt *WalkOpt) FS { +// NewFS creates a new FS from a root directory on the host filesystem. +func NewFS(root string) (FS, error) { + root, err := filepath.EvalSymlinks(root) + if err != nil { + return nil, errors.WithStack(&os.PathError{Op: "resolve", Path: root, Err: err}) + } + fi, err := os.Stat(root) + if err != nil { + return nil, err + } + if !fi.IsDir() { + return nil, errors.WithStack(&os.PathError{Op: "stat", Path: root, Err: syscall.ENOTDIR}) + } + return &fs{ root: root, - opt: opt, - } + }, nil } type fs struct { root string - opt *WalkOpt } -func (fs *fs) Walk(ctx context.Context, fn filepath.WalkFunc) error { - return Walk(ctx, fs.root, fs.opt, fn) +func (fs *fs) Walk(ctx context.Context, target string, fn gofs.WalkDirFunc) error { + seenFiles := make(map[uint64]string) + return filepath.WalkDir(filepath.Join(fs.root, target), func(path string, dirEntry gofs.DirEntry, walkErr error) (retErr error) { + defer func() { + if retErr != nil && isNotExist(retErr) { + retErr = filepath.SkipDir + } + }() + + origpath := path + path, err := filepath.Rel(fs.root, path) + if err != nil { + return err + } + // Skip root + if path == "." { + return nil + } + + var entry gofs.DirEntry + if dirEntry != nil { + entry = &DirEntryInfo{ + path: path, + origpath: origpath, + entry: dirEntry, + seenFiles: seenFiles, + } + } + + select { + case <-ctx.Done(): + return ctx.Err() + default: + if err := fn(path, entry, walkErr); err != nil { + return err + } + } + return nil + }) } func (fs *fs) Open(p string) (io.ReadCloser, error) { @@ -67,16 +117,31 @@ type subDirFS struct { dirs []Dir } -func (fs *subDirFS) Walk(ctx context.Context, fn filepath.WalkFunc) error { +func (fs *subDirFS) Walk(ctx context.Context, target string, fn gofs.WalkDirFunc) error { + first, rest, _ := strings.Cut(target, string(filepath.Separator)) + for _, d := range fs.dirs { - fi := &StatInfo{Stat: &d.Stat} + if first != "" && first != d.Stat.Path { + continue + } + + fi := &StatInfo{&d.Stat} if !fi.IsDir() { return errors.WithStack(&os.PathError{Path: d.Stat.Path, Err: syscall.ENOTDIR, Op: "walk subdir"}) } - if err := fn(d.Stat.Path, fi, nil); err != nil { + dStat := d.Stat + if err := fn(d.Stat.Path, &DirEntryInfo{Stat: &dStat}, nil); err != nil { return err } - if err := d.FS.Walk(ctx, func(p string, fi os.FileInfo, err error) error { + if err := d.FS.Walk(ctx, rest, func(p string, entry gofs.DirEntry, err error) error { + if err != nil { + return err + } + + fi, err := entry.Info() + if err != nil { + return err + } stat, ok := fi.Sys().(*types.Stat) if !ok { return errors.WithStack(&os.PathError{Path: d.Stat.Path, Err: syscall.EBADMSG, Op: "fileinfo without stat info"}) @@ -91,7 +156,7 @@ func (fs *subDirFS) Walk(ctx context.Context, fn filepath.WalkFunc) error { stat.Linkname = path.Join(d.Stat.Path, stat.Linkname) } } - return fn(filepath.Join(d.Stat.Path, p), &StatInfo{stat}, nil) + return fn(filepath.Join(d.Stat.Path, p), &DirEntryInfo{Stat: stat}, nil) }); err != nil { return err } @@ -117,3 +182,70 @@ type emptyReader struct { func (*emptyReader) Read([]byte) (int, error) { return 0, io.EOF } + +type StatInfo struct { + *types.Stat +} + +func (s *StatInfo) Name() string { + return filepath.Base(s.Stat.Path) +} +func (s *StatInfo) Size() int64 { + return s.Stat.Size_ +} +func (s *StatInfo) Mode() os.FileMode { + return os.FileMode(s.Stat.Mode) +} +func (s *StatInfo) ModTime() time.Time { + return time.Unix(s.Stat.ModTime/1e9, s.Stat.ModTime%1e9) +} +func (s *StatInfo) IsDir() bool { + return s.Mode().IsDir() +} +func (s *StatInfo) Sys() interface{} { + return s.Stat +} + +type DirEntryInfo struct { + *types.Stat + + entry gofs.DirEntry + path string + origpath string + seenFiles map[uint64]string +} + +func (s *DirEntryInfo) Name() string { + if s.Stat != nil { + return filepath.Base(s.Stat.Path) + } + return s.entry.Name() +} +func (s *DirEntryInfo) IsDir() bool { + if s.Stat != nil { + return s.Stat.IsDir() + } + return s.entry.IsDir() +} +func (s *DirEntryInfo) Type() gofs.FileMode { + if s.Stat != nil { + return gofs.FileMode(s.Stat.Mode) + } + return s.entry.Type() +} +func (s *DirEntryInfo) Info() (gofs.FileInfo, error) { + if s.Stat == nil { + fi, err := s.entry.Info() + if err != nil { + return nil, err + } + stat, err := mkstat(s.origpath, s.path, fi, s.seenFiles) + if err != nil { + return nil, err + } + s.Stat = stat + } + + st := *s.Stat + return &StatInfo{&st}, nil +} diff --git a/fs_test.go b/fs_test.go new file mode 100644 index 00000000..54e7653b --- /dev/null +++ b/fs_test.go @@ -0,0 +1,105 @@ +package fsutil + +import ( + "context" + gofs "io/fs" + "os" + "testing" + + "github.com/containerd/continuity/fs/fstest" + "github.com/stretchr/testify/require" + "github.com/tonistiigi/fsutil/types" +) + +func TestWalk(t *testing.T) { + tmpDir := t.TempDir() + + apply := fstest.Apply( + fstest.CreateDir("dir", 0700), + fstest.CreateFile("dir/foo", []byte("contents"), 0600), + ) + require.NoError(t, apply.Apply(tmpDir)) + + f, err := NewFS(tmpDir) + require.NoError(t, err) + paths := []string{} + files := []gofs.DirEntry{} + err = f.Walk(context.TODO(), "", func(path string, entry gofs.DirEntry, err error) error { + require.NoError(t, err) + paths = append(paths, path) + files = append(files, entry) + return nil + }) + require.NoError(t, err) + + require.Equal(t, paths, []string{"dir", "dir/foo"}) + require.Len(t, files, 2) + require.Equal(t, "dir", files[0].Name()) + require.Equal(t, "foo", files[1].Name()) + + fis := []gofs.FileInfo{} + for _, f := range files { + fi, err := f.Info() + require.NoError(t, err) + fis = append(fis, fi) + } + require.Equal(t, "dir", fis[0].Name()) + require.Equal(t, "foo", fis[1].Name()) + + require.Equal(t, len("contents"), int(fis[1].Size())) + + require.Equal(t, "dir", fis[0].(*StatInfo).Path) + require.Equal(t, "dir/foo", fis[1].(*StatInfo).Path) +} + +func TestWalkDir(t *testing.T) { + tmpDir := t.TempDir() + apply := fstest.Apply( + fstest.CreateDir("dir", 0700), + fstest.CreateFile("dir/foo", []byte("contents"), 0600), + ) + require.NoError(t, apply.Apply(tmpDir)) + tmpfs, err := NewFS(tmpDir) + require.NoError(t, err) + + tmpDir2 := t.TempDir() + apply = fstest.Apply( + fstest.CreateDir("dir", 0700), + fstest.CreateFile("dir/bar", []byte("contents2"), 0600), + ) + require.NoError(t, apply.Apply(tmpDir2)) + tmpfs2, err := NewFS(tmpDir2) + require.NoError(t, err) + + f, err := SubDirFS([]Dir{ + { + Stat: types.Stat{ + Mode: uint32(os.ModeDir | 0755), + Path: "1", + }, + FS: tmpfs, + }, + { + Stat: types.Stat{ + Mode: uint32(os.ModeDir | 0755), + Path: "2", + }, + FS: tmpfs2, + }, + }) + require.NoError(t, err) + paths := []string{} + files := []gofs.DirEntry{} + err = f.Walk(context.TODO(), "", func(path string, entry gofs.DirEntry, err error) error { + require.NoError(t, err) + paths = append(paths, path) + files = append(files, entry) + return nil + }) + require.NoError(t, err) + + require.Equal(t, paths, []string{"1", "1/dir", "1/dir/foo", "2", "2/dir", "2/dir/bar"}) + require.Equal(t, "1", files[0].Name()) + require.Equal(t, "dir", files[1].Name()) + require.Equal(t, "foo", files[2].Name()) +} diff --git a/receive_test.go b/receive_test.go index 0f9eb310..a36d5013 100644 --- a/receive_test.go +++ b/receive_test.go @@ -6,6 +6,7 @@ import ( "crypto/sha256" "hash" "io" + gofs "io/fs" "os" "path/filepath" "sync" @@ -20,12 +21,8 @@ import ( "golang.org/x/sync/errgroup" ) -func TestInvalidExcludePatterns(t *testing.T) { - d, err := tmpDir(changeStream([]string{ - "ADD foo file data1", - })) - assert.NoError(t, err) - defer os.RemoveAll(d) +func TestSendError(t *testing.T) { + fs := &testErrFS{err: errors.New("foo bar")} dest := t.TempDir() @@ -37,8 +34,8 @@ func TestInvalidExcludePatterns(t *testing.T) { eg.Go(func() error { defer s1.(*fakeConnProto).closeSend() - err := Send(ctx, s1, NewFS(d, &WalkOpt{ExcludePatterns: []string{"!"}}), nil) - assert.Contains(t, err.Error(), "invalid excludepatterns") + err := Send(ctx, s1, fs, nil) + assert.Contains(t, err.Error(), "foo bar") return err }) eg.Go(func() error { @@ -47,7 +44,7 @@ func TestInvalidExcludePatterns(t *testing.T) { ContentHasher: simpleSHA256Hasher, }) assert.Contains(t, err.Error(), "error from sender:") - assert.Contains(t, err.Error(), "invalid excludepatterns") + assert.Contains(t, err.Error(), "foo bar") return err }) @@ -58,8 +55,8 @@ func TestInvalidExcludePatterns(t *testing.T) { select { case <-time.After(15 * time.Second): t.Fatal("timeout") - case err = <-errCh: - assert.Contains(t, err.Error(), "invalid excludepatterns") + case err := <-errCh: + assert.Contains(t, err.Error(), "foo bar") } } @@ -72,6 +69,8 @@ func TestCopyWithSubDir(t *testing.T) { })) assert.NoError(t, err) defer os.RemoveAll(d) + fs, err := NewFS(d) + assert.NoError(t, err) dest := t.TempDir() @@ -80,7 +79,7 @@ func TestCopyWithSubDir(t *testing.T) { eg.Go(func() error { defer s1.(*fakeConnProto).closeSend() - subdir, err := SubDirFS([]Dir{{FS: NewFS(d, &WalkOpt{}), Stat: types.Stat{Path: "sub", Mode: uint32(os.ModeDir | 0755)}}}) + subdir, err := SubDirFS([]Dir{{FS: fs, Stat: types.Stat{Path: "sub", Mode: uint32(os.ModeDir | 0755)}}}) if err != nil { return err } @@ -105,6 +104,8 @@ func TestCopyDirectoryTimestamps(t *testing.T) { })) assert.NoError(t, err) defer os.RemoveAll(d) + fs, err := NewFS(d) + assert.NoError(t, err) timestamp := time.Unix(0, 0) require.NoError(t, os.Chtimes(filepath.Join(d, "foo"), timestamp, timestamp)) @@ -116,7 +117,7 @@ func TestCopyDirectoryTimestamps(t *testing.T) { eg.Go(func() error { defer s1.(*fakeConnProto).closeSend() - return Send(ctx, s1, NewFS(d, nil), nil) + return Send(ctx, s1, fs, nil) }) eg.Go(func() error { return Receive(ctx, s2, dest, ReceiveOpt{}) @@ -155,15 +156,24 @@ func TestCopySwitchDirToFile(t *testing.T) { eg, ctx := errgroup.WithContext(context.Background()) s1, s2 := sockPairProto(ctx) + fs, err := NewFS(src) + if err != nil { + return nil, err + } + fs, err = NewFilterFS(fs, &FilterOpt{ + Map: func(_ string, s *types.Stat) MapResult { + s.Uid = 0 + s.Gid = 0 + return MapResultKeep + }, + }) + if err != nil { + return nil, err + } + eg.Go(func() error { defer s1.(*fakeConnProto).closeSend() - return Send(ctx, s1, NewFS(src, &WalkOpt{ - Map: func(_ string, s *types.Stat) MapResult { - s.Uid = 0 - s.Gid = 0 - return MapResultKeep - }, - }), nil) + return Send(ctx, s1, fs, nil) }) eg.Go(func() error { return Receive(ctx, s2, dest, ReceiveOpt{ @@ -213,6 +223,16 @@ func TestCopySimple(t *testing.T) { })) assert.NoError(t, err) defer os.RemoveAll(d) + fs, err := NewFS(d) + assert.NoError(t, err) + fs, err = NewFilterFS(fs, &FilterOpt{ + Map: func(_ string, s *types.Stat) MapResult { + s.Uid = 0 + s.Gid = 0 + return MapResultKeep + }, + }) + assert.NoError(t, err) dest := t.TempDir() @@ -226,13 +246,7 @@ func TestCopySimple(t *testing.T) { eg.Go(func() error { defer s1.(*fakeConnProto).closeSend() - return Send(ctx, s1, NewFS(d, &WalkOpt{ - Map: func(_ string, s *types.Stat) MapResult { - s.Uid = 0 - s.Gid = 0 - return MapResultKeep - }, - }), nil) + return Send(ctx, s1, fs, nil) }) eg.Go(func() error { return Receive(ctx, s2, dest, ReceiveOpt{ @@ -304,13 +318,7 @@ file zzz.aa eg.Go(func() error { defer s1.(*fakeConnProto).closeSend() - return Send(ctx, s1, NewFS(d, &WalkOpt{ - Map: func(_ string, s *types.Stat) MapResult { - s.Uid = 0 - s.Gid = 0 - return MapResultKeep - }, - }), nil) + return Send(ctx, s1, fs, nil) }) eg.Go(func() error { return Receive(ctx, s2, dest, ReceiveOpt{ @@ -502,3 +510,15 @@ func simpleSHA256Hasher(s *types.Stat) (hash.Hash, error) { h.Write(dt) return h, nil } + +type testErrFS struct { + err error +} + +func (e *testErrFS) Walk(ctx context.Context, p string, fn gofs.WalkDirFunc) error { + return errors.Wrap(e.err, "invalid walk") +} + +func (e *testErrFS) Open(p string) (io.ReadCloser, error) { + return nil, errors.Wrap(e.err, "invalid open") +} diff --git a/send.go b/send.go index f1c51b83..a044d04d 100644 --- a/send.go +++ b/send.go @@ -144,7 +144,11 @@ func (s *sender) sendFile(h *sendHandle) error { func (s *sender) walk(ctx context.Context) error { var i uint32 = 0 - err := s.fs.Walk(ctx, func(path string, fi os.FileInfo, err error) error { + err := s.fs.Walk(ctx, "/", func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + fi, err := entry.Info() if err != nil { return err } diff --git a/tarwriter.go b/tarwriter.go index bd46a225..06b7bda9 100644 --- a/tarwriter.go +++ b/tarwriter.go @@ -15,10 +15,15 @@ import ( func WriteTar(ctx context.Context, fs FS, w io.Writer) error { tw := tar.NewWriter(w) - err := fs.Walk(ctx, func(path string, fi os.FileInfo, err error) error { + err := fs.Walk(ctx, "/", func(path string, entry os.DirEntry, err error) error { if err != nil && !errors.Is(err, os.ErrNotExist) { return err } + + fi, err := entry.Info() + if err != nil { + return err + } stat, ok := fi.Sys().(*types.Stat) if !ok { return errors.WithStack(&os.PathError{Path: path, Err: syscall.EBADMSG, Op: "fileinfo without stat info"})