Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Change use of Walk to WalkDir to improve disk performance #22462

Merged
merged 7 commits into from
Jan 16, 2023
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions build/generate-bindata.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,15 @@ func needsUpdate(dir, filename string) (bool, []byte) {

hasher := sha1.New()

err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
err = filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
if err != nil {
return err
}
_, _ = hasher.Write([]byte(info.Name()))
info, err := d.Info()
if err != nil {
return err
}
_, _ = hasher.Write([]byte(d.Name()))
_, _ = hasher.Write([]byte(info.ModTime().String()))
_, _ = hasher.Write([]byte(strconv.FormatInt(info.Size(), 16)))
return nil
Expand Down
14 changes: 12 additions & 2 deletions modules/log/file.go
Original file line number Diff line number Diff line change
Expand Up @@ -225,14 +225,24 @@ func compressOldLogFile(fname string, compressionLevel int) error {

func (log *FileLogger) deleteOldLog() {
dir := filepath.Dir(log.Filename)
_ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) (returnErr error) {
_ = filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) (returnErr error) {
defer func() {
if r := recover(); r != nil {
returnErr = fmt.Errorf("Unable to delete old log '%s', error: %+v", path, r)
}
}()

if !info.IsDir() && info.ModTime().Unix() < (time.Now().Unix()-60*60*24*log.Maxdays) {
if returnErr != nil {
return returnErr
}
zeripath marked this conversation as resolved.
Show resolved Hide resolved
if d.IsDir() {
return nil
}
info, err := d.Info()
if err != nil {
return err
}
if info.ModTime().Unix() < (time.Now().Unix() - 60*60*24*log.Maxdays) {
if strings.HasPrefix(filepath.Base(path), filepath.Base(log.Filename)) {
if err := util.Remove(path); err != nil {
returnErr = fmt.Errorf("Failed to remove %s: %w", path, err)
Expand Down
4 changes: 2 additions & 2 deletions modules/repository/generate.go
Original file line number Diff line number Diff line change
Expand Up @@ -173,12 +173,12 @@ func generateRepoCommit(ctx context.Context, repo, templateRepo, generateRepo *r
// Avoid walking tree if there are no globs
if len(gt.Globs()) > 0 {
tmpDirSlash := strings.TrimSuffix(filepath.ToSlash(tmpDir), "/") + "/"
if err := filepath.Walk(tmpDirSlash, func(path string, info os.FileInfo, walkErr error) error {
if err := filepath.WalkDir(tmpDirSlash, func(path string, d os.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}

if info.IsDir() {
if d.IsDir() {
return nil
}

Expand Down
4 changes: 2 additions & 2 deletions modules/storage/local.go
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ func (l *LocalStorage) URL(path, name string) (*url.URL, error) {

// IterateObjects iterates across the objects in the local storage
func (l *LocalStorage) IterateObjects(fn func(path string, obj Object) error) error {
return filepath.Walk(l.dir, func(path string, info os.FileInfo, err error) error {
return filepath.WalkDir(l.dir, func(path string, d os.DirEntry, err error) error {
if err != nil {
return err
}
Expand All @@ -141,7 +141,7 @@ func (l *LocalStorage) IterateObjects(fn func(path string, obj Object) error) er
if path == l.dir {
return nil
}
if info.IsDir() {
if d.IsDir() {
return nil
}
relPath, err := filepath.Rel(l.dir, path)
Expand Down
8 changes: 4 additions & 4 deletions routers/web/user/setting/profile.go
Original file line number Diff line number Diff line change
Expand Up @@ -280,17 +280,17 @@ func Repos(ctx *context.Context) {
repos := map[string]*repo_model.Repository{}
// We're going to iterate by pagesize.
root := user_model.UserPath(ctxUser.Name)
if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
if !info.IsDir() || path == root {
if !d.IsDir() || path == root {
return nil
}
name := info.Name()
name := d.Name()
if !strings.HasSuffix(name, ".git") {
return filepath.SkipDir
}
Expand All @@ -304,7 +304,7 @@ func Repos(ctx *context.Context) {
count++
return filepath.SkipDir
}); err != nil {
ctx.ServerError("filepath.Walk", err)
ctx.ServerError("filepath.WalkDir", err)
return
}

Expand Down
12 changes: 6 additions & 6 deletions services/repository/adopt.go
Original file line number Diff line number Diff line change
Expand Up @@ -303,31 +303,31 @@ func ListUnadoptedRepositories(query string, opts *db.ListOptions) ([]string, in

// We're going to iterate by pagesize.
root := filepath.Clean(setting.RepoRootPath)
if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
if err != nil {
return err
}
if !info.IsDir() || path == root {
if !d.IsDir() || path == root {
return nil
}

name := d.Name()

if !strings.ContainsRune(path[len(root)+1:], filepath.Separator) {
// Got a new user
if err = checkUnadoptedRepositories(userName, repoNamesToCheck, unadopted); err != nil {
return err
}
repoNamesToCheck = repoNamesToCheck[:0]

if !globUser.Match(info.Name()) {
if !globUser.Match(name) {
return filepath.SkipDir
}

userName = info.Name()
userName = name
return nil
}

name := info.Name()

if !strings.HasSuffix(name, ".git") {
return filepath.SkipDir
}
Expand Down