From 2b0d425e991dcb9e546994f09e7604b85a452447 Mon Sep 17 00:00:00 2001 From: Ben Boyter Date: Mon, 30 Sep 2024 09:04:39 +1000 Subject: [PATCH] Add in gitmodule support and prepare 3.4.0 release --- README.md | 4 +- SCC-OUTPUT-REPORT.html | 66 ++++++++-------- go.mod | 2 +- go.sum | 2 + main.go | 6 ++ processor/processor.go | 6 +- .../github.com/boyter/gocodewalker/README.md | 2 +- .../github.com/boyter/gocodewalker/UNLICENSE | 24 ------ .../github.com/boyter/gocodewalker/check.sh | 2 +- vendor/github.com/boyter/gocodewalker/file.go | 79 +++++++++++++++---- .../boyter/gocodewalker/gitmodule.go | 23 ++++++ .../github.com/boyter/gocodewalker/hidden.go | 8 +- .../boyter/gocodewalker/hidden_windows.go | 8 +- vendor/modules.txt | 2 +- 14 files changed, 153 insertions(+), 81 deletions(-) delete mode 100644 vendor/github.com/boyter/gocodewalker/UNLICENSE create mode 100644 vendor/github.com/boyter/gocodewalker/gitmodule.go diff --git a/README.md b/README.md index 6db691985..83d50249a 100644 --- a/README.md +++ b/README.md @@ -219,7 +219,7 @@ features listed below may be missing from your installation. ``` Sloc, Cloc and Code. Count lines of code in a directory with complexity estimation. -Version 3.3.4 +Version 3.4.0 Ben Boyter + Contributors Usage: @@ -260,6 +260,8 @@ Flags: -d, --no-duplicates remove duplicate files from stats and output --no-gen ignore generated files in output (implies --gen) --no-gitignore disables .gitignore file logic + --no-gitmodule disables .gitmodules file logic + --no-hborder remove horizontal borders between sections --no-ignore disables .ignore file logic --no-large ignore files over certain byte and line size set by max-line-count and max-byte-count --no-min ignore minified files in output (implies --min) diff --git a/SCC-OUTPUT-REPORT.html b/SCC-OUTPUT-REPORT.html index 423e0bc3f..9567cb97d 100644 --- a/SCC-OUTPUT-REPORT.html +++ b/SCC-OUTPUT-REPORT.html @@ -13,13 +13,13 @@ Go 30 - 9627 - 1481 - 468 - 7678 + 9637 + 1482 + 469 + 7686 1536 - 256617 - 4092 + 256846 + 4098 processor/formatters.go @@ -63,23 +63,23 @@ processor/processor.go - 667 - 140 - 103 - 424 + 671 + 141 + 104 + 426 92 - 19295 - 435 + 19411 + 438 main.go - 398 + 404 10 6 - 382 + 388 10 - 8856 - 253 + 8969 + 256 processor/detector_test.go @@ -250,16 +250,6 @@ 0 2209 35 - - processor/cocomo_test.go - - 37 - 8 - 4 - 25 - 6 - 686 - 23 processor/bloom.go @@ -270,6 +260,16 @@ 2 1062 29 + + processor/cocomo_test.go + + 37 + 8 + 4 + 25 + 6 + 686 + 23 processor/helpers_test.go @@ -324,15 +324,15 @@ Total 30 - 9627 - 1481 - 468 - 7678 + 9637 + 1482 + 469 + 7686 1536 - 256617 - 4092 + 256846 + 4098 - Estimated Cost to Develop (organic) $229,670
Estimated Schedule Effort (organic) 7.86 months
Estimated People Required (organic) 2.59
+ Estimated Cost to Develop (organic) $229,922
Estimated Schedule Effort (organic) 7.87 months
Estimated People Required (organic) 2.60
\ No newline at end of file diff --git a/go.mod b/go.mod index 8a4b3f622..e0ee3eacd 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/boyter/scc/v3 go 1.22 require ( - github.com/boyter/gocodewalker v1.3.3 + github.com/boyter/gocodewalker v1.3.4 github.com/json-iterator/go v1.1.12 github.com/mattn/go-runewidth v0.0.15 github.com/rs/zerolog v1.30.0 diff --git a/go.sum b/go.sum index 236ce13e4..fb7dfaf1e 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/boyter/gocodewalker v1.3.3 h1:yPSbWT1wGmPSC73jASY0GaJu4EDN3FROfwYUDQjTmuE= github.com/boyter/gocodewalker v1.3.3/go.mod h1:hXG8xzR1uURS+99P5/3xh3uWHjaV2XfoMMmvPyhrCDg= +github.com/boyter/gocodewalker v1.3.4 h1:52rQJhVKwTLbbwJqAvDogbILLz8GIMO2b5oWR2ikhAM= +github.com/boyter/gocodewalker v1.3.4/go.mod h1:hXG8xzR1uURS+99P5/3xh3uWHjaV2XfoMMmvPyhrCDg= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ= diff --git a/main.go b/main.go index bf25e6d41..b2898a9c4 100644 --- a/main.go +++ b/main.go @@ -110,6 +110,12 @@ func main() { false, "disables .gitignore file logic", ) + flags.BoolVar( + &processor.GitModuleIgnore, + "no-gitmodule", + false, + "disables .gitmodules file logic", + ) flags.BoolVar( &processor.CountIgnore, "count-ignore", diff --git a/processor/processor.go b/processor/processor.go index 90d7404ef..ef07c663b 100644 --- a/processor/processor.go +++ b/processor/processor.go @@ -23,7 +23,7 @@ import ( ) // Version indicates the version of the application -var Version = "3.4.0 (beta)" +var Version = "3.4.0" // Flags set via the CLI which control how the output is displayed @@ -99,6 +99,9 @@ var Ci = false // GitIgnore disables .gitignore checks var GitIgnore = false +// GitModuleIgnore disables .gitmodules checks +var GitModuleIgnore = false + // Ignore disables ignore file checks var Ignore = false @@ -606,6 +609,7 @@ func Process() { }) fileWalker.IgnoreGitIgnore = GitIgnore fileWalker.IgnoreIgnoreFile = Ignore + fileWalker.IgnoreGitModules = GitModuleIgnore fileWalker.IncludeHidden = true fileWalker.ExcludeDirectory = PathDenyList diff --git a/vendor/github.com/boyter/gocodewalker/README.md b/vendor/github.com/boyter/gocodewalker/README.md index a52c1aa23..f70551c16 100644 --- a/vendor/github.com/boyter/gocodewalker/README.md +++ b/vendor/github.com/boyter/gocodewalker/README.md @@ -61,7 +61,7 @@ for f := range fileListQueue { } ``` -All code is dual-licenced as either MIT or Unlicence. +All code is licenced as MIT. ### Error Handler diff --git a/vendor/github.com/boyter/gocodewalker/UNLICENSE b/vendor/github.com/boyter/gocodewalker/UNLICENSE deleted file mode 100644 index 00d2e135a..000000000 --- a/vendor/github.com/boyter/gocodewalker/UNLICENSE +++ /dev/null @@ -1,24 +0,0 @@ -This is free and unencumbered software released into the public domain. - -Anyone is free to copy, modify, publish, use, compile, sell, or -distribute this software, either in source code form or as a compiled -binary, for any purpose, commercial or non-commercial, and by any -means. - -In jurisdictions that recognize copyright laws, the author or authors -of this software dedicate any and all copyright interest in the -software to the public domain. We make this dedication for the benefit -of the public at large and to the detriment of our heirs and -successors. We intend this dedication to be an overt act of -relinquishment in perpetuity of all present and future rights to this -software under copyright law. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -OTHER DEALINGS IN THE SOFTWARE. - -For more information, please refer to \ No newline at end of file diff --git a/vendor/github.com/boyter/gocodewalker/check.sh b/vendor/github.com/boyter/gocodewalker/check.sh index 223556984..7214c1a6f 100644 --- a/vendor/github.com/boyter/gocodewalker/check.sh +++ b/vendor/github.com/boyter/gocodewalker/check.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# SPDX-License-Identifier: MIT OR Unlicense +# SPDX-License-Identifier: MIT set -e diff --git a/vendor/github.com/boyter/gocodewalker/file.go b/vendor/github.com/boyter/gocodewalker/file.go index a33f7f522..707ff5d3e 100644 --- a/vendor/github.com/boyter/gocodewalker/file.go +++ b/vendor/github.com/boyter/gocodewalker/file.go @@ -2,26 +2,29 @@ // such as walking the file tree obeying .ignore and .gitignore files // or looking for the root directory assuming already in a git project -// SPDX-License-Identifier: MIT OR Unlicense +// SPDX-License-Identifier: MIT package gocodewalker import ( "bytes" "errors" - "github.com/boyter/gocodewalker/go-gitignore" - "golang.org/x/sync/errgroup" + "io/fs" "os" "path" "path/filepath" "regexp" "strings" "sync" + + "github.com/boyter/gocodewalker/go-gitignore" + "golang.org/x/sync/errgroup" ) const ( - GitIgnore = ".gitignore" - Ignore = ".ignore" + GitIgnore = ".gitignore" + Ignore = ".ignore" + GitModules = ".gitmodules" ) // ErrTerminateWalk error which indicates that the walker was terminated @@ -56,6 +59,7 @@ type FileWalker struct { isWalking bool IgnoreIgnoreFile bool // Should .ignore files be respected? IgnoreGitIgnore bool // Should .gitignore files be respected? + IgnoreGitModules bool // Should .gitmodules files be respected? IncludeHidden bool // Should hidden files and directories be included/walked osOpen func(name string) (*os.File, error) osReadFile func(name string) ([]byte, error) @@ -86,6 +90,7 @@ func NewFileWalker(directory string, fileListQueue chan *File) *FileWalker { isWalking: false, IgnoreIgnoreFile: false, IgnoreGitIgnore: false, + IgnoreGitModules: false, IncludeHidden: false, osOpen: os.Open, osReadFile: os.ReadFile, @@ -117,6 +122,7 @@ func NewParallelFileWalker(directories []string, fileListQueue chan *File) *File isWalking: false, IgnoreIgnoreFile: false, IgnoreGitIgnore: false, + IgnoreGitModules: false, IncludeHidden: false, osOpen: os.Open, osReadFile: os.ReadFile, @@ -184,14 +190,14 @@ func (f *FileWalker) Start() error { for _, directory := range f.directories { d := directory // capture var eg.Go(func() error { - return f.walkDirectoryRecursive(0, d, []gitignore.GitIgnore{}, []gitignore.GitIgnore{}) + return f.walkDirectoryRecursive(0, d, []gitignore.GitIgnore{}, []gitignore.GitIgnore{}, []gitignore.GitIgnore{}) }) } err = eg.Wait() } else { if f.directory != "" { - err = f.walkDirectoryRecursive(0, f.directory, []gitignore.GitIgnore{}, []gitignore.GitIgnore{}) + err = f.walkDirectoryRecursive(0, f.directory, []gitignore.GitIgnore{}, []gitignore.GitIgnore{}, []gitignore.GitIgnore{}) } } @@ -204,7 +210,11 @@ func (f *FileWalker) Start() error { return err } -func (f *FileWalker) walkDirectoryRecursive(iteration int, directory string, gitignores []gitignore.GitIgnore, ignores []gitignore.GitIgnore) error { +func (f *FileWalker) walkDirectoryRecursive(iteration int, + directory string, + gitignores []gitignore.GitIgnore, + ignores []gitignore.GitIgnore, + moduleIgnores []gitignore.GitIgnore) error { if iteration == 1 { f.countingSemaphore <- true defer func() { @@ -231,7 +241,7 @@ func (f *FileWalker) walkDirectoryRecursive(iteration int, directory string, git } defer d.Close() - foundFiles, err := d.Readdir(-1) + foundFiles, err := d.ReadDir(-1) if err != nil { // nothing we can do with this so return nil and process as best we can if f.errorsHandler(err) { @@ -240,8 +250,8 @@ func (f *FileWalker) walkDirectoryRecursive(iteration int, directory string, git return err } - files := []os.FileInfo{} - dirs := []os.FileInfo{} + files := []fs.DirEntry{} + dirs := []fs.DirEntry{} // We want to break apart the files and directories from the // return as we loop over them differently and this avoids some @@ -254,7 +264,7 @@ func (f *FileWalker) walkDirectoryRecursive(iteration int, directory string, git } } - // Pull out all ignore and gitignore files and add them + // Pull out all ignore, gitignore and gitmodule files and add them // to out collection of gitignores to be applied for this pass // and any subdirectories // Since they can apply to the current list of files we need to ensure @@ -305,6 +315,37 @@ func (f *FileWalker) walkDirectoryRecursive(iteration int, directory string, git ignores = append(ignores, gitIgnore) } } + + // this should only happen on the first iteration + // because there should be one .gitmodules file per repository + // however we also need to support someone running in a directory of + // projects that have multiple repositories or in a go vendor + // repository etc... hence check every time + if !f.IgnoreGitModules { + if file.Name() == GitModules { + // now we need to open and parse the file + c, err := f.osReadFile(filepath.Join(directory, file.Name())) + if err != nil { + if f.errorsHandler(err) { + continue // if asked to ignore it lets continue + } + return err + } + + abs, err := filepath.Abs(directory) + if err != nil { + if f.errorsHandler(err) { + continue // if asked to ignore it lets continue + } + return err + } + + for _, gm := range extractGitModuleFolders(string(c)) { + gitIgnore := gitignore.New(strings.NewReader(gm), abs, nil) + moduleIgnores = append(moduleIgnores, gitIgnore) + } + } + } } // Process files first to start feeding whatever process is consuming @@ -370,7 +411,7 @@ func (f *FileWalker) walkDirectoryRecursive(iteration int, directory string, git // Ignore hidden files if !f.IncludeHidden { - s, err := IsHidden(file, directory) + s, err := IsHiddenDirEntry(file, directory) if err != nil { if !f.errorsHandler(err) { return err @@ -466,6 +507,12 @@ func (f *FileWalker) walkDirectoryRecursive(iteration int, directory string, git shouldIgnore = ignore.Ignore(joined) } } + for _, ignore := range moduleIgnores { + // same rules as above + if ignore.MatchIsDir(joined, true) != nil { + shouldIgnore = ignore.Ignore(joined) + } + } // start by saying we didn't find it then check each possible // choice to see if we did find it @@ -510,7 +557,7 @@ func (f *FileWalker) walkDirectoryRecursive(iteration int, directory string, git // Ignore hidden directories if !f.IncludeHidden { - s, err := IsHidden(dir, directory) + s, err := IsHiddenDirEntry(dir, directory) if err != nil { if !f.errorsHandler(err) { return err @@ -532,11 +579,11 @@ func (f *FileWalker) walkDirectoryRecursive(iteration int, directory string, git if iteration == 0 { wg.Add(1) go func(iteration int, directory string, gitignores []gitignore.GitIgnore, ignores []gitignore.GitIgnore) { - _ = f.walkDirectoryRecursive(iteration+1, joined, gitignores, ignores) + _ = f.walkDirectoryRecursive(iteration+1, joined, gitignores, ignores, moduleIgnores) wg.Done() }(iteration, joined, gitignores, ignores) } else { - err = f.walkDirectoryRecursive(iteration+1, joined, gitignores, ignores) + err = f.walkDirectoryRecursive(iteration+1, joined, gitignores, ignores, moduleIgnores) if err != nil { return err } diff --git a/vendor/github.com/boyter/gocodewalker/gitmodule.go b/vendor/github.com/boyter/gocodewalker/gitmodule.go new file mode 100644 index 000000000..361a78066 --- /dev/null +++ b/vendor/github.com/boyter/gocodewalker/gitmodule.go @@ -0,0 +1,23 @@ +package gocodewalker + +import ( + "regexp" + "strings" +) + +func extractGitModuleFolders(input string) []string { + // Compile a regular expression to match lines starting with "path =" + re := regexp.MustCompile(`^\s*path\s*=\s*(.*)`) + output := []string{} + + for _, line := range strings.Split(input, "\n") { + // Check if the line matches the "path = " pattern + if matches := re.FindStringSubmatch(line); matches != nil { + // Extract the submodule path (which is captured in the regex group) + submodulePath := strings.TrimSpace(matches[1]) + output = append(output, submodulePath) + } + } + + return output +} diff --git a/vendor/github.com/boyter/gocodewalker/hidden.go b/vendor/github.com/boyter/gocodewalker/hidden.go index 2a8347ff7..c9ad146ce 100644 --- a/vendor/github.com/boyter/gocodewalker/hidden.go +++ b/vendor/github.com/boyter/gocodewalker/hidden.go @@ -1,14 +1,20 @@ -// SPDX-License-Identifier: MIT OR Unlicense +// SPDX-License-Identifier: MIT //go:build !windows // +build !windows package gocodewalker import ( + "io/fs" "os" ) // IsHidden Returns true if file is hidden func IsHidden(file os.FileInfo, directory string) (bool, error) { + return IsHiddenDirEntry(fs.FileInfoToDirEntry(file), directory) +} + +// IsHiddenDirEntry is similar to [IsHidden], excepts it accepts [fs.DirEntry] as its argument +func IsHiddenDirEntry(file fs.DirEntry, directory string) (bool, error) { return file.Name()[0:1] == ".", nil } diff --git a/vendor/github.com/boyter/gocodewalker/hidden_windows.go b/vendor/github.com/boyter/gocodewalker/hidden_windows.go index ea0ebf909..614d6e8dc 100644 --- a/vendor/github.com/boyter/gocodewalker/hidden_windows.go +++ b/vendor/github.com/boyter/gocodewalker/hidden_windows.go @@ -1,10 +1,11 @@ -// SPDX-License-Identifier: MIT OR Unlicense +// SPDX-License-Identifier: MIT //go:build windows // +build windows package gocodewalker import ( + "io/fs" "os" "path" "syscall" @@ -12,6 +13,11 @@ import ( // IsHidden Returns true if file is hidden func IsHidden(file os.FileInfo, directory string) (bool, error) { + return IsHiddenDirEntry(fs.FileInfoToDirEntry(file), directory) +} + +// IsHiddenDirEntry is similar to [IsHidden], excepts it accepts [fs.DirEntry] as its argument +func IsHiddenDirEntry(file fs.DirEntry, directory string) (bool, error) { fullpath := path.Join(directory, file.Name()) pointer, err := syscall.UTF16PtrFromString(fullpath) if err != nil { diff --git a/vendor/modules.txt b/vendor/modules.txt index 94c2824fa..32c553cc7 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,4 +1,4 @@ -# github.com/boyter/gocodewalker v1.3.3 +# github.com/boyter/gocodewalker v1.3.4 ## explicit; go 1.20 github.com/boyter/gocodewalker github.com/boyter/gocodewalker/go-gitignore