From 94e3d7b97d1463514babcafa6c6c9773fb481938 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 22 Sep 2019 22:54:39 +0200 Subject: [PATCH 01/15] Use a local Walk function Because arduino-cli segfaults when sketchDir is a symbolic link, a simple walk function is introduced with a twofold effect: - fix the segfault - allow to use symlinks ref: https://www.google.com/search?q=golang+Walk+does+not+follow+symbolic+links --- arduino/builder/sketch.go | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/arduino/builder/sketch.go b/arduino/builder/sketch.go index 99cd3b368a4..fd61dac0921 100644 --- a/arduino/builder/sketch.go +++ b/arduino/builder/sketch.go @@ -58,6 +58,33 @@ func SketchSaveItemCpp(item *sketch.Item, destPath string) error { return nil } +func SimpleLocalWalk(root string, walkFn func(path string, info os.FileInfo, err error) error) error { + info, err := os.Stat(root) + + if err != nil { + return walkFn(root, nil, err) + } + + err = walkFn(root, info, err) + if err == filepath.SkipDir { + return nil + } + + if info.IsDir() { + files, err := ioutil.ReadDir(root) + if err == nil { + for _, file := range files { + err = SimpleLocalWalk(root + string(os.PathSeparator) + file.Name(), walkFn) + if err == filepath.SkipDir { + return nil + } + } + } + } + + return nil +} + // SketchLoad collects all the files composing a sketch. // The parameter `sketchPath` holds a path pointing to a single sketch file or a sketch folder, // the path must be absolute. @@ -86,7 +113,8 @@ func SketchLoad(sketchPath, buildPath string) (*sketch.Sketch, error) { // collect all the sketch files var files []string - err = filepath.Walk(sketchFolder, func(path string, info os.FileInfo, err error) error { + err = SimpleLocalWalk(sketchFolder, func(path string, info os.FileInfo, err error) error { + // ignore hidden files and skip hidden directories if strings.HasPrefix(info.Name(), ".") { if info.IsDir() { From 7186d73ebbe4eb7137d21f77ec7367edfe2eea8b Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 22 Sep 2019 23:08:43 +0200 Subject: [PATCH 02/15] tabs --- arduino/builder/sketch.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/arduino/builder/sketch.go b/arduino/builder/sketch.go index fd61dac0921..e787c045576 100644 --- a/arduino/builder/sketch.go +++ b/arduino/builder/sketch.go @@ -62,25 +62,25 @@ func SimpleLocalWalk(root string, walkFn func(path string, info os.FileInfo, err info, err := os.Stat(root) if err != nil { - return walkFn(root, nil, err) + return walkFn(root, nil, err) } err = walkFn(root, info, err) if err == filepath.SkipDir { - return nil + return nil } if info.IsDir() { - files, err := ioutil.ReadDir(root) - if err == nil { - for _, file := range files { - err = SimpleLocalWalk(root + string(os.PathSeparator) + file.Name(), walkFn) - if err == filepath.SkipDir { - return nil - } - } - } - } + files, err := ioutil.ReadDir(root) + if err == nil { + for _, file := range files { + err = SimpleLocalWalk(root + string(os.PathSeparator) + file.Name(), walkFn) + if err == filepath.SkipDir { + return nil + } + } + } + } return nil } From 23f02041e341adba9b8ea009b789465fa9a6522c Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 22 Sep 2019 23:14:37 +0200 Subject: [PATCH 03/15] style --- arduino/builder/sketch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arduino/builder/sketch.go b/arduino/builder/sketch.go index e787c045576..1426a9b2650 100644 --- a/arduino/builder/sketch.go +++ b/arduino/builder/sketch.go @@ -74,7 +74,7 @@ func SimpleLocalWalk(root string, walkFn func(path string, info os.FileInfo, err files, err := ioutil.ReadDir(root) if err == nil { for _, file := range files { - err = SimpleLocalWalk(root + string(os.PathSeparator) + file.Name(), walkFn) + err = SimpleLocalWalk(root+string(os.PathSeparator)+file.Name(), walkFn) if err == filepath.SkipDir { return nil } From fad3aa5b25cf22ff1f70788f677722bc5587470a Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 22 Sep 2019 23:28:05 +0200 Subject: [PATCH 04/15] comment --- arduino/builder/sketch.go | 1 + 1 file changed, 1 insertion(+) diff --git a/arduino/builder/sketch.go b/arduino/builder/sketch.go index 1426a9b2650..87a52396ef6 100644 --- a/arduino/builder/sketch.go +++ b/arduino/builder/sketch.go @@ -58,6 +58,7 @@ func SketchSaveItemCpp(item *sketch.Item, destPath string) error { return nil } +// SimpleLocalWalk locally replaces filepath.Walk and/but goes through symlinks func SimpleLocalWalk(root string, walkFn func(path string, info os.FileInfo, err error) error) error { info, err := os.Stat(root) From 7403a3930d59401cc332cbebcef4c92f58576ce7 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 22 Sep 2019 23:32:20 +0200 Subject: [PATCH 05/15] retrigger CI --- arduino/builder/sketch.go | 1 + 1 file changed, 1 insertion(+) diff --git a/arduino/builder/sketch.go b/arduino/builder/sketch.go index 87a52396ef6..1f9735739ed 100644 --- a/arduino/builder/sketch.go +++ b/arduino/builder/sketch.go @@ -60,6 +60,7 @@ func SketchSaveItemCpp(item *sketch.Item, destPath string) error { // SimpleLocalWalk locally replaces filepath.Walk and/but goes through symlinks func SimpleLocalWalk(root string, walkFn func(path string, info os.FileInfo, err error) error) error { + info, err := os.Stat(root) if err != nil { From 9c82aade5d36e24a99f0f0eb7a2b62a53da68587 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 23 Sep 2019 22:30:41 +0200 Subject: [PATCH 06/15] retrigger CI --- arduino/builder/sketch.go | 1 - 1 file changed, 1 deletion(-) diff --git a/arduino/builder/sketch.go b/arduino/builder/sketch.go index 1f9735739ed..87a52396ef6 100644 --- a/arduino/builder/sketch.go +++ b/arduino/builder/sketch.go @@ -60,7 +60,6 @@ func SketchSaveItemCpp(item *sketch.Item, destPath string) error { // SimpleLocalWalk locally replaces filepath.Walk and/but goes through symlinks func SimpleLocalWalk(root string, walkFn func(path string, info os.FileInfo, err error) error) error { - info, err := os.Stat(root) if err != nil { From 38c71e9a670c6ab327a239f0cfa2117496e87c32 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 23 Sep 2019 22:44:03 +0200 Subject: [PATCH 07/15] style --- arduino/builder/sketch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arduino/builder/sketch.go b/arduino/builder/sketch.go index 87a52396ef6..a4518621698 100644 --- a/arduino/builder/sketch.go +++ b/arduino/builder/sketch.go @@ -60,6 +60,7 @@ func SketchSaveItemCpp(item *sketch.Item, destPath string) error { // SimpleLocalWalk locally replaces filepath.Walk and/but goes through symlinks func SimpleLocalWalk(root string, walkFn func(path string, info os.FileInfo, err error) error) error { + info, err := os.Stat(root) if err != nil { @@ -115,7 +116,6 @@ func SketchLoad(sketchPath, buildPath string) (*sketch.Sketch, error) { // collect all the sketch files var files []string err = SimpleLocalWalk(sketchFolder, func(path string, info os.FileInfo, err error) error { - // ignore hidden files and skip hidden directories if strings.HasPrefix(info.Name(), ".") { if info.IsDir() { From 5503c569db9f950990e9a885e029a025ff52fa29 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Tue, 24 Sep 2019 01:37:59 +0200 Subject: [PATCH 08/15] check sketch for being a file --- arduino/builder/sketch.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arduino/builder/sketch.go b/arduino/builder/sketch.go index a4518621698..e8575a0a579 100644 --- a/arduino/builder/sketch.go +++ b/arduino/builder/sketch.go @@ -103,11 +103,13 @@ func SketchLoad(sketchPath, buildPath string) (*sketch.Sketch, error) { sketchFolder = sketchPath mainSketchFile = filepath.Join(sketchPath, stat.Name()+".ino") // in the case a dir was passed, ensure the main file exists and is readable - f, err := os.Open(mainSketchFile) + info, err := os.Stat(mainSketchFile) if err != nil { return nil, errors.Wrap(err, "unable to find the main sketch file") } - f.Close() + if info.IsDir() { + return nil, errors.Wrap(errors.New(mainSketchFile), "sketch must be a file") + } } else { sketchFolder = filepath.Dir(sketchPath) mainSketchFile = sketchPath From 75ffd4f94488d9672a7ec7b64f31a0a14210e37d Mon Sep 17 00:00:00 2001 From: david gauchard Date: Tue, 24 Sep 2019 01:51:01 +0200 Subject: [PATCH 09/15] check sketch is not a directory --- arduino/builder/sketch.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/arduino/builder/sketch.go b/arduino/builder/sketch.go index e8575a0a579..4e5875e7aa4 100644 --- a/arduino/builder/sketch.go +++ b/arduino/builder/sketch.go @@ -103,12 +103,18 @@ func SketchLoad(sketchPath, buildPath string) (*sketch.Sketch, error) { sketchFolder = sketchPath mainSketchFile = filepath.Join(sketchPath, stat.Name()+".ino") // in the case a dir was passed, ensure the main file exists and is readable - info, err := os.Stat(mainSketchFile) + f, err := os.Open(mainSketchFile) if err != nil { return nil, errors.Wrap(err, "unable to find the main sketch file") } + f.Close() + // ensure it is not a directory + info, err := os.Stat(mainSketchFile) + if err != nil { + return nil, errors.Wrap(err, "unable to check the main sketch file") + } if info.IsDir() { - return nil, errors.Wrap(errors.New(mainSketchFile), "sketch must be a file") + return nil, errors.Wrap(errors.New(mainSketchFile), "sketch must not be a directory") } } else { sketchFolder = filepath.Dir(sketchPath) From 05576c9212c0ea54d77dbf96302814448e58b22c Mon Sep 17 00:00:00 2001 From: david gauchard Date: Wed, 23 Oct 2019 23:10:01 +0200 Subject: [PATCH 10/15] additions: - act on error in walk function (OS can detect symbolic links loops (lnk -> lnk)) - add a deepness detector for non trivial loops (dir/lnk -> ../..) --- arduino/builder/sketch.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/arduino/builder/sketch.go b/arduino/builder/sketch.go index 4e5875e7aa4..f8358668332 100644 --- a/arduino/builder/sketch.go +++ b/arduino/builder/sketch.go @@ -22,6 +22,7 @@ import ( "path/filepath" "regexp" "strings" + "fmt" "github.com/arduino/arduino-cli/arduino/globals" "github.com/arduino/arduino-cli/arduino/sketch" @@ -59,7 +60,7 @@ func SketchSaveItemCpp(item *sketch.Item, destPath string) error { } // SimpleLocalWalk locally replaces filepath.Walk and/but goes through symlinks -func SimpleLocalWalk(root string, walkFn func(path string, info os.FileInfo, err error) error) error { +func SimpleLocalWalkRecursive(root string, maxDepth int, walkFn func(path string, info os.FileInfo, err error) error) error { info, err := os.Stat(root) @@ -73,10 +74,14 @@ func SimpleLocalWalk(root string, walkFn func(path string, info os.FileInfo, err } if info.IsDir() { + if maxDepth <= 0 { + return walkFn(root, info, errors.New("Filesystem bottom is too deep (directory recursion or filesystem really deep): " + root)) + } + maxDepth-- files, err := ioutil.ReadDir(root) if err == nil { for _, file := range files { - err = SimpleLocalWalk(root+string(os.PathSeparator)+file.Name(), walkFn) + err = SimpleLocalWalkRecursive(root+string(os.PathSeparator)+file.Name(), maxDepth, walkFn) if err == filepath.SkipDir { return nil } @@ -87,6 +92,11 @@ func SimpleLocalWalk(root string, walkFn func(path string, info os.FileInfo, err return nil } +func SimpleLocalWalk(root string, walkFn func(path string, info os.FileInfo, err error) error) error { + // see discussion in https://github.com/arduino/arduino-cli/pull/421 + return SimpleLocalWalkRecursive(root, 40, walkFn) +} + // SketchLoad collects all the files composing a sketch. // The parameter `sketchPath` holds a path pointing to a single sketch file or a sketch folder, // the path must be absolute. @@ -124,6 +134,12 @@ func SketchLoad(sketchPath, buildPath string) (*sketch.Sketch, error) { // collect all the sketch files var files []string err = SimpleLocalWalk(sketchFolder, func(path string, info os.FileInfo, err error) error { + + if err != nil { + fmt.Printf("\nerror: %+v\n\n", err) + return filepath.SkipDir; + } + // ignore hidden files and skip hidden directories if strings.HasPrefix(info.Name(), ".") { if info.IsDir() { From fe4cd8b6ac01ca005e32baf6ed59e693ab41773a Mon Sep 17 00:00:00 2001 From: david gauchard Date: Wed, 23 Oct 2019 23:25:46 +0200 Subject: [PATCH 11/15] fixes from "task check" --- arduino/builder/sketch.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/arduino/builder/sketch.go b/arduino/builder/sketch.go index f8358668332..74211fb780e 100644 --- a/arduino/builder/sketch.go +++ b/arduino/builder/sketch.go @@ -17,12 +17,12 @@ package builder import ( "bytes" + "fmt" "io/ioutil" "os" "path/filepath" "regexp" "strings" - "fmt" "github.com/arduino/arduino-cli/arduino/globals" "github.com/arduino/arduino-cli/arduino/sketch" @@ -59,8 +59,7 @@ func SketchSaveItemCpp(item *sketch.Item, destPath string) error { return nil } -// SimpleLocalWalk locally replaces filepath.Walk and/but goes through symlinks -func SimpleLocalWalkRecursive(root string, maxDepth int, walkFn func(path string, info os.FileInfo, err error) error) error { +func simpleLocalWalkRecursive(root string, maxDepth int, walkFn func(path string, info os.FileInfo, err error) error) error { info, err := os.Stat(root) @@ -75,13 +74,13 @@ func SimpleLocalWalkRecursive(root string, maxDepth int, walkFn func(path string if info.IsDir() { if maxDepth <= 0 { - return walkFn(root, info, errors.New("Filesystem bottom is too deep (directory recursion or filesystem really deep): " + root)) + return walkFn(root, info, errors.New("Filesystem bottom is too deep (directory recursion or filesystem really deep): "+root)) } maxDepth-- files, err := ioutil.ReadDir(root) if err == nil { for _, file := range files { - err = SimpleLocalWalkRecursive(root+string(os.PathSeparator)+file.Name(), maxDepth, walkFn) + err = simpleLocalWalkRecursive(root+string(os.PathSeparator)+file.Name(), maxDepth, walkFn) if err == filepath.SkipDir { return nil } @@ -92,9 +91,10 @@ func SimpleLocalWalkRecursive(root string, maxDepth int, walkFn func(path string return nil } +// SimpleLocalWalk locally replaces filepath.Walk and/but goes through symlinks func SimpleLocalWalk(root string, walkFn func(path string, info os.FileInfo, err error) error) error { // see discussion in https://github.com/arduino/arduino-cli/pull/421 - return SimpleLocalWalkRecursive(root, 40, walkFn) + return simpleLocalWalkRecursive(root, 40, walkFn) } // SketchLoad collects all the files composing a sketch. @@ -137,7 +137,7 @@ func SketchLoad(sketchPath, buildPath string) (*sketch.Sketch, error) { if err != nil { fmt.Printf("\nerror: %+v\n\n", err) - return filepath.SkipDir; + return filepath.SkipDir } // ignore hidden files and skip hidden directories From b51893a247991e58a6124a6ded636dde7b551182 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 28 Oct 2019 19:47:57 +0100 Subject: [PATCH 12/15] updates per review --- arduino/builder/sketch.go | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/arduino/builder/sketch.go b/arduino/builder/sketch.go index 74211fb780e..547ebe78447 100644 --- a/arduino/builder/sketch.go +++ b/arduino/builder/sketch.go @@ -17,19 +17,23 @@ package builder import ( "bytes" - "fmt" "io/ioutil" "os" "path/filepath" "regexp" "strings" + "github.com/arduino/arduino-cli/cli/feedback" "github.com/arduino/arduino-cli/arduino/globals" "github.com/arduino/arduino-cli/arduino/sketch" "github.com/pkg/errors" ) +// As currently implemented on Linux, +// the maximum number of symbolic links that will be followed while resolving a pathname is 40 +const maxFileSystemDepth = 40 + var includesArduinoH = regexp.MustCompile(`(?m)^\s*#\s*include\s*[<\"]Arduino\.h[>\"]`) // QuoteCppString returns the given string as a quoted string for use with the C @@ -59,7 +63,8 @@ func SketchSaveItemCpp(item *sketch.Item, destPath string) error { return nil } -func simpleLocalWalkRecursive(root string, maxDepth int, walkFn func(path string, info os.FileInfo, err error) error) error { +// simpleLocalWalk locally replaces filepath.Walk and/but goes through symlinks +func simpleLocalWalk(root string, maxDepth int, walkFn func(path string, info os.FileInfo, err error) error) error { info, err := os.Stat(root) @@ -80,7 +85,7 @@ func simpleLocalWalkRecursive(root string, maxDepth int, walkFn func(path string files, err := ioutil.ReadDir(root) if err == nil { for _, file := range files { - err = simpleLocalWalkRecursive(root+string(os.PathSeparator)+file.Name(), maxDepth, walkFn) + err = simpleLocalWalk(root+string(os.PathSeparator)+file.Name(), maxDepth, walkFn) if err == filepath.SkipDir { return nil } @@ -91,12 +96,6 @@ func simpleLocalWalkRecursive(root string, maxDepth int, walkFn func(path string return nil } -// SimpleLocalWalk locally replaces filepath.Walk and/but goes through symlinks -func SimpleLocalWalk(root string, walkFn func(path string, info os.FileInfo, err error) error) error { - // see discussion in https://github.com/arduino/arduino-cli/pull/421 - return simpleLocalWalkRecursive(root, 40, walkFn) -} - // SketchLoad collects all the files composing a sketch. // The parameter `sketchPath` holds a path pointing to a single sketch file or a sketch folder, // the path must be absolute. @@ -133,10 +132,10 @@ func SketchLoad(sketchPath, buildPath string) (*sketch.Sketch, error) { // collect all the sketch files var files []string - err = SimpleLocalWalk(sketchFolder, func(path string, info os.FileInfo, err error) error { + err = simpleLocalWalk(sketchFolder, maxFileSystemDepth, func(path string, info os.FileInfo, err error) error { if err != nil { - fmt.Printf("\nerror: %+v\n\n", err) + feedback.Printf("\nerror: %+v\n\n", err) return filepath.SkipDir } From cde4bec8506d3f6f9b6c88d04df25c6b2b70878e Mon Sep 17 00:00:00 2001 From: david gauchard Date: Tue, 29 Oct 2019 09:31:59 +0100 Subject: [PATCH 13/15] style --- arduino/builder/sketch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arduino/builder/sketch.go b/arduino/builder/sketch.go index 547ebe78447..ed42ed03dd8 100644 --- a/arduino/builder/sketch.go +++ b/arduino/builder/sketch.go @@ -23,9 +23,9 @@ import ( "regexp" "strings" - "github.com/arduino/arduino-cli/cli/feedback" "github.com/arduino/arduino-cli/arduino/globals" "github.com/arduino/arduino-cli/arduino/sketch" + "github.com/arduino/arduino-cli/cli/feedback" "github.com/pkg/errors" ) From d41fa12d73e10b2b1779add99d27cb8a0d795444 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Tue, 29 Oct 2019 10:50:19 +0100 Subject: [PATCH 14/15] Update arduino/builder/sketch.go Co-Authored-By: Roberto Sora --- arduino/builder/sketch.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arduino/builder/sketch.go b/arduino/builder/sketch.go index ed42ed03dd8..5760b1c2b44 100644 --- a/arduino/builder/sketch.go +++ b/arduino/builder/sketch.go @@ -135,7 +135,8 @@ func SketchLoad(sketchPath, buildPath string) (*sketch.Sketch, error) { err = simpleLocalWalk(sketchFolder, maxFileSystemDepth, func(path string, info os.FileInfo, err error) error { if err != nil { - feedback.Printf("\nerror: %+v\n\n", err) + feedback.Errorf("Error during sketch processing: %v", err) + os.Exit(errorcodes.ErrGeneric) return filepath.SkipDir } From 9de33441d21016974f2bf972ecab70da8543eaef Mon Sep 17 00:00:00 2001 From: david gauchard Date: Tue, 29 Oct 2019 11:11:45 +0100 Subject: [PATCH 15/15] fixes --- arduino/builder/sketch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arduino/builder/sketch.go b/arduino/builder/sketch.go index 5760b1c2b44..d9ad4c50385 100644 --- a/arduino/builder/sketch.go +++ b/arduino/builder/sketch.go @@ -25,6 +25,7 @@ import ( "github.com/arduino/arduino-cli/arduino/globals" "github.com/arduino/arduino-cli/arduino/sketch" + "github.com/arduino/arduino-cli/cli/errorcodes" "github.com/arduino/arduino-cli/cli/feedback" "github.com/pkg/errors" @@ -137,7 +138,6 @@ func SketchLoad(sketchPath, buildPath string) (*sketch.Sketch, error) { if err != nil { feedback.Errorf("Error during sketch processing: %v", err) os.Exit(errorcodes.ErrGeneric) - return filepath.SkipDir } // ignore hidden files and skip hidden directories