Merge pull request #494 from projectdiscovery/improve_str_normalize_func

improve str normalize func
projectdiscovery · Aug 15, 2024 · f6b40d9 · f6b40d9
2 parents 7b58775 + 7383e38
commit f6b40d9
Show file tree

Hide file tree

Showing 7 changed files with 72 additions and 16 deletions.
diff --git a/channelutil/clone_join_test.go b/channelutil/clone_join_test.go
@@ -62,7 +62,7 @@ func TestCloneCounter(t *testing.T) {
 		cloner := channelutil.NewCloneChannels[struct{}](cloneOpts)
 		err := cloner.Clone(context.TODO(), source, sinks...)
 		if err != nil {
-			t.Fatalf(err.Error())
+			t.Fatalf("%s", err.Error())
 		}
 	}
 }

diff --git a/errkit/helpers.go b/errkit/helpers.go
@@ -82,7 +82,7 @@ func Wrap(err error, message string) error {
 	}
 	x := &ErrorX{}
 	parseError(x, err)
-	x.Msgf(message)
+	x.Msgf("%s", message)
 	return x
 }
 
@@ -148,7 +148,7 @@ func WithMessage(err error, message string) error {
 	}
 	x := &ErrorX{}
 	parseError(x, err)
-	x.Msgf(message)
+	x.Msgf("%s", message)
 	return x
 }
 

diff --git a/errors/enriched.go b/errors/enriched.go
@@ -66,10 +66,10 @@ func (e *enrichedError) Wrap(err ...error) Error {
 			continue
 		}
 		if ee, ok := v.(*enrichedError); ok {
-			_ = e.Msgf(ee.errString).WithLevel(ee.Level).WithTag(ee.Tags...)
+			_ = e.Msgf("%s", ee.errString).WithLevel(ee.Level).WithTag(ee.Tags...)
 			e.StackTrace += ee.StackTrace
 		} else {
-			_ = e.Msgf(v.Error())
+			_ = e.Msgf("%s", v.Error())
 		}
 	}
 	return e
@@ -131,10 +131,10 @@ func NewWithErr(err error) Error {
 		return nil
 	}
 	if ee, ok := err.(*enrichedError); ok {
-		x := New(ee.errString).WithTag(ee.Tags...).WithLevel(ee.Level)
+		x := New("%s", ee.errString).WithTag(ee.Tags...).WithLevel(ee.Level)
 		x.(*enrichedError).StackTrace = ee.StackTrace
 	}
-	return New(err.Error())
+	return New("%s", err.Error())
 }
 
 // NewWithTag creates an error with tag

diff --git a/exec/executil.go b/exec/executil.go
@@ -172,7 +172,7 @@ func RunSafe(cmd ...string) (string, error) {
 
 	if err := cmdExec.Wait(); err != nil {
 		if _, ok := err.(*exec.ExitError); ok {
-			adbError = errkit.Append(err, errkit.New(string(errorData)), errkit.New("exit error"))
+			adbError = errkit.Append(err, errkit.New("%s", string(errorData)), errkit.New("exit error"))
 			outData = errorData
 		} else {
 			return "", errkit.Wrap(err, "process i/o error")
@@ -215,7 +215,7 @@ func RunSh(cmd ...string) (string, error) {
 
 	if err := cmdExec.Wait(); err != nil {
 		if _, ok := err.(*exec.ExitError); ok {
-			adbError = errkit.Append(err, errkit.New(string(errorData)), errkit.New("exit error"))
+			adbError = errkit.Append(err, errkit.New("%s", string(errorData)), errkit.New("exit error"))
 			outData = errorData
 		} else {
 			return "", errkit.Wrap(err, "process i/o error")
@@ -272,7 +272,7 @@ func RunPS(cmd string) (string, error) {
 
 	if err := cmdExec.Wait(); err != nil {
 		if _, ok := err.(*exec.ExitError); ok {
-			adbError = errkit.Append(err, errkit.New(string(errorData)), errkit.New("exit error"))
+			adbError = errkit.Append(err, errkit.New("%s", string(errorData)), errkit.New("exit error"))
 			outData = errorData
 		} else {
 			return "", errkit.Wrap(err, "process i/o error")

diff --git a/strings/strings_normalize.go b/strings/strings_normalize.go
@@ -2,15 +2,18 @@ package stringsutil
 
 import (
 	"strings"
+	"unicode"
 
 	"github.com/microcosm-cc/bluemonday"
 )
 
 type NormalizeOptions struct {
-	TrimSpaces bool
-	StripHTML  bool
-	Lowercase  bool
-	Uppercase  bool
+	TrimSpaces    bool
+	TrimCutset    string
+	StripHTML     bool
+	Lowercase     bool
+	Uppercase     bool
+	StripComments bool
 }
 
 var DefaultNormalizeOptions NormalizeOptions = NormalizeOptions{
@@ -25,6 +28,10 @@ func NormalizeWithOptions(data string, options NormalizeOptions) string {
 		data = strings.TrimSpace(data)
 	}
 
+	if options.TrimCutset != "" {
+		data = strings.Trim(data, options.TrimCutset)
+	}
+
 	if options.Lowercase {
 		data = strings.ToLower(data)
 	}
@@ -37,6 +44,12 @@ func NormalizeWithOptions(data string, options NormalizeOptions) string {
 		data = HTMLPolicy.Sanitize(data)
 	}
 
+	if options.StripComments {
+		if cut := strings.IndexAny(data, "#"); cut >= 0 {
+			data = strings.TrimRightFunc(data[:cut], unicode.IsSpace)
+		}
+	}
+
 	return data
 }
 

diff --git a/strings/stringsutil_test.go b/strings/stringsutil_test.go
@@ -424,3 +424,46 @@ func TestContainsAllI(t *testing.T) {
 		require.Equal(t, test.result, res)
 	}
 }
+
+func TestNormalizeWithOptions(t *testing.T) {
+	tests := []struct {
+		data    string
+		options NormalizeOptions
+		result  string
+	}{
+		{
+			data:    "  Hello World!  ",
+			options: NormalizeOptions{TrimSpaces: true},
+			result:  "Hello World!",
+		},
+		{
+			data:    "\n\t\"'` Hello World! \n\t\"'` ",
+			options: NormalizeOptions{TrimCutset: "\n\t\"'` "},
+			result:  "Hello World!",
+		},
+		{
+			data:    "  Hello World!  ",
+			options: NormalizeOptions{Lowercase: true},
+			result:  "  hello world!  ",
+		},
+		{
+			data:    "  Hello World!  ",
+			options: NormalizeOptions{Uppercase: true},
+			result:  "  HELLO WORLD!  ",
+		},
+		{
+			data:    "<b>Hello World!</b>",
+			options: NormalizeOptions{StripHTML: true},
+			result:  "Hello World!",
+		},
+		{
+			data:    "Hello World! # Comment",
+			options: NormalizeOptions{StripComments: true},
+			result:  "Hello World!",
+		},
+	}
+	for _, test := range tests {
+		res := NormalizeWithOptions(test.data, test.options)
+		require.Equal(t, test.result, res)
+	}
+}
diff --git a/update/update.go b/update/update.go
@@ -143,9 +143,9 @@ func GetToolVersionCallback(toolName, version string) func() (string, error) {
 		if toolDetails.Version == "" {
 			msg := fmt.Sprintf("something went wrong, expected version string but got empty string for GET `%v` response `%v`", updateURL, string(body))
 			if err == nil {
-				return "", errorutil.New(msg)
+				return "", errorutil.New("%s", msg)
 			}
-			return "", errorutil.NewWithErr(err).Msgf(msg)
+			return "", errorutil.NewWithErr(err).Msgf("%s", msg)
 		}
 		return toolDetails.Version, nil
 	}