From 096da1bb8ef9940de063a1db8a68a0c991e79796 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Wed, 25 Sep 2024 08:16:36 +0700 Subject: [PATCH 1/6] refactor(vardump): use `godump` lib also increate limit char to `255`. Signed-off-by: Dwi Siswanto --- go.mod | 1 + go.sum | 2 + pkg/protocols/common/utils/vardump/dump.go | 83 +++++++++++++--------- 3 files changed, 53 insertions(+), 33 deletions(-) diff --git a/go.mod b/go.mod index f2adacca21..dfd346efab 100644 --- a/go.mod +++ b/go.mod @@ -104,6 +104,7 @@ require ( github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 github.com/stretchr/testify v1.9.0 github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9 + github.com/yassinebenaid/godump v0.10.0 github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706 golang.org/x/term v0.24.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index da71e9b287..a77ec3c07d 100644 --- a/go.sum +++ b/go.sum @@ -1096,6 +1096,8 @@ github.com/xhit/go-str2duration v1.2.0/go.mod h1:3cPSlfZlUHVlneIVfePFWcJZsuwf+P1 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/yassinebenaid/godump v0.10.0 h1:FolBA+Ix5uwUiXYBBYOsf1VkT5+0f4gtFNTkYTiIR08= +github.com/yassinebenaid/godump v0.10.0/go.mod h1:dc/0w8wmg6kVIvNGAzbKH1Oa54dXQx8SNKh4dPRyW44= github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU= github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g= github.com/ysmood/fetchup v0.2.3 h1:ulX+SonA0Vma5zUFXtv52Kzip/xe7aj4vqT5AJwQ+ZQ= diff --git a/pkg/protocols/common/utils/vardump/dump.go b/pkg/protocols/common/utils/vardump/dump.go index ab4f56b113..eae5668af4 100644 --- a/pkg/protocols/common/utils/vardump/dump.go +++ b/pkg/protocols/common/utils/vardump/dump.go @@ -1,53 +1,70 @@ package vardump import ( - "strconv" "strings" "github.com/projectdiscovery/nuclei/v3/pkg/types" mapsutil "github.com/projectdiscovery/utils/maps" + "github.com/yassinebenaid/godump" ) // EnableVarDump enables var dump for debugging optionally var EnableVarDump bool -// DumpVariables writes the truncated dump of variables to a string -// in a formatted key-value manner. -// -// The values are truncated to return 50 characters from start and end. -func DumpVariables(data map[string]interface{}) string { - var counter int +// variables is a map of variables +type variables = map[string]any - buffer := &strings.Builder{} - buffer.Grow(len(data) * 78) // grow buffer to an approximate size +// DumpVariables dumps the variables in a pretty format +func DumpVariables(data variables) string { + if !EnableVarDump { + return "" + } + + d := godump.Dumper{ + Indentation: " ", + HidePrivateFields: false, + ShowPrimitiveNamedTypes: true, + } - builder := &strings.Builder{} - // sort keys for deterministic output + d.Theme = godump.Theme{ + String: godump.RGB{R: 138, G: 201, B: 38}, + Quotes: godump.RGB{R: 112, G: 214, B: 255}, + Bool: godump.RGB{R: 249, G: 87, B: 56}, + Number: godump.RGB{R: 10, G: 178, B: 242}, + Types: godump.RGB{R: 0, G: 150, B: 199}, + Address: godump.RGB{R: 205, G: 93, B: 0}, + PointerTag: godump.RGB{R: 110, G: 110, B: 110}, + Nil: godump.RGB{R: 219, G: 57, B: 26}, + Func: godump.RGB{R: 160, G: 90, B: 220}, + Fields: godump.RGB{R: 189, G: 176, B: 194}, + Chan: godump.RGB{R: 195, G: 154, B: 76}, + UnsafePointer: godump.RGB{R: 89, G: 193, B: 180}, + Braces: godump.RGB{R: 185, G: 86, B: 86}, + } + + return d.Sprint(process(data, 255)) +} + +// process is a helper function that processes the variables +// and returns a new map of variables +func process(data variables, limit int) variables { keys := mapsutil.GetSortedKeys(data) + vars := make(variables) + + if limit == 0 { + limit = 255 + } for _, k := range keys { - v := data[k] - valueString := types.ToString(v) - - counter++ - if len(valueString) > 50 { - builder.Grow(56) - builder.WriteString(valueString[0:25]) - builder.WriteString(" .... ") - builder.WriteString(valueString[len(valueString)-25:]) - valueString = builder.String() - builder.Reset() + v := types.ToString(data[k]) + v = strings.ReplaceAll(strings.ReplaceAll(v, "\r", " "), "\n", " ") + if len(v) > limit { + v = v[:limit] + v += " [...]" } - valueString = strings.ReplaceAll(strings.ReplaceAll(valueString, "\r", " "), "\n", " ") - - buffer.WriteString("\t") - buffer.WriteString(strconv.Itoa(counter)) - buffer.WriteString(". ") - buffer.WriteString(k) - buffer.WriteString(" => ") - buffer.WriteString(valueString) - buffer.WriteString("\n") + + vars[k] = v } - final := buffer.String() - return final + + return vars } From 2fdfb7835da09a41e660c4b42aa0d27208092f76 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Mon, 30 Sep 2024 13:51:58 +0700 Subject: [PATCH 2/6] feat(vardump): add global var `Limit` Signed-off-by: Dwi Siswanto --- pkg/protocols/common/utils/vardump/dump.go | 5 +---- pkg/protocols/common/utils/vardump/vars.go | 8 ++++++++ 2 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 pkg/protocols/common/utils/vardump/vars.go diff --git a/pkg/protocols/common/utils/vardump/dump.go b/pkg/protocols/common/utils/vardump/dump.go index eae5668af4..82de85e67b 100644 --- a/pkg/protocols/common/utils/vardump/dump.go +++ b/pkg/protocols/common/utils/vardump/dump.go @@ -8,9 +8,6 @@ import ( "github.com/yassinebenaid/godump" ) -// EnableVarDump enables var dump for debugging optionally -var EnableVarDump bool - // variables is a map of variables type variables = map[string]any @@ -42,7 +39,7 @@ func DumpVariables(data variables) string { Braces: godump.RGB{R: 185, G: 86, B: 86}, } - return d.Sprint(process(data, 255)) + return d.Sprint(process(data, Limit)) } // process is a helper function that processes the variables diff --git a/pkg/protocols/common/utils/vardump/vars.go b/pkg/protocols/common/utils/vardump/vars.go new file mode 100644 index 0000000000..f5e18bce69 --- /dev/null +++ b/pkg/protocols/common/utils/vardump/vars.go @@ -0,0 +1,8 @@ +package vardump + +var ( + // EnableVarDump enables var dump for debugging optionally + EnableVarDump bool + // Limit is the maximum characters to be dumped + Limit int = 255 +) From ab57ae1a094136faa1823d4b425890a8945ea7b7 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Mon, 30 Sep 2024 13:52:51 +0700 Subject: [PATCH 3/6] chore(protocols): rm newline Signed-off-by: Dwi Siswanto --- pkg/protocols/code/code.go | 2 +- pkg/protocols/common/helpers/eventcreator/eventcreator.go | 2 +- pkg/protocols/dns/request.go | 2 +- pkg/protocols/headless/engine/page_actions.go | 2 +- pkg/protocols/headless/request.go | 2 +- pkg/protocols/http/build_request.go | 2 +- pkg/protocols/javascript/js.go | 2 +- pkg/protocols/network/request.go | 2 +- pkg/protocols/ssl/ssl.go | 2 +- pkg/protocols/websocket/websocket.go | 2 +- pkg/protocols/whois/whois.go | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/protocols/code/code.go b/pkg/protocols/code/code.go index 87a0128645..4503b2117f 100644 --- a/pkg/protocols/code/code.go +++ b/pkg/protocols/code/code.go @@ -235,7 +235,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa gologger.Verbose().Msgf("[%s] Executed code on local machine %v", request.options.TemplateID, input.MetaInput.Input) if vardump.EnableVarDump { - gologger.Debug().Msgf("Code Protocol request variables: \n%s\n", vardump.DumpVariables(allvars)) + gologger.Debug().Msgf("Code Protocol request variables: %s\n", vardump.DumpVariables(allvars)) } if request.options.Options.Debug || request.options.Options.DebugRequests { diff --git a/pkg/protocols/common/helpers/eventcreator/eventcreator.go b/pkg/protocols/common/helpers/eventcreator/eventcreator.go index 078963ceed..22c0a7810d 100644 --- a/pkg/protocols/common/helpers/eventcreator/eventcreator.go +++ b/pkg/protocols/common/helpers/eventcreator/eventcreator.go @@ -24,7 +24,7 @@ func CreateEventWithAdditionalOptions(request protocols.Request, outputEvent out // Dump response variables if ran in debug mode if vardump.EnableVarDump { protoName := cases.Title(language.English).String(request.Type().String()) - gologger.Debug().Msgf("%v Protocol response variables: \n%s\n", protoName, vardump.DumpVariables(outputEvent)) + gologger.Debug().Msgf("%v Protocol response variables: %s\n", protoName, vardump.DumpVariables(outputEvent)) } for _, compiledOperator := range request.GetCompiledOperators() { if compiledOperator != nil { diff --git a/pkg/protocols/dns/request.go b/pkg/protocols/dns/request.go index 9457845270..6e82c047bc 100644 --- a/pkg/protocols/dns/request.go +++ b/pkg/protocols/dns/request.go @@ -108,7 +108,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, func (request *Request) execute(input *contextargs.Context, domain string, metadata, previous output.InternalEvent, vars map[string]interface{}, callback protocols.OutputEventCallback) error { var err error if vardump.EnableVarDump { - gologger.Debug().Msgf("DNS Protocol request variables: \n%s\n", vardump.DumpVariables(vars)) + gologger.Debug().Msgf("DNS Protocol request variables: %s\n", vardump.DumpVariables(vars)) } // Compile each request for the template based on the URL diff --git a/pkg/protocols/headless/engine/page_actions.go b/pkg/protocols/headless/engine/page_actions.go index 18ed515662..640bb76214 100644 --- a/pkg/protocols/headless/engine/page_actions.go +++ b/pkg/protocols/headless/engine/page_actions.go @@ -334,7 +334,7 @@ func (p *Page) NavigateURL(action *Action, out ActionData, allvars map[string]in allvars = generators.MergeMaps(allvars, defaultReqVars) if vardump.EnableVarDump { - gologger.Debug().Msgf("Headless Protocol request variables: \n%s\n", vardump.DumpVariables(allvars)) + gologger.Debug().Msgf("Headless Protocol request variables: %s\n", vardump.DumpVariables(allvars)) } // Evaluate the target url with all variables diff --git a/pkg/protocols/headless/request.go b/pkg/protocols/headless/request.go index 5f9b53174e..aae70aa347 100644 --- a/pkg/protocols/headless/request.go +++ b/pkg/protocols/headless/request.go @@ -122,7 +122,7 @@ func (request *Request) executeRequestWithPayloads(input *contextargs.Context, p defer instance.Close() if vardump.EnableVarDump { - gologger.Debug().Msgf("Headless Protocol request variables: \n%s\n", vardump.DumpVariables(payloads)) + gologger.Debug().Msgf("Headless Protocol request variables: %s\n", vardump.DumpVariables(payloads)) } instance.SetInteractsh(request.options.Interactsh) diff --git a/pkg/protocols/http/build_request.go b/pkg/protocols/http/build_request.go index 1b046bffd8..bc3b41244e 100644 --- a/pkg/protocols/http/build_request.go +++ b/pkg/protocols/http/build_request.go @@ -204,7 +204,7 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, finalVars := generators.MergeMaps(allVars, payloads) if vardump.EnableVarDump { - gologger.Debug().Msgf("HTTP Protocol request variables: \n%s\n", vardump.DumpVariables(finalVars)) + gologger.Debug().Msgf("HTTP Protocol request variables: %s\n", vardump.DumpVariables(finalVars)) } // Note: If possible any changes to current logic (i.e evaluate -> then parse URL) diff --git a/pkg/protocols/javascript/js.go b/pkg/protocols/javascript/js.go index 0bd26b88a5..5953c9c296 100644 --- a/pkg/protocols/javascript/js.go +++ b/pkg/protocols/javascript/js.go @@ -319,7 +319,7 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, dynamicV templateCtx.Merge(payloadValues) if vardump.EnableVarDump { - gologger.Debug().Msgf("Javascript Protocol request variables: \n%s\n", vardump.DumpVariables(payloadValues)) + gologger.Debug().Msgf("JavaScript Protocol request variables: %s\n", vardump.DumpVariables(payloadValues)) } if request.PreCondition != "" { diff --git a/pkg/protocols/network/request.go b/pkg/protocols/network/request.go index 32d4ae3494..29cdca4ad2 100644 --- a/pkg/protocols/network/request.go +++ b/pkg/protocols/network/request.go @@ -283,7 +283,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac interimValues := generators.MergeMaps(variables, payloads) if vardump.EnableVarDump { - gologger.Debug().Msgf("Network Protocol request variables: \n%s\n", vardump.DumpVariables(interimValues)) + gologger.Debug().Msgf("Network Protocol request variables: %s\n", vardump.DumpVariables(interimValues)) } inputEvents := make(map[string]interface{}) diff --git a/pkg/protocols/ssl/ssl.go b/pkg/protocols/ssl/ssl.go index 681043d3bf..5bd3befdf1 100644 --- a/pkg/protocols/ssl/ssl.go +++ b/pkg/protocols/ssl/ssl.go @@ -222,7 +222,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa payloadValues = generators.MergeMaps(variablesMap, payloadValues, request.options.Constants) if vardump.EnableVarDump { - gologger.Debug().Msgf("SSL Protocol request variables: \n%s\n", vardump.DumpVariables(payloadValues)) + gologger.Debug().Msgf("SSL Protocol request variables: %s\n", vardump.DumpVariables(payloadValues)) } finalAddress, dataErr := expressions.EvaluateByte([]byte(request.Address), payloadValues) diff --git a/pkg/protocols/websocket/websocket.go b/pkg/protocols/websocket/websocket.go index aa099ef43a..8eeeedf217 100644 --- a/pkg/protocols/websocket/websocket.go +++ b/pkg/protocols/websocket/websocket.go @@ -207,7 +207,7 @@ func (request *Request) executeRequestWithPayloads(target *contextargs.Context, } if vardump.EnableVarDump { - gologger.Debug().Msgf("Websocket Protocol request variables: \n%s\n", vardump.DumpVariables(payloadValues)) + gologger.Debug().Msgf("WebSocket Protocol request variables: %s\n", vardump.DumpVariables(payloadValues)) } finalAddress, dataErr := expressions.EvaluateByte([]byte(request.Address), payloadValues) diff --git a/pkg/protocols/whois/whois.go b/pkg/protocols/whois/whois.go index 9963ec19f8..91d0edcf8a 100644 --- a/pkg/protocols/whois/whois.go +++ b/pkg/protocols/whois/whois.go @@ -99,7 +99,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa variables := generators.MergeMaps(vars, defaultVars, optionVars, dynamicValues, request.options.Constants) if vardump.EnableVarDump { - gologger.Debug().Msgf("Whois Protocol request variables: \n%s\n", vardump.DumpVariables(variables)) + gologger.Debug().Msgf("Whois Protocol request variables: %s\n", vardump.DumpVariables(variables)) } // and replace placeholders From d04c78807ea5571b5736e986f2b8c2ba13f1290f Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Mon, 30 Sep 2024 14:04:40 +0700 Subject: [PATCH 4/6] feat(types): add `VarDumpLimit` option Signed-off-by: Dwi Siswanto --- cmd/nuclei/main.go | 1 + internal/runner/options.go | 1 + pkg/types/types.go | 2 ++ 3 files changed, 4 insertions(+) diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index 8fe5b29573..7eb4092c1a 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -405,6 +405,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringVar(&memProfile, "profile-mem", "", "optional nuclei memory profile dump file"), flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display templates loaded for scan"), flagSet.BoolVarP(&options.ShowVarDump, "show-var-dump", "svd", false, "show variables dump for debugging"), + flagSet.IntVarP(&options.VarDumpLimit, "var-dump-limit", "vdl", 255, "limit the number of characters displayed in var dump"), flagSet.BoolVarP(&options.EnablePprof, "enable-pprof", "ep", false, "enable pprof debugging server"), flagSet.CallbackVarP(printTemplateVersion, "templates-version", "tv", "shows the version of the installed nuclei-templates"), flagSet.BoolVarP(&options.HealthCheck, "health-check", "hc", false, "run diagnostic check up"), diff --git a/internal/runner/options.go b/internal/runner/options.go index 2872b96a7a..4ad62a855a 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -68,6 +68,7 @@ func ParseOptions(options *types.Options) { if options.ShowVarDump { vardump.EnableVarDump = true + vardump.Limit = options.VarDumpLimit } if options.ShowActions { gologger.Info().Msgf("Showing available headless actions: ") diff --git a/pkg/types/types.go b/pkg/types/types.go index 9cc88f49ff..f6e7ab4470 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -206,6 +206,8 @@ type Options struct { VerboseVerbose bool // ShowVarDump displays variable dump ShowVarDump bool + // VarDumpLimit limits the number of characters displayed in var dump + VarDumpLimit int // No-Color disables the colored output. NoColor bool // UpdateTemplates updates the templates installed at startup (also used by cloud to update datasources) From 726f43b0912944133d8191ed9ab7627a9c9c2731 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Mon, 30 Sep 2024 14:21:08 +0700 Subject: [PATCH 5/6] test(vardump): add test cases Signed-off-by: Dwi Siswanto --- .../common/utils/vardump/dump_test.go | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 pkg/protocols/common/utils/vardump/dump_test.go diff --git a/pkg/protocols/common/utils/vardump/dump_test.go b/pkg/protocols/common/utils/vardump/dump_test.go new file mode 100644 index 0000000000..9929fa5318 --- /dev/null +++ b/pkg/protocols/common/utils/vardump/dump_test.go @@ -0,0 +1,55 @@ +package vardump + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDumpVariables(t *testing.T) { + // Enable var dump for testing + EnableVarDump = true + + // Test case + testVars := variables{ + "string": "test", + "int": 42, + "bool": true, + "slice": []string{"a", "b", "c"}, + } + + result := DumpVariables(testVars) + + // Assertions + assert.NotEmpty(t, result) + assert.Contains(t, result, "string") + assert.Contains(t, result, "test") + assert.Contains(t, result, "int") + assert.Contains(t, result, "42") + assert.Contains(t, result, "bool") + assert.Contains(t, result, "true") + assert.Contains(t, result, "slice") + assert.Contains(t, result, "a") + assert.Contains(t, result, "b") + assert.Contains(t, result, "c") + + // Test with EnableVarDump set to false + EnableVarDump = false + result = DumpVariables(testVars) + assert.Empty(t, result) +} + +func TestProcess(t *testing.T) { + testVars := variables{ + "short": "short string", + "long": strings.Repeat("a", 300), + "number": 42, + } + + processed := process(testVars, 255) + + assert.Equal(t, "short string", processed["short"]) + assert.Equal(t, strings.Repeat("a", 255)+" [...]", processed["long"]) + assert.Equal(t, "42", processed["number"]) +} From 0b27af79a7aabcd38431fd585f2d1d93e736d704 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Mon, 30 Sep 2024 14:23:04 +0700 Subject: [PATCH 6/6] chore: tidy up mod Signed-off-by: Dwi Siswanto --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index dfd346efab..fff0c2f7f7 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/projectdiscovery/nuclei/v3 -go 1.21 +go 1.21.0 require ( github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible