Skip to content

Commit

Permalink
refactor(vardump): use godump lib (#5676)
Browse files Browse the repository at this point in the history
* refactor(vardump): use `godump` lib

also increate limit char to `255`.

Signed-off-by: Dwi Siswanto <git@dw1.io>

* feat(vardump): add global var `Limit`

Signed-off-by: Dwi Siswanto <git@dw1.io>

* chore(protocols): rm newline

Signed-off-by: Dwi Siswanto <git@dw1.io>

* feat(types): add `VarDumpLimit` option

Signed-off-by: Dwi Siswanto <git@dw1.io>

* test(vardump): add test cases

Signed-off-by: Dwi Siswanto <git@dw1.io>

* chore: tidy up mod

Signed-off-by: Dwi Siswanto <git@dw1.io>

---------

Signed-off-by: Dwi Siswanto <git@dw1.io>
  • Loading branch information
dwisiswant0 authored Oct 14, 2024
1 parent 53f56e1 commit 2c832f5
Show file tree
Hide file tree
Showing 19 changed files with 131 additions and 47 deletions.
1 change: 1 addition & 0 deletions cmd/nuclei/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -425,6 +425,7 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.StringVar(&memProfile, "profile-mem", "", "generate memory (heap) profile & trace files"),
flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display templates loaded for scan"),
flagSet.BoolVarP(&options.ShowVarDump, "show-var-dump", "svd", false, "show variables dump for debugging"),
flagSet.IntVarP(&options.VarDumpLimit, "var-dump-limit", "vdl", 255, "limit the number of characters displayed in var dump"),
flagSet.BoolVarP(&options.EnablePprof, "enable-pprof", "ep", false, "enable pprof debugging server"),
flagSet.CallbackVarP(printTemplateVersion, "templates-version", "tv", "shows the version of the installed nuclei-templates"),
flagSet.BoolVarP(&options.HealthCheck, "health-check", "hc", false, "run diagnostic check up"),
Expand Down
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/projectdiscovery/nuclei/v3

go 1.21
go 1.21.0

require (
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible
Expand Down Expand Up @@ -104,6 +104,7 @@ require (
github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466
github.com/stretchr/testify v1.9.0
github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9
github.com/yassinebenaid/godump v0.10.0
github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706
go.mongodb.org/mongo-driver v1.17.0
golang.org/x/term v0.24.0
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -1104,6 +1104,8 @@ github.com/xhit/go-str2duration v1.2.0/go.mod h1:3cPSlfZlUHVlneIVfePFWcJZsuwf+P1
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/yassinebenaid/godump v0.10.0 h1:FolBA+Ix5uwUiXYBBYOsf1VkT5+0f4gtFNTkYTiIR08=
github.com/yassinebenaid/godump v0.10.0/go.mod h1:dc/0w8wmg6kVIvNGAzbKH1Oa54dXQx8SNKh4dPRyW44=
github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU=
github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g=
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM=
Expand Down
1 change: 1 addition & 0 deletions internal/runner/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ func ParseOptions(options *types.Options) {

if options.ShowVarDump {
vardump.EnableVarDump = true
vardump.Limit = options.VarDumpLimit
}
if options.ShowActions {
gologger.Info().Msgf("Showing available headless actions: ")
Expand Down
2 changes: 1 addition & 1 deletion pkg/protocols/code/code.go
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
gologger.Verbose().Msgf("[%s] Executed code on local machine %v", request.options.TemplateID, input.MetaInput.Input)

if vardump.EnableVarDump {
gologger.Debug().Msgf("Code Protocol request variables: \n%s\n", vardump.DumpVariables(allvars))
gologger.Debug().Msgf("Code Protocol request variables: %s\n", vardump.DumpVariables(allvars))
}

if request.options.Options.Debug || request.options.Options.DebugRequests {
Expand Down
2 changes: 1 addition & 1 deletion pkg/protocols/common/helpers/eventcreator/eventcreator.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ func CreateEventWithAdditionalOptions(request protocols.Request, outputEvent out
// Dump response variables if ran in debug mode
if vardump.EnableVarDump {
protoName := cases.Title(language.English).String(request.Type().String())
gologger.Debug().Msgf("%v Protocol response variables: \n%s\n", protoName, vardump.DumpVariables(outputEvent))
gologger.Debug().Msgf("%v Protocol response variables: %s\n", protoName, vardump.DumpVariables(outputEvent))
}
for _, compiledOperator := range request.GetCompiledOperators() {
if compiledOperator != nil {
Expand Down
84 changes: 49 additions & 35 deletions pkg/protocols/common/utils/vardump/dump.go
Original file line number Diff line number Diff line change
@@ -1,53 +1,67 @@
package vardump

import (
"strconv"
"strings"

"github.com/projectdiscovery/nuclei/v3/pkg/types"
mapsutil "github.com/projectdiscovery/utils/maps"
"github.com/yassinebenaid/godump"
)

// EnableVarDump enables var dump for debugging optionally
var EnableVarDump bool
// variables is a map of variables
type variables = map[string]any

// DumpVariables writes the truncated dump of variables to a string
// in a formatted key-value manner.
//
// The values are truncated to return 50 characters from start and end.
func DumpVariables(data map[string]interface{}) string {
var counter int
// DumpVariables dumps the variables in a pretty format
func DumpVariables(data variables) string {
if !EnableVarDump {
return ""
}

d := godump.Dumper{
Indentation: " ",
HidePrivateFields: false,
ShowPrimitiveNamedTypes: true,
}

d.Theme = godump.Theme{
String: godump.RGB{R: 138, G: 201, B: 38},
Quotes: godump.RGB{R: 112, G: 214, B: 255},
Bool: godump.RGB{R: 249, G: 87, B: 56},
Number: godump.RGB{R: 10, G: 178, B: 242},
Types: godump.RGB{R: 0, G: 150, B: 199},
Address: godump.RGB{R: 205, G: 93, B: 0},
PointerTag: godump.RGB{R: 110, G: 110, B: 110},
Nil: godump.RGB{R: 219, G: 57, B: 26},
Func: godump.RGB{R: 160, G: 90, B: 220},
Fields: godump.RGB{R: 189, G: 176, B: 194},
Chan: godump.RGB{R: 195, G: 154, B: 76},
UnsafePointer: godump.RGB{R: 89, G: 193, B: 180},
Braces: godump.RGB{R: 185, G: 86, B: 86},
}

buffer := &strings.Builder{}
buffer.Grow(len(data) * 78) // grow buffer to an approximate size
return d.Sprint(process(data, Limit))
}

builder := &strings.Builder{}
// sort keys for deterministic output
// process is a helper function that processes the variables
// and returns a new map of variables
func process(data variables, limit int) variables {
keys := mapsutil.GetSortedKeys(data)
vars := make(variables)

if limit == 0 {
limit = 255
}

for _, k := range keys {
v := data[k]
valueString := types.ToString(v)

counter++
if len(valueString) > 50 {
builder.Grow(56)
builder.WriteString(valueString[0:25])
builder.WriteString(" .... ")
builder.WriteString(valueString[len(valueString)-25:])
valueString = builder.String()
builder.Reset()
v := types.ToString(data[k])
v = strings.ReplaceAll(strings.ReplaceAll(v, "\r", " "), "\n", " ")
if len(v) > limit {
v = v[:limit]
v += " [...]"
}
valueString = strings.ReplaceAll(strings.ReplaceAll(valueString, "\r", " "), "\n", " ")

buffer.WriteString("\t")
buffer.WriteString(strconv.Itoa(counter))
buffer.WriteString(". ")
buffer.WriteString(k)
buffer.WriteString(" => ")
buffer.WriteString(valueString)
buffer.WriteString("\n")

vars[k] = v
}
final := buffer.String()
return final

return vars
}
55 changes: 55 additions & 0 deletions pkg/protocols/common/utils/vardump/dump_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
package vardump

import (
"strings"
"testing"

"github.com/stretchr/testify/assert"
)

func TestDumpVariables(t *testing.T) {
// Enable var dump for testing
EnableVarDump = true

// Test case
testVars := variables{
"string": "test",
"int": 42,
"bool": true,
"slice": []string{"a", "b", "c"},
}

result := DumpVariables(testVars)

// Assertions
assert.NotEmpty(t, result)
assert.Contains(t, result, "string")
assert.Contains(t, result, "test")
assert.Contains(t, result, "int")
assert.Contains(t, result, "42")
assert.Contains(t, result, "bool")
assert.Contains(t, result, "true")
assert.Contains(t, result, "slice")
assert.Contains(t, result, "a")
assert.Contains(t, result, "b")
assert.Contains(t, result, "c")

// Test with EnableVarDump set to false
EnableVarDump = false
result = DumpVariables(testVars)
assert.Empty(t, result)
}

func TestProcess(t *testing.T) {
testVars := variables{
"short": "short string",
"long": strings.Repeat("a", 300),
"number": 42,
}

processed := process(testVars, 255)

assert.Equal(t, "short string", processed["short"])
assert.Equal(t, strings.Repeat("a", 255)+" [...]", processed["long"])
assert.Equal(t, "42", processed["number"])
}
8 changes: 8 additions & 0 deletions pkg/protocols/common/utils/vardump/vars.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
package vardump

var (
// EnableVarDump enables var dump for debugging optionally
EnableVarDump bool
// Limit is the maximum characters to be dumped
Limit int = 255
)
2 changes: 1 addition & 1 deletion pkg/protocols/dns/request.go
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
func (request *Request) execute(input *contextargs.Context, domain string, metadata, previous output.InternalEvent, vars map[string]interface{}, callback protocols.OutputEventCallback) error {
var err error
if vardump.EnableVarDump {
gologger.Debug().Msgf("DNS Protocol request variables: \n%s\n", vardump.DumpVariables(vars))
gologger.Debug().Msgf("DNS Protocol request variables: %s\n", vardump.DumpVariables(vars))
}

// Compile each request for the template based on the URL
Expand Down
2 changes: 1 addition & 1 deletion pkg/protocols/headless/engine/page_actions.go
Original file line number Diff line number Diff line change
Expand Up @@ -334,7 +334,7 @@ func (p *Page) NavigateURL(action *Action, out ActionData, allvars map[string]in
allvars = generators.MergeMaps(allvars, defaultReqVars)

if vardump.EnableVarDump {
gologger.Debug().Msgf("Headless Protocol request variables: \n%s\n", vardump.DumpVariables(allvars))
gologger.Debug().Msgf("Headless Protocol request variables: %s\n", vardump.DumpVariables(allvars))
}

// Evaluate the target url with all variables
Expand Down
2 changes: 1 addition & 1 deletion pkg/protocols/headless/request.go
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ func (request *Request) executeRequestWithPayloads(input *contextargs.Context, p
defer instance.Close()

if vardump.EnableVarDump {
gologger.Debug().Msgf("Headless Protocol request variables: \n%s\n", vardump.DumpVariables(payloads))
gologger.Debug().Msgf("Headless Protocol request variables: %s\n", vardump.DumpVariables(payloads))
}

instance.SetInteractsh(request.options.Interactsh)
Expand Down
2 changes: 1 addition & 1 deletion pkg/protocols/http/build_request.go
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context,
finalVars := generators.MergeMaps(allVars, payloads)

if vardump.EnableVarDump {
gologger.Debug().Msgf("HTTP Protocol request variables: \n%s\n", vardump.DumpVariables(finalVars))
gologger.Debug().Msgf("HTTP Protocol request variables: %s\n", vardump.DumpVariables(finalVars))
}

// Note: If possible any changes to current logic (i.e evaluate -> then parse URL)
Expand Down
2 changes: 1 addition & 1 deletion pkg/protocols/javascript/js.go
Original file line number Diff line number Diff line change
Expand Up @@ -319,7 +319,7 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, dynamicV
templateCtx.Merge(payloadValues)

if vardump.EnableVarDump {
gologger.Debug().Msgf("Javascript Protocol request variables: \n%s\n", vardump.DumpVariables(payloadValues))
gologger.Debug().Msgf("JavaScript Protocol request variables: %s\n", vardump.DumpVariables(payloadValues))
}

if request.PreCondition != "" {
Expand Down
2 changes: 1 addition & 1 deletion pkg/protocols/network/request.go
Original file line number Diff line number Diff line change
Expand Up @@ -283,7 +283,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
interimValues := generators.MergeMaps(variables, payloads)

if vardump.EnableVarDump {
gologger.Debug().Msgf("Network Protocol request variables: \n%s\n", vardump.DumpVariables(interimValues))
gologger.Debug().Msgf("Network Protocol request variables: %s\n", vardump.DumpVariables(interimValues))
}

inputEvents := make(map[string]interface{})
Expand Down
2 changes: 1 addition & 1 deletion pkg/protocols/ssl/ssl.go
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
payloadValues = generators.MergeMaps(variablesMap, payloadValues, request.options.Constants)

if vardump.EnableVarDump {
gologger.Debug().Msgf("SSL Protocol request variables: \n%s\n", vardump.DumpVariables(payloadValues))
gologger.Debug().Msgf("SSL Protocol request variables: %s\n", vardump.DumpVariables(payloadValues))
}

finalAddress, dataErr := expressions.EvaluateByte([]byte(request.Address), payloadValues)
Expand Down
2 changes: 1 addition & 1 deletion pkg/protocols/websocket/websocket.go
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ func (request *Request) executeRequestWithPayloads(target *contextargs.Context,
}

if vardump.EnableVarDump {
gologger.Debug().Msgf("Websocket Protocol request variables: \n%s\n", vardump.DumpVariables(payloadValues))
gologger.Debug().Msgf("WebSocket Protocol request variables: %s\n", vardump.DumpVariables(payloadValues))
}

finalAddress, dataErr := expressions.EvaluateByte([]byte(request.Address), payloadValues)
Expand Down
2 changes: 1 addition & 1 deletion pkg/protocols/whois/whois.go
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
variables := generators.MergeMaps(vars, defaultVars, optionVars, dynamicValues, request.options.Constants)

if vardump.EnableVarDump {
gologger.Debug().Msgf("Whois Protocol request variables: \n%s\n", vardump.DumpVariables(variables))
gologger.Debug().Msgf("Whois Protocol request variables: %s\n", vardump.DumpVariables(variables))
}

// and replace placeholders
Expand Down
2 changes: 2 additions & 0 deletions pkg/types/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,8 @@ type Options struct {
VerboseVerbose bool
// ShowVarDump displays variable dump
ShowVarDump bool
// VarDumpLimit limits the number of characters displayed in var dump
VarDumpLimit int
// No-Color disables the colored output.
NoColor bool
// UpdateTemplates updates the templates installed at startup (also used by cloud to update datasources)
Expand Down

0 comments on commit 2c832f5

Please sign in to comment.