From 33709e6adbd9a163f1837949569466feed125a4a Mon Sep 17 00:00:00 2001 From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Date: Wed, 9 Aug 2023 16:14:03 +0300 Subject: [PATCH] fix DSL filter and matcher (#1313) --- go.mod | 4 ++-- go.sum | 9 ++++++++ runner/runner.go | 35 ++++------------------------ runner/types.go | 59 +++++++++++++++++++++++++++++++++++++++++++++--- 4 files changed, 71 insertions(+), 36 deletions(-) diff --git a/go.mod b/go.mod index a15dae0..a8d2a87 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 github.com/mitchellh/mapstructure v1.5.0 github.com/projectdiscovery/asnmap v1.0.4 - github.com/projectdiscovery/dsl v0.0.14 + github.com/projectdiscovery/dsl v0.0.16 github.com/projectdiscovery/fastdialer v0.0.35 github.com/projectdiscovery/ratelimit v0.0.9 github.com/projectdiscovery/tlsx v1.1.1 @@ -111,7 +111,7 @@ require ( github.com/refraction-networking/utls v1.3.2 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect - github.com/sashabaranov/go-openai v1.13.0 // indirect + github.com/sashabaranov/go-openai v1.14.1 // indirect github.com/shirou/gopsutil/v3 v3.23.6 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect diff --git a/go.sum b/go.sum index b4d7dd4..7034627 100644 --- a/go.sum +++ b/go.sum @@ -57,6 +57,7 @@ github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdf github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= github.com/gaukas/godicttls v0.0.3 h1:YNDIf0d9adcxOijiLrEzpfZGAkNwLRzPaG6OjU7EITk= @@ -124,6 +125,7 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= @@ -193,6 +195,8 @@ github.com/projectdiscovery/clistats v0.0.19 h1:SA/qRHbmS9VEbVEPzX/ka01hZDYATL9Z github.com/projectdiscovery/clistats v0.0.19/go.mod h1:NQDAW/O7cK9xBIgk46kJjwGRkjSg5JkB8E4DvuxXr+c= github.com/projectdiscovery/dsl v0.0.14 h1:CAxCoYbIEBCuINiMR1UKA1v6ifmub3P5hCwzBmmkh0c= github.com/projectdiscovery/dsl v0.0.14/go.mod h1:3K2GmExpriruVHsVJmsTugxR7H9wVpUo8/+jWXXbSSw= +github.com/projectdiscovery/dsl v0.0.16 h1:ECymBWfB6L6M/y0X6fa+mwg2l0nCSUkfoJkesjGCYJ4= +github.com/projectdiscovery/dsl v0.0.16/go.mod h1:OiVbde6xGMM4NXnf3DUJIEqdwWppPADBSPMrxDHwRCU= github.com/projectdiscovery/fastdialer v0.0.35 h1:dCjYaZ2dOtKmIbQ7OUuf/pZiMQRHfUjjLoHrEF8CJ8g= github.com/projectdiscovery/fastdialer v0.0.35/go.mod h1:dTx0C7JRWKKO5ZxGqM0NUDzB4svmyYqGM6zcHIk2ueo= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= @@ -241,6 +245,8 @@ github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7 github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/sashabaranov/go-openai v1.13.0 h1:EAusFfnhaMaaUspUZ2+MbB/ZcVeD4epJmTOlZ+8AcAE= github.com/sashabaranov/go-openai v1.13.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= +github.com/sashabaranov/go-openai v1.14.1 h1:jqfkdj8XHnBF84oi2aNtT8Ktp3EJ0MfuVjvcMkfI0LA= +github.com/sashabaranov/go-openai v1.14.1/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= github.com/shirou/gopsutil/v3 v3.23.6 h1:5y46WPI9QBKBbK7EEccUPNXpJpNrvPuTD0O2zHEHT08= github.com/shirou/gopsutil/v3 v3.23.6/go.mod h1:j7QX50DrXYggrpN30W0Mo+I4/8U2UUIQrnrhqUeWrAU= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= @@ -336,6 +342,7 @@ github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101 h1:QuLjRpIBjqene8VvB+ github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101/go.mod h1:bRZdjnJaHWVXKEwrfAZMd0gfRjZGNhTbZwzp07s0Abw= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -452,6 +459,7 @@ google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/djherbis/times.v1 v1.3.0 h1:uxMS4iMtH6Pwsxog094W0FYldiNnfY/xba00vq6C2+o= gopkg.in/djherbis/times.v1 v1.3.0/go.mod h1:AQlg6unIsrsCEdQYhTzERy542dz6SFdQFZFv6mUY0P8= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -463,6 +471,7 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/runner/runner.go b/runner/runner.go index 7f2cb4e..a2bced9 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -26,7 +26,6 @@ import ( "github.com/PuerkitoBio/goquery" asnmap "github.com/projectdiscovery/asnmap/libs" - dsl "github.com/projectdiscovery/dsl" "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/httpx/common/customextract" "github.com/projectdiscovery/httpx/common/errorpageclassifier" @@ -742,42 +741,16 @@ func (r *Runner) RunEnumeration() { // apply matchers and filters if r.options.OutputFilterCondition != "" || r.options.OutputMatchCondition != "" { - rawMap, err := ResultToMap(resp) - if err != nil { - gologger.Warning().Msgf("Could not decode response: %s\n", err) - continue - } - dslVars, err := dslVariables() - if err != nil { - gologger.Warning().Msgf("Could not retrieve dsl variables: %s\n", err) - continue - } - flatMap := make(map[string]interface{}) - - for _, v := range dslVars { - flatMap[v] = rawMap[v] - } - if r.options.OutputMatchCondition != "" { - res, err := dsl.EvalExpr(r.options.OutputMatchCondition, flatMap) - if err != nil { - gologger.Error().Msgf("Could not evaluate match condition: %s\n", err) + matched := evalDslExpr(resp, r.options.OutputMatchCondition) + if !matched { continue - } else { - if res == false { - continue - } } } if r.options.OutputFilterCondition != "" { - res, err := dsl.EvalExpr(r.options.OutputFilterCondition, flatMap) - if err != nil { - gologger.Error().Msgf("Could not evaluate filter condition: %s\n", err) + matched := evalDslExpr(resp, r.options.OutputFilterCondition) + if matched { continue - } else { - if res == true { - continue - } } } } diff --git a/runner/types.go b/runner/types.go index f3cb4eb..2b24f37 100644 --- a/runner/types.go +++ b/runner/types.go @@ -1,12 +1,17 @@ package runner import ( + "errors" "fmt" + "os" + "strings" "time" "github.com/go-faker/faker/v4" "github.com/go-faker/faker/v4/pkg/options" "github.com/mitchellh/mapstructure" + "github.com/projectdiscovery/dsl" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/tlsx/pkg/tlsx/clients" mapsutil "github.com/projectdiscovery/utils/maps" @@ -87,7 +92,7 @@ func dslVariables() ([]string, error) { if err := faker.FakeData(&fakeResult, options.WithFieldsToIgnore(fieldsToIgnore...)); err != nil { return nil, err } - m, err := ResultToMap(fakeResult) + m, err := resultToMap(fakeResult) if err != nil { return nil, err } @@ -99,7 +104,22 @@ func dslVariables() ([]string, error) { return vars, nil } -func ResultToMap(resp Result) (map[string]any, error) { +func evalDslExpr(result Result, dslExpr string) bool { + resultMap, err := resultToMap(result) + if err != nil { + gologger.Warning().Msgf("Could not map result: %s\n", err) + return false + } + + res, err := dsl.EvalExpr(dslExpr, resultMap) + if err != nil && !ignoreErr(err) { + gologger.Error().Msgf("Could not evaluate DSL expression: %s\n", err) + return false + } + return res == true +} + +func resultToMap(resp Result) (map[string]any, error) { m := make(map[string]any) config := &mapstructure.DecoderConfig{ TagName: "json", @@ -113,5 +133,38 @@ func ResultToMap(resp Result) (map[string]any, error) { if err != nil { return nil, fmt.Errorf("error decoding: %v", err) } - return m, nil + return flatten(m), nil +} + +// mapsutil.Flatten w/o separator +func flatten(m map[string]any) map[string]any { + o := make(map[string]any) + for k, v := range m { + switch child := v.(type) { + case map[string]any: + nm := flatten(child) + for nk, nv := range nm { + o[nk] = nv + } + default: + o[k] = v + } + } + return o +} + +var ( + // showDSLErr controls whether to show hidden DSL errors or not + showDSLErr = strings.EqualFold(os.Getenv("SHOW_DSL_ERRORS"), "true") +) + +// ignoreErr checks if the error is to be ignored or not +func ignoreErr(err error) bool { + if showDSLErr { + return false + } + if errors.Is(err, dsl.ErrParsingArg) || strings.Contains(err.Error(), "No parameter") { + return true + } + return false }