From 9dbaef1f55bb09187ec4ec6e8f05cb5569f46f5a Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Wed, 20 Mar 2024 21:29:05 +0530 Subject: [PATCH 1/5] fuzz: check and handle typed slice --- pkg/fuzz/component/value.go | 34 ++++++++++++++++++++++++++++++-- pkg/fuzz/component/value_test.go | 14 +++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/pkg/fuzz/component/value.go b/pkg/fuzz/component/value.go index c6d7a9603a..763528aed9 100644 --- a/pkg/fuzz/component/value.go +++ b/pkg/fuzz/component/value.go @@ -1,9 +1,11 @@ package component import ( + "reflect" "strconv" "github.com/leslie-qiwa/flat" + "github.com/logrusorgru/aurora" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/dataformat" ) @@ -67,7 +69,11 @@ func (v *Value) SetParsedValue(key string, value string) bool { // otherwise replace it switch v := origValue.(type) { case []interface{}: - origValue = append(v, value) + // update last value + if len(v) > 0 { + v[len(v)-1] = value + } + origValue = v case string: origValue = value case int, int32, int64, float32, float64: @@ -85,7 +91,16 @@ func (v *Value) SetParsedValue(key string, value string) bool { case nil: origValue = value default: - gologger.Error().Msgf("unknown type %T for value %s", v, v) + // explicitly check for typed slice + if val, ok := IsTypedSlice(v); ok { + if len(val) > 0 { + val[len(val)-1] = value + } + origValue = val + } else { + // make it default warning instead of error + gologger.DefaultLogger.Print().Msgf("[%v] unknown type %T for value %s", aurora.BrightYellow("WARN"), v, v) + } } v.parsed[key] = origValue return true @@ -118,3 +133,18 @@ func (v *Value) Encode() (string, error) { } return toEncodeStr, nil } + +// In go, []int, []string are not implictily converted to []interface{} +// when using type assertion and they need to be handled separately. +func IsTypedSlice(v interface{}) ([]interface{}, bool) { + if reflect.ValueOf(v).Kind() == reflect.Slice { + // iterate and convert to []interface{} + slice := reflect.ValueOf(v) + interfaceSlice := make([]interface{}, slice.Len()) + for i := 0; i < slice.Len(); i++ { + interfaceSlice[i] = slice.Index(i).Interface() + } + return interfaceSlice, true + } + return nil, false +} diff --git a/pkg/fuzz/component/value_test.go b/pkg/fuzz/component/value_test.go index bafd02775c..e732643863 100644 --- a/pkg/fuzz/component/value_test.go +++ b/pkg/fuzz/component/value_test.go @@ -37,3 +37,17 @@ func TestFlatMap_FlattenUnflatten(t *testing.T) { } require.Equal(t, data, nested, "unexpected data") } + +func TestAnySlice(t *testing.T) { + data := []any{} + data = append(data, []int{1, 2, 3}) + data = append(data, []string{"foo", "bar"}) + data = append(data, []bool{true, false}) + data = append(data, []float64{1.1, 2.2, 3.3}) + + for _, d := range data { + val, ok := IsTypedSlice(d) + require.True(t, ok, "expected slice") + require.True(t, val != nil, "expected value but got nil") + } +} From c915276d7e6d97f74378ce908e6fcc08be174c06 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Thu, 21 Mar 2024 00:25:35 +0530 Subject: [PATCH 2/5] do not query encode params + fuzz/allow duplicates params --- pkg/fuzz/dataformat/form.go | 92 +++++++++++++++++++++++++++++++++---- 1 file changed, 83 insertions(+), 9 deletions(-) diff --git a/pkg/fuzz/dataformat/form.go b/pkg/fuzz/dataformat/form.go index 088a62fe11..6ff200cda9 100644 --- a/pkg/fuzz/dataformat/form.go +++ b/pkg/fuzz/dataformat/form.go @@ -1,9 +1,34 @@ package dataformat import ( + "fmt" "net/url" + "regexp" + "strconv" + "strings" + + "github.com/projectdiscovery/gologger" + urlutil "github.com/projectdiscovery/utils/url" +) + +const ( + normalizedRegex = `_(\d+)$` +) + +var ( + reNormalized = regexp.MustCompile(normalizedRegex) ) +// == Handling Duplicate Query Parameters / Form Data == +// Nuclei supports fuzzing duplicate query parameters by internally normalizing +// them and denormalizing them back when creating request this normalization +// can be leveraged to specify custom fuzzing behaviour in template as well +// if a query like `?foo=bar&foo=baz&foo=fuzzz` is provided, it will be normalized to +// foo_1=bar , foo_2=baz , foo=fuzzz (i.e last value is given original key which is usual behaviour in HTTP and its implementations) +// this way this change does not break any existing rules in template given by keys-regex or keys +// At same time if user wants to specify 2nd or 1st duplicate value in template, they can use foo_1 or foo_2 in keys-regex or keys +// Note: By default all duplicate query parameters are fuzzed + type Form struct{} var ( @@ -22,18 +47,61 @@ func (f *Form) IsType(data string) bool { // Encode encodes the data into Form format func (f *Form) Encode(data map[string]interface{}) (string, error) { - query := url.Values{} + params := urlutil.NewOrderedParams() for key, value := range data { - switch v := value.(type) { - case []interface{}: - for _, val := range v { - query.Add(key, val.(string)) + params.Set(key, fmt.Sprint(value)) + } + + normalized := map[string]map[string]string{} + for k := range data { + params.Iterate(func(key string, value []string) bool { + if strings.HasPrefix(key, k) && reNormalized.MatchString(key) { + m := map[string]string{} + if normalized[k] != nil { + m = normalized[k] + } + if len(value) == 1 { + m[key] = value[0] + } else { + m[key] = "" + } + normalized[k] = m + params.Del(key) + } + return true + }) + } + + if len(normalized) > 0 { + for k, v := range normalized { + maxIndex := -1 + for key := range v { + matches := reNormalized.FindStringSubmatch(key) + if len(matches) == 2 { + dataIdx, err := strconv.Atoi(matches[1]) + if err != nil { + gologger.Verbose().Msgf("error converting normalized index(%v) to integer: %v", matches[1], err) + continue + } + if dataIdx > maxIndex { + maxIndex = dataIdx + } + } } - case string: - query.Set(key, v) + data := make([]string, maxIndex+1) // Ensure the slice is large enough + for key, value := range v { + matches := reNormalized.FindStringSubmatch(key) + if len(matches) == 2 { + dataIdx, _ := strconv.Atoi(matches[1]) // Error already checked above + data[dataIdx-1] = value // Use dataIdx-1 since slice is 0-indexed + } + } + data[maxIndex] = fmt.Sprint(params.Get(k)) // Use maxIndex which is the last index + params.Add(k, data...) } } - encoded := query.Encode() + + encoded := params.Encode() return encoded, nil } @@ -49,7 +117,13 @@ func (f *Form) Decode(data string) (map[string]interface{}, error) { if len(value) == 1 { values[key] = value[0] } else { - values[key] = value + // in case of multiple query params in form data + // last value is considered and previous values are exposed with _1, _2, _3 etc. + // note that last value will not be included in _1, _2, _3 etc. + for i := 0; i < len(value)-1; i++ { + values[key+"_"+strconv.Itoa(i+1)] = value[i] + } + values[key] = value[len(value)-1] } } return values, nil From d98093bb287a4b65832a85d245f832aa63bb4bb8 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Thu, 21 Mar 2024 02:59:17 +0530 Subject: [PATCH 3/5] sometimes order matters ~query params --- pkg/fuzz/component/body.go | 19 +++-- pkg/fuzz/component/cookie.go | 32 ++++--- pkg/fuzz/component/headers.go | 37 +++++---- pkg/fuzz/component/path.go | 12 +-- pkg/fuzz/component/query.go | 10 ++- pkg/fuzz/component/value.go | 48 ++++++----- pkg/fuzz/dataformat/dataformat.go | 14 +++- pkg/fuzz/dataformat/dataformat_test.go | 18 ++-- pkg/fuzz/dataformat/form.go | 63 ++++++++------ pkg/fuzz/dataformat/json.go | 8 +- pkg/fuzz/dataformat/kv.go | 111 +++++++++++++++++++++++++ pkg/fuzz/dataformat/multipart.go | 36 ++++---- pkg/fuzz/dataformat/raw.go | 10 +-- pkg/fuzz/dataformat/xml.go | 14 ++-- 14 files changed, 300 insertions(+), 132 deletions(-) create mode 100644 pkg/fuzz/dataformat/kv.go diff --git a/pkg/fuzz/component/body.go b/pkg/fuzz/component/body.go index 79d8d6a881..a670fb9bdf 100644 --- a/pkg/fuzz/component/body.go +++ b/pkg/fuzz/component/body.go @@ -55,16 +55,17 @@ func (b *Body) Parse(req *retryablehttp.Request) (bool, error) { } b.value = NewValue(dataStr) - if b.value.Parsed() != nil { + tmp := b.value.Parsed() + if !tmp.IsNIL() { return true, nil } switch { - case strings.Contains(contentType, "application/json") && b.value.Parsed() == nil: + case strings.Contains(contentType, "application/json") && tmp.IsNIL(): return b.parseBody(dataformat.JSONDataFormat, req) - case strings.Contains(contentType, "application/xml") && b.value.Parsed() == nil: + case strings.Contains(contentType, "application/xml") && tmp.IsNIL(): return b.parseBody(dataformat.XMLDataFormat, req) - case strings.Contains(contentType, "multipart/form-data") && b.value.Parsed() == nil: + case strings.Contains(contentType, "multipart/form-data") && tmp.IsNIL(): return b.parseBody(dataformat.MultiPartFormDataFormat, req) } parsed, err := b.parseBody(dataformat.FormDataFormat, req) @@ -94,14 +95,16 @@ func (b *Body) parseBody(decoderName string, req *retryablehttp.Request) (bool, // Iterate iterates through the component func (b *Body) Iterate(callback func(key string, value interface{}) error) error { - for key, value := range b.value.Parsed() { + b.value.parsed.Iterate(func(key string, value any) bool { if strings.HasPrefix(key, "#_") { - continue + return true } if err := callback(key, value); err != nil { - return err + return false } - } + return true + + }) return nil } diff --git a/pkg/fuzz/component/cookie.go b/pkg/fuzz/component/cookie.go index 7269284ccc..05d8bf6bee 100644 --- a/pkg/fuzz/component/cookie.go +++ b/pkg/fuzz/component/cookie.go @@ -2,9 +2,12 @@ package component import ( "context" + "fmt" "net/http" + "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/dataformat" "github.com/projectdiscovery/retryablehttp-go" + mapsutil "github.com/projectdiscovery/utils/maps" ) // Cookie is a component for a request cookie @@ -35,28 +38,30 @@ func (c *Cookie) Parse(req *retryablehttp.Request) (bool, error) { c.req = req c.value = NewValue("") - parsedCookies := make(map[string]interface{}) + parsedCookies := mapsutil.NewOrderedMap[string, any]() for _, cookie := range req.Cookies() { - parsedCookies[cookie.Name] = cookie.Value + parsedCookies.Set(cookie.Name, cookie.Value) } - if len(parsedCookies) == 0 { + if parsedCookies.Len() == 0 { return false, nil } - c.value.SetParsed(parsedCookies, "") + c.value.SetParsed(dataformat.KVOrderedMap(&parsedCookies), "") return true, nil } // Iterate iterates through the component -func (c *Cookie) Iterate(callback func(key string, value interface{}) error) error { - for key, value := range c.value.Parsed() { +func (c *Cookie) Iterate(callback func(key string, value interface{}) error) (err error) { + c.value.parsed.Iterate(func(key string, value any) bool { // Skip ignored cookies if _, ok := defaultIgnoredCookieKeys[key]; ok { - continue + return ok } - if err := callback(key, value); err != nil { - return err + if errx := callback(key, value); errx != nil { + err = errx + return false } - } + return true + }) return nil } @@ -83,13 +88,14 @@ func (c *Cookie) Rebuild() (*retryablehttp.Request, error) { cloned := c.req.Clone(context.Background()) cloned.Header.Del("Cookie") - for key, value := range c.value.Parsed() { + c.value.parsed.Iterate(func(key string, value any) bool { cookie := &http.Cookie{ Name: key, - Value: value.(string), // Assume the value is always a string for cookies + Value: fmt.Sprint(value), // Assume the value is always a string for cookies } cloned.AddCookie(cookie) - } + return true + }) return cloned, nil } diff --git a/pkg/fuzz/component/headers.go b/pkg/fuzz/component/headers.go index 60ac980480..46ada3cebf 100644 --- a/pkg/fuzz/component/headers.go +++ b/pkg/fuzz/component/headers.go @@ -4,6 +4,7 @@ import ( "context" "strings" + "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/dataformat" "github.com/projectdiscovery/retryablehttp-go" ) @@ -40,21 +41,22 @@ func (q *Header) Parse(req *retryablehttp.Request) (bool, error) { } parsedHeaders[key] = value } - q.value.SetParsed(parsedHeaders, "") + q.value.SetParsed(dataformat.KVMap(parsedHeaders), "") return true, nil } // Iterate iterates through the component -func (q *Header) Iterate(callback func(key string, value interface{}) error) error { - for key, value := range q.value.Parsed() { +func (q *Header) Iterate(callback func(key string, value interface{}) error) (err error) { + q.value.parsed.Iterate(func(key string, value any) bool { // Skip ignored headers if _, ok := defaultIgnoredHeaderKeys[key]; ok { - continue + return ok } if err := callback(key, value); err != nil { - return err + return false } - } + return true + }) return nil } @@ -79,22 +81,23 @@ func (q *Header) Delete(key string) error { // component rebuilt func (q *Header) Rebuild() (*retryablehttp.Request, error) { cloned := q.req.Clone(context.Background()) - for key, value := range q.value.parsed { + q.value.parsed.Iterate(func(key string, value any) bool { if strings.EqualFold(key, "Host") { - cloned.Host = value.(string) + return true } - switch v := value.(type) { - case []interface{}: + if vx, ok := IsTypedSlice(value); ok { + // convert to []interface{} + value = vx + } + if v, ok := value.([]interface{}); ok { for _, vv := range v { - if cloned.Header[key] == nil { - cloned.Header[key] = make([]string, 0) - } - cloned.Header[key] = append(cloned.Header[key], vv.(string)) + cloned.Header.Add(key, vv.(string)) } - case string: - cloned.Header[key] = []string{v} + return true } - } + cloned.Header.Set(key, value.(string)) + return true + }) return cloned, nil } diff --git a/pkg/fuzz/component/path.go b/pkg/fuzz/component/path.go index c1d31cee69..bd1d1fee09 100644 --- a/pkg/fuzz/component/path.go +++ b/pkg/fuzz/component/path.go @@ -42,12 +42,14 @@ func (q *Path) Parse(req *retryablehttp.Request) (bool, error) { } // Iterate iterates through the component -func (q *Path) Iterate(callback func(key string, value interface{}) error) error { - for key, value := range q.value.Parsed() { - if err := callback(key, value); err != nil { - return err +func (q *Path) Iterate(callback func(key string, value interface{}) error) (err error) { + q.value.parsed.Iterate(func(key string, value any) bool { + if errx := callback(key, value); errx != nil { + err = errx + return false } - } + return true + }) return nil } diff --git a/pkg/fuzz/component/query.go b/pkg/fuzz/component/query.go index 0952c97986..d68962662b 100644 --- a/pkg/fuzz/component/query.go +++ b/pkg/fuzz/component/query.go @@ -47,12 +47,14 @@ func (q *Query) Parse(req *retryablehttp.Request) (bool, error) { } // Iterate iterates through the component -func (q *Query) Iterate(callback func(key string, value interface{}) error) error { - for key, value := range q.value.Parsed() { +func (q *Query) Iterate(callback func(key string, value interface{}) error) (errx error) { + q.value.parsed.Iterate(func(key string, value interface{}) bool { if err := callback(key, value); err != nil { - return err + errx = err + return false } - } + return true + }) return nil } diff --git a/pkg/fuzz/component/value.go b/pkg/fuzz/component/value.go index 763528aed9..190dcc9def 100644 --- a/pkg/fuzz/component/value.go +++ b/pkg/fuzz/component/value.go @@ -17,7 +17,7 @@ import ( // all the data values that are used in a request. type Value struct { data string - parsed map[string]interface{} + parsed dataformat.KV dataFormat string } @@ -42,27 +42,32 @@ func (v *Value) String() string { } // Parsed returns the parsed value -func (v *Value) Parsed() map[string]interface{} { +func (v *Value) Parsed() dataformat.KV { return v.parsed } // SetParsed sets the parsed value map -func (v *Value) SetParsed(parsed map[string]interface{}, dataFormat string) { +func (v *Value) SetParsed(data dataformat.KV, dataFormat string) { + v.dataFormat = dataFormat + if data.OrderedMap != nil { + v.parsed = data + return + } + parsed := data.Map flattened, err := flat.Flatten(parsed, flatOpts) if err == nil { - v.parsed = flattened + v.parsed = dataformat.KVMap(flattened) } else { - v.parsed = parsed + v.parsed = dataformat.KVMap(parsed) } - v.dataFormat = dataFormat } // SetParsedValue sets the parsed value for a key // in the parsed map func (v *Value) SetParsedValue(key string, value string) bool { - origValue, ok := v.parsed[key] - if !ok { - v.parsed[key] = value + origValue := v.parsed.Get(key) + if origValue == nil { + v.parsed.Set(key, value) return true } // If the value is a list, append to it @@ -88,8 +93,6 @@ func (v *Value) SetParsedValue(key string, value string) bool { return false } origValue = parsed - case nil: - origValue = value default: // explicitly check for typed slice if val, ok := IsTypedSlice(v); ok { @@ -102,30 +105,37 @@ func (v *Value) SetParsedValue(key string, value string) bool { gologger.DefaultLogger.Print().Msgf("[%v] unknown type %T for value %s", aurora.BrightYellow("WARN"), v, v) } } - v.parsed[key] = origValue + v.parsed.Set(key, origValue) return true } // Delete removes a key from the parsed value func (v *Value) Delete(key string) bool { - if _, ok := v.parsed[key]; !ok { - return false - } - delete(v.parsed, key) - return true + return v.parsed.Delete(key) } // Encode encodes the value into a string // using the dataformat and encoding func (v *Value) Encode() (string, error) { toEncodeStr := v.data + if v.parsed.OrderedMap != nil { + // flattening orderedmap not supported + if v.dataFormat != "" { + dataformatStr, err := dataformat.Encode(v.parsed, v.dataFormat) + if err != nil { + return "", err + } + toEncodeStr = dataformatStr + } + return toEncodeStr, nil + } - nested, err := flat.Unflatten(v.parsed, flatOpts) + nested, err := flat.Unflatten(v.parsed.Map, flatOpts) if err != nil { return "", err } if v.dataFormat != "" { - dataformatStr, err := dataformat.Encode(nested, v.dataFormat) + dataformatStr, err := dataformat.Encode(dataformat.KVMap(nested), v.dataFormat) if err != nil { return "", err } diff --git a/pkg/fuzz/dataformat/dataformat.go b/pkg/fuzz/dataformat/dataformat.go index a07abc28f4..9d3cdc3061 100644 --- a/pkg/fuzz/dataformat/dataformat.go +++ b/pkg/fuzz/dataformat/dataformat.go @@ -8,6 +8,12 @@ import ( // dataformats is a list of dataformats var dataformats map[string]DataFormat +const ( + // DefaultKey is the key i.e used when given + // data is not of k-v type + DefaultKey = "value" +) + func init() { dataformats = make(map[string]DataFormat) @@ -49,9 +55,9 @@ type DataFormat interface { // Name returns the name of the encoder Name() string // Encode encodes the data into a format - Encode(data map[string]interface{}) (string, error) + Encode(data KV) (string, error) // Decode decodes the data from a format - Decode(input string) (map[string]interface{}, error) + Decode(input string) (KV, error) } // Decoded is a decoded data format @@ -59,7 +65,7 @@ type Decoded struct { // DataFormat is the data format DataFormat string // Data is the decoded data - Data map[string]interface{} + Data KV } // Decode decodes the data from a format @@ -81,7 +87,7 @@ func Decode(data string) (*Decoded, error) { } // Encode encodes the data into a format -func Encode(data map[string]interface{}, dataformat string) (string, error) { +func Encode(data KV, dataformat string) (string, error) { if dataformat == "" { return "", errors.New("dataformat is required") } diff --git a/pkg/fuzz/dataformat/dataformat_test.go b/pkg/fuzz/dataformat/dataformat_test.go index 7595f9b962..c51a8f6f26 100644 --- a/pkg/fuzz/dataformat/dataformat_test.go +++ b/pkg/fuzz/dataformat/dataformat_test.go @@ -14,7 +14,7 @@ func TestDataformatDecodeEncode_JSON(t *testing.T) { if decoded.DataFormat != "json" { t.Fatal("unexpected data format") } - if decoded.Data["foo"] != "bar" { + if decoded.Data.Get("foo") != "bar" { t.Fatal("unexpected data") } @@ -37,11 +37,19 @@ func TestDataformatDecodeEncode_XML(t *testing.T) { if decoded.DataFormat != "xml" { t.Fatal("unexpected data format") } - if decoded.Data["foo"].(map[string]interface{})["#text"] != "bar" { - t.Fatal("unexpected data") + fooValue := decoded.Data.Get("foo") + if fooValue == nil { + t.Fatal("key 'foo' not found") } - if decoded.Data["foo"].(map[string]interface{})["-attr"] != "baz" { - t.Fatal("unexpected data") + fooMap, ok := fooValue.(map[string]interface{}) + if !ok { + t.Fatal("type assertion to map[string]interface{} failed") + } + if fooMap["#text"] != "bar" { + t.Fatal("unexpected data for '#text'") + } + if fooMap["-attr"] != "baz" { + t.Fatal("unexpected data for '-attr'") } encoded, err := Encode(decoded.Data, decoded.DataFormat) diff --git a/pkg/fuzz/dataformat/form.go b/pkg/fuzz/dataformat/form.go index 6ff200cda9..9a9c7b61cf 100644 --- a/pkg/fuzz/dataformat/form.go +++ b/pkg/fuzz/dataformat/form.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/projectdiscovery/gologger" + mapsutil "github.com/projectdiscovery/utils/maps" urlutil "github.com/projectdiscovery/utils/url" ) @@ -46,30 +47,36 @@ func (f *Form) IsType(data string) bool { } // Encode encodes the data into Form format -func (f *Form) Encode(data map[string]interface{}) (string, error) { +func (f *Form) Encode(data KV) (string, error) { params := urlutil.NewOrderedParams() - for key, value := range data { - params.Set(key, fmt.Sprint(value)) - } + + data.Iterate(func(key string, value any) bool { + params.Add(key, fmt.Sprint(value)) + return true + }) normalized := map[string]map[string]string{} - for k := range data { - params.Iterate(func(key string, value []string) bool { - if strings.HasPrefix(key, k) && reNormalized.MatchString(key) { - m := map[string]string{} - if normalized[k] != nil { - m = normalized[k] - } - if len(value) == 1 { - m[key] = value[0] - } else { - m[key] = "" + // Normalize the data + for _, origKey := range data.OrderedMap.GetKeys() { + // here origKey is base key without _1, _2 etc. + if origKey != "" && !reNormalized.MatchString(origKey) { + params.Iterate(func(key string, value []string) bool { + if strings.HasPrefix(key, origKey) && reNormalized.MatchString(key) { + m := map[string]string{} + if normalized[origKey] != nil { + m = normalized[origKey] + } + if len(value) == 1 { + m[key] = value[0] + } else { + m[key] = "" + } + normalized[origKey] = m + params.Del(key) } - normalized[k] = m - params.Del(key) - } - return true - }) + return true + }) + } } if len(normalized) > 0 { @@ -97,6 +104,8 @@ func (f *Form) Encode(data map[string]interface{}) (string, error) { } } data[maxIndex] = fmt.Sprint(params.Get(k)) // Use maxIndex which is the last index + // remove existing + params.Del(k) params.Add(k, data...) } } @@ -106,27 +115,27 @@ func (f *Form) Encode(data map[string]interface{}) (string, error) { } // Decode decodes the data from Form format -func (f *Form) Decode(data string) (map[string]interface{}, error) { +func (f *Form) Decode(data string) (KV, error) { parsed, err := url.ParseQuery(data) if err != nil { - return nil, err + return KV{}, err } - values := make(map[string]interface{}) + values := mapsutil.NewOrderedMap[string, any]() for key, value := range parsed { if len(value) == 1 { - values[key] = value[0] + values.Set(key, value[0]) } else { // in case of multiple query params in form data // last value is considered and previous values are exposed with _1, _2, _3 etc. // note that last value will not be included in _1, _2, _3 etc. for i := 0; i < len(value)-1; i++ { - values[key+"_"+strconv.Itoa(i+1)] = value[i] + values.Set(key+"_"+strconv.Itoa(i+1), value[i]) } - values[key] = value[len(value)-1] + values.Set(key, value[len(value)-1]) } } - return values, nil + return KVOrderedMap(&values), nil } // Name returns the name of the encoder diff --git a/pkg/fuzz/dataformat/json.go b/pkg/fuzz/dataformat/json.go index 3979ed5e65..99dd0430ec 100644 --- a/pkg/fuzz/dataformat/json.go +++ b/pkg/fuzz/dataformat/json.go @@ -30,16 +30,16 @@ func (j *JSON) IsType(data string) bool { } // Encode encodes the data into JSON format -func (j *JSON) Encode(data map[string]interface{}) (string, error) { - encoded, err := jsoniter.Marshal(data) +func (j *JSON) Encode(data KV) (string, error) { + encoded, err := jsoniter.Marshal(data.Map) return string(encoded), err } // Decode decodes the data from JSON format -func (j *JSON) Decode(data string) (map[string]interface{}, error) { +func (j *JSON) Decode(data string) (KV, error) { var decoded map[string]interface{} err := jsoniter.Unmarshal([]byte(data), &decoded) - return decoded, err + return KVMap(decoded), err } // Name returns the name of the encoder diff --git a/pkg/fuzz/dataformat/kv.go b/pkg/fuzz/dataformat/kv.go new file mode 100644 index 0000000000..83d227c3e4 --- /dev/null +++ b/pkg/fuzz/dataformat/kv.go @@ -0,0 +1,111 @@ +package dataformat + +import mapsutil "github.com/projectdiscovery/utils/maps" + +// KV is a key-value struct +// that is implemented or used by fuzzing package +// to represent a key-value pair +// sometimes order or key-value pair is important (query params) +// so we use ordered map to represent the data +// if it's not important/significant (ex: json,xml) we use map +// this also allows us to iteratively implement ordered map +type KV struct { + Map map[string]interface{} + OrderedMap *mapsutil.OrderedMap[string, any] +} + +// IsNIL returns true if the KV struct is nil +func (kv *KV) IsNIL() bool { + return kv.Map == nil && kv.OrderedMap == nil +} + +// IsOrderedMap returns true if the KV struct is an ordered map +func (kv *KV) IsOrderedMap() bool { + return kv.OrderedMap != nil +} + +// Set sets a value in the KV struct +func (kv *KV) Set(key string, value any) { + if kv.OrderedMap != nil { + kv.OrderedMap.Set(key, value) + return + } + if kv.Map == nil { + kv.Map = make(map[string]interface{}) + } + kv.Map[key] = value +} + +// Get gets a value from the KV struct +func (kv *KV) Get(key string) interface{} { + if kv.OrderedMap != nil { + value, ok := kv.OrderedMap.Get(key) + if !ok { + return nil + } + return value + } + return kv.Map[key] +} + +// Iterate iterates over the KV struct in insertion order +func (kv *KV) Iterate(f func(key string, value any) bool) { + if kv.OrderedMap != nil { + kv.OrderedMap.Iterate(func(key string, value any) bool { + return f(key, value) + }) + return + } + for key, value := range kv.Map { + if !f(key, value) { + break + } + } +} + +// Delete deletes a key from the KV struct +func (kv *KV) Delete(key string) bool { + if kv.OrderedMap != nil { + _, ok := kv.OrderedMap.Get(key) + if !ok { + return false + } + kv.OrderedMap.Delete(key) + return true + } + _, ok := kv.Map[key] + if !ok { + return false + } + delete(kv.Map, key) + return true +} + +// KVMap returns a new KV struct with the given map +func KVMap(data map[string]interface{}) KV { + return KV{Map: data} +} + +// KVOrderedMap returns a new KV struct with the given ordered map +func KVOrderedMap(data *mapsutil.OrderedMap[string, any]) KV { + return KV{OrderedMap: data} +} + +// ToMap converts the ordered map to a map +func ToMap(m *mapsutil.OrderedMap[string, any]) map[string]interface{} { + data := make(map[string]interface{}) + m.Iterate(func(key string, value any) bool { + data[key] = value + return true + }) + return data +} + +// ToOrderedMap converts the map to an ordered map +func ToOrderedMap(data map[string]interface{}) *mapsutil.OrderedMap[string, any] { + m := mapsutil.NewOrderedMap[string, any]() + for key, value := range data { + m.Set(key, value) + } + return &m +} diff --git a/pkg/fuzz/dataformat/multipart.go b/pkg/fuzz/dataformat/multipart.go index 658f77b321..d7e40af10c 100644 --- a/pkg/fuzz/dataformat/multipart.go +++ b/pkg/fuzz/dataformat/multipart.go @@ -6,6 +6,8 @@ import ( "io" "mime" "mime/multipart" + + mapsutil "github.com/projectdiscovery/utils/maps" ) type MultiPartForm struct { @@ -28,23 +30,30 @@ func (m *MultiPartForm) IsType(data string) bool { } // Encode encodes the data into MultiPartForm format -func (m *MultiPartForm) Encode(data map[string]interface{}) (string, error) { +func (m *MultiPartForm) Encode(data KV) (string, error) { var b bytes.Buffer w := multipart.NewWriter(&b) if err := w.SetBoundary(m.boundary); err != nil { return "", err } - for key, value := range data { + var Itererr error + data.Iterate(func(key string, value any) bool { var fw io.Writer var err error // Add field if fw, err = w.CreateFormField(key); err != nil { - return "", err + Itererr = err + return false } if _, err = fw.Write([]byte(value.(string))); err != nil { - return "", err + Itererr = err + return false } + return true + }) + if Itererr != nil { + return "", Itererr } w.Close() @@ -65,7 +74,7 @@ func (m *MultiPartForm) ParseBoundary(contentType string) error { } // Decode decodes the data from MultiPartForm format -func (m *MultiPartForm) Decode(data string) (map[string]interface{}, error) { +func (m *MultiPartForm) Decode(data string) (KV, error) { // Create a buffer from the string data b := bytes.NewBufferString(data) // The boundary parameter should be extracted from the Content-Type header of the HTTP request @@ -75,18 +84,18 @@ func (m *MultiPartForm) Decode(data string) (map[string]interface{}, error) { form, err := r.ReadForm(32 << 20) // 32MB is the max memory used to parse the form if err != nil { - return nil, err + return KV{}, err } defer func() { _ = form.RemoveAll() }() - result := make(map[string]interface{}) + result := mapsutil.NewOrderedMap[string, any]() for key, values := range form.Value { if len(values) > 1 { - result[key] = values + result.Set(key, values) } else { - result[key] = values[0] + result.Set(key, values[0]) } } for key, files := range form.File { @@ -94,20 +103,19 @@ func (m *MultiPartForm) Decode(data string) (map[string]interface{}, error) { for _, fileHeader := range files { file, err := fileHeader.Open() if err != nil { - return nil, err + return KV{}, err } defer file.Close() buffer := new(bytes.Buffer) if _, err := buffer.ReadFrom(file); err != nil { - return nil, err + return KV{}, err } fileContents = append(fileContents, buffer.String()) } - result[key] = fileContents + result.Set(key, fileContents) } - - return result, nil + return KVOrderedMap(&result), nil } // Name returns the name of the encoder diff --git a/pkg/fuzz/dataformat/raw.go b/pkg/fuzz/dataformat/raw.go index 70431528fb..1c2d5fa77e 100644 --- a/pkg/fuzz/dataformat/raw.go +++ b/pkg/fuzz/dataformat/raw.go @@ -17,15 +17,15 @@ func (r *Raw) IsType(data string) bool { } // Encode encodes the data into Raw format -func (r *Raw) Encode(data map[string]interface{}) (string, error) { - return data["value"].(string), nil +func (r *Raw) Encode(data KV) (string, error) { + return data.Get("value").(string), nil } // Decode decodes the data from Raw format -func (r *Raw) Decode(data string) (map[string]interface{}, error) { - return map[string]interface{}{ +func (r *Raw) Decode(data string) (KV, error) { + return KVMap(map[string]interface{}{ "value": data, - }, nil + }), nil } // Name returns the name of the encoder diff --git a/pkg/fuzz/dataformat/xml.go b/pkg/fuzz/dataformat/xml.go index 0609031ba8..2fb605bbde 100644 --- a/pkg/fuzz/dataformat/xml.go +++ b/pkg/fuzz/dataformat/xml.go @@ -22,13 +22,13 @@ func (x *XML) IsType(data string) bool { } // Encode encodes the data into XML format -func (x *XML) Encode(data map[string]interface{}) (string, error) { +func (x *XML) Encode(data KV) (string, error) { var header string - if value, ok := data["#_xml_header"]; ok && value != nil { + if value := data.Get("#_xml_header"); value != nil { header = value.(string) - delete(data, "#_xml_header") + data.Delete("#_xml_header") } - marshalled, err := mxj.Map(data).Xml() + marshalled, err := mxj.Map(data.Map).Xml() if err != nil { return "", err } @@ -41,7 +41,7 @@ func (x *XML) Encode(data map[string]interface{}) (string, error) { var xmlHeader = regexp.MustCompile(`\<\?(.*)\?\>`) // Decode decodes the data from XML format -func (x *XML) Decode(data string) (map[string]interface{}, error) { +func (x *XML) Decode(data string) (KV, error) { var prefixStr string prefix := xmlHeader.FindAllStringSubmatch(data, -1) if len(prefix) > 0 { @@ -50,10 +50,10 @@ func (x *XML) Decode(data string) (map[string]interface{}, error) { decoded, err := mxj.NewMapXml([]byte(data)) if err != nil { - return nil, err + return KV{}, err } decoded["#_xml_header"] = prefixStr - return decoded, nil + return KVMap(decoded), nil } // Name returns the name of the encoder From f1fc62ebae98f80c1fef909a5bcb78d1bae49cab Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Thu, 21 Mar 2024 03:21:31 +0530 Subject: [PATCH 4/5] component: fix broken iterator --- pkg/fuzz/component/body.go | 6 +++--- pkg/fuzz/component/cookie.go | 2 +- pkg/fuzz/component/headers.go | 5 +++-- pkg/fuzz/component/path.go | 2 +- pkg/fuzz/component/query.go | 2 +- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/pkg/fuzz/component/body.go b/pkg/fuzz/component/body.go index a670fb9bdf..32d59a01cb 100644 --- a/pkg/fuzz/component/body.go +++ b/pkg/fuzz/component/body.go @@ -94,18 +94,18 @@ func (b *Body) parseBody(decoderName string, req *retryablehttp.Request) (bool, } // Iterate iterates through the component -func (b *Body) Iterate(callback func(key string, value interface{}) error) error { +func (b *Body) Iterate(callback func(key string, value interface{}) error) (errx error) { b.value.parsed.Iterate(func(key string, value any) bool { if strings.HasPrefix(key, "#_") { return true } if err := callback(key, value); err != nil { + errx = err return false } return true - }) - return nil + return } // SetValue sets a value in the component diff --git a/pkg/fuzz/component/cookie.go b/pkg/fuzz/component/cookie.go index 05d8bf6bee..b50087e36d 100644 --- a/pkg/fuzz/component/cookie.go +++ b/pkg/fuzz/component/cookie.go @@ -62,7 +62,7 @@ func (c *Cookie) Iterate(callback func(key string, value interface{}) error) (er } return true }) - return nil + return } // SetValue sets a value in the component diff --git a/pkg/fuzz/component/headers.go b/pkg/fuzz/component/headers.go index 46ada3cebf..9f921bee28 100644 --- a/pkg/fuzz/component/headers.go +++ b/pkg/fuzz/component/headers.go @@ -46,18 +46,19 @@ func (q *Header) Parse(req *retryablehttp.Request) (bool, error) { } // Iterate iterates through the component -func (q *Header) Iterate(callback func(key string, value interface{}) error) (err error) { +func (q *Header) Iterate(callback func(key string, value interface{}) error) (errx error) { q.value.parsed.Iterate(func(key string, value any) bool { // Skip ignored headers if _, ok := defaultIgnoredHeaderKeys[key]; ok { return ok } if err := callback(key, value); err != nil { + errx = err return false } return true }) - return nil + return } // SetValue sets a value in the component diff --git a/pkg/fuzz/component/path.go b/pkg/fuzz/component/path.go index bd1d1fee09..9521a089f4 100644 --- a/pkg/fuzz/component/path.go +++ b/pkg/fuzz/component/path.go @@ -50,7 +50,7 @@ func (q *Path) Iterate(callback func(key string, value interface{}) error) (err } return true }) - return nil + return } // SetValue sets a value in the component diff --git a/pkg/fuzz/component/query.go b/pkg/fuzz/component/query.go index d68962662b..3fb2f350d4 100644 --- a/pkg/fuzz/component/query.go +++ b/pkg/fuzz/component/query.go @@ -55,7 +55,7 @@ func (q *Query) Iterate(callback func(key string, value interface{}) error) (err } return true }) - return nil + return } // SetValue sets a value in the component From 30d9f1232b9e1bbe13b2c7d712435cb3b98caf1a Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Thu, 21 Mar 2024 03:36:01 +0530 Subject: [PATCH 5/5] result upload add meta params --- internal/pdcp/writer.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/pdcp/writer.go b/internal/pdcp/writer.go index fa0fb17251..bd0a7d4712 100644 --- a/internal/pdcp/writer.go +++ b/internal/pdcp/writer.go @@ -13,10 +13,12 @@ import ( "time" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/retryablehttp-go" pdcpauth "github.com/projectdiscovery/utils/auth/pdcp" errorutil "github.com/projectdiscovery/utils/errors" + updateutils "github.com/projectdiscovery/utils/update" urlutil "github.com/projectdiscovery/utils/url" ) @@ -217,6 +219,8 @@ func (u *UploadWriter) getRequest(bin []byte) (*retryablehttp.Request, error) { if err != nil { return nil, errorutil.NewWithErr(err).Msgf("could not create cloud upload request") } + // add pdtm meta params + req.URL.RawQuery = updateutils.GetpdtmParams(config.Version) req.Header.Set(pdcpauth.ApiKeyHeaderName, u.creds.APIKey) req.Header.Set("Content-Type", "application/octet-stream") req.Header.Set("Accept", "application/json")