Fuzzing layer enhancements + input-types support

.gitignore

@@ -30,6 +30,9 @@ pkg/protocols/headless/engine/.cache
 /tsgen
 /scrapefuncs
 /integration_tests/.cache/
-/integration_tests/.nuclei-config/
 /*.yaml
-/pkg/protocols/headless/engine/.nuclei-config/
+**/*-config
+**/*-cache
+/fuzzplayground
+integration_tests/fuzzplayground
+

Makefile

@@ -42,6 +42,8 @@ jsupdate:
 ts:
 	$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "tsgen" pkg/js/devtools/tsgen/cmd/tsgen/main.go
 	./tsgen -dir pkg/js/libs -out pkg/js/generated/ts
+fuzzplayground:
+	$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "fuzzplayground" cmd/tools/fuzzplayground/main.go
 memogen:
 	$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "memogen" cmd/memogen/memogen.go
 	./memogen -src pkg/js/libs -tpl cmd/memogen/function.tpl

README.md

@@ -113,12 +113,17 @@ Usage:
 
 Flags:
 TARGET:
-   -u, -target string[]             target URLs/hosts to scan
-   -l, -list string                 path to file containing a list of target URLs/hosts to scan (one per line)
-   -eh, -exclude-hosts string[]     hosts to exclude to scan from the input list (ip, cidr, hostname)
-   -resume string                   resume scan using resume.cfg (clustering will be disabled)
-   -sa, -scan-all-ips               scan all the IP's associated with dns record
-   -iv, -ip-version string[]        IP version to scan of hostname (4,6) - (default 4)
+   -u, -target string[]          target URLs/hosts to scan
+   -l, -list string              path to file containing a list of target URLs/hosts to scan (one per line)
+   -eh, -exclude-hosts string[]  hosts to exclude to scan from the input list (ip, cidr, hostname)
+   -resume string                resume scan using resume.cfg (clustering will be disabled)
+   -sa, -scan-all-ips            scan all the IP's associated with dns record
+   -iv, -ip-version string[]     IP version to scan of hostname (4,6) - (default 4)
+
+TARGET-FORMAT:
+   -im, -input-mode string        mode of input file (list, burp, jsonl, yaml, openapi, swagger) (default "list")
+   -ro, -required-only            use only required fields in input format when generating requests
+   -sfv, -skip-format-validation  skip format validation (like missing vars) when parsing input file
 
 TEMPLATES:
    -nt, -new-templates                    run only new templates added in latest nuclei-templates release
@@ -134,6 +139,7 @@ TEMPLATES:
    -tl                                    list all available templates
    -sign                                  signs the templates with the private key defined in NUCLEI_SIGNATURE_PRIVATE_KEY env variable
    -code                                  enable loading code protocol-based templates
+   -dut, -disable-unsigned-templates      disable running unsigned templates or templates with mismatched signature
 
 FILTERING:
    -a, -author string[]               templates to run based on authors (comma-separated, file)
@@ -142,8 +148,8 @@ FILTERING:
    -itags, -include-tags string[]     tags to be executed even if they are excluded either by default or configuration
    -id, -template-id string[]         templates to run based on template ids (comma-separated, file, allow-wildcard)
    -eid, -exclude-id string[]         templates to exclude based on template ids (comma-separated, file)
-   -it, -include-templates string[]   templates to be executed even if they are excluded either by default or configuration
-   -et, -exclude-templates string[]   template or template directory to exclude (comma-separated, file)
+   -it, -include-templates string[]   path to template file or directory to be executed even if they are excluded either by default or configuration
+   -et, -exclude-templates string[]   path to template file or directory to exclude (comma-separated, file)
    -em, -exclude-matchers string[]    template matchers to exclude in result
    -s, -severity value[]              templates to run based on severity. Possible values: info, low, medium, high, critical, unknown
    -es, -exclude-severity value[]     templates to exclude based on severity. Possible values: info, low, medium, high, critical, unknown
@@ -215,6 +221,7 @@ INTERACTSH:
 FUZZING:
    -ft, -fuzzing-type string  overrides fuzzing type set in template (replace, prefix, postfix, infix)
    -fm, -fuzzing-mode string  overrides fuzzing mode set in template (multiple, single)
+   -fuzz                      enable loading fuzzing templates
 
 UNCOVER:
    -uc, -uncover                  enable uncover engine
@@ -231,6 +238,8 @@ RATE-LIMIT:
    -c, -concurrency int               maximum number of templates to be executed in parallel (default 25)
    -hbs, -headless-bulk-size int      maximum number of headless hosts to be analyzed in parallel per template (default 10)
    -headc, -headless-concurrency int  maximum number of headless templates to be executed in parallel (default 10)
+   -jsc, -js-concurrency int          maximum number of javascript runtimes to be executed in parallel (default 120)
+   -pc, -payload-concurrency int      max payload concurrency for each template (default 25)
 
 OPTIMIZATIONS:
    -timeout int                     time to wait in seconds before timeout (default 10)
@@ -292,22 +301,26 @@ CLOUD:
    -cup, -cloud-upload    upload scan results to pdcp dashboard
    -sid, -scan-id string  upload scan results to given scan id
 
+AUTHENTICATION:
+   -sf, -secret-file string[]  path to config file containing secrets for nuclei authenticated scan
+   -ps, -prefetch-secrets      prefetch secrets from the secrets file
+
 
 EXAMPLES:
 Run nuclei on single host:
-	$ nuclei -target example.com
+   $ nuclei -target example.com
 
 Run nuclei with specific template directories:
-	$ nuclei -target example.com -t http/cves/ -t ssl
+   $ nuclei -target example.com -t http/cves/ -t ssl
 
 Run nuclei against a list of hosts:
-	$ nuclei -list hosts.txt
+   $ nuclei -list hosts.txt
 
 Run nuclei with a JSON output:
-	$ nuclei -target example.com -json-export output.json
+   $ nuclei -target example.com -json-export output.json
 
 Run nuclei with sorted Markdown outputs (with environment variables):
-	$ MARKDOWN_EXPORT_SORT_MODE=template nuclei -target example.com -markdown-export nuclei_report/
+   $ MARKDOWN_EXPORT_SORT_MODE=template nuclei -target example.com -markdown-export nuclei_report/
 
 Additional documentation is available at: https://docs.nuclei.sh/getting-started/running
 ```

cmd/integration-test/fuzz.go

@@ -12,13 +12,40 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
 )
 
+const (
+	targetFile = "fuzz/testData/ginandjuice.proxify.yaml"
+)
+
 var fuzzingTestCases = []TestCaseInfo{
 	{Path: "fuzz/fuzz-mode.yaml", TestCase: &fuzzModeOverride{}},
 	{Path: "fuzz/fuzz-type.yaml", TestCase: &fuzzTypeOverride{}},
 	{Path: "fuzz/fuzz-query.yaml", TestCase: &httpFuzzQuery{}},
 	{Path: "fuzz/fuzz-headless.yaml", TestCase: &HeadlessFuzzingQuery{}},
 	{Path: "fuzz/fuzz-header-basic.yaml", TestCase: &FuzzHeaderBasic{}},
 	{Path: "fuzz/fuzz-header-multiple.yaml", TestCase: &FuzzHeaderMultiple{}},
+	// for fuzzing we should prioritize adding test case related backend
+	// logic in fuzz playground server instead of adding them here
+	{Path: "fuzz/fuzz-query-num-replace.yaml", TestCase: &genericFuzzTestCase{expectedResults: 2}},
+	{Path: "fuzz/fuzz-host-header-injection.yaml", TestCase: &genericFuzzTestCase{expectedResults: 1}},
+	{Path: "fuzz/fuzz-path-sqli.yaml", TestCase: &genericFuzzTestCase{expectedResults: 1}},
+	{Path: "fuzz/fuzz-cookie-error-sqli.yaml", TestCase: &genericFuzzTestCase{expectedResults: 1}},
+	{Path: "fuzz/fuzz-body-json-sqli.yaml", TestCase: &genericFuzzTestCase{expectedResults: 1}},
+	{Path: "fuzz/fuzz-body-multipart-form-sqli.yaml", TestCase: &genericFuzzTestCase{expectedResults: 1}},
+	{Path: "fuzz/fuzz-body-params-sqli.yaml", TestCase: &genericFuzzTestCase{expectedResults: 1}},
+	{Path: "fuzz/fuzz-body-xml-sqli.yaml", TestCase: &genericFuzzTestCase{expectedResults: 1}},
+	{Path: "fuzz/fuzz-body-generic-sqli.yaml", TestCase: &genericFuzzTestCase{expectedResults: 4}},
+}
+
+type genericFuzzTestCase struct {
+	expectedResults int
+}
+
+func (g *genericFuzzTestCase) Execute(filePath string) error {
+	results, err := testutils.RunNucleiWithArgsAndGetResults(debug, "-t", filePath, "-l", targetFile, "-im", "yaml")
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(results, g.expectedResults)
 }
 
 type httpFuzzQuery struct{}
@@ -34,7 +61,7 @@ func (h *httpFuzzQuery) Execute(filePath string) error {
 	ts := httptest.NewTLSServer(router)
 	defer ts.Close()
 
-	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL+"/?id=example", debug)
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL+"/?id=example", debug, "-fuzz")
 	if err != nil {
 		return err
 	}
@@ -53,7 +80,7 @@ func (h *fuzzModeOverride) Execute(filePath string) error {
 	})
 	ts := httptest.NewTLSServer(router)
 	defer ts.Close()
-	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL+"/?id=example&name=nuclei", debug, "-fuzzing-mode", "single", "-jsonl")
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL+"/?id=example&name=nuclei", debug, "-fuzzing-mode", "single", "-jsonl", "-fuzz")
 	if err != nil {
 		return err
 	}
@@ -98,7 +125,7 @@ func (h *fuzzTypeOverride) Execute(filePath string) error {
 	})
 	ts := httptest.NewTLSServer(router)
 	defer ts.Close()
-	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL+"?id=example", debug, "-fuzzing-type", "replace", "-jsonl")
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL+"?id=example", debug, "-fuzzing-type", "replace", "-jsonl", "-fuzz")
 	if err != nil {
 		return err
 	}
@@ -143,7 +170,7 @@ func (h *HeadlessFuzzingQuery) Execute(filePath string) error {
 	ts := httptest.NewTLSServer(router)
 	defer ts.Close()
 
-	got, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL+"?url=https://scanme.sh", debug, "-headless")
+	got, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL+"?url=https://scanme.sh", debug, "-headless", "-fuzz")
 	if err != nil {
 		return err
 	}
@@ -164,7 +191,7 @@ func (h *FuzzHeaderBasic) Execute(filePath string) error {
 	ts := httptest.NewTLSServer(router)
 	defer ts.Close()
 
-	got, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
+	got, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-fuzz")
 	if err != nil {
 		return err
 	}
@@ -192,7 +219,7 @@ func (h *FuzzHeaderMultiple) Execute(filePath string) error {
 	ts := httptest.NewTLSServer(router)
 	defer ts.Close()
 
-	got, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
+	got, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-fuzz")
 	if err != nil {
 		return err
 	}

cmd/integration-test/integration-test.go

@@ -9,7 +9,9 @@ import (
 
 	"github.com/logrusorgru/aurora"
 
+	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
+	"github.com/projectdiscovery/nuclei/v3/pkg/testutils/fuzzplayground"
 	sliceutil "github.com/projectdiscovery/utils/slice"
 )
 
@@ -53,6 +55,10 @@ var (
 		"flow":            flowTestcases,
 		"javascript":      jsTestcases,
 	}
+	// flakyTests are run with a retry count of 3
+	flakyTests = map[string]bool{
+		"protocols/http/self-contained-file-input.yaml": true,
+	}
 
 	// For debug purposes
 	runProtocol          = ""
@@ -78,6 +84,18 @@ func main() {
 		os.Exit(1)
 	}
 
+	// start fuzz playground server
+	defer fuzzplayground.Cleanup()
+	server := fuzzplayground.GetPlaygroundServer()
+	defer server.Close()
+	go func() {
+		if err := server.Start("localhost:8082"); err != nil {
+			if !strings.Contains(err.Error(), "Server closed") {
+				gologger.Fatal().Msgf("Could not start server: %s\n", err)
+			}
+		}
+	}()
+
 	customTestsList := normalizeSplit(customTests)
 
 	failedTestTemplatePaths := runTests(customTestsList)
@@ -150,6 +168,8 @@ func runTests(customTemplatePaths []string) []string {
 				var err error
 				if proto == "interactsh" || strings.Contains(testCaseInfo.Path, "interactsh") {
 					failedTemplatePath, err = executeWithRetry(testCaseInfo.TestCase, testCaseInfo.Path, interactshRetryCount)
+				} else if flakyTests[testCaseInfo.Path] {
+					failedTemplatePath, err = executeWithRetry(testCaseInfo.TestCase, testCaseInfo.Path, interactshRetryCount)
 				} else {
 					failedTemplatePath, err = execute(testCaseInfo.TestCase, testCaseInfo.Path)
 				}

cmd/integration-test/library.go

@@ -19,11 +19,10 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
 	"github.com/projectdiscovery/nuclei/v3/pkg/core"
-	"github.com/projectdiscovery/nuclei/v3/pkg/core/inputs"
+	"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/parsers"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit"
@@ -126,8 +125,7 @@ func executeNucleiAsLibrary(templatePath, templateURL string) ([]string, error)
 	}
 	store.Load()
 
-	input := &inputs.SimpleInputProvider{Inputs: []*contextargs.MetaInput{{Input: templateURL}}}
-	_ = engine.Execute(store.Templates(), input)
+	_ = engine.Execute(store.Templates(), provider.NewSimpleInputProviderWithUrls(templateURL))
 	engine.WorkPool().Wait() // Wait for the scan to finish
 
 	return results, nil

cmd/integration-test/workflow.go

@@ -137,7 +137,7 @@ type workflowSharedCookies struct{}
 
 // Execute executes a test case and returns an error if occurred
 func (h *workflowSharedCookies) Execute(filePath string) error {
-	handleFunc := func(name string, w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+	handleFunc := func(name string, w http.ResponseWriter, _ *http.Request, _ httprouter.Params) {
 		cookie := &http.Cookie{Name: name, Value: name}
 		http.SetCookie(w, cookie)
 	}

cmd/nuclei/main.go

@@ -22,6 +22,7 @@ import (
 	"github.com/projectdiscovery/interactsh/pkg/client"
 	"github.com/projectdiscovery/nuclei/v3/internal/runner"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
+	"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
 	"github.com/projectdiscovery/nuclei/v3/pkg/installer"
 	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl"
@@ -46,6 +47,9 @@ var (
 )
 
 func main() {
+	// enables CLI specific configs mostly interactive behavior
+	config.CurrentAppMode = config.AppModeCLI
+
 	if err := runner.ConfigureOptions(); err != nil {
 		gologger.Fatal().Msgf("Could not initialize options: %s\n", err)
 	}
@@ -201,6 +205,12 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.StringSliceVarP(&options.IPVersion, "ip-version", "iv", nil, "IP version to scan of hostname (4,6) - (default 4)", goflags.CommaSeparatedStringSliceOptions),
 	)
 
+	flagSet.CreateGroup("target-format", "Target-Format",
+		flagSet.StringVarP(&options.InputFileMode, "input-mode", "im", "list", fmt.Sprintf("mode of input file (%v)", provider.SupportedInputFormats())),
+		flagSet.BoolVarP(&options.FormatUseRequiredOnly, "required-only", "ro", false, "use only required fields in input format when generating requests"),
+		flagSet.BoolVarP(&options.SkipFormatValidation, "skip-format-validation", "sfv", false, "skip format validation (like missing vars) when parsing input file"),
+	)
+
 	flagSet.CreateGroup("templates", "Templates",
 		flagSet.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "run only new templates added in latest nuclei-templates release"),
 		flagSet.StringSliceVarP(&options.NewTemplatesWithVersion, "new-templates-version", "ntv", nil, "run new templates added in specific version", goflags.CommaSeparatedStringSliceOptions),
@@ -226,8 +236,8 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.StringSliceVarP(&options.IncludeTags, "include-tags", "itags", nil, "tags to be executed even if they are excluded either by default or configuration", goflags.FileNormalizedStringSliceOptions), // TODO show default deny list
 		flagSet.StringSliceVarP(&options.IncludeIds, "template-id", "id", nil, "templates to run based on template ids (comma-separated, file, allow-wildcard)", goflags.FileNormalizedStringSliceOptions),
 		flagSet.StringSliceVarP(&options.ExcludeIds, "exclude-id", "eid", nil, "templates to exclude based on template ids (comma-separated, file)", goflags.FileNormalizedStringSliceOptions),
-		flagSet.StringSliceVarP(&options.IncludeTemplates, "include-templates", "it", nil, "templates to be executed even if they are excluded either by default or configuration", goflags.FileCommaSeparatedStringSliceOptions),
-		flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude-templates", "et", nil, "template or template directory to exclude (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
+		flagSet.StringSliceVarP(&options.IncludeTemplates, "include-templates", "it", nil, "path to template file or directory to be executed even if they are excluded either by default or configuration", goflags.FileCommaSeparatedStringSliceOptions),
+		flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude-templates", "et", nil, "path to template file or directory to exclude (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
 		flagSet.StringSliceVarP(&options.ExcludeMatchers, "exclude-matchers", "em", nil, "template matchers to exclude in result", goflags.FileCommaSeparatedStringSliceOptions),
 		flagSet.VarP(&options.Severities, "severity", "s", fmt.Sprintf("templates to run based on severity. Possible values: %s", severity.GetSupportedSeverities().String())),
 		flagSet.VarP(&options.ExcludeSeverities, "exclude-severity", "es", fmt.Sprintf("templates to exclude based on severity. Possible values: %s", severity.GetSupportedSeverities().String())),
@@ -303,6 +313,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 	flagSet.CreateGroup("fuzzing", "Fuzzing",
 		flagSet.StringVarP(&options.FuzzingType, "fuzzing-type", "ft", "", "overrides fuzzing type set in template (replace, prefix, postfix, infix)"),
 		flagSet.StringVarP(&options.FuzzingMode, "fuzzing-mode", "fm", "", "overrides fuzzing mode set in template (multiple, single)"),
+		flagSet.BoolVar(&options.FuzzTemplates, "fuzz", false, "enable loading fuzzing templates"),
 	)
 
 	flagSet.CreateGroup("uncover", "Uncover",
@@ -394,6 +405,11 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.StringVarP(&options.ScanID, "scan-id", "sid", "", "upload scan results to given scan id"),
 	)
 
+	flagSet.CreateGroup("Authentication", "Authentication",
+		flagSet.StringSliceVarP(&options.SecretsFile, "secret-file", "sf", nil, "path to config file containing secrets for nuclei authenticated scan", goflags.CommaSeparatedStringSliceOptions),
+		flagSet.BoolVarP(&options.PreFetchSecrets, "prefetch-secrets", "ps", false, "prefetch secrets from the secrets file"),
+	)
+
 	flagSet.SetCustomHelpText(`EXAMPLES:
 Run nuclei on single host:
 	$ nuclei -target example.com
@@ -468,6 +484,14 @@ Additional documentation is available at: https://docs.nuclei.sh/getting-started
 		config.DefaultConfig.SetTemplatesDir(options.NewTemplatesDirectory)
 	}
 
+	if len(options.SecretsFile) > 0 {
+		for _, secretFile := range options.SecretsFile {
+			if !fileutil.FileExists(secretFile) {
+				gologger.Fatal().Msgf("given secrets file '%s' does not exist", options.SecretsFile)
+			}
+		}
+	}
+
 	cleanupOldResumeFiles()
 	return flagSet
 }