Fuzzing layer enhancements + input-types support

.gitignore

@@ -30,6 +30,9 @@ pkg/protocols/headless/engine/.cache
 /tsgen
 /scrapefuncs
 /integration_tests/.cache/
-/integration_tests/.nuclei-config/
 /*.yaml
-/pkg/protocols/headless/engine/.nuclei-config/
+**/*-config
+**/*-cache
+/fuzzplayground
+integration_tests/fuzzplayground
+

Makefile

@@ -42,6 +42,8 @@ jsupdate:
 ts:
 	$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "tsgen" pkg/js/devtools/tsgen/cmd/tsgen/main.go
 	./tsgen -dir pkg/js/libs -out pkg/js/generated/ts
+fuzzplayground:
+	$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "fuzzplayground" cmd/tools/fuzzplayground/main.go
 memogen:
 	$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "memogen" cmd/memogen/memogen.go
 	./memogen -src pkg/js/libs -tpl cmd/memogen/function.tpl

cmd/integration-test/fuzz.go

@@ -12,13 +12,40 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
 )
 
+const (
+	targetFile = "fuzz/testData/ginandjuice.proxify.yaml"
+)
+
 var fuzzingTestCases = []TestCaseInfo{
 	{Path: "fuzz/fuzz-mode.yaml", TestCase: &fuzzModeOverride{}},
 	{Path: "fuzz/fuzz-type.yaml", TestCase: &fuzzTypeOverride{}},
 	{Path: "fuzz/fuzz-query.yaml", TestCase: &httpFuzzQuery{}},
 	{Path: "fuzz/fuzz-headless.yaml", TestCase: &HeadlessFuzzingQuery{}},
 	{Path: "fuzz/fuzz-header-basic.yaml", TestCase: &FuzzHeaderBasic{}},
 	{Path: "fuzz/fuzz-header-multiple.yaml", TestCase: &FuzzHeaderMultiple{}},
+	// for fuzzing we should prioritize adding test case related backend
+	// logic in fuzz playground server instead of adding them here
+	{Path: "fuzz/fuzz-query-num-replace.yaml", TestCase: &genericFuzzTestCase{expectedResults: 2}},
+	{Path: "fuzz/fuzz-host-header-injection.yaml", TestCase: &genericFuzzTestCase{expectedResults: 1}},
+	{Path: "fuzz/fuzz-path-sqli.yaml", TestCase: &genericFuzzTestCase{expectedResults: 1}},
+	{Path: "fuzz/fuzz-cookie-error-sqli.yaml", TestCase: &genericFuzzTestCase{expectedResults: 1}},
+	{Path: "fuzz/fuzz-body-json-sqli.yaml", TestCase: &genericFuzzTestCase{expectedResults: 1}},
+	{Path: "fuzz/fuzz-body-multipart-form-sqli.yaml", TestCase: &genericFuzzTestCase{expectedResults: 1}},
+	{Path: "fuzz/fuzz-body-params-sqli.yaml", TestCase: &genericFuzzTestCase{expectedResults: 1}},
+	{Path: "fuzz/fuzz-body-xml-sqli.yaml", TestCase: &genericFuzzTestCase{expectedResults: 1}},
+	{Path: "fuzz/fuzz-body-generic-sqli.yaml", TestCase: &genericFuzzTestCase{expectedResults: 4}},
+}
+
+type genericFuzzTestCase struct {
+	expectedResults int
+}
+
+func (g *genericFuzzTestCase) Execute(filePath string) error {
+	results, err := testutils.RunNucleiWithArgsAndGetResults(debug, "-t", filePath, "-l", targetFile, "-im", "yaml")
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(results, g.expectedResults)
 }
 
 type httpFuzzQuery struct{}
@@ -34,7 +61,7 @@ func (h *httpFuzzQuery) Execute(filePath string) error {
 	ts := httptest.NewTLSServer(router)
 	defer ts.Close()
 
-	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL+"/?id=example", debug)
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL+"/?id=example", debug, "-fuzz")
 	if err != nil {
 		return err
 	}
@@ -53,7 +80,7 @@ func (h *fuzzModeOverride) Execute(filePath string) error {
 	})
 	ts := httptest.NewTLSServer(router)
 	defer ts.Close()
-	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL+"/?id=example&name=nuclei", debug, "-fuzzing-mode", "single", "-jsonl")
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL+"/?id=example&name=nuclei", debug, "-fuzzing-mode", "single", "-jsonl", "-fuzz")
 	if err != nil {
 		return err
 	}
@@ -98,7 +125,7 @@ func (h *fuzzTypeOverride) Execute(filePath string) error {
 	})
 	ts := httptest.NewTLSServer(router)
 	defer ts.Close()
-	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL+"?id=example", debug, "-fuzzing-type", "replace", "-jsonl")
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL+"?id=example", debug, "-fuzzing-type", "replace", "-jsonl", "-fuzz")
 	if err != nil {
 		return err
 	}
@@ -143,7 +170,7 @@ func (h *HeadlessFuzzingQuery) Execute(filePath string) error {
 	ts := httptest.NewTLSServer(router)
 	defer ts.Close()
 
-	got, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL+"?url=https://scanme.sh", debug, "-headless")
+	got, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL+"?url=https://scanme.sh", debug, "-headless", "-fuzz")
 	if err != nil {
 		return err
 	}
@@ -164,7 +191,7 @@ func (h *FuzzHeaderBasic) Execute(filePath string) error {
 	ts := httptest.NewTLSServer(router)
 	defer ts.Close()
 
-	got, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
+	got, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-fuzz")
 	if err != nil {
 		return err
 	}
@@ -192,7 +219,7 @@ func (h *FuzzHeaderMultiple) Execute(filePath string) error {
 	ts := httptest.NewTLSServer(router)
 	defer ts.Close()
 
-	got, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
+	got, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-fuzz")
 	if err != nil {
 		return err
 	}

cmd/integration-test/integration-test.go

@@ -9,7 +9,9 @@ import (
 
 	"github.com/logrusorgru/aurora"
 
+	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
+	"github.com/projectdiscovery/nuclei/v3/pkg/testutils/fuzzplayground"
 	sliceutil "github.com/projectdiscovery/utils/slice"
 )
 
@@ -53,6 +55,10 @@ var (
 		"flow":            flowTestcases,
 		"javascript":      jsTestcases,
 	}
+	// flakyTests are run with a retry count of 3
+	flakyTests = map[string]bool{
+		"protocols/http/self-contained-file-input.yaml": true,
+	}
 
 	// For debug purposes
 	runProtocol          = ""
@@ -78,6 +84,18 @@ func main() {
 		os.Exit(1)
 	}
 
+	// start fuzz playground server
+	defer fuzzplayground.Cleanup()
+	server := fuzzplayground.GetPlaygroundServer()
+	defer server.Close()
+	go func() {
+		if err := server.Start("localhost:8082"); err != nil {
+			if !strings.Contains(err.Error(), "Server closed") {
+				gologger.Fatal().Msgf("Could not start server: %s\n", err)
+			}
+		}
+	}()
+
 	customTestsList := normalizeSplit(customTests)
 
 	failedTestTemplatePaths := runTests(customTestsList)
@@ -150,6 +168,8 @@ func runTests(customTemplatePaths []string) []string {
 				var err error
 				if proto == "interactsh" || strings.Contains(testCaseInfo.Path, "interactsh") {
 					failedTemplatePath, err = executeWithRetry(testCaseInfo.TestCase, testCaseInfo.Path, interactshRetryCount)
+				} else if flakyTests[testCaseInfo.Path] {
+					failedTemplatePath, err = executeWithRetry(testCaseInfo.TestCase, testCaseInfo.Path, interactshRetryCount)
 				} else {
 					failedTemplatePath, err = execute(testCaseInfo.TestCase, testCaseInfo.Path)
 				}

cmd/integration-test/library.go

@@ -19,11 +19,10 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
 	"github.com/projectdiscovery/nuclei/v3/pkg/core"
-	"github.com/projectdiscovery/nuclei/v3/pkg/core/inputs"
+	"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/parsers"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit"
@@ -126,8 +125,7 @@ func executeNucleiAsLibrary(templatePath, templateURL string) ([]string, error)
 	}
 	store.Load()
 
-	input := &inputs.SimpleInputProvider{Inputs: []*contextargs.MetaInput{{Input: templateURL}}}
-	_ = engine.Execute(store.Templates(), input)
+	_ = engine.Execute(store.Templates(), provider.NewSimpleInputProviderWithUrls(templateURL))
 	engine.WorkPool().Wait() // Wait for the scan to finish
 
 	return results, nil

cmd/integration-test/workflow.go

@@ -137,7 +137,7 @@ type workflowSharedCookies struct{}
 
 // Execute executes a test case and returns an error if occurred
 func (h *workflowSharedCookies) Execute(filePath string) error {
-	handleFunc := func(name string, w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+	handleFunc := func(name string, w http.ResponseWriter, _ *http.Request, _ httprouter.Params) {
 		cookie := &http.Cookie{Name: name, Value: name}
 		http.SetCookie(w, cookie)
 	}

cmd/nuclei/main.go

@@ -22,6 +22,7 @@ import (
 	"github.com/projectdiscovery/interactsh/pkg/client"
 	"github.com/projectdiscovery/nuclei/v3/internal/runner"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
+	"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
 	"github.com/projectdiscovery/nuclei/v3/pkg/installer"
 	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl"
@@ -46,6 +47,9 @@ var (
 )
 
 func main() {
+	// enables CLI specific configs mostly interactive behavior
+	config.CurrentAppMode = config.AppModeCLI
+
 	if err := runner.ConfigureOptions(); err != nil {
 		gologger.Fatal().Msgf("Could not initialize options: %s\n", err)
 	}
@@ -201,6 +205,12 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.StringSliceVarP(&options.IPVersion, "ip-version", "iv", nil, "IP version to scan of hostname (4,6) - (default 4)", goflags.CommaSeparatedStringSliceOptions),
 	)
 
+	flagSet.CreateGroup("target-format", "Target-Format",
+		flagSet.StringVarP(&options.InputFileMode, "input-mode", "im", "list", fmt.Sprintf("mode of input file (%v)", provider.SupportedInputFormats())),
+		flagSet.BoolVarP(&options.FormatUseRequiredOnly, "required-only", "ro", false, "use only required fields in input format when generating requests"),
+		flagSet.BoolVarP(&options.SkipFormatValidation, "skip-format-validation", "sfv", false, "skip format validation (like missing vars) when parsing input file"),
+	)
+
 	flagSet.CreateGroup("templates", "Templates",
 		flagSet.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "run only new templates added in latest nuclei-templates release"),
 		flagSet.StringSliceVarP(&options.NewTemplatesWithVersion, "new-templates-version", "ntv", nil, "run new templates added in specific version", goflags.CommaSeparatedStringSliceOptions),
@@ -302,6 +312,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 	flagSet.CreateGroup("fuzzing", "Fuzzing",
 		flagSet.StringVarP(&options.FuzzingType, "fuzzing-type", "ft", "", "overrides fuzzing type set in template (replace, prefix, postfix, infix)"),
 		flagSet.StringVarP(&options.FuzzingMode, "fuzzing-mode", "fm", "", "overrides fuzzing mode set in template (multiple, single)"),
+		flagSet.BoolVar(&options.FuzzTemplates, "fuzz", false, "enable and run fuzzing templates"),
 	)
 
 	flagSet.CreateGroup("uncover", "Uncover",
@@ -392,6 +403,11 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.StringVarP(&options.ScanID, "scan-id", "sid", "", "upload scan results to given scan id"),
 	)
 
+	flagSet.CreateGroup("Authentication", "Authentication",
+		flagSet.StringSliceVar(&options.SecretsFile, "secrets", nil, "path to file containing secrets for nuclei authenticated scan", goflags.CommaSeparatedStringSliceOptions),
+		flagSet.BoolVarP(&options.PreFetchSecrets, "prefetch-secrets", "ps", false, "prefetch secrets from the secrets file"),
+	)
+
 	flagSet.SetCustomHelpText(`EXAMPLES:
 Run nuclei on single host:
 	$ nuclei -target example.com
@@ -418,7 +434,7 @@ Additional documentation is available at: https://docs.nuclei.sh/getting-started
 	goflags.DisableAutoConfigMigration = true
 	_ = flagSet.Parse()
 
-    // api key hierarchy: cli flag > env var > .pdcp/credential file
+	// api key hierarchy: cli flag > env var > .pdcp/credential file
 	if pdcpauth == "true" {
 		runner.AuthWithPDCP()
 	} else if len(pdcpauth) == 36 {
@@ -466,6 +482,14 @@ Additional documentation is available at: https://docs.nuclei.sh/getting-started
 		config.DefaultConfig.SetTemplatesDir(options.NewTemplatesDirectory)
 	}
 
+	if len(options.SecretsFile) > 0 {
+		for _, secretFile := range options.SecretsFile {
+			if !fileutil.FileExists(secretFile) {
+				gologger.Fatal().Msgf("given secrets file '%s' does not exist", options.SecretsFile)
+			}
+		}
+	}
+
 	cleanupOldResumeFiles()
 	return flagSet
 }

cmd/tools/fuzzplayground/main.go

@@ -1,94 +1,27 @@
 package main
 
 import (
-	"fmt"
-	"io"
-	"os/exec"
-	"strconv"
-	"strings"
+	"flag"
 
-	"github.com/labstack/echo/v4"
-	"github.com/labstack/echo/v4/middleware"
-	"github.com/projectdiscovery/retryablehttp-go"
+	_ "github.com/mattn/go-sqlite3"
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v3/pkg/testutils/fuzzplayground"
 )
 
-func main() {
-	e := echo.New()
-	e.Use(middleware.Recover())
-	e.Use(middleware.Logger())
-
-	e.GET("/", indexHandler)
-	e.GET("/info", infoHandler)
-	e.GET("/redirect", redirectHandler)
-	e.GET("/request", requestHandler)
-	e.GET("/email", emailHandler)
-	e.GET("/permissions", permissionsHandler)
-	if err := e.Start("localhost:8082"); err != nil {
-		panic(err)
-	}
-}
-
-var bodyTemplate = `<html>
-<head>
-<title>Fuzz Playground</title>
-</head>
-<body>
-%s
-</body>
-</html>`
-
-func indexHandler(ctx echo.Context) error {
-	return ctx.HTML(200, fmt.Sprintf(bodyTemplate, `<h1>Fuzzing Playground</h1><hr>
-<ul>
-<li><a href="/info?name=test&another=value&random=data">Info Page XSS</a></li>
-<li><a href="/redirect?redirect_url=/info?name=redirected_from_url">Redirect Page OpenRedirect</a></li>
-<li><a href="/request?url=https://example.com">Request Page SSRF</a></li>
-<li><a href="/email?text=important_user">Email Page SSTI</a></li>
-<li><a href="/permissions?cmd=whoami">Permissions Page CMDI</a></li>
-</ul>
-`))
-}
-
-func infoHandler(ctx echo.Context) error {
-	return ctx.HTML(200, fmt.Sprintf(bodyTemplate, fmt.Sprintf("Name of user: %s%s%s", ctx.QueryParam("name"), ctx.QueryParam("another"), ctx.QueryParam("random"))))
-}
+var (
+	addr string
+)
 
-func redirectHandler(ctx echo.Context) error {
-	url := ctx.QueryParam("redirect_url")
-	return ctx.Redirect(302, url)
-}
+func main() {
+	flag.StringVar(&addr, "addr", "localhost:8082", "playground server address")
+	flag.Parse()
 
-func requestHandler(ctx echo.Context) error {
-	url := ctx.QueryParam("url")
-	data, err := retryablehttp.DefaultClient().Get(url)
-	if err != nil {
-		return ctx.HTML(500, err.Error())
-	}
-	defer data.Body.Close()
+	defer fuzzplayground.Cleanup()
+	server := fuzzplayground.GetPlaygroundServer()
+	defer server.Close()
 
-	body, _ := io.ReadAll(data.Body)
-	return ctx.HTML(200, fmt.Sprintf(bodyTemplate, string(body)))
-}
-
-func emailHandler(ctx echo.Context) error {
-	text := ctx.QueryParam("text")
-	if strings.Contains(text, "{{") {
-		trimmed := strings.SplitN(strings.Trim(text[strings.Index(text, "{"):], "{}"), "*", 2)
-		if len(trimmed) < 2 {
-			return ctx.HTML(500, "invalid template")
-		}
-		first, _ := strconv.Atoi(trimmed[0])
-		second, _ := strconv.Atoi(trimmed[1])
-		text = strconv.Itoa(first * second)
+	// Start the server
+	if err := server.Start(addr); err != nil {
+		gologger.Fatal().Msgf("Could not start server: %s\n", err)
 	}
-	return ctx.HTML(200, fmt.Sprintf(bodyTemplate, fmt.Sprintf("Text: %s", text)))
-}
-
-func permissionsHandler(ctx echo.Context) error {
-	command := ctx.QueryParam("cmd")
-	fields := strings.Fields(command)
-	cmd := exec.Command(fields[0], fields[1:]...)
-	data, _ := cmd.CombinedOutput()
-
-	return ctx.HTML(200, fmt.Sprintf(bodyTemplate, string(data)))
 }