Merge pull request #5529 from projectdiscovery/dev

v3.3.1

.github/ISSUE_TEMPLATE/issue-report.md

@@ -18,6 +18,11 @@ labels: 'Type: Bug'
 <!-- We only accept issues that are reproducible on the latest version of nuclei. -->
 <!-- You can find the latest version of project at https://github.com/projectdiscovery/nuclei/releases/ -->
 
+## Nuclei command:
+
+<!-- Provide the exact command you used to run Nuclei. -->
+<!-- Please redact any literal target hosts/URLs or other sensitive information. -->
+
 ### Current Behavior:
 <!-- A concise description of what you're experiencing. -->
 

.github/workflows/build-test.yml

@@ -12,28 +12,23 @@ jobs:
     name: Test Builds
     strategy:
       matrix:
-        go-version: [1.21.x]
         os: [ubuntu-latest, windows-latest, macOS-latest]
-
     runs-on: ${{ matrix.os }}
     steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
       - name: Set up Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: ${{ matrix.go-version }}
+        uses: projectdiscovery/actions/setup/go@v1
 
-      - name: Set up Python   # required for running python code in py-snippet.yaml integration test
+      # required for running python code in py-snippet.yaml integration test
+      - name: Set up Python
         uses: actions/setup-python@v4
         with:
           python-version: '3.10'
 
-      - name: Check out code
-        uses: actions/checkout@v3
-
-      - name: Go Mod hygiene
-        run: |
-          go clean -modcache
-          go mod tidy
+      - name: Verify Go modules
+        run: make verify
 
       - name: Build
         run: go build .
@@ -65,9 +60,10 @@ jobs:
         run: go run .
         working-directory: examples/simple/
 
-      - name: Example SDK Advanced
-        run: go run .
-        working-directory: examples/advanced/
+      # Temporarily disabled very flaky in github actions
+      # - name: Example SDK Advanced
+      #   run: go run .
+      #   working-directory: examples/advanced/
 
       - name: Example SDK with speed control
         run: go run .

.github/workflows/functional-test.yml

@@ -15,13 +15,11 @@ jobs:
       matrix:
         os: [ubuntu-latest, windows-latest, macOS-latest]
     steps:
-      - name: Set up Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: 1.21.x
-
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: projectdiscovery/actions/setup/go@v1
 
       - name: Functional Tests
         env:

.github/workflows/lint-test.yml

@@ -12,16 +12,11 @@ jobs:
     name: Lint Test
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: 1.21.x
-
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: projectdiscovery/actions/setup/go@v1
 
       - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v4.0.0
-        with:
-          version: latest
-          args: --timeout 5m
+        uses: projectdiscovery/actions/golangci-lint@v1

.github/workflows/performance-test.yaml

@@ -11,24 +11,19 @@ jobs:
     name: Test Performance
     strategy:
       matrix:
-        go-version: [1.21.x]
         os: [ubuntu-latest, macOS-latest]
 
     runs-on: ${{ matrix.os }}
     if: github.repository == 'projectdiscovery/nuclei'
     steps:
-      - name: Set up Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: ${{ matrix.go-version }}
-
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: projectdiscovery/actions/setup/go@v1
 
-      - name: Go Mod hygine
-        run: |
-          go clean -modcache
-          go mod tidy
+      - name: Verify Go modules
+        run: make verify
 
       # Max GH exection time 6H => timeout after that
       - name: Running performance with big list

.github/workflows/publish-docs.yaml

@@ -11,14 +11,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with: 
           fetch-depth: 0
 
       - name: "Set up Go"
-        uses: actions/setup-go@v4
-        with: 
-          go-version: 1.21.x
+        uses: projectdiscovery/actions/setup/go@v1
+
+      - name: "Set up Git"
+        uses: projectdiscovery/actions/setup/git@v1
 
       - name: Generate YAML Syntax Documentation
         id: generate-docs
@@ -35,8 +36,6 @@ jobs:
       - name: Commit files
         if: steps.generate-docs.outputs.CHANGES > 0
         run: |
-          git config --local user.email "action@github.com"
-          git config --local user.name "GitHub Action"
           git add SYNTAX-REFERENCE.md nuclei-jsonschema.json
           git commit -m "Auto Generate Syntax Docs + JSONSchema [$(date)] :robot:" -a
 

.github/workflows/release-binary.yml

@@ -9,20 +9,17 @@ on:
 jobs: 
   release: 
     runs-on: ubuntu-latest-16-cores
-
     steps: 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with: 
           fetch-depth: 0
 
-      - uses: actions/setup-go@v4
-        with: 
-          go-version: 1.21.x
+      - name: Set up Go
+        uses: projectdiscovery/actions/setup/go@v1
 
-      - uses: goreleaser/goreleaser-action@v4
+      - uses: projectdiscovery/actions/goreleaser@v1
         with: 
-          args: "release --clean"
-          version: latest
+          release: true
         env: 
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
           SLACK_WEBHOOK: "${{ secrets.RELEASE_SLACK_WEBHOOK }}"

.github/workflows/release-test.yml

@@ -12,17 +12,12 @@ jobs:
     runs-on: ubuntu-latest-16-cores
     steps:
       - name: "Check out code"
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with: 
           fetch-depth: 0
 
       - name: Set up Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: 1.21.x
+        uses: projectdiscovery/actions/setup/go@v1
 
-      - name: release test
-        uses: goreleaser/goreleaser-action@v4
-        with:
-          args: "release --clean --snapshot"
-          version: latest
+      - name: Release snapshot
+        uses: projectdiscovery/actions/goreleaser@v1

.github/workflows/template-validate.yml

@@ -10,10 +10,9 @@ jobs:
   build:
     runs-on: ubuntu-latest-16-cores
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v4
-        with: 
-          go-version: 1.21.x
+      - uses: actions/checkout@v4
+
+      - uses: projectdiscovery/actions/setup/go@v1
 
       - name: Template Validation
         run: |

Makefile

@@ -11,7 +11,7 @@ ifneq ($(shell go env GOOS),darwin)
 LDFLAGS := -extldflags "-static"
 endif
 
-.PHONY: all build build-stats scan-charts docs test integration functional tidy devtools jsupdate ts fuzzplayground memogen dsl-docs
+.PHONY: all build build-stats scan-charts docs test integration functional tidy verify devtools jsupdate ts fuzzplayground memogen dsl-docs
 
 all: build
 build:
@@ -39,6 +39,8 @@ functional:
 	cd cmd/functional-test; bash run.sh
 tidy:
 	$(GOMOD) tidy
+verify: tidy
+	$(GOMOD) verify
 devtools:
 	$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "bindgen" pkg/js/devtools/bindgen/cmd/bindgen/main.go
 	$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "tsgen" pkg/js/devtools/tsgen/cmd/tsgen/main.go

README.md

@@ -179,6 +179,7 @@ OUTPUT:
    -se, -sarif-export string     file to export results in SARIF format
    -je, -json-export string      file to export results in JSON format
    -jle, -jsonl-export string    file to export results in JSONL(ine) format
+   -rd, -redact string[]         redact given list of keys from query parameter, request header and body
 
 CONFIGURATIONS:
    -config string                        path to the nuclei configuration file
@@ -310,6 +311,7 @@ STATISTICS:
 
 CLOUD:
    -auth                      configure projectdiscovery cloud (pdcp) api key (default true)
+   -tid, -team-id string      upload scan results to given team id (optional) (default "none")
    -cup, -cloud-upload        upload scan results to pdcp dashboard
    -sid, -scan-id string      upload scan results to existing scan id (optional)
    -sname, -scan-name string  scan name to set (optional)

SYNTAX-REFERENCE.md

@@ -1404,6 +1404,19 @@ Valid values:
 
 <div class="dd">
 
+<code>skip-secret-file</code>  <i>bool</i>
+
+</div>
+<div class="dt">
+
+SkipSecretFile skips the authentication or authorization configured in the secret file.
+
+</div>
+
+<hr />
+
+<div class="dd">
+
 <code>cookie-reuse</code>  <i>bool</i>
 
 </div>

cmd/nuclei/main.go

@@ -12,6 +12,7 @@ import (
 	"strings"
 	"time"
 
+	_pdcp "github.com/projectdiscovery/nuclei/v3/internal/pdcp"
 	"github.com/projectdiscovery/utils/auth/pdcp"
 	"github.com/projectdiscovery/utils/env"
 	_ "github.com/projectdiscovery/utils/pprof"
@@ -272,6 +273,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.StringVarP(&options.SarifExport, "sarif-export", "se", "", "file to export results in SARIF format"),
 		flagSet.StringVarP(&options.JSONExport, "json-export", "je", "", "file to export results in JSON format"),
 		flagSet.StringVarP(&options.JSONLExport, "jsonl-export", "jle", "", "file to export results in JSONL(ine) format"),
+		flagSet.StringSliceVarP(&options.Redact, "redact", "rd", nil, "redact given list of keys from query parameter, request header and body", goflags.CommaSeparatedStringSliceOptions),
 	)
 
 	flagSet.CreateGroup("configs", "Configurations",
@@ -417,6 +419,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 
 	flagSet.CreateGroup("cloud", "Cloud",
 		flagSet.DynamicVar(&pdcpauth, "auth", "true", "configure projectdiscovery cloud (pdcp) api key"),
+		flagSet.StringVarP(&options.TeamID, "team-id", "tid", _pdcp.TeamIDEnv, "upload scan results to given team id (optional)"),
 		flagSet.BoolVarP(&options.EnableCloudUpload, "cloud-upload", "cup", false, "upload scan results to pdcp dashboard"),
 		flagSet.StringVarP(&options.ScanID, "scan-id", "sid", "", "upload scan results to existing scan id (optional)"),
 		flagSet.StringVarP(&options.ScanName, "scan-name", "sname", "", "scan name to set (optional)"),

cmd/tmc/main.go

@@ -162,7 +162,7 @@ func process(opts options) error {
 			var updated bool // if max-requests is updated
 			dataString, updated, err = parseAndAddMaxRequests(templateCatalog, path, dataString)
 			if err != nil {
-				gologger.Info().Label("max-request").Msgf(logErrMsg(path, err, opts.debug, errFile))
+				gologger.Info().Label("max-request").Msg(logErrMsg(path, err, opts.debug, errFile))
 			} else {
 				if updated {
 					gologger.Info().Label("max-request").Msgf("✅ updated template: %s\n", path)
@@ -255,7 +255,7 @@ func enhanceTemplate(data string) (string, bool, error) {
 		return data, false, errorutil.New("validation failed").WithTag("validate")
 	}
 	if templateResp.Error.Name != "" {
-		return data, false, errorutil.New(templateResp.Error.Name)
+		return data, false, errorutil.New("%s", templateResp.Error.Name)
 	}
 	if strings.TrimSpace(templateResp.Enhanced) == "" && !templateResp.Lint {
 		if templateResp.LintError.Reason != "" {
@@ -289,7 +289,7 @@ func formatTemplate(data string) (string, bool, error) {
 		return data, false, errorutil.New("validation failed").WithTag("validate")
 	}
 	if templateResp.Error.Name != "" {
-		return data, false, errorutil.New(templateResp.Error.Name)
+		return data, false, errorutil.New("%s", templateResp.Error.Name)
 	}
 	if strings.TrimSpace(templateResp.Updated) == "" && !templateResp.Lint {
 		if templateResp.LintError.Reason != "" {
@@ -345,7 +345,7 @@ func validateTemplate(data string) (bool, error) {
 		return false, errorutil.New("validation failed").WithTag("validate")
 	}
 	if validateResp.Error.Name != "" {
-		return false, errorutil.New(validateResp.Error.Name)
+		return false, errorutil.New("%s", validateResp.Error.Name)
 	}
 	return false, errorutil.New("template validation failed")
 }

examples/advanced/advanced.go

@@ -1,13 +1,24 @@
 package main
 
 import (
+	"context"
+
 	nuclei "github.com/projectdiscovery/nuclei/v3/lib"
+	"github.com/projectdiscovery/nuclei/v3/pkg/installer"
 	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 func main() {
+	ctx := context.Background()
+	// when running nuclei in parallel for first time it is a good practice to make sure
+	// templates exists first
+	tm := installer.TemplateManager{}
+	if err := tm.FreshInstallIfNotExists(); err != nil {
+		panic(err)
+	}
+
 	// create nuclei engine with options
-	ne, err := nuclei.NewThreadSafeNucleiEngine()
+	ne, err := nuclei.NewThreadSafeNucleiEngineCtx(ctx)
 	if err != nil {
 		panic(err)
 	}