MongoDB Reporting

cmd/nuclei/issue-tracker-config.yaml

@@ -162,3 +162,14 @@
 #  duplicate-issue-check: false
 #  # open-state-id is the ID of the open state in Linear
 #  open-state-id: ""
+#mongodb:
+#  # the connection string to the MongoDB database
+#  # (e.g., mongodb://root:example@localhost:27017/nuclei?ssl=false&authSource=admin)
+#  connection-string: ""
+#  # the name of the collection to store the issues
+#  collection-name: ""
+#  # excludes the Request and Response from the results (helps with filesize)
+#  omit-raw: false
+#  # determines the number of results to be kept in memory before writing it to the database or 0 to
+#  # persist all in memory and write all results at the end (default)
+#  batch-size: 0

go.mod

@@ -105,6 +105,7 @@ require (
 	github.com/stretchr/testify v1.9.0
 	github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9
 	github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706
+	go.mongodb.org/mongo-driver v1.17.0
 	golang.org/x/term v0.24.0
 	gopkg.in/yaml.v3 v3.0.1
 	moul.io/http2curl v1.0.0
@@ -195,6 +196,7 @@ require (
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/moby/term v0.5.0 // indirect
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
+	github.com/montanaflynn/stats v0.7.1 // indirect
 	github.com/muesli/reflow v0.3.0 // indirect
 	github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
@@ -228,9 +230,13 @@ require (
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.11 // indirect
 	github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
+	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
+	github.com/xdg-go/scram v1.1.2 // indirect
+	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
+	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
 	github.com/ysmood/fetchup v0.2.3 // indirect
 	github.com/ysmood/got v0.34.1 // indirect
 	github.com/yuin/goldmark v1.7.4 // indirect

go.sum

@@ -755,6 +755,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
+github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
+github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
 github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8=
 github.com/mreiferson/go-httpclient v0.0.0-20201222173833-5e475fde3a4d/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8=
 github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s=
@@ -1085,6 +1087,12 @@ github.com/xanzy/go-gitlab v0.107.0 h1:P2CT9Uy9yN9lJo3FLxpMZ4xj6uWcpnigXsjvqJ6nd
 github.com/xanzy/go-gitlab v0.107.0/go.mod h1:wKNKh3GkYDMOsGmnfuX+ITCmDuSDWFO0G+C4AygL9RY=
 github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
 github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
+github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
+github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
+github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
@@ -1098,6 +1106,8 @@ github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMx
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU=
 github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g=
+github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM=
+github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI=
 github.com/ysmood/fetchup v0.2.3 h1:ulX+SonA0Vma5zUFXtv52Kzip/xe7aj4vqT5AJwQ+ZQ=
 github.com/ysmood/fetchup v0.2.3/go.mod h1:xhibcRKziSvol0H1/pj33dnKrYyI2ebIvz5cOOkYGns=
 github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=
@@ -1150,6 +1160,8 @@ go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0=
 go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ=
 go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
+go.mongodb.org/mongo-driver v1.17.0 h1:Hp4q2MCjvY19ViwimTs00wHi7G4yzxh4/2+nTx8r40k=
+go.mongodb.org/mongo-driver v1.17.0/go.mod h1:wwWm/+BuOddhcq3n68LKRmgk2wXzmF6s0SFOa0GINL4=
 go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=

pkg/reporting/exporters/mongo/mongo.go

@@ -0,0 +1,154 @@
+package mongo
+
+import (
+	"context"
+	"github.com/pkg/errors"
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v3/pkg/output"
+	"go.mongodb.org/mongo-driver/mongo"
+	"net/url"
+	"os"
+	"strings"
+	"sync"
+
+	mongooptions "go.mongodb.org/mongo-driver/mongo/options"
+)
+
+type Exporter struct {
+	options    *Options
+	mutex      *sync.Mutex
+	rows       []output.ResultEvent
+	collection *mongo.Collection
+	connection *mongo.Client
+}
+
+// Options contains the configuration options for MongoDB exporter client
+type Options struct {
+	// ConnectionString is the connection string to the MongoDB database
+	ConnectionString string `yaml:"connection-string"`
+	// CollectionName is the name of the MongoDB collection in which to store the results
+	CollectionName string `yaml:"collection-name"`
+	// OmitRaw excludes the Request and Response from the results (helps with filesize)
+	OmitRaw bool `yaml:"omit-raw"`
+	// BatchSize determines the number of results to be kept in memory before writing it to the database or 0 to
+	//	persist all in memory and write all results at the end (default)
+	BatchSize int `yaml:"batch-size"`
+}
+
+// New creates a new MongoDB exporter integration client based on options.
+func New(options *Options) (*Exporter, error) {
+	exporter := &Exporter{
+		mutex:   &sync.Mutex{},
+		options: options,
+		rows:    []output.ResultEvent{},
+	}
+
+	// If the environment variable for the connection string is set, then use that instead. This allows for easier
+	// management of sensitive items such as credentials
+	envConnectionString := os.Getenv("MONGO_CONNECTION_STRING")
+	if envConnectionString != "" {
+		options.ConnectionString = envConnectionString
+		gologger.Info().Msgf("Using connection string from environment variable MONGO_CONNECTION_STRING")
+	}
+
+	// Create the connection to the database
+	clientOptions := mongooptions.Client().ApplyURI(options.ConnectionString)
+
+	// Create a new client and connect to the MongoDB server
+	client, err := mongo.Connect(context.TODO(), clientOptions)
+	if err != nil {
+		gologger.Error().Msgf("Error creating MongoDB client: %s", err)
+	}
+
+	// Ensure the connection is valid
+	err = client.Ping(context.Background(), nil)
+	if err != nil {
+		gologger.Error().Msgf("Error connecting to MongoDB: %s", err)
+	}
+
+	// Get the database from the connection string to set the database and collection
+	parsed, err := url.Parse(options.ConnectionString)
+	if err != nil {
+		gologger.Error().Msgf("Error parsing connection string: %s", options.ConnectionString)
+		return nil, err
+	}
+
+	databaseName := strings.TrimPrefix(parsed.Path, "/")
+
+	if databaseName == "" {
+		gologger.Error().Msgf("Error getting database name from connection string: %s", options.ConnectionString)
+		return nil, errors.New("error getting database name from connection string")
+	}
+
+	exporter.connection = client
+	exporter.collection = client.Database(databaseName).Collection(options.CollectionName)
+
+	return exporter, nil
+}
+
+// Export writes a result document to the configured MongoDB collection
+// in the database configured by the connection string
+func (exporter *Exporter) Export(event *output.ResultEvent) error {
+	exporter.mutex.Lock()
+	defer exporter.mutex.Unlock()
+
+	if exporter.options.OmitRaw {
+		event.Request = ""
+		event.Response = ""
+	}
+
+	// Add the row to the queue to be processed
+	exporter.rows = append(exporter.rows, *event)
+
+	// If the batch size is greater than 0 and the number of rows has reached the batch, flush it to the database
+	if exporter.options.BatchSize > 0 && len(exporter.rows) >= exporter.options.BatchSize {
+		err := exporter.WriteRows()
+		if err != nil {
+			// The error is already logged, return it to bubble up to the caller
+			return err
+		}
+	}
+
+	return nil
+}
+
+// WriteRows writes all rows from the rows list to the MongoDB collection and removes them from the list
+func (exporter *Exporter) WriteRows() error {
+	// Loop through the rows and write them, removing them as they're entered
+	for len(exporter.rows) > 0 {
+		data := exporter.rows[0]
+
+		// Write the data to the database
+		_, err := exporter.collection.InsertOne(context.TODO(), data)
+		if err != nil {
+			gologger.Fatal().Msgf("Error inserting record into MongoDB collection: %s", err)
+			return err
+		}
+
+		// Remove the item from the list
+		exporter.rows = exporter.rows[1:]
+	}
+
+	return nil
+}
+
+func (exporter *Exporter) Close() error {
+	exporter.mutex.Lock()
+	defer exporter.mutex.Unlock()
+
+	// Write all pending rows
+	err := exporter.WriteRows()
+	if err != nil {
+		// The error is already logged, return it to bubble up to the caller
+		return err
+	}
+
+	// Close the database connection
+	err = exporter.connection.Disconnect(context.TODO())
+	if err != nil {
+		gologger.Error().Msgf("Error disconnecting from MongoDB: %s", err)
+		return err
+	}
+
+	return nil
+}

pkg/reporting/options.go

@@ -5,6 +5,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonexporter"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonl"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown"
+	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/mongo"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/sarif"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/splunk"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters"
@@ -44,6 +45,8 @@ type Options struct {
 	JSONExporter *jsonexporter.Options `yaml:"json"`
 	// JSONLExporter contains configuration options for JSONL Exporter Module
 	JSONLExporter *jsonl.Options `yaml:"jsonl"`
+	// MongoDBExporter containers the configuration options for the MongoDB Exporter Module
+	MongoDBExporter *mongo.Options `yaml:"mongodb"`
 
 	HttpClient *retryablehttp.Client `yaml:"-"`
 	OmitRaw    bool                  `yaml:"-"`

pkg/reporting/reporting.go

@@ -2,6 +2,7 @@ package reporting
 
 import (
 	"fmt"
+	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/mongo"
 	"os"
 	"strings"
 	"sync/atomic"
@@ -166,6 +167,13 @@ func New(options *Options, db string, doNotDedupe bool) (Client, error) {
 		}
 		client.exporters = append(client.exporters, exporter)
 	}
+	if options.MongoDBExporter != nil {
+		exporter, err := mongo.New(options.MongoDBExporter)
+		if err != nil {
+			return nil, errorutil.NewWithErr(err).Wrap(ErrExportClientCreation)
+		}
+		client.exporters = append(client.exporters, exporter)
+	}
 
 	if doNotDedupe {
 		return client, nil
@@ -212,6 +220,7 @@ func CreateConfigIfNotExists() error {
 		SplunkExporter:        &splunk.Options{},
 		JSONExporter:          &json_exporter.Options{},
 		JSONLExporter:         &jsonl.Options{},
+		MongoDBExporter:       &mongo.Options{},
 	}
 	reportingFile, err := os.Create(reportingConfig)
 	if err != nil {