Support multiple output format

.gitignore

@@ -7,3 +7,5 @@ cmd/functional-test/httpx_dev
 cmd/functional-test/functional-test
 cmd/functional-test/httpx
 cmd/functional-test/*.cfg
+
+.devcontainer

README.md

@@ -164,6 +164,7 @@ UPDATE:
 
 OUTPUT:
    -o, -output string                  file to write output results
+   -oa, -output-all                    filename to write output results in all formats
    -sr, -store-response                store http response to output directory
    -srd, -store-response-dir string    store http response to custom directory
    -csv                                store output in csv format

cmd/integration-test/http.go

@@ -5,10 +5,12 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"os"
 	"strings"
 
 	"github.com/julienschmidt/httprouter"
 	"github.com/projectdiscovery/httpx/internal/testutils"
+	fileutil "github.com/projectdiscovery/utils/file"
 )
 
 var httpTestcases = map[string]testutils.TestCase{
@@ -30,6 +32,7 @@ var httpTestcases = map[string]testutils.TestCase{
 	"Multiple Custom Header":                                                              &customHeader{inputData: []string{"-debug-req", "-H", "'user-agent: test'", "-H", "'foo: bar'"}, expectedOutput: []string{"User-Agent: test", "Foo: bar"}},
 	"Output Match Condition":                                                              &outputMatchCondition{inputData: []string{"-silent", "-mdc", "\"status_code == 200\""}},
 	"Output Filter Condition":                                                             &outputFilterCondition{inputData: []string{"-silent", "-fdc", "\"status_code == 400\""}},
+	"Output All":                                                                          &outputAll{},
 }
 
 type standardHttpGet struct {
@@ -377,3 +380,42 @@ func (h *outputFilterCondition) Execute() error {
 	}
 	return nil
 }
+
+type outputAll struct {
+}
+
+func (h *outputAll) Execute() error {
+	var ts *httptest.Server
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
+		w.Header().Add("Content-Type", "application/json")
+		w.WriteHeader(200)
+		fmt.Fprint(w, `{"status": "ok"}`)
+	}))
+	ts = httptest.NewServer(router)
+	defer ts.Close()
+
+	fileName := "test_output_all"
+	_, hErr := testutils.RunHttpxAndGetResults(ts.URL, false, []string{"-o", fileName, "-oa"}...)
+	if hErr != nil {
+		return hErr
+	}
+
+	expectedFiles := []string{fileName, fileName + ".json", fileName + ".csv"}
+	var actualFiles []string
+
+	for _, file := range expectedFiles {
+		if fileutil.FileExists(file) {
+			actualFiles = append(actualFiles, file)
+		}
+	}
+	if len(actualFiles) != 3 {
+		return errIncorrectResultsCount(actualFiles)
+	}
+
+	for _, file := range actualFiles {
+		_ = os.Remove(file)
+	}
+
+	return nil
+}

runner/options.go

@@ -146,6 +146,7 @@ type Options struct {
 	filterStatusCode          []int
 	filterContentLength       []int
 	Output                    string
+	OutputAll                 bool
 	StoreResponseDir          string
 	HTTPProxy                 string
 	SocksProxy                string
@@ -372,6 +373,7 @@ func ParseOptions() *Options {
 
 	flagSet.CreateGroup("output", "Output",
 		flagSet.StringVarP(&options.Output, "output", "o", "", "file to write output results"),
+		flagSet.BoolVarP(&options.OutputAll, "output-all", "oa", false, "filename to write output results in all formats"),
 		flagSet.BoolVarP(&options.StoreResponse, "store-response", "sr", false, "store http response to output directory"),
 		flagSet.StringVarP(&options.StoreResponseDir, "store-response-dir", "srd", "", "store http response to custom directory"),
 		flagSet.BoolVar(&options.CSVOutput, "csv", false, "store output in csv format"),
@@ -437,6 +439,15 @@ func ParseOptions() *Options {
 
 	_ = flagSet.Parse()
 
+	if options.OutputAll && options.Output == "" {
+		gologger.Fatal().Msg("Please specify an output file using -o/-output when using -oa/-output-all")
+	}
+
+	if options.OutputAll {
+		options.JSONOutput = true
+		options.CSVOutput = true
+	}
+
 	if cfgFile != "" {
 		if !fileutil.FileExists(cfgFile) {
 			gologger.Fatal().Msgf("given config file '%s' does not exist", cfgFile)
@@ -507,11 +518,6 @@ func (options *Options) ValidateOptions() error {
 		return fmt.Errorf("file '%s' does not exist", options.InputRawRequest)
 	}
 
-	multiOutput := options.CSVOutput && options.JSONOutput
-	if multiOutput {
-		return fmt.Errorf("results can only be displayed in one format: 'JSON' or 'CSV'")
-	}
-
 	if options.Silent {
 		incompatibleFlagsList := flagsIncompatibleWithSilent(options)
 		if len(incompatibleFlagsList) > 0 {

runner/runner.go

@@ -621,38 +621,63 @@ func (r *Runner) RunEnumeration() {
 			}
 		}()
 
-		var f, indexFile, indexScreenshotFile *os.File
+		var plainFile, jsonFile, csvFile, indexFile, indexScreenshotFile *os.File
 
-		if r.options.Output != "" {
-			var err error
-			if r.options.Resume {
-				f, err = os.OpenFile(r.options.Output, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
-			} else {
-				f, err = os.Create(r.options.Output)
+		if r.options.Output != "" && r.options.OutputAll {
+			plainFile = openOrCreateFile(r.options.Resume, r.options.Output)
+			defer plainFile.Close()
+			jsonFile = openOrCreateFile(r.options.Resume, r.options.Output+".json")
+			defer jsonFile.Close()
+			csvFile = openOrCreateFile(r.options.Resume, r.options.Output+".csv")
+			defer csvFile.Close()
+		}
+
+		jsonOrCsv := (r.options.JSONOutput || r.options.CSVOutput)
+		jsonAndCsv := (r.options.JSONOutput && r.options.CSVOutput)
+		if r.options.Output != "" && plainFile == nil && !jsonOrCsv {
+			plainFile = openOrCreateFile(r.options.Resume, r.options.Output)
+			defer plainFile.Close()
+		}
+
+		if r.options.Output != "" && r.options.JSONOutput && jsonFile == nil {
+			ext := ""
+			if jsonAndCsv {
+				ext = ".json"
 			}
-			if err != nil {
-				gologger.Fatal().Msgf("Could not open/create output file '%s': %s\n", r.options.Output, err)
+			jsonFile = openOrCreateFile(r.options.Resume, r.options.Output+ext)
+			defer jsonFile.Close()
+		}
+
+		if r.options.Output != "" && r.options.CSVOutput && csvFile == nil {
+			ext := ""
+			if jsonAndCsv {
+				ext = ".csv"
 			}
-			defer f.Close() //nolint
+			csvFile = openOrCreateFile(r.options.Resume, r.options.Output+ext)
+			defer csvFile.Close()
 		}
+
 		if r.options.CSVOutput {
 			outEncoding := strings.ToLower(r.options.CSVOutputEncoding)
 			switch outEncoding {
 			case "": // no encoding do nothing
 			case "utf-8", "utf8":
 				bomUtf8 := []byte{0xEF, 0xBB, 0xBF}
-				_, err := f.Write(bomUtf8)
+				_, err := csvFile.Write(bomUtf8)
 				if err != nil {
 					gologger.Fatal().Msgf("err on file write: %s\n", err)
 				}
 			default: // unknown encoding
 				gologger.Fatal().Msgf("unknown csv output encoding: %s\n", r.options.CSVOutputEncoding)
 			}
 			header := Result{}.CSVHeader()
-			gologger.Silent().Msgf("%s\n", header)
-			if f != nil {
+			if !r.options.OutputAll && !jsonAndCsv {
+				gologger.Silent().Msgf("%s\n", header)
+			}
+
+			if csvFile != nil {
 				//nolint:errcheck // this method needs a small refactor to reduce complexity
-				f.WriteString(header + "\n")
+				csvFile.WriteString(header + "\n")
 			}
 		}
 		if r.options.StoreResponseDir != "" {
@@ -859,18 +884,40 @@ func (r *Runner) RunEnumeration() {
 					}
 				}
 			}
-			row := resp.str
+
+			if !jsonOrCsv || jsonAndCsv || r.options.OutputAll {
+				gologger.Silent().Msgf("%s\n", resp.str)
+			}
+
+			//nolint:errcheck // this method needs a small refactor to reduce complexity
+			if plainFile != nil {
+				plainFile.WriteString(resp.str + "\n")
+			}
+
 			if r.options.JSONOutput {
-				row = resp.JSON(&r.scanopts)
-			} else if r.options.CSVOutput {
-				row = resp.CSVRow(&r.scanopts)
+				row := resp.JSON(&r.scanopts)
+
+				if !r.options.OutputAll && !jsonAndCsv {
+					gologger.Silent().Msgf("%s\n", row)
+				}
 
+				//nolint:errcheck // this method needs a small refactor to reduce complexity
+				if jsonFile != nil {
+					jsonFile.WriteString(row + "\n")
+				}
 			}
 
-			gologger.Silent().Msgf("%s\n", row)
-			if f != nil {
+			if r.options.CSVOutput {
+				row := resp.CSVRow(&r.scanopts)
+
+				if !r.options.OutputAll && !jsonAndCsv {
+					gologger.Silent().Msgf("%s\n", row)
+				}
+
 				//nolint:errcheck // this method needs a small refactor to reduce complexity
-				f.WriteString(row + "\n")
+				if csvFile != nil {
+					csvFile.WriteString(row + "\n")
+				}
 			}
 
 			for _, nextStep := range nextSteps {
@@ -975,6 +1022,20 @@ func (r *Runner) RunEnumeration() {
 	wgoutput.Wait()
 }
 
+func openOrCreateFile(resume bool, filename string) *os.File {
+	var err error
+	var f *os.File
+	if resume {
+		f, err = os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
+	} else {
+		f, err = os.Create(filename)
+	}
+	if err != nil {
+		gologger.Fatal().Msgf("Could not open/create output file '%s': %s\n", filename, err)
+	}
+	return f
+}
+
 func (r *Runner) GetScanOpts() ScanOptions {
 	return r.scanopts
 }
@@ -1614,7 +1675,10 @@ retry:
 	hashesMap := make(map[string]interface{})
 	if scanopts.Hashes != "" {
 		hs := strings.Split(scanopts.Hashes, ",")
-		builder.WriteString(" [")
+		outputHashes := !(r.options.JSONOutput || r.options.OutputAll)
+		if outputHashes {
+			builder.WriteString(" [")
+		}
 		for index, hashType := range hs {
 			var (
 				hashHeader, hashBody string
@@ -1643,17 +1707,21 @@ retry:
 			if hashBody != "" {
 				hashesMap[fmt.Sprintf("body_%s", hashType)] = hashBody
 				hashesMap[fmt.Sprintf("header_%s", hashType)] = hashHeader
-				if !scanopts.OutputWithNoColor {
-					builder.WriteString(aurora.Magenta(hashBody).String())
-				} else {
-					builder.WriteString(hashBody)
-				}
-				if index != len(hs)-1 {
-					builder.WriteString(",")
+				if outputHashes {
+					if !scanopts.OutputWithNoColor {
+						builder.WriteString(aurora.Magenta(hashBody).String())
+					} else {
+						builder.WriteString(hashBody)
+					}
+					if index != len(hs)-1 {
+						builder.WriteString(",")
+					}
 				}
 			}
 		}
-		builder.WriteRune(']')
+		if outputHashes {
+			builder.WriteRune(']')
+		}
 	}
 	if scanopts.OutputLinesCount {
 		builder.WriteString(" [")