diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a39e441..cff1fb2 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -11,7 +11,7 @@ updates: directory: "/" schedule: interval: "weekly" - target-branch: "dev" + target-branch: "dep" commit-message: prefix: "chore" include: "scope" @@ -23,7 +23,7 @@ updates: directory: "/" schedule: interval: "daily" - target-branch: "dev" + target-branch: "dep" commit-message: prefix: "chore" include: "scope" @@ -35,7 +35,7 @@ updates: directory: "/" schedule: interval: "weekly" - target-branch: "dev" + target-branch: "dep" commit-message: prefix: "chore" include: "scope" diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index d2518d2..362b5c3 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -53,3 +53,10 @@ jobs: - name: Race Condition Tests run: go build -race . working-directory: cmd/httpx/ + + - name: release test + uses: goreleaser/goreleaser-action@v4 + with: + args: "release --clean --snapshot" + version: latest + workdir: . diff --git a/.github/workflows/dep-auto-merge.yml b/.github/workflows/dep-auto-merge.yml new file mode 100644 index 0000000..3622902 --- /dev/null +++ b/.github/workflows/dep-auto-merge.yml @@ -0,0 +1,25 @@ +name: 🤖 dep-auto-merge + +on: + pull_request: + branches: + - main + workflow_dispatch: + +permissions: + pull-requests: write + issues: write + repository-projects: write + +jobs: + automerge: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + token: ${{ secrets.DEPENDABOT_PAT }} + + - uses: ahmadnassri/action-dependabot-auto-merge@v2 + with: + github-token: ${{ secrets.DEPENDABOT_PAT }} + target: all \ No newline at end of file diff --git a/.gitignore b/.gitignore index 028e9b1..064196c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .idea/ .vscode/ +dist cmd/httpx/httpx integration_tests/httpx integration_tests/integration-test @@ -7,3 +8,5 @@ cmd/functional-test/httpx_dev cmd/functional-test/functional-test cmd/functional-test/httpx cmd/functional-test/*.cfg + +.devcontainer \ No newline at end of file diff --git a/.goreleaser.yml b/.goreleaser.yml index 78f2618..852f6cb 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -28,8 +28,7 @@ builds: archives: - format: zip - replacements: - darwin: macOS + name_template: '{{ .ProjectName }}_{{ .Version }}_{{ if eq .Os "darwin" }}macOS{{ else }}{{ .Os }}{{ end }}_{{ .Arch }}' checksum: algorithm: sha256 diff --git a/Dockerfile b/Dockerfile index ee83651..196ba43 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Base -FROM golang:1.20.5-alpine AS builder +FROM golang:1.20.6-alpine AS builder RUN apk add --no-cache git build-base gcc musl-dev WORKDIR /app diff --git a/README.md b/README.md index df00b6e..fe8b485 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,11 @@ go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest ``` +| :exclamation: **Disclaimer** | +|---------------------------------| +| **This project is in active development**. Expect breaking changes with releases. Review the changelog before updating. | +| This project was primarily built to be used as a standalone CLI tool. **Running it as a service may pose security risks.** It's recommended to use with caution and additional security measures. | + # Usage ```sh @@ -165,6 +170,7 @@ UPDATE: OUTPUT: -o, -output string file to write output results + -oa, -output-all filename to write output results in all formats -sr, -store-response store http response to output directory -srd, -store-response-dir string store http response to custom directory -csv store output in csv format diff --git a/cmd/integration-test/http.go b/cmd/integration-test/http.go index 654d50c..62bc34e 100644 --- a/cmd/integration-test/http.go +++ b/cmd/integration-test/http.go @@ -5,10 +5,12 @@ import ( "io" "net/http" "net/http/httptest" + "os" "strings" "github.com/julienschmidt/httprouter" "github.com/projectdiscovery/httpx/internal/testutils" + fileutil "github.com/projectdiscovery/utils/file" ) var httpTestcases = map[string]testutils.TestCase{ @@ -30,6 +32,7 @@ var httpTestcases = map[string]testutils.TestCase{ "Multiple Custom Header": &customHeader{inputData: []string{"-debug-req", "-H", "'user-agent: test'", "-H", "'foo: bar'"}, expectedOutput: []string{"User-Agent: test", "Foo: bar"}}, "Output Match Condition": &outputMatchCondition{inputData: []string{"-silent", "-mdc", "\"status_code == 200\""}}, "Output Filter Condition": &outputFilterCondition{inputData: []string{"-silent", "-fdc", "\"status_code == 400\""}}, + "Output All": &outputAll{}, } type standardHttpGet struct { @@ -377,3 +380,42 @@ func (h *outputFilterCondition) Execute() error { } return nil } + +type outputAll struct { +} + +func (h *outputAll) Execute() error { + var ts *httptest.Server + router := httprouter.New() + router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(200) + fmt.Fprint(w, `{"status": "ok"}`) + })) + ts = httptest.NewServer(router) + defer ts.Close() + + fileName := "test_output_all" + _, hErr := testutils.RunHttpxAndGetResults(ts.URL, false, []string{"-o", fileName, "-oa"}...) + if hErr != nil { + return hErr + } + + expectedFiles := []string{fileName, fileName + ".json", fileName + ".csv"} + var actualFiles []string + + for _, file := range expectedFiles { + if fileutil.FileExists(file) { + actualFiles = append(actualFiles, file) + } + } + if len(actualFiles) != 3 { + return errIncorrectResultsCount(actualFiles) + } + + for _, file := range actualFiles { + _ = os.Remove(file) + } + + return nil +} diff --git a/go.mod b/go.mod index 646a90b..6027fb1 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/projectdiscovery/mapcidr v1.1.2 github.com/projectdiscovery/rawhttp v0.1.17 github.com/projectdiscovery/retryablehttp-go v1.0.18 - github.com/projectdiscovery/wappalyzergo v0.0.102 + github.com/projectdiscovery/wappalyzergo v0.0.105 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 go.etcd.io/bbolt v1.3.7 // indirect @@ -44,10 +44,10 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/projectdiscovery/asnmap v1.0.4 github.com/projectdiscovery/dsl v0.0.12 - github.com/projectdiscovery/fastdialer v0.0.32 + github.com/projectdiscovery/fastdialer v0.0.35 github.com/projectdiscovery/ratelimit v0.0.9 - github.com/projectdiscovery/tlsx v1.1.0 - github.com/projectdiscovery/utils v0.0.42 + github.com/projectdiscovery/tlsx v1.1.1 + github.com/projectdiscovery/utils v0.0.43 github.com/stretchr/testify v1.8.4 github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101 go.uber.org/multierr v1.11.0 @@ -68,14 +68,14 @@ require ( github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/charmbracelet/glamour v0.6.0 // indirect - github.com/cheggaaa/pb/v3 v3.1.2 // indirect + github.com/cheggaaa/pb/v3 v3.1.4 // indirect github.com/cloudflare/cfssl v1.6.4 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dlclark/regexp2 v1.8.1 // indirect github.com/dsnet/compress v0.0.1 // indirect - github.com/fatih/color v1.14.1 // indirect + github.com/fatih/color v1.15.0 // indirect github.com/gaukas/godicttls v0.0.3 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/golang/protobuf v1.5.3 // indirect @@ -91,7 +91,7 @@ require ( github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect github.com/mholt/archiver v3.1.1+incompatible // indirect github.com/minio/selfupdate v0.6.0 // indirect diff --git a/go.sum b/go.sum index 89be62c..51ece8f 100644 --- a/go.sum +++ b/go.sum @@ -35,8 +35,8 @@ github.com/bits-and-blooms/bitset v1.3.1 h1:y+qrlmq3XsWi+xZqSaueaE8ry8Y127iMxlMf github.com/bits-and-blooms/bloom/v3 v3.4.0 h1:9zesenPR5M3SLIJ/esQ84o1eSVFY5Rw5d+pa1tiXQNA= github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc= github.com/charmbracelet/glamour v0.6.0/go.mod h1:taqWV4swIMMbWALc0m7AfE9JkPSU8om2538k9ITBxOc= -github.com/cheggaaa/pb/v3 v3.1.2 h1:FIxT3ZjOj9XJl0U4o2XbEhjFfZl7jCVCDOGq1ZAB7wQ= -github.com/cheggaaa/pb/v3 v3.1.2/go.mod h1:SNjnd0yKcW+kw0brSusraeDd5Bf1zBfxAzTL2ss3yQ4= +github.com/cheggaaa/pb/v3 v3.1.4 h1:DN8j4TVVdKu3WxVwcRKu0sG00IIU6FewoABZzXbRQeo= +github.com/cheggaaa/pb/v3 v3.1.4/go.mod h1:6wVjILNBaXMs8c21qRiaUM8BR82erfgau1DQ4iUXmSA= github.com/cloudflare/cfssl v1.6.4 h1:NMOvfrEjFfC63K3SGXgAnFdsgkmiq4kATme5BfcqrO8= github.com/cloudflare/cfssl v1.6.4/go.mod h1:8b3CQMxfWPAeom3zBnGJ6sd+G1NkL5TXqmDXacb+1J0= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= @@ -54,8 +54,8 @@ github.com/dlclark/regexp2 v1.8.1/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnm github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= -github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= -github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= @@ -133,8 +133,8 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= @@ -193,8 +193,8 @@ github.com/projectdiscovery/clistats v0.0.19 h1:SA/qRHbmS9VEbVEPzX/ka01hZDYATL9Z github.com/projectdiscovery/clistats v0.0.19/go.mod h1:NQDAW/O7cK9xBIgk46kJjwGRkjSg5JkB8E4DvuxXr+c= github.com/projectdiscovery/dsl v0.0.12 h1:F3S94FKyakMMtRNuob+HbW0XmibBE3zwWBw+b10x2gg= github.com/projectdiscovery/dsl v0.0.12/go.mod h1:UQxYzKD9oy/xs86rHMfCcVb+JoPJ8qUhxm9AejdsvFw= -github.com/projectdiscovery/fastdialer v0.0.32 h1:2sMAXLUcdyHMmXh46PkoRRewBBjZBMiraawSHDT/fjs= -github.com/projectdiscovery/fastdialer v0.0.32/go.mod h1:ttLvt0xnpNQAStYYQ6ElIBHfSXHuPEiXBkLH/OLbYlc= +github.com/projectdiscovery/fastdialer v0.0.35 h1:dCjYaZ2dOtKmIbQ7OUuf/pZiMQRHfUjjLoHrEF8CJ8g= +github.com/projectdiscovery/fastdialer v0.0.35/go.mod h1:dTx0C7JRWKKO5ZxGqM0NUDzB4svmyYqGM6zcHIk2ueo= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= @@ -220,12 +220,12 @@ github.com/projectdiscovery/retryabledns v1.0.30/go.mod h1:+Aqc0TjKGcTtP0HtXE8o1 github.com/projectdiscovery/retryablehttp-go v1.0.18 h1:3IUxyIOOUVSGEBm4pV0cQSk1i/DausZdHePdGDip0Lg= github.com/projectdiscovery/retryablehttp-go v1.0.18/go.mod h1:oE3dmYWMadFWzaIfG1IqINsYAzUWYUtdI4PJ2xo7cXg= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= -github.com/projectdiscovery/tlsx v1.1.0 h1:6L5VKpHaoqvIHN6lH9zi7jIvph1JwYMYZOIpWBJBG6I= -github.com/projectdiscovery/tlsx v1.1.0/go.mod h1:C9xTbU2t54Anmvuq+4jxevR5rzqpp6XUUtV7G9J5CTE= -github.com/projectdiscovery/utils v0.0.42 h1:NK506tyhI3vGH5Z6S69VTa1U/Y+VFY6vy0opg69Xy7Q= -github.com/projectdiscovery/utils v0.0.42/go.mod h1:zlRoARdARkoSa0rkoyDFPxbJ4QlqPfJnoo5pih+/FYc= -github.com/projectdiscovery/wappalyzergo v0.0.102 h1:ABjZghof2U2yzGNL+q5ouWHEardLd2o53Ukgrf8CZzE= -github.com/projectdiscovery/wappalyzergo v0.0.102/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA= +github.com/projectdiscovery/tlsx v1.1.1 h1:4q14vu2A+TnQjhYI68I3yCUss3UM0fmrkmnJKqoYRQ8= +github.com/projectdiscovery/tlsx v1.1.1/go.mod h1:x2S3KajTVxH5Tm4lbBoX4EumY/gh+cGzfBUhlCuNtdY= +github.com/projectdiscovery/utils v0.0.43 h1:KjZ17rffipb32ti7IQUS7oOXDUqilPw7dMGAmBvZayI= +github.com/projectdiscovery/utils v0.0.43/go.mod h1:HtUI1pyNCgQUuwZuxDILQ4NSUaFcfBh0TuCK/ZQTS6Q= +github.com/projectdiscovery/wappalyzergo v0.0.105 h1:8Uag57UUzZSjzKgaOZKLB+vW1VlXUcwbDKgy2fE+VUs= +github.com/projectdiscovery/wappalyzergo v0.0.105/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA= github.com/refraction-networking/utls v1.3.2 h1:o+AkWB57mkcoW36ET7uJ002CpBWHu0KPxi6vzxvPnv8= github.com/refraction-networking/utls v1.3.2/go.mod h1:fmoaOww2bxzzEpIKOebIsnBvjQpqP7L2vcm/9KUfm/E= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= diff --git a/runner/options.go b/runner/options.go index c8f4dd0..8a0e110 100644 --- a/runner/options.go +++ b/runner/options.go @@ -146,6 +146,7 @@ type Options struct { filterStatusCode []int filterContentLength []int Output string + OutputAll bool StoreResponseDir string HTTPProxy string SocksProxy string @@ -374,6 +375,7 @@ func ParseOptions() *Options { flagSet.CreateGroup("output", "Output", flagSet.StringVarP(&options.Output, "output", "o", "", "file to write output results"), + flagSet.BoolVarP(&options.OutputAll, "output-all", "oa", false, "filename to write output results in all formats"), flagSet.BoolVarP(&options.StoreResponse, "store-response", "sr", false, "store http response to output directory"), flagSet.StringVarP(&options.StoreResponseDir, "store-response-dir", "srd", "", "store http response to custom directory"), flagSet.BoolVar(&options.CSVOutput, "csv", false, "store output in csv format"), @@ -439,6 +441,15 @@ func ParseOptions() *Options { _ = flagSet.Parse() + if options.OutputAll && options.Output == "" { + gologger.Fatal().Msg("Please specify an output file using -o/-output when using -oa/-output-all") + } + + if options.OutputAll { + options.JSONOutput = true + options.CSVOutput = true + } + if cfgFile != "" { if !fileutil.FileExists(cfgFile) { gologger.Fatal().Msgf("given config file '%s' does not exist", cfgFile) @@ -509,11 +520,6 @@ func (options *Options) ValidateOptions() error { return fmt.Errorf("file '%s' does not exist", options.InputRawRequest) } - multiOutput := options.CSVOutput && options.JSONOutput - if multiOutput { - return fmt.Errorf("results can only be displayed in one format: 'JSON' or 'CSV'") - } - if options.Silent { incompatibleFlagsList := flagsIncompatibleWithSilent(options) if len(incompatibleFlagsList) > 0 { diff --git a/runner/runner.go b/runner/runner.go index 7be2c1b..a4a3720 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -626,27 +626,49 @@ func (r *Runner) RunEnumeration() { } }() - var f, indexFile, indexScreenshotFile *os.File + var plainFile, jsonFile, csvFile, indexFile, indexScreenshotFile *os.File - if r.options.Output != "" { - var err error - if r.options.Resume { - f, err = os.OpenFile(r.options.Output, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) - } else { - f, err = os.Create(r.options.Output) + if r.options.Output != "" && r.options.OutputAll { + plainFile = openOrCreateFile(r.options.Resume, r.options.Output) + defer plainFile.Close() + jsonFile = openOrCreateFile(r.options.Resume, r.options.Output+".json") + defer jsonFile.Close() + csvFile = openOrCreateFile(r.options.Resume, r.options.Output+".csv") + defer csvFile.Close() + } + + jsonOrCsv := (r.options.JSONOutput || r.options.CSVOutput) + jsonAndCsv := (r.options.JSONOutput && r.options.CSVOutput) + if r.options.Output != "" && plainFile == nil && !jsonOrCsv { + plainFile = openOrCreateFile(r.options.Resume, r.options.Output) + defer plainFile.Close() + } + + if r.options.Output != "" && r.options.JSONOutput && jsonFile == nil { + ext := "" + if jsonAndCsv { + ext = ".json" } - if err != nil { - gologger.Fatal().Msgf("Could not open/create output file '%s': %s\n", r.options.Output, err) + jsonFile = openOrCreateFile(r.options.Resume, r.options.Output+ext) + defer jsonFile.Close() + } + + if r.options.Output != "" && r.options.CSVOutput && csvFile == nil { + ext := "" + if jsonAndCsv { + ext = ".csv" } - defer f.Close() //nolint + csvFile = openOrCreateFile(r.options.Resume, r.options.Output+ext) + defer csvFile.Close() } + if r.options.CSVOutput { outEncoding := strings.ToLower(r.options.CSVOutputEncoding) switch outEncoding { case "": // no encoding do nothing case "utf-8", "utf8": bomUtf8 := []byte{0xEF, 0xBB, 0xBF} - _, err := f.Write(bomUtf8) + _, err := csvFile.Write(bomUtf8) if err != nil { gologger.Fatal().Msgf("err on file write: %s\n", err) } @@ -654,10 +676,13 @@ func (r *Runner) RunEnumeration() { gologger.Fatal().Msgf("unknown csv output encoding: %s\n", r.options.CSVOutputEncoding) } header := Result{}.CSVHeader() - gologger.Silent().Msgf("%s\n", header) - if f != nil { + if !r.options.OutputAll && !jsonAndCsv { + gologger.Silent().Msgf("%s\n", header) + } + + if csvFile != nil { //nolint:errcheck // this method needs a small refactor to reduce complexity - f.WriteString(header + "\n") + csvFile.WriteString(header + "\n") } } if r.options.StoreResponseDir != "" { @@ -873,18 +898,40 @@ func (r *Runner) RunEnumeration() { } } } - row := resp.str + + if !jsonOrCsv || jsonAndCsv || r.options.OutputAll { + gologger.Silent().Msgf("%s\n", resp.str) + } + + //nolint:errcheck // this method needs a small refactor to reduce complexity + if plainFile != nil { + plainFile.WriteString(resp.str + "\n") + } + if r.options.JSONOutput { - row = resp.JSON(&r.scanopts) - } else if r.options.CSVOutput { - row = resp.CSVRow(&r.scanopts) + row := resp.JSON(&r.scanopts) + if !r.options.OutputAll && !jsonAndCsv { + gologger.Silent().Msgf("%s\n", row) + } + + //nolint:errcheck // this method needs a small refactor to reduce complexity + if jsonFile != nil { + jsonFile.WriteString(row + "\n") + } } - gologger.Silent().Msgf("%s\n", row) - if f != nil { + if r.options.CSVOutput { + row := resp.CSVRow(&r.scanopts) + + if !r.options.OutputAll && !jsonAndCsv { + gologger.Silent().Msgf("%s\n", row) + } + //nolint:errcheck // this method needs a small refactor to reduce complexity - f.WriteString(row + "\n") + if csvFile != nil { + csvFile.WriteString(row + "\n") + } } for _, nextStep := range nextSteps { @@ -1019,6 +1066,19 @@ func logFilteredErrorPage(url string) { return } } +func openOrCreateFile(resume bool, filename string) *os.File { + var err error + var f *os.File + if resume { + f, err = os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) + } else { + f, err = os.Create(filename) + } + if err != nil { + gologger.Fatal().Msgf("Could not open/create output file '%s': %s\n", filename, err) + } + return f +} func (r *Runner) GetScanOpts() ScanOptions { return r.scanopts @@ -1659,7 +1719,10 @@ retry: hashesMap := make(map[string]interface{}) if scanopts.Hashes != "" { hs := strings.Split(scanopts.Hashes, ",") - builder.WriteString(" [") + outputHashes := !(r.options.JSONOutput || r.options.OutputAll) + if outputHashes { + builder.WriteString(" [") + } for index, hashType := range hs { var ( hashHeader, hashBody string @@ -1688,17 +1751,21 @@ retry: if hashBody != "" { hashesMap[fmt.Sprintf("body_%s", hashType)] = hashBody hashesMap[fmt.Sprintf("header_%s", hashType)] = hashHeader - if !scanopts.OutputWithNoColor { - builder.WriteString(aurora.Magenta(hashBody).String()) - } else { - builder.WriteString(hashBody) - } - if index != len(hs)-1 { - builder.WriteString(",") + if outputHashes { + if !scanopts.OutputWithNoColor { + builder.WriteString(aurora.Magenta(hashBody).String()) + } else { + builder.WriteString(hashBody) + } + if index != len(hs)-1 { + builder.WriteString(",") + } } } } - builder.WriteRune(']') + if outputHashes { + builder.WriteRune(']') + } } if scanopts.OutputLinesCount { builder.WriteString(" [")