Skip to content

Commit

Permalink
Merge branch 'dev' into error_page_classifier
Browse files Browse the repository at this point in the history
  • Loading branch information
dogancanbakir committed Jul 17, 2023
2 parents 2718a9a + 924eb5b commit 82c7e5d
Show file tree
Hide file tree
Showing 5 changed files with 210 additions and 6 deletions.
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ require (
github.com/projectdiscovery/gologger v1.1.11
github.com/projectdiscovery/hmap v0.0.13
github.com/projectdiscovery/mapcidr v1.1.2
github.com/projectdiscovery/rawhttp v0.1.15
github.com/projectdiscovery/rawhttp v0.1.17
github.com/projectdiscovery/retryablehttp-go v1.0.18
github.com/projectdiscovery/wappalyzergo v0.0.102
github.com/remeh/sizedwaitgroup v1.0.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -213,8 +213,8 @@ github.com/projectdiscovery/networkpolicy v0.0.6 h1:yDvm0XCrS9HeemRrBS+J+22surzV
github.com/projectdiscovery/networkpolicy v0.0.6/go.mod h1:8HJQ/33Pi7v3a3MRWIQGXzpj+zHw2d60TysEL4qdoQk=
github.com/projectdiscovery/ratelimit v0.0.9 h1:28t2xDHUnyss1irzqPG3Oxz5hkRjl+3Q2I/aes7nau8=
github.com/projectdiscovery/ratelimit v0.0.9/go.mod h1:f98UxLsHt0dWrHTbRDxos4+RvOLE0UFpyECfrfKBz1I=
github.com/projectdiscovery/rawhttp v0.1.15 h1:wW6U+M98NHtD0ZlSFJ49vS24gpSNZ6KZV3TZNPVCpAc=
github.com/projectdiscovery/rawhttp v0.1.15/go.mod h1:f57f8nG7oV8PqrhKmI1duKIT28mdpZauytslt8gP/7s=
github.com/projectdiscovery/rawhttp v0.1.17 h1:f9B5oh86TelOqN4p/RjTjPgwoMYJG9bIrSFC8oOgL8s=
github.com/projectdiscovery/rawhttp v0.1.17/go.mod h1:Wzgdg0OW83RjT73ujY3brKnb7U3aJYDKmCQgbOqv2AY=
github.com/projectdiscovery/retryabledns v1.0.30 h1:7bc8Lq3r/qzw4LdXXAxKtQa52iGiEx1WasZLVCO6Oj0=
github.com/projectdiscovery/retryabledns v1.0.30/go.mod h1:+Aqc0TjKGcTtP0HtXE8o1GzrjAHhSno6hSF+L63TBtI=
github.com/projectdiscovery/retryablehttp-go v1.0.18 h1:3IUxyIOOUVSGEBm4pV0cQSk1i/DausZdHePdGDip0Lg=
Expand Down
69 changes: 66 additions & 3 deletions runner/runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"encoding/csv"
"encoding/json"
"fmt"
"html/template"
"io"
"net"
"net/http"
Expand All @@ -30,8 +31,10 @@ import (
"github.com/projectdiscovery/httpx/common/customextract"
"github.com/projectdiscovery/httpx/common/errorpageclassifier"
"github.com/projectdiscovery/httpx/common/hashes/jarm"
"github.com/projectdiscovery/httpx/static"
"github.com/projectdiscovery/mapcidr/asn"
errorutil "github.com/projectdiscovery/utils/errors"
osutil "github.com/projectdiscovery/utils/os"

"github.com/Mzack9999/gcache"
"github.com/logrusorgru/aurora"
Expand Down Expand Up @@ -608,12 +611,21 @@ func (r *Runner) RunEnumeration() {
}

// output routine
wgoutput := sizedwaitgroup.New(1)
wgoutput := sizedwaitgroup.New(2)
wgoutput.Add()

output := make(chan Result)
go func(output chan Result) {
nextStep := make(chan Result)

go func(output chan Result, nextSteps ...chan Result) {
defer wgoutput.Done()

defer func() {
for _, nextStep := range nextSteps {
close(nextStep)
}
}()

var f, indexFile, indexScreenshotFile *os.File

if r.options.Output != "" {
Expand Down Expand Up @@ -874,8 +886,59 @@ func (r *Runner) RunEnumeration() {
//nolint:errcheck // this method needs a small refactor to reduce complexity
f.WriteString(row + "\n")
}

for _, nextStep := range nextSteps {
nextStep <- resp
}
}
}(output, nextStep)

// HTML Summary
// - needs output of previous routine
// - separate goroutine due to incapability of go templates to render from file
wgoutput.Add()
go func(output chan Result) {
defer wgoutput.Done()

if r.options.Screenshot {
screenshotHtmlPath := filepath.Join(r.options.StoreResponseDir, "screenshot", "screenshot.html")
screenshotHtml, err := os.Create(screenshotHtmlPath)
if err != nil {
gologger.Warning().Msgf("Could not create HTML file %s\n", err)
}
defer screenshotHtml.Close()

templateMap := template.FuncMap{
"safeURL": func(u string) template.URL {
if osutil.IsWindows() {
u = fmt.Sprintf("file:///%s", u)
}
return template.URL(u)
},
}
tmpl, err := template.
New("screenshotTemplate").
Funcs(templateMap).
Parse(static.HtmlTemplate)
if err != nil {
gologger.Warning().Msgf("Could not create HTML template: %v\n", err)
}

if err = tmpl.Execute(screenshotHtml, struct {
Options Options
Output chan Result
}{
Options: *r.options,
Output: output,
}); err != nil {
gologger.Warning().Msgf("Could not execute HTML template: %v\n", err)
}
}

// fallthrough if anything is left in the buffer unblocks if screenshot is false
for range output {
}
}(output)
}(nextStep)

wg := sizedwaitgroup.New(r.options.Threads)

Expand Down
133 changes: 133 additions & 0 deletions static/html-summary.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
<!DOCTYPE html>
<html>

<head>
<title>Screenshot Table</title>
<style>
body {
display: flex;
justify-content: center;
align-items: center;
}

table {
margin-top: 20px;
border-collapse: collapse;
}

th,
td {
padding: 10px;
text-align: center;
border: 1px solid black;
}

.thumbnail {
width: 400px;
height: 300px;
object-fit: cover;
}
</style>
</head>

<body>

<table style="margin: 20px auto; border-collapse: collapse">
<thead>
<tr>
<th style="padding: 10px; text-align: center; border: 1px solid black">
<strong>Response Info</strong>
</th>
<th style="padding: 10px; text-align: center; border: 1px solid black">
<strong>Screenshot</strong>
</th>
</tr>
</thead>
<tbody>
{{ $ExtractTitle := .Options.ExtractTitle }}
{{ $OutputStatusCode := .Options.StatusCode }}
{{ $OutputContentLength := .Options.ContentLength }}
{{ $Favicon := .Options.Favicon }}
{{ $OutputResponseTime := .Options.OutputResponseTime }}
{{ $OutputLinesCount := .Options.OutputLinesCount }}
{{ $OutputWordsCount := .Options.OutputWordsCount }}
{{ $OutputServerHeader := .Options.OutputServerHeader }}
{{ $TechDetect := .Options.TechDetect }}
{{range .Output}}
{{if ne .ScreenshotPath ""}}
<tr>
<td style="padding: 10px; border: 1px solid black">
<ul style="list-style-type: none; padding-left: 0">
<li>
<strong>Host:</strong>
<a style="text-decoration: none; color: blue">{{.URL}}</a>
</li>
{{if $ExtractTitle}}
<li>
<strong>Title:</strong>
<a style="text-decoration: none; color: blue">{{.Title}}</a>
</li>
{{end}}
{{if $OutputStatusCode}}
<li>
<strong>Status Code:</strong>
<a style="text-decoration: none; color: blue">{{.StatusCode}}</a>
</li>
{{end}}
{{if $OutputContentLength}}
<li>
<strong>Content-Length:</strong>
<a style="text-decoration: none; color: blue">{{.ContentLength}}</a>
</li>
{{end}}
{{if $Favicon}}
<li>
<strong>Favicon:</strong>
<a style="text-decoration: none; color: blue">{{.FavIconMMH3}}</a>
</li>
{{end}}
{{if $OutputResponseTime}}
<li>
<strong>Response Time:</strong>
<a style="text-decoration: none; color: blue">{{.ResponseTime}}</a>
</li>
{{end}}
{{if $OutputLinesCount}}
<li>
<strong>Total Lines:</strong>
<a style="text-decoration: none; color: blue">{{.Lines}}</a>
</li>
{{end}}
{{if $OutputWordsCount}}
<li>
<strong>Words Count:</strong>
<a style="text-decoration: none; color: blue">{{.Words}}</a>
</li>
{{end}}
{{if $OutputServerHeader}}
<li>
<strong>Webserver:</strong>
<a style="text-decoration: none; color: blue">{{.WebServer}}</a>
</li>
{{end}}
{{if $TechDetect}}
<li>
<strong>Technologies:</strong>
<a style="text-decoration: none; color: blue">{{.Technologies}}</a>
</li>
{{end}}
</ul>
</td>
<td style="padding: 10px; border: 1px solid black">
<a href="{{.ScreenshotPath | safeURL}}" target="_blank">
<img src="{{.ScreenshotPath | safeURL}}" alt="Screenshot" style="width: 400px; height: 300px" />
</a>
</td>
</tr>
{{end}}
{{end}}
</tbody>
</table>
</body>

</html>
8 changes: 8 additions & 0 deletions static/static.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
package static

import (
_ "embed"
)

//go:embed html-summary.html
var HtmlTemplate string

0 comments on commit 82c7e5d

Please sign in to comment.