From f8e9bcdba25ef272a57b21d6414e5e52d93cd549 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Wed, 11 Sep 2024 19:45:25 +0700 Subject: [PATCH 1/4] refactor(common): use `ParseRequestURI` instead when `NormalizeCacheValue` also it exports the method Signed-off-by: Dwi Siswanto --- .../common/hosterrorscache/hosterrorscache.go | 43 ++++++++++++------- .../hosterrorscache/hosterrorscache_test.go | 4 +- 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/pkg/protocols/common/hosterrorscache/hosterrorscache.go b/pkg/protocols/common/hosterrorscache/hosterrorscache.go index 1630e97c7d..90c9730e7f 100644 --- a/pkg/protocols/common/hosterrorscache/hosterrorscache.go +++ b/pkg/protocols/common/hosterrorscache/hosterrorscache.go @@ -75,24 +75,35 @@ func (c *Cache) Close() { c.failedTargets.Purge() } -func (c *Cache) normalizeCacheValue(value string) string { - finalValue := value - if strings.HasPrefix(value, "http") { - if parsed, err := url.Parse(value); err == nil { - hostname := parsed.Host - finalPort := parsed.Port() - if finalPort == "" { - if parsed.Scheme == "https" { - finalPort = "443" - } else { - finalPort = "80" - } - hostname = net.JoinHostPort(parsed.Host, finalPort) +// NormalizeCacheValue processes the input value and returns a normalized cache +// value. +func (c *Cache) NormalizeCacheValue(value string) string { + var normalizedValue string = value + + u, err := url.ParseRequestURI(value) + if err != nil || u.Host == "" { + u, err2 := url.ParseRequestURI("https://" + value) + if err2 != nil { + return normalizedValue + } + + normalizedValue = u.Host + err = nil + } else { + port := u.Port() + if port == "" { + switch u.Scheme { + case "https": + normalizedValue = net.JoinHostPort(u.Host, "443") + case "http": + normalizedValue = net.JoinHostPort(u.Host, "80") } - finalValue = hostname + } else { + normalizedValue = u.Host } } - return finalValue + + return normalizedValue } // ErrUnresponsiveHost is returned when a host is unresponsive @@ -166,7 +177,7 @@ func (c *Cache) GetKeyFromContext(ctx *contextargs.Context, err error) string { address = tmp.String() } } - finalValue := c.normalizeCacheValue(address) + finalValue := c.NormalizeCacheValue(address) return finalValue } diff --git a/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go b/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go index 112690d875..3c93177674 100644 --- a/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go +++ b/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go @@ -109,7 +109,7 @@ func TestCacheMarkFailedConcurrent(t *testing.T) { // the cache is not atomic during items creation, so we pre-create them with counter to zero for _, test := range tests { - normalizedValue := cache.normalizeCacheValue(test.host) + normalizedValue := cache.NormalizeCacheValue(test.host) newItem := &cacheItem{errors: atomic.Int32{}} newItem.errors.Store(0) _ = cache.failedTargets.Set(normalizedValue, newItem) @@ -131,7 +131,7 @@ func TestCacheMarkFailedConcurrent(t *testing.T) { for _, test := range tests { require.True(t, cache.Check(newCtxArgs(test.host))) - normalizedCacheValue := cache.normalizeCacheValue(test.host) + normalizedCacheValue := cache.NormalizeCacheValue(test.host) failedTarget, err := cache.failedTargets.Get(normalizedCacheValue) require.Nil(t, err) require.NotNil(t, failedTarget) From b7c7918cffbe68f97cee1d12e6d1dc22df0f5a5a Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Sat, 14 Sep 2024 04:57:14 +0700 Subject: [PATCH 2/4] refactor(runner): adjust `max-host-error` if gt `concurrency` Signed-off-by: Dwi Siswanto --- internal/runner/runner.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index b36d8ed584..e94b132df3 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -501,8 +501,17 @@ func (r *Runner) RunEnumeration() error { } if r.options.ShouldUseHostError() { - cache := hosterrorscache.New(r.options.MaxHostError, hosterrorscache.DefaultMaxHostsCount, r.options.TrackError) + maxHostError := r.options.MaxHostError + if r.options.TemplateThreads > maxHostError { + gologger.Print().Msgf("[%v] The concurrency value is higher than max-host-error", r.colorizer.BrightYellow("WRN")) + gologger.Info().Msg("Adjusting max-host-error to the concurrency value") + + maxHostError = r.options.TemplateThreads + } + + cache := hosterrorscache.New(maxHostError, hosterrorscache.DefaultMaxHostsCount, r.options.TrackError) cache.SetVerbose(r.options.Verbose) + r.hostErrors = cache executorOpts.HostErrorsCache = cache } From 146f6ae1e62a5ba45c65edf11e53f847913c2eed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Sun, 15 Sep 2024 15:49:10 +0300 Subject: [PATCH 3/4] fix lint --- pkg/protocols/common/hosterrorscache/hosterrorscache.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/protocols/common/hosterrorscache/hosterrorscache.go b/pkg/protocols/common/hosterrorscache/hosterrorscache.go index 90c9730e7f..bcfa27dbe4 100644 --- a/pkg/protocols/common/hosterrorscache/hosterrorscache.go +++ b/pkg/protocols/common/hosterrorscache/hosterrorscache.go @@ -88,7 +88,6 @@ func (c *Cache) NormalizeCacheValue(value string) string { } normalizedValue = u.Host - err = nil } else { port := u.Port() if port == "" { From 15840193385a1bac3c8fa762208fa48725155642 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Mon, 16 Sep 2024 04:01:32 +0700 Subject: [PATCH 4/4] chore(runner): expose adjusted `max-host-error` value Signed-off-by: Dwi Siswanto --- internal/runner/runner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index e94b132df3..0b6da592d3 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -504,7 +504,7 @@ func (r *Runner) RunEnumeration() error { maxHostError := r.options.MaxHostError if r.options.TemplateThreads > maxHostError { gologger.Print().Msgf("[%v] The concurrency value is higher than max-host-error", r.colorizer.BrightYellow("WRN")) - gologger.Info().Msg("Adjusting max-host-error to the concurrency value") + gologger.Info().Msgf("Adjusting max-host-error to the concurrency value: %d", r.options.TemplateThreads) maxHostError = r.options.TemplateThreads }