Merge pull request #4575 from projectdiscovery/feat-fastdialer-exclude

pass exclude list to layer 4 fast dialer

pkg/core/inputs/hybrid/hmap.go

@@ -18,12 +18,12 @@ import (
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/hmap/filekv"
 	"github.com/projectdiscovery/hmap/store/hybrid"
-	"github.com/projectdiscovery/mapcidr"
 	"github.com/projectdiscovery/mapcidr/asn"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/uncover"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	"github.com/projectdiscovery/nuclei/v3/pkg/utils/expand"
 	uncoverlib "github.com/projectdiscovery/uncover"
 	fileutil "github.com/projectdiscovery/utils/file"
 	iputil "github.com/projectdiscovery/utils/ip"
@@ -121,10 +121,10 @@ func (i *Input) initializeInputSources(opts *Options) error {
 	for _, target := range options.Targets {
 		switch {
 		case iputil.IsCIDR(target):
-			ips := i.expandCIDRInputValue(target)
+			ips := expand.CIDR(target)
 			i.addTargets(ips)
 		case asn.IsASN(target):
-			ips := i.expandASNInputValue(target)
+			ips := expand.ASN(target)
 			i.addTargets(ips)
 		default:
 			i.Set(target)
@@ -174,10 +174,10 @@ func (i *Input) initializeInputSources(opts *Options) error {
 		for _, target := range options.ExcludeTargets {
 			switch {
 			case iputil.IsCIDR(target):
-				ips := i.expandCIDRInputValue(target)
+				ips := expand.CIDR(target)
 				i.removeTargets(ips)
 			case asn.IsASN(target):
-				ips := i.expandASNInputValue(target)
+				ips := expand.ASN(target)
 				i.removeTargets(ips)
 			default:
 				i.Del(target)
@@ -195,10 +195,10 @@ func (i *Input) scanInputFromReader(reader io.Reader) {
 		item := scanner.Text()
 		switch {
 		case iputil.IsCIDR(item):
-			ips := i.expandCIDRInputValue(item)
+			ips := expand.CIDR(item)
 			i.addTargets(ips)
 		case asn.IsASN(item):
-			ips := i.expandASNInputValue(item)
+			ips := expand.ASN(item)
 			i.addTargets(ips)
 		default:
 			i.Set(item)
@@ -489,26 +489,6 @@ func (i *Input) Scan(callback func(value *contextargs.MetaInput) bool) {
 	}
 }
 
-// expandCIDRInputValue expands CIDR and stores expanded IPs
-func (i *Input) expandCIDRInputValue(value string) []string {
-	var ips []string
-	ipsCh, _ := mapcidr.IPAddressesAsStream(value)
-	for ip := range ipsCh {
-		ips = append(ips, ip)
-	}
-	return ips
-}
-
-// expandASNInputValue expands CIDRs for given ASN and stores expanded IPs
-func (i *Input) expandASNInputValue(value string) []string {
-	var ips []string
-	cidrs, _ := asn.GetCIDRsForASNNum(value)
-	for _, cidr := range cidrs {
-		ips = append(ips, i.expandCIDRInputValue(cidr.String())...)
-	}
-	return ips
-}
-
 func (i *Input) addTargets(targets []string) {
 	for _, target := range targets {
 		i.Set(target)

pkg/core/inputs/hybrid/hmap_test.go

@@ -12,10 +12,11 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	"github.com/projectdiscovery/nuclei/v3/pkg/utils/expand"
 	"github.com/stretchr/testify/require"
 )
 
-func Test_expandCIDRInputValue(t *testing.T) {
+func Test_expandCIDR(t *testing.T) {
 	tests := []struct {
 		cidr     string
 		expected []string
@@ -33,7 +34,7 @@ func Test_expandCIDRInputValue(t *testing.T) {
 		require.Nil(t, err, "could not create temporary input file")
 		input := &Input{hostMap: hm}
 
-		ips := input.expandCIDRInputValue(tt.cidr)
+		ips := expand.CIDR(tt.cidr)
 		input.addTargets(ips)
 		// scan
 		got := []string{}
@@ -170,7 +171,7 @@ func Test_expandASNInputValue(t *testing.T) {
 		require.Nil(t, err, "could not create temporary input file")
 		input := &Input{hostMap: hm}
 		// get the IP addresses for ASN number
-		ips := input.expandASNInputValue(tt.asn)
+		ips := expand.ASN(tt.asn)
 		input.addTargets(ips)
 		// scan the hmap
 		got := []string{}

pkg/protocols/common/protocolstate/state.go

@@ -9,8 +9,10 @@ import (
 	"golang.org/x/net/proxy"
 
 	"github.com/projectdiscovery/fastdialer/fastdialer"
+	"github.com/projectdiscovery/mapcidr/asn"
 	"github.com/projectdiscovery/networkpolicy"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	"github.com/projectdiscovery/nuclei/v3/pkg/utils/expand"
 )
 
 // Dialer is a shared fastdialer instance for host DNS resolution
@@ -102,6 +104,15 @@ func Init(options *types.Options) error {
 	if options.RestrictLocalNetworkAccess {
 		opts.Deny = append(networkpolicy.DefaultIPv4DenylistRanges, networkpolicy.DefaultIPv6DenylistRanges...)
 	}
+	for _, excludeTarget := range options.ExcludeTargets {
+		switch {
+		case asn.IsASN(excludeTarget):
+			opts.Deny = append(opts.Deny, expand.ASN(excludeTarget)...)
+		default:
+			opts.Deny = append(opts.Deny, excludeTarget)
+		}
+	}
+
 	opts.WithDialerHistory = true
 	opts.SNIName = options.SNI
 

pkg/utils/expand/expand.go

@@ -0,0 +1,26 @@
+package expand
+
+import (
+	"github.com/projectdiscovery/mapcidr"
+	"github.com/projectdiscovery/mapcidr/asn"
+)
+
+// Expands CIDR to IPs
+func CIDR(value string) []string {
+	var ips []string
+	ipsCh, _ := mapcidr.IPAddressesAsStream(value)
+	for ip := range ipsCh {
+		ips = append(ips, ip)
+	}
+	return ips
+}
+
+// Expand ASN to IPs
+func ASN(value string) []string {
+	var ips []string
+	cidrs, _ := asn.GetCIDRsForASNNum(value)
+	for _, cidr := range cidrs {
+		ips = append(ips, CIDR(cidr.String())...)
+	}
+	return ips
+}