add dns tools (#38)

* add dns tool fact readers
* add dig dns tool
* add dnsenum tool
* simplify some rule conditions
* add dnsrecon tool
* add fierce dns tool
* add TODO for parsing dnsrecon json output
* parse dnsrecon json output, fixes to dns commands
---------

Signed-off-by: Patrick Double <pat@patdouble.com>

README.md

@@ -12,7 +12,7 @@ shadycompass - https://github.com/double16/shadycompass
 Press enter/return at the prompt to refresh data.
 
 [*] Reading hosts from tests/fixtures/etchosts/hosts
-[*] Reading nmap facts from tests/fixtures/nmap/open-ports.xml
+[*] Reading nmap facts from tests/fixtures/nmap/all/open-ports.xml
 [*] Reading hosts from /etc/hosts
 
 [$]  1. feroxbuster -u http://shadycompass.test:8080 -o feroxbuster-8080-shadycompass.test.txt --scan-limit 4 --insecure

shadycompass/__init__.py

@@ -37,10 +37,20 @@ def update_facts(self):
         for file_path in self.file_metadata.find_changes():
             if os.path.exists(file_path):
                 for fact_reader in fact_reader_registry:
-                    the_facts = fact_reader.read_facts(file_path)
-                    for the_fact in the_facts:
-                        the_fact.update({'file_path': file_path})
-                        self.declare(the_fact)
+                    try:
+                        the_facts = fact_reader.read_facts(file_path)
+                        for the_fact in the_facts:
+                            if the_fact is None:
+                                print(
+                                    f'[!] returned fact is None from {file_path}, {type(fact_reader)}, implementation error, file report at https://github.com/double16/shadycompass/issues',
+                                    file=sys.stderr)
+                            else:
+                                the_fact.update({'file_path': file_path})
+                                self.declare(the_fact)
+                    except BaseException as e:
+                        print(
+                            f'[!] error parsing {file_path}, {type(fact_reader)}, implementation error, file report at https://github.com/double16/shadycompass/issues',
+                            file=sys.stderr)
             else:
                 # retract facts for files that have been removed
                 for fact in self.facts.values():

shadycompass/config/__init__.py

@@ -20,6 +20,7 @@ class ToolCategory(object):
     port_scanner = 'port_scanner'
     http_buster = 'http_buster'
     http_spider = 'http_spider'
+    http_public_spider = 'http_public_spider'
     vhost_scanner = 'vhost_scanner'
     vuln_scanner = 'vuln_scanner'
     smb_scanner = 'smb_scanner'
@@ -50,6 +51,8 @@ def tool_category_priority(category: str) -> int:
             return 700
         case ToolCategory.http_spider:
             return 600
+        case ToolCategory.http_public_spider:
+            return 601
         case ToolCategory.http_buster:
             return 500
         case ToolCategory.smb_scanner:

shadycompass/facts/__init__.py

@@ -6,7 +6,7 @@
 
 from experta import Fact, Field
 
-from shadycompass.rules.library import METHOD_POP, METHOD_IMAP, METHOD_SMTP
+from shadycompass.rules.library import METHOD_POP, METHOD_IMAP, METHOD_SMTP, METHOD_DNS
 
 HTTP_PATTERN = re.compile(r'https?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*(),]|(%[0-9a-fA-F][0-9a-fA-F]))+')
 PRODUCT_PATTERN = re.compile(r'([A-Za-z0-9.-]+)/([0-9]+[.][A-Za-z0-9.]+)')
@@ -24,20 +24,25 @@ def read_facts(self, file_path: str) -> list[Fact]:
 fact_reader_registry: list[FactReader] = list()
 
 
-def check_file_signature(file_path: str, signature) -> bool:
+def check_file_signature(file_path: str, *signatures) -> bool:
     try:
-        if isinstance(signature, bytes):
+        if isinstance(signatures[0], bytes):
             open_flags = 'rb'
         else:
             open_flags = 'rt'
 
         with open(file_path, open_flags) as f:
             content = f.read(4096)
 
-        if isinstance(signature, re.Pattern):
-            return signature.search(content) is not None
-        else:
-            return str(signature) in content
+        for sig in signatures:
+            if isinstance(sig, re.Pattern):
+                if sig.search(content) is None:
+                    return False
+            else:
+                if str(sig) not in content:
+                    return False
+
+        return True
     except UnicodeDecodeError:
         return False
 
@@ -57,12 +62,45 @@ class TargetIPv6Network(Fact):
     network = Field(str, mandatory=True)
 
 
+class HostnameIPv4Resolution(Fact):
+    hostname = Field(str, mandatory=True)
+    addr = Field(str, mandatory=True)
+    implied = Field(bool, mandatory=False, default=True)
+
+    def get_hostname(self) -> str:
+        return self.get('hostname')
+
+    def get_addr(self) -> str:
+        return self.get('addr')
+
+    def is_implied(self) -> bool:
+        return self.get('implied')
+
+
+class HostnameIPv6Resolution(Fact):
+    hostname = Field(str, mandatory=True)
+    addr = Field(str, mandatory=True)
+    implied = Field(bool, mandatory=False, default=True)
+
+    def get_hostname(self) -> str:
+        return self.get('hostname')
+
+    def get_addr(self) -> str:
+        return self.get('addr')
+
+    def is_implied(self) -> bool:
+        return self.get('implied')
+
+
 class TargetHostname(Fact):
     hostname = Field(str, mandatory=True)
 
     def get_hostname(self) -> str:
         return self.get('hostname')
 
+    def get_resolution(self, hostname: str, implied: bool) -> None:
+        return None
+
 
 class TargetIPv4Address(Fact):
     addr = Field(str, mandatory=True)
@@ -77,6 +115,9 @@ def is_private_ip(self):
         except ValueError:
             return False  # Invalid IP address
 
+    def get_resolution(self, hostname: str, implied: bool) -> HostnameIPv4Resolution:
+        return HostnameIPv4Resolution(hostname=hostname, addr=self.get_addr(), implied=implied)
+
 
 class TargetIPv6Address(Fact):
     addr = Field(str, mandatory=True)
@@ -91,35 +132,8 @@ def is_private_ip(self):
         except ValueError:
             return False  # Invalid IP address
 
-
-class HostnameIPv4Resolution(Fact):
-    hostname = Field(str, mandatory=True)
-    addr = Field(str, mandatory=True)
-    implied = Field(bool, mandatory=False, default=True)
-
-    def get_hostname(self) -> str:
-        return self.get('hostname')
-
-    def get_addr(self) -> str:
-        return self.get('addr')
-
-    def is_implied(self) -> bool:
-        return self.get('implied')
-
-
-class HostnameIPv6Resolution(Fact):
-    hostname = Field(str, mandatory=True)
-    addr = Field(str, mandatory=True)
-    implied = Field(bool, mandatory=False, default=True)
-
-    def get_hostname(self) -> str:
-        return self.get('hostname')
-
-    def get_addr(self) -> str:
-        return self.get('addr')
-
-    def is_implied(self) -> bool:
-        return self.get('implied')
+    def get_resolution(self, hostname: str, implied: bool) -> HostnameIPv6Resolution:
+        return HostnameIPv6Resolution(hostname=hostname, addr=self.get_addr(), implied=implied)
 
 
 class HasIpService(Fact):
@@ -155,13 +169,11 @@ class HttpService(TcpIpService, HasTLS):
 
 
 class DomainTcpIpService(TcpIpService):
-    methodology_links = [
-        'https://book.hacktricks.xyz/network-services-pentesting/pentesting-dns'
-    ]
+    methodology_links = METHOD_DNS
 
 
 class DomainUdpIpService(UdpIpService):
-    methodology_links = DomainTcpIpService.methodology_links
+    methodology_links = METHOD_DNS
 
 
 class SshService(TcpIpService):

shadycompass/facts/all.py

@@ -1,3 +1,7 @@
+import shadycompass.facts.dns_scanner.dig  # noqa: F401
+import shadycompass.facts.dns_scanner.dnsenum  # noqa: F401
+import shadycompass.facts.dns_scanner.dnsrecon  # noqa: F401
+import shadycompass.facts.dns_scanner.fierce  # noqa: F401
 import shadycompass.facts.etc_hosts  # noqa: F401
 import shadycompass.facts.http_buster.dirb  # noqa: F401
 import shadycompass.facts.http_buster.feroxbuster  # noqa: F401

shadycompass/facts/dns_scanner/dig.py

@@ -0,0 +1,49 @@
+import re
+
+from experta import Fact
+
+from shadycompass.config import ToolCategory
+from shadycompass.facts import FactReader, fact_reader_registry, check_file_signature, guess_target, TargetHostname, \
+    ScanPresent
+
+_DNS_LINE_PATTERN = re.compile(r'^(\S+)[.]\s+\d+\s+IN\s+A+\s+(\S+)$', re.MULTILINE)
+_SERVER_PATTERN = re.compile(r'SERVER:\s+(\S+)#(\d+)\((\S+)\)\s+\(([A-Z]+)\)')
+
+
+class DigReader(FactReader):
+    def read_facts(self, file_path: str) -> list[Fact]:
+        if not check_file_signature(file_path, '<<>> DiG '):
+            return []
+        print(f"[*] Reading dig findings from {file_path}")
+        targets = set()
+        resolutions = set()
+        result = []
+        with open(file_path, 'rt') as file:
+            for line in file.readlines():
+                m = _SERVER_PATTERN.search(line)
+                if m:
+                    hostname_args = dict()
+                    target = guess_target(m.group(3))
+                    targets.add(target)
+                    if isinstance(target, TargetHostname):
+                        hostname_args['hostname'] = target.get_hostname()
+                    result.append(ScanPresent(category=ToolCategory.dns_scanner, name='dig',
+                                              addr=m.group(1), port=int(m.group(2)),
+                                              **hostname_args))
+                    continue
+                if line.startswith(';'):
+                    continue
+                m = _DNS_LINE_PATTERN.search(line)
+                if m:
+                    target = guess_target(m.group(2))
+                    targets.add(target)
+                    hostname = m.group(1).rstrip('.')
+                    targets.add(TargetHostname(hostname=hostname))
+                    resolutions.add(target.get_resolution(hostname, False))
+
+        result.extend(targets)
+        result.extend(resolutions)
+        return result
+
+
+fact_reader_registry.append(DigReader())

shadycompass/facts/dns_scanner/dnsenum.py

@@ -0,0 +1,49 @@
+import re
+import xml.etree.ElementTree as ET
+
+from experta import Fact
+
+from shadycompass.config import ToolCategory
+from shadycompass.facts import FactReader, fact_reader_registry, check_file_signature, guess_target, TargetHostname, \
+    ScanPresent
+
+DNSENUM_FILENAME_PATTERN = re.compile(r'dnsenum(?:-subdomains)?-([^/\\]+[.][a-z]{2,6})(?:-[\w-]+?)?[.]\w{3,5}$')
+
+
+class DnsEnumReader(FactReader):
+    def read_facts(self, file_path: str) -> list[Fact]:
+        if not check_file_signature(file_path, '<magictree class="MtBranchObject">'):
+            return []
+        print(f"[*] Reading dnsenum subdomain findings from {file_path}")
+        m = DNSENUM_FILENAME_PATTERN.search(file_path)
+        dns_hostname = m.group(1)
+
+        result = []
+        targets = set()
+        resolutions = set()
+        tree = ET.parse(file_path)
+        for host_el in tree.findall('.//host'):
+            addr = host_el.text.strip()
+            hostname_el = host_el.find('hostname')
+            if hostname_el is not None:
+                hostname = hostname_el.text.strip()
+                target = guess_target(addr)
+                targets.add(target)
+                targets.add(TargetHostname(hostname=hostname))
+                resolutions.add(target.get_resolution(hostname, False))
+
+        result.extend(targets)
+        result.extend(resolutions)
+
+        if len(result) > 0:
+            dns_hostname_args = {}
+            if isinstance(guess_target(dns_hostname), TargetHostname):
+                dns_hostname_args['hostname'] = dns_hostname
+            else:
+                dns_hostname_args['addr'] = dns_hostname
+            result.append(ScanPresent(category=ToolCategory.dns_scanner, name='dnsenum', port=53, **dns_hostname_args))
+
+        return result
+
+
+fact_reader_registry.append(DnsEnumReader())