Add configurable word lists (#68)

* add reload command
* improve location of test results
* add configurable word lists

---------

Signed-off-by: Patrick Double <pat@patdouble.com>

.github/workflows/app.yaml

@@ -47,16 +47,16 @@ jobs:
         run: |
           pip install -r requirements.txt
           pip install -r dev_requirements.txt
-          PYTHONPATH=$(pwd) pytest tests --junitxml=junit/test-results-${{ matrix.python-version }}.xml --cov=shadycompass --cov-branch --cov-report=xml:coverage-${{ matrix.python-version }}.xml --cov-report=html:htmlcov/${{ matrix.python-version }} --cov-report=term-missing
+          RESULT_DIR="test-results/${{ matrix.python-version }}"
+          mkdir -p "${RESULT_DIR}"
+          PYTHONPATH=$(pwd) pytest tests --junitxml=${RESULT_DIR}/test-results.xml --cov=shadycompass --cov-branch --cov-report=xml:${RESULT_DIR}/coverage.xml --cov-report=html:${RESULT_DIR}/htmlcov --cov-report=term-missing
 
       - name: Upload pytest test results
         uses: actions/upload-artifact@v4
         with:
           name: pytest-results-${{ matrix.python-version }}
           path: |
-            junit/test-results-${{ matrix.python-version }}.xml
-            coverage-${{ matrix.python-version }}.xml
-            htmlcov/${{ matrix.python-version }}
+            test-results/${{ matrix.python-version }}
         # Use always() to always run this step to publish test results when there are test failures
         if: ${{ always() }}
 

.gitignore

@@ -133,6 +133,7 @@ pip-delete-this-directory.txt
 # Unit test / coverage reports
 htmlcov/
 junit/
+test-results/
 .tox/
 .nox/
 .coverage

README.md

@@ -129,6 +129,8 @@ users
     displays the users that have been found
 emails
     displays the emails that have been found
+reload
+    reload from all files, only needed if recommendations aren't updated properly
 ```
 
 ### info 
@@ -427,9 +429,14 @@ tests/fixtures shadycompass > emails
 - bin@shadycompass.test
 ```
 
+### reload
+
+Shadycompass is designed to update recommendations based on changes in files. Sometimes the logic isn't quite right.
+Run the `reload` command to reload all files.
+
 ### facts
 
-Displays the things shady compass knows about to make recommendations. This is used for debugging purposes.
+Displays the things shadycompass knows about to make recommendations. This is used for debugging purposes.
 
 ## Configuration
 
@@ -467,3 +474,18 @@ shadycompass> set global production true
 
 shadycompass> unset production
  ```
+
+### word lists
+
+Some tools use word lists for brute forcing such as HTTP busting, sub domain enumeration, etc. The files can be
+configured using the `set` command.
+
+```
+shadycompass> set global wordlists.subdomain /usr/share/seclists/Discovery/DNS/subdomains-top1million-5000.txt
+
+shadycompass> set wordlists.file /usr/share/seclists/Discovery/Web-Content/raft-medium-files.txt
+
+shadycompass> set wordlists.username /usr/share/seclists/Usernames/xato-net-10-million-usernames-dup.txt
+
+shadycompass> set wordlists.password /usr/share/seclists/Passwords/xato-net-10-million-passwords-100000.txt
+```

dev_requirements.txt

@@ -1,3 +1,4 @@
 pytest==8.1.1
 pytest-cov==5.0.0
-ruff==0.4.4
+ruff==0.5.1
+parameterized==0.9.0

requirements.txt

@@ -1,2 +1,2 @@
 git+https://github.com/openmotics/om-experta@d35d53708a46482e1ee4e3a4bc1a36bc03492913#egg=om-experta
-prompt_toolkit==3.0.43
+prompt_toolkit==3.0.47

shadycompass.py

@@ -9,6 +9,8 @@
 from prompt_toolkit.history import InMemoryHistory
 
 from shadycompass import ShadyCompassOps, get_local_config_path, get_global_config_path, ToolAvailable
+from shadycompass.config import OPTION_RATELIMIT, OPTION_PRODUCTION, SECTION_WORDLISTS, OPTION_WORDLIST_FILE, \
+    OPTION_WORDLIST_USERNAME, OPTION_WORDLIST_PASSWORD, OPTION_WORDLIST_SUBDOMAIN
 
 
 def shadycompass_cli(args: list[str]) -> int:
@@ -23,27 +25,33 @@ def shadycompass_cli(args: list[str]) -> int:
     history = InMemoryHistory()
     ops = ShadyCompassOps(parsed.directories)
     commands = ['exit', 'quit', 'save', 'use', 'option', 'set', 'unset', 'reset', 'info', 'facts', 'tools', 'targets',
-                'services', 'products', 'urls', 'users', 'emails']
-    config_names = {'ratelimit', 'production'}
+                'services', 'products', 'urls', 'users', 'emails', 'reload']
+    config_names = {
+        OPTION_RATELIMIT, OPTION_PRODUCTION,
+        SECTION_WORDLISTS + '.' + OPTION_WORDLIST_FILE, SECTION_WORDLISTS + '.' + OPTION_WORDLIST_SUBDOMAIN,
+        SECTION_WORDLISTS + '.' + OPTION_WORDLIST_USERNAME, SECTION_WORDLISTS + '.' + OPTION_WORDLIST_PASSWORD,
+    }
+    config_dict = {name: None for name in config_names}
     tools = set(map(lambda e: e.get_name(), filter(lambda e: isinstance(e, ToolAvailable), ops.engine.facts.values())))
+    tools_dict = {tool: None for tool in tools}
     completer = NestedCompleter.from_nested_dict({
         **{command:None for command in commands},
         'use': {
             'global': tools,
-            **{tool: None for tool in tools},
+            **tools_dict,
         },
         'option': {
             'global': tools,
-            **{tool: None for tool in tools},
+            **tools_dict,
         },
         'info': tools,
         'set': {
             'global': config_names,
-            **{name: None for name in config_names},
+            **config_dict,
         },
         'unset': {
             'global': config_names,
-            **{name: None for name in config_names},
+            **config_dict,
         },
     })
 
@@ -122,6 +130,9 @@ def shadycompass_cli(args: list[str]) -> int:
                     ops.show_users(user_command)
                 elif user_command[0] == 'emails':
                     ops.show_emails(user_command)
+                elif user_command[0] == 'reload':
+                    ops.reload()
+                    break
 
                 else:
                     print(f'''
@@ -162,6 +173,8 @@ def shadycompass_cli(args: list[str]) -> int:
     displays the users that have been found
 emails
     displays the emails that have been found
+reload
+    reload from all files, only needed if recommendations aren't updated properly
 facts
     show current facts (useful for debugging)
 ''')

shadycompass/__init__.py

@@ -78,6 +78,10 @@ def update_facts(self):
         for fact in retract_queue:
             self.retract(fact)
 
+    def reset(self, **kwargs):
+        super().reset(**kwargs)
+        self.file_metadata.reset()
+
     def _save_config(self, facts: list[ConfigFact], config_path: str):
         config = ConfigParser()
         for fact in facts:
@@ -289,6 +293,18 @@ def print_banner(self):
 """, file=self.fd_out)
 
     def refresh(self):
+        """
+        Update the facts from changed files and run the rules.
+        """
+        self.engine.update_facts()
+        self.engine.run()
+
+    def reload(self):
+        """
+        Clear the facts and load all from files. This should not be needed if the rules are correct to handle
+        incremental changes.
+        """
+        self.engine.reset()
         self.engine.update_facts()
         self.engine.run()
 

shadycompass/config/__init__.py

@@ -15,6 +15,13 @@
 OPTION_PRODUCTION = 'production'
 OPTION_VALUE_ALL = '*'
 
+# wordlist configuration
+SECTION_WORDLISTS = 'wordlists'
+OPTION_WORDLIST_FILE = 'file'
+OPTION_WORDLIST_USERNAME = 'username'
+OPTION_WORDLIST_PASSWORD = 'password'
+OPTION_WORDLIST_SUBDOMAIN = 'subdomain'
+
 
 class ToolCategory(object):
     port_scanner = 'port_scanner'
@@ -264,6 +271,22 @@ def get_hostname(self) -> str:
         return self.get('hostname')
 
 
+class PreferredWordlist(Fact):
+    """
+    Identifies a preferred wordlist for brute forcing. This fact is expected to be present for all categories whether
+    the default or configured value. Therefore, rules do not need to check for non-existence.
+    """
+    category = Field(str, mandatory=True)
+    path = Field(str, mandatory=True)
+    default = Field(bool, mandatory=True)
+
+    def get_path(self) -> str:
+        return self.get('path')
+
+    def is_default(self) -> bool:
+        return bool(self.get('default'))
+
+
 class ConfigRules(IRules, ABC):
     def _get_tools(self, category: str) -> list[ToolAvailable]:
         tools = []
@@ -321,3 +344,55 @@ def tool_chosen(self, f1):
     )
     def retract_tool(self, f1):
         self.retract(f1)
+
+    def _declare_preferred_wordlist(self, category: str, path: str, default: bool):
+        retract_queue = []
+        for fact in filter(
+                lambda f: isinstance(f, PreferredWordlist) and f.get('category') == category,
+                self.get_facts()):
+            retract_queue.append(fact)
+        for fact in retract_queue:
+            self.retract(fact)
+        self.declare(PreferredWordlist(category=category, path=path, default=default))
+
+    @Rule(
+        NOT(ConfigFact(section=SECTION_WORDLISTS, option=OPTION_WORDLIST_FILE)),
+        salience=200
+    )
+    def preferred_wordlist_file_default(self):
+        self._declare_preferred_wordlist(OPTION_WORDLIST_FILE, "raft-large-files.txt", True)
+
+    @Rule(
+        NOT(ConfigFact(section=SECTION_WORDLISTS, option=OPTION_WORDLIST_USERNAME)),
+        salience=200
+    )
+    def preferred_wordlist_username_default(self):
+        self._declare_preferred_wordlist(OPTION_WORDLIST_USERNAME, "xato-net-10-million-usernames.txt", True)
+
+    @Rule(
+        NOT(ConfigFact(section=SECTION_WORDLISTS, option=OPTION_WORDLIST_PASSWORD)),
+        salience=200
+    )
+    def preferred_wordlist_password_default(self):
+        self._declare_preferred_wordlist(OPTION_WORDLIST_PASSWORD, "rockyou.txt", True)
+
+    @Rule(
+        NOT(ConfigFact(section=SECTION_WORDLISTS, option=OPTION_WORDLIST_SUBDOMAIN)),
+        salience=200
+    )
+    def preferred_wordlist_subdomain_default(self):
+        self._declare_preferred_wordlist(OPTION_WORDLIST_SUBDOMAIN, "subdomains-top1million-110000.txt", True)
+
+    @Rule(
+        ConfigFact(section=SECTION_WORDLISTS, option=MATCH.category, value=MATCH.path, global0=False),
+        salience=100
+    )
+    def preferred_wordlist_local(self, category, path):
+        self._declare_preferred_wordlist(category, path, False)
+
+    @Rule(
+        ConfigFact(section=SECTION_WORDLISTS, option=MATCH.category, value=MATCH.path, global0=True),
+        NOT(PreferredWordlist(category=MATCH.category, default=False)),
+    )
+    def preferred_wordlist_global(self, category, path):
+        self._declare_preferred_wordlist(category, path, False)

shadycompass/facts/filemetadata.py

@@ -48,6 +48,9 @@ def find_changes(self) -> list[str]:
             self.files.pop(path)
         return changes
 
+    def reset(self):
+        self.files.clear()
+
     def _check_file_change(self, file_path: str) -> list[str]:
         if file_path in self.files:
             file_meta_data = self.files[file_path]

shadycompass/rules/dns_scanner/dnsenum.py

@@ -3,7 +3,7 @@
 from experta import DefFacts, Rule, AS, NOT, MATCH
 
 from shadycompass import ToolAvailable
-from shadycompass.config import ToolCategory
+from shadycompass.config import ToolCategory, PreferredWordlist, OPTION_WORDLIST_SUBDOMAIN
 from shadycompass.facts import ScanNeeded, TargetDomain, RateLimitEnable, PublicTarget
 from shadycompass.rules.conditions import TOOL_PREF, TOOL_CONF
 from shadycompass.rules.irules import IRules
@@ -26,7 +26,7 @@ def dnsenum_available(self):
         )
 
     def _declare_dnsenum(self, f1: ScanNeeded, domain: TargetDomain, ratelimit: RateLimitEnable = None,
-                         public: PublicTarget = None):
+                         public: PublicTarget = None, wordlist: PreferredWordlist = None):
         addr = f1.get_addr()
         addr_file_name_part = f'-{addr}-{f1.get_port()}'
 
@@ -38,14 +38,15 @@ def _declare_dnsenum(self, f1: ScanNeeded, domain: TargetDomain, ratelimit: Rate
         more_options = []
         if ratelimit:
             more_options.append(['--threads', '1'])
+        if wordlist:
+            more_options.append(['-f', wordlist.get_path()])
 
         command_line = self.resolve_command_line(
             self.dnsenum_tool_name,
             [
                 '--dnsserver', f1.get_addr(),
                 *enum_options,
                 '-o', f'dnsenum{addr_file_name_part}-subdomains-{domain.get_domain()}.xml',
-                '-f', '/usr/share/seclists/Discovery/DNS/subdomains-top1million-110000.txt'
             ], *more_options
         )
         command_line.extend([domain.get_domain()])
@@ -61,44 +62,50 @@ def _declare_dnsenum(self, f1: ScanNeeded, domain: TargetDomain, ratelimit: Rate
     @Rule(
         AS.f1 << ScanNeeded(category=ToolCategory.dns_scanner, addr=MATCH.addr, port=53),
         AS.domain << TargetDomain(),
+        AS.wordlist << PreferredWordlist(category=OPTION_WORDLIST_SUBDOMAIN),
         TOOL_PREF(ToolCategory.dns_scanner, dnsenum_tool_name),
         TOOL_CONF(ToolCategory.dns_scanner, dnsenum_tool_name),
         NOT(RateLimitEnable(addr=MATCH.addr)),
         NOT(PublicTarget(addr=MATCH.addr)),
     )
-    def run_dnsenum(self, f1: ScanNeeded, domain: TargetDomain):
-        self._declare_dnsenum(f1, domain)
+    def run_dnsenum(self, f1: ScanNeeded, domain: TargetDomain, wordlist: PreferredWordlist):
+        self._declare_dnsenum(f1, domain, wordlist=wordlist)
 
     @Rule(
         AS.f1 << ScanNeeded(category=ToolCategory.dns_scanner, addr=MATCH.addr, port=53),
         AS.domain << TargetDomain(),
         AS.public << PublicTarget(addr=MATCH.addr),
+        AS.wordlist << PreferredWordlist(category=OPTION_WORDLIST_SUBDOMAIN),
         TOOL_PREF(ToolCategory.dns_scanner, dnsenum_tool_name),
         TOOL_CONF(ToolCategory.dns_scanner, dnsenum_tool_name),
         NOT(RateLimitEnable(addr=MATCH.addr)),
     )
-    def run_dnsenum_public(self, f1: ScanNeeded, domain: TargetDomain, public: PublicTarget):
-        self._declare_dnsenum(f1, domain, public=public)
+    def run_dnsenum_public(self, f1: ScanNeeded, domain: TargetDomain, public: PublicTarget,
+                           wordlist: PreferredWordlist):
+        self._declare_dnsenum(f1, domain, public=public, wordlist=wordlist)
 
     @Rule(
         AS.f1 << ScanNeeded(category=ToolCategory.dns_scanner, addr=MATCH.addr, port=53),
         AS.domain << TargetDomain(),
         AS.ratelimit << RateLimitEnable(addr=MATCH.addr),
+        AS.wordlist << PreferredWordlist(category=OPTION_WORDLIST_SUBDOMAIN),
         TOOL_PREF(ToolCategory.dns_scanner, dnsenum_tool_name),
         TOOL_CONF(ToolCategory.dns_scanner, dnsenum_tool_name),
         NOT(PublicTarget(addr=MATCH.addr)),
     )
-    def run_dnsenum_ratelimit(self, f1: ScanNeeded, domain: TargetDomain, ratelimit: RateLimitEnable):
-        self._declare_dnsenum(f1, domain, ratelimit=ratelimit)
+    def run_dnsenum_ratelimit(self, f1: ScanNeeded, domain: TargetDomain, ratelimit: RateLimitEnable,
+                              wordlist: PreferredWordlist):
+        self._declare_dnsenum(f1, domain, ratelimit=ratelimit, wordlist=wordlist)
 
     @Rule(
         AS.f1 << ScanNeeded(category=ToolCategory.dns_scanner, addr=MATCH.addr, port=53),
         AS.domain << TargetDomain(),
         AS.ratelimit << RateLimitEnable(addr=MATCH.addr),
         AS.public << PublicTarget(addr=MATCH.addr),
+        AS.wordlist << PreferredWordlist(category=OPTION_WORDLIST_SUBDOMAIN),
         TOOL_PREF(ToolCategory.dns_scanner, dnsenum_tool_name),
         TOOL_CONF(ToolCategory.dns_scanner, dnsenum_tool_name),
     )
     def run_dnsenum_public_ratelimit(self, f1: ScanNeeded, domain: TargetDomain, ratelimit: RateLimitEnable,
-                                     public: PublicTarget):
-        self._declare_dnsenum(f1, domain, ratelimit=ratelimit, public=public)
+                                     public: PublicTarget, wordlist: PreferredWordlist):
+        self._declare_dnsenum(f1, domain, ratelimit=ratelimit, public=public, wordlist=wordlist)