T5212 Port adding request headers and GET/POST arguments to webhooks …

…to v3 (#199)

* add request headers and args to webhooks

* minor changes and catch-ups with v2

.github/workflows/test.yml

@@ -4,7 +4,7 @@ on:
   push:
     branches:
       - "T4627_py3_main"
-      - "T5234_v3_deploy_workflow"
+      - "T5212_webhook_post_args"
 
 jobs:
   tests:

.pre-commit-config.yaml

@@ -18,7 +18,7 @@ repos:
     -   id: check-yaml
     -   id: debug-statements
     -   id: no-commit-to-branch
-        # Github only allows branch protection for teams or enterprise.
+        # GitHub only allows branch protection for teams or enterprise.
         args: ['--pattern', '^(?!T\d+.*)']
 -   repo: https://github.com/psf/black
     rev: 22.3.0

canarytokens/channel.py

@@ -262,14 +262,15 @@ def format_webhook_canaryalert(
         TokenAlertDetailsSlack, TokenAlertDetailGeneric, TokenAlertDetailsGoogleChat
     ]:
         # TODO: Need to add `host` and `protocol` that can be used to manage the token.
+        slack_hook_base_url = "https://hooks.slack.com"
         googlechat_hook_base_url = "https://chat.googleapis.com"
         details = cls.gather_alert_details(
             canarydrop,
             protocol=protocol,
             host=host,
         )
         if canarydrop.alert_webhook_url and (
-            "https://hooks.slack.com" in canarydrop.alert_webhook_url
+            str(canarydrop.alert_webhook_url).startswith(slack_hook_base_url)
         ):
             return format_as_slack_canaryalert(details=details)
         elif canarydrop.alert_webhook_url and (

canarytokens/models.py

@@ -926,6 +926,8 @@ class AdditionalInfo(BaseModel):
     # TODO: split this off - additional info can be handled separately.
     #       See `AWSKeyAdditionalInfo`
     mysql_client: Optional[dict[str, list[str]]]
+    r: Optional[list[str]]
+    l: Optional[list[str]]
 
     def serialize_for_v2(self) -> dict:
         data = json_safe_dict(self)
@@ -1129,6 +1131,8 @@ class SQLServerTokenHit(TokenHit):
 class WebBugTokenHit(TokenHit):
     token_type: Literal[TokenTypes.WEB] = TokenTypes.WEB
     useragent: str
+    request_headers: Optional[dict[str, str]]
+    request_args: Optional[dict[str, str]]
     additional_info: AdditionalInfo = AdditionalInfo()
 
     class Config:
@@ -1361,9 +1365,20 @@ def get_additional_data_for_notification(self) -> Dict[str, str]:
             Dict[str, str]: Key value pairs to include in webhook / email info.
         """
         latest_hit = self.latest_hit()
-        if latest_hit is None:
-            return {}
-        return {"useragent": latest_hit.useragent}
+        additional_data = {}
+        if latest_hit is not None:
+            additional_data["useragent"] = latest_hit.useragent
+            additional_data["location"] = (
+                latest_hit.additional_info.l if latest_hit.additional_info.l else None
+            )
+            additional_data["referer"] = (
+                latest_hit.additional_info.r if latest_hit.additional_info.r else None
+            )
+            additional_data["request_headers"] = latest_hit.request_headers
+            additional_data["request_args"] = (
+                latest_hit.request_args if latest_hit.request_args else {}
+            )
+        return additional_data
 
 
 class ClonedWebTokenHistory(TokenHistory[ClonedWebTokenHit]):
@@ -1580,9 +1595,14 @@ def add_widgets(self, widgets_info: Optional[Dict[str, str]] = {}) -> None:
         for (label, text) in widgets_info.items():
             if not label or not text:
                 continue
+            message_text = (
+                json.dumps(text) if isinstance(text, dict) else "{}".format(text)
+            )
             self.widgets.append(
                 GoogleChatWidget(
-                    decoratedText=GoogleChatDecoratedText(topLabel=label, text=text)
+                    decoratedText=GoogleChatDecoratedText(
+                        topLabel=label, text=message_text
+                    )
                 )
             )
 

canarytokens/tokens.py

@@ -325,12 +325,27 @@ def _grab_http_general_info(request: Request):
         src_ip_chain = [o.strip() for o in src_ips.split(",")]
         # TODO: 'ts_key' -> which tokens fire this?
         hit_time = request.args.get("ts_key", [datetime.utcnow().strftime("%s")])[0]
+        flatten_singletons = lambda l: l[0] if len(l) == 1 else l  # noqa: E731
+        request_headers = {
+            k.decode(): flatten_singletons([s.decode() for s in v])
+            for k, v in request.requestHeaders.getAllRawHeaders()
+        }
+        request_args = (
+            {
+                k.decode(): ",".join([s.decode() for s in v])
+                for k, v in request.args.items()
+            }
+            if isinstance(request.args, dict)
+            else {}
+        )
         return {
             "useragent": useragent,
             "x_forwarded_for": src_ip_chain,
             "src_ip": src_ip,
             "time_of_hit": hit_time,
             "is_tor_relay": is_tor_relay,
+            "request_headers": request_headers,
+            "request_args": request_args,
         }
 
     @staticmethod
@@ -442,7 +457,7 @@ def _get_response_for_slow_redirect(
     @staticmethod
     def _get_info_for_web(request):
         http_general_info = Canarytoken._grab_http_general_info(request=request)
-        return http_general_info, {"useragent": http_general_info["useragent"]}
+        return http_general_info, {}
 
     @staticmethod
     def _get_response_for_web(

tests/units/test_http_channel.py

@@ -65,11 +65,11 @@ def test_channel_http_GET(setup_db, settings, frontend_settings, token_type):
         (TokenTypes.FAST_REDIRECT, {}),
         (
             TokenTypes.SLOW_REDIRECT,
-            {b"l": ["https://test.com"], b"r": ["https://test.com"]},
+            {b"l": [b"https://test.com"], b"r": [b"https://test.com"]},
         ),
         (
             TokenTypes.CLONEDSITE,
-            {b"l": ["https://test.com"], b"r": ["https://test.com"]},
+            {b"l": [b"https://test.com"], b"r": [b"https://test.com"]},
         ),
     ],
 )

tests/units/test_models.py

@@ -488,7 +488,13 @@ def test_all_requests_have_a_response():
                 "useragent": "python 3.10",
                 "geo_info": GeoIPBogonInfo(ip="127.0.0.1", bogon=True),
             },
-            {"useragent": "python 3.10"},
+            {
+                "location": None,
+                "referer": None,
+                "request_args": {},
+                "request_headers": None,
+                "useragent": "python 3.10",
+            },
         ),
         (
             Log4ShellTokenHistory,