T5182: Email and domain blocking port to v3 (#169)

* add initial email and domain blocking

* add unit test for email blocking and unblocking

* add error codes to responses (frontend still doesn't show the right thing)

* add v3 deploy workflow

* remove redundant build workflow

* add unit test for email blocking

* add /legal and banners, make script runnable directly

* enable tests, add integration test for error codes

* separate poetry call from install

use pipx to install poetry on windows

* exclude skipped tests from coverage

.github/workflows/build_docker.yml

@@ -5,6 +5,7 @@ on:
     branches:
       - "master"
       - "dev"
+      - "dev_v3"
 
   workflow_dispatch:
     inputs:
@@ -76,7 +77,7 @@ jobs:
             CACHE_BUSTER_COMMIT=${{ github.sha }}
 
   dev-deploy:
-    if: github.repository == 'thinkst/canarytokens' && github.ref == 'refs/heads/dev'
+    if: github.repository == 'thinkst/canarytokens' && github.ref == 'refs/heads/dev_v3'
     runs-on: [self-hosted, dev]
     needs: build
     steps:
@@ -86,12 +87,12 @@ jobs:
           ./canarytokensdb_s3backup.sh
 
           cd /home/ubuntu/canarytokens-docker
-          sed "s/thinkst\/canarytokens:dev/thinkst\/canarytokens:${GITHUB_REF##*/}/g" docker-compose-letsencrypt.yml.tpl > docker-compose-letsencrypt.yml
-          sed -i'' "s/CANARY_DEV_BUILD_ID=.*/CANARY_DEV_BUILD_ID=${GITHUB_SHA:0:8}/" frontend.env
+          sed "s/thinkst\/canarytokens:dev_v3/thinkst\/canarytokens:${GITHUB_REF##*/}/g" docker-compose-v3.yml.tpl > docker-compose-v3.yml
+          sed -i'' "s/CANARY_DEV_BUILD_ID=.*/CANARY_DEV_BUILD_ID=${GITHUB_SHA:0:8}/" backend-v3.env
           sudo docker pull thinkst/canarytokens:${GITHUB_REF##*/}
-          sudo docker-compose -f docker-compose-letsencrypt.yml pull
-          sudo docker-compose -f docker-compose-letsencrypt.yml down
-          sudo docker-compose -f docker-compose-letsencrypt.yml up -d
+          sudo docker-compose -f docker-compose-v3.yml pull
+          sudo docker-compose -f docker-compose-v3.yml down
+          sudo docker-compose -f docker-compose-v3.yml up -d
           sudo docker system prune -f -a
 
   staging-deploy:

.github/workflows/test.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - "T4627_py3_main"
+      - "T5182_email_block_list_py3"
 
 jobs:
   tests:
@@ -146,15 +147,18 @@ jobs:
       - uses: actions/checkout@v2
       - name: Install deps
         run: |
-          (Invoke-WebRequest -Uri https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py -UseBasicParsing).Content | python -
-          $env:path = $env:path + ";C:\Users\runneradmin\.poetry\bin"; poetry config virtualenvs.in-project true
+          python -m pip install --user pipx
+          python -m pipx ensurepath
+          python -m pipx install poetry==1.3.2
+          # (Invoke-WebRequest -Uri https://install.python-poetry.org -UseBasicParsing).Content | python -
       - name: Install python dependencies
         # poetry cache clear --all pypi ref: https://stackoverflow.com/questions/72551057/poetry-gives-toomanyindirects-error-suddenly
         # Remove when this is resolved.
         run: |
-          $env:path = $env:path + ";C:\Users\runneradmin\.poetry\bin"; poetry cache clear --all pypi
-          $env:path = $env:path + ";C:\Users\runneradmin\.poetry\bin"; poetry install -E 'twisted web'
+          poetry config virtualenvs.in-project true
+          poetry cache clear --all pypi
+          poetry install -E 'twisted web'
       - name: Integration Tests
         run: |
           $env:LIVE = 'True'
-          $env:path = $env:path + ";C:\Users\runneradmin\.poetry\bin"; poetry run coverage run --source=.\tests\integration -m pytest .\tests\integration\test_custom_binary.py  --runv2
+          poetry run coverage run --source=.\tests\integration -m pytest .\tests\integration\test_custom_binary.py  --runv2

backend/app.py

@@ -27,7 +27,7 @@
 from fastapi.security import APIKeyQuery
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
-from pydantic import HttpUrl, parse_obj_as
+from pydantic import HttpUrl, ValidationError, parse_obj_as
 from sentry_sdk.integrations.asgi import SentryAsgiMiddleware
 from sentry_sdk.integrations.fastapi import FastApiIntegration
 from sentry_sdk.integrations.redis import RedisIntegration
@@ -125,6 +125,8 @@
     add_canary_path_element,
     get_all_canary_domains,
     get_all_canary_sites,
+    is_email_blocked,
+    is_valid_email,
     remove_canary_domain,
     save_canarydrop,
     validate_webhook,
@@ -293,32 +295,69 @@ def generate_page(request: Request) -> HTMLResponse:
     "/generate",
     tags=["Create Canarytokens"],
 )
-async def generate(request: Request) -> AnyTokenResponse:
+async def generate(request: Request) -> AnyTokenResponse:  # noqa: C901  # gen is large
     """
     Whatt
     """
+    response_error = (
+        lambda error, message: JSONResponse(  # noqa: E731  # lambda is cleaner
+            {
+                "error": str(error),
+                "error_message": message,
+                "url": "",
+                "url_components": None,
+                "token": "",
+                "email": "",
+                "hostname": "",
+                "auth": "",
+            }
+        )
+    )
+
     if request.headers.get("Content-Type", "application/json") == "application/json":
-        data = await request.json()
-        token_request_details = parse_obj_as(AnyTokenRequest, data)
+        token_request_data = await request.json()
     else:
         # Need a mutable copy of the form data
-        token_request_form = dict(await request.form())
-        token_request_form["token_type"] = token_request_form.pop(
-            "type", token_request_form.get("token_type", None)
+        token_request_data = dict(await request.form())
+        token_request_data["token_type"] = token_request_data.pop(
+            "type", token_request_data.get("token_type", None)
         )
 
-        token_request_details = parse_obj_as(AnyTokenRequest, token_request_form)
+    try:
+        token_request_details = parse_obj_as(AnyTokenRequest, token_request_data)
+    except ValidationError:  # DESIGN: can we specialise on what went wrong?
+        return response_error(1, "No email/webhook supplied or malformed request")
+
+    if not token_request_details.memo:
+        return response_error(2, "No memo supplied")
 
     if token_request_details.webhook_url:
         try:
             validate_webhook(
                 token_request_details.webhook_url, token_request_details.token_type
             )
         except requests.exceptions.HTTPError:
-            raise HTTPException(status_code=400, detail="Failed to validate webhook")
+            # raise HTTPException(status_code=400, detail="Failed to validate webhook")
+            return response_error(
+                3, "Invalid webhook supplied. Confirm you can POST to this URL."
+            )
         except requests.exceptions.ConnectTimeout:
-            raise HTTPException(
-                status_code=400, detail="Failed to validate webhook - timed out."
+            # raise HTTPException(
+            #     status_code=400, detail="Failed to validate webhook - timed out."
+            # )
+            return response_error(
+                3, "Webhook timed out. Confirm you can POST to this URL."
+            )
+
+    if token_request_details.email:
+        if not is_valid_email(token_request_details.email):
+            return response_error(5, "Invalid email supplied")
+
+        if is_email_blocked(token_request_details.email):
+            # raise HTTPException(status_code=400, detail="Email is blocked.")
+            return response_error(
+                6,
+                "Blocked email supplied. Please see our Acceptable Use Policy at https://canarytokens.org/legal",
             )
     # TODO: refactor this. KUBECONFIG token creates it's own token
     # value and cannot follow same path as before.
@@ -458,6 +497,15 @@ async def settings_post(
         return JSONResponse({"message": "failure"}, status_code=400)
 
 
+@app.get(
+    "/legal",
+    tags=["Canarytokens legal page"],
+    response_class=HTMLResponse,
+)
+def legal_page(request: Request) -> HTMLResponse:
+    return templates.TemplateResponse("legal.html", {"request": request})
+
+
 @app.get(
     "/download",
     tags=["Canarytokens Downloads"],