Merge pull request #6 from akvo/implement-routes

Implement proxy routes

backend/app/dependencies.py

@@ -0,0 +1,11 @@
+from collections.abc import Callable
+
+from app.s3 import S3Bucket
+
+
+def make_bucket(bucket: str, access_key_id: str, secret_access_key: str) -> S3Bucket:
+    return S3Bucket(bucket, access_key_id, secret_access_key)
+
+
+def bucket_factory() -> Callable[[str, str, str], S3Bucket]:
+    return make_bucket

backend/app/flow_config.py

@@ -2,6 +2,7 @@
 
 import json
 import os
+import re
 import tempfile
 from glob import glob
 from xml.etree import ElementTree
@@ -15,6 +16,10 @@
 AWS_SECRET = "awsSecretKey"
 GCP_CREDENTIAL = "gcpCredentialFile"
 
+instanceUrlPattern = re.compile(
+    r"^(https?://)?(?P<alias>.+)+\.(akvoflow\.org|appspot\.com)$"
+)
+
 
 def populate(*, source: str = SOURCE_PATH, destination: str = CONFIG_FILE) -> None:
     files = glob(f"{source}/*/survey.properties")
@@ -25,11 +30,17 @@ def populate(*, source: str = SOURCE_PATH, destination: str = CONFIG_FILE) -> No
         app_id = _get_app_id(path)
         if not app_id:
             continue
+        matches = instanceUrlPattern.match(props.get("instanceUrl", ""))
+        if not matches:
+            continue
+        alias = matches.group("alias").strip()
+        if not alias:
+            continue
         gcp_credential_file = glob(f"{path}/{app_id}*.json")
         if not gcp_credential_file:
             continue
         props[GCP_CREDENTIAL] = gcp_credential_file[0]
-        configs[app_id] = props
+        configs[alias] = props
     with open(destination, "w") as out:
         json.dump(configs, out)
 
@@ -46,9 +57,7 @@ def get_config(
 
 def refresh(*, source: str = SOURCE_PATH, destination: str = CONFIG_FILE) -> None:
     repo = Git(source)
-    print(repo)
     repo.pull(rebase=True)
-    print(repo.pull.__dict__)
     populate(source=source, destination=destination)
 
 
@@ -65,7 +74,11 @@ def _get_app_id(source_path: str) -> str | None:
     if not isinstance(xml_root, ElementTree.Element):
         return None
     app_element = xml_root.find("{http://appengine.google.com/ns/1.0}application")
-    return app_element.text if isinstance(app_element, ElementTree.Element) else ""
+    return (
+        str(app_element.text).strip()
+        if isinstance(app_element, ElementTree.Element)
+        else None
+    )
 
 
 def _get_xml_root(filename: str) -> ElementTree.Element | None:

backend/app/main.py

@@ -1,8 +1,127 @@
-from fastapi import FastAPI
+from collections.abc import Callable
+from typing import Annotated
 
+from fastapi import Depends, FastAPI, HTTPException, Path, UploadFile, status
+from starlette.responses import StreamingResponse
+
+from app.dependencies import bucket_factory
+from app.flow_config import get_config
+from app.messages import ResultMessage
+from app.s3 import S3Bucket
+
+FormIdParam = Annotated[str, Path(pattern=r"^\d+$")]
 app = FastAPI()
 
 
+def validate_form_id(form_id: str) -> None:
+    # TODO: form id validation
+    if not form_id:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)  # pragma: no cover
+
+
+def get_config_for(instance: str) -> dict[str, str]:
+    config = get_config(instance)
+    if not config:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
+
+    return config
+
+
+async def upload(
+    instance: str,
+    form_id: str,
+    file: UploadFile,
+    folder: str,
+    make_bucket: Callable[[str, str, str], S3Bucket],
+) -> ResultMessage:
+    config = get_config_for(instance)
+    validate_form_id(form_id)
+    bucket = make_bucket(
+        str(config.get("awsBucket")),
+        str(config.get("awsAccessKeyId")),
+        str(config.get("awsSecretKey")),
+    )
+    extra_args = {"ContentType": file.content_type}
+    if folder == "images":
+        extra_args["ACL"] = "public-read"
+    file_key = f"{folder}/{str(file.filename)}"
+    bucket.upload(file.file, file_key, extra_args)
+    return ResultMessage.success("OK!")
+
+
+@app.put("/{instance}/devicezip/{form_id}/", status_code=status.HTTP_201_CREATED)
+async def put_devicezip(
+    instance: str,
+    form_id: FormIdParam,
+    file: UploadFile,
+    make_bucket: Annotated[
+        Callable[[str, str, str], S3Bucket], Depends(bucket_factory)
+    ],
+) -> ResultMessage:
+    return await upload(instance, form_id, file, "devicezip", make_bucket)
+
+
+@app.put("/{instance}/images/{form_id}/", status_code=status.HTTP_201_CREATED)
+async def put_images(
+    instance: str,
+    form_id: FormIdParam,
+    file: UploadFile,
+    make_bucket: Annotated[
+        Callable[[str, str, str], S3Bucket], Depends(bucket_factory)
+    ],
+) -> ResultMessage:
+    return await upload(instance, form_id, file, "images", make_bucket)
+
+
+@app.get("/{instance}/surveys/{form_id}.zip")
+async def get_survey_form(
+    instance: str,
+    form_id: FormIdParam,
+    make_bucket: Annotated[
+        Callable[[str, str, str], S3Bucket], Depends(bucket_factory)
+    ],
+) -> StreamingResponse:
+    config = get_config_for(instance)
+    validate_form_id(form_id)
+    bucket = make_bucket(
+        str(config.get("awsBucket")),
+        str(config.get("awsAccessKeyId")),
+        str(config.get("awsSecretKey")),
+    )
+
+    try:
+        res = bucket.download(f"surveys/{form_id}.zip")
+        return StreamingResponse(
+            content=res["Body"].iter_chunks(), media_type=res["ContentType"]
+        )
+    except Exception as e:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND) from e
+
+
+@app.get("/{instance}/images/{filename}")
+async def get_image(
+    instance: str,
+    filename: str,
+    make_bucket: Annotated[
+        Callable[[str, str, str], S3Bucket], Depends(bucket_factory)
+    ],
+) -> StreamingResponse:
+    config = get_config_for(instance)
+    bucket = make_bucket(
+        str(config.get("awsBucket")),
+        str(config.get("awsAccessKeyId")),
+        str(config.get("awsSecretKey")),
+    )
+
+    try:
+        res = bucket.download(f"images/{filename}")
+        return StreamingResponse(
+            content=res["Body"].iter_chunks(), media_type=res["ContentType"]
+        )
+    except Exception as e:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND) from e
+
+
 @app.get("/healtz", include_in_schema=False)
-async def healt_check() -> dict[str, str]:
-    return {"message": "OK!"}
+async def healt_check() -> ResultMessage:
+    return ResultMessage.success("OK!")

backend/app/messages.py

@@ -0,0 +1,27 @@
+from enum import StrEnum, auto
+from typing import Self
+
+from pydantic import BaseModel
+
+
+class MessageStatus(StrEnum):
+    SUCCESS = auto()
+    FAIL = auto()
+    ERROR = auto()
+
+
+class ResultMessage(BaseModel):
+    status: MessageStatus
+    message: str | None = None
+
+    @classmethod
+    def success(cls, message: str) -> Self:
+        return cls(status=MessageStatus.SUCCESS, message=message)
+
+    @classmethod
+    def fail(cls, message: str) -> Self:
+        return cls(status=MessageStatus.FAIL, message=message)
+
+    @classmethod
+    def error(cls, message: str) -> Self:
+        return cls(status=MessageStatus.ERROR, message=message)

backend/app/s3.py

@@ -0,0 +1,21 @@
+from typing import Any, BinaryIO
+
+import boto3
+
+
+class S3Bucket:
+    def __init__(self, bucket: str, access_key_id: str, secret_access_key: str):
+        self.bucket = bucket
+        self.client = boto3.client(
+            "s3",
+            aws_access_key_id=access_key_id,
+            aws_secret_access_key=secret_access_key,
+        )
+
+    def upload(
+        self, fileobj: BinaryIO, key: str, extra: dict[str, Any] | None = None
+    ) -> None:
+        self.client.upload_fileobj(fileobj, self.bucket, key, ExtraArgs=extra)
+
+    def download(self, key: str):  # type: ignore
+        return self.client.get_object(Bucket=self.bucket, Key=key)

backend/pyproject.toml

@@ -3,14 +3,17 @@ name = "akvo-flow-s3-proxy"
 version = "0.1.0"
 requires-python = ">=3.12"
 dependencies = [
+  "boto3",
   "fastapi",
   "GitPython",
   "google-cloud-datastore",
+  "python-multipart",
   "uvicorn",
 ]
 
 [project.optional-dependencies]
 dev = [
+  "boto3-stubs[s3]",
   "coverage",
   "httpx",
   "ipython",