feat: Several optimizations for valkey

- Add vertical autoscaler to track resource usage of valkey
- Increase TTL for valkey data
- Clear cache when git model is updated
- Mount data and config properly to valkey
- Fallback to traditional loading if saving or loading in valkey fails

.gitignore

@@ -17,3 +17,4 @@ certs/*
 /helm/charts/*
 /logs/*
 .env
+autoscaler

Makefile

@@ -152,6 +152,12 @@ create-cluster: registry
 	kubectl cluster-info
 	kubectl config set-context --current --namespace=$(NAMESPACE)
 
+install-vpa:
+	git clone https://github.com/kubernetes/autoscaler.git
+	cd autoscaler/vertical-pod-autoscaler
+	./hack/vpa-up.sh
+	kubectl --namespace=kube-system get pods | grep vpa
+
 delete-cluster:
 	k3d cluster list $(CLUSTER_NAME) 2>&- && k3d cluster delete $(CLUSTER_NAME)
 

backend/Makefile

@@ -5,6 +5,9 @@ DB_PORT = 5432
 DB_PASSWORD = dev
 DB_USER = dev
 DB_NAME = dev
+
+VALKEY_PASSWORD ?= password
+
 VENV = .venv
 
 HOST ?= 127.0.0.1
@@ -34,11 +37,16 @@ database:
 		postgres
 
 valkey:
-	docker start valkey || \
+	TMP_FILE=$$(mktemp)
+	echo "requirepass $(VALKEY_PASSWORD)" > $$TMP_FILE
+	docker start capella-collab-valkey || \
 	docker run -d \
-		--name valkey \
+		--name capella-collab-valkey \
 		-p $(VALKEY_PORT):6379 \
-		valkey/valkey:latest
+		-v $$TMP_FILE:/usr/local/etc/valkey/valkey.conf \
+		valkey/valkey:latest \
+		valkey-server \
+		/usr/local/etc/valkey/valkey.conf
 
 app:
 	if [ -d "$(VENV)/bin" ]; then

backend/capellacollab/config/models.py

@@ -289,7 +289,9 @@ class DatabaseConfig(BaseConfig):
 
 
 class ValkeyConfig(BaseConfig):
-    url: str = pydantic.Field(default="valkey://localhost:6379/0")
+    url: str = pydantic.Field(
+        default="valkey://default:password@localhost:6379/0"
+    )
 
 
 class InitialConfig(BaseConfig):

backend/capellacollab/projects/toolmodels/diagrams/routes.py

@@ -45,6 +45,7 @@ async def get_diagram_metadata(
         job_id, last_updated, diagram_metadata_entries = (
             await handler.get_file_or_artifact(
                 trusted_file_path="diagram_cache/index.json",
+                logger=logger,
                 job_name="update_capella_diagram_cache",
                 file_revision=f"diagram-cache/{handler.revision}",
             )
@@ -91,6 +92,7 @@ async def get_diagram(
     try:
         file_or_artifact = await handler.get_file_or_artifact(
             trusted_file_path=file_path,
+            logger=logger,
             job_name="update_capella_diagram_cache",
             job_id=job_id,
             file_revision=f"diagram-cache/{handler.revision}",

backend/capellacollab/projects/toolmodels/modelbadge/routes.py

@@ -40,7 +40,9 @@ async def get_model_complexity_badge(
 ):
     try:
         file_or_artifact = await git_handler.get_file_or_artifact(
-            "model-complexity-badge.svg", "generate-model-badge"
+            trusted_file_path="model-complexity-badge.svg",
+            job_name="generate-model-badge",
+            logger=logger,
         )
         return responses.SVGResponse(content=file_or_artifact[2])
     except Exception:

backend/capellacollab/projects/toolmodels/modelsources/git/crud.py

@@ -63,12 +63,10 @@ def update_git_model(
     git_model: models.DatabaseGitModel,
     put_model: models.PutGitModel,
 ) -> models.DatabaseGitModel:
+    git_model.path = put_model.path
     git_model.entrypoint = put_model.entrypoint
     git_model.revision = put_model.revision
-
-    if put_model.path != git_model.path:
-        git_model.path = put_model.path
-        git_model.repository_id = None
+    git_model.repository_id = None
 
     if put_model.password:
         git_model.username = put_model.username

backend/capellacollab/projects/toolmodels/modelsources/git/handler/cache.py

@@ -2,36 +2,52 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import datetime
+import logging
+
+import valkey.exceptions
 
 from capellacollab.core import database
 
+DEFAULT_TTL = datetime.timedelta(days=90)
+
 
 class GitValkeyCache:
-    def __init__(self, git_model_id: int) -> None:
+    def __init__(
+        self,
+        git_model_id: int,
+    ) -> None:
         self._valkey = database.get_valkey()
         self.git_model_id = git_model_id
         super().__init__()
 
     def get_file_data(
-        self, file_path: str, revision: str
+        self, file_path: str, revision: str, logger: logging.LoggerAdapter
     ) -> tuple[datetime.datetime, bytes] | None:
-        file_data = self._valkey.hmget(
-            name=self._get_file_key(file_path, revision),
-            keys=["last_updated", "content"],
-        )
+        try:
+            file_data = self._valkey.hmget(
+                name=self._get_file_key(file_path, revision),
+                keys=["last_updated", "content"],
+            )
+        except valkey.exceptions.ValkeyError:
+            logger.exception("Failed to load file data from valkey")
+            return None
         if (last_update := file_data[0]) and (content := file_data[1]):
             last_update = datetime.datetime.fromisoformat(last_update)
             return last_update, content
 
         return None
 
     def get_artifact_data(
-        self, job_id: str, file_path: str
+        self, job_id: str, file_path: str, logger: logging.LoggerAdapter
     ) -> tuple[datetime.datetime, bytes] | None:
-        artifact_data = self._valkey.hmget(
-            name=self._get_artifact_key(job_id, file_path),
-            keys=["started_at", "content"],
-        )
+        try:
+            artifact_data = self._valkey.hmget(
+                name=self._get_artifact_key(job_id, file_path),
+                keys=["started_at", "content"],
+            )
+        except valkey.exceptions.ValkeyError:
+            logger.exception("Failed to load artifact data from valkey")
+            return None
         if (started_at := artifact_data[0]) and (content := artifact_data[1]):
             started_at = datetime.datetime.fromisoformat(started_at)
             return started_at, content
@@ -44,34 +60,44 @@ def put_file_data(
         last_updated: datetime.datetime,
         content: bytes,
         revision: str,
-        ttl: int = 3600,
+        logger: logging.LoggerAdapter,
     ) -> None:
-        self._valkey.hset(
-            name=self._get_file_key(file_path, revision),
-            mapping={
-                "last_updated": last_updated.isoformat(),
-                "content": content,
-            },
-        )
-        self._valkey.expire(
-            name=self._get_file_key(file_path, revision), time=ttl
-        )
+        try:
+            self._valkey.hset(
+                name=self._get_file_key(file_path, revision),
+                mapping={
+                    "last_updated": last_updated.isoformat(),
+                    "content": content,
+                },
+            )
+            self._valkey.expire(
+                name=self._get_file_key(file_path, revision), time=DEFAULT_TTL
+            )
+        except valkey.exceptions.ValkeyError:
+            logger.exception("Failed to save file data to valkey")
 
     def put_artifact_data(
         self,
         job_id: str,
         file_path: str,
         started_at: datetime.datetime,
         content: bytes,
-        ttl: int = 3600,
+        logger: logging.LoggerAdapter,
     ) -> None:
-        self._valkey.hset(
-            name=self._get_artifact_key(job_id, file_path),
-            mapping={"started_at": started_at.isoformat(), "content": content},
-        )
-        self._valkey.expire(
-            name=self._get_artifact_key(job_id, file_path), time=ttl
-        )
+        try:
+            self._valkey.hset(
+                name=self._get_artifact_key(job_id, file_path),
+                mapping={
+                    "started_at": started_at.isoformat(),
+                    "content": content,
+                },
+            )
+            self._valkey.expire(
+                name=self._get_artifact_key(job_id, file_path),
+                time=DEFAULT_TTL,
+            )
+        except valkey.exceptions.ValkeyError:
+            logger.exception("Failed to save artifact data to valkey")
 
     def clear(self) -> None:
         for key in self._valkey.scan_iter(match=f"{self.git_model_id}:*"):

backend/capellacollab/projects/toolmodels/modelsources/git/handler/handler.py

@@ -5,6 +5,7 @@
 
 import abc
 import datetime
+import logging
 
 import requests
 
@@ -118,7 +119,10 @@ def get_started_at_for_job(self, job_id: str) -> datetime.datetime:
         """
 
     async def get_file(
-        self, trusted_file_path: str, revision: str | None = None
+        self,
+        trusted_file_path: str,
+        logger: logging.LoggerAdapter,
+        revision: str | None = None,
     ) -> tuple[datetime.datetime, bytes]:
         if not revision:
             revision = self.revision
@@ -127,15 +131,17 @@ async def get_file(
             trusted_file_path, revision
         )
 
-        if file_data := self.cache.get_file_data(trusted_file_path, revision):
+        if file_data := self.cache.get_file_data(
+            trusted_file_path, revision, logger
+        ):
             last_updated_cache, content_cache = file_data
 
             if last_updated == last_updated_cache:
                 return last_updated_cache, content_cache
 
         content = self.get_file_from_repository(trusted_file_path, revision)
         self.cache.put_file_data(
-            trusted_file_path, last_updated, content, revision
+            trusted_file_path, last_updated, content, revision, logger
         )
 
         return last_updated, content
@@ -144,6 +150,7 @@ async def get_artifact(
         self,
         trusted_file_path: str,
         job_name: str,
+        logger: logging.LoggerAdapter,
         job_id: str | None = None,
     ) -> tuple[str, datetime.datetime, bytes]:
         if not job_id:
@@ -154,13 +161,13 @@ async def get_artifact(
             started_at = self.get_started_at_for_job(job_id)
 
         if artifact_data := self.cache.get_artifact_data(
-            job_id, trusted_file_path
+            job_id, trusted_file_path, logger
         ):
             return job_id, artifact_data[0], artifact_data[1]
 
         content = self.get_artifact_from_job(job_id, trusted_file_path)
         self.cache.put_artifact_data(
-            job_id, trusted_file_path, started_at, content
+            job_id, trusted_file_path, started_at, content, logger
         )
 
         return job_id, started_at, content
@@ -169,17 +176,22 @@ async def get_file_or_artifact(
         self,
         trusted_file_path: str,
         job_name: str,
+        logger: logging.LoggerAdapter,
         job_id: str | None = None,
         file_revision: str | None = None,
     ) -> tuple[str | None, datetime.datetime, bytes]:
         if not job_id:
             try:
-                file = await self.get_file(trusted_file_path, file_revision)
+                file = await self.get_file(
+                    trusted_file_path, logger, file_revision
+                )
                 return (None, file[0], file[1])
             except (
                 requests.HTTPError,
                 git_exceptions.GitRepositoryFileNotFoundError,
             ):
                 pass
 
-        return await self.get_artifact(trusted_file_path, job_name, job_id)
+        return await self.get_artifact(
+            trusted_file_path, job_name, logger, job_id
+        )

backend/capellacollab/projects/toolmodels/modelsources/git/routes.py

@@ -144,6 +144,7 @@ def update_git_model_by_id(
     db: orm.Session = fastapi.Depends(database.get_db),
 ) -> models.DatabaseGitModel:
     git_util.verify_path_prefix(db, put_git_model.path)
+    cache.GitValkeyCache(git_model_id=db_git_model.id).clear()
     return crud.update_git_model(db, db_git_model, put_git_model)
 
 
@@ -175,5 +176,5 @@ def delete_git_model_by_id(
 ):
     if backups_crud.get_pipelines_for_git_model(db, db_git_model):
         raise exceptions.GitRepositoryUsedInPipelines(db_git_model.id)
-
+    cache.GitValkeyCache(git_model_id=db_git_model.id).clear()
     crud.delete_git_model(db, db_git_model)

backend/pyproject.toml

@@ -170,6 +170,7 @@ disable = [
   "too-few-public-methods",
   "too-many-ancestors",
   "too-many-arguments",
+  "too-many-positional-arguments",
   "too-many-boolean-expressions",
   "too-many-branches",
   "too-many-instance-attributes",

backend/tests/projects/toolmodels/conftest.py

@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import datetime
-import typing as t
+import logging
 
 import pytest
 import responses
@@ -25,12 +25,12 @@ def __init__(self, *args, **kwargs) -> None:
             super().__init__()
 
         def get_file_data(
-            self, file_path: str, revision: str
+            self, file_path: str, revision: str, logger: logging.LoggerAdapter
         ) -> tuple[datetime.datetime, bytes] | None:
             return MockGitValkeyCache.cache.get(f"f:{file_path}", None)
 
         def get_artifact_data(
-            self, job_id: str, file_path: str
+            self, job_id: str, file_path: str, logger: logging.LoggerAdapter
         ) -> tuple[datetime.datetime, bytes] | None:
             return MockGitValkeyCache.cache.get(f"a:{file_path}:{job_id}")
 
@@ -40,7 +40,7 @@ def put_file_data(
             last_updated: datetime.datetime,
             content: bytes,
             revision: str,
-            ttl: int = 3600,
+            logger: logging.LoggerAdapter,
         ) -> None:
             MockGitValkeyCache.cache[f"f:{file_path}"] = (
                 last_updated,
@@ -53,7 +53,7 @@ def put_artifact_data(
             file_path: str,
             started_at: datetime.datetime,
             content: bytes,
-            ttl: int = 3600,
+            logger: logging.LoggerAdapter,
         ) -> None:
             MockGitValkeyCache.cache[f"a:{file_path}:{job_id}"] = (
                 started_at,