feat: use cloud storage from amalthea (#387)

Co-authored-by: Samuel Gaist <samuel.gaist@idiap.ch>
SwissDataScienceCenter · Oct 2, 2024 · 8174633 · 8174633
1 parent 319d133
commit 8174633
Show file tree

Hide file tree

Showing 19 changed files with 225 additions and 126 deletions.
diff --git a/bases/renku_data_services/data_api/app.py b/bases/renku_data_services/data_api/app.py
@@ -142,6 +142,7 @@ def register_all_handlers(app: Sanic, config: Config) -> Sanic:
         nb_config=config.nb_config,
         internal_gitlab_authenticator=config.gitlab_authenticator,
         git_repo=config.git_repositories_repo,
+        rp_repo=config.rp_repo,
     )
     notebooks_new = NotebooksNewBP(
         name="notebooks",
@@ -150,6 +151,7 @@ def register_all_handlers(app: Sanic, config: Config) -> Sanic:
         nb_config=config.nb_config,
         project_repo=config.project_repo,
         session_repo=config.session_repo,
+        storage_repo=config.storage_v2_repo,
         rp_repo=config.rp_repo,
         internal_gitlab_authenticator=config.gitlab_authenticator,
     )

diff --git a/components/renku_data_services/notebooks/api.spec.yaml b/components/renku_data_services/notebooks/api.spec.yaml
@@ -1022,11 +1022,12 @@ components:
         storage_id:
           allOf:
             - "$ref": "#/components/schemas/Ulid"
-            - description: If the storage_id is provided then this config must replace an existing storage config in the session
+            - description: The storage ID is used to know which storage config from the DB should be overriden
       required:
         - configuration
         - source_path
         - target_path
+        - storage_id
     ServerName:
       type: string
       minLength: 5

diff --git a/components/renku_data_services/notebooks/api/amalthea_patches/git_sidecar.py b/components/renku_data_services/notebooks/api/amalthea_patches/git_sidecar.py
@@ -22,7 +22,7 @@ async def main(server: "UserServer") -> list[dict[str, Any]]:
     commit_sha = getattr(server, "commit_sha", None)
 
     volume_mount = {
-        "mountPath": server.work_dir.absolute().as_posix(),
+        "mountPath": server.work_dir.as_posix(),
         "name": "workspace",
     }
     if gl_project_path:
@@ -51,7 +51,7 @@ async def main(server: "UserServer") -> list[dict[str, Any]]:
                         "env": [
                             {
                                 "name": "GIT_RPC_MOUNT_PATH",
-                                "value": server.work_dir.absolute().as_posix(),
+                                "value": server.work_dir.as_posix(),
                             },
                             {
                                 "name": "GIT_RPC_PORT",

diff --git a/components/renku_data_services/notebooks/api/amalthea_patches/init_containers.py b/components/renku_data_services/notebooks/api/amalthea_patches/init_containers.py
@@ -35,11 +35,11 @@ async def git_clone_container_v2(server: "UserServer") -> dict[str, Any] | None:
     env = [
         {
             "name": f"{prefix}WORKSPACE_MOUNT_PATH",
-            "value": server.workspace_mount_path.absolute().as_posix(),
+            "value": server.workspace_mount_path.as_posix(),
         },
         {
             "name": f"{prefix}MOUNT_PATH",
-            "value": server.work_dir.absolute().as_posix(),
+            "value": server.work_dir.as_posix(),
         },
         {
             "name": f"{prefix}LFS_AUTO_FETCH",
@@ -134,7 +134,7 @@ async def git_clone_container_v2(server: "UserServer") -> dict[str, Any] | None:
         },
         "volumeMounts": [
             {
-                "mountPath": server.workspace_mount_path.absolute().as_posix(),
+                "mountPath": server.workspace_mount_path.as_posix(),
                 "name": amalthea_session_work_volume,
             },
             *etc_cert_volume_mount,
@@ -161,11 +161,11 @@ async def git_clone_container(server: "UserServer") -> dict[str, Any] | None:
     env = [
         {
             "name": f"{prefix}WORKSPACE_MOUNT_PATH",
-            "value": server.workspace_mount_path.absolute().as_posix(),
+            "value": server.workspace_mount_path.as_posix(),
         },
         {
             "name": f"{prefix}MOUNT_PATH",
-            "value": server.work_dir.absolute().as_posix(),
+            "value": server.work_dir.as_posix(),
         },
         {
             "name": f"{prefix}LFS_AUTO_FETCH",
@@ -260,7 +260,7 @@ async def git_clone_container(server: "UserServer") -> dict[str, Any] | None:
         },
         "volumeMounts": [
             {
-                "mountPath": server.workspace_mount_path.absolute().as_posix(),
+                "mountPath": server.workspace_mount_path.as_posix(),
                 "name": "workspace",
             },
             *etc_cert_volume_mount,

diff --git a/components/renku_data_services/notebooks/api/amalthea_patches/jupyter_server.py b/components/renku_data_services/notebooks/api/amalthea_patches/jupyter_server.py
@@ -43,7 +43,7 @@ def env(server: "UserServer") -> list[dict[str, Any]]:
             "path": "/statefulset/spec/template/spec/containers/0/env/-",
             "value": {
                 "name": "NOTEBOOK_DIR",
-                "value": server.work_dir.absolute().as_posix(),
+                "value": server.work_dir.as_posix(),
             },
         },
         {
@@ -53,7 +53,7 @@ def env(server: "UserServer") -> list[dict[str, Any]]:
             # relative to $HOME.
             "value": {
                 "name": "MOUNT_PATH",
-                "value": server.work_dir.absolute().as_posix(),
+                "value": server.work_dir.as_posix(),
             },
         },
         {
@@ -223,7 +223,7 @@ def rstudio_env_variables(server: "UserServer") -> list[dict[str, Any]]:
                     "path": "/statefulset/spec/template/spec/containers/0/volumeMounts/-",
                     "value": {
                         "name": secret_name,
-                        "mountPath": mount_location.absolute().as_posix(),
+                        "mountPath": mount_location.as_posix(),
                         "subPath": mount_location.name,
                         "readOnly": True,
                     },

diff --git a/components/renku_data_services/notebooks/api/classes/cloud_storage/__init__.py b/components/renku_data_services/notebooks/api/classes/cloud_storage/__init__.py
@@ -6,17 +6,15 @@
 class ICloudStorageRequest(Protocol):
     """The abstract class for cloud storage."""
 
-    exists: bool
     mount_folder: str
-    source_folder: str
-    bucket: str
+    source_path: str
 
     def get_manifest_patch(
         self,
         base_name: str,
         namespace: str,
-        labels: dict[str, str] = {},
-        annotations: dict[str, str] = {},
+        labels: dict[str, str] | None = None,
+        annotations: dict[str, str] | None = None,
     ) -> list[dict[str, Any]]:
         """The patches applied to a jupyter server to insert the storage in the session."""
         ...
diff --git a/components/renku_data_services/notebooks/api/classes/image.py b/components/renku_data_services/notebooks/api/classes/image.py
@@ -4,7 +4,7 @@
 import re
 from dataclasses import dataclass, field
 from enum import Enum
-from pathlib import Path
+from pathlib import PurePosixPath
 from typing import Any, Optional, Self, cast
 
 import requests
@@ -101,7 +101,7 @@ def get_image_config(self, image: "Image") -> Optional[dict[str, Any]]:
             return None
         return cast(dict[str, Any], res.json())
 
-    def image_workdir(self, image: "Image") -> Optional[Path]:
+    def image_workdir(self, image: "Image") -> Optional[PurePosixPath]:
         """Query the docker API to get the workdir of an image."""
         config = self.get_image_config(image)
         if config is None:
@@ -112,7 +112,7 @@ def image_workdir(self, image: "Image") -> Optional[Path]:
         workdir = nested_config.get("WorkingDir", "/")
         if workdir == "":
             workdir = "/"
-        return Path(workdir)
+        return PurePosixPath(workdir)
 
     def with_oauth2_token(self, oauth2_token: str) -> "ImageRepoDockerAPI":
         """Return a docker API instance with the token as authentication."""

diff --git a/components/renku_data_services/notebooks/api/classes/k8s_client.py b/components/renku_data_services/notebooks/api/classes/k8s_client.py
@@ -353,10 +353,13 @@ def __init__(self, url: str, server_type: type[_SessionType]):
         self.url = url
         self.client = httpx.AsyncClient()
         self.server_type: type[_SessionType] = server_type
+        self.url_path_name = "servers"
+        if server_type == AmaltheaSessionV1Alpha1:
+            self.url_path_name = "sessions"
 
     async def list_servers(self, safe_username: str) -> list[_SessionType]:
         """List the jupyter servers."""
-        url = urljoin(self.url, f"/users/{safe_username}/servers")
+        url = urljoin(self.url, f"/users/{safe_username}/{self.url_path_name}")
         try:
             res = await self.client.get(url, timeout=10)
         except httpx.RequestError as err:
@@ -374,7 +377,7 @@ async def list_servers(self, safe_username: str) -> list[_SessionType]:
 
     async def get_server(self, name: str) -> _SessionType | None:
         """Get a specific jupyter server."""
-        url = urljoin(self.url, f"/servers/{name}")
+        url = urljoin(self.url, f"/{self.url_path_name}/{name}")
         try:
             res = await self.client.get(url, timeout=10)
         except httpx.RequestError as err:
@@ -487,10 +490,9 @@ async def delete_server(self, server_name: str, safe_username: str) -> None:
         """Delete the server."""
         server = await self.get_server(server_name, safe_username)
         if not server:
-            raise MissingResourceError(
-                f"Cannot find server {server_name} for user " f"{safe_username} in order to delete it."
-            )
-        return await self.renku_ns_client.delete_server(server_name)
+            return None
+        await self.renku_ns_client.delete_server(server_name)
+        return None
 
     async def patch_tokens(self, server_name: str, renku_tokens: RenkuTokens, gitlab_token: GitlabToken) -> None:
         """Patch the Renku and Gitlab access tokens used in a session."""

diff --git a/components/renku_data_services/notebooks/api/classes/server.py b/components/renku_data_services/notebooks/api/classes/server.py
@@ -3,7 +3,7 @@
 from abc import ABC
 from collections.abc import Sequence
 from itertools import chain
-from pathlib import Path
+from pathlib import PurePosixPath
 from typing import Any
 from urllib.parse import urljoin, urlparse
 
@@ -44,8 +44,8 @@ def __init__(
         user_secrets: K8sUserSecrets | None,
         cloudstorage: Sequence[ICloudStorageRequest],
         k8s_client: K8sClient,
-        workspace_mount_path: Path,
-        work_dir: Path,
+        workspace_mount_path: PurePosixPath,
+        work_dir: PurePosixPath,
         config: _NotebooksConfig,
         internal_gitlab_user: APIUser,
         using_default_image: bool = False,
@@ -205,7 +205,7 @@ async def _get_session_manifest(self) -> dict[str, Any]:
                 "pvc": {
                     "enabled": True,
                     "storageClassName": self.config.sessions.storage.pvs_storage_class,
-                    "mountPath": self.workspace_mount_path.absolute().as_posix(),
+                    "mountPath": self.workspace_mount_path.as_posix(),
                 },
             }
         else:
@@ -214,7 +214,7 @@ async def _get_session_manifest(self) -> dict[str, Any]:
                 "size": storage_size,
                 "pvc": {
                     "enabled": False,
-                    "mountPath": self.workspace_mount_path.absolute().as_posix(),
+                    "mountPath": self.workspace_mount_path.as_posix(),
                 },
             }
         # Authentication
@@ -257,7 +257,7 @@ async def _get_session_manifest(self) -> dict[str, Any]:
                 "jupyterServer": {
                     "defaultUrl": self.server_options.default_url,
                     "image": self.image,
-                    "rootDir": self.work_dir.absolute().as_posix(),
+                    "rootDir": self.work_dir.as_posix(),
                     "resources": self.server_options.to_k8s_resources(
                         enforce_cpu_limits=self.config.sessions.enforce_cpu_limits
                     ),
@@ -378,8 +378,8 @@ def __init__(
         user_secrets: K8sUserSecrets | None,
         cloudstorage: Sequence[ICloudStorageRequest],
         k8s_client: K8sClient,
-        workspace_mount_path: Path,
-        work_dir: Path,
+        workspace_mount_path: PurePosixPath,
+        work_dir: PurePosixPath,
         config: _NotebooksConfig,
         gitlab_project: Project | None,
         internal_gitlab_user: APIUser,
@@ -503,8 +503,8 @@ def __init__(
         user_secrets: K8sUserSecrets | None,
         cloudstorage: Sequence[ICloudStorageRequest],
         k8s_client: K8sClient,
-        workspace_mount_path: Path,
-        work_dir: Path,
+        workspace_mount_path: PurePosixPath,
+        work_dir: PurePosixPath,
         repositories: list[Repository],
         config: _NotebooksConfig,
         internal_gitlab_user: APIUser,