squashme: notebooks changes

This includes major edits to the notebooks code to work with the data service.
SwissDataScienceCenter · Aug 27, 2024 · dcc7d03 · dcc7d03
1 parent 6f3b3c0
commit dcc7d03
Show file tree

Hide file tree

Showing 182 changed files with 11,906 additions and 3,866 deletions.
diff --git a/.devcontainer/.poetry_cache/.keep b/.devcontainer/.poetry_cache/.keep
@@ -0,0 +1 @@
+
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -1,33 +1,42 @@
-// For format details, see https://aka.ms/devcontainer.json. For config options, see the
-// README at: https://github.com/devcontainers/templates/tree/main/src/postgres
 {
-	"name": "Python 3 & PostgreSQL",
+	"name": "Data services devcontainer",
 	"dockerComposeFile": "docker-compose.yml",
 	"service": "data_service",
-	"remoteUser": "renku",
-	"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
-	// Features to add to the dev container. More info: https://containers.dev/features.
+	"containerUser": "root",
+	"workspaceFolder": "/workspace",
+	"shutdownAction": "stopCompose",
 	"features": {
 		"ghcr.io/devcontainers-contrib/features/poetry:2": {},
-		"ghcr.io/devcontainers/features/git:1": {},
-		"ghcr.io/jungaretti/features/make:1": {},
-		"ghcr.io/rocker-org/devcontainer-features/apt-packages:1": {
-			"packages": "procps,gcc,python3-dev,curl,ca-certificates,gpg"
-		},
 		"ghcr.io/devcontainers-contrib/features/bash-command:1": {
-			"command": "apt update && apt install -y curl ca-certificates gpg && curl https://apt.fury.io/authzed/gpg.key | apt-key add - && echo 'deb https://apt.fury.io/authzed/ * *' > /etc/apt/sources.list.d/fury.list && apt update && apt install -y zed && curl https://pkg.authzed.com/apt/gpg.key | apt-key add - && echo 'deb https://pkg.authzed.com/apt/ * *' > /etc/apt/sources.list.d/authzed.list && apt update && apt install -y spicedb lsof"
-		}
+			"command": "poetry self add poetry-polylith-plugin"
+		},
+		"ghcr.io/devcontainers/features/docker-in-docker:2": {},
+		"ghcr.io/mpriscella/features/kind:1": {},
+		"ghcr.io/devcontainers-contrib/features/gh-release:1": {
+			"repo": "authzed/zed",
+			"binaryNames": "zed"
+		},
+		"ghcr.io/devcontainers-contrib/features/spicedb:1": {},
+		"ghcr.io/devcontainers/features/kubectl-helm-minikube:1": {
+			"minikube": "none"
+		},
+		"ghcr.io/eitsupi/devcontainer-features/jq-likes:2": {
+			"jqVersion": "latest",
+			"yqVersion": "latest"
+		},
+		"ghcr.io/dhoeric/features/k9s:1": {},
+		"ghcr.io/EliiseS/devcontainer-features/bash-profile:1": {
+			"command": "alias k=kubectl"
+		},
+		"ghcr.io/devcontainers-contrib/features/rclone:1": {}
 	},
-	// Use 'forwardPorts' to make a list of ports inside the container available locally.
-	// This can be used to network with other containers or the host.
-	//"forwardPorts": [8000, 8080, 5432],
-	// Use 'postCreateCommand' to run commands after the container is created.
-	"postCreateCommand": "poetry self add poetry-polylith-plugin && poetry install --with dev && echo 'cd /workspaces/renku-data-services/'>~/.bashrc",
-	// Configure tool-specific properties.
+	"overrideFeatureInstallOrder": [
+		"ghcr.io/devcontainers-contrib/features/poetry",
+		"ghcr.io/devcontainers-contrib/features/bash-command"
+	],
+	"postCreateCommand": "poetry install --with dev",
 	"customizations": {
-		// Configure properties specific to VS Code.
 		"vscode": {
-			// Add the IDs of extensions you want installed when the container is created.
 			"extensions": [
 				"ms-python.python",
 				"ms-python.flake8",
@@ -39,6 +48,4 @@
 			]
 		}
 	}
-	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
-	//"remoteUser": "root"
 }
diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml
@@ -2,30 +2,29 @@ version: '3.8'
 
 services:
   data_service:
-    build:
-      context: ..
-      dockerfile: projects/renku_data_service/Dockerfile
-      args:
-        - USER_UID=${USER_UID:-1000}
-        - USER_GID=${USER_GID:-1000}
+    image: "mcr.microsoft.com/devcontainers/python:3.12-bookworm"
     volumes:
-      - ../..:/workspaces:cached
+      - ../:/workspace:cached
+      - .poetry_cache/:/poetry_cache:delegated
     # Overrides default command so things don't shut down after the process ends.
     entrypoint: sleep infinity
     environment:
       DB_USER: renku
       DB_NAME: renku
       DB_PASSWORD: renku
-      DB_HOST: 127.0.0.1 
+      DB_HOST: 127.0.0.1
       CORS_ALLOW_ALL_ORIGINS: "true"
-      ALEMBIC_CONFIG: /workspaces/renku-data-services/components/renku_data_services/migrations/alembic.ini
-      AUTHZ_DB_HOST: 127.0.0.1 
-      AUTHZ_DB_GRPC_PORT: "50051" 
-      AUTHZ_DB_KEY: renku 
+      ALEMBIC_CONFIG: /workspace/components/renku_data_services/migrations/alembic.ini
+      AUTHZ_DB_HOST: 127.0.0.1
+      AUTHZ_DB_GRPC_PORT: "50051"
+      AUTHZ_DB_KEY: renku
       AUTHZ_DB_NO_TLS_CONNECTION: "true"
       ZED_ENDPOINT: 127.0.0.1:50051
       ZED_TOKEN: renku
       ZED_INSECURE: "true"
+      POETRY_CACHE_DIR: "/poetry_cache"
+      NB_SERVER_OPTIONS__DEFAULTS_PATH: /workspace/server_defaults.json
+      NB_SERVER_OPTIONS__UI_CHOICES_PATH: /workspace/server_options.json
     network_mode: service:db
     depends_on:
       - db
@@ -46,14 +45,15 @@ services:
       - "8080:8080"
       - "5678:5678"
       - "50051:50051"
+      - "8888:80"
 
   swagger:
     image: swaggerapi/swagger-ui
     environment:
       SWAGGER_JSON_URL: http://localhost:8000/api/data/spec.json
       PORT: "8080"
     network_mode: service:db
-  
+
   authz:
     image: authzed/spicedb:latest-debug
     restart: unless-stopped

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -14,3 +14,8 @@ updates:
     poetry:
       patterns:
         - "*"
+- package-ecosystem: "github-actions"
+  directory: "/"
+  schedule:
+    interval: "weekly"
+    day: "sunday"
diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml
@@ -25,13 +25,15 @@ jobs:
       renku-graph: ${{ steps.deploy-comment.outputs.renku-graph}}
       renku-notebooks: ${{ steps.deploy-comment.outputs.renku-notebooks}}
       renku-ui: ${{ steps.deploy-comment.outputs.renku-ui}}
+      amalthea-sessions: ${{ steps.deploy-comment.outputs.amalthea-sessions}}
+      amalthea: ${{ steps.deploy-comment.outputs.amalthea}}
       test-enabled: ${{ steps.deploy-comment.outputs.test-enabled}}
       test-cypress-enabled: ${{ steps.deploy-comment.outputs.test-cypress-enabled}}
       persist: ${{ steps.deploy-comment.outputs.persist}}
       extra-values: ${{ steps.deploy-comment.outputs.extra-values}}
     steps:
       - id: deploy-comment
-        uses: SwissDataScienceCenter/renku-actions/check-pr-description@v1.11.3
+        uses: SwissDataScienceCenter/renku-actions/check-pr-description@v1.12.1
         with:
           string: /deploy
           pr_ref: ${{ github.event.number }}
@@ -44,7 +46,7 @@ jobs:
       name: renku-ci-ds-${{ github.event.number }}
       url: https://renku-ci-ds-${{ github.event.number }}.dev.renku.ch
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Find deplyoment url
         uses: peter-evans/find-comment@v1
         id: deploymentUrlMessage
@@ -61,7 +63,7 @@ jobs:
           body: |
             You can access the deployment of this PR at https://renku-ci-ds-${{ github.event.number }}.dev.renku.ch
       - name: Build and deploy
-        uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.11.3
+        uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.12.1
         env:
           RANCHER_PROJECT_ID: ${{ secrets.CI_RANCHER_PROJECT }}
           DOCKER_PASSWORD: ${{ secrets.RENKU_DOCKER_PASSWORD }}
@@ -84,14 +86,16 @@ jobs:
           renku_graph: "${{ needs.check-deploy.outputs.renku-graph }}"
           renku_notebooks: "${{ needs.check-deploy.outputs.renku-notebooks }}"
           renku_data_services: "@${{ github.head_ref }}"
+          amalthea: "${{ needs.check-deploy.outputs.amalthea }}"
+          amalthea-sessions: "${{ needs.check-deploy.outputs.amalthea-sessions }}"
           extra_values: "${{ needs.check-deploy.outputs.extra-values }}"
 
   selenium-acceptance-tests:
     needs: [check-deploy, deploy-pr]
     if: github.event.action != 'closed' && needs.check-deploy.outputs.pr-contains-string == 'true' && needs.check-deploy.outputs.test-enabled == 'true'
     runs-on: ubuntu-22.04
     steps:
-      - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.11.3
+      - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.12.1
         with:
           kubeconfig: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }}
           renku-release: renku-ci-ds-${{ github.event.number }}
@@ -118,7 +122,7 @@ jobs:
     steps:
       - name: Extract Renku repository reference
         run: echo "RENKU_REFERENCE=`echo '${{ needs.check-deploy.outputs.renku }}' | cut -d'@' -f2`" >> $GITHUB_ENV
-      - uses: SwissDataScienceCenter/renku-actions/test-renku-cypress@v1.11.3
+      - uses: SwissDataScienceCenter/renku-actions/test-renku-cypress@v1.12.1
         with:
           e2e-target: ${{ matrix.tests }}
           renku-reference: ${{ env.RENKU_REFERENCE }}
@@ -146,7 +150,7 @@ jobs:
           body: |
             Tearing down the temporary RenkuLab deplyoment for this PR.
       - name: renku teardown
-        uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.11.3
+        uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.12.1
         env:
           HELM_RELEASE_REGEX: "^renku-ci-ds-${{ github.event.number }}$"
           GITLAB_TOKEN: ${{ secrets.DEV_GITLAB_TOKEN }}

diff --git a/.github/workflows/save_cache.yml b/.github/workflows/save_cache.yml
@@ -0,0 +1,41 @@
+name: Create cache from commits on main
+
+on:
+  push:
+    branches:
+      - main
+      - chore-add-kind
+  workflow_dispatch:
+
+
+jobs:
+  save-poetry-cache:
+    runs-on: ubuntu-latest
+    env:
+      CACHE_KEY: main-branch-poetry-cache-ubuntu
+      CACHE_PATH: .devcontainer/.poetry_cache
+      DEVCONTAINER_IMAGE_CACHE: ghcr.io/swissdatasciencecenter/renku-data-services/devcontainer
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install python deps
+        uses: devcontainers/ci@v0.3
+        with:
+          runCmd: poetry install --with dev
+          push: always
+          skipContainerUserIdUpdate: false
+          imageName: ${{ env.DEVCONTAINER_IMAGE_CACHE }}
+          cacheFrom: ${{ env.DEVCONTAINER_IMAGE_CACHE }}
+      - uses: actions/cache/save@v3
+        name: Create cache
+        with:
+          path: ${{ env.CACHE_PATH }}
+          key: ${{ env.CACHE_KEY }}