From 0facb98c584cd22827d33bf81ab172466ebffcd1 Mon Sep 17 00:00:00 2001 From: Chandrasekhar Ramakrishnan Date: Thu, 27 Jan 2022 15:24:37 +0100 Subject: [PATCH 1/6] minor: more fine-grained handling of errors in notebook start --- client/src/api-client/notebook-servers.js | 7 +++- client/src/notebooks/Notebooks.container.js | 44 +++++++++++++-------- 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/client/src/api-client/notebook-servers.js b/client/src/api-client/notebook-servers.js index 9d0209fae8..4f5e8b418e 100644 --- a/client/src/api-client/notebook-servers.js +++ b/client/src/api-client/notebook-servers.js @@ -112,8 +112,11 @@ function addNotebookServersMethods(client) { }).then(resp => { return resp.data; }).catch(error => { - if (error.errorData && error.errorData.messages && error.errorData.messages.error) - throw new Error(error.errorData.messages.error); + if (error.errorData && error.errorData.messages && error.errorData.messages.error) { + const err = new Error(error.errorData.messages.error); + err.cause = error; + throw err; + } }); }; diff --git a/client/src/notebooks/Notebooks.container.js b/client/src/notebooks/Notebooks.container.js index fb5a6ff99f..e565f719ac 100644 --- a/client/src/notebooks/Notebooks.container.js +++ b/client/src/notebooks/Notebooks.container.js @@ -683,26 +683,36 @@ class StartNotebookServer extends Component { //* To avoid flickering UI, just set a temporary state and display a loading wheel. this.setState({ "starting": true, launchError: null }); this.internalStartServer().catch((error) => { - // Some failures just go away. Try again to see if it works the second time. - setTimeout(() => { - this.internalStartServer().catch((error) => { - // crafting notification - const fullError = `An error occurred when trying to start a new session. - Error message: "${error.message}", Stack trace: "${error.stack}"`; - this.notifications.addError( - this.notifications.Topics.SESSION_START, - "Unable to start the session.", - this.props.location.pathname, "Try again", - null, // always toast - fullError); - this.setState({ "starting": false, launchError: error.message }); - if (this.autostart && !this.state.autostartTried) - this.setState({ autostartTried: true }); - }); - }, 3000); + if (error.cause && error.cause.response && error.cause.response.status) { + if (error.cause.response.status === 500) { + // Some failures just go away. Try again to see if it works the second time. + setTimeout(() => { + this.internalStartServer().catch((error) => { + this.handleNotebookStartError(error); + }); + }, 3000); + } + else { this.handleNotebookStartError(error); } + } + else { this.handleNotebookStartError(error); } }); } + handleNotebookStartError(error) { + // crafting notification + const fullError = `An error occurred when trying to start a new session. + Error message: "${error.message}", Stack trace: "${error.stack}"`; + this.notifications.addError( + this.notifications.Topics.SESSION_START, + "Unable to start the session.", + this.props.location.pathname, "Try again", + null, // always toast + fullError); + this.setState({ "starting": false, launchError: error.message }); + if (this.autostart && !this.state.autostartTried) + this.setState({ autostartTried: true }); + } + toggleMergedBranches() { const currentSetting = this.model.get("filters.includeMergedBranches"); this.coordinator.setMergedBranches(!currentSetting); From 8cb8ca785d6be14652dd155852ba5f47eb10ee64 Mon Sep 17 00:00:00 2001 From: Chandrasekhar Ramakrishnan Date: Thu, 3 Feb 2022 17:34:01 +0100 Subject: [PATCH 2/6] tests: cypress test for new session UI --- .../fixtures/session/pipeline-jobs.json | 90 ++++++++++++++++ e2e/cypress/fixtures/session/pipelines.json | 13 +++ e2e/cypress/fixtures/session/renku.ini | 2 + .../fixtures/session/server-options.json | 46 ++++++++ e2e/cypress/integration/local/session.spec.ts | 48 +++++++++ .../support/renkulab-fixtures/index.ts | 7 +- .../support/renkulab-fixtures/projects.ts | 12 +++ .../support/renkulab-fixtures/session.ts | 100 ++++++++++++++++++ 8 files changed, 315 insertions(+), 3 deletions(-) create mode 100644 e2e/cypress/fixtures/session/pipeline-jobs.json create mode 100644 e2e/cypress/fixtures/session/pipelines.json create mode 100644 e2e/cypress/fixtures/session/renku.ini create mode 100644 e2e/cypress/fixtures/session/server-options.json create mode 100644 e2e/cypress/integration/local/session.spec.ts create mode 100644 e2e/cypress/support/renkulab-fixtures/session.ts diff --git a/e2e/cypress/fixtures/session/pipeline-jobs.json b/e2e/cypress/fixtures/session/pipeline-jobs.json new file mode 100644 index 0000000000..190cf2c615 --- /dev/null +++ b/e2e/cypress/fixtures/session/pipeline-jobs.json @@ -0,0 +1,90 @@ +[ + { + "id": 182909, + "status": "success", + "stage": "build", + "name": "image_build", + "ref": "master", + "tag": false, + "coverage": null, + "allow_failure": false, + "created_at": "2022-01-20T14:14:58.450Z", + "started_at": "2022-01-20T14:15:02.605Z", + "finished_at": "2022-01-20T14:16:03.136Z", + "duration": 60.531174, + "queued_duration": 3.619096, + "user": { + "id": 69, + "username": "e2e", + "name": "E2E User", + "state": "active", + "avatar_url": "https://secure.gravatar.com/avatar/67294be6fc8a46e7dd5ae6a3cf11a0ae?s=80\u0026d=identicon", + "web_url": "https://dev.renku.ch/gitlab/e2e", + "created_at": "2021-12-06T10:28:48.021Z", + "bio": "", + "location": null, + "public_email": "", + "skype": "", + "linkedin": "", + "twitter": "", + "website_url": "", + "organization": null, + "job_title": "", + "pronouns": null, + "bot": false, + "work_information": null, + "followers": 0, + "following": 0, + "local_time": "4:28 PM" + }, + "commit": { + "id": "172a784d465a7bd45bacc165df2b64a591ac6b18", + "short_id": "172a784d", + "created_at": "2022-01-20T14:14:54.000+00:00", + "parent_ids": [], + "title": "service: renku init -n \"local-test-project\" -s \"https://github.com/SwissDataScienceCenter/renku-p...", + "message": "service: renku init -n \"local-test-project\" -s \"https://github.com/SwissDataScienceCenter/renku-p...", + "author_name": "e2e User", + "author_email": "e2e@renku.ch", + "authored_date": "2022-01-20T14:14:54.000+00:00", + "committer_name": "renku 0.16.2", + "committer_email": "https://github.com/swissdatasciencecenter/renku-python/tree/v0.16.2", + "committed_date": "2022-01-20T14:14:54.000+00:00", + "trailers": {}, + "web_url": "https://dev.renku.ch/gitlab/e2e/local-test-project/-/commit/172a784d465a7bd45bacc165df2b64a591ac6b18" + }, + "pipeline": { + "id": 182743, + "project_id": 39646, + "sha": "172a784d465a7bd45bacc165df2b64a591ac6b18", + "ref": "master", + "status": "success", + "source": "push", + "created_at": "2022-01-20T14:14:58.405Z", + "updated_at": "2022-01-20T14:16:03.339Z", + "web_url": "https://dev.renku.ch/gitlab/e2e/local-test-project/-/pipelines/182743" + }, + "web_url": "https://dev.renku.ch/gitlab/e2e/local-test-project/-/jobs/182909", + "artifacts": [ + { + "file_type": "trace", + "size": 5360, + "filename": "job.log", + "file_format": null + } + ], + "runner": { + "id": 76, + "description": "gitlab/gl-runner-0", + "ip_address": "10.42.31.240", + "active": true, + "is_shared": true, + "runner_type": "instance_type", + "name": "gitlab-runner", + "online": false, + "status": "offline" + }, + "artifacts_expire_at": null, + "tag_list": [] + } +] diff --git a/e2e/cypress/fixtures/session/pipelines.json b/e2e/cypress/fixtures/session/pipelines.json new file mode 100644 index 0000000000..62b574ef80 --- /dev/null +++ b/e2e/cypress/fixtures/session/pipelines.json @@ -0,0 +1,13 @@ +[ + { + "id": 182743, + "project_id": 39646, + "sha": "172a784d465a7bd45bacc165df2b64a591ac6b18", + "ref": "master", + "status": "success", + "source": "push", + "created_at": "2022-01-20T14:14:58.405Z", + "updated_at": "2022-01-20T14:16:03.339Z", + "web_url": "https://dev.renku.ch/gitlab/e2e/local-test-project/-/pipelines/182743" + } +] diff --git a/e2e/cypress/fixtures/session/renku.ini b/e2e/cypress/fixtures/session/renku.ini new file mode 100644 index 0000000000..5a3d87dcf4 --- /dev/null +++ b/e2e/cypress/fixtures/session/renku.ini @@ -0,0 +1,2 @@ +[renku "interactive"] +default_url = /lab diff --git a/e2e/cypress/fixtures/session/server-options.json b/e2e/cypress/fixtures/session/server-options.json new file mode 100644 index 0000000000..94449e3489 --- /dev/null +++ b/e2e/cypress/fixtures/session/server-options.json @@ -0,0 +1,46 @@ +{ + "cpu_request": { + "default": 0.1, + "displayName": "Number of CPUs", + "options": [0.1, 0.5], + "order": 2, + "type": "enum" + }, + "defaultUrl": { + "default": "/lab", + "displayName": "Default Environment", + "options": ["/lab", "/rstudio"], + "order": 1, + "type": "enum" + }, + "disk_request": { + "default": "1G", + "displayName": "Amount of Storage", + "options": ["1G", "4G", "16G"], + "order": 3, + "type": "enum" + }, + "gpu_request": { + "default": 0, + "displayName": "Number of GPUs", + "options": [0], + "order": 5, + "type": "enum" + }, + "lfs_auto_fetch": { + "default": false, + "displayName": "Automatically fetch LFS data", + "order": 6, + "type": "boolean" + }, + "mem_request": { + "default": "1G", + "displayName": "Amount of Memory", + "options": ["1G", "2G"], + "order": 4, + "type": "enum" + }, + "s3mounts": { + "enabled": true + } +} diff --git a/e2e/cypress/integration/local/session.spec.ts b/e2e/cypress/integration/local/session.spec.ts new file mode 100644 index 0000000000..c0447e29c6 --- /dev/null +++ b/e2e/cypress/integration/local/session.spec.ts @@ -0,0 +1,48 @@ +/// +/*! + * Copyright 2022 - Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Fixtures from "../../support/renkulab-fixtures"; + +describe("launch sessions", () => { + const fixtures = new Fixtures(cy); + beforeEach(() => { + fixtures.config().versions().userTest(); + fixtures.projects().landingUserProjects().projectTest(); + fixtures.projectMigrationUpToDate(); + fixtures.sessionAutosave().sessionServersEmpty().renkuIni(); + fixtures.sessionPipelines(); + cy.visit("/projects/e2e/local-test-project"); + }); + + it("displays new session page", () => { + fixtures.sessionServerOptions(); + cy.visit("/projects/e2e/local-test-project/sessions/new"); + cy.contains("Do you want to select the branch, commit, or image?").should( + "be.visible" + ); + }); + + it("displays cloud storage options", () => { + fixtures.sessionServerOptions(true); + cy.visit("/projects/e2e/local-test-project/sessions/new"); + cy.contains( + "Do you want to select the branch, commit, or image, or configure cloud storage?" + ).should("be.visible"); + }); +}); diff --git a/e2e/cypress/support/renkulab-fixtures/index.ts b/e2e/cypress/support/renkulab-fixtures/index.ts index a34666d5a1..9498f0e0ca 100644 --- a/e2e/cypress/support/renkulab-fixtures/index.ts +++ b/e2e/cypress/support/renkulab-fixtures/index.ts @@ -20,10 +20,11 @@ * Common fixtures defined in one place. */ import BaseFixtures from "./fixtures"; -import { User } from "./user"; -import { Projects } from "./projects"; import { Datasets } from "./datasets"; +import { Projects } from "./projects"; +import { Session } from "./session"; +import { User } from "./user"; -const Fixtures = Datasets(Projects(User(BaseFixtures))); +const Fixtures = Datasets(Projects(Session(User(BaseFixtures)))); export default Fixtures; diff --git a/e2e/cypress/support/renkulab-fixtures/projects.ts b/e2e/cypress/support/renkulab-fixtures/projects.ts index d09b806ad1..93e572a82d 100644 --- a/e2e/cypress/support/renkulab-fixtures/projects.ts +++ b/e2e/cypress/support/renkulab-fixtures/projects.ts @@ -103,6 +103,7 @@ function Projects(Parent: T) { projectTestContents( names = { coreServiceVersionName: "getCoreServiceVersion", + coreService8VersionName: "getCoreService8Version", projectBranchesName: "getProjectBranches", projectCommitsName: "getProjectCommits", projectReadmeCommits: "getProjectReadmeCommits", @@ -112,6 +113,7 @@ function Projects(Parent: T) { ) { const { coreServiceVersionName, + coreService8VersionName, projectBranchesName, projectCommitsName, readmeName @@ -146,6 +148,14 @@ function Projects(Parent: T) { supported_project_version: coreVersion } } + }).as(coreService8VersionName); + cy.intercept("/ui-server/api/renku/9/version", { + body: { + result: { + latest_version: "1.0.4", + supported_project_version: 9.0 + } + } }).as(coreServiceVersionName); return this; } @@ -153,6 +163,7 @@ function Projects(Parent: T) { projectTest( names = { coreServiceVersionName: "getCoreServiceVersion", + coreService8VersionName: "getCoreService8Version", projectBranchesName: "getProjectBranches", projectCommitsName: "getProjectCommits", projectName: "getProject", @@ -175,6 +186,7 @@ function Projects(Parent: T) { projectTestObserver( names = { coreServiceVersionName: "getCoreServiceVersion", + coreService8VersionName: "getCoreService8Version", projectBranchesName: "getProjectBranches", projectCommitsName: "getProjectCommits", projectName: "getProject", diff --git a/e2e/cypress/support/renkulab-fixtures/session.ts b/e2e/cypress/support/renkulab-fixtures/session.ts new file mode 100644 index 0000000000..d66cca5687 --- /dev/null +++ b/e2e/cypress/support/renkulab-fixtures/session.ts @@ -0,0 +1,100 @@ +/*! + * Copyright 2022 - Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { FixturesConstructor } from "./fixtures"; + +/** + * Fixtures for Sessions + */ + +function Session(Parent: T) { + return class SessionFixtures extends Parent { + renkuIni(name = "getRenkuIni") { + cy.intercept( + // eslint-disable-next-line max-len + "/ui-server/api/projects/e2e%2Flocal-test-project/repository/files/.renku%2Frenku.ini/raw?ref=172a784d465a7bd45bacc165df2b64a591ac6b18", + { + fixture: "session/renku.ini" + } + ).as(name); + return this; + } + + sessionAutosave(name = "getSessionAutosave") { + cy.intercept( + "/ui-server/api/notebooks/e2e%2Flocal-test-project/autosave", + { + body: { + autosaves: [], + pvsSupport: true + } + } + ).as(name); + return this; + } + + sessionPipelines( + names = { + sessionPipelineJobsName: "sessionPipelineJobsName", + sessionPipelinesName: "getSessionPipelines" + } + ) { + const { sessionPipelineJobsName, sessionPipelinesName } = names; + cy.intercept( + "/ui-server/api/projects/e2e%2Flocal-test-project/pipelines?sha=172a784d465a7bd45bacc165df2b64a591ac6b18", + { + fixture: "session/pipelines.json" + } + ).as(sessionPipelinesName); + cy.intercept( + "/ui-server/api/projects/e2e%2Flocal-test-project/pipelines/182743/jobs", + { + fixture: "session/pipeline-jobs.json" + } + ).as(sessionPipelineJobsName); + return this; + } + + sessionServersEmpty(name = "getSessionServers") { + cy.intercept( + //"/ui-server/api/notebooks/servers?namespace=e2e&project=local-test-project", + "/ui-server/api/notebooks/servers?namespace=e2e&project=local-test-project*", + { + body: { servers: {} } + } + ).as(name); + return this; + } + + sessionServerOptions(cloudStorage?, name = "getSessionServerOptions") { + cy.fixture("session/server-options.json").then((options) => { + if (cloudStorage == null) delete options["s3mounts"]; + else if (!cloudStorage) options["s3mounts"] = false; + + cy.intercept( + "GET", + "/ui-server/api/notebooks/server_options", + options + ).as(name); + }); + return this; + } + }; +} + +export { Session }; From 6c14de158f72b03989d8a86d02d616d6451420fd Mon Sep 17 00:00:00 2001 From: Chandrasekhar Ramakrishnan Date: Thu, 27 Jan 2022 14:49:21 +0100 Subject: [PATCH 3/6] feat: support mounting cloud storage in sessions (#1499) (#1518) --- client/.eslintrc.json | 1 + client/src/model/RenkuModels.js | 2 +- client/src/notebooks/NotebookStart.present.js | 32 +- client/src/notebooks/Notebooks.container.js | 17 +- client/src/notebooks/Notebooks.state.js | 10 +- .../notebooks/ObjectStoresConfig.present.js | 301 ++++++++++++++++++ .../fixtures/session/server-options.json | 6 +- e2e/cypress/integration/local/session.spec.ts | 1 + .../support/renkulab-fixtures/session.ts | 6 +- 9 files changed, 361 insertions(+), 15 deletions(-) create mode 100644 client/src/notebooks/ObjectStoresConfig.present.js diff --git a/client/.eslintrc.json b/client/.eslintrc.json index b64e5897dc..d03ded51e1 100644 --- a/client/.eslintrc.json +++ b/client/.eslintrc.json @@ -167,6 +167,7 @@ "checkbox", "ckeditor", "cktextarea", + "cloudstorage", "codemirror", "craco", "dagre", diff --git a/client/src/model/RenkuModels.js b/client/src/model/RenkuModels.js index 45a544c032..c9684d171d 100644 --- a/client/src/model/RenkuModels.js +++ b/client/src/model/RenkuModels.js @@ -362,7 +362,7 @@ const notebooksSchema = new Schema({ commit: { initial: {} }, discard: { initial: false }, options: { initial: {} }, - + objectStoresConfiguration: { initial: [] }, includeMergedBranches: { initial: false }, displayedCommits: { initial: 25 }, } diff --git a/client/src/notebooks/NotebookStart.present.js b/client/src/notebooks/NotebookStart.present.js index ad6e32131e..cc0b349639 100644 --- a/client/src/notebooks/NotebookStart.present.js +++ b/client/src/notebooks/NotebookStart.present.js @@ -29,7 +29,6 @@ import { faInfoCircle, faLink, faRedo, faSyncAlt, } from "@fortawesome/free-solid-svg-icons"; -import { NotebooksHelper } from "./index"; import { StatusHelper } from "../model/Model"; import { InfoAlert, SuccessAlert, WarnAlert } from "../utils/components/Alert"; import { ButtonWithMenu } from "../utils/components/Button"; @@ -40,12 +39,17 @@ import { Loader } from "../utils/components/Loader"; import { ThrottledTooltip } from "../utils/components/Tooltip"; import { Url } from "../utils/helpers/url"; import Time from "../utils/helpers/Time"; +import { NotebooksHelper } from "./index"; +import { ObjectStoresConfigurationButton, ObjectStoresConfigurationModal } from "./ObjectStoresConfig.present"; // * StartNotebookServer code * // function StartNotebookServer(props) { - const { deleteAutosave, setCommit, setIgnorePipeline, toggleShowAdvanced } = props.handlers; + const { autosaves, autoStarting, pipelines, message, showObjectStoreModal } = props; const { branch, commit } = props.filters; - const { autosaves, autoStarting, pipelines, message } = props; + const { objectStoresConfiguration } = props.filters; + const { deleteAutosave, setCommit, setIgnorePipeline, toggleShowAdvanced } = props.handlers; + const { toggleShowObjectStoresConfigModal } = props.handlers; + if (autoStarting) return (); @@ -66,10 +70,15 @@ function StartNotebookServer(props) { (
{message}
) : null; const disabled = fetching.branches || fetching.commits; + const s3MountsConfig = props.options.global.cloudstorage?.s3; + const cloudStorageAvailable = s3MountsConfig?.enabled ?? false; + const showAdvancedMessage = cloudStorageAvailable ? + "Do you want to select the branch, commit, or image, or configure cloud storage?" : + "Do you want to select the branch, commit, or image?"; const buttonMessage = props.showAdvanced ? - "Hide branch, commit, and image settings" : - "Do you want to select the branch, commit, or image?"; + "Hide advanced settings" : + showAdvancedMessage; const advancedSelection = ( @@ -81,6 +90,19 @@ function StartNotebookServer(props) { {show.pipelines ? : null} + {cloudStorageAvailable ? + + + + : + null + } + + ; +} + +/** + * Check if the endpoint is valid. + * @param {object} cloudStoreConfig + */ +function isCloudStorageEndpointValid(cloudStoreConfig) { + return (cloudStoreConfig["endpoint"].length > 0); +} + +/** + * Check if the bucket is valid. + * @param {object} cloudStoreConfig + */ +function isCloudStorageBucketValid(cloudStoreConfig) { + return (cloudStoreConfig["bucket"].length > 0); +} + +function ObjectStoreRow({ credentials, index, onChangeValue, onDeleteValue }) { + + function changeHandler(field) { + return (e) => onChangeValue(index, field, e.target.value); + } + const validationState = { + endpoint: isCloudStorageEndpointValid(credentials), + bucket: isCloudStorageBucketValid(credentials), + }; + + return + + + + { + (validationState.endpoint) ? + null : + Please enter an endpoint + } + + + + + + { + (validationState.bucket) ? + null : + Please enter a bucket + } + + + + + + + + + + + + ; +} + +function emptyObjectStoreCredentials() { + return { bucket: "", endpoint: "", access_key: "", secret_key: "" }; +} + +function ObjectStoresTable({ objectStoresConfiguration, onChangeValue, onDeleteValue }) { + + return + + + + + + + + + + + { + objectStoresConfiguration.map((cl, i) => { + return ; + }) + } + +
EndpointBucket NameAccess KeySecret Key
; +} + +/** + * Check if the storesConfig are valid. Return true if so, false if not. + * @param {array} storesConfig + */ +function validateStoresConfig(storesConfig) { + for (const cs of storesConfig) { + if (!isCloudStorageEndpointValid(cs)) return false; + if (!isCloudStorageBucketValid(cs)) return false; + } + return true; +} + +function filterConfig(storesConfig) { + const keys = ["bucket", "endpoint", "access_key", "secret_key"]; + // remove any rows that contain only empty values + const filteredConfig = storesConfig.toJS().filter((cs) => { + const nonEmpty = keys.map((k) => cs[k].length > 0); + return nonEmpty.some((e) => e); + }).map((cs) => { + // remove tailing spaces + keys.map((k) => cs[k] = cs[k].trim()); + return cs; + }); + return filteredConfig; +} + +function saveValidStoresConfig(storesConfig, setObjectStoresConfiguration, + setSaveStatusMessage, toggleShowObjectStoresConfigModal) { + const filteredConfig = filterConfig(storesConfig).filter((cs) => { + // remove invalid rows that contain only empty values + if (!isCloudStorageEndpointValid(cs)) return false; + if (!isCloudStorageBucketValid(cs)) return false; + return true; + }); + + setObjectStoresConfiguration(filteredConfig); + setSaveStatusMessage(""); + toggleShowObjectStoresConfigModal(); +} + + +function saveStoresConfig(storesConfig, setObjectStoresConfiguration, + setSaveStatusMessage, toggleShowObjectStoresConfigModal) { + // const keys = ["bucket", "endpoint", "access_key", "secret_key"]; + // remove any rows that contain only empty values + const filteredConfig = filterConfig(storesConfig); + if (!validateStoresConfig(filteredConfig)) { + setSaveStatusMessage("Please fix all credentials before saving."); + return; + } + + // TODO check that the credentials work on save: this needs ui-server support because of CORS + // if (filteredConfig.length > 0) { + // const cs = filteredConfig[0]; + // const s3Params = { + // apiVersion: "2006-03-01", + // endpoint: cs["endpoint"] + // }; + // if (cs["access_key"] != null) { + // s3Params["accessKeyId"] = cs["access_key"]; + // s3Params["secretAccessKey"] = cs["secret_key"]; + // } + // const s3 = new S3(s3Params); + // s3.listObjects({ Bucket: cs["bucket"] }, (err, data) => { + // if (err) console.log(err, err.stack); + // else console.log(data); + // }); + // } + + setObjectStoresConfiguration(filteredConfig); + setSaveStatusMessage(""); + toggleShowObjectStoresConfigModal(); +} + +function ObjectStoresConfigurationModal({ objectStoresConfiguration, showObjectStoreModal, + toggleShowObjectStoresConfigModal, setObjectStoresConfiguration }) { + const [storesConfig, setStoresConfig] = useState([]); + useEffect(() => { + const initialCredentials = (objectStoresConfiguration.length > 0) ? + List(objectStoresConfiguration) : + List([emptyObjectStoreCredentials()]); + setStoresConfig(initialCredentials); + }, [objectStoresConfiguration]); + const onChangeValue = (index, field, value) => { + const old = Map(storesConfig.get(index)); + const newElt = old.set(field, value).toJS(); + setStoresConfig(storesConfig.set(index, newElt)); + }; + const onDeleteValue = (index) => { + let c = storesConfig.remove(index); + if (c.size < 1) c = List([emptyObjectStoreCredentials()]); + setStoresConfig(c); + }; + const onAddValue = () => { + setStoresConfig(storesConfig.push(emptyObjectStoreCredentials())); + }; + + const onClose = () => { + saveValidStoresConfig(storesConfig, setObjectStoresConfiguration, + setSaveStatusMessage, toggleShowObjectStoresConfigModal); + }; + + const [saveStatusMessage, setSaveStatusMessage] = useState(""); + const onSave = () => { + saveStoresConfig(storesConfig, setObjectStoresConfiguration, + setSaveStatusMessage, toggleShowObjectStoresConfigModal); + }; + + return
+ + Object Store Configuration + +

+ Provide credentials to use cloud storage like {" "} + AWS S3, Google Cloud Storage, or Azure Blob Storage. +

+ + + + {saveStatusMessage} + + + + +
; +} + + +export { ObjectStoresConfigurationButton, ObjectStoresConfigurationModal }; diff --git a/e2e/cypress/fixtures/session/server-options.json b/e2e/cypress/fixtures/session/server-options.json index 94449e3489..b6d6d87065 100644 --- a/e2e/cypress/fixtures/session/server-options.json +++ b/e2e/cypress/fixtures/session/server-options.json @@ -40,7 +40,9 @@ "order": 4, "type": "enum" }, - "s3mounts": { - "enabled": true + "cloudstorage": { + "s3": { + "enabled": true + } } } diff --git a/e2e/cypress/integration/local/session.spec.ts b/e2e/cypress/integration/local/session.spec.ts index c0447e29c6..43a16e662f 100644 --- a/e2e/cypress/integration/local/session.spec.ts +++ b/e2e/cypress/integration/local/session.spec.ts @@ -41,6 +41,7 @@ describe("launch sessions", () => { it("displays cloud storage options", () => { fixtures.sessionServerOptions(true); cy.visit("/projects/e2e/local-test-project/sessions/new"); + cy.wait("@getSessionPipelineJobsName", { timeout: 10000 }); cy.contains( "Do you want to select the branch, commit, or image, or configure cloud storage?" ).should("be.visible"); diff --git a/e2e/cypress/support/renkulab-fixtures/session.ts b/e2e/cypress/support/renkulab-fixtures/session.ts index d66cca5687..11b8f2f401 100644 --- a/e2e/cypress/support/renkulab-fixtures/session.ts +++ b/e2e/cypress/support/renkulab-fixtures/session.ts @@ -50,7 +50,7 @@ function Session(Parent: T) { sessionPipelines( names = { - sessionPipelineJobsName: "sessionPipelineJobsName", + sessionPipelineJobsName: "getSessionPipelineJobsName", sessionPipelinesName: "getSessionPipelines" } ) { @@ -83,8 +83,8 @@ function Session(Parent: T) { sessionServerOptions(cloudStorage?, name = "getSessionServerOptions") { cy.fixture("session/server-options.json").then((options) => { - if (cloudStorage == null) delete options["s3mounts"]; - else if (!cloudStorage) options["s3mounts"] = false; + if (cloudStorage == null) delete options["cloudstorage"]; + else if (!cloudStorage) options["cloudstorage"]["s3"] = false; cy.intercept( "GET", From ab2596b2e75148257d973c53b8ea2a88b50f3721 Mon Sep 17 00:00:00 2001 From: Chandrasekhar Ramakrishnan Date: Thu, 27 Jan 2022 14:49:21 +0100 Subject: [PATCH 4/6] feat: support mounting cloud storage in sessions (#1499) (#1518) --- client/src/notebooks/Notebooks.state.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/client/src/notebooks/Notebooks.state.js b/client/src/notebooks/Notebooks.state.js index ec982e9dcc..58bb288c09 100644 --- a/client/src/notebooks/Notebooks.state.js +++ b/client/src/notebooks/Notebooks.state.js @@ -928,8 +928,11 @@ class NotebooksCoordinator { startServer() { const options = { serverOptions: this.model.get("filters.options"), - s3mounts: this.model.get("filters.objectStoresConfiguration") }; + const cloudstorage = this.model.get("filters.objectStoresConfiguration"); + if (cloudstorage.length > 0) + options["cloudstorage"] = cloudstorage; + const filters = this.model.get("filters"); const namespace = filters.namespace; const project = filters.project; From 40a10082bc2ba2a052d3b78789dcb42a6a5f90ca Mon Sep 17 00:00:00 2001 From: Chandrasekhar Ramakrishnan Date: Wed, 16 Feb 2022 11:32:26 +0100 Subject: [PATCH 5/6] tests: fix project tests by removing unnecessary wait --- client/.eslintrc.json | 2 + .../notebooks/ObjectStoresConfig.present.js | 75 +++++++++++++------ e2e/cypress/integration/local/project.spec.ts | 3 - 3 files changed, 54 insertions(+), 26 deletions(-) diff --git a/client/.eslintrc.json b/client/.eslintrc.json index d03ded51e1..75ce3ef50d 100644 --- a/client/.eslintrc.json +++ b/client/.eslintrc.json @@ -161,6 +161,7 @@ "autosuggest", "backend", "bool", + "borderless", "cancellable", "cancelled", "cheatsheet", @@ -223,6 +224,7 @@ "noreferrer", "nowrap", "nullable", + "objectstores", "onloadend", "papermill", "pathname", diff --git a/client/src/notebooks/ObjectStoresConfig.present.js b/client/src/notebooks/ObjectStoresConfig.present.js index 46bba0ccae..ac979733b2 100644 --- a/client/src/notebooks/ObjectStoresConfig.present.js +++ b/client/src/notebooks/ObjectStoresConfig.present.js @@ -66,6 +66,19 @@ function isCloudStorageEndpointValid(cloudStoreConfig) { return (cloudStoreConfig["endpoint"].length > 0); } +function EndpointMessage({ validationState }) { + if (!validationState.endpoint) return Please enter an endpoint; + return (validationState.bucket) ? + Data mounted at: : + null; +} + +function BucketMessage({ credentials, validationState }) { + return (validationState.bucket) ? + /cloudstorage/{credentials.bucket} : + Please enter an bucket; +} + /** * Check if the bucket is valid. * @param {object} cloudStoreConfig @@ -84,18 +97,14 @@ function ObjectStoreRow({ credentials, index, onChangeValue, onDeleteValue }) { bucket: isCloudStorageBucketValid(credentials), }; - return + return - { - (validationState.endpoint) ? - null : - Please enter an endpoint - } + @@ -105,11 +114,7 @@ function ObjectStoreRow({ credentials, index, onChangeValue, onDeleteValue }) { id={`s3-bucket-${index}`} name="bucket" bsSize="sm" value={credentials.bucket} onChange={changeHandler("bucket")} invalid={!validationState.bucket} /> - { - (validationState.bucket) ? - null : - Please enter a bucket - } + @@ -139,7 +144,7 @@ function emptyObjectStoreCredentials() { function ObjectStoresTable({ objectStoresConfiguration, onChangeValue, onDeleteValue }) { - return + return
@@ -166,11 +171,20 @@ function ObjectStoresTable({ objectStoresConfiguration, onChangeValue, onDeleteV * @param {array} storesConfig */ function validateStoresConfig(storesConfig) { + const invalidConfigs = {}; + const bucketNamesMap = {}; for (const cs of storesConfig) { - if (!isCloudStorageEndpointValid(cs)) return false; - if (!isCloudStorageBucketValid(cs)) return false; + if (!isCloudStorageEndpointValid(cs)) invalidConfigs[cs.bucket] = cs; + else if (!isCloudStorageBucketValid(cs)) invalidConfigs[cs.bucket] = cs; + if (bucketNamesMap[cs.bucket]) bucketNamesMap[cs.bucket].push(cs); + else bucketNamesMap[cs.bucket] = [cs]; } - return true; + + const conflictingConfigs = {}; + Object.keys(bucketNamesMap).forEach((k) => { + if (bucketNamesMap[k].length > 1) conflictingConfigs[k] = bucketNamesMap[k]; + }); + return { invalidConfigs, conflictingConfigs }; } function filterConfig(storesConfig) { @@ -189,10 +203,12 @@ function filterConfig(storesConfig) { function saveValidStoresConfig(storesConfig, setObjectStoresConfiguration, setSaveStatusMessage, toggleShowObjectStoresConfigModal) { - const filteredConfig = filterConfig(storesConfig).filter((cs) => { - // remove invalid rows that contain only empty values - if (!isCloudStorageEndpointValid(cs)) return false; - if (!isCloudStorageBucketValid(cs)) return false; + // remove any rows that contain only empty values + let filteredConfig = filterConfig(storesConfig); + const validatedStores = validateStoresConfig(filteredConfig); + filteredConfig = filteredConfig.filter((cs) => { + if (validatedStores.conflictingConfigs[cs.bucket]) return false; + if (validatedStores.invalidConfigs[cs.bucket]) return false; return true; }); @@ -207,7 +223,16 @@ function saveStoresConfig(storesConfig, setObjectStoresConfiguration, // const keys = ["bucket", "endpoint", "access_key", "secret_key"]; // remove any rows that contain only empty values const filteredConfig = filterConfig(storesConfig); - if (!validateStoresConfig(filteredConfig)) { + const validatedStores = validateStoresConfig(filteredConfig); + const conflictingBucketNames = Object.keys(validatedStores.conflictingConfigs); + if (conflictingBucketNames.length > 0) { + const namesStr = conflictingBucketNames.join(", "); + setSaveStatusMessage(Bucket names must be unique; multiple rows share {namesStr}.); + return; + } + + const invalidBucketNames = Object.keys(validatedStores.invalidConfigs); + if (invalidBucketNames.length > 0) { setSaveStatusMessage("Please fix all credentials before saving."); return; } @@ -238,6 +263,10 @@ function saveStoresConfig(storesConfig, setObjectStoresConfiguration, function ObjectStoresConfigurationModal({ objectStoresConfiguration, showObjectStoreModal, toggleShowObjectStoresConfigModal, setObjectStoresConfiguration }) { const [storesConfig, setStoresConfig] = useState([]); + const setStoresConfigAndClearMessage = (value) => { + setStoresConfig(value); + setSaveStatusMessage(""); + }; useEffect(() => { const initialCredentials = (objectStoresConfiguration.length > 0) ? List(objectStoresConfiguration) : @@ -247,12 +276,12 @@ function ObjectStoresConfigurationModal({ objectStoresConfiguration, showObjectS const onChangeValue = (index, field, value) => { const old = Map(storesConfig.get(index)); const newElt = old.set(field, value).toJS(); - setStoresConfig(storesConfig.set(index, newElt)); + setStoresConfigAndClearMessage(storesConfig.set(index, newElt)); }; const onDeleteValue = (index) => { let c = storesConfig.remove(index); if (c.size < 1) c = List([emptyObjectStoreCredentials()]); - setStoresConfig(c); + setStoresConfigAndClearMessage(c); }; const onAddValue = () => { setStoresConfig(storesConfig.push(emptyObjectStoreCredentials())); @@ -286,7 +315,7 @@ function ObjectStoresConfigurationModal({ objectStoresConfiguration, showObjectS setCredentials={setStoresConfig} /> - {saveStatusMessage} + {saveStatusMessage}
Endpoint