feat(datasets): user can import datasets though the UI

src/api-client/dataset.js

@@ -39,7 +39,7 @@ export default function addDatasetMethods(client) {
     });
   };
 
-  client.addFilesToDataset = (projectUrl, datasetName, filesList, signal) => {
+  client.addFilesToDataset = (projectUrl, datasetName, filesList) => {
     let headers = client.getBasicHeaders();
     headers.append("Content-Type", "application/json");
     headers.append("X-Requested-With", "XMLHttpRequest");
@@ -90,32 +90,77 @@ export default function addDatasetMethods(client) {
       if (response.data.error !== undefined)
         return response;
 
-      project_id = response.data.result.project_id;
-      return client.clientFetch(`${client.baseUrl}/renku/datasets.create`, {
-        method: "POST",
-        headers: headers,
-        body: JSON.stringify({
-          "dataset_name": renkuDataset.name,
-          "description": renkuDataset.description,
-          "project_id": project_id
-        })
-      });
-
+        project_id = response.data.result.project_id;
+        return client.clientFetch(`${client.baseUrl}/renku/datasets.create`, {
+          method: "POST",
+          headers: headers,
+          body: JSON.stringify({
+            "dataset_name": renkuDataset.name,
+            "description": renkuDataset.description,
+            "project_id": project_id
+          })
+        });
     })
       .then(response => {
         if (response.data.error) { return response; }
         else
-          if (renkuDataset.files.length > 0) {
-            return client.clientFetch(`${client.baseUrl}/renku/datasets.add`, {
-              method: "POST",
-              headers: headers,
-              body: JSON.stringify({
-                "dataset_name": renkuDataset.name,
-                "files": renkuDataset.files,
-                "project_id": project_id
-              })
-            });
-          } return response;
+        if (renkuDataset.files.length > 0) {
+          return client.clientFetch(`${client.baseUrl}/renku/datasets.add`, {
+            method: "POST",
+            headers: headers,
+            body: JSON.stringify({
+              "dataset_name": renkuDataset.name,
+              "files": renkuDataset.files,
+              "project_id": project_id
+            })
+          });
+        } return response;
       });
   };
+
+  client.datasetImport = (projectUrl, renkuDataset) => {
+    let headers = client.getBasicHeaders();
+    headers.append("Content-Type", "application/json");
+    headers.append("X-Requested-With", "XMLHttpRequest");
+
+    let project_id;
+
+    return client.clientFetch(`${client.baseUrl}/renku/cache.project_clone`, {
+      method: "POST",
+      headers: headers,
+      body: JSON.stringify({
+        depth: 1,
+        git_url: projectUrl
+      })
+    }).then(response => {
+      if (response.data.error !== undefined)
+        return response;
+
+        project_id = response.data.result.project_id;
+        //here import
+        return client.clientFetch(`${client.baseUrl}/renku/datasets.import`, {
+          method: "POST",
+          headers: headers,
+          body: JSON.stringify({
+            "dataset_uri": renkuDataset.uri,
+            "project_id": project_id
+          })
+        });
+
+    });
+  };
+
+
+  client.getJobStatus = (job_id) => {
+    let headers = client.getBasicHeaders();
+    headers.append("Content-Type", "application/json");
+    headers.append("X-Requested-With", "XMLHttpRequest");
+
+    return client.clientFetch(`${client.baseUrl}/renku/jobs/${job_id}`, {
+      method: "GET",
+      headers: headers
+    }).then(response => {
+      return response.data.result;
+    });
+  };
 }

src/dataset/Dataset.present.js

@@ -132,14 +132,13 @@ export default function DatasetView(props) {
           }).catch(error => {
             if (fetchError === null) {
               if (!unmounted && error.case === API_ERRORS.notFoundError) {
-                setFetchError(
-                  "Error 404: The dataset that was selected does not exist or could not be accessed." +
-                  "If you just created the dataset try reloading the page."
-                );
-              }
+                setFetchError("Error 404: The dataset that was selected does" +
+                " not exist or could not be accessed. If you just created or" +
+                " imported the dataset try reloading the page.");
+            }
               else if (!unmounted && error.case === API_ERRORS.internalServerError) {
                 setFetchError("Error 500: The dataset that was selected couldn't be fetched.");
-              }
+            }
             }
           });
       }
@@ -156,10 +155,8 @@ export default function DatasetView(props) {
           }).catch(error => {
             if (fetchError === null) {
               if (!unmounted && error.case === API_ERRORS.notFoundError) {
-                setFetchError(
-                  "Error 404: The dataset that was selected does not exist or could not be accessed." +
-                  "If you just created the dataset try reloading the page."
-                );
+                setFetchError("Error 404: The dataset that was selected does not exist or" +
+                +" could not be accessed. If you just created or imported the dataset try reloading the page.");
               }
               else if (!unmounted && error.case === API_ERRORS.internalServerError) {
                 setFetchError("Error 500: The dataset that was selected couldn't be fetched.");
@@ -198,15 +195,12 @@ export default function DatasetView(props) {
     return <Alert color="danger">{fetchError}</Alert>;
   if (dataset === undefined) return <Loader />;
   if (dataset === null) {
-    return (
-      <Alert color="danger">
-        Error 404: The dataset that was selected does not exist or could not
-        be accessed.<br /> <br /> If you just created the dataset
-        try <Button color="danger" size="sm" onClick={
-          () => window.location.reload()
-        }>reloading</Button> the page.</Alert>
-    );
+    return <Alert color="danger">
+      Error 404: The dataset that was selected does not exist or could notbe accessed.<br /> <br />
+      If you just created or imported the dataset try <Button color="danger" size="sm"
+      onClick={()=> window.location.reload()}>reloading</Button> the page.</Alert>;
   }
+
   return <Col>
     <Row>
       <Col md={8} sm={12}>

src/model/RenkuModels.js

@@ -314,6 +314,26 @@ const issueFormSchema = new Schema({
   }
 });
 
+const datasetImportFormSchema = new Schema({
+  uri: {
+    initial: "",
+    name: "uri",
+    label: "URI/DOI",
+    edit: false,
+    help: "This will import the dataset with the DOI (Digital Object Identifier) 10.5281/zenodo.3352150" +
+      "and make it locally available. Dataverse and Zenodo are supported, with DOIs (e.g. 10.5281/zenodo.3352150" +
+      " or doi:10.5281/zenodo.3352150) and full URLs (e.g. http://zenodo.org/record/3352150). A tag with the" +
+      " remote version of the dataset is automatically created.",
+    type: FormGenerator.FieldTypes.TEXT,
+    // parseFun: expression => FormGenerator.Parsers.slugFromTitle(expression),
+    validators: [{
+      id: "uri-length",
+      isValidFun: expression => FormGenerator.Validators.isNotEmpty(expression),
+      alert: "URI is too short"
+    }]
+  }
+});
+
 
 export { userSchema, metaSchema, displaySchema, newProjectSchema, projectSchema, forkProjectSchema };
-export { notebooksSchema, projectsSchema, datasetFormSchema, issueFormSchema };
+export { notebooksSchema, projectsSchema, datasetFormSchema, issueFormSchema, datasetImportFormSchema };

src/project/Project.js

@@ -42,6 +42,7 @@ import Fork from "./fork";
 import ShowDataset from "../dataset/Dataset.container";
 import NewDataset from "./datasets/new/index";
 import EditDataset from "./datasets/edit/index";
+import ImportDataset from "./datasets/import/index";
 
 
 const subRoutes = {
@@ -293,6 +294,7 @@ class View extends Component {
       overviewDatasetsUrl: `${baseUrl}/overview/datasets`,
       datasetsUrl: `${datasetsUrl}`,
       newDatasetUrl: `${datasetsUrl}/new`,
+      importDatasetUrl: `${datasetsUrl}/import`,
       datasetUrl: `${datasetsUrl}/:datasetId`,
       editDatasetUrl: `${datasetsUrl}/:datasetId/modify`,
       issueNewUrl: `${collaborationUrl}/issues/issue_new`,
@@ -466,6 +468,24 @@ class View extends Component {
         httpProjectUrl={httpProjectUrl}
       />,
 
+      importDataset: (p) => <ImportDataset
+        key="datasetnew" {...subProps}
+        progress={graphProgress}
+        maintainer={maintainer}
+        accessLevel={accessLevel}
+        forked={forked}
+        insideProject={true}
+        datasets={datasets}
+        reFetchProject={this.fetchAll.bind(this)}
+        lineagesUrl={subUrls.lineagesUrl}
+        fileContentUrl={subUrls.fileContentUrl}
+        projectsUrl={subUrls.projectsUrl}
+        selectedDataset={p.match.params.datasetId}
+        client={this.props.client}
+        history={this.props.history}
+        httpProjectUrl={httpProjectUrl}
+      />,
+
       mrList: <ConnectedMergeRequestList key="mrList" store={this.projectState.reduxStore}
         mergeRequestsOverviewUrl={subUrls.mergeRequestsOverviewUrl} />,
 

src/project/Project.present.js

@@ -562,6 +562,7 @@ class ProjectDatasetsNav extends Component {
       datasets={this.props.core.datasets}
       datasetsUrl={this.props.datasetsUrl}
       newDatasetUrl={this.props.newDatasetUrl}
+      importDatasetUrl={this.props.importDatasetUrl}
       visibility={this.props.visibility}
     />;
   }
@@ -571,9 +572,11 @@ class ProjectViewDatasets extends Component {
   render() {
     return <Switch>
       <Route exact path={this.props.newDatasetUrl}
-        render={p => this.props.newDataset(p)} />
+          render={p => this.props.newDataset(p)} />
+      <Route exact path={this.props.importDatasetUrl}
+          render={p => this.props.importDataset(p)} />
       <Route path={this.props.editDatasetUrl}
-        render={p => this.props.editDataset(p)} />
+          render={p => this.props.editDataset(p)} />
       <Route path={this.props.datasetsUrl} render={props =>
         <ProjectViewDatasetsList {...this.props} {...props} />} />
     </Switch>;
@@ -594,10 +597,11 @@ class ProjectViewDatasetsList extends Component {
       return <Col sm={12} md={8} lg={10}>
         <Alert timeout={0} color="primary">
           No datasets found for this project. <br /><br />
-          <FontAwesomeIcon icon={faInfoCircle} /> If you recently activated the knowledge graph or
+          <FontAwesomeIcon icon={faInfoCircle} />  If you recently activated the knowledge graph or
           added the datasets try refreshing the page. <br /><br />
-          You can also click on the button to create
-          a <Link className="btn btn-primary btn-sm" to={this.props.newDatasetUrl}>New Dataset</Link>
+          You can also click on the button to create a
+          <Link className="btn btn-primary btn-sm" to={this.props.newDatasetUrl}>New Dataset</Link>
+          or <Link className="btn btn-primary btn-sm" to={this.props.importDatasetUrl}>Import Dataset</Link>
         </Alert>
       </Col>;
     }

src/project/Project.state.js

@@ -208,8 +208,8 @@ class ProjectModel extends StateModel {
   }
 
   fetchProjectDatasets(client) { //from KG
-    if (this.get("transient.requests.datasets") === SpecialPropVal.UPDATING) return;
-    this.setUpdating({ transient: { requests: { datasets: true } } });
+    if (this.get("core.datasets") === SpecialPropVal.UPDATING) return;
+    this.setUpdating({ core: { datasets: true } });
     return client.getProjectDatasetsFromKG(this.get("core.path_with_namespace"))
       .then(datasets => {
         const updatedState = { datasets: datasets, transient: { requests: { datasets: false } } };

src/project/datasets/DatasetsListView.js

@@ -23,19 +23,22 @@ function DatasetListRow(props) {
 
 function CreateDatasetButton(props) {
   if (props.visibility.accessLevel >= ACCESS_LEVELS.MAINTAINER) {
-return <span className="float-right throw-right-in-flex">
-  <UncontrolledButtonDropdown size="sm">
-    <DropdownToggle color="primary" className="alternateToggleStyle" caret>
-      <FontAwesomeIcon icon={faPlus} style={{ color: "white", backgroundColor: "#5561A6" }} />
-    </DropdownToggle>
-    <DropdownMenu right={false}>
-      <DropdownItem>
-        <Link to={props.newDatasetUrl}>New Dataset</Link>
-      </DropdownItem>
-    </DropdownMenu>
-  </UncontrolledButtonDropdown>
-</span>;
-}
+    return <span className="float-right throw-right-in-flex">
+      <UncontrolledButtonDropdown size="sm">
+        <DropdownToggle color="primary" className="alternateToggleStyle" caret>
+          <FontAwesomeIcon icon={faPlus} style={{ color: "white", backgroundColor: "#5561A6" }} />
+        </DropdownToggle>
+        <DropdownMenu right={false}>
+          <DropdownItem>
+            <Link to={props.newDatasetUrl}>New Dataset</Link>
+          </DropdownItem>
+          <DropdownItem>
+            <Link to={props.importDatasetUrl}>Import Dataset</Link>
+          </DropdownItem>
+        </DropdownMenu>
+      </UncontrolledButtonDropdown>
+    </span>;
+  }
   return null;
 }
 
@@ -53,7 +56,10 @@ export default function DatasetsListView(props) {
         <span className="tree-header-title text-truncate">
           Datasets List
         </span>
-        <CreateDatasetButton visibility={props.visibility} newDatasetUrl={props.newDatasetUrl}/>
+        <CreateDatasetButton
+          visibility={props.visibility}
+          newDatasetUrl={props.newDatasetUrl}
+          importDatasetUrl={props.importDatasetUrl}/>
       </div>
       <nav>
         {

src/project/datasets/import/DatasetImport.container.js

@@ -0,0 +1,45 @@
+/*!
+ * Copyright 2020 - Swiss Data Science Center (SDSC)
+ * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+ * Eidgenössische Technische Hochschule Zürich (ETHZ).
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ *  incubator-renku-ui
+ *
+ *  DatasetImport.container.js
+ *  Container components for new dataset.
+ */
+
+import React from "react";
+import { datasetImportFormSchema } from "../../../model/RenkuModels";
+import DatasetImport from "./DatasetImport.present";
+
+function ImportDataset(props) {
+
+  return <DatasetImport
+    datasetImportFormSchema={datasetImportFormSchema}
+    user={props.user}
+    projectPathWithNamespace={props.projectPathWithNamespace}
+    client={props.client}
+    history={props.history}
+    accessLevel={props.accessLevel}
+    reFetchProject={props.reFetchProject}
+    httpProjectUrl={props.httpProjectUrl}
+  />;
+}
+
+
+export default ImportDataset;