feat(datasets): user can import datasets though the UI

src/api-client/dataset.js

@@ -10,6 +10,7 @@ export default function addDatasetMethods(client) {
       queryParams
     })
   }
+
 
   client.uploadFile = (file) => {
     const data = new FormData();
@@ -30,6 +31,7 @@ export default function addDatasetMethods(client) {
     })
   }
 
+
   client.addFilesToDataset = (projectUrl, datasetName, filesList) => {
     let headers = client.getBasicHeaders();
     headers.append('Content-Type', 'application/json');
@@ -43,25 +45,26 @@ export default function addDatasetMethods(client) {
         git_url: projectUrl
       })
     }).then(response => {
-      if(response.data.error) 
+      if (response.data.error)
         return response;
-      else 
-      if(filesList.length > 0){
+      else
+      if (filesList.length > 0) {
         return client.clientFetch(`${client.baseUrl}/renku/datasets.add`, {
           method: 'POST',
           headers: headers,
           body: JSON.stringify({
-            "dataset_name":datasetName,
-            "files":filesList,
-            "project_id":response.data.result.project_id
+            "dataset_name": datasetName,
+            "files": filesList,
+            "project_id": response.data.result.project_id
 
           })
         })
-      } else 
+      } else
         return response;
     })
   }
 
+
   client.postDataset = (projectUrl, renkuDataset) => {
     let headers = client.getBasicHeaders();
     headers.append('Content-Type', 'application/json');
@@ -77,37 +80,84 @@ export default function addDatasetMethods(client) {
         git_url: projectUrl
       })
     }).then(response => {
-      if(response.data.error !== undefined){
+      if (response.data.error !== undefined) {
         return response;
       } else {
-        project_id= response.data.result.project_id;
+        project_id = response.data.result.project_id;
         return client.clientFetch(`${client.baseUrl}/renku/datasets.create`, {
           method: 'POST',
           headers: headers,
           body: JSON.stringify({
-            "dataset_name":renkuDataset.name,
-            "description":renkuDataset.description,
+            "dataset_name": renkuDataset.name,
+            "description": renkuDataset.description,
             "project_id": project_id
           })
         })
-        }
+      }
     })
       .then(response => {
-        if(response.data.error) 
+        if (response.data.error)
           return response;
-        else 
-          if(renkuDataset.files.length > 0){
-            return client.clientFetch(`${client.baseUrl}/renku/datasets.add`, {
-              method: 'POST',
-              headers: headers,
-              body: JSON.stringify({
-                "dataset_name":renkuDataset.name,
-                "files":renkuDataset.files,
-                "project_id": project_id
-              })
+        else
+        if (renkuDataset.files.length > 0) {
+          return client.clientFetch(`${client.baseUrl}/renku/datasets.add`, {
+            method: 'POST',
+            headers: headers,
+            body: JSON.stringify({
+              "dataset_name": renkuDataset.name,
+              "files": renkuDataset.files,
+              "project_id": project_id
             })
-          } else 
-            return response;
+          })
+        } else
+          return response;
       })
-    }
+  }
+
+
+  client.datasetImport = (projectUrl, renkuDataset) => {
+    let headers = client.getBasicHeaders();
+    headers.append('Content-Type', 'application/json');
+    headers.append('X-Requested-With', 'XMLHttpRequest');
+
+    let project_id;
+
+    return client.clientFetch(`${client.baseUrl}/renku/cache.project_clone`, {
+      method: 'POST',
+      headers: headers,
+      body: JSON.stringify({
+        depth: 1,
+        git_url: projectUrl
+      })
+    }).then(response => {
+      if (response.data.error !== undefined) {
+        return response;
+      } else {
+        project_id = response.data.result.project_id;
+        //here import
+        return client.clientFetch(`${client.baseUrl}/renku/datasets.import`, {
+          method: 'POST',
+          headers: headers,
+          body: JSON.stringify({
+            "dataset_uri": renkuDataset.uri,
+            "project_id": project_id
+          })
+        })
+      }
+    })
+  }
+
+
+  client.getJobStatus = (job_id) => {
+    let headers = client.getBasicHeaders();
+    headers.append('Content-Type', 'application/json');
+    headers.append('X-Requested-With', 'XMLHttpRequest');
+
+    return client.clientFetch(`${client.baseUrl}/renku/jobs/${job_id}`, {
+      method: 'GET',
+      headers: headers
+    }).then(response => {
+      return response.data.result;
+    })
+  }
 }

src/dataset/Dataset.present.js

@@ -132,7 +132,7 @@ export default function DatasetView(props){
           }).catch(error => {
             if(fetchError === null){
               if (!unmounted && error.case === API_ERRORS.notFoundError){
-                setFetchError("Error 404: The dataset that was selected does not exist or could not be accessed. If you just created the dataset try reloading the page.");}
+                setFetchError("Error 404: The dataset that was selected does not exist or could not be accessed. If you just created or imported the dataset try reloading the page.");}
               else if(!unmounted && error.case === API_ERRORS.internalServerError){
                 setFetchError("Error 500: The dataset that was selected couldn't be fetched.");}
             }
@@ -148,7 +148,7 @@ export default function DatasetView(props){
           }).catch(error => {
             if(fetchError === null){
               if (!unmounted && error.case === API_ERRORS.notFoundError){
-                setFetchError("Error 404: The dataset that was selected does not exist or could not be accessed. If you just created the dataset try reloading the page.");}
+                setFetchError("Error 404: The dataset that was selected does not exist or could not be accessed. If you just created or imported the dataset try reloading the page.");}
               else if(!unmounted && error.case === API_ERRORS.internalServerError){
                 setFetchError("Error 500: The dataset that was selected couldn't be fetched.");}
             }
@@ -184,7 +184,7 @@ export default function DatasetView(props){
     return <Alert color="danger">{fetchError}</Alert>;
   if(dataset === undefined) return <Loader />;
   if(dataset === null)
-    return <Alert color="danger">Error 404: The dataset that was selected does not exist or could not be accessed.<br /> <br /> If you just created the dataset try <Button color="danger" size="sm" onClick={()=> window.location.reload()}>reloading</Button> the page.</Alert>
+    return <Alert color="danger">Error 404: The dataset that was selected does not exist or could not be accessed.<br /> <br /> If you just created or imported the dataset try <Button color="danger" size="sm" onClick={()=> window.location.reload()}>reloading</Button> the page.</Alert>
 
   return <Col>
     <Row>

src/model/RenkuModels.js

@@ -314,6 +314,23 @@ const issueFormSchema = new Schema({
   }
 })
 
+const datasetImportFormSchema = new Schema({
+  uri: {
+    initial: "",
+    name: 'uri',
+    label: 'URI/DOI',
+    edit: false,
+    help:'This will import the dataset with the DOI (Digital Object Identifier) 10.5281/zenodo.3352150 and make it locally available. Dataverse and Zenodo are supported, with DOIs (e.g. 10.5281/zenodo.3352150 or doi:10.5281/zenodo.3352150) and full URLs (e.g. http://zenodo.org/record/3352150). A tag with the remote version of the dataset is automatically created.',
+    type: FormGenerator.FieldTypes.TEXT,
+    // parseFun: expression => FormGenerator.Parsers.slugFromTitle(expression),
+    validators: [{
+      id: 'uri-length',
+      isValidFun: expression => FormGenerator.Validators.isNotEmpty(expression),
+      alert: 'URI is too short'
+    }]
+  }
+});
+
 
 export { userSchema, metaSchema, displaySchema, newProjectSchema, projectSchema, forkProjectSchema };
-export { notebooksSchema, projectsSchema, datasetFormSchema, issueFormSchema };
+export { notebooksSchema, projectsSchema, datasetFormSchema, issueFormSchema, datasetImportFormSchema };

src/project/Project.js

@@ -41,6 +41,7 @@ import { ShowFile } from '../file';
 import Fork from './fork';
 import ShowDataset from '../dataset/Dataset.container';
 import NewDataset from './datasets/new/index';
+import ImportDataset from './datasets/import/index';
 import EditDataset from './datasets/edit/index';
 
 
@@ -293,6 +294,7 @@ class View extends Component {
       overviewDatasetsUrl: `${baseUrl}/overview/datasets`,
       datasetsUrl: `${datasetsUrl}`,
       newDatasetUrl: `${datasetsUrl}/new`,
+      importDatasetUrl: `${datasetsUrl}/import`,
       datasetUrl: `${datasetsUrl}/:datasetId`,
       editDatasetUrl: `${datasetsUrl}/:datasetId/modify`,
       issueNewUrl: `${collaborationUrl}/issues/issue_new`,
@@ -464,6 +466,24 @@ class View extends Component {
         httpProjectUrl={httpProjectUrl}
       />,
 
+      importDataset: (p) => <ImportDataset
+        key="datasetnew"  {...subProps}
+        progress={graphProgress}
+        maintainer={maintainer}
+        accessLevel={accessLevel}
+        forked={forked}
+        insideProject={true}
+        datasets={datasets}
+        reFetchProject={this.fetchAll.bind(this)}
+        lineagesUrl={subUrls.lineagesUrl}
+        fileContentUrl={subUrls.fileContentUrl}
+        projectsUrl={subUrls.projectsUrl}
+        selectedDataset={p.match.params.datasetId}
+        client={this.props.client}
+        history={this.props.history}
+        httpProjectUrl={httpProjectUrl}
+      />,
+
       mrList: <ConnectedMergeRequestList key="mrList" store={this.projectState.reduxStore}
         mergeRequestsOverviewUrl={subUrls.mergeRequestsOverviewUrl} />,
 

src/project/Project.present.js

@@ -562,6 +562,7 @@ class ProjectDatasetsNav extends Component {
       datasets={this.props.core.datasets}
       datasetsUrl={this.props.datasetsUrl}
       newDatasetUrl={this.props.newDatasetUrl}
+      importDatasetUrl={this.props.importDatasetUrl}
       visibility={this.props.visibility}
     />;
   }
@@ -572,6 +573,8 @@ class ProjectViewDatasets extends Component {
     return <Switch>
         <Route exact path={this.props.newDatasetUrl}
           render={p => this.props.newDataset(p)} />
+        <Route exact path={this.props.importDatasetUrl}
+          render={p => this.props.importDataset(p)} />
         <Route path={this.props.editDatasetUrl}
           render={p => this.props.editDataset(p)} />
         <Route path={this.props.datasetsUrl} render={ props =>
@@ -595,7 +598,7 @@ class ProjectViewDatasetsList extends Component {
         <Alert timeout={0} color="primary">
 					No datasets found for this project. <br /><br />
 					<FontAwesomeIcon icon={faInfoCircle} />  If you recently activated the knowledge graph or added the datasets try refreshing the page. <br /><br />
-					You can also click on the button to create a <Link className="btn btn-primary btn-sm" to={this.props.newDatasetUrl}>New Dataset</Link>
+					You can also click on the button to create a <Link className="btn btn-primary btn-sm" to={this.props.newDatasetUrl}>New Dataset</Link> or <Link className="btn btn-primary btn-sm" to={this.props.importDatasetUrl}>Import Dataset</Link>
         </Alert>
       </Col>;
     }

src/project/datasets/DatasetsListView.js

@@ -32,6 +32,9 @@ function CreateDatasetButton(props){
           <DropdownItem>
             <Link to={props.newDatasetUrl}>New Dataset</Link>
           </DropdownItem>
+          <DropdownItem>
+            <Link to={props.importDatasetUrl}>Import Dataset</Link>
+          </DropdownItem>
         </DropdownMenu>
       </UncontrolledButtonDropdown>
     </span>;
@@ -52,7 +55,7 @@ export default function DatasetsListView(props){
         <span className="tree-header-title text-truncate">
           Datasets List
         </span>
-        <CreateDatasetButton visibility={props.visibility} newDatasetUrl={props.newDatasetUrl}/>
+        <CreateDatasetButton visibility={props.visibility} newDatasetUrl={props.newDatasetUrl} importDatasetUrl={props.importDatasetUrl}/>
       </div> 
       <nav>
         {

src/project/datasets/import/DatasetImport.container.js

@@ -0,0 +1,45 @@
+/*!
+ * Copyright 2020 - Swiss Data Science Center (SDSC)
+ * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+ * Eidgenössische Technische Hochschule Zürich (ETHZ).
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ *  incubator-renku-ui
+ *
+ *  DatasetImport.container.js
+ *  Container components for new dataset.
+ */
+
+import React from 'react';
+import { datasetImportFormSchema } from '../../../model/RenkuModels';
+import DatasetImport from './DatasetImport.present';
+
+function ImportDataset(props) {
+
+  return <DatasetImport
+    datasetImportFormSchema={datasetImportFormSchema}
+    user={props.user}
+    projectPathWithNamespace={props.projectPathWithNamespace}
+    client={props.client}
+    history={props.history}
+    accessLevel={props.accessLevel}
+    reFetchProject={props.reFetchProject}
+    httpProjectUrl={props.httpProjectUrl}
+  />;
+}
+
+
+export default ImportDataset;