Query for datasets using RTK-Query ( #2415) (#2453)

* refactor: prepare to migrate to rtk-query for dataset
* refactor: fetch dataset files using rtk query
* refactor: simplify ProjectDatasetView logic
* refactor: fetch datasets from KG using rtk query

client/.eslintrc.json

@@ -19,7 +19,13 @@
     "plugin:react/recommended",
     "plugin:jest/recommended"
   ],
-  "plugins": ["react", "react-hooks", "jest", "spellcheck", "@typescript-eslint"],
+  "plugins": [
+    "react",
+    "react-hooks",
+    "jest",
+    "spellcheck",
+    "@typescript-eslint"
+  ],
   "settings": {
     "react": {
       "version": "detect"
@@ -224,6 +230,7 @@
           "Ravey",
           "rect",
           "redux",
+          "reduxjs",
           "remark2rehype",
           "renderer",
           "renderers",

client/src/dataset/Dataset.present.js

@@ -46,16 +46,21 @@ import { EntityDeleteButtonButton } from "../components/entities/Buttons";
 
 function DisplayFiles(props) {
   if (!props.files || !props.files?.hasPart) return null;
-  if (props.files?.fetching || props.loadingDatasets) return "LOADING FILES...";
+  if (props.isFilesFetching || props.loadingDatasets) return "LOADING FILES...";
 
 
-  if (props.files.fetchError !== null) {
-    const error = props.files.fetchError;
+  if (props.filesFetchError != null) {
+    const error = props.filesFetchError;
     let errorObject;
-    if (CoreError.isValid(error))
+    if (CoreError.isValid(error)) {
       errorObject = (<CoreErrorAlert error={error}/>);
-    else
-      errorObject = (<span><strong>Error fetching dataset files:</strong> {props.files.fetchError.message}</span>);
+    }
+    else {
+      errorObject = (<span>
+        <strong>Error fetching dataset files:</strong>{" "}
+        {JSON.stringify(props.filesFetchError)}
+      </span>);
+    }
 
     return (
       <Card key="datasetDetails" className="border-rk-light mb-4">
@@ -419,6 +424,8 @@ export default function DatasetView(props) {
         fileContentUrl={props.fileContentUrl}
         lineagesUrl={props.lineagesUrl}
         files={props.files}
+        isFilesFetching={props.isFilesFetching}
+        filesFetchError={props.filesFetchError}
         insideProject={props.insideProject}
         loadingDatasets={props.loadingDatasets}
       />

client/src/features/project/Project.d.ts

@@ -1,22 +1,121 @@
-type IDataset = {
-  created_at: string; // could be created?
-  creators: string[];
+/*!
+ * Copyright 2023 - Swiss Data Science Center (SDSC)
+ * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+ * Eidgenössische Technische Hochschule Zürich (ETHZ).
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+interface Creator {
+  affiliation: string | null;
+  email: string;
+}
+
+type DatasetAbstract = {
+  annotations: string[];
   description: string;
-  exists?: boolean;
-  hasPart?: boolean;
   identifier: string;
-  insideKg: boolean;
-  keywords: string;
+  keywords: string[];
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   mediaContent: any;
   name: string;
-  published: string;
-  sameAs?: string;
   title: string;
+};
+
+interface DatasetCore extends DatasetAbstract {
+  creators: Creator[];
+  created_at: string;
+}
+
+interface DatasetKg extends DatasetAbstract {
+  created: string;
+  hasPart: Part[];
+  published: Published;
+  url: string;
+  usedIn: UsedIn;
+  sameAs?: string;
+}
+
+interface IDataset extends DatasetAbstract {
+  created: string;
+  exists: boolean;
+  insideKg: boolean;
+  hasPart?: Part[];
+  published?: Published;
+  sameAs?: string;
   url?: string;
-  usedIn?: string;
+  usedIn?: UsedIn;
+}
+
+type IDatasetFiles = {
+  fetched: boolean;
+  fetching: boolean;
+  files: IDatasetFile[];
+};
+
+type IDatasetFile = {
+  path: string;
+  added: string;
+  name: string;
 };
 
+type IMigration = {
+  check: {
+    core_renku_version?: string;
+    project_supported?: boolean;
+    project_renku_version?: string;
+    core_compatibility_status: {
+      project_metadata_version?: string;
+      migration_required?: boolean;
+      current_metadata_version?: string;
+    };
+    dockerfile_renku_status: {
+      latest_renku_version?: string;
+      dockerfile_renku_version?: string;
+      automated_dockerfile_update?: boolean;
+      newer_renku_available?: boolean;
+    };
+    template_status: {
+      newer_template_available?: boolean;
+      template_id?: string;
+      automated_template_update?: boolean;
+      template_ref?: string;
+      project_template_version?: string;
+      template_source?: string;
+      latest_template_version?: string;
+    };
+  };
+  core: {
+    versionUrl: string | null;
+    backendAvailable: boolean | null;
+    error: boolean | null;
+    fetched: boolean | null;
+    fetching: boolean | null;
+  };
+  migrating: boolean;
+  migration_status: unknown | null;
+  migration_error: unknown | null;
+};
+
+interface Part {
+  atLocation: string;
+}
+
+interface Published {
+  creator: Creator[];
+  datePublished?: string;
+}
+
 type StateModelProject = {
   branches: unknown;
   datasets: {
@@ -26,7 +125,7 @@ type StateModelProject = {
   };
   filesTree?: {
     hash: Record<string, unknown>;
-  }
+  };
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   forkedFromProject?: any;
   lockStatus: unknown;
@@ -44,7 +143,18 @@ type StateModelProject = {
   migration: unknown;
   webhook: {
     progress: unknown;
-  }
+  };
+};
+
+type UsedIn = {
+  _links: [
+    {
+      rel: string;
+      href: string;
+    }
+  ];
+  path: string;
+  name: string;
 };
 
-export type { IDataset, StateModelProject };
+export type { DatasetCore, DatasetKg, IDataset, IDatasetFile, IDatasetFiles, IMigration, StateModelProject };

client/src/features/project/dataset/ProjectDatasetShow.tsx

@@ -2,10 +2,18 @@ import React from "react";
 import { RootStateOrAny, useSelector } from "react-redux";
 
 import { ACCESS_LEVELS } from "../../../api-client";
-import ShowDataset from "../../../dataset/Dataset.container";
+import DatasetView from "../../../dataset/Dataset.present";
 import { Url } from "../../../utils/helpers/url";
 
-import type { StateModelProject } from "../Project.d";
+import { useGetDatasetFilesQuery, useGetDatasetKgQuery } from "../projectCoreApi";
+import type { DatasetCore, DatasetKg, IDataset, IMigration, StateModelProject } from "../Project.d";
+
+type IDatasetCoordinator = {
+  fetchDataset: (id: string, datasets: DatasetCore[], fetchKG: boolean) => void;
+  fetchDatasetFilesFromCoreService: (id: string, httpProjectUrl: string, versionUrl: string) => void;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  get(key: string): any;
+};
 
 type ProjectDatasetShowProps = {
   datasetCoordinator: unknown;
@@ -17,6 +25,123 @@ type ProjectDatasetShowProps = {
   projectInsideKg: boolean;
 };
 
+type ProjectDatasetViewProps = {
+  datasetCoordinator: IDatasetCoordinator;
+  datasets: DatasetCore[];
+  datasetId: string;
+  fileContentUrl: string;
+  graphStatus: boolean;
+  history: unknown;
+  httpProjectUrl: string;
+  lineagesUrl: string;
+  location: unknown;
+  lockStatus: unknown;
+  logged: unknown;
+  maintainer: boolean;
+  migration: IMigration;
+  model: unknown;
+  overviewStatusUrl: string;
+  projectId: string;
+  projectInsideKg: boolean;
+  projectPathWithNamespace: string;
+  projectsUrl: string;
+};
+
+function findDataset(name: string, datasets: DatasetCore[]) {
+  return datasets.find((d) => d.name === name);
+}
+
+function findDatasetId(name?: string, datasets?: DatasetCore[]) {
+  if (name == null || datasets == null) return undefined;
+  const dataset = findDataset(name, datasets);
+  return dataset?.identifier;
+}
+
+function mergeCoreAndKgDatasets(coreDataset?: DatasetCore, kgDataset?: DatasetKg) {
+  if (coreDataset == null) {
+    if (kgDataset == null) return undefined;
+    const dataset: IDataset = { exists: true, insideKg: true, ...kgDataset };
+    return dataset;
+  }
+
+  const dataset: IDataset = {
+    created: coreDataset.created_at,
+    exists: true,
+    insideKg: kgDataset != null,
+    ...coreDataset,
+  };
+  dataset.published = {
+    creator: coreDataset.creators,
+  };
+  if (kgDataset) {
+    dataset.url = kgDataset.url;
+    dataset.sameAs = kgDataset.sameAs;
+    dataset.usedIn = kgDataset.usedIn;
+    dataset.published.datePublished =
+      kgDataset.published && kgDataset.published.datePublished ? kgDataset.published.datePublished : undefined;
+  }
+  return dataset;
+}
+
+function ProjectDatasetView(props: ProjectDatasetViewProps) {
+  const coreDataset = findDataset(props.datasetId, props.datasets);
+  const datasetId = findDatasetId(props.datasetId, props.datasets);
+  const migration = props.migration;
+  // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+  const versionUrl = migration.core.versionUrl!;
+  const {
+    data: kgDataset,
+    error: kgFetchError,
+    isFetching: isKgFetching,
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+  } = useGetDatasetKgQuery({ id: datasetId! }, { skip: !datasetId });
+  const currentDataset = mergeCoreAndKgDatasets(coreDataset, kgDataset);
+  //  const { }
+  const datasetName = currentDataset?.name;
+  const {
+    data: datasetFiles,
+    error: filesFetchError,
+    isFetching: isFilesFetching,
+  } = useGetDatasetFilesQuery(
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    { git_url: props.httpProjectUrl, name: datasetName!, versionUrl },
+    { skip: !datasetName }
+  );
+
+  const loadingDatasets = currentDataset == null || currentDataset.identifier !== datasetId || isKgFetching;
+  return (
+    <DatasetView
+      client={undefined}
+      dataset={currentDataset}
+      files={datasetFiles}
+      isFilesFetching={isFilesFetching}
+      filesFetchError={filesFetchError}
+      datasets={props.datasets}
+      fetchError={kgFetchError}
+      fetchedKg={kgDataset != null}
+      fileContentUrl={props.fileContentUrl}
+      history={props.history}
+      httpProjectUrl={props.httpProjectUrl}
+      identifier={datasetId}
+      insideProject={true}
+      lineagesUrl={props.lineagesUrl}
+      loadingDatasets={loadingDatasets}
+      location={props.location}
+      lockStatus={props.lockStatus}
+      logged={props.logged}
+      maintainer={props.maintainer}
+      migration={migration}
+      model={props.model}
+      overviewStatusUrl={props.overviewStatusUrl}
+      progress={undefined}
+      projectId={props.projectId}
+      projectInsideKg={props.projectInsideKg}
+      projectPathWithNamespace={props.projectPathWithNamespace}
+      projectsUrl={props.projectsUrl}
+    />
+  );
+}
+
 function ProjectDatasetShow(props: ProjectDatasetShowProps) {
   const project = useSelector((state: RootStateOrAny) => state.stateModel.project as StateModelProject);
   const user = useSelector((state: RootStateOrAny) => state.stateModel.user);
@@ -39,23 +164,23 @@ function ProjectDatasetShow(props: ProjectDatasetShowProps) {
   const lineagesUrl = lineageUrl.substring(0, lineageUrl.length - 1);
   const overviewStatusUrl = Url.get(Url.pages.project.overview.status, projectUrlProps);
   const projectsUrl = Url.get(Url.pages.projects);
+  if (props.datasetCoordinator == null) return null;
   return (
-    <ShowDataset
+    <ProjectDatasetView
       key="datasetPreview"
-      datasets={datasets}
-      datasetCoordinator={props.datasetCoordinator}
+      datasets={datasets as DatasetCore[]}
+      datasetCoordinator={props.datasetCoordinator as IDatasetCoordinator}
       datasetId={props.datasetId}
       fileContentUrl={fileContentUrl}
       graphStatus={props.graphStatus}
       history={props.history}
       httpProjectUrl={httpProjectUrl}
-      insideProject={true}
       lineagesUrl={lineagesUrl}
       location={props.location}
       lockStatus={lockStatus}
       logged={user.logged}
       maintainer={maintainer}
-      migration={migration}
+      migration={migration as IMigration}
       model={props.model}
       overviewStatusUrl={overviewStatusUrl}
       projectId={projectId}