Merge branch 'main' into abcdj-aliaschange

README.md

@@ -11,6 +11,13 @@
 [![All Contributors](https://img.shields.io/badge/all_contributors-12-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 
+---
+
+### *:loudspeaker: A new catalog is coming...*
+
+We're working on a newer, leaner, more modular, and more interoperable solution to the same challenge that the current `datalad-catalog` aims to address. This new development is taking place within the broader context of making DataLad datasets interoperable with linked and semantic (meta)data. For more background, see [this issue](https://github.com/psychoinformatics-de/datalad-concepts/issues/115). To keep up to date, follow progress at [`psychoinformatics-de/datalad-concepts`](https://github.com/psychoinformatics-de/datalad-concepts), [`psychoinformatics-de/shacl-vue`](https://github.com/psychoinformatics-de/shacl-vue), and in the [new development branch](https://github.com/datalad/datalad-catalog/tree/revolution). Because of this redirected focus, `datalad-catalog` itself will be downscaled by focusing on maintenance and assessing the priority of new features on a case-by-case basis.
+
+---
 
 <picture>
   <source media="(prefers-color-scheme: light)" srcset="docs/source/_static/datacat0_hero.svg">
@@ -186,7 +193,8 @@ cd datalad-catalog
 pip install -r requirements-devel.txt
 ```
 
-This installs `sphinx` and related packages for documentation building, `coverage` for code coverage, and `pytest` for testing.
+This installs `sphinx` and related packages for documentation building, `coverage` for code coverage,
+`black` for linting, and `pytest` for testing.
 
 ### Contribution process
 
@@ -195,10 +203,12 @@ To make a contribution to the code or documentation, please:
 - create an issue describing the bug/feature
 - fork the project repository,
 - create a branch from `main`,
-- check that tests succeed: from the project root directory, run `pytest`
 - commit your changes,
-- push to your fork
+- check that linting tests succeed: from the project root directory, run `black .`
+- check that tests succeed: from the project root directory, run `python -m pytest`
+- push your commits to your fork
 - create a pull request with a clear description of the changes
+- check that all continuous integration tests succeed on the pull request
 
 ## Contributors ✨
 

datalad_catalog/__init__.py

@@ -62,6 +62,12 @@
             "catalog-remove",
             "catalog_remove",
         ),
+        (
+            "datalad_catalog.tree",
+            "Tree",
+            "catalog-tree",
+            "catalog_tree",
+        ),
         (
             "datalad_catalog.translate",
             "MetaTranslate",

datalad_catalog/catalog/assets/app_component_dataset.js

@@ -77,6 +77,8 @@ const datasetView = () =>
               console.debug("Active dataset:");
               dataset = this.selectedDataset;
               console.debug(this.selectedDataset);
+              dataset_options = dataset.dataset_options
+
               disp_dataset = {};
               // Set name to unknown if not available
               if (!dataset.hasOwnProperty("name") || !dataset["name"]) {
@@ -134,61 +136,117 @@ const datasetView = () =>
                 dataset.dataset_id + "_" + dataset.dataset_version;
               disp_dataset.download_filename =
                 "dataset_" + disp_dataset.id_and_version + ".json";
-              // URL
-              disp_dataset.is_github = false; // Github / gitlab / url / binder
-              disp_dataset.is_gitlab = false; // Github / gitlab / url / binder
-              disp_dataset.is_gin = false; // GIN
-              disp_dataset.url = "";
-              if (
-                dataset.hasOwnProperty("url") &&
-                (dataset["url"] instanceof Array || Array.isArray(dataset["url"])) &&
-                dataset["url"].length > 0
-              ) {
-                for (var i = 0; i < dataset.url.length; i++) {
-                  if (dataset.url[i].toLowerCase().indexOf("github") >= 0) {
-                    disp_dataset.is_github = true;
-                    disp_dataset.url = dataset.url[i];
-                    disp_dataset.url = disp_dataset.url.replace('git@github.com:', 'https://github.com');
-                  }
-                  if (dataset.url[i].toLowerCase().indexOf("gin.g-node") >= 0) {
-                    disp_dataset.is_gin = true;
-                    disp_dataset.url = dataset.url[i];
-                    disp_dataset.url = disp_dataset.url.replace('ssh://', '');
-                    disp_dataset.url = disp_dataset.url.replace('git@gin.g-node.org:', 'https://gin.g-node.org');
-                    disp_dataset.url = disp_dataset.url.replace('git@gin.g-node.org', 'https://gin.g-node.org');
-                    disp_dataset.url = disp_dataset.url.replace('.git', '');
-                  }
-                }
-                if (!disp_dataset.url) {
+
+              // -------
+              // BUTTONS
+              // -------
+              // show all buttons by default, unless logic dictates otherwise
+
+              // first get dataset options
+              dataset_options = dataset.config.dataset_options;
+
+              console.log("Dataset options (on dataset or inferred from catalog)")
+              console.log(dataset_options)
+
+              // then initialise show/hide flags
+              disp_dataset.show_datalad = true; // Download with DataLad button
+              disp_dataset.show_github = true; // Github
+              disp_dataset.show_gitlab = true; // Gitlab
+              disp_dataset.show_gin = true; // GIN.g-node
+              disp_dataset.show_binder = true; // MyBinder
+              disp_dataset.show_cite = true; // 'Cite' button
+              disp_dataset.show_export = true; // 'Export metadata' button
+              disp_dataset.show_access_request = true; // 'Request access' button
+              disp_dataset.show_homepage = true; // 'Homepage' button
+              disp_dataset.show_download = true; // 'Download' button
+
+              // URL (this is the field for the datalad dataset url to clone from)
+              // If URL does not exist, several buttons cannot be shown
+              // If URL is an array, first select first element
+              if (dataset.hasOwnProperty("url")) {
+                if (
+                  (dataset["url"] instanceof Array || Array.isArray(dataset["url"])) &&
+                  dataset["url"].length > 0
+                ) {
                   disp_dataset.url = dataset.url[0];
+                } else {
+                  disp_dataset.url = dataset.url;
                 }
-              } else {
-                disp_dataset.url = dataset.url;
-                if (disp_dataset.url && dataset.url.toLowerCase().indexOf("gin.g-node") >= 0) {
-                  disp_dataset.is_gin = true;
+                // show/hide datalad button
+                disp_dataset.show_datalad = dataset_options.include_datalad ?? true
+                // show/hide github button
+                if (disp_dataset.url.toLowerCase().indexOf("github") >= 0) {
+                  disp_dataset.url = disp_dataset.url.replace('git@github.com:', 'https://github.com');
+                  disp_dataset.show_github = dataset_options.include_github ?? true
+                } else {
+                  disp_dataset.show_github = false;
+                }
+                // show/hide GIN button
+                if (disp_dataset.url.toLowerCase().indexOf("gin.g-node") >= 0) {
                   disp_dataset.url = disp_dataset.url.replace('ssh://', '');
                   disp_dataset.url = disp_dataset.url.replace('git@gin.g-node.org:', 'https://gin.g-node.org');
-                  disp_dataset.url = disp_dataset.url.replace('git@gin.g-node.org', 'https://gin.g-node.org');
                   disp_dataset.url = disp_dataset.url.replace('.git', '');
+                  disp_dataset.show_gin = dataset_options.include_gin ?? true
+                } else {
+                  disp_dataset.show_gin = false;
                 }
+                // show/hide gitlab button
+                if (disp_dataset.url.toLowerCase().indexOf("gitlab") >= 0) {
+                  disp_dataset.show_gitlab = dataset_options.include_gitlab ?? true
+                } else {
+                  disp_dataset.show_gitlab = false;
+                }
+              } else{
+                // none of these buttons can be shown without a URL
+                disp_dataset.show_datalad = false;
+                disp_dataset.show_github = false;
+                disp_dataset.show_gitlab = false;
+                disp_dataset.show_gin = false;
               }
-              // Description
-              if (
-                dataset.hasOwnProperty("description") &&
-                (dataset["description"] instanceof Array ||
-                  Array.isArray(dataset["description"])) &&
-                dataset["description"].length > 0
-              ) {
-                disp_dataset.description = dataset.description;
-                disp_dataset.selected_description = disp_dataset.description[0];
-                this.selectDescription(disp_dataset.selected_description);
+              // Show binder button: (if disp_dataset.url exists OR if dataset has a notebook specified in metadata) AND config specifies (or is missing)
+              disp_dataset.show_binder_button = false
+              if ( disp_dataset.url || disp_dataset.hasOwnProperty("notebooks") && disp_dataset.notebooks.length > 0 ) {
+                disp_dataset.show_binder = dataset_options.include_binder ?? true
+              } else {
+                disp_dataset.show_binder = false;
               }
-              if (
-                (dataset.hasOwnProperty("description") &&
-                  dataset["description"] instanceof String) ||
-                typeof dataset["description"] === "string"
-              ) {
-                this.description_ready = true;
+              // Show cite button: if the dataset.doi exists AND config specifies (or is missing)
+              disp_dataset.show_cite = dataset.doi ? (dataset_options.include_cite ?? true) : false
+
+              // Show homepage button: if the dataset.homepage_url exists AND config specifies (or is missing)
+              if (dataset.hasOwnProperty("homepage_url")) {
+                if (
+                  (dataset["homepage_url"] instanceof Array || Array.isArray(dataset["homepage_url"])) &&
+                  dataset["homepage_url"].length > 0
+                ) {
+                  disp_dataset.homepage_url = dataset.homepage_url[0];
+                } else {
+                  disp_dataset.homepage_url = dataset.homepage_url;
+                }
+              }
+              disp_dataset.show_homepage = disp_dataset.homepage_url ? (dataset_options.include_homepage ?? true) : false
+              // Show download button: if the dataset.download_url exists AND config specifies (or is missing)
+              if (dataset.hasOwnProperty("download_url")) {
+                if (
+                  (dataset["download_url"] instanceof Array || Array.isArray(dataset["download_url"])) &&
+                  dataset["download_url"].length > 0
+                ) {
+                  disp_dataset.download_url = dataset.download_url[0];
+                } else {
+                  disp_dataset.download_url = dataset.download_url;
+                }
+              }
+              disp_dataset.show_download = disp_dataset.download_url ? (dataset_options.include_download ?? true) : false
+              // Show export button: if config specifies (or is missing)
+              disp_dataset.show_export = dataset_options.include_metadata_export ?? true
+              // Show/hide config for "Request access" button:
+              // if the (access_request_contact exists OR access_request_url exists)
+              // AND config specifies (or is missing)
+              if ( dataset.hasOwnProperty("access_request_contact") && dataset["access_request_contact"] ||
+              dataset.hasOwnProperty("access_request_url") && dataset["access_request_url"] ) {
+                disp_dataset.show_access_request = dataset_options.include_access_request ?? true
+              } else {
+                disp_dataset.show_access_request = false;
               }
               // Create href mailto for request access contact
               if (
@@ -197,7 +255,6 @@ const datasetView = () =>
               ) {
                 var email_to = dataset.access_request_contact.email
                 var email_subject = "Access request: " + disp_dataset.short_name
-
                 disp_dataset.access_request_mailto =
                   "mailto:" + 
                   email_to +
@@ -208,25 +265,24 @@ const datasetView = () =>
                   "%20" + 
                   dataset.access_request_contact.familyName;
               }
-              // Rendering options for dataset page
-              if (this.$root.hasOwnProperty("dataset_options") && this.$root.dataset_options.hasOwnProperty("include_metadata_export")) {
-                disp_dataset.show_export = this.$root.dataset_options.include_metadata_export
-              }
-              else {
-                disp_dataset.show_export = false
-              }
-              // Determine show/hide confirg for "Request access" button
-              if (dataset.config?.hasOwnProperty("dataset_options") && dataset.config.dataset_options.hasOwnProperty("include_access_request")) {
-                disp_dataset.show_access_request = dataset.config.dataset_options.include_access_request
-              }
-              else {
-                // default should be to display the access request button, if access request contact/url are included
-                disp_dataset.show_access_request = true
+
+              // Description
+              if (
+                dataset.hasOwnProperty("description") &&
+                (dataset["description"] instanceof Array ||
+                  Array.isArray(dataset["description"])) &&
+                dataset["description"].length > 0
+              ) {
+                disp_dataset.description = dataset.description;
+                disp_dataset.selected_description = disp_dataset.description[0];
+                this.selectDescription(disp_dataset.selected_description);
               }
-              // Show / hide binder button: if disp_dataset.url exists OR if dataset has a notebook specified in metadata
-              disp_dataset.show_binder_button = false
-              if ( disp_dataset.url || disp_dataset.hasOwnProperty("notebooks") && disp_dataset.notebooks.length > 0 ) {
-                disp_dataset.show_binder_button = true
+              if (
+                (dataset.hasOwnProperty("description") &&
+                  dataset["description"] instanceof String) ||
+                typeof dataset["description"] === "string"
+              ) {
+                this.description_ready = true;
               }
 
               // Set correct URL query string to mirrorif keyword(s) included in query parameters
@@ -423,7 +479,7 @@ const datasetView = () =>
             history.replaceState(
               {},
               null,
-              current_route.path + query_string
+              (router.options.base + current_route.path + query_string).replace("//", "/")
             )
             console.debug("- After: Vue Route query params: %s", JSON.stringify(Object.assign({}, this.$route.query)))
             let url_qp2 = new URL(document.location.toString()).searchParams
@@ -1049,15 +1105,20 @@ const datasetView = () =>
           // set the root data for available tabs
           available_tabs_lower = available_tabs
           this.$root.selectedDataset.available_tabs = available_tabs_lower
-          // Now get dataset config if it exists
+          // Now get dataset config if it exists, else set to catalog-level config
           dataset_config_path = metadata_dir + "/" + sDs.dataset_id + "/" + sDs.dataset_version + "/config.json";
           configresponse = await fetch(dataset_config_path, {cache: "no-cache"});
           if (configresponse.status == 404) {
-            this.$root.selectedDataset.config = {};
+            this.$root.selectedDataset.config = this.$root.catalog_config;
           } else {
             configtext = await configresponse.text();
             config = JSON.parse(configtext);
-            this.$root.selectedDataset.config = config;
+            this.$root.selectedDataset.config = {...this.$root.catalog_config, ...config};
+            if (config.dataset_options) {
+              // dataset options exist in catalog level config and dataset-level config
+              // they need to be merged, with dataset-level taking priority
+              this.$root.selectedDataset.config.dataset_options = {...this.$root.catalog_config.dataset_options, ...config.dataset_options};
+            }
           }
           // Set the correct tab to be rendered
           correct_tab = to.query.hasOwnProperty("tab") ? to.query.tab : null
@@ -1204,15 +1265,20 @@ const datasetView = () =>
           available_tabs_lower = available_tabs
           // set the root data for available tabs
           this.$root.selectedDataset.available_tabs = available_tabs_lower
-          // Now get dataset config if it exists
+          // Now get dataset config if it exists, else set to catalog-level config
           dataset_config_path = metadata_dir + "/" + sDs.dataset_id + "/" + sDs.dataset_version + "/config.json";
           configresponse = await fetch(dataset_config_path, {cache: "no-cache"});
           if (configresponse.status == 404) {
-            this.$root.selectedDataset.config = {};
+            this.$root.selectedDataset.config = this.$root.catalog_config;
           } else {
             configtext = await configresponse.text();
             config = JSON.parse(configtext);
-            this.$root.selectedDataset.config = config;
+            this.$root.selectedDataset.config = {...this.$root.catalog_config, ...config};
+            if (config.dataset_options) {
+              // dataset options exist in catalog level config and dataset-level config
+              // they need to be merged, with dataset-level taking priority
+              this.$root.selectedDataset.config.dataset_options = {...this.$root.catalog_config.dataset_options, ...config.dataset_options};
+            }
           }
           // ---
           // Note for future: Handle route query parameters (tab and keyword) here?

datalad_catalog/catalog/assets/app_globals.js

@@ -2,9 +2,9 @@
 // Data //
 /********/
 
-const template_dir = "/templates";
-const config_file = "/config.json";
-const metadata_dir = "/metadata";
+const template_dir = "templates";
+const config_file = "config.json";
+const metadata_dir = "metadata";
 const superdatasets_file = metadata_dir + "/super.json";
 const SPLIT_INDEX = 3;
 const SHORT_NAME_LENGTH = 0; // number of characters in name to display, zero if all
@@ -13,7 +13,7 @@ const default_config = {
   catalog_url: "https://datalad-catalog.netlify.app/",
   link_color: "#fba304",
   link_hover_color: "#af7714",
-  logo_path: "/artwork/catalog_logo.svg",
+  logo_path: "artwork/catalog_logo.svg",
   social_links: {
     about: null,
     documentation: "https://docs.datalad.org/projects/catalog/en/latest/",

datalad_catalog/catalog/assets/app_router.js

@@ -50,7 +50,7 @@ const routes = [
 // Create router
 const router = new VueRouter({
   mode: 'history',
-  base: '/',
+  base: window.location.pathname.split('dataset/')[0],
   routes: routes,
   scrollBehavior(to, from, savedPosition) {
     return { x: 0, y: 0, behavior: "auto" };

datalad_catalog/catalog/config.json

@@ -11,7 +11,14 @@
         "x": "https://x.com/datalad"
     },
     "dataset_options": {
-        "include_metadata_export": true
+        "include_datalad": true,
+        "include_github": true,
+        "include_gitlab": true,
+        "include_gin": true,
+        "include_binder": true,
+        "include_cite": true,
+        "include_metadata_export": true,
+        "include_access_request": true
     },
     "property_sources": {
         "dataset": {