From 28930c07ead87f70353132481d307d6f9aaf6491 Mon Sep 17 00:00:00 2001 From: Lazlo Westerhof Date: Mon, 18 Nov 2024 08:36:37 +0100 Subject: [PATCH] =?UTF-8?q?Sunset=20intake=20module=20=F0=9F=AB=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../workflows/build-push-image-provider.yml | 2 - docker/images/yoda_irods_icat/Dockerfile | 4 - .../yoda_irods_icat/job_clearintakelocks.r | 29 ---- .../images/yoda_irods_icat/job_movetovault.r | 67 --------- docker/images/yoda_irods_icat/rules_uu.cfg | 1 - .../images/yoda_irods_icat/stage-uploads.sh | 1 - docker/images/yoda_portal/yoda_portal_init.sh | 2 - docker/run-cronjob.sh | 4 - docs/administration/configuring-yoda.md | 8 -- docs/design/api/index.md | 1 - docs/design/index.md | 1 - docs/design/overview/index.md | 5 - docs/design/overview/intake.md | 135 ------------------ .../processes/asynchronous-processes.md | 14 +- .../allinone/group_vars/allinone.yml | 3 - .../el8-database/group_vars/el8.yml | 3 - .../development/full/group_vars/full.yml | 3 - .../single_fqdn/group_vars/single_fqdn.yml | 3 - .../development/surf/group_vars/surf.yml | 3 - .../development/ubuntu/group_vars/ubuntu.yml | 3 - roles/yoda_portal/defaults/main.yml | 4 - roles/yoda_portal/templates/flask.cfg.j2 | 2 - .../templates/yoda-portal-vhost.conf.j2 | 4 - roles/yoda_rulesets/defaults/main.yml | 5 - .../files/run-intake-movetovault.sh | 14 -- roles/yoda_rulesets/tasks/yoda-ruleset.yml | 44 ++---- .../templates/job_clearintakelocks.r.j2 | 29 ---- .../templates/job_movetovault.r.j2 | 71 --------- roles/yoda_rulesets/templates/rules_uu.cfg.j2 | 1 - roles/yoda_test/files/job_checksums.r | 21 --- roles/yoda_test/files/job_clearintakelocks.r | 29 ---- roles/yoda_test/files/job_movetovault.r | 67 --------- roles/yoda_test/tasks/install-data.yml | 16 +-- 33 files changed, 23 insertions(+), 576 deletions(-) delete mode 100644 docker/images/yoda_irods_icat/job_clearintakelocks.r delete mode 100644 docker/images/yoda_irods_icat/job_movetovault.r delete mode 100644 docs/design/overview/intake.md delete mode 100644 roles/yoda_rulesets/files/run-intake-movetovault.sh delete mode 100644 roles/yoda_rulesets/templates/job_clearintakelocks.r.j2 delete mode 100644 roles/yoda_rulesets/templates/job_movetovault.r.j2 delete mode 100644 roles/yoda_test/files/job_checksums.r delete mode 100644 roles/yoda_test/files/job_clearintakelocks.r delete mode 100644 roles/yoda_test/files/job_movetovault.r diff --git a/.github/workflows/build-push-image-provider.yml b/.github/workflows/build-push-image-provider.yml index fb39bfea6..fd2be0931 100644 --- a/.github/workflows/build-push-image-provider.yml +++ b/.github/workflows/build-push-image-provider.yml @@ -12,7 +12,6 @@ on: - 'docker/images/yoda_irods_icat/**' - 'roles/irods_completion/files/irods_completion.sh' - 'roles/pam_python/files/pam_python.so' - - 'roles/yoda_rulesets/files/run-intake-movetovault.sh' jobs: push-image: @@ -55,4 +54,3 @@ jobs: tags: ghcr.io/utrechtuniversity/yoda-provider:${{ steps.extract_branch.outputs.branch }} build-args: | TAG=${{ steps.extract_branch.outputs.branch }} - diff --git a/docker/images/yoda_irods_icat/Dockerfile b/docker/images/yoda_irods_icat/Dockerfile index c2a507b82..83ec751f6 100644 --- a/docker/images/yoda_irods_icat/Dockerfile +++ b/docker/images/yoda_irods_icat/Dockerfile @@ -177,10 +177,6 @@ RUN for script in scheduled-copytovault.sh admin-remove-orphan-vault-if-empty.sh do install -m 0755 /etc/irods/yoda-ruleset/tools/${script} /var/lib/irods/msiExecCmd_bin/${script} ; \ done && \ /etc/irods/yoda-ruleset/tools/setup_tokens.sh /etc/irods/yoda-ruleset/accesstokens.db test -COPY stage/run-intake-movetovault.sh /tmp/run-intake-movetovault.sh -COPY job_movetovault.r /var/lib/irods/.irods/job_movetovault.r -COPY job_clearintakelocks.r /var/lib/irods/.irods/job_clearintakelocks.r -RUN install -o irods -g irods -m 0755 /tmp/run-intake-movetovault.sh /var/lib/irods/.irods/run-intake-movetovault.sh COPY cronjob-revision-cleanup.sh /tmp/cronjob-revision-cleanup.sh RUN install -o irods -g irods -m 0755 /tmp/cronjob-revision-cleanup.sh /var/lib/irods/.irods/cronjob-revision-cleanup.sh COPY service_account.config /tmp/service_account.config diff --git a/docker/images/yoda_irods_icat/job_clearintakelocks.r b/docker/images/yoda_irods_icat/job_clearintakelocks.r deleted file mode 100644 index bd5a67fe5..000000000 --- a/docker/images/yoda_irods_icat/job_clearintakelocks.r +++ /dev/null @@ -1,29 +0,0 @@ -# \file -# \brief job -# \author Sietse Snel -# \copyright Copyright (c) 2022, Utrecht university. All rights reserved -# \license GPLv3, see LICENSE -# -# This file should be executed as part of a recurring crontab job -# as the irods admin user (i.e. irods user type rodsadmin) -# e.g. run every 5 minutes -# -# It clears all locks on intake folders after job_movetovault has run, in -# order to facilitate retries after transient errors -# - -uuYcClearIntakeLocks { - # intake areas can be added to the grouplist as needed - *grouplist = list ("initial","test"); - *zone = $rodsZoneClient; - - foreach (*grp in *grouplist) { - *intakeRoot = "/*zone/home/grp-intake-*grp"; - *vaultRoot = "/*zone/home/grp-vault-*grp"; - # uuUnlock also succeeds if lock doesn't exist, so no need to verify that lock exists first - uuUnlock(*vaultRoot); - } -} - -input *intakeRoot='dummy' -output ruleExecOut diff --git a/docker/images/yoda_irods_icat/job_movetovault.r b/docker/images/yoda_irods_icat/job_movetovault.r deleted file mode 100644 index fb8299433..000000000 --- a/docker/images/yoda_irods_icat/job_movetovault.r +++ /dev/null @@ -1,67 +0,0 @@ -# \file -# \brief job -# \author Ton Smeele -# \copyright Copyright (c) 2015, Utrecht university. All rights reserved -# \license GPLv3, see LICENSE -# -# if another instance of the job is running then the vault will be -# locked and silently ignored -# - -uuYcRunIntake2Vault { - # intake areas can be added to the grouplist as needed - *grouplist = list ("initial", "test", "test2"); - *grp_prefixes = list ("grp-intake-", "intake-"); - *zone = $rodsZoneClient; - - foreach (*grp in *grouplist) { - *intakeRoot = ""; - *vlt_name = ""; - *found = false; - - foreach (*prefix in *grp_prefixes) { - *grp_name = *prefix ++ *grp; - - if (*prefix == "intake-" ) { - *vlt_name = "vault-" ++ *grp; - } - else { - *vlt_name = "grp-vault-" ++ *grp; - } - - foreach(*row in SELECT USER_GROUP_NAME - WHERE USER_GROUP_NAME = '*grp_name' - AND USER_TYPE = 'rodsgroup') { - - *path=*row.USER_GROUP_NAME; - *found = true; - break; - } - if (*found) { - break; - } - } - - if ( *found ) { - *intakeRoot = "/*zone/home/*grp_name"; - *vaultRoot = "/*zone/home/*vlt_name"; - writeLine("serverLog","RunIntake2Vault for *intakeRoot -> *vaultRoot ..."); - - uuLock(*vaultRoot, *status); - if (*status == 0) { - # we have a lock - uuYc2Vault(*intakeRoot, *vaultRoot, *status); - if (*status == 0 ) then *result = "ok" else *result = "ERROR (*status)"; - writeLine("serverLog","RunIntake2Vault for *intakeRoot result = *result"); - uuUnlock(*vaultRoot); - } - } - else { - writeLine("serverLog", "Intake2vault was unable to find group *grp."); - } - } -} - - -input *intakeRoot='dummy' -output ruleExecOut diff --git a/docker/images/yoda_irods_icat/rules_uu.cfg b/docker/images/yoda_irods_icat/rules_uu.cfg index 10dfd1287..dff1a76c6 100644 --- a/docker/images/yoda_irods_icat/rules_uu.cfg +++ b/docker/images/yoda_irods_icat/rules_uu.cfg @@ -39,7 +39,6 @@ eus_api_tls_verify = 'false' enable_deposit = 'true' enable_open_search = 'false' -enable_intake = 'true' enable_datarequest = 'true' yoda_portal_fqdn = 'portal.yoda' diff --git a/docker/images/yoda_irods_icat/stage-uploads.sh b/docker/images/yoda_irods_icat/stage-uploads.sh index 5a9a6c245..fe04ef8e8 100755 --- a/docker/images/yoda_irods_icat/stage-uploads.sh +++ b/docker/images/yoda_irods_icat/stage-uploads.sh @@ -16,4 +16,3 @@ fi cp ../../../roles/pam_python/files/pam_python.so stage cp ../../../roles/irods_completion/files/irods_completion.sh stage -cp ../../../roles/yoda_rulesets/files/run-intake-movetovault.sh stage diff --git a/docker/images/yoda_portal/yoda_portal_init.sh b/docker/images/yoda_portal/yoda_portal_init.sh index 2105eb801..aca333ac6 100644 --- a/docker/images/yoda_portal/yoda_portal_init.sh +++ b/docker/images/yoda_portal/yoda_portal_init.sh @@ -92,8 +92,6 @@ YODA_COMMIT = '$YODA_COMMIT' RESEARCH_ENABLED = True OPEN_SEARCH_ENABLED = False DEPOSIT_ENABLED = True -INTAKE_ENABLED = True -INTAKE_EXT_TIMEOUT = 1800 DATAREQUEST_ENABLED = True TOKENS_ENABLED = True TOKEN_LIFETIME = 72 diff --git a/docker/run-cronjob.sh b/docker/run-cronjob.sh index e5bf6ae8e..9f47817cf 100755 --- a/docker/run-cronjob.sh +++ b/docker/run-cronjob.sh @@ -25,10 +25,6 @@ case "$1" in docker exec "$EXEC_OPTIONS" provider.yoda sudo -iu irods /bin/bash /etc/irods/yoda-ruleset/tools/notification/notification-data-access-token-expiry.sh ;; - intakevault) - docker exec "$EXEC_OPTIONS" provider.yoda sudo -iu irods /var/lib/irods/.irods/run-intake-movetovault.sh - ;; - publication) docker exec "$EXEC_OPTIONS" provider.yoda sudo -iu irods /bin/irule -r irods_rule_engine_plugin-irods_rule_language-instance -F /etc/irods/yoda-ruleset/tools/process-publication.r ;; diff --git a/docs/administration/configuring-yoda.md b/docs/administration/configuring-yoda.md index 4f56cecc7..293a22bd0 100644 --- a/docs/administration/configuring-yoda.md +++ b/docs/administration/configuring-yoda.md @@ -282,14 +282,6 @@ Variable | Description -----------------------------|--------------------- enable_deposit | Enable deposit module -### Intake module configuration - -Variable | Description ------------------------------|--------------------- -enable_intake | Enable intake module -intake_groups | List of intake groups (without the "grp-intake-" prefix) -intake_extended_timeout | If the intake module is enabled, portal iRODS session timeouts and portal VHost timeouts will be changed to this value (in seconds), in order to be able to handle long synchronous operations, such as intake scans (default: 1800) - ### Datarequest module configuration Variable | Description diff --git a/docs/design/api/index.md b/docs/design/api/index.md index a863f1701..00d3b2083 100644 --- a/docs/design/api/index.md +++ b/docs/design/api/index.md @@ -11,5 +11,4 @@ has_toc: false - [Core API documentation](https://petstore.swagger.io/?url=https://utrechtuniversity.github.io/yoda-ruleset/api_core.json) ### Module API -- [Intake API documentation](https://petstore.swagger.io/?url=https://utrechtuniversity.github.io/yoda-ruleset/api_intake.json) - [Datarequest API documentation](https://petstore.swagger.io/?url=https://utrechtuniversity.github.io/yoda-ruleset/api_datarequest.json) diff --git a/docs/design/index.md b/docs/design/index.md index d1bc1c035..84060e8af 100644 --- a/docs/design/index.md +++ b/docs/design/index.md @@ -15,7 +15,6 @@ has_toc: false - [Research space](overview/research-space.md) - [Vault space](overview/vault-space.md) - [Statistics module](overview/statistics.md) -- [Intake module](overview/intake.md) - [Datarequests module](overview/data-requests.md) - [Group manager](overview/group-manager.md) - [Authentication](overview/authentication.md) diff --git a/docs/design/overview/index.md b/docs/design/overview/index.md index dddae25c6..5ec042b08 100644 --- a/docs/design/overview/index.md +++ b/docs/design/overview/index.md @@ -39,11 +39,6 @@ The user module provides settings and notifications. ## Additional modules -### Intake -This module processes uploaded YOUth programme experiment data, and facilitate its archival in the vault by a data manager. - -[Technical design](intake.md){: .btn } - ### Datarequest Module adding support for the submission, review and approval of requests to obtain research data stored in Yoda. diff --git a/docs/design/overview/intake.md b/docs/design/overview/intake.md deleted file mode 100644 index c77dbec21..000000000 --- a/docs/design/overview/intake.md +++ /dev/null @@ -1,135 +0,0 @@ ---- -grand_parent: Software Design -parent: System Overview -nav_order: 8 ---- -# Intake module - -## Introduction - -This page describes the specifications of the Yoda intake module. The main purpose of this module -is to process uploaded YOUth programme experiment data, and facilitate its archival in the vault by a data manager. - -Experiment data would first be uploaded into an intake -folder by a lab assistant or other employee ("intaker"). The intake module has a scan function the -data manager can use to extract dataset information from file and folder names so that data can be grouped -by dataset. Finally, the data manager can use the intake module to lock datasets in the Yoda vault. - -The rest of this page contains specifications of the expected data object and collection names. - -## Data object and collection name specifications - -### Allowed characters - -Data object names (file names) and collection names (folder names) should consist of only these characters: -- Letters, either upper case or lower case (Latin alphabet: a-z, A-Z) -- Digits: 0-9 -- Underscores (\_) -- Hyphens (-) -- Dots (.) - -### WEPV data in names - -Datasets are identified by WEPV properties (Wave, Experiment, Pseudo code, Version) which are derived from data object -and/or collection names. The wave, experiment and pseudo code elements are compulsory. The version element is optional, -and has a default value of "Raw". Either underscores ("\_") or hyphens ("-") can be used as separators between WEPV -elements in data object or collection names. WEPV elements may be defined at different levels in the tree. - -WEPV elements in a path are processed from left to right. The first folder which has complete WEPV data (i.e. wave, experiment and -pseudo code are present in the path of the folder, including its parent folders) is considered to be the -root folder of the dataset. If only the data object itself has complete WEPV data, the data object itself is considered -to be the root of the dataset. - -Here is an example of a data object in a dataset: - - Y:/grp-vault-youth/malu/20w/echo/Y2015M05/D27/20w_B54321/I0000000.vol - ^^^^^^^^^^ - -In this example, the experiment type element ("echo") is defined at a different level from the wave ("20w") and -pseudo ID ("B54321") element. The folder indicated by the carets is considered to be the root folder of the dataset, -since it is the topmost folder with complete WEPV element data. - -#### Wave values - -Waves before birth have a "w" (week) suffix, whereas waves after birth have a month ("m") or year ("y") suffix. - -The 9y, 12y and 15y waves are part of the adolescent cohort, whereas the other waves are part of the baby / child cohort. - -Valid waves are: -- 20w -- 30w -- 0m -- 5m -- 10m -- 3y -- 6y -- 9y -- 12y -- 15y - -#### Experiment type values - -Valid experiment types are: - -- pci -- echo -- faceemo -- coherence -- infprogap -- infsgaze -- infpop -- chprogap -- chantigap -- chsgaze -- pciconflict -- pcivacation -- peabody -- discount -- cyberball -- trustgame -- other -- inhibmockbehav -- inhibmribehav -- emotionmribehav -- emotionmriscan -- anatomymriscan -- restingstatemriscan -- dtiamriscan -- dtipmriscan -- mriqcreport -- mriqceval -- vasmri -- vasmock -- looklisten -- handgame -- infpeabody -- delaygratification -- dtimriscan -- inhibmriscan -- chdualet -- functionalmriscan -- infdualet - -The "other" type is used for questionnaires, logs and unknown types of data. - -#### Pseudo code format - -Pseudo codes consist of a letter "a" (adolescent), "b" (baby) or "p" (adolescent pilot) followed -by five digits. - -#### Version format - -Version elements consist of "ver", followed by a version name. The version element is optional, and -defaults to a version named "Raw". The version name must begin with an upper case letter. The -version name must consist of only alphanumeric characters. - -Examples: -- verRaw -- verVersionName - -## Acknowledgements and provenance - -This page is based on the original specification document titled _Yoda — Youth Cohort Data Intake Specificaties_, -by Jonas Sweep, Chris Smeele and Ton Smeele, version 0.12. Information regarding quality criteria specific to experiment -types has not been copied from the original document, since quality checks specific to experiment types have been removed -from the intake module. diff --git a/docs/design/processes/asynchronous-processes.md b/docs/design/processes/asynchronous-processes.md index df8740288..e312ae005 100644 --- a/docs/design/processes/asynchronous-processes.md +++ b/docs/design/processes/asynchronous-processes.md @@ -19,7 +19,6 @@ This page contains an overview of asynchronous processes in Yoda. * [Statistics job](#statistics-job) * [Archiving](#archiving) * [Archiving - retry-copy-to-vault job](#archiving-retry-copy-job) -* [Archiving - intake to vault job](#archiving-intake-to-vault-job) * [Archiving - publication job](#archiving-publication-job) @@ -141,7 +140,7 @@ and communities. ## Archiving and publication -Asynchronous jobs are also used to copy data packages from a research or intake folder to the vault, +Asynchronous jobs are also used to copy data packages from a research folder to the vault, as well as to process publications. @@ -158,17 +157,6 @@ as well as to process publications. By default, groups that are to be copied to the vault are marked with a metadata attribute named `cronjob_copy_to_vault`. - - -### Intake copy to vault job - -| | | -|---|---| -| Script | /var/lib/irods/.irods/job_movetovault.r | -| Purpose | copy data packages from intake groups to the vault | -| Lock file | uses lock attribute on the vault folder | -| Typically started by | cronjob, runs every five minutes | - ### Process publication job diff --git a/environments/development/allinone/group_vars/allinone.yml b/environments/development/allinone/group_vars/allinone.yml index 13ccaa0a2..75a73a747 100644 --- a/environments/development/allinone/group_vars/allinone.yml +++ b/environments/development/allinone/group_vars/allinone.yml @@ -35,9 +35,6 @@ opensearch_server: combined.yoda.test # Deposit module configuration enable_deposit: true # Enable deposit module -# Intake module configuration -enable_intake: true # Enable intake module - # Datarequest module configuration enable_datarequest: true # Enable datarequest module datarequest_help_contact_name: PLACEHOLDER # Help contact name diff --git a/environments/development/el8-database/group_vars/el8.yml b/environments/development/el8-database/group_vars/el8.yml index 9d7df09af..3c11df825 100644 --- a/environments/development/el8-database/group_vars/el8.yml +++ b/environments/development/el8-database/group_vars/el8.yml @@ -30,9 +30,6 @@ opensearch_server: combined.yoda.test # Deposit module configuration enable_deposit: true # Enable deposit module -# Intake module configuration -enable_intake: true # Enable intake module - # Datarequest module configuration enable_datarequest: true # Enable datarequest module datarequest_help_contact_name: PLACEHOLDER # Help contact name diff --git a/environments/development/full/group_vars/full.yml b/environments/development/full/group_vars/full.yml index fb281b31a..74f971638 100644 --- a/environments/development/full/group_vars/full.yml +++ b/environments/development/full/group_vars/full.yml @@ -32,9 +32,6 @@ opensearch_server: portal.yoda.test # Deposit module configuration enable_deposit: true # Enable deposit module -# Intake module configuration -enable_intake: true # Enable intake module - # Datarequest module configuration enable_datarequest: true # Enable datarequest module datarequest_help_contact_name: PLACEHOLDER # Help contact name diff --git a/environments/development/single_fqdn/group_vars/single_fqdn.yml b/environments/development/single_fqdn/group_vars/single_fqdn.yml index cdb316210..2c89603ea 100644 --- a/environments/development/single_fqdn/group_vars/single_fqdn.yml +++ b/environments/development/single_fqdn/group_vars/single_fqdn.yml @@ -37,9 +37,6 @@ opensearch_server: combined.yoda.test # Deposit module configuration enable_deposit: true # Enable deposit module -# Intake module configuration -enable_intake: true # Enable intake module - # Datarequest module configuration enable_datarequest: true # Enable datarequest module datarequest_help_contact_name: PLACEHOLDER # Help contact name diff --git a/environments/development/surf/group_vars/surf.yml b/environments/development/surf/group_vars/surf.yml index 2bb2e6d98..e0d911a09 100644 --- a/environments/development/surf/group_vars/surf.yml +++ b/environments/development/surf/group_vars/surf.yml @@ -32,9 +32,6 @@ opensearch_server: PLACEHOLDER # Deposit module configuration enable_deposit: false # Surf doesn't use deposit module -# Intake module configuration -enable_intake: false # Surf doesn't use intake module - # Datarequest module configuration enable_datarequest: false # Surf doesn't use DR module datarequest_help_contact_name: PLACEHOLDER # Help contact name diff --git a/environments/development/ubuntu/group_vars/ubuntu.yml b/environments/development/ubuntu/group_vars/ubuntu.yml index 1e4e01f49..8a18b318c 100644 --- a/environments/development/ubuntu/group_vars/ubuntu.yml +++ b/environments/development/ubuntu/group_vars/ubuntu.yml @@ -32,9 +32,6 @@ opensearch_server: combined.yoda.test # Deposit module configuration enable_deposit: true # Enable deposit module -# Intake module configuration -enable_intake: true # Enable intake module - # Datarequest module configuration enable_datarequest: true # Enable datarequest module datarequest_help_contact_name: PLACEHOLDER # Help contact name diff --git a/roles/yoda_portal/defaults/main.yml b/roles/yoda_portal/defaults/main.yml index 5fa6c1a35..73326c45f 100644 --- a/roles/yoda_portal/defaults/main.yml +++ b/roles/yoda_portal/defaults/main.yml @@ -70,10 +70,6 @@ enable_open_search: false # Enable OpenSearch module # Deposit module configuration enable_deposit: false # Enable deposit module -# Intake module configuration -enable_intake: false # Enable intake module -intake_extended_timeout: 1800 - # Datarequest module configuration enable_datarequest: false # Enable datarequest module datarequest_help_contact_name: PLACEHOLDER # Help contact name diff --git a/roles/yoda_portal/templates/flask.cfg.j2 b/roles/yoda_portal/templates/flask.cfg.j2 index 7725b492b..eb4e766cd 100644 --- a/roles/yoda_portal/templates/flask.cfg.j2 +++ b/roles/yoda_portal/templates/flask.cfg.j2 @@ -10,8 +10,6 @@ YODA_COMMIT = '{{ yoda_commit }}' RESEARCH_ENABLED = {{ enable_research }} OPEN_SEARCH_ENABLED = {{ enable_open_search }} DEPOSIT_ENABLED = {{ enable_deposit }} -INTAKE_ENABLED = {{ enable_intake }} -INTAKE_EXT_TIMEOUT = {{ intake_extended_timeout }} DATAREQUEST_ENABLED = {{ enable_datarequest }} TOKENS_ENABLED = {{ enable_tokens }} TOKEN_LIFETIME = {{ token_lifetime }} diff --git a/roles/yoda_portal/templates/yoda-portal-vhost.conf.j2 b/roles/yoda_portal/templates/yoda-portal-vhost.conf.j2 index 963fb8bc1..1c3870473 100644 --- a/roles/yoda_portal/templates/yoda-portal-vhost.conf.j2 +++ b/roles/yoda_portal/templates/yoda-portal-vhost.conf.j2 @@ -19,10 +19,6 @@ Require all granted -{% if enable_intake %} - Timeout {{ intake_extended_timeout }} -{% endif %} - Alias "/static/" "/var/www/yoda/static/" Order allow,deny diff --git a/roles/yoda_rulesets/defaults/main.yml b/roles/yoda_rulesets/defaults/main.yml index 9499a58e8..0f3f0c7bc 100644 --- a/roles/yoda_rulesets/defaults/main.yml +++ b/roles/yoda_rulesets/defaults/main.yml @@ -145,11 +145,6 @@ enable_open_search: true # Enable OpenSearch module # Deposit module configuration enable_deposit: false # Enable deposit module -# Intake module configuration -enable_intake: false # Enable intake module -intake_groups: [] # List of intake_groups. Use the names without "grp-intake-" - # For example: "projectabc" for "grp-intake-projectabc" - # Data Package Reference enable_data_package_reference: false # Enable Data Package References for vault data packages diff --git a/roles/yoda_rulesets/files/run-intake-movetovault.sh b/roles/yoda_rulesets/files/run-intake-movetovault.sh deleted file mode 100644 index d6e867ea3..000000000 --- a/roles/yoda_rulesets/files/run-intake-movetovault.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -/bin/irule -r irods_rule_engine_plugin-irods_rule_language-instance \ - -F /var/lib/irods/.irods/job_movetovault.r \ - >>$HOME/log/job_movetovault.log 2>&1 - -# Check if the job is still running. This can happen if the previous -# job hasn't finished before this one started. In this case, the previous -# job will clear the locks when it's finished, so this job should -# leave the locks in place. -if pgrep -u irods -f /var/lib/irods/.irods/job_movetovault.r >& /dev/null -then echo "job_movetovault still running; not clearing locks ..." -else /bin/irule -r irods_rule_engine_plugin-irods_rule_language-instance \ - -F /var/lib/irods/.irods/job_clearintakelocks.r 2>&1 -fi diff --git a/roles/yoda_rulesets/tasks/yoda-ruleset.yml b/roles/yoda_rulesets/tasks/yoda-ruleset.yml index 9686c6a12..eafa993f9 100644 --- a/roles/yoda_rulesets/tasks/yoda-ruleset.yml +++ b/roles/yoda_rulesets/tasks/yoda-ruleset.yml @@ -367,30 +367,20 @@ mode: '0644' -- name: Install script for moving intake collections to the vault +- name: Ensure job_movetovault.r is absent become_user: "{{ irods_service_account }}" become: true - ansible.builtin.template: - src: job_movetovault.r.j2 - dest: '{{ irods_directory.stdout }}/.irods/job_movetovault.r' - owner: '{{ irods_service_account }}' - group: '{{ irods_service_account }}' - mode: '0644' - # This script is provided by the test playbook for development/test servers. - when: enable_intake and yoda_environment not in ["development", "testing"] + ansible.builtin.file: + path: '{{ irods_directory.stdout }}/.irods/job_movetovault.r' + state: absent -- name: Install script for clearing locks after moving intake collections +- name: Ensure job_clearintakelocks.r is absent become_user: "{{ irods_service_account }}" become: true - ansible.builtin.template: - src: job_clearintakelocks.r.j2 - dest: '{{ irods_directory.stdout }}/.irods/job_clearintakelocks.r' - owner: '{{ irods_service_account }}' - group: '{{ irods_service_account }}' - mode: '0644' - # This script is provided by the test playbook for development/test servers. - when: enable_intake and yoda_environment not in ["development", "testing"] + ansible.builtin.file: + path: '{{ irods_directory.stdout }}/.irods/job_clearintakelocks.r' + state: absent - name: Configure cronjob to process publications @@ -434,28 +424,22 @@ state: '{{ "present" if enable_revision_cleanup else "absent" }}' -- name: Ensure cronjob script for moving collections from intake to vault is installed +- name: Ensure run-intake-movetovault.sh is absent become_user: "{{ irods_service_account }}" become: true - ansible.builtin.copy: - src: run-intake-movetovault.sh - dest: '{{ irods_directory.stdout }}/.irods/run-intake-movetovault.sh' - owner: '{{ irods_service_account }}' - group: '{{ irods_service_account }}' - mode: '0755' - when: enable_intake or yoda_environment in ["development", "testing"] + ansible.builtin.file: + path: '{{ irods_directory.stdout }}/.irods/run-intake-movetovault.sh' + state: absent -- name: Configure cronjob for moving collections from intake to vault +- name: Ensure cronjob for moving collections from intake is absent become_user: "{{ irods_service_account }}" become: true - # Name kept for historical reasons, so that the playbook updates old-style - # cronjobs that invoked the rule directly. ansible.builtin.cron: name: 'job_movetovault.r' minute: '*/5' job: '{{ irods_directory.stdout }}/.irods/run-intake-movetovault.sh >/dev/null 2>&1' - state: '{{ "present" if enable_intake else "absent" }}' + state: absent - name: Set configuration AVUs for research ruleset to system collection diff --git a/roles/yoda_rulesets/templates/job_clearintakelocks.r.j2 b/roles/yoda_rulesets/templates/job_clearintakelocks.r.j2 deleted file mode 100644 index 75c21ca4b..000000000 --- a/roles/yoda_rulesets/templates/job_clearintakelocks.r.j2 +++ /dev/null @@ -1,29 +0,0 @@ -# \file -# \brief job -# \author Sietse Snel -# \copyright Copyright (c) 2022, Utrecht university. All rights reserved -# \license GPLv3, see LICENSE -# -# This file should be executed as part of a recurring crontab job -# as the irods admin user (i.e. irods user type rodsadmin) -# e.g. run every 5 minutes -# -# It clears all locks on intake folders after job_movetovault has run, in -# order to facilitate retries after transient errors -# - -uuYcClearIntakeLocks { - # intake areas can be added to the grouplist as needed - *grouplist = list ( {{ intake_groups | map("regex_replace","(.+)",'\"\\1\"') | join(',') }} ); - *zone = $rodsZoneClient; - - foreach (*grp in *grouplist) { - *intakeRoot = "/*zone/home/grp-intake-*grp"; - *vaultRoot = "/*zone/home/grp-vault-*grp"; - # uuUnlock also succeeds if lock doesn't exist, so no need to verify that lock exists first - uuUnlock(*vaultRoot); - } -} - -input *intakeRoot='dummy' -output ruleExecOut diff --git a/roles/yoda_rulesets/templates/job_movetovault.r.j2 b/roles/yoda_rulesets/templates/job_movetovault.r.j2 deleted file mode 100644 index 3b65da1a6..000000000 --- a/roles/yoda_rulesets/templates/job_movetovault.r.j2 +++ /dev/null @@ -1,71 +0,0 @@ -# \file -# \brief job -# \author Ton Smeele -# \copyright Copyright (c) 2015, Utrecht university. All rights reserved -# \license GPLv3, see LICENSE -# -# This file should be executed as part of a recurring crontab job -# as the irods admin user (i.e. irods user type rodsadmin) -# e.g. run every 5 minutes -# -# if another instance of the job is running then the vault will be -# locked and silently ignored -# - -uuYcRunIntake2Vault { - # intake areas can be added to the grouplist as needed - *grouplist = list ( {{ intake_groups | map("regex_replace","(.+)",'\"\\1\"') | join(',') }} ); - *grp_prefixes = list ("grp-intake-", "intake-"); - *zone = $rodsZoneClient; - - foreach (*grp in *grouplist) { - *intakeRoot = ""; - *vlt_name = ""; - *found = false; - - foreach (*prefix in *grp_prefixes) { - *grp_name = *prefix ++ *grp; - - if (*prefix == "intake-" ) { - *vlt_name = "vault-" ++ *grp; - } - else { - *vlt_name = "grp-vault-" ++ *grp; - } - - foreach(*row in SELECT USER_GROUP_NAME - WHERE USER_GROUP_NAME = '*grp_name' - AND USER_TYPE = 'rodsgroup') { - - *path=*row.USER_GROUP_NAME; - *found = true; - break; - } - if (*found) { - break; - } - } - - if ( *found ) { - *intakeRoot = "/*zone/home/*grp_name"; - *vaultRoot = "/*zone/home/*vlt_name"; - writeLine("serverLog","RunIntake2Vault for *intakeRoot -> *vaultRoot ..."); - - uuLock(*vaultRoot, *status); - if (*status == 0) { - # we have a lock - uuYc2Vault(*intakeRoot, *vaultRoot, *status); - if (*status == 0 ) then *result = "ok" else *result = "ERROR (*status)"; - writeLine("serverLog","RunIntake2Vault for *intakeRoot result = *result"); - uuUnlock(*vaultRoot); - } - } - else { - writeLine("serverLog", "Intake2vault was unable to find group *grp."); - } - } -} - - -input *intakeRoot='dummy' -output ruleExecOut diff --git a/roles/yoda_rulesets/templates/rules_uu.cfg.j2 b/roles/yoda_rulesets/templates/rules_uu.cfg.j2 index e2312b1ef..d87c3aaf0 100644 --- a/roles/yoda_rulesets/templates/rules_uu.cfg.j2 +++ b/roles/yoda_rulesets/templates/rules_uu.cfg.j2 @@ -41,7 +41,6 @@ eus_api_tls_verify = '{{ ["false", "true"][eus_api_tls_verify|int] } enable_deposit = '{{ ["false", "true"][enable_deposit|int] }}' enable_open_search = '{{ ["false", "true"][enable_open_search|int] }}' -enable_intake = '{{ ["false", "true"][enable_intake|int] }}' enable_datarequest = '{{ ["false", "true"][enable_datarequest|int] }}' yoda_portal_fqdn = '{{ yoda_portal_fqdn }}' diff --git a/roles/yoda_test/files/job_checksums.r b/roles/yoda_test/files/job_checksums.r deleted file mode 100644 index 4702fc528..000000000 --- a/roles/yoda_test/files/job_checksums.r +++ /dev/null @@ -1,21 +0,0 @@ -# \file job_checksums.r -# \brief Youth Cohort - generate checksum index of Vault -# \author Ton Smeele -# \copyright Copyright (c) 2016, Utrecht University. All rights reserved. -# \license GPLv3, see LICENSE - -job_checksums { - *zone = $rodsZoneClient; - *root = "/*zone/home/grp-vault-initial"; -# NB: logfile location for test purpose, requires new position upon production! - *logfile = "/*zone/home/grp-datamanager-initial/checksums.txt"; - uuYcGenerateDatasetsIndex(*root, *logfile, *status); - writeLine("stdout","return status is *status"); - *root = "/*zone/home/grp-vault-test"; - *logfile = "/*zone/home/grp-datamanager-test/checksums.txt"; - uuYcGenerateDatasetsIndex(*root, *logfile, *status); - writeLine("stdout","return status is *status"); -} - -input null -output ruleExecOut \ No newline at end of file diff --git a/roles/yoda_test/files/job_clearintakelocks.r b/roles/yoda_test/files/job_clearintakelocks.r deleted file mode 100644 index bd5a67fe5..000000000 --- a/roles/yoda_test/files/job_clearintakelocks.r +++ /dev/null @@ -1,29 +0,0 @@ -# \file -# \brief job -# \author Sietse Snel -# \copyright Copyright (c) 2022, Utrecht university. All rights reserved -# \license GPLv3, see LICENSE -# -# This file should be executed as part of a recurring crontab job -# as the irods admin user (i.e. irods user type rodsadmin) -# e.g. run every 5 minutes -# -# It clears all locks on intake folders after job_movetovault has run, in -# order to facilitate retries after transient errors -# - -uuYcClearIntakeLocks { - # intake areas can be added to the grouplist as needed - *grouplist = list ("initial","test"); - *zone = $rodsZoneClient; - - foreach (*grp in *grouplist) { - *intakeRoot = "/*zone/home/grp-intake-*grp"; - *vaultRoot = "/*zone/home/grp-vault-*grp"; - # uuUnlock also succeeds if lock doesn't exist, so no need to verify that lock exists first - uuUnlock(*vaultRoot); - } -} - -input *intakeRoot='dummy' -output ruleExecOut diff --git a/roles/yoda_test/files/job_movetovault.r b/roles/yoda_test/files/job_movetovault.r deleted file mode 100644 index fb8299433..000000000 --- a/roles/yoda_test/files/job_movetovault.r +++ /dev/null @@ -1,67 +0,0 @@ -# \file -# \brief job -# \author Ton Smeele -# \copyright Copyright (c) 2015, Utrecht university. All rights reserved -# \license GPLv3, see LICENSE -# -# if another instance of the job is running then the vault will be -# locked and silently ignored -# - -uuYcRunIntake2Vault { - # intake areas can be added to the grouplist as needed - *grouplist = list ("initial", "test", "test2"); - *grp_prefixes = list ("grp-intake-", "intake-"); - *zone = $rodsZoneClient; - - foreach (*grp in *grouplist) { - *intakeRoot = ""; - *vlt_name = ""; - *found = false; - - foreach (*prefix in *grp_prefixes) { - *grp_name = *prefix ++ *grp; - - if (*prefix == "intake-" ) { - *vlt_name = "vault-" ++ *grp; - } - else { - *vlt_name = "grp-vault-" ++ *grp; - } - - foreach(*row in SELECT USER_GROUP_NAME - WHERE USER_GROUP_NAME = '*grp_name' - AND USER_TYPE = 'rodsgroup') { - - *path=*row.USER_GROUP_NAME; - *found = true; - break; - } - if (*found) { - break; - } - } - - if ( *found ) { - *intakeRoot = "/*zone/home/*grp_name"; - *vaultRoot = "/*zone/home/*vlt_name"; - writeLine("serverLog","RunIntake2Vault for *intakeRoot -> *vaultRoot ..."); - - uuLock(*vaultRoot, *status); - if (*status == 0) { - # we have a lock - uuYc2Vault(*intakeRoot, *vaultRoot, *status); - if (*status == 0 ) then *result = "ok" else *result = "ERROR (*status)"; - writeLine("serverLog","RunIntake2Vault for *intakeRoot result = *result"); - uuUnlock(*vaultRoot); - } - } - else { - writeLine("serverLog", "Intake2vault was unable to find group *grp."); - } - } -} - - -input *intakeRoot='dummy' -output ruleExecOut diff --git a/roles/yoda_test/tasks/install-data.yml b/roles/yoda_test/tasks/install-data.yml index 5221d2a59..fe4ca1843 100644 --- a/roles/yoda_test/tasks/install-data.yml +++ b/roles/yoda_test/tasks/install-data.yml @@ -132,7 +132,7 @@ ansible.builtin.command: iput -f ~/testdata/{{ item }} /{{ irods_zone }}/home/research-initial/testdata with_items: - - 'creatures.json' + - 'creatures.json' - 'image.txt' - 'large-file.html' - 'lorem.txt' @@ -194,26 +194,26 @@ ignore_errors: true -- name: Ensure intake module rule files are present +- name: Ensure intake module rule files are absent become_user: '{{ irods_service_account }}' become: true - ansible.builtin.copy: - src: '{{ item }}' - dest: '~/.irods/{{ item }}' - mode: 0644 - with_items: + ansible.builtin.file: + path: '~/.irods/{{ item }}' + state: absent + loop: - 'job_movetovault.r' - 'job_checksums.r' - 'job_clearintakelocks.r' -- name: Configure cronjob to make checksums of datapackages in grp-vault-* +- name: Ensure cronjob to make checksums of datapackages in grp-vault-* is absent become_user: '{{ irods_service_account }}' become: true ansible.builtin.cron: name: 'job_checksums.r' minute: '*/7' job: '/bin/irule -r irods_rule_engine_plugin-irods_rule_language-instance -F {{ irods_directory.stdout }}/.irods/job_checksums.r >> /var/lib/irods/log/job_checksums.log 2>&1' + state: absent - name: Run storage statistics job