From f23719ff449af50c599f173db643056ab718039d Mon Sep 17 00:00:00 2001 From: Sietse Snel Date: Mon, 2 Dec 2024 17:04:31 +0100 Subject: [PATCH 01/16] docs: fix missing newline at end of file --- docs/administration/configuring-yoda.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/administration/configuring-yoda.md b/docs/administration/configuring-yoda.md index cf630871b..22255f0b4 100644 --- a/docs/administration/configuring-yoda.md +++ b/docs/administration/configuring-yoda.md @@ -519,4 +519,4 @@ yoda_rulesets_vault_copy_multithread_enabled | Enable multithreading when copyin Variable | Description --------------------|----------------------------------------------------------- -enable_yoda_report | Enable installation of yoda report for financial reporting \ No newline at end of file +enable_yoda_report | Enable installation of yoda report for financial reporting From adaf09f6852c5fcf643058706e5534f2782dfa1d Mon Sep 17 00:00:00 2001 From: Sietse Snel Date: Mon, 2 Dec 2024 17:12:24 +0100 Subject: [PATCH 02/16] Fix DAT expiry notification cronjob settings The cronjob was set to run once a month. However, by default, data access tokens are valid for three days. So we need to send notifications more often if we want users to reliably receive a notification before their token expires. --- roles/yoda_rulesets/tasks/yoda-ruleset.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/yoda_rulesets/tasks/yoda-ruleset.yml b/roles/yoda_rulesets/tasks/yoda-ruleset.yml index eafa993f9..ea2228771 100644 --- a/roles/yoda_rulesets/tasks/yoda-ruleset.yml +++ b/roles/yoda_rulesets/tasks/yoda-ruleset.yml @@ -584,7 +584,6 @@ name: 'notification-data-access-token-expiry.sh' minute: '45' hour: '04' - day: '1' job: '/bin/bash /etc/irods/yoda-ruleset/tools/notification/notification-data-access-token-expiry.sh >/dev/null 2>&1' state: '{{ "present" if enable_tokens else "absent" }}' From e9a10e4a835f1ba2efdb28adbff4aeae5df3b529 Mon Sep 17 00:00:00 2001 From: Sietse Snel Date: Mon, 2 Dec 2024 16:52:45 +0100 Subject: [PATCH 03/16] YDA-6036: Add portal timeout parameter Add a virtual host timeout parameter for the Yoda portal. --- docs/administration/configuring-yoda.md | 1 + roles/yoda_portal/defaults/main.yml | 3 +++ roles/yoda_portal/templates/yoda-portal-vhost.conf.j2 | 1 + 3 files changed, 5 insertions(+) diff --git a/docs/administration/configuring-yoda.md b/docs/administration/configuring-yoda.md index 22255f0b4..39c33ae74 100644 --- a/docs/administration/configuring-yoda.md +++ b/docs/administration/configuring-yoda.md @@ -157,6 +157,7 @@ tcp_keepalive_intvl | IPv4 TCP keepalives: time between keepalives yoda_theme | The theme to use for the Yoda Portal. See also [the theme documentation](../design/overview/theme-packages.md). By default, Yoda uses the UU theme. yoda_theme_path | Path where themes for the Yoda Portal are retrieved from. See [the theme documentation](../design/overview/theme-packages.md) for more information. portal_session_cookie_samesite | Samesite setting for session cookies Yoda Portal. Should be 'Lax' if OIDC is enabled and identity provider is in different domain. Otherwise it should be 'Strict'. Default value: 'Strict'. +yoda_portal_timeout | The web server timeout for requests to the Yoda portal, in seconds (default: 60) yoda_portal_wsgi_daemon_threads | The number of threads to be created to handle requests in each daemon process (default: 15) yoda_portal_upload_part_files | Whether the portal uploader function should upload multi-chunk files as .part files initially and rename them to their final name later (boolean value, default: true). It is generally recommended to keep this enabled, so that users can easily see when an upload failed and the result is partial. However, on storage systems where renaming data objects takes much time, such as S3 object storage in consistent mode, it may be necessary to switch use of .part files off. yoda_portal_monitor_signal_file | Path to the signal file for the portal monitoring thread. If this file is present, the monitor thread will start logging technical support information to the directory configured in `yoda_portal_monitor_output_dir`. Set to empty string to disable the monitoring thread. Default value: /var/www/yoda/show-tech.sig diff --git a/roles/yoda_portal/defaults/main.yml b/roles/yoda_portal/defaults/main.yml index 73326c45f..75ce95ab9 100644 --- a/roles/yoda_portal/defaults/main.yml +++ b/roles/yoda_portal/defaults/main.yml @@ -82,6 +82,9 @@ token_lifetime: 72 # Lifetime of data access tok # SRAM Configuration enable_sram: false # Enable SRAM configuration +# Apache timeout for Yoda portal vhost in seconds +yoda_portal_timeout: 60 + # Monitor thread configuration for extracting tech support information yoda_portal_monitor_output_dir: /tmp yoda_portal_monitor_signal_file: /var/www/yoda/show-tech.sig diff --git a/roles/yoda_portal/templates/yoda-portal-vhost.conf.j2 b/roles/yoda_portal/templates/yoda-portal-vhost.conf.j2 index 1c3870473..15f136ce1 100644 --- a/roles/yoda_portal/templates/yoda-portal-vhost.conf.j2 +++ b/roles/yoda_portal/templates/yoda-portal-vhost.conf.j2 @@ -13,6 +13,7 @@ DocumentRoot "/var/www/yoda/public" ServerName {{ yoda_portal_fqdn }} + Timeout {{ yoda_portal_timeout }} AllowOverride All From 0287f68e0495068d923aade352372c0c2101dbbe Mon Sep 17 00:00:00 2001 From: Sietse Snel Date: Thu, 5 Dec 2024 08:34:05 +0100 Subject: [PATCH 04/16] GoCommands: update to v0.10.7 --- roles/irods_gocommands/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/irods_gocommands/defaults/main.yml b/roles/irods_gocommands/defaults/main.yml index 37303346b..f8a5c9399 100644 --- a/roles/irods_gocommands/defaults/main.yml +++ b/roles/irods_gocommands/defaults/main.yml @@ -4,5 +4,5 @@ irods_service_account: irods irods_enable_gocommands: false -irods_gocommands_version: 0.10.5 -irods_gocommands_archive_checksum: "sha256:fef135ea54aad9fbbe8a2f375cf534e61e0c273a1df01375af462f34b49161a5" +irods_gocommands_version: 0.10.7 +irods_gocommands_archive_checksum: "sha256:1569dfb4784a1f8ba6ce8d035cc16b439ee2c95f258a30880c1e546a8d58fb0b" From e688dcb89990fb76f5bfc8ac59e255e45dac5cce Mon Sep 17 00:00:00 2001 From: Sietse Snel Date: Mon, 9 Dec 2024 17:26:16 +0100 Subject: [PATCH 05/16] YDA-5994: modify Postfix role to allow queue cmd Modify the Postfix role to ensure that the AppArmor profile allows viewing and flushing the mail queue using the postqueue command. Also ensure that any AppArmor profile changes will be reloaded after modifications. A part of the issue was already fixed before in the changes for ticket YDA-5971. --- roles/postfix/handlers/main.yml | 6 ++++++ roles/postfix/tasks/main.yml | 2 ++ roles/postfix/templates/usr.sbin.postfix.j2 | 4 +++- roles/postfix/templates/usr.sbin.postqueue.j2 | 6 +++--- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/roles/postfix/handlers/main.yml b/roles/postfix/handlers/main.yml index fab0348f9..2b92bb4d4 100644 --- a/roles/postfix/handlers/main.yml +++ b/roles/postfix/handlers/main.yml @@ -8,6 +8,12 @@ when: not ansible_check_mode +- name: Reload Apparmor + ansible.builtin.service: + name: apparmor + state: reloaded + + - name: Systemd daemon reload ansible.builtin.systemd: daemon_reload: true diff --git a/roles/postfix/tasks/main.yml b/roles/postfix/tasks/main.yml index b8004b36f..f666d606d 100644 --- a/roles/postfix/tasks/main.yml +++ b/roles/postfix/tasks/main.yml @@ -180,6 +180,7 @@ - usr.sbin.postalias - usr.sbin.sendmail.postfix register: postfix_apparmor_default + notify: Reload Apparmor when: not ansible_check_mode and ansible_os_family == "Debian" @@ -202,6 +203,7 @@ - src: 'usr.sbin.sendmail.j2' dest: 'usr.sbin.sendmail' register: postfix_apparmor_custom + notify: Reload Apparmor when: ansible_os_family == "Debian" diff --git a/roles/postfix/templates/usr.sbin.postfix.j2 b/roles/postfix/templates/usr.sbin.postfix.j2 index ca3ad8753..6329554ca 100644 --- a/roles/postfix/templates/usr.sbin.postfix.j2 +++ b/roles/postfix/templates/usr.sbin.postfix.j2 @@ -1,5 +1,5 @@ # {{ ansible_managed }} -# Last Modified: Tue Oct 22 15:17:36 2024 +# Last Modified: Mon Dec 9 14:18:33 2024 #include # vim:syntax=apparmor @@ -24,6 +24,7 @@ signal send set=term peer=/usr/sbin/postfix//null-/usr/lib/postfix/sbin/pickup, signal send set=term peer=/usr/sbin/postfix//null-/usr/lib/postfix/sbin/qmgr, signal send set=term peer=/usr/sbin/postfix//null-/usr/lib/postfix/sbin/scache, + signal send set=term peer=/usr/sbin/postfix//null-/usr/lib/postfix/sbin/showq, signal send set=term peer=/usr/sbin/postfix//null-/usr/lib/postfix/sbin/smtp, signal send set=term peer=/usr/sbin/postfix//null-/usr/lib/postfix/sbin/tlsmgr, signal send set=term peer=/usr/sbin/postfix//null-/usr/lib/postfix/sbin/trivial-rewrite, @@ -69,6 +70,7 @@ owner /etc/postfix/postfix-files.d/ r, owner /etc/postfix/sasl/ r, owner /usr/sbin/sendmail r, + owner /var/lib/postfix/__db.smtp_scache.db rw, owner /var/lib/postfix/master.lock rwk, owner /var/lib/postfix/prng_exch k, owner /var/lib/postfix/prng_exch rw, diff --git a/roles/postfix/templates/usr.sbin.postqueue.j2 b/roles/postfix/templates/usr.sbin.postqueue.j2 index 57c904d0f..d5640fa27 100644 --- a/roles/postfix/templates/usr.sbin.postqueue.j2 +++ b/roles/postfix/templates/usr.sbin.postqueue.j2 @@ -1,5 +1,5 @@ # {{ ansible_managed }} -# Last Modified: Tue Oct 22 13:36:14 2024 +# Last Modified: Mon Dec 9 14:13:02 2024 #include # ------------------------------------------------------------------ @@ -31,8 +31,8 @@ /var/spool/postfix/maildrop r, /var/spool/postfix/maildrop/* rwl, /var/spool/postfix/pid r, - /var/spool/postfix/public/pickup w, - /var/spool/postfix/public/qmgr w, + /var/spool/postfix/public/pickup rw, + /var/spool/postfix/public/qmgr rw, /var/spool/postfix/public/showq rw, } From 5919a30a919e91f8980c7bf95f09956971d3eac5 Mon Sep 17 00:00:00 2001 From: Sietse Snel Date: Mon, 9 Dec 2024 12:53:21 +0100 Subject: [PATCH 06/16] YDA-6045: fix Apache global TLS configuration The Apache global TLS configuration was not applied on Ubuntu 20.04 LTS, which resulted in the session cache not being created. This change activates the global TLS configuration, including the configuration of the session cache, so that we have session resumption. We also needed to re-arrange Listen statements for TCP ports to ensure each port has at most one Listen statement --- roles/apache/files/ports.conf.focal | 10 ++++++++++ roles/apache/tasks/main-tasks.yml | 20 ++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 roles/apache/files/ports.conf.focal diff --git a/roles/apache/files/ports.conf.focal b/roles/apache/files/ports.conf.focal new file mode 100644 index 000000000..d7b9e405e --- /dev/null +++ b/roles/apache/files/ports.conf.focal @@ -0,0 +1,10 @@ +# If you just change the port or add more ports here, you will likely also +# have to change the VirtualHost statement in +# /etc/apache2/sites-enabled/000-default.conf + +Listen 80 + +# Listen statements for ports 443 and 8443 are in SSL module configuration and the +# EUS vhost file. + +# vim: syntax=apache ts=4 sw=4 sts=4 sr noet diff --git a/roles/apache/tasks/main-tasks.yml b/roles/apache/tasks/main-tasks.yml index 3b63df48b..cc3a75296 100644 --- a/roles/apache/tasks/main-tasks.yml +++ b/roles/apache/tasks/main-tasks.yml @@ -58,6 +58,16 @@ when: ansible_os_family == 'Debian' +- name: Copy Apache ports configuration file (Ubuntu) + ansible.builtin.copy: + src: ports.conf.focal + dest: /etc/apache2/ports.conf + owner: root + group: root + mode: '0644' + when: ansible_os_family == 'Debian' + + - name: Ensure autoindex.conf is absent ansible.builtin.file: path: '{{ item }}' @@ -89,6 +99,16 @@ with_items: - headers - ssl + - socache_shmcb + when: ansible_os_family == 'Debian' + + +- name: Ensure Apache global SSL configuration is loaded (Ubuntu) + ansible.builtin.file: + src: "/etc/apache2/mods-available/ssl.conf" + dest: "/etc/apache2/mods-enabled/ssl.conf" + state: link + notify: Restart Apache webserver when: ansible_os_family == 'Debian' From 75e7cd4d4d45140d83338bea4f7df893f28e1c20 Mon Sep 17 00:00:00 2001 From: kaur16 Date: Wed, 20 Nov 2024 20:24:20 +0100 Subject: [PATCH 07/16] Update roles for Ubuntu 24.04 and iRODS 4.3.3 --- Vagrantfile | 8 ++- .../{combined => combined.yoda.test} | 0 environments/development/allinone/hosts | 10 +-- library/irods_indexing.py | 7 +- roles/acme_certificates/meta/main.yml | 4 +- roles/apache/templates/usr.sbin.apache2.j2 | 7 +- roles/certificates/meta/main.yml | 4 +- roles/certificates/tasks/main-tasks.yml | 13 +++- roles/common/meta/main.yml | 4 +- roles/common/tasks/basics-redhat.yml | 10 +++ roles/common/tasks/irods-debian.yml | 2 +- roles/composable_resources/meta/main.yml | 4 +- roles/dms_archive_mock/meta/main.yml | 4 +- roles/dms_tape_archive/meta/main.yml | 4 +- roles/icat_database_checker/meta/main.yml | 4 +- roles/icat_database_checker/tasks/main.yml | 17 +---- roles/irods_arb/meta/main.yml | 4 +- roles/irods_arb/tasks/main.yml | 4 +- roles/irods_consistency_check/meta/main.yml | 4 +- roles/irods_consistency_check/tasks/main.yml | 17 +---- roles/irods_database/meta/main.yml | 4 +- roles/irods_gocommands/meta/main.yml | 4 +- roles/irods_icat/defaults/main.yml | 2 +- roles/irods_icat/meta/main.yml | 7 +- roles/irods_icat/tasks/main.yml | 6 +- roles/irods_icat/tasks/setup_pam.yml | 8 +-- .../templates/irods_indexing.pref.j2 | 6 -- .../templates/setup_irods_provider.json.j2 | 6 +- roles/irods_icat/vars/Debian.yml | 14 ++-- roles/irods_icat/vars/RedHat.yml | 14 ++-- roles/irods_icommands/meta/main.yml | 4 +- roles/irods_icommands/vars/Debian.yml | 2 +- roles/irods_icommands/vars/RedHat.yml | 2 +- roles/irods_microservices/meta/main.yml | 4 +- roles/irods_microservices/vars/Debian.yml | 16 ++--- roles/irods_remove_old_version/meta/main.yml | 4 +- .../irods_remove_old_version/vars/Debian.yml | 22 +++--- .../irods_remove_old_version/vars/RedHat.yml | 22 +++--- roles/irods_resource/defaults/main.yml | 2 +- roles/irods_resource/meta/main.yml | 8 +-- roles/irods_resource/vars/Debian.yml | 6 +- roles/irods_resource/vars/RedHat.yml | 6 +- roles/irods_resource_plugin_s3/meta/main.yml | 4 +- .../irods_resource_plugin_s3/vars/Debian.yml | 2 +- .../irods_resource_plugin_s3/vars/RedHat.yml | 2 +- roles/irods_runtime/meta/main.yml | 6 +- roles/irods_runtime/tasks/main-tasks.yml | 36 +++++----- roles/irods_runtime/vars/Debian.yml | 6 +- roles/irods_runtime/vars/RedHat.yml | 6 +- .../defaults/main.yml | 22 ------ roles/irods_ubuntu_dependencies/meta/main.yml | 15 ---- .../tasks/main-tasks.yml | 72 ------------------- .../irods_ubuntu_dependencies/tasks/main.yml | 11 --- roles/mailpit/meta/main.yml | 3 +- roles/minio/meta/main.yml | 4 +- roles/nvm/meta/main.yml | 4 +- roles/opensearch/meta/main.yml | 4 +- roles/pam_python/meta/main.yml | 4 +- roles/pgbouncer/meta/main.yml | 4 +- roles/postfix/meta/main.yml | 4 +- roles/postgresql/meta/main.yml | 4 +- roles/postgresql/tasks/setup-redhat.yml | 14 ++-- roles/postgresql_odbc/meta/main.yml | 4 +- roles/postgresql_odbc/tasks/setup-debian.yml | 2 + roles/postgresql_odbc/tasks/setup-redhat.yml | 1 + roles/postgresql_repository/defaults/main.yml | 2 +- roles/postgresql_repository/meta/main.yml | 4 +- roles/python2/meta/main.yml | 17 ----- roles/python2/tasks/install-debian.yml | 24 ------- roles/python2/tasks/install-redhat.yml | 18 ----- roles/python2/tasks/main-tasks.yml | 11 --- roles/python2/tasks/main.yml | 11 --- roles/python3/tasks/install-debian.yml | 1 - roles/python_irodsclient/meta/main.yml | 5 +- roles/python_irodsclient/tasks/main-tasks.yml | 18 +---- roles/python_irodsclient/vars/Debian.yml | 1 - roles/python_irodsclient/vars/RedHat.yml | 1 - roles/sqlcipher/meta/main.yml | 2 +- roles/sqlcipher/tasks/install-ubuntu.yml | 25 +------ roles/yoda_davrods/meta/main.yml | 4 +- roles/yoda_davrods/vars/Debian.yml | 2 +- roles/yoda_davrods/vars/RedHat.yml | 8 +-- .../defaults/main.yml | 2 +- .../yoda_external_user_service/tasks/main.yml | 20 ++++-- .../vars/Debian.yml | 2 +- roles/yoda_moai/defaults/main.yml | 2 +- roles/yoda_moai/tasks/main.yml | 38 ++++------ roles/yoda_moai/templates/moai.wsgi.j2 | 4 -- roles/yoda_moai/vars/Debian.yml | 2 +- roles/yoda_portal/defaults/main.yml | 4 +- roles/yoda_portal/tasks/main.yml | 21 +----- roles/yoda_portal/vars/Debian.yml | 2 +- roles/yoda_rulesets/defaults/main.yml | 2 +- roles/yoda_rulesets/meta/main.yml | 1 - roles/yoda_rulesets/tasks/main.yml | 49 +------------ roles/yoda_rulesets/tasks/yoda-ruleset.yml | 12 ++-- roles/yoda_rulesets/vars/Debian.yml | 1 - roles/yoda_rulesets/vars/RedHat.yml | 1 - roles/yoda_test/tasks/run-revision-job.yml | 2 +- roles/yoda_test_users_eus/meta/main.yml | 4 +- roles/yoda_test_users_eus/tasks/main.yml | 3 +- roles/yoda_web_mock/meta/main.yml | 4 +- roles/yoda_web_mock/tasks/main.yml | 17 +++-- roles/yoda_web_mock/vars/Debian.yml | 2 +- roles/yoda_web_mock_datacite/meta/main.yml | 4 +- roles/yoda_web_mock_sram/meta/main.yml | 4 +- 106 files changed, 287 insertions(+), 604 deletions(-) rename environments/development/allinone/host_vars/{combined => combined.yoda.test} (100%) delete mode 100644 roles/irods_ubuntu_dependencies/defaults/main.yml delete mode 100644 roles/irods_ubuntu_dependencies/meta/main.yml delete mode 100644 roles/irods_ubuntu_dependencies/tasks/main-tasks.yml delete mode 100644 roles/irods_ubuntu_dependencies/tasks/main.yml delete mode 100644 roles/python2/meta/main.yml delete mode 100644 roles/python2/tasks/install-debian.yml delete mode 100644 roles/python2/tasks/install-redhat.yml delete mode 100644 roles/python2/tasks/main-tasks.yml delete mode 100644 roles/python2/tasks/main.yml diff --git a/Vagrantfile b/Vagrantfile index d0a23ae2b..ec6ca6e87 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -7,7 +7,7 @@ VAGRANTFILE_API_VERSION = "2" ENV['VAGRANT_DEFAULT_PROVIDER'] = "libvirt" -BOX = 'generic/ubuntu2004' +BOX = 'alvistack/ubuntu-24.04' GUI = false CPU = 2 RAM = 4096 @@ -47,6 +47,10 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| machine.vm.hostname = name + DOMAIN machine.vm.network 'private_network', ip: ipaddr, netmask: NETMASK machine.vm.synced_folder ".", "/vagrant", disabled: true + machine.vm.provision "shell" do |s| + s.inline = "sudo hostnamectl hostname $1" + s.args = name + DOMAIN + end machine.vm.provision "shell", inline: "sudo timedatectl set-timezone Europe/Amsterdam" machine.vm.provision "shell", @@ -64,7 +68,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| controller.vm.provider :virtualbox do |vbox| vbox.customize ["guestproperty", "set", :id, "/VirtualBox/GuestAdd/VBoxService/--timesync-set-threshold", 10000] end - controller.vm.box = 'generic/ubuntu2004' + controller.vm.box = 'alvistack/ubuntu-24.04' controller.vm.hostname = "controller" controller.vm.network :private_network, ip: "192.168.56.5", netmask: NETMASK controller.vm.provision "shell", privileged: false, path: "vagrant/provision_controller.sh" diff --git a/environments/development/allinone/host_vars/combined b/environments/development/allinone/host_vars/combined.yoda.test similarity index 100% rename from environments/development/allinone/host_vars/combined rename to environments/development/allinone/host_vars/combined.yoda.test diff --git a/environments/development/allinone/hosts b/environments/development/allinone/hosts index 2468f0745..610d8cee0 100644 --- a/environments/development/allinone/hosts +++ b/environments/development/allinone/hosts @@ -12,26 +12,26 @@ allinone_eus # Define portal host here: [allinone_portal] -combined +combined.yoda.test # Define database host here: [allinone_database] -combined +combined.yoda.test # Define icat host here: [allinone_icat] -combined +combined.yoda.test # Define resource host here: [allinone_resource] # Define public host here: [allinone_public] -combined +combined.yoda.test # Define external user service host here: [allinone_eus] -combined +combined.yoda.test ############### # All instances diff --git a/library/irods_indexing.py b/library/irods_indexing.py index 3adb073bc..8eb6d3edb 100644 --- a/library/irods_indexing.py +++ b/library/irods_indexing.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (c) 2021 Utrecht University +# Copyright (c) 2021-2024 Utrecht University # GNU General Public License v3.0 ANSIBLE_METADATA = { @@ -56,11 +56,6 @@ def main(): "plugin_name": "irods_rule_engine_plugin-elasticsearch", "plugin_specific_configuration": config }, - { - "instance_name": "irods_rule_engine_plugin-document_type-instance", - "plugin_name": "irods_rule_engine_plugin-document_type", - "plugin_specific_configuration": {} - }, default_policy ]) changed = True diff --git a/roles/acme_certificates/meta/main.yml b/roles/acme_certificates/meta/main.yml index f28b1866d..af9ced19f 100644 --- a/roles/acme_certificates/meta/main.yml +++ b/roles/acme_certificates/meta/main.yml @@ -8,6 +8,6 @@ galaxy_info: min_ansible_version: '2.11' platforms: - name: EL - version: 7 + version: 8 - name: Ubuntu - version: focal + version: noble diff --git a/roles/apache/templates/usr.sbin.apache2.j2 b/roles/apache/templates/usr.sbin.apache2.j2 index 03c01a8da..5c21e3903 100644 --- a/roles/apache/templates/usr.sbin.apache2.j2 +++ b/roles/apache/templates/usr.sbin.apache2.j2 @@ -1,5 +1,5 @@ # {{ ansible_managed }} -# Last Modified: Fri Oct 18 15:12:08 2024 +# Last Modified: Tue Nov 26 15:11:10 2024 #include # vim:syntax=apparmor @@ -25,6 +25,8 @@ capability kill, signal send set=term peer=unconfined, + signal send set=usr1 peer=unconfined, + signal send set=winch peer=unconfined, deny owner /etc/*.load r, @@ -38,6 +40,8 @@ /usr/bin/uname mrix, /usr/sbin/ldconfig* mrix, /var/log/apache2/*.log w, + /var/run/apache2/ r, + /var/run/apache2/** rw, /var/www/landingpages/** r, /var/www/yoda/** r, owner /dev/shm/* mrwl, @@ -46,6 +50,7 @@ owner /home/yodadeployment/flask_session/* rw, owner /opt/irods-externals/** mr, owner /run/apache2/apache2.pid rw, + owner /run/apache2/apache2.pid.* rw, owner /var/lib/davrods/__db.lockdb_locallock rw, owner /var/lib/davrods/lockdb_locallock rw, owner /var/www/extuser/** rw, diff --git a/roles/certificates/meta/main.yml b/roles/certificates/meta/main.yml index bb57b98e2..fc0ff29dc 100644 --- a/roles/certificates/meta/main.yml +++ b/roles/certificates/meta/main.yml @@ -9,7 +9,7 @@ galaxy_info: platforms: - name: EL version: - - 7 - 8 + - 9 - name: Ubuntu - version: focal + version: noble diff --git a/roles/certificates/tasks/main-tasks.yml b/roles/certificates/tasks/main-tasks.yml index 6e51767dd..5f3294317 100644 --- a/roles/certificates/tasks/main-tasks.yml +++ b/roles/certificates/tasks/main-tasks.yml @@ -9,14 +9,21 @@ ansible.builtin.package: name: python-cryptography state: present - when: (ansible_distribution_major_version == "7" and ansible_os_family == "RedHat") or ansible_os_family == "Debian" + when: ansible_distribution_major_version == "7" and ansible_os_family == "RedHat" -- name: Ensure dependencies for Ansible OpenSSL module are present (EL8) +- name: Ensure dependencies for Ansible OpenSSL module are present (EL8 and Debian) ansible.builtin.package: name: python3-cryptography state: present - when: ansible_distribution_major_version == "8" and ansible_os_family == "RedHat" + when: (ansible_distribution_major_version == "8" and ansible_os_family == "RedHat") or ansible_os_family == "Debian" + + +- name: Ensure dependencies for Ansible OpenSSL module are present (EL9) + ansible.builtin.package: + name: python3-cryptography + state: present + when: ansible_distribution_major_version == "9" and ansible_os_family == "RedHat" - name: Ensure an OpenSSL 4096 bits RSA private key is present diff --git a/roles/common/meta/main.yml b/roles/common/meta/main.yml index 357c6448a..37dcd2cee 100644 --- a/roles/common/meta/main.yml +++ b/roles/common/meta/main.yml @@ -9,7 +9,7 @@ galaxy_info: platforms: - name: EL version: - - 7 - 8 + - 9 - name: Ubuntu - version: focal + version: noble diff --git a/roles/common/tasks/basics-redhat.yml b/roles/common/tasks/basics-redhat.yml index d6e18436c..8fef0952a 100644 --- a/roles/common/tasks/basics-redhat.yml +++ b/roles/common/tasks/basics-redhat.yml @@ -21,6 +21,16 @@ when: ansible_distribution_major_version == "8" +- name: Ensure Ansible dependencies are installed (EL9) + ansible.builtin.package: + name: + - python3-libselinux + - python3-libsemanage + - python3-policycoreutils + state: present + when: ansible_distribution_major_version == "9" + + - name: Ensure basics (vim, bind-utils, etc.) are installed ansible.builtin.package: name: diff --git a/roles/common/tasks/irods-debian.yml b/roles/common/tasks/irods-debian.yml index c225cdb31..366905be9 100644 --- a/roles/common/tasks/irods-debian.yml +++ b/roles/common/tasks/irods-debian.yml @@ -9,6 +9,6 @@ - name: Ensure iRODS packages APT repository is enabled ansible.builtin.apt_repository: - repo: deb [arch=amd64] https://packages.irods.org/apt/ bionic main + repo: deb [arch=amd64] https://packages.irods.org/apt/ noble main filename: renci-irods state: present diff --git a/roles/composable_resources/meta/main.yml b/roles/composable_resources/meta/main.yml index 02bad01e2..387e2ad36 100644 --- a/roles/composable_resources/meta/main.yml +++ b/roles/composable_resources/meta/main.yml @@ -8,9 +8,9 @@ galaxy_info: min_ansible_version: '2.11' platforms: - name: EL - version: 7 + version: 8 - name: Ubuntu - version: focal + version: noble dependencies: diff --git a/roles/dms_archive_mock/meta/main.yml b/roles/dms_archive_mock/meta/main.yml index cbe95ae60..8767e5fa5 100644 --- a/roles/dms_archive_mock/meta/main.yml +++ b/roles/dms_archive_mock/meta/main.yml @@ -8,9 +8,9 @@ galaxy_info: min_ansible_version: '2.11' platforms: - name: EL - version: 7 + version: 8 - name: Ubuntu - version: focal + version: noble dependencies: diff --git a/roles/dms_tape_archive/meta/main.yml b/roles/dms_tape_archive/meta/main.yml index 01d3417f6..0d9ffef73 100644 --- a/roles/dms_tape_archive/meta/main.yml +++ b/roles/dms_tape_archive/meta/main.yml @@ -8,6 +8,6 @@ galaxy_info: min_ansible_version: '2.11' platforms: - name: EL - version: 7 + version: 8 - name: Ubuntu - version: focal + version: noble diff --git a/roles/icat_database_checker/meta/main.yml b/roles/icat_database_checker/meta/main.yml index 50364fd8e..41ef3a53c 100644 --- a/roles/icat_database_checker/meta/main.yml +++ b/roles/icat_database_checker/meta/main.yml @@ -8,9 +8,9 @@ galaxy_info: min_ansible_version: '2.11' platforms: - name: EL - version: 7 + version: 8 - name: Ubuntu - version: focal + version: noble dependencies: diff --git a/roles/icat_database_checker/tasks/main.yml b/roles/icat_database_checker/tasks/main.yml index e8f24db73..311187052 100644 --- a/roles/icat_database_checker/tasks/main.yml +++ b/roles/icat_database_checker/tasks/main.yml @@ -5,25 +5,10 @@ ansible.builtin.include_vars: "{{ ansible_os_family }}.yml" -- name: Check if icat-database-checker virtual environment is running Python 3.6 - ansible.builtin.stat: - path: '/var/lib/irods/icat-database-checker/bin/pip3.6' - register: idbc_venv_python36 - - -# For upgrade to Yoda 1.9 -- name: Move old icat-database-checker virtual environment - become_user: '{{ irods_service_account }}' - become: true - ansible.builtin.command: # noqa no-changed-when - cmd: "mv /var/lib/irods/icat-database-checker /var/lib/irods/icat-database-checker.backup-py3.6" - when: idbc_venv_python36.stat.exists - - - name: Ensure iCAT database checker virtualenv exists become_user: '{{ irods_service_account }}' become: true - ansible.builtin.command: "{{ icat_database_checker_python3_path }} -m virtualenv /var/lib/irods/icat-database-checker" + ansible.builtin.command: "{{ icat_database_checker_python3_path }} -m venv /var/lib/irods/icat-database-checker" args: creates: /var/lib/irods/icat-database-checker diff --git a/roles/irods_arb/meta/main.yml b/roles/irods_arb/meta/main.yml index f69847c1c..8492b4ba1 100644 --- a/roles/irods_arb/meta/main.yml +++ b/roles/irods_arb/meta/main.yml @@ -8,9 +8,9 @@ galaxy_info: min_ansible_version: '2.11' platforms: - name: EL - version: 7 + version: 8 - name: Ubuntu - version: focal + version: noble dependencies: diff --git a/roles/irods_arb/tasks/main.yml b/roles/irods_arb/tasks/main.yml index 15ac112be..a553c01a7 100644 --- a/roles/irods_arb/tasks/main.yml +++ b/roles/irods_arb/tasks/main.yml @@ -15,10 +15,10 @@ state: '{{ "present" if irods_arb_enabled else "absent" }}' -- name: Ensure psutil is installed (Python 3) +- name: Ensure psutil is installed become_user: '{{ irods_service_account }}' become: true ansible.builtin.pip: name: "psutil==5.9.5" executable: "{{ irods_arb_pip3_location }}" - extra_args: --user + extra_args: --user --break-system-packages diff --git a/roles/irods_consistency_check/meta/main.yml b/roles/irods_consistency_check/meta/main.yml index 84ef086e3..3ab4c4819 100644 --- a/roles/irods_consistency_check/meta/main.yml +++ b/roles/irods_consistency_check/meta/main.yml @@ -8,9 +8,9 @@ galaxy_info: min_ansible_version: '2.11' platforms: - name: EL - version: 7 + version: 8 - name: Ubuntu - version: focal + version: noble dependencies: diff --git a/roles/irods_consistency_check/tasks/main.yml b/roles/irods_consistency_check/tasks/main.yml index b44ce0075..b86d0c384 100644 --- a/roles/irods_consistency_check/tasks/main.yml +++ b/roles/irods_consistency_check/tasks/main.yml @@ -5,25 +5,10 @@ ansible.builtin.include_vars: "{{ ansible_os_family }}.yml" -- name: Check if ichk virtual environment is running Python 3.6 - ansible.builtin.stat: - path: '/var/lib/irods/irods-consistency-check/bin/pip3.6' - register: ichk_venv_python36 - - -# For upgrade to Yoda 1.9 -- name: Move old ichk virtual environment - become_user: '{{ irods_service_account }}' - become: true - ansible.builtin.command: # noqa no-changed-when - cmd: "mv /var/lib/irods/irods-consistency-check /var/lib/irods/irods-consistency-check.backup-py3.6" - when: ichk_venv_python36.stat.exists - - - name: Ensure iRODS consistency check virtualenv exists become_user: '{{ irods_service_account }}' become: true - ansible.builtin.command: "{{ irods_consistency_check_python3_path }} -m virtualenv /var/lib/irods/irods-consistency-check" + ansible.builtin.command: "{{ irods_consistency_check_python3_path }} -m venv /var/lib/irods/irods-consistency-check" args: creates: /var/lib/irods/irods-consistency-check diff --git a/roles/irods_database/meta/main.yml b/roles/irods_database/meta/main.yml index 8e755c887..ba97f49fa 100644 --- a/roles/irods_database/meta/main.yml +++ b/roles/irods_database/meta/main.yml @@ -9,10 +9,10 @@ galaxy_info: platforms: - name: EL version: - - 7 - 8 + - 9 - name: Ubuntu - version: focal + version: noble dependencies: diff --git a/roles/irods_gocommands/meta/main.yml b/roles/irods_gocommands/meta/main.yml index 466730275..92e169751 100644 --- a/roles/irods_gocommands/meta/main.yml +++ b/roles/irods_gocommands/meta/main.yml @@ -8,6 +8,6 @@ galaxy_info: min_ansible_version: "2.7" platforms: - name: EL - version: 7 + version: 8 - name: Ubuntu - version: focal + version: noble diff --git a/roles/irods_icat/defaults/main.yml b/roles/irods_icat/defaults/main.yml index dab8668b8..50e6b70e9 100644 --- a/roles/irods_icat/defaults/main.yml +++ b/roles/irods_icat/defaults/main.yml @@ -11,7 +11,7 @@ irods_database_name: ICAT irods_database_port: "{{ 6432 if enable_pgbouncer else 5432 }}" irods_database_user: irodsdb # The iRODS database username irods_database_password: irodsdev # The password for the iRODS database username -irods_authentication_scheme: PAM # iRODS authentication method: "Native" or "PAM" +irods_authentication_scheme: pam_password # iRODS authentication method: "native" or "pam_password" irods_zone: tempZone # The name of the iRODS Zone irods_default_resc: irodsResc # iRODS default resource name irods_port_range_begin: 20000 diff --git a/roles/irods_icat/meta/main.yml b/roles/irods_icat/meta/main.yml index a134dcf11..46fa7c269 100644 --- a/roles/irods_icat/meta/main.yml +++ b/roles/irods_icat/meta/main.yml @@ -8,17 +8,14 @@ galaxy_info: min_ansible_version: '2.11' platforms: - name: EL - version: 7 + version: 8 - name: Ubuntu - version: focal + version: noble dependencies: - - role: irods_ubuntu_dependencies - when: ansible_os_family == 'Debian' - role: irods_remove_old_version - role: irods_database - role: irods_icommands - - role: python2 - role: python_irodsclient - role: sqlcipher diff --git a/roles/irods_icat/tasks/main.yml b/roles/irods_icat/tasks/main.yml index e83f994ff..057e55260 100644 --- a/roles/irods_icat/tasks/main.yml +++ b/roles/irods_icat/tasks/main.yml @@ -45,7 +45,7 @@ name: - "{{ irods_idp_package_new }}" - "{{ irods_esp_package_new }}" - - "{{ irods_dtp_package_new }}" +# - "{{ irods_dtp_package_new }}" state: present when: not ansible_check_mode and enable_open_search @@ -132,7 +132,7 @@ - name: Configure iRODS iCAT server become: true - ansible.builtin.command: python /var/lib/irods/scripts/setup_irods.py --json_configuration_file=/etc/irods/setup_irods_provider.json + ansible.builtin.command: python3 /var/lib/irods/scripts/setup_irods.py --json_configuration_file=/etc/irods/setup_irods_provider.json args: creates: /etc/irods/service_account.config run_once: true @@ -539,4 +539,4 @@ - name: Setup PAM ansible.builtin.include_tasks: setup_pam.yml - when: irods_authentication_scheme == "PAM" + when: irods_authentication_scheme == "pam_password" diff --git a/roles/irods_icat/tasks/setup_pam.yml b/roles/irods_icat/tasks/setup_pam.yml index 35e09e267..8fb7c5fd6 100644 --- a/roles/irods_icat/tasks/setup_pam.yml +++ b/roles/irods_icat/tasks/setup_pam.yml @@ -65,10 +65,10 @@ when: enable_tokens and ansible_os_family == 'Debian' and '1.0.4' in pysqlcipher3_version.stdout -- name: Ensure custom build of pysqlcipher3 is installed globally for token authentication script - ansible.builtin.command: # noqa no-changed-when - cmd: python2 -m easy_install https://github.com/UtrechtUniversity/pysqlcipher3/releases/download/v1.2.1/pysqlcipher3-1.2.1-py2.7-linux-x86_64.egg - when: enable_tokens and ansible_os_family == 'Debian' and '1.2.1' not in pysqlcipher3_version.stdout +# - name: Ensure custom build of pysqlcipher3 is installed globally for token authentication script +# ansible.builtin.command: # noqa no-changed-when +# cmd: python3 -m easy_install https://github.com/UtrechtUniversity/pysqlcipher3/releases/download/v1.2.1/pysqlcipher3-1.2.1-py2.7-linux-x86_64.egg +# when: enable_tokens and ansible_os_family == 'Debian' and '1.2.1' not in pysqlcipher3_version.stdout - name: Ensure token authentication script is present diff --git a/roles/irods_icat/templates/irods_indexing.pref.j2 b/roles/irods_icat/templates/irods_indexing.pref.j2 index 8d90de55e..9d80f1bfa 100644 --- a/roles/irods_icat/templates/irods_indexing.pref.j2 +++ b/roles/irods_icat/templates/irods_indexing.pref.j2 @@ -11,9 +11,3 @@ Explanation: Pin added by role: irods_icat Package: {{ irods_esp[0] }} Pin: version {{ irods_esp[1] }} Pin-Priority: 999 - -{% set irods_dtp = irods_dtp_package_new.split('=') %} -Explanation: Pin added by role: irods_icat -Package: {{ irods_dtp[0] }} -Pin: version {{ irods_dtp[1] }} -Pin-Priority: 999 diff --git a/roles/irods_icat/templates/setup_irods_provider.json.j2 b/roles/irods_icat/templates/setup_irods_provider.json.j2 index 43780043a..bbb684871 100644 --- a/roles/irods_icat/templates/setup_irods_provider.json.j2 +++ b/roles/irods_icat/templates/setup_irods_provider.json.j2 @@ -50,6 +50,9 @@ "default_resource_name": "{{ irods_default_resc }}", "environment_variables": {}, "federation": [], + "host_resolution": { + "host_entries": [] + }, "match_hash_policy": "compatible", "negotiation_key": "{{ irods_negotiation_key }}", "plugin_configuration": { @@ -113,5 +116,6 @@ "zone_name": "{{ irods_zone }}", "zone_port": {{ irods_icat_port }}, "zone_user": "rods" - } + }, + "default_resource_name": "{{ irods_default_resc }}" } diff --git a/roles/irods_icat/vars/Debian.yml b/roles/irods_icat/vars/Debian.yml index 997197981..c4d40bc4c 100644 --- a/roles/irods_icat/vars/Debian.yml +++ b/roles/irods_icat/vars/Debian.yml @@ -8,10 +8,10 @@ pam_radius_package: libpam-radius-auth irods_icat_pip2_path: /usr/local/bin/pip2 -irods_runtime_package_new: irods-runtime=4.2.12-1~bionic -irods_server_package_new: irods-server=4.2.12-1~bionic -irods_prep_package_new: irods-rule-engine-plugin-python=4.2.12.0-1~bionic -irods_pgp_package_new: irods-database-plugin-postgres=4.2.12-1~bionic -irods_idp_package_new: irods-rule-engine-plugin-indexing=4.2.12.0-1~bionic -irods_esp_package_new: irods-rule-engine-plugin-elasticsearch=4.2.12.0-1~bionic -irods_dtp_package_new: irods-rule-engine-plugin-document-type=4.2.12.0-1~bionic +irods_runtime_package_new: irods-runtime=4.3.3-0~noble +irods_server_package_new: irods-server=4.3.3-0~noble +irods_prep_package_new: irods-rule-engine-plugin-python=4.3.3.0-0+4.3.3~noble +irods_pgp_package_new: irods-database-plugin-postgres=4.3.3-0~noble +irods_idp_package_new: irods-rule-engine-plugin-indexing=4.3.3.0-0+4.3.3~noble +irods_esp_package_new: irods-rule-engine-plugin-elasticsearch=4.3.3.0-0+4.3.3~noble +# irods_dtp_package_new: irods-rule-engine-plugin-document-type=4.2.12.0-1~bionic diff --git a/roles/irods_icat/vars/RedHat.yml b/roles/irods_icat/vars/RedHat.yml index 74786eb09..cffcb855a 100644 --- a/roles/irods_icat/vars/RedHat.yml +++ b/roles/irods_icat/vars/RedHat.yml @@ -8,10 +8,10 @@ pam_radius_package: pam_radius irods_icat_pip2_path: /usr/bin/pip -irods_runtime_package_new: irods-runtime-4.2.12-1 -irods_server_package_new: irods-server-4.2.12-1 -irods_prep_package_new: irods-rule-engine-plugin-python-4.2.12.0-1 -irods_pgp_package_new: irods-database-plugin-postgres-4.2.12-1 -irods_idp_package_new: irods-rule-engine-plugin-indexing-4.2.12.0-1 -irods_esp_package_new: irods-rule-engine-plugin-elasticsearch-4.2.12.0-1 -irods_dtp_package_new: irods-rule-engine-plugin-document-type-4.2.12.0-1 +irods_runtime_package_new: irods-runtime-4.3.3-0 +irods_server_package_new: irods-server-4.3.3-0 +irods_prep_package_new: irods-rule-engine-plugin-python-4.3.3.0-0 +irods_pgp_package_new: irods-database-plugin-postgres-4.3.3-0 +irods_idp_package_new: irods-rule-engine-plugin-indexing-4.3.3.0-0 +irods_esp_package_new: irods-rule-engine-plugin-elasticsearch-4.3.3.0-0 +# irods_dtp_package_new: irods-rule-engine-plugin-document-type-4.2.12.0-1 diff --git a/roles/irods_icommands/meta/main.yml b/roles/irods_icommands/meta/main.yml index a01d55dc0..d5a0ee239 100644 --- a/roles/irods_icommands/meta/main.yml +++ b/roles/irods_icommands/meta/main.yml @@ -8,9 +8,9 @@ galaxy_info: min_ansible_version: '2.11' platforms: - name: EL - version: 7 + version: 8 - name: Ubuntu - version: focal + version: noble dependencies: diff --git a/roles/irods_icommands/vars/Debian.yml b/roles/irods_icommands/vars/Debian.yml index 3c843ddbb..46321b1b7 100644 --- a/roles/irods_icommands/vars/Debian.yml +++ b/roles/irods_icommands/vars/Debian.yml @@ -1,4 +1,4 @@ --- # copyright Utrecht University -irods_icommands_package_new: irods-icommands=4.2.12-1~bionic +irods_icommands_package_new: irods-icommands=4.3.3-0~noble diff --git a/roles/irods_icommands/vars/RedHat.yml b/roles/irods_icommands/vars/RedHat.yml index c85b83ddb..6adc99f0e 100644 --- a/roles/irods_icommands/vars/RedHat.yml +++ b/roles/irods_icommands/vars/RedHat.yml @@ -1,4 +1,4 @@ --- # copyright Utrecht University -irods_icommands_package_new: irods-icommands-4.2.12-1 +irods_icommands_package_new: irods-icommands-4.3.3-0 diff --git a/roles/irods_microservices/meta/main.yml b/roles/irods_microservices/meta/main.yml index 199926a84..819f29ca8 100644 --- a/roles/irods_microservices/meta/main.yml +++ b/roles/irods_microservices/meta/main.yml @@ -8,9 +8,9 @@ galaxy_info: min_ansible_version: '2.11' platforms: - name: EL - version: 7 + version: 8 - name: Ubuntu - version: focal + version: noble dependencies: diff --git a/roles/irods_microservices/vars/Debian.yml b/roles/irods_microservices/vars/Debian.yml index 8195b332c..5961aab7c 100644 --- a/roles/irods_microservices/vars/Debian.yml +++ b/roles/irods_microservices/vars/Debian.yml @@ -4,13 +4,13 @@ irods_microservices_data: irods_sudo_microservices: package: irods-sudo-microservices - version: 4.2.12_1.0.0-1 - url: https://github.com/UtrechtUniversity/irods-sudo-microservices/releases/download/4.2.12_1.0.0/irods-sudo-microservices-4.2.12-1.0.0-1.deb - filename: irods-sudo-microservices-4.2.12_1.0.0-1.rpm - checksum: sha256:508cf7284353b832a95e4f818760bf631456607feb5946af0297e849207fdfb4 + version: 4.3.3-1.0.0-1 + url: https://github.com/UtrechtUniversity/irods-sudo-microservices/releases/download/4.3.3_1.0.0/irods-sudo-microservices-4.3.3-1.0.0-1.deb + filename: irods-sudo-microservices-4.3.3-1.0.0-1.deb + checksum: sha256:039ccbb892590208b88c800b8a197cc5e75bd40ac9304edf019f9fc527ead22c irods_uu_microservices: package: irods-uu-microservices - version: 4.2.12-1.2.0-0 - url: https://github.com/UtrechtUniversity/irods-uu-microservices/releases/download/v1.2.0/irods-uu-microservices-4.2.12-1.2.0-0.deb - filename: irods-uu-microservices-4.2.12-1.2.0-0.deb - checksum: sha256:6b2334535d953bc70a083873adbbe1de57707fd088d7d5d52379a412001c13ff + version: 4.3.3-1.2.0-0 + url: https://github.com/UtrechtUniversity/irods-uu-microservices/releases/download/v1.2.0/irods-uu-microservices-4.3.3-1.2.0-0.deb + filename: irods-uu-microservices-4.3.3-1.2.0-0.deb + checksum: sha256:beaaa5171e9068f55fc81662057d578b03e56e938b438c739ac7f962ec56b868 diff --git a/roles/irods_remove_old_version/meta/main.yml b/roles/irods_remove_old_version/meta/main.yml index 2bc84806c..770e3b2a9 100644 --- a/roles/irods_remove_old_version/meta/main.yml +++ b/roles/irods_remove_old_version/meta/main.yml @@ -8,6 +8,6 @@ galaxy_info: min_ansible_version: '2.11' platforms: - name: EL - version: 7 + version: 8 - name: Ubuntu - version: focal + version: noble diff --git a/roles/irods_remove_old_version/vars/Debian.yml b/roles/irods_remove_old_version/vars/Debian.yml index 12559099a..b9ca43e13 100644 --- a/roles/irods_remove_old_version/vars/Debian.yml +++ b/roles/irods_remove_old_version/vars/Debian.yml @@ -1,14 +1,14 @@ --- # copyright Utrecht University -irods_uu_microservices_package_old: irods-uu-microservices=4.2.11_0.8.2-1 -irods_sudo_microservices_package_old: irods-sudo-microservices=4.2.11_1.0.0-1 -irods_runtime_package_old: irods-runtime=4.2.11-1~bionic -irods_server_package_old: irods-server=4.2.11-1~bionic -irods_icommands_package_old: irods-icommands=4.2.11-1~bionic -irods_prep_package_old: irods-rule-engine-plugin-python=4.2.11.1-1~bionic -irods_pgp_package_old: irods-database-plugin-postgres=4.2.11-1~bionic -irods_idp_package_old: irods-rule-engine-plugin-indexing=4.2.11.0-1~bionic -irods_esp_package_old: irods-rule-engine-plugin-elasticsearch=4.2.11.0-1~bionic -irods_dtp_package_old: irods-rule-engine-plugin-document-type=4.2.11.0-1~bionic -irods_davrods_package_old: davrods=4.2.11_1.5.0-1 +irods_uu_microservices_package_old: irods-uu-microservices=4.2.12-1.2.0-0 +irods_sudo_microservices_package_old: irods-sudo-microservices=4.2.12_1.0.0-1 +irods_runtime_package_old: irods-runtime=4.2.12-1~bionic +irods_server_package_old: irods-server=4.2.12-1~bionic +irods_icommands_package_old: irods-icommands=4.2.12-1~bionic +irods_prep_package_old: irods-rule-engine-plugin-python=4.2.12.0-1~bionic +irods_pgp_package_old: irods-database-plugin-postgres=4.2.12-1~bionic +irods_idp_package_old: irods-rule-engine-plugin-indexing=4.2.12.0-1~bionic +irods_esp_package_old: irods-rule-engine-plugin-elasticsearch=4.2.12.0-1~bionic +irods_dtp_package_old: irods-rule-engine-plugin-document-type=4.2.12.0-1~bionic +irods_davrods_package_old: davrods=4.2.12_1.5.1-1 diff --git a/roles/irods_remove_old_version/vars/RedHat.yml b/roles/irods_remove_old_version/vars/RedHat.yml index 01f066f31..d1914f451 100644 --- a/roles/irods_remove_old_version/vars/RedHat.yml +++ b/roles/irods_remove_old_version/vars/RedHat.yml @@ -1,14 +1,14 @@ --- # copyright Utrecht University -irods_uu_microservices_package_old: irods-uu-microservices-4.2.11_0.8.2-1 -irods_sudo_microservices_package_old: irods-sudo-microservices-4.2.11_1.0.0-1 -irods_runtime_package_old: irods-runtime-4.2.11-1 -irods_server_package_old: irods-server-4.2.11-1 -irods_icommands_package_old: irods-icommands-4.2.11-1 -irods_prep_package_old: irods-rule-engine-plugin-python-4.2.11.1-1 -irods_pgp_package_old: irods-database-plugin-postgres-4.2.11-1 -irods_idp_package_old: irods-rule-engine-plugin-indexing-4.2.11.0-1 -irods_esp_package_old: irods-rule-engine-plugin-elasticsearch-4.2.11.0-1 -irods_dtp_package_old: irods-rule-engine-plugin-document-type-4.2.11.0-1 -irods_davrods_package_old: davrods-4.2.11_1.5.0-1 +irods_uu_microservices_package_old: irods-uu-microservices-4.2.12-1.2.0-0 +irods_sudo_microservices_package_old: irods-sudo-microservices-4.2.12_1.0.0-1 +irods_runtime_package_old: irods-runtime-4.2.12-1 +irods_server_package_old: irods-server-4.2.12-1 +irods_icommands_package_old: irods-icommands-4.2.12-1 +irods_prep_package_old: irods-rule-engine-plugin-python-4.2.12.0-1 +irods_pgp_package_old: irods-database-plugin-postgres-4.2.12-1 +irods_idp_package_old: irods-rule-engine-plugin-indexing-4.2.12.0-1 +irods_esp_package_old: irods-rule-engine-plugin-elasticsearch-4.2.12.0-1 +irods_dtp_package_old: irods-rule-engine-plugin-document-type-4.2.12.0-1 +irods_davrods_package_old: davrods-4.2.12_1.5.1-1 diff --git a/roles/irods_resource/defaults/main.yml b/roles/irods_resource/defaults/main.yml index 8d8afc24b..fc9e31826 100644 --- a/roles/irods_resource/defaults/main.yml +++ b/roles/irods_resource/defaults/main.yml @@ -5,7 +5,7 @@ irods_admin: rods # iRODS admin username irods_password: rods # iRODS admin password irods_icat_fqdn: icat.yoda.test -irods_authentication_scheme: PAM # iRODS authentication method: "Native" or "PAM" +irods_authentication_scheme: pam_password # iRODS authentication method: "native" or "pam_password" irods_zone: tempZone # The name of the iRODS Zone irods_default_resc: irodsResc # iRODS default resource name irods_port_range_begin: 20000 diff --git a/roles/irods_resource/meta/main.yml b/roles/irods_resource/meta/main.yml index 2bda80eee..081c8332a 100644 --- a/roles/irods_resource/meta/main.yml +++ b/roles/irods_resource/meta/main.yml @@ -8,15 +8,13 @@ galaxy_info: min_ansible_version: '2.11' platforms: - name: EL - version: 7 + version: 8 - name: Ubuntu - version: focal + version: noble dependencies: - - role: irods_ubuntu_dependencies - when: ansible_os_family == 'Debian' - role: irods_remove_old_version - role: irods_icommands - - role: python2 + - role: python3 - role: python_irodsclient diff --git a/roles/irods_resource/vars/Debian.yml b/roles/irods_resource/vars/Debian.yml index d9d8ed403..bcf19cd69 100644 --- a/roles/irods_resource/vars/Debian.yml +++ b/roles/irods_resource/vars/Debian.yml @@ -4,6 +4,6 @@ openssl_private_dir: '/etc/ssl/private/' openssl_certs_dir: '/etc/ssl/certs' -irods_runtime_package_new: irods-runtime=4.2.12-1~bionic -irods_server_package_new: irods-server=4.2.12-1~bionic -irods_prep_package_new: irods-rule-engine-plugin-python=4.2.12.0-1~bionic +irods_runtime_package_new: irods-runtime=4.3.3-0~noble +irods_server_package_new: irods-server=4.3.3-0~noble +irods_prep_package_new: irods-rule-engine-plugin-python=4.3.3.0-0~noble diff --git a/roles/irods_resource/vars/RedHat.yml b/roles/irods_resource/vars/RedHat.yml index 24412e0c3..3a1d7b2c4 100644 --- a/roles/irods_resource/vars/RedHat.yml +++ b/roles/irods_resource/vars/RedHat.yml @@ -4,6 +4,6 @@ openssl_private_dir: '/etc/pki/tls/private' openssl_certs_dir: '/etc/pki/tls/certs' -irods_runtime_package_new: irods-runtime-4.2.12-1 -irods_server_package_new: irods-server-4.2.12-1 -irods_prep_package_new: irods-rule-engine-plugin-python-4.2.12.0-1 +irods_runtime_package_new: irods-runtime-4.3.3-0 +irods_server_package_new: irods-server-4.3.3-0 +irods_prep_package_new: irods-rule-engine-plugin-python-4.3.3.0-0 diff --git a/roles/irods_resource_plugin_s3/meta/main.yml b/roles/irods_resource_plugin_s3/meta/main.yml index 73d236864..cc41dc448 100644 --- a/roles/irods_resource_plugin_s3/meta/main.yml +++ b/roles/irods_resource_plugin_s3/meta/main.yml @@ -8,9 +8,9 @@ galaxy_info: min_ansible_version: '2.11' platforms: - name: EL - version: 7 + version: 8 - name: Ubuntu - version: focal + version: noble dependencies: diff --git a/roles/irods_resource_plugin_s3/vars/Debian.yml b/roles/irods_resource_plugin_s3/vars/Debian.yml index e751c4caf..224e0b572 100644 --- a/roles/irods_resource_plugin_s3/vars/Debian.yml +++ b/roles/irods_resource_plugin_s3/vars/Debian.yml @@ -1,4 +1,4 @@ --- # copyright Utrecht University -irods_resource_plugin_s3_package: irods-resource-plugin-s3=4.2.12.0-1~bionic +irods_resource_plugin_s3_package: irods-resource-plugin-s3=4.3.3.0-0~noble diff --git a/roles/irods_resource_plugin_s3/vars/RedHat.yml b/roles/irods_resource_plugin_s3/vars/RedHat.yml index b3fd20712..cc616be66 100644 --- a/roles/irods_resource_plugin_s3/vars/RedHat.yml +++ b/roles/irods_resource_plugin_s3/vars/RedHat.yml @@ -1,4 +1,4 @@ --- # copyright Utrecht University # -irods_resource_plugin_s3_package: irods-resource-plugin-s3-4.2.12.0-1 +irods_resource_plugin_s3_package: irods-resource-plugin-s3-4.3.3.0-0 diff --git a/roles/irods_runtime/meta/main.yml b/roles/irods_runtime/meta/main.yml index c4406fcf3..3aae13216 100644 --- a/roles/irods_runtime/meta/main.yml +++ b/roles/irods_runtime/meta/main.yml @@ -8,12 +8,10 @@ galaxy_info: min_ansible_version: '2.11' platforms: - name: EL - version: 7 + version: 8 - name: Ubuntu - version: focal + version: noble dependencies: - - role: irods_ubuntu_dependencies - when: ansible_os_family == 'Debian' - role: irods_remove_old_version diff --git a/roles/irods_runtime/tasks/main-tasks.yml b/roles/irods_runtime/tasks/main-tasks.yml index 8ed8fe293..7e776c10d 100644 --- a/roles/irods_runtime/tasks/main-tasks.yml +++ b/roles/irods_runtime/tasks/main-tasks.yml @@ -29,21 +29,21 @@ when: ansible_os_family == 'Debian' -- name: Determine current checksum of irods_server library - ansible.builtin.command: "sha256sum /usr/lib/libirods_server.so.4.2.12" - register: libirods_server_checksum - changed_when: false - - -- name: Install local patch for iRODS 4.2.12 - ansible.builtin.get_url: - url: "{{ irods_runtime_server_patch_url }}" - dest: /usr/lib/libirods_server.so.4.2.12 - checksum: "sha256:{{ irods_runtime_server_patch_sha256sum }}" - owner: 'root' - group: 'root' - mode: '0644' - force: true - backup: true - notify: Restart iRODS - when: not ansible_check_mode and libirods_server_checksum.stdout.split()[0] != irods_runtime_server_patch_sha256sum +# - name: Determine current checksum of irods_server library +# ansible.builtin.command: "sha256sum /usr/lib/libirods_server.so.4.2.12" +# register: libirods_server_checksum +# changed_when: false + + +# - name: Install local patch for iRODS 4.2.12 +# ansible.builtin.get_url: +# url: "{{ irods_runtime_server_patch_url }}" +# dest: /usr/lib/libirods_server.so.4.2.12 +# checksum: "sha256:{{ irods_runtime_server_patch_sha256sum }}" +# owner: 'root' +# group: 'root' +# mode: '0644' +# force: true +# backup: true +# notify: Restart iRODS +# when: not ansible_check_mode and libirods_server_checksum.stdout.split()[0] != irods_runtime_server_patch_sha256sum diff --git a/roles/irods_runtime/vars/Debian.yml b/roles/irods_runtime/vars/Debian.yml index 83758f39b..6d1296a6b 100644 --- a/roles/irods_runtime/vars/Debian.yml +++ b/roles/irods_runtime/vars/Debian.yml @@ -1,6 +1,6 @@ --- # copyright Utrecht University -irods_runtime_package_new: irods-runtime=4.2.12-1~bionic -irods_runtime_server_patch_url: https://yoda.uu.nl/irods-patches/libirods_server.so.4.2.12.ubuntu2004 -irods_runtime_server_patch_sha256sum: 44b656e07954d41fc36a23400101690a5b7c150aa3ab8887b3df24b338525e9c +irods_runtime_package_new: irods-runtime=4.3.3-0~noble +# irods_runtime_server_patch_url: https://yoda.uu.nl/irods-patches/libirods_server.so.4.2.12.ubuntu2004 +# irods_runtime_server_patch_sha256sum: 44b656e07954d41fc36a23400101690a5b7c150aa3ab8887b3df24b338525e9c diff --git a/roles/irods_runtime/vars/RedHat.yml b/roles/irods_runtime/vars/RedHat.yml index 9087abbdf..dccd80778 100644 --- a/roles/irods_runtime/vars/RedHat.yml +++ b/roles/irods_runtime/vars/RedHat.yml @@ -1,6 +1,6 @@ --- # copyright Utrecht University -irods_runtime_package_new: irods-runtime-4.2.12-1 -irods_runtime_server_patch_url: https://yoda.uu.nl/irods-patches/libirods_server.so.4.2.12 -irods_runtime_server_patch_sha256sum: 11bb77ff1f17faac1052b891cf6d75a0f55a55814a4ecbad53e1469aca96c1d2 +irods_runtime_package_new: irods-runtime-4.3.3-0 +# irods_runtime_server_patch_url: https://yoda.uu.nl/irods-patches/libirods_server.so.4.2.12 +# irods_runtime_server_patch_sha256sum: 11bb77ff1f17faac1052b891cf6d75a0f55a55814a4ecbad53e1469aca96c1d2 diff --git a/roles/irods_ubuntu_dependencies/defaults/main.yml b/roles/irods_ubuntu_dependencies/defaults/main.yml deleted file mode 100644 index 9d9213a4d..000000000 --- a/roles/irods_ubuntu_dependencies/defaults/main.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -# copyright Utrecht University - -irods_ubuntu_libssl_package_name: "libssl1.0.0" -irods_ubuntu_libssl_package_filename: "libssl1.0.0_1.0.2n-1ubuntu5.13_amd64.deb" -irods_ubuntu_libssl_package_url_prefix: "http://security.ubuntu.com/ubuntu/pool/main/o/openssl1.0" - -irods_ubuntu_py_jsonschema_package_name: "py_jsonschema" -irods_ubuntu_py_jsonschema_package_filename: "python-jsonschema_2.3.0-1build1_all.deb" -irods_ubuntu_py_jsonschema_package_url_prefix: "http://security.ubuntu.com/ubuntu/pool/main/p/python-jsonschema" - -irods_ubuntu_py_odbc_package_name: "py_odbc" -irods_ubuntu_py_odbc_package_filename: "python-pyodbc_4.0.17-1_amd64.deb" -irods_ubuntu_py_odbc_package_url_prefix: "http://security.ubuntu.com/ubuntu/pool/universe/p/pyodbc" - -irods_ubuntu_py_requests_package_name: "py_requests" -irods_ubuntu_py_requests_package_filename: "python-requests_2.18.4-2ubuntu0.1_all.deb" -irods_ubuntu_py_requests_package_url_prefix: "http://security.ubuntu.com/ubuntu/pool/main/r/requests" - -irods_ubuntu_py_urllib_package_name: "py_urllib3" -irods_ubuntu_py_urllib_package_filename: "python-urllib3_1.22-1ubuntu0.18.04.2_all.deb" -irods_ubuntu_py_urllib_package_url_prefix: "http://security.ubuntu.com/ubuntu/pool/main/p/python-urllib3" diff --git a/roles/irods_ubuntu_dependencies/meta/main.yml b/roles/irods_ubuntu_dependencies/meta/main.yml deleted file mode 100644 index d82a137d6..000000000 --- a/roles/irods_ubuntu_dependencies/meta/main.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -# copyright Utrecht University - -galaxy_info: - author: Sietse Snel - description: Install specific dependencies that are needed to run iRODS on Ubuntu - license: GPLv3 - min_ansible_version: '2.11' - platforms: - - name: Ubuntu - version: focal - - -dependencies: - - role: common diff --git a/roles/irods_ubuntu_dependencies/tasks/main-tasks.yml b/roles/irods_ubuntu_dependencies/tasks/main-tasks.yml deleted file mode 100644 index bf07c3728..000000000 --- a/roles/irods_ubuntu_dependencies/tasks/main-tasks.yml +++ /dev/null @@ -1,72 +0,0 @@ ---- -# copyright Utrecht University - - -- name: Check whether OpenSSL 1.0 has already been installed - ansible.builtin.command: # noqa no-changed-when - cmd: "dpkg-query -W --showformat='${Status}\n' {{ irods_ubuntu_libssl_package_name }}" - register: irods_ubuntu_libssl_status - failed_when: false - changed_when: false - - -- name: Install OpenSSL 1.0 - ansible.builtin.apt: - deb: "{{ irods_ubuntu_libssl_package_url_prefix }}/{{ irods_ubuntu_libssl_package_filename }}" - when: "'install ok installed' not in irods_ubuntu_libssl_status.stdout and irods_ubuntu_libssl_status.rc == 1" - - -- name: Check whether JSONschema for Python 2 has already been installed - ansible.builtin.command: # noqa no-changed-when - cmd: "dpkg-query -W --showformat='${Status}\n' {{ irods_ubuntu_py_jsonschema_package_name }}" - register: irods_ubuntu_py_jsonschema_status - failed_when: false - changed_when: false - - -- name: Install JSONSchema for Python 2 - ansible.builtin.apt: - deb: "{{ irods_ubuntu_py_jsonschema_package_url_prefix }}/{{ irods_ubuntu_py_jsonschema_package_filename }}" - when: "'install ok installed' not in irods_ubuntu_py_jsonschema_status.stdout and irods_ubuntu_py_jsonschema_status.rc == 1" - - -- name: Check whether ODBC for Python 2 has already been installed - ansible.builtin.command: # noqa no-changed-when - cmd: "dpkg-query -W --showformat='${Status}\n' {{ irods_ubuntu_py_odbc_package_name }}" - register: irods_ubuntu_py_odbc_status - failed_when: false - changed_when: false - - -- name: Install ODBC for Python 2 - ansible.builtin.apt: - deb: "{{ irods_ubuntu_py_odbc_package_url_prefix }}/{{ irods_ubuntu_py_odbc_package_filename }}" - when: "'install ok installed' not in irods_ubuntu_py_odbc_status.stdout and irods_ubuntu_py_odbc_status.rc == 1" - - -- name: Check whether URLLib 3 for Python 2 has already been installed - ansible.builtin.command: # noqa no-changed-when - cmd: "dpkg-query -W --showformat='${Status}\n' {{ irods_ubuntu_py_urllib_package_name }}" - register: irods_ubuntu_py_urllib_status - failed_when: false - changed_when: false - - -- name: Install URLLib 3 for Python 2 - ansible.builtin.apt: - deb: "{{ irods_ubuntu_py_urllib_package_url_prefix }}/{{ irods_ubuntu_py_urllib_package_filename }}" - when: "'install ok installed' not in irods_ubuntu_py_urllib_status.stdout and irods_ubuntu_py_urllib_status.rc == 1" - - -- name: Check whether Requests for Python 2 has already been installed - ansible.builtin.command: # noqa no-changed-when - cmd: "dpkg-query -W --showformat='${Status}\n' {{ irods_ubuntu_py_requests_package_name }}" - register: irods_ubuntu_py_requests_status - failed_when: false - changed_when: false - - -- name: Install Requests for Python 2 - ansible.builtin.apt: - deb: "{{ irods_ubuntu_py_requests_package_url_prefix }}/{{ irods_ubuntu_py_requests_package_filename }}" - when: "'install ok installed' not in irods_ubuntu_py_requests_status.stdout and irods_ubuntu_py_requests_status.rc == 1" diff --git a/roles/irods_ubuntu_dependencies/tasks/main.yml b/roles/irods_ubuntu_dependencies/tasks/main.yml deleted file mode 100644 index 685e1edea..000000000 --- a/roles/irods_ubuntu_dependencies/tasks/main.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -# copyright Utrecht University - -- name: Run irods_ubuntu_dependencies tasks if role has not run yet - ansible.builtin.include_tasks: main-tasks.yml - when: '"yoda_irods_ubuntu_dependencies_role_has_run" not in hostvars[inventory_hostname]' - -- name: Register that irods_ubuntu_dependencies role has run - ansible.builtin.set_fact: - yoda_irods_ubuntu_dependencies_role_has_run: true - when: '"yoda_irods_ubuntu_dependencies_role_has_run" not in hostvars[inventory_hostname]' diff --git a/roles/mailpit/meta/main.yml b/roles/mailpit/meta/main.yml index c1f88e5a5..31a11d02e 100644 --- a/roles/mailpit/meta/main.yml +++ b/roles/mailpit/meta/main.yml @@ -9,11 +9,10 @@ galaxy_info: platforms: - name: EL version: - - 7 - 8 - 9 - name: Ubuntu - version: focal + version: noble dependencies: diff --git a/roles/minio/meta/main.yml b/roles/minio/meta/main.yml index 58dc78750..2532a8f4a 100644 --- a/roles/minio/meta/main.yml +++ b/roles/minio/meta/main.yml @@ -8,9 +8,9 @@ galaxy_info: min_ansible_version: '2.11' platforms: - name: EL - version: 7 + version: 8 - name: Ubuntu - version: focal + version: noble dependencies: diff --git a/roles/nvm/meta/main.yml b/roles/nvm/meta/main.yml index 4faf31c5c..f4248348c 100644 --- a/roles/nvm/meta/main.yml +++ b/roles/nvm/meta/main.yml @@ -8,9 +8,9 @@ galaxy_info: min_ansible_version: '2.11' platforms: - name: EL - version: 7 + version: 8 - name: Ubuntu - version: focal + version: noble dependencies: - role: yoda_portal diff --git a/roles/opensearch/meta/main.yml b/roles/opensearch/meta/main.yml index 757999e17..61fc96ae3 100644 --- a/roles/opensearch/meta/main.yml +++ b/roles/opensearch/meta/main.yml @@ -8,6 +8,6 @@ galaxy_info: min_ansible_version: '2.11' platforms: - name: EL - version: 7 + version: 8 - name: Ubuntu - version: focal + version: noble diff --git a/roles/pam_python/meta/main.yml b/roles/pam_python/meta/main.yml index 7b32b0e3d..a77b14976 100644 --- a/roles/pam_python/meta/main.yml +++ b/roles/pam_python/meta/main.yml @@ -8,6 +8,6 @@ galaxy_info: min_ansible_version: '2.11' platforms: - name: EL - version: 7 + version: 8 - name: Ubuntu - version: focal + version: noble diff --git a/roles/pgbouncer/meta/main.yml b/roles/pgbouncer/meta/main.yml index b6b1e26db..b47dfa7f7 100644 --- a/roles/pgbouncer/meta/main.yml +++ b/roles/pgbouncer/meta/main.yml @@ -9,10 +9,10 @@ galaxy_info: platforms: - name: EL version: - - 7 - 8 + - 9 - name: Ubuntu - version: focal + version: noble dependencies: - role: postgresql diff --git a/roles/postfix/meta/main.yml b/roles/postfix/meta/main.yml index 74195120c..072729744 100644 --- a/roles/postfix/meta/main.yml +++ b/roles/postfix/meta/main.yml @@ -9,10 +9,10 @@ galaxy_info: platforms: - name: EL version: - - 7 + - 8 - 9 - name: Ubuntu - version: focal + version: noble dependencies: diff --git a/roles/postgresql/meta/main.yml b/roles/postgresql/meta/main.yml index b02c8d3a4..a08a5aa05 100644 --- a/roles/postgresql/meta/main.yml +++ b/roles/postgresql/meta/main.yml @@ -9,10 +9,10 @@ galaxy_info: platforms: - name: EL version: - - 7 - 8 + - 9 - name: Ubuntu - version: focal + version: noble dependencies: - role: certificates diff --git a/roles/postgresql/tasks/setup-redhat.yml b/roles/postgresql/tasks/setup-redhat.yml index b89bc5cc2..b2d6abcdd 100644 --- a/roles/postgresql/tasks/setup-redhat.yml +++ b/roles/postgresql/tasks/setup-redhat.yml @@ -18,15 +18,15 @@ when: not ansible_check_mode -- name: Ensure PsycoPG2 is installed (EL7) - ansible.builtin.package: - name: python-psycopg2 - state: present - when: "ansible_distribution_major_version == '7' and not ansible_check_mode" +# - name: Ensure PsycoPG2 is installed (EL7) +# ansible.builtin.package: +# name: python-psycopg2 +# state: present +# when: "ansible_distribution_major_version == '7' and not ansible_check_mode" -- name: Ensure PsycoPG2 is installed (EL8) +- name: Ensure PsycoPG2 is installed (EL8 and EL9) ansible.builtin.package: name: python3-psycopg2 state: present - when: "ansible_distribution_major_version == '8' and not ansible_check_mode" + when: "(ansible_distribution_major_version == '8' or ansible_distribution_major_version == '9') and not ansible_check_mode" diff --git a/roles/postgresql_odbc/meta/main.yml b/roles/postgresql_odbc/meta/main.yml index c913fd86e..9b2a871a5 100644 --- a/roles/postgresql_odbc/meta/main.yml +++ b/roles/postgresql_odbc/meta/main.yml @@ -9,10 +9,10 @@ galaxy_info: platforms: - name: EL version: - - 7 - 8 + - 9 - name: Ubuntu - version: focal + version: noble dependencies: - role: postgresql_repository diff --git a/roles/postgresql_odbc/tasks/setup-debian.yml b/roles/postgresql_odbc/tasks/setup-debian.yml index f02461b40..0ad388630 100644 --- a/roles/postgresql_odbc/tasks/setup-debian.yml +++ b/roles/postgresql_odbc/tasks/setup-debian.yml @@ -6,4 +6,6 @@ ansible.builtin.package: name: - unixodbc + - unixodbc-dev + - odbcinst state: present diff --git a/roles/postgresql_odbc/tasks/setup-redhat.yml b/roles/postgresql_odbc/tasks/setup-redhat.yml index b70e939c2..d4ce2d58c 100644 --- a/roles/postgresql_odbc/tasks/setup-redhat.yml +++ b/roles/postgresql_odbc/tasks/setup-redhat.yml @@ -14,5 +14,6 @@ name: - "postgresql{{ pgsql_version }}-odbc" - unixODBC + - unixODBC-devel state: present when: not ansible_check_mode diff --git a/roles/postgresql_repository/defaults/main.yml b/roles/postgresql_repository/defaults/main.yml index 74266362a..1324a92ca 100644 --- a/roles/postgresql_repository/defaults/main.yml +++ b/roles/postgresql_repository/defaults/main.yml @@ -9,4 +9,4 @@ postgresql_rpm_key_url: https://download.postgresql.org/pub/repos/yum/keys/PGDG- # PostgreSQL APT package repo postgresql_apt_signing_key: https://www.postgresql.org/media/keys/ACCC4CF8.asc -postgresql_apt_repo: deb http://apt.postgresql.org/pub/repos/apt focal-pgdg main +postgresql_apt_repo: deb http://apt.postgresql.org/pub/repos/apt noble-pgdg main diff --git a/roles/postgresql_repository/meta/main.yml b/roles/postgresql_repository/meta/main.yml index a09a21067..c507d1906 100644 --- a/roles/postgresql_repository/meta/main.yml +++ b/roles/postgresql_repository/meta/main.yml @@ -9,7 +9,7 @@ galaxy_info: platforms: - name: EL version: - - 7 - 8 + - 9 - name: Ubuntu - version: focal + version: noble diff --git a/roles/python2/meta/main.yml b/roles/python2/meta/main.yml deleted file mode 100644 index e0727ddd1..000000000 --- a/roles/python2/meta/main.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -# copyright Utrecht University - -galaxy_info: - author: Sietse Snel - description: Install Python 2.7 - license: GPLv3 - min_ansible_version: '2.11' - platforms: - - name: EL - version: 7 - - name: Ubuntu - version: focal - -dependencies: - # For apt update - - role: common diff --git a/roles/python2/tasks/install-debian.yml b/roles/python2/tasks/install-debian.yml deleted file mode 100644 index e16abec1b..000000000 --- a/roles/python2/tasks/install-debian.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -# copyright Utrecht University - -- name: Ensure Python 2.7 is installed - ansible.builtin.apt: - name: - - gcc - - python2.7 - - python2.7-dev - state: present - - -- name: Download get-pip script for Python 2.7 - ansible.builtin.get_url: - url: 'https://bootstrap.pypa.io/pip/2.7/get-pip.py' - dest: '/usr/local/bin/get-pip.py' - checksum: 'sha256:40ee07eac6674b8d60fce2bbabc148cf0e2f1408c167683f110fd608b8d6f416' - mode: '0755' - - -- name: Install pip for Python 2.7 - ansible.builtin.command: - cmd: /usr/bin/python2.7 /usr/local/bin/get-pip.py - creates: /usr/local/bin/pip2 diff --git a/roles/python2/tasks/install-redhat.yml b/roles/python2/tasks/install-redhat.yml deleted file mode 100644 index a90f842e0..000000000 --- a/roles/python2/tasks/install-redhat.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -# copyright Utrecht University - -- name: Ensure Python 2.7 is installed - ansible.builtin.yum: - name: - - gcc - - python - - python-pip - state: present - - -- name: Update Python 2.7 pip version - ansible.builtin.pip: - name: - - pip==20.2.4 - executable: /usr/bin/pip - state: present diff --git a/roles/python2/tasks/main-tasks.yml b/roles/python2/tasks/main-tasks.yml deleted file mode 100644 index d46f0d6af..000000000 --- a/roles/python2/tasks/main-tasks.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -# copyright Utrecht University - -- name: Install Python 2.7 for Debian family - ansible.builtin.include_tasks: install-debian.yml - when: ansible_os_family == 'Debian' - - -- name: Install Python 2.7 for RedHat family - ansible.builtin.include_tasks: install-redhat.yml - when: ansible_os_family == 'RedHat' diff --git a/roles/python2/tasks/main.yml b/roles/python2/tasks/main.yml deleted file mode 100644 index 4019b52ac..000000000 --- a/roles/python2/tasks/main.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -# copyright Utrecht University - -- name: Run python2 tasks if role has not run yet - ansible.builtin.include_tasks: main-tasks.yml - when: '"yoda_python2_role_has_run" not in hostvars[inventory_hostname]' - -- name: Register that python2 role has run - ansible.builtin.set_fact: - yoda_python2_role_has_run: true - when: '"yoda_python2_role_has_run" not in hostvars[inventory_hostname]' diff --git a/roles/python3/tasks/install-debian.yml b/roles/python3/tasks/install-debian.yml index b28c3a932..260e4cdad 100644 --- a/roles/python3/tasks/install-debian.yml +++ b/roles/python3/tasks/install-debian.yml @@ -9,6 +9,5 @@ - python3-dev - python3-pip - python3-venv - - python3-virtualenv - python3-wheel state: present diff --git a/roles/python_irodsclient/meta/main.yml b/roles/python_irodsclient/meta/main.yml index a3239533f..9f7b9122a 100644 --- a/roles/python_irodsclient/meta/main.yml +++ b/roles/python_irodsclient/meta/main.yml @@ -8,10 +8,9 @@ galaxy_info: min_ansible_version: '2.11' platforms: - name: EL - version: 7 + version: 8 - name: Ubuntu - version: focal + version: noble dependencies: - - role: python2 - role: python3 diff --git a/roles/python_irodsclient/tasks/main-tasks.yml b/roles/python_irodsclient/tasks/main-tasks.yml index 64fdc01a5..d44c5ba08 100644 --- a/roles/python_irodsclient/tasks/main-tasks.yml +++ b/roles/python_irodsclient/tasks/main-tasks.yml @@ -5,24 +5,10 @@ ansible.builtin.include_vars: "{{ ansible_os_family }}.yml" -- name: Ensure python-irodsclient dependencies are installed (Python 2) - ansible.builtin.pip: - name: - - setuptools==44.1.1 - executable: "{{ python_irodsclient_pip2_path }}" - state: present - - -- name: Ensure python-irodsclient is installed (Python 2) - ansible.builtin.pip: - name: - - "python-irodsclient=={{ python_irodsclient_version }}" - executable: "{{ python_irodsclient_pip2_path }}" - state: present - - +# TODO install in venv - name: Ensure python-irodsclient is installed (Python 3) ansible.builtin.pip: name: "python_irodsclient=={{ python_irodsclient_version }}" executable: "{{ python_irodsclient_pip3_path }}" + extra_args: "--break-system-packages" when: not ansible_check_mode diff --git a/roles/python_irodsclient/vars/Debian.yml b/roles/python_irodsclient/vars/Debian.yml index 90f520a44..45d0ff83e 100644 --- a/roles/python_irodsclient/vars/Debian.yml +++ b/roles/python_irodsclient/vars/Debian.yml @@ -1,5 +1,4 @@ --- # copyright Utrecht University -python_irodsclient_pip2_path: /usr/local/bin/pip2 python_irodsclient_pip3_path: /usr/bin/pip3 diff --git a/roles/python_irodsclient/vars/RedHat.yml b/roles/python_irodsclient/vars/RedHat.yml index a9cc19331..2a3af67de 100644 --- a/roles/python_irodsclient/vars/RedHat.yml +++ b/roles/python_irodsclient/vars/RedHat.yml @@ -1,5 +1,4 @@ --- # copyright Utrecht University -python_irodsclient_pip2_path: /usr/bin/pip python_irodsclient_pip3_path: /usr/local/bin/pip3 diff --git a/roles/sqlcipher/meta/main.yml b/roles/sqlcipher/meta/main.yml index 5b3e2e595..54ce35b37 100644 --- a/roles/sqlcipher/meta/main.yml +++ b/roles/sqlcipher/meta/main.yml @@ -14,4 +14,4 @@ galaxy_info: dependencies: - - role: python2 + - role: python3 diff --git a/roles/sqlcipher/tasks/install-ubuntu.yml b/roles/sqlcipher/tasks/install-ubuntu.yml index 9fe3be93a..2be90dfd7 100644 --- a/roles/sqlcipher/tasks/install-ubuntu.yml +++ b/roles/sqlcipher/tasks/install-ubuntu.yml @@ -5,31 +5,10 @@ ansible.builtin.include_vars: "{{ ansible_os_family }}.yml" -# Ubuntu 20.04 LTS includes SQLCipher 3 by default. We need SQLCipher 4, -# so that we can migrate token databases created on SQLCipher 4 to Ubuntu -# 20.04 LTS servers. -- name: Ensure standard SQLCipher packages are not installed +- name: Ensure SQLCipher packages are installed ansible.builtin.package: name: - - libsqlcipher0 - sqlcipher + - libsqlcipher1 - libsqlcipher-dev - state: absent - - -- name: Download SQLCipher packages - ansible.builtin.get_url: - url: '{{ item.value.url }}' - dest: '{{ sqlcipher_package_dir }}/{{ item.value.filename }}' - checksum: '{{ item.value.checksum }}' - mode: '0644' - when: item.value.package not in ansible_facts.packages or item.value.version != ansible_facts.packages[item.value.package][0]['version'] - with_dict: '{{ sqlcipher_packages }}' - - -- name: Install SQLCipher from downloaded package files - ansible.builtin.apt: - deb: '{{ sqlcipher_package_dir }}/{{ item.value.filename }}' state: present - when: not ansible_check_mode and (item.value.package not in ansible_facts.packages or item.value.version != ansible_facts.packages[item.value.package][0]['version']) - with_dict: '{{ sqlcipher_packages }}' diff --git a/roles/yoda_davrods/meta/main.yml b/roles/yoda_davrods/meta/main.yml index f9da46e27..b321e9042 100644 --- a/roles/yoda_davrods/meta/main.yml +++ b/roles/yoda_davrods/meta/main.yml @@ -8,9 +8,9 @@ galaxy_info: min_ansible_version: '2.11' platforms: - name: EL - version: 7 + version: 8 - name: Ubuntu - version: focal + version: noble dependencies: diff --git a/roles/yoda_davrods/vars/Debian.yml b/roles/yoda_davrods/vars/Debian.yml index 99313ba5a..1a5860268 100644 --- a/roles/yoda_davrods/vars/Debian.yml +++ b/roles/yoda_davrods/vars/Debian.yml @@ -7,7 +7,7 @@ openssl_certs_dir: '/etc/ssl/certs' yoda_davrods_boost_package: libboost-all-dev yoda_davrods_jansson_package: libjansson4 -yoda_davrods_version: 4.2.12-1.5.1 +yoda_davrods_version: 4.3.3-1.5.1 yoda_davrods_site_config_dir: /etc/apache2/sites-available yoda_davrods_site_enabled_config_dir: /etc/apache2/sites-enabled diff --git a/roles/yoda_davrods/vars/RedHat.yml b/roles/yoda_davrods/vars/RedHat.yml index d77516faa..0a91e178e 100644 --- a/roles/yoda_davrods/vars/RedHat.yml +++ b/roles/yoda_davrods/vars/RedHat.yml @@ -12,10 +12,10 @@ rpm_dest_dir: /tmp # davrods RPM location and checksum davrods: - package: davrods-4.2.12_1.5.1-1 - url: https://github.com/UtrechtUniversity/davrods/releases/download/4.2.12_1.5.1/ - filename: davrods-4.2.12_1.5.1-1.rpm - checksum: sha256:dcd1e17aa1295ec1347c050d3e7936d670379ee51b1ec0ec15ce5d9311837e34 + package: davrods-4.3.3_1.5.1-1 + url: https://github.com/UtrechtUniversity/davrods/releases/tag/4.3.3_1.5.1/ + filename: davrods-4.3.3_1.5.1-1.rpm + checksum: sha256:ea5f3667123fe73a01f705d7d684689569ca5d3a4eea27b96678f3583593da62 yoda_davrods_site_config_dir: /etc/httpd/conf.d yoda_davrods_config_file: davrods-vhost.conf diff --git a/roles/yoda_external_user_service/defaults/main.yml b/roles/yoda_external_user_service/defaults/main.yml index 95a333a3b..08aa83559 100644 --- a/roles/yoda_external_user_service/defaults/main.yml +++ b/roles/yoda_external_user_service/defaults/main.yml @@ -3,7 +3,7 @@ yoda_deployment_user: yodadeployment yoda_version: development -yoda_eus_version: "{{ yoda_version }}" +yoda_eus_version: "development-irods-4.3" yoda_eus_fqdn: eus.yoda.test yoda_eus_port: 443 diff --git a/roles/yoda_external_user_service/tasks/main.yml b/roles/yoda_external_user_service/tasks/main.yml index 3419a947d..775bb66cf 100644 --- a/roles/yoda_external_user_service/tasks/main.yml +++ b/roles/yoda_external_user_service/tasks/main.yml @@ -197,22 +197,30 @@ notify: Restart Apache webserver -- name: Ensure Yoda EUS virtualenv exists +- name: Ensure Yoda EUS venv exists become_user: '{{ yoda_deployment_user }}' become: true - ansible.builtin.command: "virtualenv --python {{ yoda_external_user_service_python3_path }} /var/www/extuser/yoda-external-user-service/venv" + ansible.builtin.command: "{{ yoda_external_user_service_python3_path }} -m venv /var/www/extuser/yoda-external-user-service/venv" args: - creates: "/var/www/extuser/yoda-external-user-service/venv" + creates: "/var/www/extuser/yoda-external-user-service/venv/bin/activate_this.py" -- name: Upgrade pip in EUS virtual environment +- name: Upgrade pip and virtualenv in Yoda EUS virtual environment become_user: '{{ yoda_deployment_user }}' become: true ansible.builtin.pip: name: - - pip==23.0.1 + - pip==24.3.1 + - virtualenv==20.28.0 executable: /var/www/extuser/yoda-external-user-service/venv/bin/pip3 - when: not ansible_check_mode + + +- name: Ensure Yoda EUS virtualenv exists + become_user: '{{ yoda_deployment_user }}' + become: true + ansible.builtin.command: "/var/www/extuser/yoda-external-user-service/venv/bin/python3 -m virtualenv /var/www/extuser/yoda-external-user-service/venv" + args: + creates: "/var/www/extuser/yoda-external-user-service/venv/bin/activate_this.py" # Change detection in git repositories is unreliable in some Ansible versions, so we always diff --git a/roles/yoda_external_user_service/vars/Debian.yml b/roles/yoda_external_user_service/vars/Debian.yml index 32eef90d2..b95d08920 100644 --- a/roles/yoda_external_user_service/vars/Debian.yml +++ b/roles/yoda_external_user_service/vars/Debian.yml @@ -5,7 +5,7 @@ openssl_private_dir: '/etc/ssl/private/' openssl_certs_dir: '/etc/ssl/certs' yoda_external_user_service_python3_path: /usr/bin/python3 -yoda_external_user_service_python3_include_path: /usr/include/python3.8 +yoda_external_user_service_python3_include_path: /usr/include/python3.12 yoda_external_user_service_site_config_dir: /etc/apache2/sites-available yoda_external_user_service_site_enabled_config_dir: /etc/apache2/sites-enabled yoda_external_user_service_site_config_file: 003-yoda-eus-vhost.conf diff --git a/roles/yoda_moai/defaults/main.yml b/roles/yoda_moai/defaults/main.yml index f138ec6b9..049fec2ed 100644 --- a/roles/yoda_moai/defaults/main.yml +++ b/roles/yoda_moai/defaults/main.yml @@ -1,7 +1,7 @@ --- # copyright Utrecht University -yoda_moai_version: "{{ yoda_version }}" +yoda_moai_version: "development-irods-4.3" yoda_moai_user: moai yoda_moai_home: /var/www/moai yoda_moai_database_connection_string: "sqlite:///{{ yoda_moai_home }}/moai.db" diff --git a/roles/yoda_moai/tasks/main.yml b/roles/yoda_moai/tasks/main.yml index 85ae031f4..bd7264c68 100644 --- a/roles/yoda_moai/tasks/main.yml +++ b/roles/yoda_moai/tasks/main.yml @@ -47,42 +47,32 @@ register: repochanges -- name: Check if MOAI virtual environment is running Python 2.7 - ansible.builtin.stat: - path: '/var/www/moai/yoda-moai/venv/bin/pip2.7' - register: moai_venv_python27 - - -- name: Check if MOAI virtual environment is running Python 3.6 - ansible.builtin.stat: - path: '/var/www/moai/yoda-moai/venv/bin/pip3.6' - register: moai_venv_python36 - - -# For upgrade to Yoda 1.9 -- name: Move old MOAI virtual environment - ansible.builtin.command: # noqa no-changed-when - cmd: "mv /var/www/moai/yoda-moai/venv /var/www/moai/venv.backup-pre-1.9" - when: moai_venv_python27.stat.exists or moai_venv_python36.stat.exists - - -- name: Ensure Yoda MOAI virtualenv exists +- name: Ensure Yoda MOAI venv exists become_user: '{{ yoda_moai_user }}' become: true - ansible.builtin.command: "virtualenv --python {{ yoda_moai_python3_path }} {{ yoda_moai_home }}/yoda-moai/venv" + ansible.builtin.command: "{{ yoda_moai_python3_path }} -m venv {{ yoda_moai_home }}/yoda-moai/venv" args: - creates: "{{ yoda_moai_home }}/yoda-moai/venv" + creates: "{{ yoda_moai_home }}/yoda-moai/venv/bin/activate_this.py" -- name: Upgrade pip in virtual environment +- name: Upgrade pip and virtualenv in Yoda MOAI environment become_user: '{{ yoda_moai_user }}' become: true ansible.builtin.pip: name: - - pip==23.0.1 + - pip==24.3.1 + - virtualenv==20.28.0 executable: /var/www/moai/yoda-moai/venv/bin/pip3 +- name: Ensure Yoda MOAI virtualenv exists + become_user: '{{ yoda_moai_user }}' + become: true + ansible.builtin.command: "{{ yoda_moai_home }}/yoda-moai/venv/bin/python3 -m virtualenv {{ yoda_moai_home }}/yoda-moai/venv" + args: + creates: "{{ yoda_moai_home }}/yoda-moai/venv/bin/activate_this.py" + + # We use the PySqlite3 dialect to avoid compatibility issues between SQLAlchemy # and the old SQlite version bundled with CentOS 7 - name: Ensure PySQLite3 is installed in MOAI virtual environment diff --git a/roles/yoda_moai/templates/moai.wsgi.j2 b/roles/yoda_moai/templates/moai.wsgi.j2 index d83be1bc7..bd48c2fee 100644 --- a/roles/yoda_moai/templates/moai.wsgi.j2 +++ b/roles/yoda_moai/templates/moai.wsgi.j2 @@ -2,10 +2,6 @@ import configparser import os import sys -activate_this = '/var/www/moai/yoda-moai/venv/bin/activate_this.py' -with open(activate_this) as file_: - exec(file_.read(), dict(__file__=activate_this)) - from paste.deploy import loadapp from logging.config import fileConfig diff --git a/roles/yoda_moai/vars/Debian.yml b/roles/yoda_moai/vars/Debian.yml index 05e25ed03..6a8550972 100644 --- a/roles/yoda_moai/vars/Debian.yml +++ b/roles/yoda_moai/vars/Debian.yml @@ -5,7 +5,7 @@ openssl_private_dir: '/etc/ssl/private/' openssl_certs_dir: '/etc/ssl/certs' yoda_moai_python3_path: /usr/bin/python3 -yoda_moai_python3_include_path: /usr/include/python3.8 +yoda_moai_python3_include_path: /usr/include/python3.12 yoda_moai_sqlite3_include_path: /usr/include yoda_moai_sqlite3_lib_path: /usr/lib/x86_64-linux-gnu diff --git a/roles/yoda_portal/defaults/main.yml b/roles/yoda_portal/defaults/main.yml index 75ce95ab9..1806914b1 100644 --- a/roles/yoda_portal/defaults/main.yml +++ b/roles/yoda_portal/defaults/main.yml @@ -28,7 +28,7 @@ yoda_theme_mapping: wur: "Wageningen University & Research" # Yoda portal -yoda_portal_version: "{{ yoda_version }}" +yoda_portal_version: "development-irods-4.3" yoda_portal_log_api_call_duration: false yoda_portal_path: /var/www/yoda # Path to location of portal yoda_config_path: '/var/www/yoda/config' # Path to portal's shared configuration @@ -42,7 +42,7 @@ yoda_portal_wsgi_daemon_threads: 15 # iRODS configuration. irods_default_resc: irodsResc # iRODS default resource name -irods_authentication_scheme: PAM # iRODS authentication method: "Native" or "PAM" +irods_authentication_scheme: pam_password # iRODS authentication method: "native" or "pam_password" irods_zone: tempZone # The name of the iRODS Zone irods_icat_fqdn: icat.yoda.test # iRODS iCAT fully qualified domain name (FQDN) irods_icat_port: 1247 diff --git a/roles/yoda_portal/tasks/main.yml b/roles/yoda_portal/tasks/main.yml index 52ae6199a..72847567a 100644 --- a/roles/yoda_portal/tasks/main.yml +++ b/roles/yoda_portal/tasks/main.yml @@ -30,25 +30,10 @@ register: portalchanges -- name: Check if Portal virtual environment is running Python 3.6 - ansible.builtin.stat: - path: '/var/www/yoda/venv/bin/pip3.6' - register: portal_venv_python36 - - -# For upgrade to Yoda 1.9 -- name: Move old portal virtual environment - become_user: '{{ yoda_deployment_user }}' - become: true - ansible.builtin.command: # noqa no-changed-when - cmd: "mv /var/www/yoda/venv /var/www/yoda/venv.backup-pre-1.9" - when: portal_venv_python36.stat.exists - - - name: Ensure Yoda portal virtualenv exists become_user: "{{ yoda_deployment_user }}" become: true - ansible.builtin.command: "virtualenv --python {{ yoda_portal_python3_path }} /var/www/yoda/venv" + ansible.builtin.command: "{{ yoda_portal_python3_path }} -m venv /var/www/yoda/venv" args: creates: /var/www/yoda/venv @@ -58,7 +43,7 @@ become: true ansible.builtin.pip: name: - - pip==23.0.1 + - pip==24.3.1 executable: /var/www/yoda/venv/bin/pip3 @@ -68,7 +53,7 @@ ansible.builtin.pip: requirements: /var/www/yoda/requirements.txt virtualenv: '/var/www/yoda/venv' - virtualenv_python: python3.8 + virtualenv_python: python3.12 environment: C_INCLUDE_PATH: "{{ yoda_portal_python3_include_path }}" notify: Restart Apache webserver diff --git a/roles/yoda_portal/vars/Debian.yml b/roles/yoda_portal/vars/Debian.yml index 114f5facc..5ae363d03 100644 --- a/roles/yoda_portal/vars/Debian.yml +++ b/roles/yoda_portal/vars/Debian.yml @@ -5,7 +5,7 @@ openssl_private_dir: '/etc/ssl/private/' openssl_certs_dir: '/etc/ssl/certs' yoda_portal_python3_path: /usr/bin/python3 -yoda_portal_python3_include_path: /usr/include/python3.8 +yoda_portal_python3_include_path: /usr/include/python3.12 yoda_portal_site_config_dir: /etc/apache2/sites-available yoda_portal_site_enabled_config_dir: /etc/apache2/sites-enabled yoda_portal_site_config_file: 001-yoda-portal-vhost.conf diff --git a/roles/yoda_rulesets/defaults/main.yml b/roles/yoda_rulesets/defaults/main.yml index 0f3f0c7bc..e8b40ae91 100644 --- a/roles/yoda_rulesets/defaults/main.yml +++ b/roles/yoda_rulesets/defaults/main.yml @@ -10,7 +10,7 @@ core_rulesets: - name: yoda-ruleset repo: https://github.com/UtrechtUniversity/yoda-ruleset.git ruleset_name: rules-uu - version: "{{ yoda_ruleset_version }}" + version: "irods-4.3" install_scripts: true - name: core ruleset_name: core diff --git a/roles/yoda_rulesets/meta/main.yml b/roles/yoda_rulesets/meta/main.yml index 40fb37aff..afd1ad532 100644 --- a/roles/yoda_rulesets/meta/main.yml +++ b/roles/yoda_rulesets/meta/main.yml @@ -14,7 +14,6 @@ galaxy_info: dependencies: - - role: python2 - role: python3 - role: python_irodsclient - role: sqlcipher diff --git a/roles/yoda_rulesets/tasks/main.yml b/roles/yoda_rulesets/tasks/main.yml index c18f597d2..81af9bc58 100644 --- a/roles/yoda_rulesets/tasks/main.yml +++ b/roles/yoda_rulesets/tasks/main.yml @@ -105,55 +105,8 @@ become: true ansible.builtin.pip: requirements: /etc/irods/yoda-ruleset/requirements.txt - extra_args: --user - executable: "{{ yoda_rulesets_pip2_path }}" - - -- name: Ensure pysqlcipher3 is installed - become_user: '{{ irods_service_account }}' - become: true - ansible.builtin.pip: - name: pysqlcipher3==1.0.4 - executable: "{{ yoda_rulesets_pip2_path }}" - extra_args: --user - when: ansible_os_family == 'RedHat' - - -- name: Check installed version of pysqlcipher3 - become_user: "{{ irods_service_account }}" - become: true - ansible.builtin.shell: "{{ yoda_rulesets_pip2_path }} show pysqlcipher3 | grep Version | cut -d ' ' -f 2" - ignore_errors: true - changed_when: false - register: pysqlcipher3_version - when: ansible_os_family == 'Debian' - - -- name: Ensure PyPi build of pysqlcipher3 is absent - become_user: "{{ irods_service_account }}" - become: true - ansible.builtin.pip: - name: pysqlcipher3==1.0.4 - executable: "{{ yoda_rulesets_pip2_path }}" - state: absent - when: ansible_os_family == 'Debian' and '1.0.4' in pysqlcipher3_version.stdout - - -- name: Ensure custom build of pysqlcipher3 is installed globally for ruleset - become_user: "{{ irods_service_account }}" - become: true - ansible.builtin.command: # noqa no-changed-when - cmd: python2 -m easy_install --user https://github.com/UtrechtUniversity/pysqlcipher3/releases/download/v1.2.1/pysqlcipher3-1.2.1-py2.7-linux-x86_64.egg - when: ansible_os_family == 'Debian' and '1.2.1' not in pysqlcipher3_version.stdout - - -- name: Ensure Python 3 jsonschema is installed - become_user: '{{ irods_service_account }}' - become: true - ansible.builtin.pip: - name: jsonschema==4.19.1 + extra_args: --user --break-system-packages executable: "{{ yoda_rulesets_pip3_path }}" - extra_args: --user - name: Link ruleset directory diff --git a/roles/yoda_rulesets/tasks/yoda-ruleset.yml b/roles/yoda_rulesets/tasks/yoda-ruleset.yml index ea2228771..deec612de 100644 --- a/roles/yoda_rulesets/tasks/yoda-ruleset.yml +++ b/roles/yoda_rulesets/tasks/yoda-ruleset.yml @@ -88,12 +88,12 @@ become: true ansible.builtin.copy: remote_src: true - src: "/etc/irods/yoda-ruleset/tools/{{ item }}" + src: "/etc/irods/yoda-ruleset/tools/admin/{{ item }}" dest: "/var/lib/irods/msiExecCmd_bin/{{ item }}" mode: '0755' when: not ansible_check_mode with_items: - - scheduled-copytovault.sh + - admin-scheduled-copytovault.sh - admin-remove-orphan-vault-if-empty.sh - admin-vaultactions.sh - admin-vaultingest.sh @@ -114,7 +114,7 @@ become: true ansible.builtin.copy: remote_src: true - src: "/etc/irods/yoda-ruleset/tools/{{ item }}" + src: "/etc/irods/yoda-ruleset/tools/admin/{{ item }}" dest: "/var/lib/irods/msiExecCmd_bin/{{ item }}" mode: '0755' when: not ansible_check_mode and (enable_data_package_archive or enable_data_package_download) @@ -497,7 +497,7 @@ name: 'asynchronous-replication-{{ item }}' minute: '*/5' job: > - /bin/python /etc/irods/yoda-ruleset/tools/async-data-replicate.py + /bin/python3 /etc/irods/yoda-ruleset/tools/async-data-replicate.py --balance_id_min={{ (item * (64 / (async_replication_jobs | int)) - (64 / (async_replication_jobs | int))) | int + 1 }} --balance_id_max={{ (item * (64 / (async_replication_jobs | int))) | int }} --batch_size_limit={{ ((async_replication_batch_size | int) / (async_replication_jobs | int)) | int }} @@ -514,7 +514,7 @@ name: 'asynchronous-revisions-{{ item }}' minute: '6,16,26,36,46,56' job: > - /bin/python /etc/irods/yoda-ruleset/tools/async-data-revision.py + /bin/python3 /etc/irods/yoda-ruleset/tools/async-data-revision.py --balance_id_min={{ (item * (64 / (async_revision_jobs | int)) - (64 / (async_revision_jobs | int))) | int + 1 }} --balance_id_max={{ (item * (64 / (async_revision_jobs | int))) | int }} --batch_size_limit={{ ((async_revision_batch_size | int) / (async_revision_jobs | int)) | int }} @@ -660,7 +660,7 @@ become: true ansible.builtin.copy: remote_src: true - src: "/etc/irods/yoda-ruleset/tools/{{ item }}" + src: "/etc/irods/yoda-ruleset/tools/admin/{{ item }}" dest: "/var/lib/irods/msiExecCmd_bin/{{ item }}" mode: '0755' when: enable_datarequest and not ansible_check_mode diff --git a/roles/yoda_rulesets/vars/Debian.yml b/roles/yoda_rulesets/vars/Debian.yml index 11b81d9fd..9b7fd019a 100644 --- a/roles/yoda_rulesets/vars/Debian.yml +++ b/roles/yoda_rulesets/vars/Debian.yml @@ -1,6 +1,5 @@ --- # copyright Utrecht University -yoda_rulesets_pip2_path: /usr/local/bin/pip2 yoda_rulesets_pip3_path: /usr/bin/pip3 yoda_rulesets_python3_interpreter: /usr/bin/python3 diff --git a/roles/yoda_rulesets/vars/RedHat.yml b/roles/yoda_rulesets/vars/RedHat.yml index 2ae47fbb9..ca8b68494 100644 --- a/roles/yoda_rulesets/vars/RedHat.yml +++ b/roles/yoda_rulesets/vars/RedHat.yml @@ -1,6 +1,5 @@ --- # copyright Utrecht University -yoda_rulesets_pip2_path: /usr/bin/pip yoda_rulesets_pip3_path: /usr/local/bin/pip3 yoda_rulesets_python3_interpreter: /usr/local/bin/python3 diff --git a/roles/yoda_test/tasks/run-revision-job.yml b/roles/yoda_test/tasks/run-revision-job.yml index aff6fa214..ecf1b26a9 100644 --- a/roles/yoda_test/tasks/run-revision-job.yml +++ b/roles/yoda_test/tasks/run-revision-job.yml @@ -5,4 +5,4 @@ become_user: '{{ irods_service_account }}' become: true ansible.builtin.command: - /bin/python /etc/irods/yoda-ruleset/tools/async-data-revision.py + /bin/python3 /etc/irods/yoda-ruleset/tools/async-data-revision.py diff --git a/roles/yoda_test_users_eus/meta/main.yml b/roles/yoda_test_users_eus/meta/main.yml index b36e48037..771d6f099 100644 --- a/roles/yoda_test_users_eus/meta/main.yml +++ b/roles/yoda_test_users_eus/meta/main.yml @@ -8,9 +8,9 @@ galaxy_info: min_ansible_version: '2.11' platforms: - name: EL - version: 7 + version: 8 - name: Ubuntu - version: focal + version: noble dependencies: diff --git a/roles/yoda_test_users_eus/tasks/main.yml b/roles/yoda_test_users_eus/tasks/main.yml index 28f8d3578..d0357738b 100644 --- a/roles/yoda_test_users_eus/tasks/main.yml +++ b/roles/yoda_test_users_eus/tasks/main.yml @@ -27,9 +27,10 @@ - name: Install passlib and dependencies ansible.builtin.pip: name: - - bcrypt==3.1.7 + - bcrypt==4.2.1 - passlib executable: "{{ yoda_test_pip3_path }}" + extra_args: --break-system-packages state: present diff --git a/roles/yoda_web_mock/meta/main.yml b/roles/yoda_web_mock/meta/main.yml index 659aa160f..3273b2aa0 100644 --- a/roles/yoda_web_mock/meta/main.yml +++ b/roles/yoda_web_mock/meta/main.yml @@ -8,9 +8,9 @@ galaxy_info: min_ansible_version: '2.11' platforms: - name: EL - version: 7 + version: 8 - name: Ubuntu - version: focal + version: noble dependencies: diff --git a/roles/yoda_web_mock/tasks/main.yml b/roles/yoda_web_mock/tasks/main.yml index 27c4a7988..c4c2b9a71 100644 --- a/roles/yoda_web_mock/tasks/main.yml +++ b/roles/yoda_web_mock/tasks/main.yml @@ -32,24 +32,33 @@ force: true -- name: Ensure Yoda web mock virtualenv exists +- name: Ensure Yoda web mock venv exists become_user: '{{ yoda_deployment_user }}' become: true - ansible.builtin.command: "virtualenv --python {{ yoda_web_mock_python3_path }} /var/www/webmock/venv" + ansible.builtin.command: "python3 -m venv /var/www/webmock/venv" args: creates: "/var/www/webmock/venv" -- name: Upgrade pip in Yoda web mock environment +- name: Upgrade pip and virtualenv in Yoda web mock environment become_user: '{{ yoda_deployment_user }}' become: true ansible.builtin.pip: name: - - pip==23.0.1 + - pip==24.3.1 + - virtualenv==20.28.0 executable: /var/www/webmock/venv/bin/pip3 when: not ansible_check_mode +- name: Ensure Yoda web mock virtualenv exists + become_user: '{{ yoda_deployment_user }}' + become: true + ansible.builtin.command: "{{ yoda_web_mock_python3_path }} -m virtualenv /var/www/webmock/venv" + args: + creates: "/var/www/webmock/venv/bin/activate_this.py" + + # Change detection in git repositories is unreliable in some Ansible versions, so we always # run this task, just to be sure. - name: Ensure Yoda web mock dependencies are installed diff --git a/roles/yoda_web_mock/vars/Debian.yml b/roles/yoda_web_mock/vars/Debian.yml index 00a2c49ba..e2a6625e3 100644 --- a/roles/yoda_web_mock/vars/Debian.yml +++ b/roles/yoda_web_mock/vars/Debian.yml @@ -1,4 +1,4 @@ --- # copyright Utrecht University -yoda_web_mock_python3_path: /usr/bin/python3 +yoda_web_mock_python3_path: /var/www/webmock/venv/bin/python3 diff --git a/roles/yoda_web_mock_datacite/meta/main.yml b/roles/yoda_web_mock_datacite/meta/main.yml index 28b3d5d71..623a25576 100644 --- a/roles/yoda_web_mock_datacite/meta/main.yml +++ b/roles/yoda_web_mock_datacite/meta/main.yml @@ -8,9 +8,9 @@ galaxy_info: min_ansible_version: '2.11' platforms: - name: EL - version: 7 + version: 8 - name: Ubuntu - version: focal + version: noble dependencies: diff --git a/roles/yoda_web_mock_sram/meta/main.yml b/roles/yoda_web_mock_sram/meta/main.yml index 913c33936..e6627d3a8 100644 --- a/roles/yoda_web_mock_sram/meta/main.yml +++ b/roles/yoda_web_mock_sram/meta/main.yml @@ -8,9 +8,9 @@ galaxy_info: min_ansible_version: '2.11' platforms: - name: EL - version: 7 + version: 8 - name: Ubuntu - version: focal + version: noble dependencies: From 4732fcd58f556d7f6eab6bc9c938b999cda31077 Mon Sep 17 00:00:00 2001 From: Lazlo Westerhof Date: Tue, 10 Dec 2024 13:25:00 +0100 Subject: [PATCH 08/16] Update Docker Setup for Ubuntu 24.04 and iRODS 4.3.3 --- docker/down.sh | 2 +- docker/images/davrods/Dockerfile | 33 +----- docker/images/yoda_eus/Dockerfile | 23 ++-- docker/images/yoda_irods_icat/Dockerfile | 105 +++++------------- .../images/yoda_irods_icat/irods-icat-init.sh | 17 +-- docker/images/yoda_portal/Dockerfile | 8 +- docker/images/yoda_public/Dockerfile | 16 +-- docker/images/yoda_web_mock/Dockerfile | 14 ++- docker/run-cronjob.sh | 2 +- docker/up.sh | 2 +- .../defaults/main.yml | 2 +- roles/yoda_moai/defaults/main.yml | 2 +- roles/yoda_portal/defaults/main.yml | 2 +- roles/yoda_rulesets/defaults/main.yml | 2 +- 14 files changed, 83 insertions(+), 147 deletions(-) diff --git a/docker/down.sh b/docker/down.sh index 043cfe063..8c1b525b2 100755 --- a/docker/down.sh +++ b/docker/down.sh @@ -1,2 +1,2 @@ #!/bin/bash -docker-compose down "$@" +docker compose down "$@" diff --git a/docker/images/davrods/Dockerfile b/docker/images/davrods/Dockerfile index 4e5fc9023..4d70332f1 100644 --- a/docker/images/davrods/Dockerfile +++ b/docker/images/davrods/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 +FROM ubuntu:24.04 LABEL maintainer="Yoda team " # Network settings @@ -7,11 +7,11 @@ EXPOSE 443 # Application settings ARG TAG=development -ENV IRODS_PACKAGE_VERSION="4.2.12-1~bionic" -ENV APT_REPO_CONFIG="deb [arch=amd64] https://packages.irods.org/apt/ bionic main" +ENV IRODS_PACKAGE_VERSION="4.3.3-0~noble" +ENV APT_REPO_CONFIG="deb [arch=amd64] https://packages.irods.org/apt/ noble main" ENV APT_IRODS_REPO_SIGNING_KEY_LOC="https://packages.irods.org/irods-signing-key.asc" -ENV DAVRODS_APT_URL="https://github.com/UtrechtUniversity/davrods/releases/download/4.2.12_1.5.1" -ENV DAVRODS_APT_PACKAGE="davrods-4.2.12-1.5.1.deb" +ENV DAVRODS_APT_URL="https://github.com/UtrechtUniversity/davrods/releases/download/4.3.3_1.5.1" +ENV DAVRODS_APT_PACKAGE="davrods-4.3.3-1.5.1.deb" ENV DEBIAN_FRONTEND="noninteractive" # Update packages @@ -24,7 +24,7 @@ RUN apt-get install -y wget vim # Install Python 3 # hadolint ignore=DL3033 RUN apt-get install -y gcc python3 python3-dev python3-pip python3-virtualenv \ - python3-wheel python3.8-venv + python3-wheel python3.12-venv # Install and configure Apache # hadolint ignore=DL3033 @@ -38,27 +38,6 @@ RUN ln -s /etc/apache2/mods-available/dav_fs.load /etc/apache2/mods-enabled/dav_ RUN ln -s /etc/apache2/mods-available/dav_lock.load /etc/apache2/mods-enabled/dav_lock.load RUN ln -s /etc/apache2/mods-available/davrods.load /etc/apache2/mods-enabled/davrods.load -# Install iRODS dependencies -RUN apt-get -y install aptitude libboost-locale-dev libpython2-stdlib libpython2.7-minimal \ - libpython2.7-stdlib \ - python-is-python2 python-six python2 python2-minimal python2.7 python2.7-minimal \ - python-certifi python-chardet python-idna python-pkg-resources python-setuptools -ENV PY_URLLIB_PREFIX="http://security.ubuntu.com/ubuntu/pool/main/p/python-urllib3" -ENV PY_URLLIB_FILENAME="python-urllib3_1.22-1ubuntu0.18.04.2_all.deb" -ENV PY_REQUESTS_PREFIX="http://security.ubuntu.com/ubuntu/pool/main/r/requests" -ENV PY_REQUESTS_FILENAME="python-requests_2.18.4-2ubuntu0.1_all.deb" -ENV OPENSSL_PREFIX="http://security.ubuntu.com/ubuntu/pool/main/o/openssl1.0" -ENV OPENSSL_FILENAME="libssl1.0.0_1.0.2n-1ubuntu5.13_amd64.deb" -RUN wget -q ${PY_URLLIB_PREFIX}/${PY_URLLIB_FILENAME} && \ - wget -q ${PY_REQUESTS_PREFIX}/${PY_REQUESTS_FILENAME} && \ - wget -q ${OPENSSL_PREFIX}/${OPENSSL_FILENAME} && \ - dpkg -i ${PY_URLLIB_FILENAME} && \ - dpkg -i ${PY_REQUESTS_FILENAME} && \ - dpkg -i ${OPENSSL_FILENAME} && \ - rm ${PY_URLLIB_FILENAME} && \ - rm ${PY_REQUESTS_FILENAME} && \ - rm ${OPENSSL_FILENAME} - # Install iRODS components: iCommands and runtime SHELL ["/bin/bash", "-o", "pipefail", "-c"] # hadolint ignore=DL3033 diff --git a/docker/images/yoda_eus/Dockerfile b/docker/images/yoda_eus/Dockerfile index 62d975a34..526a2de1d 100644 --- a/docker/images/yoda_eus/Dockerfile +++ b/docker/images/yoda_eus/Dockerfile @@ -1,12 +1,13 @@ -FROM ubuntu:20.04 +FROM ubuntu:24.04 +# FROM --platform=linux/amd64 ubuntu:24.04 LABEL maintainer="Yoda team " # Application settings ARG TAG=development -ENV YODA_EUS_REPO "https://github.com/UtrechtUniversity/yoda-external-user-service.git" -ENV YODA_EUS_BRANCH "$TAG" -ENV YODA_PORTAL_REPO "https://github.com/UtrechtUniversity/yoda-portal.git" -ENV YODA_PORTAL_BRANCH "$TAG" +ENV YODA_EUS_REPO="https://github.com/UtrechtUniversity/yoda-external-user-service.git" +ENV YODA_EUS_BRANCH="$TAG" +ENV YODA_PORTAL_REPO="https://github.com/UtrechtUniversity/yoda-portal.git" +ENV YODA_PORTAL_BRANCH="$TAG" ENV DEBIAN_FRONTEND="noninteractive" SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -19,19 +20,19 @@ RUN apt-get update # Install common tools (PostgreSQL is needed for psql commands in init script) # hadolint ignore=DL3033 -RUN apt-get install -y wget git netcat gnupg vim +RUN apt-get install -y wget git netcat-traditional gnupg vim # Install PostgreSQL 15 for checks whether database container is up, as well as troubleshooting # hadolint ignore=DL3033 RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - -RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ focal-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list +RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ noble-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list RUN apt-get update -RUN apt-get -y install postgresql-15 python3-psycopg2 acl +RUN apt-get -y install postgresql-15 libpq-dev python3-psycopg2 acl # Install Python 3 # hadolint ignore=DL3033 RUN apt-get install -y gcc python3 python3-dev python3-pip python3-virtualenv \ - python3-wheel python3.8-venv virtualenv + python3-wheel python3.12-venv virtualenv # Install and configure Apache # hadolint ignore=DL3033 @@ -57,7 +58,9 @@ RUN useradd yodadeployment && \ COPY yoda-external-user-service-vhost.conf /etc/apache2/sites-available/003-yoda-eus-vhost.conf RUN ln -s /etc/apache2/sites-available/003-yoda-eus-vhost.conf /etc/apache2/sites-enabled/003-yoda-eus-vhost.conf RUN virtualenv --python /usr/bin/python3 /var/www/extuser/yoda-external-user-service/venv && \ - /var/www/extuser/yoda-external-user-service/venv/bin/pip3 install --no-cache-dir pip==23.0.1 && \ + /var/www/extuser/yoda-external-user-service/venv/bin/pip3 install --no-cache-dir pip==24.3.1 && \ + /var/www/extuser/yoda-external-user-service/venv/bin/pip3 install --no-cache-dir virtualenv==20.28.0 && \ + /var/www/extuser/yoda-external-user-service/venv/bin/python3 -m virtualenv /var/www/extuser/yoda-external-user-service/venv && \ /var/www/extuser/yoda-external-user-service/venv/bin/pip3 install --no-cache-dir /var/www/extuser/yoda-external-user-service # Make copy of data on volumes that can optionally be bind mounted, so the container init script diff --git a/docker/images/yoda_irods_icat/Dockerfile b/docker/images/yoda_irods_icat/Dockerfile index 83ec751f6..35e65d23c 100644 --- a/docker/images/yoda_irods_icat/Dockerfile +++ b/docker/images/yoda_irods_icat/Dockerfile @@ -1,20 +1,19 @@ -FROM ubuntu:20.04 +FROM ubuntu:24.04 LABEL maintainer="Yoda team " SHELL ["/bin/bash", "-o", "pipefail", "-c"] # Application settings ARG TAG=development -ENV IRODS_VERSION "4.2.12-1~bionic" -ENV IRODS_PREP_VERSION "4.2.12.0-1~bionic" -ENV IRODS_INDEX_VERSION "4.2.12.0-1~bionic" -ENV IRODS_MSVC_IRODS_VERSION "4.2.12" +ENV IRODS_VERSION "4.3.3-0~noble" +ENV IRODS_INDEX_PREP_VERSION "4.3.3.0-0+4.3.3~noble" +ENV IRODS_MSVC_IRODS_VERSION "4.3.3" ENV IRODS_SUDO_MSVC_VERSION "1.0.0" ENV IRODS_UU_MSVC_VERSION "1.2.0" ENV IRODS_UU_MSVC_VERSION_RELEASE "0" -ENV APT_REPO_CONFIG="deb [arch=amd64] https://packages.irods.org/apt/ bionic main" +ENV APT_REPO_CONFIG="deb [arch=amd64] https://packages.irods.org/apt/ noble main" ENV APT_IRODS_REPO_SIGNING_KEY_LOC="https://packages.irods.org/irods-signing-key.asc" ENV YODA_RULESET_REPO "https://github.com/UtrechtUniversity/yoda-ruleset.git" -ENV YODA_RULESET_BRANCH "$TAG" +ENV YODA_RULESET_BRANCH="$TAG" ENV PRE_BUILD_RULESET_DEPENDENCIES "yes" ENV DEBIAN_FRONTEND="noninteractive" @@ -39,17 +38,12 @@ RUN apt-get update # Install common tools # hadolint ignore=DL3033 -RUN apt-get install -y wget git sudo netcat gcc vim +RUN apt-get install -y wget git sudo netcat-traditional gcc vim -# Install Python 2.7 -RUN apt-get install -y python2 python2-dev && \ - wget -q -O /usr/local/bin/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \ - /usr/bin/python2 /usr/local/bin/get-pip.py - -# Install Python 3.8 +# Install Python 3.12 # hadolint ignore=DL3033 RUN apt-get install -y gcc python3 python3-dev python3-pip python3-virtualenv \ - python3-wheel python3.8-venv + python3-wheel python3.12-venv # Upload PAM Python library COPY stage/pam_python.so /tmp/pam_python.so @@ -58,44 +52,12 @@ RUN install -m 0755 -o root -g root /tmp/pam_python.so /usr/lib/x86_64-linux-gn # Install PostgreSQL 15 for ODBC drivers, checks whether database container is up, as well as troubleshooting # hadolint ignore=DL3033 RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - -RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ focal-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list +RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ noble-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list RUN apt-get update -RUN apt-get -y install postgresql-15 unixodbc python3-psycopg2 acl +RUN apt-get -y install postgresql-15 unixodbc unixodbc-dev odbcinst python3-psycopg2 acl COPY odbc-settings.txt /tmp/odbc-settings.txt RUN odbcinst -i -d -r < /tmp/odbc-settings.txt -# Install iRODS dependencies -RUN apt-get -y install aptitude libboost-locale-dev libpython2-stdlib libpython2.7-minimal \ - libpython2.7-stdlib \ - python-is-python2 python-six python2 python2-minimal python2.7 python2.7-minimal \ - python-certifi python-chardet python-idna python-pkg-resources python-setuptools \ - python-funcsigs python-mock python-pbr libodbc1 -ENV PY_URLLIB_PREFIX="http://security.ubuntu.com/ubuntu/pool/main/p/python-urllib3" -ENV PY_URLLIB_FILENAME="python-urllib3_1.22-1ubuntu0.18.04.2_all.deb" -ENV PY_REQUESTS_PREFIX="http://security.ubuntu.com/ubuntu/pool/main/r/requests" -ENV PY_REQUESTS_FILENAME="python-requests_2.18.4-2ubuntu0.1_all.deb" -ENV OPENSSL_PREFIX="http://security.ubuntu.com/ubuntu/pool/main/o/openssl1.0" -ENV OPENSSL_FILENAME="libssl1.0.0_1.0.2n-1ubuntu5.13_amd64.deb" -ENV PY_JSONSCHEMA_PREFIX="http://security.ubuntu.com/ubuntu/pool/main/p/python-jsonschema" -ENV PY_JSONSCHEMA_FILENAME="python-jsonschema_2.3.0-1build1_all.deb" -ENV PY_ODBC_PREFIX="http://security.ubuntu.com/ubuntu/pool/universe/p/pyodbc" -ENV PY_ODBC_FILENAME="python-pyodbc_4.0.17-1_amd64.deb" -RUN wget -q ${PY_URLLIB_PREFIX}/${PY_URLLIB_FILENAME} && \ - wget -q ${PY_REQUESTS_PREFIX}/${PY_REQUESTS_FILENAME} && \ - wget -q ${OPENSSL_PREFIX}/${OPENSSL_FILENAME} && \ - wget -q ${PY_JSONSCHEMA_PREFIX}/${PY_JSONSCHEMA_FILENAME} && \ - wget -q ${PY_ODBC_PREFIX}/${PY_ODBC_FILENAME} && \ - dpkg -i ${PY_URLLIB_FILENAME} && \ - dpkg -i ${PY_REQUESTS_FILENAME} && \ - dpkg -i ${OPENSSL_FILENAME} && \ - dpkg -i ${PY_JSONSCHEMA_FILENAME} && \ - dpkg -i ${PY_ODBC_FILENAME} && \ - rm ${PY_URLLIB_FILENAME} && \ - rm ${PY_REQUESTS_FILENAME} && \ - rm ${OPENSSL_FILENAME} && \ - rm ${PY_JSONSCHEMA_FILENAME} && \ - rm ${PY_ODBC_FILENAME} - # Install iRODS # hadolint ignore=DL3033,DL3047 RUN useradd -d /var/lib/irods irods && \ @@ -106,13 +68,11 @@ RUN useradd -d /var/lib/irods irods && \ do echo "Installing package ${package} and its dependencies ..."; \ apt-get -y install "$package=${IRODS_VERSION}" ; \ done && \ - for package in irods-rule-engine-plugin-indexing irods-rule-engine-plugin-elasticsearch irods-rule-engine-plugin-document-type; \ + for package in irods-rule-engine-plugin-indexing irods-rule-engine-plugin-elasticsearch; \ do echo "Installing package ${package} and its dependencies ..."; \ - apt-get -y install "$package=${IRODS_INDEX_VERSION}"; \ - done && \ - apt-get -y install "irods-rule-engine-plugin-python=${IRODS_PREP_VERSION}" && \ - wget https://yoda.uu.nl/irods-patches/libirods_server.so.4.2.12.ubuntu2004 && \ - install -m 0644 libirods_server.so.4.2.12.ubuntu2004 /usr/lib/libirods_server.so.4.2.12 + apt-get -y install "$package=${IRODS_INDEX_PREP_VERSION}"; \ + done + # Install iRODS microservices for Yoda # hadolint ignore=DL3033 @@ -135,20 +95,10 @@ RUN install -m 0644 -o root -g root /tmp/irods_completion.sh /etc/profile.d/irod # Set up PAM authentication # hadolint ignore=DL3033 -RUN python -m easy_install --user https://github.com/UtrechtUniversity/pysqlcipher3/releases/download/v1.2.1/pysqlcipher3-1.2.1-py2.7-linux-x86_64.egg COPY is-user-external.sh /tmp/is-user-external.sh COPY external-auth.py /tmp/external-auth.py RUN install -m 0750 -o root -g irods /tmp/is-user-external.sh /usr/local/bin/is-user-external.sh && \ install -m 0750 -o root -g irods /tmp/external-auth.py /usr/local/bin/external-auth.py && \ - wget -q https://yoda.uu.nl/packages-irods-4.2.12-focal/sqlcipher4.deb && \ - dpkg -i sqlcipher4.deb && \ - rm sqlcipher4.deb && \ - wget -q https://yoda.uu.nl/packages-irods-4.2.12-focal/libsqlcipher4.deb && \ - dpkg -i libsqlcipher4.deb && \ - rm libsqlcipher4.deb && \ - wget -q https://yoda.uu.nl/packages-irods-4.2.12-focal/libsqlcipher4-dev.deb && \ - dpkg -i libsqlcipher4-dev.deb && \ - rm libsqlcipher4-dev.deb && \ chmod 04755 /usr/sbin/irodsPamAuthCheck COPY irods.pam /tmp/irods.pam RUN install -m 0644 /tmp/irods.pam /etc/pam.d/irods @@ -161,19 +111,18 @@ COPY irods_environment.json /var/lib/irods/.irods/irods_environment.json # Install and configure rulesets RUN git clone -b ${YODA_RULESET_BRANCH} ${YODA_RULESET_REPO} /etc/irods/yoda-ruleset # hadolint ignore=DL3004 -RUN sudo -u irods python -m pip --no-cache-dir install --user pip==20.2.4 && \ - sudo -u irods python -m pip --no-cache-dir install --user setuptools==44.1.1 && \ - sudo -u irods python -m pip --no-cache-dir install --user python-irodsclient==2.2.0 && \ - sudo -u irods python -m pip --no-cache-dir install --user -r /etc/irods/yoda-ruleset/requirements.txt && \ - sudo -u irods python -m easy_install --user https://github.com/UtrechtUniversity/pysqlcipher3/releases/download/v1.2.1/pysqlcipher3-1.2.1-py2.7-linux-x86_64.egg && \ - sudo -u irods python3 -m pip install --user jsonschema==4.17.3 && \ - sudo -u irods python3 -m pip install --user python-irodsclient==2.2.0 && \ - sudo -u irods python3 -m pip install --user psutil==5.9.5 +RUN sudo -u irods python3 -m pip --no-cache-dir install --user --break-system-packages pip==24.3.1 && \ + sudo -u irods python3 -m pip --no-cache-dir install --user --break-system-packages -r /etc/irods/yoda-ruleset/requirements.txt && \ + sudo -u irods python3 -m pip install --user --break-system-packages python-irodsclient==2.2.0 && \ + sudo -u irods python3 -m pip install --user --break-system-packages psutil==6.1.0 COPY core.py.template /etc/irods/core.py COPY core.re.template /etc/irods/core.re -RUN for script in scheduled-copytovault.sh admin-remove-orphan-vault-if-empty.sh admin-vaultactions.sh \ - admin-vaultingest.sh enable-indexing.sh disable-indexing.sh \ - admin-datarequestactions.sh admin-datarequest-temp-write-permission.sh ; \ +RUN apt-get -y install sqlcipher libsqlcipher1 libsqlcipher-dev +RUN for script in admin-scheduled-copytovault.sh admin-remove-orphan-vault-if-empty.sh admin-vaultactions.sh \ + admin-vaultingest.sh admin-datarequestactions.sh admin-datarequest-temp-write-permission.sh ; \ + do install -m 0755 /etc/irods/yoda-ruleset/tools/admin/${script} /var/lib/irods/msiExecCmd_bin/${script} ; \ + done +RUN for script in enable-indexing.sh disable-indexing.sh; \ do install -m 0755 /etc/irods/yoda-ruleset/tools/${script} /var/lib/irods/msiExecCmd_bin/${script} ; \ done && \ /etc/irods/yoda-ruleset/tools/setup_tokens.sh /etc/irods/yoda-ruleset/accesstokens.db test @@ -195,9 +144,9 @@ RUN chown -R irods:irods /var/lib/irods /etc/irods # Pre-build ruleset dependencies RUN test "$PRE_BUILD_RULESET_DEPENDENCIES" = "yes" && git clone --branch "$YODA_RULESET_BRANCH" "$YODA_RULESET_REPO" /var/lib/irods/yoda-ruleset # hadolint ignore=DL3004 -RUN test "$PRE_BUILD_RULESET_DEPENDENCIES" = "yes" && sudo -u irods pip2 install --user attrs==21.4.0 +RUN test "$PRE_BUILD_RULESET_DEPENDENCIES" = "yes" && sudo -u irods pip3 install --user --break-system-packages attrs==24.2.0 # hadolint ignore=DL3004 -RUN test "$PRE_BUILD_RULESET_DEPENDENCIES" = "yes" && sudo -u irods pip2 install --user -r /var/lib/irods/yoda-ruleset/requirements.txt +RUN test "$PRE_BUILD_RULESET_DEPENDENCIES" = "yes" && sudo -u irods pip3 install --user --break-system-packages -r /var/lib/irods/yoda-ruleset/requirements.txt # Create test accounts # hadolint ignore=SC2016 diff --git a/docker/images/yoda_irods_icat/irods-icat-init.sh b/docker/images/yoda_irods_icat/irods-icat-init.sh index 47f6a6fdd..7f39063ad 100644 --- a/docker/images/yoda_irods_icat/irods-icat-init.sh +++ b/docker/images/yoda_irods_icat/irods-icat-init.sh @@ -42,7 +42,10 @@ fi # Download test vault and iCAT data before_update "Downloading data" -mkdir /download +if [ ! -d /download ] +then + mkdir /download +fi wget -q "https://yoda.uu.nl/yoda-docker/${DATA_VERSION}.vault.tar.gz" -O "/download/${DATA_VERSION}.vault.tar.gz" progress_update "Downloaded vault test data." wget -q "https://yoda.uu.nl/yoda-docker/${DATA_VERSION}.icat.sql.gz" -O "/download/${DATA_VERSION}.icat.sql.gz" @@ -83,11 +86,11 @@ progress_update "iCAT database data loaded" INSTALL_TIMESTAMP=$(date +'%Y-%m-%dT%H:%M:%S.000000') cat > /var/lib/irods/VERSION.json << VERSION { - "catalog_schema_version": 8, + "catalog_schema_version": 8, "commit_id": "2ed549ca7fe455aaa7755becc6c14b233dcbc0b4", - "configuration_schema_version": 3, - "installation_time": "$INSTALL_TIMESTAMP", - "irods_version": "4.2.12" + "configuration_schema_version": 3, + "installation_time": "$INSTALL_TIMESTAMP", + "irods_version": "4.3.3" } VERSION chown irods:irods /var/lib/irods/VERSION.json @@ -131,8 +134,8 @@ make install progress_update "Ruleset updated" before_update "Updating ruleset dependencies" -sudo -u irods pip2 install --user attrs==21.4.0 -sudo -u irods pip2 install --user -r /etc/irods/yoda-ruleset/requirements.txt +sudo -u irods pip3 install --user --break-system-packages attrs==24.2.0 +sudo -u irods pip3 install --user --break-system-packages -r /etc/irods/yoda-ruleset/requirements.txt progress_update "Ruleset dependencies updated" touch /container_initialized diff --git a/docker/images/yoda_portal/Dockerfile b/docker/images/yoda_portal/Dockerfile index 2b57d00b4..9dc8a6f2e 100644 --- a/docker/images/yoda_portal/Dockerfile +++ b/docker/images/yoda_portal/Dockerfile @@ -1,9 +1,9 @@ -FROM ubuntu:20.04 +FROM ubuntu:24.04 LABEL maintainer="Yoda team " # Application settings ARG TAG=development -ENV YODA_PORTAL_REPO "https://github.com/UtrechtUniversity/yoda-portal.git" +ENV YODA_PORTAL_REPO="https://github.com/UtrechtUniversity/yoda-portal.git" ENV YODA_PORTAL_BRANCH="$TAG" ENV DEBIAN_FRONTEND="noninteractive" @@ -21,7 +21,7 @@ RUN apt-get install -y wget git vim # Install Python 3 # hadolint ignore=DL3033 RUN apt-get install -y gcc python3 python3-dev python3-pip python3-virtualenv \ - python3-wheel python3.8-venv virtualenv + python3-wheel python3.12-venv virtualenv # Install and configure Apache # hadolint ignore=DL3033 @@ -52,7 +52,7 @@ RUN mkdir -p /var/www/yoda/config && \ COPY yoda-portal-vhost.conf /etc/apache2/sites-available/001-yoda-portal-vhost.conf RUN ln -s /etc/apache2/sites-available/001-yoda-portal-vhost.conf /etc/apache2/sites-enabled/001-yoda-portal-vhost.conf RUN virtualenv --python /usr/bin/python3 /var/www/yoda/venv && \ - /var/www/yoda/venv/bin/pip3 install --no-cache-dir pip==23.0.1 && \ + /var/www/yoda/venv/bin/pip3 install --no-cache-dir pip==24.3.1 && \ /var/www/yoda/venv/bin/pip3 install --no-cache-dir -r /var/www/yoda/requirements.txt # Make copy of data on volumes that can optionally be bind mounted, so the container init script diff --git a/docker/images/yoda_public/Dockerfile b/docker/images/yoda_public/Dockerfile index 0537e42fb..730c82115 100644 --- a/docker/images/yoda_public/Dockerfile +++ b/docker/images/yoda_public/Dockerfile @@ -1,10 +1,10 @@ -FROM ubuntu:20.04 +FROM ubuntu:24.04 LABEL maintainer="Yoda team " # Application settings ARG TAG=development -ENV MOAI_REPO "https://github.com/UtrechtUniversity/yoda-moai.git" -ENV MOAI_BRANCH "$TAG" +ENV MOAI_REPO="https://github.com/UtrechtUniversity/yoda-moai.git" +ENV MOAI_BRANCH="$TAG" ENV DEBIAN_FRONTEND="noninteractive" # Network settings @@ -16,12 +16,12 @@ RUN apt-get update # Install common tools # hadolint ignore=DL3033 -RUN apt-get install -y wget git netcat sudo vim +RUN apt-get install -y wget git netcat-traditional sudo vim # Install Python 3 # hadolint ignore=DL3033 RUN apt-get install -y gcc python3 python3-dev python3-pip python3-virtualenv \ - python3-wheel python3.8-venv virtualenv + python3-wheel python3.12-venv virtualenv # Install and configure Apache # hadolint ignore=DL3033 @@ -49,15 +49,15 @@ RUN mkdir -p /var/www/moai /var/www/moai/metadata && \ chmod 0755 /var/www/moai /var/www/moai/metadata && \ chown -R yodadeployment:yodadeployment /var/www/moai && \ virtualenv --python /usr/bin/python3 /var/www/moai/yoda-moai/venv && \ - /var/www/moai/yoda-moai/venv/bin/pip3 install pip==23.0.1 + /var/www/moai/yoda-moai/venv/bin/pip3 install pip==24.3.1 ## Install PySQLite3 for MOAI -ENV C_INCLUDE_PATH /usr/include/python3.8:/usr/include +ENV C_INCLUDE_PATH /usr/include/python3.12:/usr/include ENV LIBRARY_PATH /usr/lib/x86_64-linux-gnu RUN /var/www/moai/yoda-moai/venv/bin/pip3 install pysqlite3==0.5.0 ## Install MOAI itself -ENV C_INCLUDE_PATH /usr/include/python3.8 +ENV C_INCLUDE_PATH /usr/include/python3.12 RUN /var/www/moai/yoda-moai/venv/bin/pip3 install -e /var/www/moai/yoda-moai ## Configure and initialize MOAI diff --git a/docker/images/yoda_web_mock/Dockerfile b/docker/images/yoda_web_mock/Dockerfile index 0ffa9114a..d8fa5784c 100644 --- a/docker/images/yoda_web_mock/Dockerfile +++ b/docker/images/yoda_web_mock/Dockerfile @@ -1,10 +1,10 @@ -FROM ubuntu:20.04 +FROM ubuntu:24.04 LABEL maintainer="Yoda team " # Application settings ARG TAG=development -ENV YODA_MOCK_REPO "https://github.com/UtrechtUniversity/yoda-web-mock.git" -ENV YODA_MOCK_BRANCH "$TAG" +ENV YODA_MOCK_REPO="https://github.com/UtrechtUniversity/yoda-web-mock.git" +ENV YODA_MOCK_BRANCH="$TAG" ENV DEBIAN_FRONTEND="noninteractive" # Network settings @@ -16,12 +16,12 @@ RUN apt-get update # Install common tools (PostgreSQL is needed for psql commands in init script) # hadolint ignore=DL3033 -RUN apt-get install -y wget git netcat gnupg vim +RUN apt-get install -y wget git netcat-traditional gnupg vim # Install Python 3 # hadolint ignore=DL3033 RUN apt-get install -y gcc python3 python3-dev python3-pip python3-virtualenv \ - python3-wheel python3.8-venv virtualenv + python3-wheel python3.12-venv virtualenv # Install and configure Apache # hadolint ignore=DL3033 @@ -45,7 +45,9 @@ RUN useradd yodadeployment && \ COPY yoda-web-mock-vhost.conf /etc/apache2/sites-available/yoda-web-mock-vhost.conf RUN ln -s /etc/apache2/sites-available/yoda-web-mock-vhost.conf /etc/apache2/sites-enabled/yoda-web-mock-vhost.conf RUN virtualenv --python /usr/bin/python3 /var/www/webmock/venv && \ - /var/www/webmock/venv/bin/pip3 install pip==23.0.1 && \ + /var/www/webmock/venv/bin/pip3 install pip==24.3.1 && \ + /var/www/webmock/venv/bin/pip3 install virtualenv==20.28.0 && \ + /var/www/webmock/venv/bin/python3 -m virtualenv /var/www/webmock/venv && \ /var/www/webmock/venv/bin/pip3 install -e /var/www/webmock/yoda-web-mock # Make copy of data on volumes that can optionally be bind mounted, so the container init script diff --git a/docker/run-cronjob.sh b/docker/run-cronjob.sh index 9f47817cf..184b2c685 100755 --- a/docker/run-cronjob.sh +++ b/docker/run-cronjob.sh @@ -30,7 +30,7 @@ case "$1" in ;; revision) - docker exec "$EXEC_OPTIONS" provider.yoda sudo -iu irods /bin/python /etc/irods/yoda-ruleset/tools/async-data-revision.py -v + docker exec "$EXEC_OPTIONS" provider.yoda sudo -iu irods /bin/python3 /etc/irods/yoda-ruleset/tools/async-data-revision.py -v ;; revisioncleanup) diff --git a/docker/up.sh b/docker/up.sh index 7d083edad..61d472be2 100755 --- a/docker/up.sh +++ b/docker/up.sh @@ -1,2 +1,2 @@ #!/bin/bash -docker-compose up "$@" +docker compose up "$@" diff --git a/roles/yoda_external_user_service/defaults/main.yml b/roles/yoda_external_user_service/defaults/main.yml index 08aa83559..95a333a3b 100644 --- a/roles/yoda_external_user_service/defaults/main.yml +++ b/roles/yoda_external_user_service/defaults/main.yml @@ -3,7 +3,7 @@ yoda_deployment_user: yodadeployment yoda_version: development -yoda_eus_version: "development-irods-4.3" +yoda_eus_version: "{{ yoda_version }}" yoda_eus_fqdn: eus.yoda.test yoda_eus_port: 443 diff --git a/roles/yoda_moai/defaults/main.yml b/roles/yoda_moai/defaults/main.yml index 049fec2ed..f138ec6b9 100644 --- a/roles/yoda_moai/defaults/main.yml +++ b/roles/yoda_moai/defaults/main.yml @@ -1,7 +1,7 @@ --- # copyright Utrecht University -yoda_moai_version: "development-irods-4.3" +yoda_moai_version: "{{ yoda_version }}" yoda_moai_user: moai yoda_moai_home: /var/www/moai yoda_moai_database_connection_string: "sqlite:///{{ yoda_moai_home }}/moai.db" diff --git a/roles/yoda_portal/defaults/main.yml b/roles/yoda_portal/defaults/main.yml index 1806914b1..252f27e2a 100644 --- a/roles/yoda_portal/defaults/main.yml +++ b/roles/yoda_portal/defaults/main.yml @@ -28,7 +28,7 @@ yoda_theme_mapping: wur: "Wageningen University & Research" # Yoda portal -yoda_portal_version: "development-irods-4.3" +yoda_portal_version: "{{ yoda_version }}" yoda_portal_log_api_call_duration: false yoda_portal_path: /var/www/yoda # Path to location of portal yoda_config_path: '/var/www/yoda/config' # Path to portal's shared configuration diff --git a/roles/yoda_rulesets/defaults/main.yml b/roles/yoda_rulesets/defaults/main.yml index e8b40ae91..0f3f0c7bc 100644 --- a/roles/yoda_rulesets/defaults/main.yml +++ b/roles/yoda_rulesets/defaults/main.yml @@ -10,7 +10,7 @@ core_rulesets: - name: yoda-ruleset repo: https://github.com/UtrechtUniversity/yoda-ruleset.git ruleset_name: rules-uu - version: "irods-4.3" + version: "{{ yoda_ruleset_version }}" install_scripts: true - name: core ruleset_name: core From 3d253c7c1f6672ffd812d2e3ab6fcfb860a2d90a Mon Sep 17 00:00:00 2001 From: Sietse Snel Date: Thu, 12 Dec 2024 15:17:14 +0100 Subject: [PATCH 09/16] Update revision docs: remove J strategy The J revision strategy no longer exists in the ruleset. Remove it from the docs --- docs/administration/configuring-yoda.md | 2 +- roles/yoda_rulesets/defaults/main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/administration/configuring-yoda.md b/docs/administration/configuring-yoda.md index 39c33ae74..5703a9e30 100644 --- a/docs/administration/configuring-yoda.md +++ b/docs/administration/configuring-yoda.md @@ -260,7 +260,7 @@ async_replication_max_rss | Limit the memory usage (in bytes) of a replicat Variable | Description -------------------------------|--------------------------------------------- enable_revisions | Enable asynchronous revisions: yes (1) or no (0) -revision_strategy | Revision strategy: A, B, J or Simple (default: B) +revision_strategy | Revision strategy: A, B or Simple (default: B) async_revision_jobs | Number of asynchronous revision jobs, when decreasing the number of jobs, manually remove jobs from the crontab (default: 1) async_revision_batch_size | Asynchronous revision jobs batch size (default: 1000) async_revision_verbose_mode | Run asynchronous revision job in verbose mode (default: true) diff --git a/roles/yoda_rulesets/defaults/main.yml b/roles/yoda_rulesets/defaults/main.yml index 0f3f0c7bc..b89277792 100644 --- a/roles/yoda_rulesets/defaults/main.yml +++ b/roles/yoda_rulesets/defaults/main.yml @@ -30,7 +30,7 @@ async_replication_max_rss: 1000000000 # Limit the memory usage of a replication # Revision configuration enable_revisions: 1 # Enable asynchronous revisions: yes (1) or no (0) -revision_strategy: B # Revision strategy: A, B, J or Simple +revision_strategy: B # Revision strategy: A, B or Simple async_revision_jobs: 1 # Number of asynchronous revision jobs, when decreasing the number of jobs, manually remove jobs from the crontab async_revision_batch_size: 1000 # Asynchronous revision jobs batch size async_revision_verbose_mode: true # Run asynchronous revision job in verbose mode From fbe26de0063b4a62fa488940e302a0c362731ff7 Mon Sep 17 00:00:00 2001 From: Lazlo Westerhof Date: Thu, 12 Dec 2024 16:00:10 +0100 Subject: [PATCH 10/16] YDA-6061: update portal config for Yoda portal Docker --- docker/images/yoda_portal/yoda_portal_init.sh | 60 +++++++++++-------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/docker/images/yoda_portal/yoda_portal_init.sh b/docker/images/yoda_portal/yoda_portal_init.sh index aca333ac6..9bc128990 100644 --- a/docker/images/yoda_portal/yoda_portal_init.sh +++ b/docker/images/yoda_portal/yoda_portal_init.sh @@ -88,6 +88,7 @@ from flask import current_app as app SECRET_KEY = '$SECRET_KEY' PORTAL_TITLE_TEXT = 'Yoda - Dev (Docker)' YODA_VERSION = 'development' +YODA_ENVIRONMENT = 'development' YODA_COMMIT = '$YODA_COMMIT' RESEARCH_ENABLED = True OPEN_SEARCH_ENABLED = False @@ -95,7 +96,10 @@ DEPOSIT_ENABLED = True DATAREQUEST_ENABLED = True TOKENS_ENABLED = True TOKEN_LIFETIME = 72 -JSON_SORT_KEYS = False # Check if this is still needed with Python v3.7? +SRAM_ENABLED = False + +# Logging configuration +LOG_API_CALL_DURATION = True # Flask-Session configuration SESSION_TYPE = 'filesystem' @@ -113,7 +117,7 @@ IRODS_ICAT_PORT = '1247' IRODS_DEFAULT_ZONE = 'tempZone' IRODS_DEFAULT_RESC = 'irodsResc' IRODS_SSL_CA_FILE = '/etc/ssl/certs/localhost_and_chain.crt' -IRODS_AUTH_SCHEME = 'PAM' +IRODS_AUTH_SCHEME = 'pam_password' IRODS_CLIENT_OPTIONS_FOR_SSL = { "irods_client_server_policy": "CS_NEG_REQUIRE", "irods_client_server_negotiation": "request_server_negotiation", @@ -137,35 +141,37 @@ IRODS_SESSION_OPTIONS = { } # OIDC configuration -OIDC_ENABLED = True -OIDC_DOMAINS = ['yoda.dev'] -OIDC_CLIENT_ID = 'myClientId' -OIDC_CLIENT_SECRET = 'myClientPassword' -OIDC_CALLBACK_URI = 'https://portal.yoda:8443/user/callback' -OIDC_AUTH_BASE_URI = 'https://oauth.mocklab.io/oauth/authorize' -OIDC_AUTH_URI = 'https://oauth.mocklab.io/oauth/authorize?response_type=code&client_id=myClientId&redirect_uri=https://portal.yoda.test/user/callback&scope=openid&acr_values=' -OIDC_LOGIN_HINT = True -OIDC_TOKEN_URI = 'https://oauth.mocklab.io/oauth/token' -OIDC_SCOPES = 'openid' -OIDC_ACR_VALUES = '' -OIDC_USERINFO_URI = 'https://oauth.mocklab.io/userinfo' -OIDC_EMAIL_FIELD = 'email' -OIDC_JWKS_URI = 'https://oauth.mocklab.io/.well-known/jwks.json' -OIDC_JWT_ISSUER = 'https://oauth.mocklab.io' -OIDC_JWT_OPTIONS = { +OIDC_ENABLED = True +OIDC_DOMAINS = ['*.yoda.dev'] +OIDC_ALWAYS_REDIRECT = False +OIDC_CLIENT_ID = 'myClientId' +OIDC_CLIENT_SECRET = 'myClientPassword' +OIDC_CALLBACK_URI = 'https://portal.yoda:8443/user/callback' +OIDC_AUTH_BASE_URI = 'https://oauth.wiremockapi.cloud/oauth/authorize' +OIDC_AUTH_URI = 'https://oauth.wiremockapi.cloud/oauth/authorize?response_type=code&client_id=myClientId&redirect_uri=https://portal.yoda:8443/user/callback&scope=openid&acr_values=' +OIDC_LOGIN_HINT = True +OIDC_TOKEN_URI = 'https://oauth.wiremockapi.cloud/oauth/token' +OIDC_SCOPES = 'openid' +OIDC_ACR_VALUES = '' +OIDC_USERINFO_URI = 'https://oauth.wiremockapi.cloud/userinfo' +OIDC_EMAIL_FIELD = 'email' +OIDC_JWKS_URI = 'https://oauth.wiremockapi.cloud/.well-known/jwks.json' +OIDC_JWT_ISSUER = 'https://oauth.wiremockapi.cloud' +OIDC_JWT_OPTIONS = { "require_exp": True, #check that exp (expiration) claim is present "require_iat": False, #check that iat (issued at) claim is present "require_nbf": False, #check that nbf (not before) claim is present - "verify_aud": True, #check that aud (audience) claim matches audience - "verify_iat": False, #check that iat (issued at) claim value is an integer - "verify_exp": True, #check that exp (expiration) claim value is OK - "verify_iss": True, #check that iss (issuer) claim matches issuer - "verify_signature": True #verify the JWT cryptographic signature + "verify_aud": True, #check that aud (audience) claim matches audience + "verify_iat": False, #check that iat (issued at) claim value is an integer + "verify_exp": True, #check that exp (expiration) claim value is OK + "verify_iss": True, #check that iss (issuer) claim matches issuer + "verify_signature": True #verify the JWT cryptographic signature } -# Yoda portal -YODA_PORTAL_PATH = '/var/www/yoda' # Path to location of portal -YODA_CONFIG_PATH = '/var/www/yoda/config' # Path to portal's shared configuration +# Portal configuration +YODA_PORTAL_PATH = '/var/www/yoda' # Path to location of portal +YODA_CONFIG_PATH = '/var/www/yoda/config' # Path to portal's shared configuration + # Portal theme configuration YODA_THEME_PATH = '/var/www/yoda/themes' # Path to location of themes @@ -192,6 +198,7 @@ YODA_EUS_FQDN = 'eus.yoda.test' DATAREQUEST_HELP_CONTACT_NAME = 'PLACEHOLDER' DATAREQUEST_HELP_CONTACT_EMAIL = 'PLACEHOLDER' +# Upload function configuration UPLOAD_PART_FILES = True # Text file extensions configuration @@ -200,6 +207,7 @@ TEXT_FILE_EXTENSIONS = ['bash', 'csv', 'c', 'cpp', 'csharp', 'css', 'diff', 'for # Monitor thread configuration MONITOR_OUTPUT_DIR = "/tmp" MONITOR_SIGNAL_FILE = "/var/www/yoda/show-tech.sig" + FLASKCFG progress_update "Portal configured" From 4162ca2f212bce3c3812dcdd28d7e43c4198e356 Mon Sep 17 00:00:00 2001 From: Lazlo Westerhof Date: Fri, 13 Dec 2024 08:36:08 +0100 Subject: [PATCH 11/16] YDA-6061: update iRODS configuration files and install missing plugin for Yoda provider Docker --- docker/images/yoda_irods_icat/Dockerfile | 14 +- .../images/yoda_irods_icat/irods-icat-init.sh | 5 +- .../yoda_irods_icat/irods_environment.json | 36 ++--- docker/images/yoda_irods_icat/rules_uu.cfg | 16 +-- .../images/yoda_irods_icat/server_config.json | 123 +++++++++--------- 5 files changed, 97 insertions(+), 97 deletions(-) diff --git a/docker/images/yoda_irods_icat/Dockerfile b/docker/images/yoda_irods_icat/Dockerfile index 35e65d23c..7986ea922 100644 --- a/docker/images/yoda_irods_icat/Dockerfile +++ b/docker/images/yoda_irods_icat/Dockerfile @@ -5,7 +5,7 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] # Application settings ARG TAG=development ENV IRODS_VERSION "4.3.3-0~noble" -ENV IRODS_INDEX_PREP_VERSION "4.3.3.0-0+4.3.3~noble" +ENV IRODS_PLUS_VERSION "4.3.3.0-0+4.3.3~noble" ENV IRODS_MSVC_IRODS_VERSION "4.3.3" ENV IRODS_SUDO_MSVC_VERSION "1.0.0" ENV IRODS_UU_MSVC_VERSION "1.2.0" @@ -40,10 +40,13 @@ RUN apt-get update # hadolint ignore=DL3033 RUN apt-get install -y wget git sudo netcat-traditional gcc vim +# Install rsyslog +# hadolint ignore=DL3033 +RUN apt-get install -y rsyslog + # Install Python 3.12 # hadolint ignore=DL3033 -RUN apt-get install -y gcc python3 python3-dev python3-pip python3-virtualenv \ - python3-wheel python3.12-venv +RUN apt-get install -y gcc python3 python3-dev python3-pip python3.12-venv python3-virtualenv python3-wheel python3-magic # Upload PAM Python library COPY stage/pam_python.so /tmp/pam_python.so @@ -68,12 +71,11 @@ RUN useradd -d /var/lib/irods irods && \ do echo "Installing package ${package} and its dependencies ..."; \ apt-get -y install "$package=${IRODS_VERSION}" ; \ done && \ - for package in irods-rule-engine-plugin-indexing irods-rule-engine-plugin-elasticsearch; \ + for package in irods-rule-engine-plugin-python irods-rule-engine-plugin-indexing irods-rule-engine-plugin-elasticsearch; \ do echo "Installing package ${package} and its dependencies ..."; \ - apt-get -y install "$package=${IRODS_INDEX_PREP_VERSION}"; \ + apt-get -y install "$package=${IRODS_PLUS_VERSION}"; \ done - # Install iRODS microservices for Yoda # hadolint ignore=DL3033 RUN apt-get -y install libc++1 libjansson4 && \ diff --git a/docker/images/yoda_irods_icat/irods-icat-init.sh b/docker/images/yoda_irods_icat/irods-icat-init.sh index 7f39063ad..4d896c660 100644 --- a/docker/images/yoda_irods_icat/irods-icat-init.sh +++ b/docker/images/yoda_irods_icat/irods-icat-init.sh @@ -42,10 +42,7 @@ fi # Download test vault and iCAT data before_update "Downloading data" -if [ ! -d /download ] -then - mkdir /download -fi +mkdir /download wget -q "https://yoda.uu.nl/yoda-docker/${DATA_VERSION}.vault.tar.gz" -O "/download/${DATA_VERSION}.vault.tar.gz" progress_update "Downloaded vault test data." wget -q "https://yoda.uu.nl/yoda-docker/${DATA_VERSION}.icat.sql.gz" -O "/download/${DATA_VERSION}.icat.sql.gz" diff --git a/docker/images/yoda_irods_icat/irods_environment.json b/docker/images/yoda_irods_icat/irods_environment.json index e06b728dd..68b541fec 100644 --- a/docker/images/yoda_irods_icat/irods_environment.json +++ b/docker/images/yoda_irods_icat/irods_environment.json @@ -1,22 +1,22 @@ { - "irods_client_server_negotiation": "request_server_negotiation", - "irods_client_server_policy": "CS_NEG_REQUIRE", - "irods_cwd": "/tempZone/home/rods", - "irods_default_hash_scheme": "SHA256", - "irods_default_resource": "irodsResc", - "irods_encryption_algorithm": "AES-256-CBC", - "irods_encryption_key_size": 32, - "irods_encryption_num_hash_rounds": 16, - "irods_encryption_salt_size": 8, - "irods_home": "/tempZone/home/rods", + "irods_client_server_negotiation": "request_server_negotiation", + "irods_client_server_policy": "CS_NEG_REQUIRE", + "irods_cwd": "/tempZone/home/rods", + "irods_default_hash_scheme": "SHA256", + "irods_default_resource": "irodsResc", + "irods_encryption_algorithm": "AES-256-CBC", + "irods_encryption_key_size": 32, + "irods_encryption_num_hash_rounds": 16, + "irods_encryption_salt_size": 8, + "irods_home": "/tempZone/home/rods", "irods_host": "provider.yoda", - "irods_match_hash_policy": "compatible", - "irods_port": 1247, - "irods_ssl_ca_certificate_file": "/etc/irods/localhost_and_chain.crt", - "irods_ssl_certificate_chain_file": "/etc/irods/localhost_and_chain.crt", - "irods_ssl_certificate_key_file": "/etc/irods/localhost.key", - "irods_ssl_dh_params_file": "/etc/irods/dhparams.pem", - "irods_ssl_verify_server": "none", - "irods_user_name": "rods", + "irods_match_hash_policy": "compatible", + "irods_port": 1247, + "irods_ssl_ca_certificate_file": "/etc/irods/localhost_and_chain.crt", + "irods_ssl_certificate_chain_file": "/etc/irods/localhost_and_chain.crt", + "irods_ssl_certificate_key_file": "/etc/irods/localhost.key", + "irods_ssl_dh_params_file": "/etc/irods/dhparams.pem", + "irods_ssl_verify_server": "none", + "irods_user_name": "rods", "irods_zone_name": "tempZone" } diff --git a/docker/images/yoda_irods_icat/rules_uu.cfg b/docker/images/yoda_irods_icat/rules_uu.cfg index dff1a76c6..5f2d842cb 100644 --- a/docker/images/yoda_irods_icat/rules_uu.cfg +++ b/docker/images/yoda_irods_icat/rules_uu.cfg @@ -55,23 +55,20 @@ token_expiration_notification = '24' enable_data_package_archive = 'false' enable_data_package_download = 'false' -data_package_archive_resource = '' -data_package_archive_fqdn = 'provider.yoda' -data_package_archive_minimum = '1024' -data_package_archive_maximum = '-1' + +async_replication_delay_time = '3600' +async_replication_max_rss = '1000000000' +async_revision_delay_time = '0' +async_revision_max_rss = '1000000000' enable_inactivity_notification = 'true' inactivity_cutoff_months = '3' temporary_files = '._* .DS_Store Thumbs.db' -external_users_domain_filter = 'uu.nl acc.uu.nl' +external_users_domain_filter = '*.yoda.dev' enable_sram = 'false' -sram_rest_api_url = 'https://sram-mock.yoda' -sram_api_key = 'PLACEHOLDER' -sram_verbose_logging = 'true' -sram_tls_verify = 'false' arb_enabled = 'false' arb_exempt_resources = '' @@ -87,4 +84,5 @@ user_max_connections_enabled = 'false' user_max_connections_number = '4' text_file_extensions = 'bash csv c cpp csharp css diff fortran gams gauss go graphql ini irpf90 java js json julia julia-repl kotlin less lua makefile markdown md mathematica matlab maxima mizar objectivec openscad perl php php-template plaintext txt python py python-repl r ruby rust sas scilab scss shell sh sql stan stata swift typescript ts vbnet wasm xml yaml html' +notifications_enabled = 'true' python3_interpreter = '/usr/bin/python3' diff --git a/docker/images/yoda_irods_icat/server_config.json b/docker/images/yoda_irods_icat/server_config.json index 340dccf9b..e842445f8 100644 --- a/docker/images/yoda_irods_icat/server_config.json +++ b/docker/images/yoda_irods_icat/server_config.json @@ -1,95 +1,98 @@ { "advanced_settings": { - "default_log_rotation_in_days": 5, - "default_number_of_transfer_threads": 4, - "default_temporary_password_lifetime_in_seconds": 120, - "maximum_number_of_concurrent_rule_engine_server_processes": 4, - "maximum_size_for_single_buffer_in_megabytes": 32, - "maximum_temporary_password_lifetime_in_seconds": 1000, - "rule_engine_server_execution_time_in_seconds": 120, - "rule_engine_server_sleep_time_in_seconds": 10, - "transfer_buffer_size_for_parallel_transfer_in_megabytes": 4, + "default_log_rotation_in_days": 5, + "default_number_of_transfer_threads": 4, + "default_temporary_password_lifetime_in_seconds": 120, + "maximum_number_of_concurrent_rule_engine_server_processes": 4, + "maximum_size_for_single_buffer_in_megabytes": 32, + "maximum_temporary_password_lifetime_in_seconds": 1000, + "rule_engine_server_execution_time_in_seconds": 120, + "rule_engine_server_sleep_time_in_seconds": 10, + "transfer_buffer_size_for_parallel_transfer_in_megabytes": 4, "transfer_chunk_size_for_parallel_transfer_in_megabytes": 40 - }, + }, "catalog_provider_hosts": [ "provider.yoda" - ], - "catalog_service_role": "provider", - "default_dir_mode": "0750", - "default_file_mode": "0600", - "default_hash_scheme": "SHA256", - "default_resource_name": "irodsResc", - "environment_variables": {}, - "federation": [], + ], + "catalog_service_role": "provider", + "default_dir_mode": "0750", + "default_file_mode": "0600", + "default_hash_scheme": "SHA256", + "default_resource_name": "irodsResc", + "environment_variables": {}, + "federation": [], + "host_resolution": { + "host_entries": [] + }, "icat_host": "provider.yoda", - "match_hash_policy": "compatible", - "negotiation_key": "458ce56d4d4b631fac7c1a12b7a1f8a1", + "match_hash_policy": "compatible", + "negotiation_key": "458ce56d4d4b631fac7c1a12b7a1f8a1", "plugin_configuration": { - "authentication": {}, + "authentication": {}, "database": { "postgres": { - "db_host": "db.yoda", - "db_name": "ICAT", - "db_odbc_driver": "PostgreSQL", - "db_password": "yodadev", - "db_port": 5432, + "db_host": "db.yoda", + "db_name": "ICAT", + "db_odbc_driver": "PostgreSQL", + "db_password": "yodadev", + "db_port": 5432, "db_username": "irodsdb" } - }, - "network": {}, - "resource": {}, + }, + "network": {}, + "resource": {}, "rule_engines": [ { - "instance_name": "irods_rule_engine_plugin-irods_rule_language-instance", - "plugin_name": "irods_rule_engine_plugin-irods_rule_language", + "instance_name": "irods_rule_engine_plugin-irods_rule_language-instance", + "plugin_name": "irods_rule_engine_plugin-irods_rule_language", "plugin_specific_configuration": { "re_data_variable_mapping_set": [ "core" - ], + ], "re_function_name_mapping_set": [ "core" - ], + ], "re_rulebase_set": [ - "rules-uu", + "rules-uu", "core" - ], + ], "regexes_for_supported_peps": [ - "ac[^ ]*", - "msi[^ ]*", + "ac[^ ]*", + "msi[^ ]*", "[^ ]*pep_[^ ]*_(pre|post)" ] - }, + }, "shared_memory_instance": "irods_rule_language_rule_engine" - }, + }, { - "instance_name": "irods_rule_engine_plugin-python-instance", - "plugin_name": "irods_rule_engine_plugin-python", + "instance_name": "irods_rule_engine_plugin-python-instance", + "plugin_name": "irods_rule_engine_plugin-python", "plugin_specific_configuration": {} - }, + }, { - "instance_name": "irods_rule_engine_plugin-cpp_default_policy-instance", - "plugin_name": "irods_rule_engine_plugin-cpp_default_policy", + "instance_name": "irods_rule_engine_plugin-cpp_default_policy-instance", + "plugin_name": "irods_rule_engine_plugin-cpp_default_policy", "plugin_specific_configuration": {} } ] - }, + }, "rule_engine_namespaces": [ "" - ], - "schema_name": "server_config", + ], + "schema_name": "server_config", "schema_validation_base_uri": "off", - "schema_version": "v3", - "server_control_plane_encryption_algorithm": "AES-256-CBC", - "server_control_plane_encryption_num_hash_rounds": 16, - "server_control_plane_key": "8273a89fbac6364eaa0a7af10d07feeb", - "server_control_plane_port": 1248, - "server_control_plane_timeout_milliseconds": 10000, - "server_port_range_end": 20199, - "server_port_range_start": 20000, - "xmsg_port": 1279, - "zone_auth_scheme": "native", - "zone_key": "bf68b8f4ada270902708a513fd339f0d", - "zone_name": "tempZone", - "zone_port": 1247, + "schema_version": "v3", + "server_control_plane_encryption_algorithm": "AES-256-CBC", + "server_control_plane_encryption_num_hash_rounds": 16, + "server_control_plane_key": "8273a89fbac6364eaa0a7af10d07feeb", + "server_control_plane_port": 1248, + "server_control_plane_timeout_milliseconds": 10000, + "server_port_range_end": 20199, + "server_port_range_start": 20000, + "xmsg_port": 1279, + "zone_auth_scheme": "native", + "zone_key": "bf68b8f4ada270902708a513fd339f0d", + "zone_name": "tempZone", + "zone_port": 1247, "zone_user": "rods" } From e4e0bf971f3ba38e574669fe886c82aa9ed8af82 Mon Sep 17 00:00:00 2001 From: Lazlo Westerhof Date: Mon, 16 Dec 2024 16:00:14 +0100 Subject: [PATCH 12/16] Mailpit: update to v1.21.7 --- docker/images/mailpit/build.sh | 2 +- roles/mailpit/defaults/main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/images/mailpit/build.sh b/docker/images/mailpit/build.sh index f59b3949c..8c45aab2e 100755 --- a/docker/images/mailpit/build.sh +++ b/docker/images/mailpit/build.sh @@ -4,7 +4,7 @@ set -e set -u set -x -export MAILPIT_VERSION=1.21.2 +export MAILPIT_VERSION=1.21.7 export DOCKER_SCAN_SUGGEST=false DOCKER_TAG="$1" diff --git a/roles/mailpit/defaults/main.yml b/roles/mailpit/defaults/main.yml index d3e26a64e..4749e2530 100644 --- a/roles/mailpit/defaults/main.yml +++ b/roles/mailpit/defaults/main.yml @@ -3,7 +3,7 @@ enable_mailpit: false -mailpit_version: v1.21.2 # Mailpit releases: https://github.com/axllent/mailpit/releases +mailpit_version: v1.21.7 # Mailpit releases: https://github.com/axllent/mailpit/releases mailpit_max_messages: 10000 # Maximum number of messages to store. Mailpit will periodically delete the oldest messages if greater than this. mailpit_smtp_bind_address: 0.0.0.0 # SMTP bind interface. mailpit_smtp_port: 25 # SMTP port. From 6be2f28dde02dc451e9aeb87543c9893d2a84dee Mon Sep 17 00:00:00 2001 From: Lazlo Westerhof Date: Thu, 12 Dec 2024 11:34:01 +0100 Subject: [PATCH 13/16] YDA-6060: add resource modification support to iRODS resource Ansible module --- library/irods_resource.py | 106 +++++++++++++--------- roles/composable_resources/tasks/main.yml | 1 + 2 files changed, 63 insertions(+), 44 deletions(-) diff --git a/library/irods_resource.py b/library/irods_resource.py index 6cd5967c4..894e96148 100644 --- a/library/irods_resource.py +++ b/library/irods_resource.py @@ -1,12 +1,8 @@ #!/usr/bin/python -# Copyright (c) 2017-2022 Utrecht University +# Copyright (c) 2017-2024 Utrecht University # GNU General Public License v3.0 -ANSIBLE_METADATA = { - 'metadata_version': '1.1', - 'supported_by': 'community', - 'status': ['preview'] -} +ANSIBLE_METADATA = {"metadata_version": "1.1", "supported_by": "community", "status": ["preview"]} from ansible.module_utils.basic import * @@ -23,7 +19,7 @@ def get_session(): - env_file = os.path.expanduser('~/.irods/irods_environment.json') + env_file = os.path.expanduser("~/.irods/irods_environment.json") with open(env_file) as data_file: ienv = json.load(data_file) return (iRODSSession(irods_env_file=env_file), ienv) @@ -39,9 +35,11 @@ def main(): children=dict(default=None, type="list"), resource_type=dict(default=None), context=dict(default=None), - state=dict(default="present") - ), - supports_check_mode=True) + modify=dict(default=False), + state=dict(default="present"), + ), + supports_check_mode=True, + ) name = module.params["name"] host = module.params["host"] @@ -55,15 +53,14 @@ def main(): if not context: context = None + modify = module.params["modify"] state = module.params["state"] if IRODSCLIENT_AVAILABLE: try: session, ienv = get_session() except iRODSException: - module.fail_json( - msg="Could not establish irods connection. Please check ~/.irods/irods_environment.json" - ) + module.fail_json(msg="Could not establish irods connection. Please check ~/.irods/irods_environment.json") else: module.fail_json(msg="python-irodsclient needs to be installed") @@ -73,32 +70,51 @@ def main(): try: resource = session.resources.get(name) except ResourceDoesNotExist: - if state == 'present' and not module.check_mode: - resource = session.resources.create( - name, resource_type, host=host, - path=vault_path, context=context) + if state == "present" and not module.check_mode: + resource = session.resources.create(name, resource_type, host=host, path=vault_path, context=context) changed = True - elif state == 'absent': + elif state == "absent": module.exit_json(changed=False, msg="Resource {} is not present".format(name)) else: - if state == 'absent': - module.fail_json(msg="python-irodsclient fails to remove resources in version 0.6") - elif state == 'present': + if state == "absent": + module.fail_json(msg="This module does not remove resources, use iadmin rmresc") + elif state == "present": if host != resource.location: - warnings.append( - "Resource {name} has location set to '{resource.location}' instead of '{host}'" - .format(**locals())) + if modify: + resource = session.resources.modify(name, "host", host) + changed = True + else: + warnings.append( + "Resource {name} has location set to '{resource.location}' instead of '{host}'".format(**locals()) + ) if vault_path != resource.vault_path: - warnings.append( - "Resource {name} has vault_path set to '{resource.vault_path}' instead of '{vault_path}'" - .format(**locals())) + if modify: + resource = session.resources.modify(name, "path", vault_path) + changed = True + else: + warnings.append( + "Resource {name} has vault_path set to '{resource.vault_path}' instead of '{vault_path}'".format( + **locals() + ) + ) if resource_type != resource.type: - warnings.append("Resource {name} has resource_type set to '{resource.type}' instead of '{resource_type}'" - .format(**locals())) + if modify: + resource = session.resources.modify(name, "type", resource_type) + changed = True + else: + warnings.append( + "Resource {name} has resource_type set to '{resource.type}' instead of '{resource_type}'".format( + **locals() + ) + ) if context != resource.context: - warnings.append( - "Resource {name} has context set to '{resource.context}' instead of '{context}'" - .format(**locals())) + if modify: + resource = session.resources.modify(name, "context", context) + changed = True + else: + warnings.append( + "Resource {name} has context set to '{resource.context}' instead of '{context}'".format(**locals()) + ) # Build list of resource children names. names = [] @@ -115,16 +131,18 @@ def main(): changed = True module.exit_json( - changed=changed, - resource=dict( - name=resource.name, - zone=resource.zone_name, - parent=resource.parent, - context=resource.context, - status=resource.status), - warnings=warnings, - irods_environment=ienv) - - -if __name__ == '__main__': + changed=changed, + resource=dict( + name=resource.name, + zone=resource.zone_name, + parent=resource.parent, + context=resource.context, + status=resource.status, + ), + warnings=warnings, + irods_environment=ienv, + ) + + +if __name__ == "__main__": main() diff --git a/roles/composable_resources/tasks/main.yml b/roles/composable_resources/tasks/main.yml index de4f0e13f..94e775885 100644 --- a/roles/composable_resources/tasks/main.yml +++ b/roles/composable_resources/tasks/main.yml @@ -12,6 +12,7 @@ vault_path: "{{ item.vault_path | default('EMPTY_RESC_PATH') }}" children: "{{ item.children | default([]) }}" context: "{{ item.context | default('') }}" + modify: yoda_environment == "development" with_items: "{{ irods_resources }}" when: not ansible_check_mode From 934e24b80374e14408e55ae9e0261595942c7536 Mon Sep 17 00:00:00 2001 From: Lazlo Westerhof Date: Thu, 12 Dec 2024 14:02:17 +0100 Subject: [PATCH 14/16] YDA-6065: update core.re for iRODS 4.3.3 --- .../images/yoda_irods_icat/core.re.template | 114 +++--------------- roles/irods_icat/templates/core.re.j2 | 114 +++--------------- 2 files changed, 34 insertions(+), 194 deletions(-) diff --git a/docker/images/yoda_irods_icat/core.re.template b/docker/images/yoda_irods_icat/core.re.template index e109c1aaa..9515fffa6 100644 --- a/docker/images/yoda_irods_icat/core.re.template +++ b/docker/images/yoda_irods_icat/core.re.template @@ -1,15 +1,10 @@ -# iRODS Rule Base -# The new rule language is used to express all policies -# Recovery procedures are included for a micro-service after " ::: " +# iRODS core.re Rule Base # -#Test Rules -printHello { print_hello; } - -# -# -# These are sys admin rules for creating and deleting users and renaming -# the local zone. -acPreConnect(*OUT) { *OUT="CS_NEG_DONT_CARE"; } +# Defines the SSL (client-server negotiation) policy for this server +# - CS_NEG_REFUSE - No SSL +# - CS_NEG_DONT_CARE - Depends on the Client request +# - CS_NEG_REQUIRE - Require SSL +acPreConnect(*OUT) { *OUT="CS_NEG_REFUSE"; } acCreateUser { acPreProcForCreateUser; @@ -120,13 +115,13 @@ acCheckPasswordStrength(*password) { } # The following are rules for data object operation # Note that the msiOprDisallowed microservice can be used by all the rules # to disallow the execution of certain actions. -# 1) acSetRescSchemeForCreate - This is the preprossing rule for creating a +# 1) acSetRescSchemeForCreate - This is the preprocessing rule for creating a # data object. It can be used for setting the resource selection scheme -# when creating a data object which is used by the put, copy and -# replicate operations. Currently, three preprocessing functions can be +# when creating a data object which is used by the put, copy, and +# replication operations. Currently, three preprocessing functions can be # used by this rule: # msiSetNoDirectRescInp(rescList) - sets a list of resources that cannot -# be used by a normal user directly. More than one resources can be +# be used by a normal user directly. More than one resource can be # input using the character "%" as separator. e.g., resc1%resc2%resc3. # This function is optional, but if used, should be the first function # to execute because it screens the resource input. @@ -134,12 +129,12 @@ acCheckPasswordStrength(*password) { } # From version 2.3 onward, this function is no longer mandatory, but # if it is used, if should be executed right after the screening # function msiSetNoDirectRescInp. -# defaultResc - the resource to use if no resource is input. A "null" -# means there is no defaultResc. More than one resources can be +# defaultRescList - the resource to use if no resource is input. A "null" +# means there is no defaultResc. More than one resource can be # input using the character "%" as separator. # optionStr - Can be "forced", "preferred" or "null". A "forced" input -# means the defaultResc will be used regardless of the user input. -# The forced action only apply to to users with normal privilege. +# means the defaultRescList will be used regardless of the user input. +# The forced action only applies to users with normal privilege. # msiSetRescSortScheme(sortScheme) - set the scheme for # for selecting the best resource to use when creating a data object. # sortScheme - The sorting scheme. Valid scheme are "default", @@ -147,7 +142,7 @@ acCheckPasswordStrength(*password) { } # cache class of resource on the top of the list. The "byLoad" scheme will put # the least loaded resource on the top of the list: in order to work properly, # the RMS system must be switched on in order to pick up the load information -# for each server in the resource group list. +# for each server in the resource list. # The scheme "random" and "byRescClass" can be applied in sequence. e.g., # msiSetRescSortScheme(random); msiSetRescSortScheme(byRescClass) # will select randomly a cache class resource and put it on the @@ -179,22 +174,10 @@ acSetRescSchemeForRepl {msiSetDefaultResc("irodsResc","forced"); } # msiSetDataObjAvoidResc(avoidResc) - set the resource to avoid when # opening an object. The copy stored in this resource will not be picked # unless this is the only copy. -# msiSortDataObj(sortingScheme) - Sort the copies of the data object using -# this scheme. Currently, "random" and "byRescClass" sorting scheme are -# supported. If "byRescClass" is set, data objects in the "cache" -# resources will be placed ahead of of those in the "archive" resources. -# The sorting schemes can also be chained. e.g., -# msiSortDataObj(random); msiSortDataObj(byRescClass) means that -# the data objects will be sorted randomly first and then separated -# by class. -# msiStageDataObj(cacheResc) - stage a copy of the data object in the -# cacheResc before opening the data object. # The $writeFlag session variable has been created to be used as a condition # for differentiating between open for read ($writeFlag == "0") and # write ($writeFlag == "1"). e.g. : -# acPreprocForDataObjOpen {ON($writeFlag == "0") {msiStageDataObj("demoResc8"); } } # acPreprocForDataObjOpen {ON($writeFlag == "1") { } } -# acPreprocForDataObjOpen {msiSortDataObj("random"); msiSetDataObjPreferredResc("xyz%demoResc8%abc"); msiStageDataObj("demoResc8"); } # acPreprocForDataObjOpen {msiSetDataObjPreferredResc("demoResc7%demoResc8"); } acPreprocForDataObjOpen { } # acPreprocForDataObjOpen {msiGetSessionVarValue("all","all"); } @@ -221,35 +204,12 @@ acSetMultiReplPerResc { } # in sequence by these rules. # msiExtractNaraMetadata - extract and register metadata from the just # upload NARA files. -# msiSysReplDataObj(replResc, flag) - can be used to replicate a copy of -# the file just uploaded or copied data object to the specified replResc -# Valid values for the "flag" input are "all", "updateRepl" and -# "rbudpTransfer". More than one flag values can be set using the -# "%" character as separator. e.g., "all%updateRepl". "updateRepl" means -# update an existing stale copy to the latest copy. The "all" flag means -# replicate to all resources in a resource group or update all stale -# copies if the "updateRepl" flag is also set. "rbudpTransfer" means -# the RBUDP protocol will be used for the transfer. -# A "null" input means a single will be made in one of the resource -# in the resource group. -# It may be desirable to do replication only if the dataObject is stored -# in a resource group. For example, the following rule can be used: -# acPostProcForPut {ON($rescGroupName != "") {msiSysReplDataObj($rescGroupName,"all"); } } -# -# msiSysChksumDataObj - checksum the just uploaded or copied data object. -# acPostProcForPut {msiSysChksumDataObj; msiSysReplDataObj("demoResc8","all"); } -# acPostProcForPut {msiSysReplDataObj("demoResc8","all"); } -# acPostProcForPut {msiSysChksumDataObj; } -# acPostProcForPut {delay("") {msiSysReplDataObj('demoResc8','all'); } } # acWriteLine(*A,*B) {writeLine(*A,*B); } # acPostProcForPut {delay("1m") {acWriteLine('serverLog','delayed by a minute message1'); acWriteLine('serverLog','delayed by a minute message2'); } } -# acPostProcForPut {ON($objPath like "/tempZone/home/rods/nvo/*") {delay("1m") {msiSysReplDataObj('nvoReplResc','null'); } } } -# acPostProcForPut {msiSysReplDataObj("demoResc8","all"); } #acPostProcForPut {msiSetDataTypeFromExt; } -#acPostProcForPut {ON($objPath like "/tempZone/home/rods/tg/*") {msiSysReplDataObj("nvoReplResc","null"); } } #acPostProcForPut {ON($objPath like "/tempZone/home/rods/mytest/*") {writeLine("serverLog","File Path is "++$filePath); } } #acPostProcForPut {ON($objPath like "/tempZone/home/rods/mytest/*") {writeLine("serverLog","File Path is "++$filePath); msiSplitPath($filePath,*fileDir,*fileName); msiExecCmd("send.sh", "*fileDir *fileName", "null", "null","null",*Junk); writeLine("serverLog","After File Path is *fileDir *fileName"); } } -#acPostProcForPut { ON($objPath like "\*txt") {writeLine("serverLog","File $objPath"); } } +# acPostProcForPut { ON($objPath like "\*txt") {writeLine("serverLog","File $objPath"); } } acPostProcForPut { } acPostProcForCopy { } acPostProcForFilePathReg { } @@ -378,15 +338,6 @@ acChkHostAccessControl { } # acSetVaultPathPolicy {msiSetRandomScheme; } acSetVaultPathPolicy {msiSetGraftPathScheme("no","1"); } # -# 17) acSetReServerNumProc - This rule set the policy for the number of processes -# to use when running jobs in the irodsReServer. The irodsReServer can now -# multi-task such that one or two long running jobs cannot block the execution -# of other jobs. One function can be called: -# msiSetReServerNumProc(numProc) - numProc can be "default" or a number -# in the range 1-4. numProc will be set to 1 if "default" is the input. -# -acSetReServerNumProc {msiSetReServerNumProc("default"); } -# # 18) acPreProcForCollCreate - This is the PreProcessing rule for creating # a collection. Currently there is no function written specifically # for this rule. @@ -715,37 +666,6 @@ getSessionVar(*name, *output) { *output = eval("str($"++*name++")"); } -# ---------------------------------------------------------------------------- -# These rules are for testing only -#acDataObjCreate {acSetCreateConditions; acDOC; } -acSetCreateConditions {msiGetNewObjDescriptor ::: recover_msiGetNewObjDescriptor; acSetResourceList; } -acDOC {msiPhyDataObjCreate ::: recover_msiPhyDataObjCreate; acRegisterData ::: msiRollback; msiCommit; } -acSetResourceList {msiSetResourceList; } -acSetCopyNumber {msiSetCopyNumber; } -acRegisterData {msiRegisterData ::: msiRollback; } -# -#These are actions for getting iCAT results for performing iRODS operations. -#These rules generate the genQueryOut_ structure for each action for the given condition -# -acGetIcatResults(*Action,*Condition,*GenQOut) {ON((*Action == "replicate") %% (*Action == "trim") %% (*Action == "chksum") %% (*Action == "copy") %% (*Action == "remove")) {msiMakeQuery("DATA_NAME, COLL_NAME",*Condition,*Query); msiExecStrCondQuery(*Query, *GenQOut); cut; } } -acGetIcatResults(*Action,*Condition,*GenQOut) {ON(*Action == "chksumRescLoc") {msiMakeQuery("DATA_NAME, COLL_NAME, RESC_LOC",*Condition,*Query); msiExecStrCondQuery(*Query, *GenQOut); cut; } } -acGetIcatResults(*Action,*Condition,*GenQOut) {ON(*Action == "list") {msiMakeQuery("DATA_NAME, COLL_NAME, DATA_RESC_NAME, DATA_REPL_NUM, DATA_SIZE",*Condition,*Query); msiExecStrCondQuery(*Query, *GenQOut); cut; } } -# -#rules for purging a file which have expired -# -acPurgeFiles(*Condition) {ON((*Condition == "null") %% (*Condition == "")) {msiGetIcatTime(*Time,"unix"); acGetIcatResults("remove","DATA_EXPIRY < '*Time'",*List); foreach(*List) {msiDataObjUnlink(*List,*Status); msiGetValByKey(*List,"DATA_NAME",*D); msiGetValByKey(*List,"COLL_NAME",*E); writeLine("stdout","Purged File *E/*D at *Time"); } } } -acPurgeFiles(*Condition) {msiGetIcatTime(*Time,"unix"); acGetIcatResults("remove","DATA_EXPIRY < '*Time' AND *Condition",*List); foreach(*List) {msiDataObjUnlink(*List,*Status); msiGetValByKey(*List,"DATA_NAME",*D); msiGetValByKey(*List,"COLL_NAME",*E); writeLine("stdout","Purged File *E/*D at *Time"); } } -acConvertToInt(*R) {assign(*A,$sysUidClient); assign($sysUidClient,*R); assign(*K, $sysUidClient); assign(*R,*K); assign($sysUidClient,*A); } - -# -# rule for running a workflow -# -acRunWorkFlow(*File, *R_BUF) { - msiDataObjOpen("objPath=*File++++openFlags=O_RDONLY",*S_FD); - msiDataObjRead(*S_FD,33554412,*R_BUF); - msiDataObjClose(*S_FD,*Status2); -} - acPostProcForParallelTransferReceived(*leaf_resource) {} acPostProcForDataCopyReceived(*leaf_resource) {} @@ -757,7 +677,7 @@ acPostProcForDataCopyReceived(*leaf_resource) {} # =-=-=-=-=-=-=- # policy controlling when a dataObject is staged to cache from archive in a compound coordinating resource -# - the default is to stage when cache is not present ("when_necessary") +# - the default is to stage when cache is stale or not present ("when_necessary") # =-=-=-=-=-=-=- # pep_resource_resolve_hierarchy_pre(*INSTANCE, *CONTEXT, *OUT, *OPERATION, *HOST, *PARSER, *VOTE){*OUT="compound_resource_cache_refresh_policy=when_necessary";} # default # pep_resource_resolve_hierarchy_pre(*INSTANCE, *CONTEXT, *OUT, *OPERATION, *HOST, *PARSER, *VOTE){*OUT="compound_resource_cache_refresh_policy=always";} diff --git a/roles/irods_icat/templates/core.re.j2 b/roles/irods_icat/templates/core.re.j2 index db7f4e316..8be96338d 100644 --- a/roles/irods_icat/templates/core.re.j2 +++ b/roles/irods_icat/templates/core.re.j2 @@ -1,16 +1,11 @@ # {{ ansible_managed }} -# iRODS Rule Base -# The new rule language is used to express all policies -# Recovery procedures are included for a micro-service after " ::: " +# iRODS core.re Rule Base # -#Test Rules -printHello { print_hello; } - -# -# -# These are sys admin rules for creating and deleting users and renaming -# the local zone. -acPreConnect(*OUT) { *OUT="CS_NEG_DONT_CARE"; } +# Defines the SSL (client-server negotiation) policy for this server +# - CS_NEG_REFUSE - No SSL +# - CS_NEG_DONT_CARE - Depends on the Client request +# - CS_NEG_REQUIRE - Require SSL +acPreConnect(*OUT) { *OUT="CS_NEG_REFUSE"; } acCreateUser { acPreProcForCreateUser; @@ -121,13 +116,13 @@ acCheckPasswordStrength(*password) { } # The following are rules for data object operation # Note that the msiOprDisallowed microservice can be used by all the rules # to disallow the execution of certain actions. -# 1) acSetRescSchemeForCreate - This is the preprossing rule for creating a +# 1) acSetRescSchemeForCreate - This is the preprocessing rule for creating a # data object. It can be used for setting the resource selection scheme -# when creating a data object which is used by the put, copy and -# replicate operations. Currently, three preprocessing functions can be +# when creating a data object which is used by the put, copy, and +# replication operations. Currently, three preprocessing functions can be # used by this rule: # msiSetNoDirectRescInp(rescList) - sets a list of resources that cannot -# be used by a normal user directly. More than one resources can be +# be used by a normal user directly. More than one resource can be # input using the character "%" as separator. e.g., resc1%resc2%resc3. # This function is optional, but if used, should be the first function # to execute because it screens the resource input. @@ -135,12 +130,12 @@ acCheckPasswordStrength(*password) { } # From version 2.3 onward, this function is no longer mandatory, but # if it is used, if should be executed right after the screening # function msiSetNoDirectRescInp. -# defaultResc - the resource to use if no resource is input. A "null" -# means there is no defaultResc. More than one resources can be +# defaultRescList - the resource to use if no resource is input. A "null" +# means there is no defaultResc. More than one resource can be # input using the character "%" as separator. # optionStr - Can be "forced", "preferred" or "null". A "forced" input -# means the defaultResc will be used regardless of the user input. -# The forced action only apply to to users with normal privilege. +# means the defaultRescList will be used regardless of the user input. +# The forced action only applies to users with normal privilege. # msiSetRescSortScheme(sortScheme) - set the scheme for # for selecting the best resource to use when creating a data object. # sortScheme - The sorting scheme. Valid scheme are "default", @@ -148,7 +143,7 @@ acCheckPasswordStrength(*password) { } # cache class of resource on the top of the list. The "byLoad" scheme will put # the least loaded resource on the top of the list: in order to work properly, # the RMS system must be switched on in order to pick up the load information -# for each server in the resource group list. +# for each server in the resource list. # The scheme "random" and "byRescClass" can be applied in sequence. e.g., # msiSetRescSortScheme(random); msiSetRescSortScheme(byRescClass) # will select randomly a cache class resource and put it on the @@ -180,22 +175,10 @@ acSetRescSchemeForRepl {msiSetDefaultResc("{{ irods_default_resc }}","forced"); # msiSetDataObjAvoidResc(avoidResc) - set the resource to avoid when # opening an object. The copy stored in this resource will not be picked # unless this is the only copy. -# msiSortDataObj(sortingScheme) - Sort the copies of the data object using -# this scheme. Currently, "random" and "byRescClass" sorting scheme are -# supported. If "byRescClass" is set, data objects in the "cache" -# resources will be placed ahead of of those in the "archive" resources. -# The sorting schemes can also be chained. e.g., -# msiSortDataObj(random); msiSortDataObj(byRescClass) means that -# the data objects will be sorted randomly first and then separated -# by class. -# msiStageDataObj(cacheResc) - stage a copy of the data object in the -# cacheResc before opening the data object. # The $writeFlag session variable has been created to be used as a condition # for differentiating between open for read ($writeFlag == "0") and # write ($writeFlag == "1"). e.g. : -# acPreprocForDataObjOpen {ON($writeFlag == "0") {msiStageDataObj("demoResc8"); } } # acPreprocForDataObjOpen {ON($writeFlag == "1") { } } -# acPreprocForDataObjOpen {msiSortDataObj("random"); msiSetDataObjPreferredResc("xyz%demoResc8%abc"); msiStageDataObj("demoResc8"); } # acPreprocForDataObjOpen {msiSetDataObjPreferredResc("demoResc7%demoResc8"); } acPreprocForDataObjOpen { } # acPreprocForDataObjOpen {msiGetSessionVarValue("all","all"); } @@ -222,35 +205,12 @@ acSetMultiReplPerResc { } # in sequence by these rules. # msiExtractNaraMetadata - extract and register metadata from the just # upload NARA files. -# msiSysReplDataObj(replResc, flag) - can be used to replicate a copy of -# the file just uploaded or copied data object to the specified replResc -# Valid values for the "flag" input are "all", "updateRepl" and -# "rbudpTransfer". More than one flag values can be set using the -# "%" character as separator. e.g., "all%updateRepl". "updateRepl" means -# update an existing stale copy to the latest copy. The "all" flag means -# replicate to all resources in a resource group or update all stale -# copies if the "updateRepl" flag is also set. "rbudpTransfer" means -# the RBUDP protocol will be used for the transfer. -# A "null" input means a single will be made in one of the resource -# in the resource group. -# It may be desirable to do replication only if the dataObject is stored -# in a resource group. For example, the following rule can be used: -# acPostProcForPut {ON($rescGroupName != "") {msiSysReplDataObj($rescGroupName,"all"); } } -# -# msiSysChksumDataObj - checksum the just uploaded or copied data object. -# acPostProcForPut {msiSysChksumDataObj; msiSysReplDataObj("demoResc8","all"); } -# acPostProcForPut {msiSysReplDataObj("demoResc8","all"); } -# acPostProcForPut {msiSysChksumDataObj; } -# acPostProcForPut {delay("") {msiSysReplDataObj('demoResc8','all'); } } # acWriteLine(*A,*B) {writeLine(*A,*B); } # acPostProcForPut {delay("1m") {acWriteLine('serverLog','delayed by a minute message1'); acWriteLine('serverLog','delayed by a minute message2'); } } -# acPostProcForPut {ON($objPath like "/tempZone/home/rods/nvo/*") {delay("1m") {msiSysReplDataObj('nvoReplResc','null'); } } } -# acPostProcForPut {msiSysReplDataObj("demoResc8","all"); } #acPostProcForPut {msiSetDataTypeFromExt; } -#acPostProcForPut {ON($objPath like "/tempZone/home/rods/tg/*") {msiSysReplDataObj("nvoReplResc","null"); } } #acPostProcForPut {ON($objPath like "/tempZone/home/rods/mytest/*") {writeLine("serverLog","File Path is "++$filePath); } } #acPostProcForPut {ON($objPath like "/tempZone/home/rods/mytest/*") {writeLine("serverLog","File Path is "++$filePath); msiSplitPath($filePath,*fileDir,*fileName); msiExecCmd("send.sh", "*fileDir *fileName", "null", "null","null",*Junk); writeLine("serverLog","After File Path is *fileDir *fileName"); } } -#acPostProcForPut { ON($objPath like "\*txt") {writeLine("serverLog","File $objPath"); } } +# acPostProcForPut { ON($objPath like "\*txt") {writeLine("serverLog","File $objPath"); } } acPostProcForPut { } acPostProcForCopy { } acPostProcForFilePathReg { } @@ -379,15 +339,6 @@ acChkHostAccessControl { } # acSetVaultPathPolicy {msiSetRandomScheme; } acSetVaultPathPolicy {msiSetGraftPathScheme("no","1"); } # -# 17) acSetReServerNumProc - This rule set the policy for the number of processes -# to use when running jobs in the irodsReServer. The irodsReServer can now -# multi-task such that one or two long running jobs cannot block the execution -# of other jobs. One function can be called: -# msiSetReServerNumProc(numProc) - numProc can be "default" or a number -# in the range 1-4. numProc will be set to 1 if "default" is the input. -# -acSetReServerNumProc {msiSetReServerNumProc("default"); } -# # 18) acPreProcForCollCreate - This is the PreProcessing rule for creating # a collection. Currently there is no function written specifically # for this rule. @@ -716,37 +667,6 @@ getSessionVar(*name, *output) { *output = eval("str($"++*name++")"); } -# ---------------------------------------------------------------------------- -# These rules are for testing only -#acDataObjCreate {acSetCreateConditions; acDOC; } -acSetCreateConditions {msiGetNewObjDescriptor ::: recover_msiGetNewObjDescriptor; acSetResourceList; } -acDOC {msiPhyDataObjCreate ::: recover_msiPhyDataObjCreate; acRegisterData ::: msiRollback; msiCommit; } -acSetResourceList {msiSetResourceList; } -acSetCopyNumber {msiSetCopyNumber; } -acRegisterData {msiRegisterData ::: msiRollback; } -# -#These are actions for getting iCAT results for performing iRODS operations. -#These rules generate the genQueryOut_ structure for each action for the given condition -# -acGetIcatResults(*Action,*Condition,*GenQOut) {ON((*Action == "replicate") %% (*Action == "trim") %% (*Action == "chksum") %% (*Action == "copy") %% (*Action == "remove")) {msiMakeQuery("DATA_NAME, COLL_NAME",*Condition,*Query); msiExecStrCondQuery(*Query, *GenQOut); cut; } } -acGetIcatResults(*Action,*Condition,*GenQOut) {ON(*Action == "chksumRescLoc") {msiMakeQuery("DATA_NAME, COLL_NAME, RESC_LOC",*Condition,*Query); msiExecStrCondQuery(*Query, *GenQOut); cut; } } -acGetIcatResults(*Action,*Condition,*GenQOut) {ON(*Action == "list") {msiMakeQuery("DATA_NAME, COLL_NAME, DATA_RESC_NAME, DATA_REPL_NUM, DATA_SIZE",*Condition,*Query); msiExecStrCondQuery(*Query, *GenQOut); cut; } } -# -#rules for purging a file which have expired -# -acPurgeFiles(*Condition) {ON((*Condition == "null") %% (*Condition == "")) {msiGetIcatTime(*Time,"unix"); acGetIcatResults("remove","DATA_EXPIRY < '*Time'",*List); foreach(*List) {msiDataObjUnlink(*List,*Status); msiGetValByKey(*List,"DATA_NAME",*D); msiGetValByKey(*List,"COLL_NAME",*E); writeLine("stdout","Purged File *E/*D at *Time"); } } } -acPurgeFiles(*Condition) {msiGetIcatTime(*Time,"unix"); acGetIcatResults("remove","DATA_EXPIRY < '*Time' AND *Condition",*List); foreach(*List) {msiDataObjUnlink(*List,*Status); msiGetValByKey(*List,"DATA_NAME",*D); msiGetValByKey(*List,"COLL_NAME",*E); writeLine("stdout","Purged File *E/*D at *Time"); } } -acConvertToInt(*R) {assign(*A,$sysUidClient); assign($sysUidClient,*R); assign(*K, $sysUidClient); assign(*R,*K); assign($sysUidClient,*A); } - -# -# rule for running a workflow -# -acRunWorkFlow(*File, *R_BUF) { - msiDataObjOpen("objPath=*File++++openFlags=O_RDONLY",*S_FD); - msiDataObjRead(*S_FD,33554412,*R_BUF); - msiDataObjClose(*S_FD,*Status2); -} - acPostProcForParallelTransferReceived(*leaf_resource) {} acPostProcForDataCopyReceived(*leaf_resource) {} @@ -758,7 +678,7 @@ acPostProcForDataCopyReceived(*leaf_resource) {} # =-=-=-=-=-=-=- # policy controlling when a dataObject is staged to cache from archive in a compound coordinating resource -# - the default is to stage when cache is not present ("when_necessary") +# - the default is to stage when cache is stale or not present ("when_necessary") # =-=-=-=-=-=-=- # pep_resource_resolve_hierarchy_pre(*INSTANCE, *CONTEXT, *OUT, *OPERATION, *HOST, *PARSER, *VOTE){*OUT="compound_resource_cache_refresh_policy=when_necessary";} # default # pep_resource_resolve_hierarchy_pre(*INSTANCE, *CONTEXT, *OUT, *OPERATION, *HOST, *PARSER, *VOTE){*OUT="compound_resource_cache_refresh_policy=always";} From 8cd4ed0399c3bacac37dbc0e7f8f24df377ef0c2 Mon Sep 17 00:00:00 2001 From: claravox Date: Tue, 17 Dec 2024 13:50:23 +0100 Subject: [PATCH 15/16] Docs: update ansible link --- docs/development/setting-up-development-environment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/development/setting-up-development-environment.md b/docs/development/setting-up-development-environment.md index 993bb6695..62f33f8d2 100644 --- a/docs/development/setting-up-development-environment.md +++ b/docs/development/setting-up-development-environment.md @@ -10,7 +10,7 @@ Setting up a Yoda development environment is easy, you only need the following: * [Vagrant](https://www.vagrantup.com/docs/installation/) (>= 2.0) On GNU/Linux or macOS you also need: -* [Ansible](https://docs.ansible.com/ansible/intro_installation.html) (>= 2.11) +* [Ansible](https://docs.ansible.com/ansible/latest/installation_guide/index.html) (>= 2.11) The guide below will deploy an 'allinone' instance (all functional roles in one virtual machine) with the default configuration. From d9cc4e15015e277651a8a2a25bcf8dc88edebe83 Mon Sep 17 00:00:00 2001 From: Lazlo Westerhof Date: Tue, 17 Dec 2024 15:24:28 +0100 Subject: [PATCH 16/16] iRODS iCAT: fix acPreConnect policy --- roles/irods_icat/templates/core.re.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/irods_icat/templates/core.re.j2 b/roles/irods_icat/templates/core.re.j2 index 8be96338d..02a22f298 100644 --- a/roles/irods_icat/templates/core.re.j2 +++ b/roles/irods_icat/templates/core.re.j2 @@ -5,7 +5,7 @@ # - CS_NEG_REFUSE - No SSL # - CS_NEG_DONT_CARE - Depends on the Client request # - CS_NEG_REQUIRE - Require SSL -acPreConnect(*OUT) { *OUT="CS_NEG_REFUSE"; } +acPreConnect(*OUT) { *OUT="CS_NEG_DONT_CARE"; } acCreateUser { acPreProcForCreateUser;