Merge branch 'development' into epos-msl

Vagrantfile

@@ -7,7 +7,7 @@
 VAGRANTFILE_API_VERSION = "2"
 ENV['VAGRANT_DEFAULT_PROVIDER'] = "libvirt"
 
-BOX = 'generic/ubuntu2004'
+BOX = 'alvistack/ubuntu-24.04'
 GUI = false
 CPU = 2
 RAM = 4096
@@ -47,6 +47,10 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
       machine.vm.hostname = name + DOMAIN
       machine.vm.network 'private_network', ip: ipaddr, netmask: NETMASK
       machine.vm.synced_folder ".", "/vagrant", disabled: true
+      machine.vm.provision "shell" do |s|
+        s.inline = "sudo hostnamectl hostname $1"
+        s.args   = name + DOMAIN
+      end
       machine.vm.provision "shell",
         inline: "sudo timedatectl set-timezone Europe/Amsterdam"
       machine.vm.provision "shell",
@@ -64,7 +68,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
       controller.vm.provider :virtualbox do |vbox|
         vbox.customize ["guestproperty", "set", :id, "/VirtualBox/GuestAdd/VBoxService/--timesync-set-threshold", 10000]
       end
-      controller.vm.box = 'generic/ubuntu2004'
+      controller.vm.box = 'alvistack/ubuntu-24.04'
       controller.vm.hostname = "controller"
       controller.vm.network :private_network, ip: "192.168.56.5", netmask: NETMASK
       controller.vm.provision "shell", privileged: false, path: "vagrant/provision_controller.sh"

docker/down.sh

@@ -1,2 +1,2 @@
 #!/bin/bash
-docker-compose down "$@"
+docker compose down "$@"

docker/images/davrods/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:20.04
+FROM ubuntu:24.04
 LABEL maintainer="Yoda team <yoda@uu.nl>"
 
 # Network settings
@@ -7,11 +7,11 @@ EXPOSE 443
 
 # Application settings
 ARG TAG=development
-ENV IRODS_PACKAGE_VERSION="4.2.12-1~bionic"
-ENV APT_REPO_CONFIG="deb [arch=amd64] https://packages.irods.org/apt/ bionic main"
+ENV IRODS_PACKAGE_VERSION="4.3.3-0~noble"
+ENV APT_REPO_CONFIG="deb [arch=amd64] https://packages.irods.org/apt/ noble main"
 ENV APT_IRODS_REPO_SIGNING_KEY_LOC="https://packages.irods.org/irods-signing-key.asc"
-ENV DAVRODS_APT_URL="https://github.com/UtrechtUniversity/davrods/releases/download/4.2.12_1.5.1"
-ENV DAVRODS_APT_PACKAGE="davrods-4.2.12-1.5.1.deb"
+ENV DAVRODS_APT_URL="https://github.com/UtrechtUniversity/davrods/releases/download/4.3.3_1.5.1"
+ENV DAVRODS_APT_PACKAGE="davrods-4.3.3-1.5.1.deb"
 ENV DEBIAN_FRONTEND="noninteractive"
 
 # Update packages
@@ -24,7 +24,7 @@ RUN apt-get install -y wget vim
 # Install Python 3
 # hadolint ignore=DL3033
 RUN apt-get install -y gcc python3 python3-dev python3-pip python3-virtualenv \
-                       python3-wheel python3.8-venv
+                       python3-wheel python3.12-venv
 
 # Install and configure Apache
 # hadolint ignore=DL3033
@@ -38,27 +38,6 @@ RUN ln -s /etc/apache2/mods-available/dav_fs.load /etc/apache2/mods-enabled/dav_
 RUN ln -s /etc/apache2/mods-available/dav_lock.load /etc/apache2/mods-enabled/dav_lock.load
 RUN ln -s /etc/apache2/mods-available/davrods.load /etc/apache2/mods-enabled/davrods.load
 
-# Install iRODS dependencies
-RUN apt-get -y install aptitude libboost-locale-dev libpython2-stdlib libpython2.7-minimal \
-          libpython2.7-stdlib \
-          python-is-python2 python-six python2 python2-minimal python2.7 python2.7-minimal \
-          python-certifi python-chardet python-idna python-pkg-resources python-setuptools
-ENV PY_URLLIB_PREFIX="http://security.ubuntu.com/ubuntu/pool/main/p/python-urllib3"
-ENV PY_URLLIB_FILENAME="python-urllib3_1.22-1ubuntu0.18.04.2_all.deb"
-ENV PY_REQUESTS_PREFIX="http://security.ubuntu.com/ubuntu/pool/main/r/requests"
-ENV PY_REQUESTS_FILENAME="python-requests_2.18.4-2ubuntu0.1_all.deb"
-ENV OPENSSL_PREFIX="http://security.ubuntu.com/ubuntu/pool/main/o/openssl1.0"
-ENV OPENSSL_FILENAME="libssl1.0.0_1.0.2n-1ubuntu5.13_amd64.deb"
-RUN wget -q ${PY_URLLIB_PREFIX}/${PY_URLLIB_FILENAME} && \
-    wget -q ${PY_REQUESTS_PREFIX}/${PY_REQUESTS_FILENAME} && \
-    wget -q ${OPENSSL_PREFIX}/${OPENSSL_FILENAME} && \
-    dpkg -i ${PY_URLLIB_FILENAME} && \
-    dpkg -i ${PY_REQUESTS_FILENAME} && \
-    dpkg -i ${OPENSSL_FILENAME} && \
-    rm ${PY_URLLIB_FILENAME} && \
-    rm ${PY_REQUESTS_FILENAME} && \
-    rm ${OPENSSL_FILENAME}
-
 # Install iRODS components: iCommands and runtime
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 # hadolint ignore=DL3033

docker/images/mailpit/build.sh

@@ -4,7 +4,7 @@ set -e
 set -u
 set -x
 
-export MAILPIT_VERSION=1.21.2
+export MAILPIT_VERSION=1.21.7
 export DOCKER_SCAN_SUGGEST=false
 DOCKER_TAG="$1"
 

docker/images/yoda_eus/Dockerfile

@@ -1,12 +1,13 @@
-FROM ubuntu:20.04
+FROM ubuntu:24.04
+# FROM --platform=linux/amd64 ubuntu:24.04
 LABEL maintainer="Yoda team <yoda@uu.nl>"
 
 # Application settings
 ARG TAG=development
-ENV YODA_EUS_REPO "https://github.com/UtrechtUniversity/yoda-external-user-service.git"
-ENV YODA_EUS_BRANCH "$TAG"
-ENV YODA_PORTAL_REPO "https://github.com/UtrechtUniversity/yoda-portal.git"
-ENV YODA_PORTAL_BRANCH "$TAG"
+ENV YODA_EUS_REPO="https://github.com/UtrechtUniversity/yoda-external-user-service.git"
+ENV YODA_EUS_BRANCH="$TAG"
+ENV YODA_PORTAL_REPO="https://github.com/UtrechtUniversity/yoda-portal.git"
+ENV YODA_PORTAL_BRANCH="$TAG"
 ENV DEBIAN_FRONTEND="noninteractive"
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
@@ -19,19 +20,19 @@ RUN apt-get update
 
 # Install common tools (PostgreSQL is needed for psql commands in init script)
 # hadolint ignore=DL3033
-RUN apt-get install -y wget git netcat gnupg vim
+RUN apt-get install -y wget git netcat-traditional gnupg vim
 
 # Install PostgreSQL 15 for checks whether database container is up, as well as troubleshooting
 # hadolint ignore=DL3033
 RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
-RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ focal-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list
+RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ noble-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list
 RUN apt-get update
-RUN apt-get -y install postgresql-15 python3-psycopg2 acl
+RUN apt-get -y install postgresql-15 libpq-dev python3-psycopg2 acl
 
 # Install Python 3
 # hadolint ignore=DL3033
 RUN apt-get install -y gcc python3 python3-dev python3-pip python3-virtualenv \
-                       python3-wheel python3.8-venv virtualenv
+                       python3-wheel python3.12-venv virtualenv
 
 # Install and configure Apache
 # hadolint ignore=DL3033
@@ -57,7 +58,9 @@ RUN useradd yodadeployment && \
 COPY yoda-external-user-service-vhost.conf /etc/apache2/sites-available/003-yoda-eus-vhost.conf
 RUN ln -s /etc/apache2/sites-available/003-yoda-eus-vhost.conf /etc/apache2/sites-enabled/003-yoda-eus-vhost.conf
 RUN virtualenv --python /usr/bin/python3  /var/www/extuser/yoda-external-user-service/venv && \
-    /var/www/extuser/yoda-external-user-service/venv/bin/pip3 install --no-cache-dir pip==23.0.1 && \
+    /var/www/extuser/yoda-external-user-service/venv/bin/pip3 install --no-cache-dir pip==24.3.1 && \
+    /var/www/extuser/yoda-external-user-service/venv/bin/pip3 install --no-cache-dir virtualenv==20.28.0 && \
+    /var/www/extuser/yoda-external-user-service/venv/bin/python3 -m virtualenv /var/www/extuser/yoda-external-user-service/venv && \
     /var/www/extuser/yoda-external-user-service/venv/bin/pip3 install --no-cache-dir /var/www/extuser/yoda-external-user-service
 
 # Make copy of data on volumes that can optionally be bind mounted, so the container init script

docker/images/yoda_irods_icat/Dockerfile

@@ -1,20 +1,19 @@
-FROM ubuntu:20.04
+FROM ubuntu:24.04
 LABEL maintainer="Yoda team <yoda@uu.nl>"
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 # Application settings
 ARG TAG=development
-ENV IRODS_VERSION "4.2.12-1~bionic"
-ENV IRODS_PREP_VERSION "4.2.12.0-1~bionic"
-ENV IRODS_INDEX_VERSION "4.2.12.0-1~bionic"
-ENV IRODS_MSVC_IRODS_VERSION "4.2.12"
+ENV IRODS_VERSION "4.3.3-0~noble"
+ENV IRODS_PLUS_VERSION "4.3.3.0-0+4.3.3~noble"
+ENV IRODS_MSVC_IRODS_VERSION "4.3.3"
 ENV IRODS_SUDO_MSVC_VERSION "1.0.0"
 ENV IRODS_UU_MSVC_VERSION "1.2.0"
 ENV IRODS_UU_MSVC_VERSION_RELEASE "0"
-ENV APT_REPO_CONFIG="deb [arch=amd64] https://packages.irods.org/apt/ bionic main"
+ENV APT_REPO_CONFIG="deb [arch=amd64] https://packages.irods.org/apt/ noble main"
 ENV APT_IRODS_REPO_SIGNING_KEY_LOC="https://packages.irods.org/irods-signing-key.asc"
 ENV YODA_RULESET_REPO "https://github.com/UtrechtUniversity/yoda-ruleset.git"
-ENV YODA_RULESET_BRANCH "$TAG"
+ENV YODA_RULESET_BRANCH="$TAG"
 ENV PRE_BUILD_RULESET_DEPENDENCIES "yes"
 ENV DEBIAN_FRONTEND="noninteractive"
 
@@ -39,17 +38,15 @@ RUN apt-get update
 
 # Install common tools
 # hadolint ignore=DL3033
-RUN apt-get install -y wget git sudo netcat gcc vim
+RUN apt-get install -y wget git sudo netcat-traditional gcc vim
 
-# Install Python 2.7
-RUN apt-get install -y python2 python2-dev && \
-    wget -q -O /usr/local/bin/get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py && \
-    /usr/bin/python2 /usr/local/bin/get-pip.py
+# Install rsyslog
+# hadolint ignore=DL3033
+RUN apt-get install -y rsyslog
 
-# Install Python 3.8
+# Install Python 3.12
 # hadolint ignore=DL3033
-RUN apt-get install -y gcc python3 python3-dev python3-pip python3-virtualenv \
-                       python3-wheel python3.8-venv
+RUN apt-get install -y gcc python3 python3-dev python3-pip python3.12-venv python3-virtualenv python3-wheel python3-magic
 
 # Upload PAM Python library
 COPY stage/pam_python.so /tmp/pam_python.so
@@ -58,44 +55,12 @@ RUN install -m 0755 -o root -g root /tmp/pam_python.so  /usr/lib/x86_64-linux-gn
 # Install PostgreSQL 15 for ODBC drivers, checks whether database container is up, as well as troubleshooting
 # hadolint ignore=DL3033
 RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
-RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ focal-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list
+RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ noble-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list
 RUN apt-get update
-RUN apt-get -y install postgresql-15 unixodbc python3-psycopg2 acl
+RUN apt-get -y install postgresql-15 unixodbc unixodbc-dev odbcinst python3-psycopg2 acl
 COPY odbc-settings.txt /tmp/odbc-settings.txt
 RUN odbcinst -i -d -r < /tmp/odbc-settings.txt
 
-# Install iRODS dependencies
-RUN apt-get -y install aptitude libboost-locale-dev libpython2-stdlib libpython2.7-minimal \
-          libpython2.7-stdlib \
-          python-is-python2 python-six python2 python2-minimal python2.7 python2.7-minimal \
-          python-certifi python-chardet python-idna python-pkg-resources python-setuptools \
-          python-funcsigs python-mock python-pbr libodbc1
-ENV PY_URLLIB_PREFIX="http://security.ubuntu.com/ubuntu/pool/main/p/python-urllib3"
-ENV PY_URLLIB_FILENAME="python-urllib3_1.22-1ubuntu0.18.04.2_all.deb"
-ENV PY_REQUESTS_PREFIX="http://security.ubuntu.com/ubuntu/pool/main/r/requests"
-ENV PY_REQUESTS_FILENAME="python-requests_2.18.4-2ubuntu0.1_all.deb"
-ENV OPENSSL_PREFIX="http://security.ubuntu.com/ubuntu/pool/main/o/openssl1.0"
-ENV OPENSSL_FILENAME="libssl1.0.0_1.0.2n-1ubuntu5.13_amd64.deb"
-ENV PY_JSONSCHEMA_PREFIX="http://security.ubuntu.com/ubuntu/pool/main/p/python-jsonschema"
-ENV PY_JSONSCHEMA_FILENAME="python-jsonschema_2.3.0-1build1_all.deb"
-ENV PY_ODBC_PREFIX="http://security.ubuntu.com/ubuntu/pool/universe/p/pyodbc"
-ENV PY_ODBC_FILENAME="python-pyodbc_4.0.17-1_amd64.deb"
-RUN wget -q ${PY_URLLIB_PREFIX}/${PY_URLLIB_FILENAME} && \
-    wget -q ${PY_REQUESTS_PREFIX}/${PY_REQUESTS_FILENAME} && \
-    wget -q ${OPENSSL_PREFIX}/${OPENSSL_FILENAME} && \
-    wget -q ${PY_JSONSCHEMA_PREFIX}/${PY_JSONSCHEMA_FILENAME} && \
-    wget -q ${PY_ODBC_PREFIX}/${PY_ODBC_FILENAME} && \
-    dpkg -i ${PY_URLLIB_FILENAME} && \
-    dpkg -i ${PY_REQUESTS_FILENAME} && \
-    dpkg -i ${OPENSSL_FILENAME} && \
-    dpkg -i ${PY_JSONSCHEMA_FILENAME} && \
-    dpkg -i ${PY_ODBC_FILENAME} && \
-    rm ${PY_URLLIB_FILENAME} && \
-    rm ${PY_REQUESTS_FILENAME} && \
-    rm ${OPENSSL_FILENAME} && \
-    rm ${PY_JSONSCHEMA_FILENAME} && \
-    rm ${PY_ODBC_FILENAME}
-
 # Install iRODS
 # hadolint ignore=DL3033,DL3047
 RUN useradd -d /var/lib/irods irods && \
@@ -106,13 +71,10 @@ RUN useradd -d /var/lib/irods irods && \
     do echo "Installing package ${package} and its dependencies ..."; \
        apt-get -y install "$package=${IRODS_VERSION}" ; \
     done && \
-    for package in irods-rule-engine-plugin-indexing irods-rule-engine-plugin-elasticsearch irods-rule-engine-plugin-document-type; \
+    for package in irods-rule-engine-plugin-python irods-rule-engine-plugin-indexing irods-rule-engine-plugin-elasticsearch; \
     do echo "Installing package ${package} and its dependencies ..."; \
-       apt-get -y install "$package=${IRODS_INDEX_VERSION}";  \
-       done && \
-    apt-get -y install "irods-rule-engine-plugin-python=${IRODS_PREP_VERSION}"  && \
-    wget https://yoda.uu.nl/irods-patches/libirods_server.so.4.2.12.ubuntu2004 && \
-    install -m 0644 libirods_server.so.4.2.12.ubuntu2004 /usr/lib/libirods_server.so.4.2.12
+       apt-get -y install "$package=${IRODS_PLUS_VERSION}";  \
+       done
 
 # Install iRODS microservices for Yoda
 # hadolint ignore=DL3033
@@ -135,20 +97,10 @@ RUN install -m 0644 -o root -g root /tmp/irods_completion.sh /etc/profile.d/irod
 
 # Set up PAM authentication
 # hadolint ignore=DL3033
-RUN python -m easy_install --user https://github.com/UtrechtUniversity/pysqlcipher3/releases/download/v1.2.1/pysqlcipher3-1.2.1-py2.7-linux-x86_64.egg
 COPY is-user-external.sh /tmp/is-user-external.sh
 COPY external-auth.py /tmp/external-auth.py
 RUN install -m 0750 -o root -g irods /tmp/is-user-external.sh /usr/local/bin/is-user-external.sh && \
     install -m 0750 -o root -g irods /tmp/external-auth.py /usr/local/bin/external-auth.py && \
-    wget -q https://yoda.uu.nl/packages-irods-4.2.12-focal/sqlcipher4.deb && \
-    dpkg -i sqlcipher4.deb && \
-    rm sqlcipher4.deb && \
-    wget -q https://yoda.uu.nl/packages-irods-4.2.12-focal/libsqlcipher4.deb && \
-    dpkg -i libsqlcipher4.deb && \
-    rm libsqlcipher4.deb && \
-    wget -q https://yoda.uu.nl/packages-irods-4.2.12-focal/libsqlcipher4-dev.deb && \
-    dpkg -i libsqlcipher4-dev.deb && \
-    rm libsqlcipher4-dev.deb && \
     chmod 04755 /usr/sbin/irodsPamAuthCheck
 COPY irods.pam /tmp/irods.pam
 RUN install -m 0644 /tmp/irods.pam /etc/pam.d/irods
@@ -161,19 +113,18 @@ COPY irods_environment.json /var/lib/irods/.irods/irods_environment.json
 # Install and configure rulesets
 RUN git clone -b ${YODA_RULESET_BRANCH} ${YODA_RULESET_REPO} /etc/irods/yoda-ruleset
 # hadolint ignore=DL3004
-RUN sudo -u irods python -m pip --no-cache-dir install --user pip==20.2.4 && \
-    sudo -u irods python -m pip --no-cache-dir install --user setuptools==44.1.1 && \
-    sudo -u irods python -m pip --no-cache-dir install --user python-irodsclient==2.2.0 && \
-    sudo -u irods python -m pip --no-cache-dir install --user -r /etc/irods/yoda-ruleset/requirements.txt && \
-    sudo -u irods python -m easy_install --user https://github.com/UtrechtUniversity/pysqlcipher3/releases/download/v1.2.1/pysqlcipher3-1.2.1-py2.7-linux-x86_64.egg && \
-    sudo -u irods python3 -m pip install --user jsonschema==4.17.3 && \
-    sudo -u irods python3 -m pip install --user python-irodsclient==2.2.0 && \
-    sudo -u irods python3 -m pip install --user psutil==5.9.5
+RUN sudo -u irods python3 -m pip --no-cache-dir install --user --break-system-packages pip==24.3.1 && \
+    sudo -u irods python3 -m pip --no-cache-dir install --user --break-system-packages -r /etc/irods/yoda-ruleset/requirements.txt && \
+    sudo -u irods python3 -m pip install --user --break-system-packages python-irodsclient==2.2.0 && \
+    sudo -u irods python3 -m pip install --user --break-system-packages psutil==6.1.0
 COPY core.py.template /etc/irods/core.py
 COPY core.re.template /etc/irods/core.re
-RUN for script in scheduled-copytovault.sh admin-remove-orphan-vault-if-empty.sh admin-vaultactions.sh \
-                  admin-vaultingest.sh enable-indexing.sh disable-indexing.sh  \
-                  admin-datarequestactions.sh admin-datarequest-temp-write-permission.sh ; \
+RUN apt-get -y install sqlcipher libsqlcipher1 libsqlcipher-dev
+RUN for script in admin-scheduled-copytovault.sh admin-remove-orphan-vault-if-empty.sh admin-vaultactions.sh \
+    admin-vaultingest.sh admin-datarequestactions.sh admin-datarequest-temp-write-permission.sh ; \
+    do install -m 0755 /etc/irods/yoda-ruleset/tools/admin/${script} /var/lib/irods/msiExecCmd_bin/${script} ; \
+    done
+RUN for script in enable-indexing.sh disable-indexing.sh; \
     do install -m 0755 /etc/irods/yoda-ruleset/tools/${script} /var/lib/irods/msiExecCmd_bin/${script} ; \
     done && \
     /etc/irods/yoda-ruleset/tools/setup_tokens.sh /etc/irods/yoda-ruleset/accesstokens.db test
@@ -195,9 +146,9 @@ RUN chown -R irods:irods /var/lib/irods /etc/irods
 # Pre-build ruleset dependencies
 RUN test "$PRE_BUILD_RULESET_DEPENDENCIES" = "yes" && git clone --branch "$YODA_RULESET_BRANCH" "$YODA_RULESET_REPO" /var/lib/irods/yoda-ruleset
 # hadolint ignore=DL3004
-RUN test "$PRE_BUILD_RULESET_DEPENDENCIES" = "yes" && sudo -u irods pip2 install --user attrs==21.4.0
+RUN test "$PRE_BUILD_RULESET_DEPENDENCIES" = "yes" && sudo -u irods pip3 install --user --break-system-packages attrs==24.2.0
 # hadolint ignore=DL3004
-RUN test "$PRE_BUILD_RULESET_DEPENDENCIES" = "yes" && sudo -u irods pip2 install --user -r /var/lib/irods/yoda-ruleset/requirements.txt
+RUN test "$PRE_BUILD_RULESET_DEPENDENCIES" = "yes" && sudo -u irods pip3 install --user --break-system-packages -r /var/lib/irods/yoda-ruleset/requirements.txt
 
 # Create test accounts
 # hadolint ignore=SC2016