Transfer the project from the SCC to the NHR partition

.env

@@ -4,8 +4,7 @@ OPERANDI_DB_ROOT_PASS=db_operandi
 OPERANDI_DB_URL=mongodb://db_operandi:db_operandi@localhost:27017
 OPERANDI_HARVESTER_DEFAULT_USERNAME=harvester_operandi
 OPERANDI_HARVESTER_DEFAULT_PASSWORD=harvester_operandi
-OPERANDI_HPC_USERNAME=mmustaf
-OPERANDI_HPC_PROJECT_USERNAME=u11874
+OPERANDI_HPC_PROJECT_USERNAME=u12198
 OPERANDI_HPC_SSH_KEYPATH=/home/mm/.ssh/gwdg-cluster
 OPERANDI_HPC_PROJECT_NAME=operandi
 OPERANDI_LOGS_DIR=/tmp/operandi_logs

.github/workflows/tests/.env

@@ -4,10 +4,9 @@ OPERANDI_DB_ROOT_PASS=db_operandi
 OPERANDI_DB_URL=mongodb://db_operandi:db_operandi@localhost:27017
 OPERANDI_HARVESTER_DEFAULT_USERNAME=harvester_operandi
 OPERANDI_HARVESTER_DEFAULT_PASSWORD=harvester_operandi
-OPERANDI_HPC_USERNAME=mmustaf
-OPERANDI_HPC_PROJECT_USERNAME=u11874
+OPERANDI_HPC_PROJECT_USERNAME=u12198
 OPERANDI_HPC_SSH_KEYPATH=/home/runner/.ssh/key_hpc
-OPERANDI_HPC_PROJECT_NAME=operandi
+OPERANDI_HPC_PROJECT_NAME=operandi_test_cicd
 OPERANDI_LOGS_DIR=/tmp/operandi_logs
 OPERANDI_RABBITMQ_CONFIG_JSON=./src/rabbitmq_definitions.json
 OPERANDI_RABBITMQ_URL=amqp://operandi_user:operandi_password@localhost:5672/

docker-compose.yml

@@ -31,8 +31,8 @@ services:
     healthcheck:
       test: rabbitmq-diagnostics check_port_connectivity
       interval: 1s
-      timeout: 3s
-      retries: 30
+      timeout: 5s
+      retries: 120
 
   operandi-mongodb:
     image: "mongo"
@@ -50,15 +50,15 @@ services:
     healthcheck:
       test: echo 'db.runCommand("ping").ok' | mongosh localhost:27017/test --quiet
       interval: 1s
-      timeout: 3s
-      retries: 30
+      timeout: 5s
+      retries: 120
 
   operandi-server:
     image: operandi-server
     container_name: operandi-server
     build:
       context: ./src
-      dockerfile: ./Dockerfile_server  
+      dockerfile: ./Dockerfile_server
     depends_on:
       operandi-rabbitmq:
         condition: service_healthy
@@ -102,7 +102,6 @@ services:
       - OPERANDI_DB_NAME=${OPERANDI_DB_NAME}
       - OPERANDI_DB_URL=${OPERANDI_DB_URL}
       - OPERANDI_HPC_PROJECT_NAME=${OPERANDI_HPC_PROJECT_NAME}
-      - OPERANDI_HPC_USERNAME=${OPERANDI_HPC_USERNAME}
       - OPERANDI_HPC_PROJECT_USERNAME=${OPERANDI_HPC_PROJECT_USERNAME}
       - OPERANDI_HPC_SSH_KEYPATH=/home/root/.ssh/gwdg_hpc_key
       - OPERANDI_LOGS_DIR=${OPERANDI_LOGS_DIR}

docker-compose_image_based.yml

@@ -31,8 +31,8 @@ services:
     healthcheck:
       test: rabbitmq-diagnostics check_port_connectivity
       interval: 1s
-      timeout: 3s
-      retries: 30
+      timeout: 5s
+      retries: 120
 
   operandi-mongodb:
     image: "mongo"
@@ -50,8 +50,8 @@ services:
     healthcheck:
       test: echo 'db.runCommand("ping").ok' | mongosh localhost:27017/test --quiet
       interval: 1s
-      timeout: 3s
-      retries: 30
+      timeout: 5s
+      retries: 120
 
   operandi-server:
     image: ghcr.io/subugoe/operandi-server:main
@@ -96,7 +96,6 @@ services:
       - OPERANDI_DB_NAME=${OPERANDI_DB_NAME}
       - OPERANDI_DB_URL=${OPERANDI_DB_URL}
       - OPERANDI_HPC_PROJECT_NAME=${OPERANDI_HPC_PROJECT_NAME}
-      - OPERANDI_HPC_USERNAME=${OPERANDI_HPC_USERNAME}
       - OPERANDI_HPC_PROJECT_USERNAME=${OPERANDI_HPC_PROJECT_USERNAME}
       - OPERANDI_HPC_SSH_KEYPATH=/home/root/.ssh/gwdg_hpc_key
       - OPERANDI_LOGS_DIR=${OPERANDI_LOGS_DIR}

docker.env

@@ -4,8 +4,7 @@ OPERANDI_DB_ROOT_PASS=db_operandi
 OPERANDI_DB_URL=mongodb://db_operandi:db_operandi@mongo-db-host:27017
 OPERANDI_HARVESTER_DEFAULT_USERNAME=harvester_operandi
 OPERANDI_HARVESTER_DEFAULT_PASSWORD=harvester_operandi
-OPERANDI_HPC_USERNAME=mmustaf
-OPERANDI_HPC_PROJECT_USERNAME=u11874
+OPERANDI_HPC_PROJECT_USERNAME=u12198
 OPERANDI_HPC_SSH_KEYPATH=/home/mm/.ssh/gwdg-cluster
 OPERANDI_HPC_PROJECT_NAME=operandi
 OPERANDI_LOGS_DIR=/tmp/operandi_logs

src/broker/operandi_broker/job_status_worker.py

@@ -10,7 +10,7 @@
     DBHPCSlurmJob, DBWorkflowJob, DBWorkspace,
     sync_db_initiate_database, sync_db_get_hpc_slurm_job, sync_db_get_workflow_job, sync_db_get_workspace,
     sync_db_update_hpc_slurm_job, sync_db_update_workflow_job, sync_db_update_workspace)
-from operandi_utils.hpc import HPCExecutor, HPCTransfer
+from operandi_utils.hpc import NHRExecutor, NHRTransfer
 from operandi_utils.rabbitmq import get_connection_consumer
 
 
@@ -51,9 +51,9 @@ def run(self):
             signal.signal(signal.SIGTERM, self.signal_handler)
 
             sync_db_initiate_database(self.db_url)
-            self.hpc_executor = HPCExecutor(tunnel_host='localhost', tunnel_port=self.tunnel_port_executor)
+            self.hpc_executor = NHRExecutor()
             self.log.info("HPC executor connection successful.")
-            self.hpc_io_transfer = HPCTransfer(tunnel_host='localhost', tunnel_port=self.tunnel_port_transfer)
+            self.hpc_io_transfer = NHRTransfer()
             self.log.info("HPC transfer connection successful.")
 
             self.rmq_consumer = get_connection_consumer(rabbitmq_url=self.rmq_url)

src/broker/operandi_broker/worker.py

@@ -10,10 +10,10 @@
 from operandi_utils.database import (
     sync_db_initiate_database, sync_db_get_workflow, sync_db_get_workspace, sync_db_create_hpc_slurm_job,
     sync_db_update_workflow_job, sync_db_update_workspace)
-from operandi_utils.hpc import HPCExecutor, HPCTransfer
+from operandi_utils.hpc import NHRExecutor, NHRTransfer
 from operandi_utils.hpc.constants import (
-    HPC_JOB_DEADLINE_TIME_REGULAR, HPC_JOB_DEADLINE_TIME_TEST, HPC_JOB_QOS_SHORT, HPC_JOB_QOS_DEFAULT
-)
+    HPC_BATCH_SUBMIT_WORKFLOW_JOB, HPC_JOB_DEADLINE_TIME_REGULAR, HPC_JOB_DEADLINE_TIME_TEST, HPC_JOB_QOS_SHORT,
+    HPC_JOB_QOS_DEFAULT)
 from operandi_utils.rabbitmq import get_connection_consumer
 
 
@@ -58,9 +58,9 @@ def run(self):
             signal.signal(signal.SIGTERM, self.signal_handler)
 
             sync_db_initiate_database(self.db_url)
-            self.hpc_executor = HPCExecutor(tunnel_host='localhost', tunnel_port=self.tunnel_port_executor)
+            self.hpc_executor = NHRExecutor()
             self.log.info("HPC executor connection successful.")
-            self.hpc_io_transfer = HPCTransfer(tunnel_host='localhost', tunnel_port=self.tunnel_port_transfer)
+            self.hpc_io_transfer = NHRTransfer()
             self.log.info("HPC transfer connection successful.")
 
             self.rmq_consumer = get_connection_consumer(rabbitmq_url=self.rmq_url)
@@ -214,8 +214,6 @@ def prepare_and_trigger_slurm_job(
         # self.hpc_io_transfer = HPCTransfer(tunel_host='localhost', tunel_port=4023)
         # self.log.info("HPC transfer connection renewed successfully.")
 
-        hpc_batch_script_path = self.hpc_io_transfer.put_batch_script(batch_script_id="batch_submit_workflow_job.sh")
-
         try:
             sync_db_update_workspace(find_workspace_id=workspace_id, state=StateWorkspace.TRANSFERRING_TO_HPC)
             sync_db_update_workflow_job(find_job_id=workflow_job_id, job_state=StateJob.TRANSFERRING_TO_HPC)
@@ -228,8 +226,8 @@ def prepare_and_trigger_slurm_job(
         try:
             # NOTE: The paths below must be a valid existing path inside the HPC
             slurm_job_id = self.hpc_executor.trigger_slurm_job(
-                batch_script_path=hpc_batch_script_path, workflow_job_id=workflow_job_id,
-                nextflow_script_path=workflow_script_path, workspace_id=workspace_id, mets_basename=workspace_base_mets,
+                workflow_job_id=workflow_job_id, nextflow_script_path=workflow_script_path,
+                workspace_id=workspace_id, mets_basename=workspace_base_mets,
                 input_file_grp=input_file_grp, nf_process_forks=nf_process_forks, ws_pages_amount=ws_pages_amount,
                 use_mets_server=use_mets_server, file_groups_to_remove=file_groups_to_remove, cpus=cpus, ram=ram,
                 job_deadline_time=job_deadline_time, partition=partition, qos=qos)
@@ -239,7 +237,7 @@ def prepare_and_trigger_slurm_job(
         try:
             sync_db_create_hpc_slurm_job(
                 workflow_job_id=workflow_job_id, hpc_slurm_job_id=slurm_job_id,
-                hpc_batch_script_path=hpc_batch_script_path,
+                hpc_batch_script_path=HPC_BATCH_SUBMIT_WORKFLOW_JOB,
                 hpc_slurm_workspace_path=join(self.hpc_io_transfer.slurm_workspaces_dir, workflow_job_id))
         except Exception as error:
             raise Exception(f"Failed to save the hpc slurm job in DB: {error}")

src/server/operandi_server/models/base.py

@@ -2,7 +2,7 @@
 from typing import Optional
 
 from operandi_utils import StateJob
-from operandi_utils.hpc.constants import HPC_JOB_DEFAULT_PARTITION
+from operandi_utils.hpc.constants import HPC_NHR_JOB_DEFAULT_PARTITION
 
 from ..constants import DEFAULT_FILE_GRP, DEFAULT_METS_BASENAME
 
@@ -35,6 +35,6 @@ class WorkflowArguments(BaseModel):
 
 
 class SbatchArguments(BaseModel):
-    partition: str = HPC_JOB_DEFAULT_PARTITION  # partition to be used
+    partition: str = HPC_NHR_JOB_DEFAULT_PARTITION  # partition to be used
     cpus: int = 4  # cpus per job allocated by default
     ram: int = 32  # RAM (in GB) per job allocated by default

src/utils/operandi_utils/hpc/__init__.py

@@ -1,9 +1,5 @@
-__all__ = [
-    "HPCConnector",
-    "HPCExecutor",
-    "HPCTransfer"
-]
+__all__ = ["NHRConnector", "NHRExecutor", "NHRTransfer"]
 
-from operandi_utils.hpc.connector import HPCConnector
-from operandi_utils.hpc.executor import HPCExecutor
-from operandi_utils.hpc.transfer import HPCTransfer
+from operandi_utils.hpc.nhr_connector import NHRConnector
+from operandi_utils.hpc.nhr_executor import NHRExecutor
+from operandi_utils.hpc.nhr_transfer import NHRTransfer

src/utils/operandi_utils/hpc/batch_scripts/batch_check_ocrd_all_version.sh

@@ -1,17 +1,20 @@
 #!/bin/bash
-#SBATCH --constraint scratch
-#SBATCH --partition medium
-#SBATCH --time 0:05:00
-#SBATCH --output /scratch1/projects/project_pwieder_ocr/batch_job_logs/batch_check_ocrd_all_version_job-%J.txt
+#SBATCH --partition standard96:shared
+#SBATCH --time 00:05:00
+#SBATCH --qos 2h
+#SBATCH --output check_ocrd_all_version_job-%J.txt
 #SBATCH --cpus-per-task 1
-#STABCH --mem 16G
+#SBATCH --mem 16G
 
 set -e
 
 hostname
 /opt/slurm/etc/scripts/misc/slurm_resources
 
 module purge
-module load singularity
+module load apptainer
+SIF_PATH="/mnt/lustre-emmy-hdd/projects/project_pwieder_ocr_nhr/ocrd_all_maximum_image.sif"
 
-singularity exec "/scratch1/projects/project_pwieder_ocr/ocrd_all_maximum_image.sif" ocrd --version
+apptainer exec "$SIF_PATH" ocrd --version
+apptainer exec "$SIF_PATH" ocrd-tesserocr-recognize --dump-module-dir
+apptainer exec "$SIF_PATH" ls -la /models

src/utils/operandi_utils/hpc/batch_scripts/batch_create_ocrd_all_maximum_sif.sh

@@ -1,23 +1,23 @@
 #!/bin/bash
-#SBATCH --constraint scratch
-#SBATCH --partition medium
-#SBATCH --time 02:00:00
-#SBATCH --output /scratch1/projects/project_pwieder_ocr/batch_job_logs/batch_create_ocrd_all_sif_job-%J.txt
+#SBATCH --partition standard96:shared
+#SBATCH --time 2:00:00
+#SBATCH --output create_ocrd_all_sif_job-%J.txt
 #SBATCH --cpus-per-task 16
 #SBATCH --mem 64G
 
 set -e
 
 module purge
-module load singularity
+module load apptainer
 
 hostname
 /opt/slurm/etc/scripts/misc/slurm_resources
 
-SINGULARITY_CACHE_DIR="/scratch1/projects/project_pwieder_ocr"
-SIF_NAME="ocrd_all_maximum_image.sif"
+APPTAINER_TMPDIR="$LOCAL_TMPDIR"
+APPTAINER_CACHE_DIR="/mnt/lustre-emmy-hdd/projects/project_pwieder_ocr_nhr"
+SIF_NAME="ocrd_all_maximum_image_new.sif"
 OCRD_ALL_MAXIMUM_IMAGE="docker://ocrd/all:latest"
 
-cd "${SINGULARITY_CACHE_DIR}" || exit
-singularity build --disable-cache "${SIF_NAME}" "${OCRD_ALL_MAXIMUM_IMAGE}"
-singularity exec "${SIF_NAME}" ocrd --version
+cd "${APPTAINER_CACHE_DIR}" || exit
+apptainer build --disable-cache "${SIF_NAME}" "${OCRD_ALL_MAXIMUM_IMAGE}"
+apptainer exec "${SIF_NAME}" ocrd --version

src/utils/operandi_utils/hpc/batch_scripts/batch_download_ocrd_all_models.sh

@@ -1,22 +1,21 @@
 #!/bin/bash
-#SBATCH --constraint scratch
-#SBATCH --partition medium
-#SBATCH --time 06:00:00
-#SBATCH --output /scratch1/projects/project_pwieder_ocr/batch_job_logs/batch_download_all_ocrd_models_job-%J.txt
+#SBATCH --partition standard96:shared
+#SBATCH --time 6:00:00
+#SBATCH --output download_all_ocrd_models_job-%J.txt
 #SBATCH --cpus-per-task 16
-#SBATCH --mem 64G
+#SBATCH --mem 32G
 
 set -e
 
 module purge
-module load singularity
+module load apptainer
 
 hostname
 /opt/slurm/etc/scripts/misc/slurm_resources
 
 # This sif file is generated with another batch script
-SIF_PATH="/scratch1/projects/project_pwieder_ocr/ocrd_all_maximum_image.sif"
-OCRD_MODELS_DIR="/scratch1/projects/project_pwieder_ocr/ocrd_models"
+SIF_PATH="/mnt/lustre-emmy-hdd/projects/project_pwieder_ocr_nhr/ocrd_all_maximum_image.sif"
+OCRD_MODELS_DIR="/mnt/lustre-emmy-hdd/projects/project_pwieder_ocr_nhr/ocrd_models"
 OCRD_MODELS_DIR_IN_DOCKER="/usr/local/share"
 
 if [ ! -f "${SIF_PATH}" ]; then
@@ -28,11 +27,9 @@ if [ ! -d "${OCRD_MODELS_DIR}" ]; then
   mkdir -p "${OCRD_MODELS_DIR}"
 fi
 
-# Download all available ocrd models
-singularity exec --bind "${OCRD_MODELS_DIR}:${OCRD_MODELS_DIR_IN_DOCKER}" "${SIF_PATH}" ocrd resmgr download '*'
-# Download models for ocrd-tesserocr-recognize which are not downloaded with the '*' glob
-singularity exec --bind "${OCRD_MODELS_DIR}:${OCRD_MODELS_DIR_IN_DOCKER}" "${SIF_PATH}" ocrd resmgr download ocrd-tesserocr-recognize '*'
-# Download models for ocrd-kraken-recognize which are not downloaded with the '*' glob
-singularity exec --bind "${OCRD_MODELS_DIR}:${OCRD_MODELS_DIR_IN_DOCKER}" "${SIF_PATH}" ocrd resmgr download ocrd-kraken-recognize '*'
-# Download models for ocrd-calamari-recognize which are not downloaded with the '*' glob
-singularity exec --bind "${OCRD_MODELS_DIR}:${OCRD_MODELS_DIR_IN_DOCKER}" "${SIF_PATH}" ocrd resmgr download ocrd-calamari-recognize '*'
+apptainer exec --bind "${OCRD_MODELS_DIR}:${OCRD_MODELS_DIR_IN_DOCKER}" "${SIF_PATH}" ocrd resmgr download -o '*'
+apptainer exec --bind "${OCRD_MODELS_DIR}:${OCRD_MODELS_DIR_IN_DOCKER}" "${SIF_PATH}" ocrd resmgr download -o ocrd-tesserocr-recognize '*'
+apptainer exec --bind "${OCRD_MODELS_DIR}:${OCRD_MODELS_DIR_IN_DOCKER}" "${SIF_PATH}" ocrd resmgr download -o ocrd-calamari-recognize '*'
+apptainer exec --bind "${OCRD_MODELS_DIR}:${OCRD_MODELS_DIR_IN_DOCKER}" "${SIF_PATH}" ocrd resmgr download -o ocrd-kraken-recognize '*'
+apptainer exec --bind "${OCRD_MODELS_DIR}:${OCRD_MODELS_DIR_IN_DOCKER}" "${SIF_PATH}" ocrd resmgr download -o ocrd-sbb-binarize '*'
+apptainer exec --bind "${OCRD_MODELS_DIR}:${OCRD_MODELS_DIR_IN_DOCKER}" "${SIF_PATH}" ocrd resmgr download -o ocrd-cis-ocropy-recognize '*'