rucio: integrate sidecar container

AUTHORS.rst

@@ -4,6 +4,7 @@ Authors
 The list of contributors in alphabetical order:
 
 - `Adelina Lintuluoto <https://orcid.org/0000-0002-0726-1452>`_
+- `Agisilaos Kounelis <https://orcid.org/0000-0001-9312-3189>`_
 - `Anton Khodak <https://orcid.org/0000-0003-3263-4553>`_
 - `Diego Rodriguez <https://orcid.org/0000-0003-0649-2002>`_
 - `Dinos Kousidis <https://orcid.org/0000-0002-4914-4289>`_

CHANGES.rst

@@ -4,6 +4,7 @@ Changes
 Version 0.9.0 (UNRELEASED)
 --------------------------
 
+- Adds support for Rucio
 - Adds support for specifying ``slurm_partition`` and ``slurm_time`` for Slurm compute backend jobs.
 - Changes ``reana-auth-vomsproxy`` sidecar to latest version to support accessing ESCAPE VOMS.
 - Changes default Slurm partition to ``inf-short``.

docs/openapi.json

@@ -84,6 +84,9 @@
           "default": "",
           "type": "string"
         },
+        "rucio": {
+          "type": "boolean"
+        },
         "shared_file_system": {
           "default": true,
           "type": "boolean"

reana_job_controller/config.py

@@ -69,6 +69,25 @@
 VOMSPROXY_CERT_CACHE_FILENAME = "x509up_proxy"
 """Name of the voms-proxy certificate cache file."""
 
+RUCIO_CONTAINER_IMAGE = os.getenv(
+    "RUCIO_CONTAINER_IMAGE", "reanahub/reana-auth-rucio:1.0.0"
+)
+"""Default docker image of RUCIO sidecar container."""
+
+RUCIO_CONTAINER_NAME = "reana-auth-rucio"
+"""Name of RUCIO sidecar container."""
+
+RUCIO_CACHE_LOCATION = "/rucio_cache/"
+"""Directory of Rucio cache.
+
+This directory is shared between job & Rucio container."""
+
+RUCIO_CFG_CACHE_FILENAME = "rucio.cfg"
+"""Name of the RUCIO configuration cache file."""
+
+RUCIO_CERN_BUNDLE_CACHE_FILENAME = "CERN-bundle.pem"
+"""Name of the CERN Bundle cache file."""
+
 IMAGE_PULL_SECRETS = os.getenv("IMAGE_PULL_SECRETS", "").split(",")
 """Docker image pull secrets which allow the usage of private images."""
 

reana_job_controller/kubernetes_job_manager.py

@@ -76,6 +76,7 @@ def __init__(
         kubernetes_uid=None,
         kubernetes_memory_limit=None,
         voms_proxy=False,
+        rucio=False,
         kubernetes_job_timeout: Optional[int] = None,
         **kwargs,
     ):
@@ -110,6 +111,9 @@ def __init__(
         :param voms_proxy: Decides if a voms-proxy certificate should be
             provided for job.
         :type voms_proxy: bool
+        :param rucio: Decides if a rucio environment should be provided
+            for job.
+        :type rucio: bool
         """
         super(KubernetesJobManager, self).__init__(
             docker_img=docker_img,
@@ -125,6 +129,7 @@ def __init__(
         self.shared_file_system = shared_file_system
         self.kerberos = kerberos
         self.voms_proxy = voms_proxy
+        self.rucio = rucio
         self.set_user_id(kubernetes_uid)
         self.set_memory_limit(kubernetes_memory_limit)
         self.workflow_uuid = workflow_uuid
@@ -224,6 +229,9 @@ def execute(self):
         if self.voms_proxy:
             self._add_voms_proxy_init_container(secrets_volume_mount, secret_env_vars)
 
+        if self.rucio:
+            self._add_rucio_init_container(secrets_volume_mount, secret_env_vars)
+
         if REANA_RUNTIME_JOBS_KUBERNETES_NODE_LABEL:
             self.job["spec"]["template"]["spec"][
                 "nodeSelector"
@@ -351,7 +359,7 @@ def add_volumes(self, volumes):
             self.job["spec"]["template"]["spec"]["volumes"].append(volume)
 
     def _add_krb5_init_container(self, secrets_store):
-        """Add  sidecar container for a job."""
+        """Add sidecar container for a job."""
         krb5_config = get_kerberos_k8s_config(
             secrets_store,
             kubernetes_uid=self.kubernetes_uid,
@@ -372,7 +380,7 @@ def _add_krb5_init_container(self, secrets_store):
         )
 
     def _add_voms_proxy_init_container(self, secrets_volume_mount, secret_env_vars):
-        """Add  sidecar container for a job."""
+        """Add sidecar container for a job."""
         ticket_cache_volume = {"name": "voms-proxy-cache", "emptyDir": {}}
         volume_mounts = [
             {
@@ -426,6 +434,64 @@ def _add_voms_proxy_init_container(self, secrets_volume_mount, secret_env_vars):
             voms_proxy_container
         )
 
+    def _add_rucio_init_container(self, secrets_volume_mount, secret_env_vars):
+        """Add sidecar container for a job."""
+        ticket_cache_volume = {"name": "rucio-cache", "emptyDir": {}}
+        volume_mounts = [
+            {
+                "name": ticket_cache_volume["name"],
+                "mountPath": current_app.config["RUCIO_CACHE_LOCATION"],
+            }
+        ]
+
+        rucio_config_file_path = os.path.join(
+            current_app.config["RUCIO_CACHE_LOCATION"],
+            current_app.config["RUCIO_CFG_CACHE_FILENAME"],
+        )
+
+        cern_bundle_path = os.path.join(
+            current_app.config["RUCIO_CACHE_LOCATION"],
+            current_app.config["RUCIO_CERN_BUNDLE_CACHE_FILENAME"],
+        )
+
+        rucio_account = os.environ.get("RUCIO_USERNAME")
+        voms_proxy_vo = os.environ.get("VONAME")
+
+        rucio_config_container = {
+            "image": current_app.config["RUCIO_CONTAINER_IMAGE"],
+            "command": ["/bin/bash"],
+            "args": [
+                "-c",
+                "export RUCIO_CFG_ACCOUNT={rucio_account} \
+                    RUCIO_CFG_RUCIO_HOST=https://{voms_proxy_vo}-rucio.cern.ch \
+                    RUCIO_CFG_AUTH_HOST=https://{voms_proxy_vo}-rucio-auth.cern.ch; \
+                cp /etc/pki/tls/certs/CERN-bundle.pem {cern_bundle_path}; \
+                j2 /opt/user/rucio.cfg.j2 > {rucio_config_file_path}".format(
+                    rucio_account=rucio_account,
+                    voms_proxy_vo=voms_proxy_vo,
+                    cern_bundle_path=cern_bundle_path,
+                    rucio_config_file_path=rucio_config_file_path,
+                ),
+            ],
+            "name": current_app.config["RUCIO_CONTAINER_NAME"],
+            "imagePullPolicy": "IfNotPresent",
+            "volumeMounts": [secrets_volume_mount] + volume_mounts,
+            "env": secret_env_vars,
+        }
+
+        self.job["spec"]["template"]["spec"]["volumes"].extend([ticket_cache_volume])
+        self.job["spec"]["template"]["spec"]["containers"][0]["volumeMounts"].extend(
+            volume_mounts
+        )
+
+        self.job["spec"]["template"]["spec"]["containers"][0]["env"].append(
+            {"name": "RUCIO_CONFIG", "value": rucio_config_file_path}
+        )
+
+        self.job["spec"]["template"]["spec"]["initContainers"].append(
+            rucio_config_container
+        )
+
     def set_user_id(self, kubernetes_uid):
         """Set user id for job pods. UIDs < 100 are refused for security."""
         if kubernetes_uid and kubernetes_uid >= 100:

reana_job_controller/schemas.py

@@ -45,6 +45,7 @@ class JobRequest(Schema):
     compute_backend = fields.Str(required=False)
     kerberos = fields.Bool(required=False)
     voms_proxy = fields.Bool(required=False)
+    rucio = fields.Bool(required=False)
     kubernetes_uid = fields.Int(required=False)
     kubernetes_memory_limit = fields.Str(required=False)
     kubernetes_job_timeout = fields.Int(required=False)