fix: adding exec_entrypoint script

raystack · Feb 28, 2023 · ce373c2 · ce373c2
1 parent 04919f5
commit ce373c2
Show file tree

Hide file tree

Showing 3 changed files with 38 additions and 26 deletions.
diff --git a/entrypoint_init_container.sh b/entrypoint_init_container.sh
@@ -22,3 +22,27 @@ optimus job run-input "$JOB_NAME" --project-name \
 	"$PROJECT" --output-dir "$JOB_DIR" \
 	--type "$INSTANCE_TYPE" --name "$INSTANCE_NAME" \
 	--scheduled-at "$SCHEDULED_AT" --host "$OPTIMUS_HOST"
+
+touch $JOB_DIR/exec_entrypoint.sh
+chmod 755 $JOB_DIR/exec_entrypoint.sh
+
+cat > $JOB_DIR/exec_entrypoint.sh <<EOF
+#!/bin/bash
+echo "exec entrypoint -------"
+# TODO: this doesnt support using back quote sign in env vars, fix it
+echo "-- exporting env"
+set -o allexport
+source "$JOB_DIR/in/.env"
+set +o allexport
+
+echo "-- current envs"
+cat "$JOB_DIR/in/.env"
+
+echo "-- exporting env with secret"
+set -o allexport
+source "$JOB_DIR/in/.secret"
+set +o allexport
+
+echo "-- running unit"
+exec $(eval echo "$@")
+EOF
diff --git a/ext/scheduler/airflow/dag/dag.py.tmpl b/ext/scheduler/airflow/dag/dag.py.tmpl
@@ -101,13 +101,7 @@ JOB_DIR = "/data"
 IMAGE_PULL_POLICY = "IfNotPresent"
 INIT_CONTAINER_IMAGE = "odpf/optimus:{{.Version}}"
 INIT_CONTAINER_ENTRYPOINT = "/opt/entrypoint_init_container.sh"
-
-def get_entrypoint_cmd(plugin_entrypoint):
-    path_config = JOB_DIR + "/in/.env"
-    path_secret = JOB_DIR + "/in/.secret"
-    entrypoint = "set -o allexport; source {path_config}; set +o allexport; cat {path_config}; ".format(path_config=path_config)
-    entrypoint += "set -o allexport; source {path_secret}; set +o allexport; ".format(path_secret=path_secret)
-    return entrypoint + plugin_entrypoint
+EXEC_CONTAINER_ENTRYPOINT = JOB_DIR + "/exec_entrypoint.sh"
 
 volume = k8s.V1Volume(
     name='asset-volume',
@@ -150,8 +144,8 @@ init_container = k8s.V1Container(
     image_pull_policy=IMAGE_PULL_POLICY,
     namespace=conf.get('kubernetes', 'namespace', fallback="default"),
     image={{  .Task.Image | quote}},
-    cmds=["/bin/sh"],
-    arguments=["-c", get_entrypoint_cmd("""{{.Task.Entrypoint}} """)],
+    cmds=[EXEC_CONTAINER_ENTRYPOINT],
+    arguments=["""{{.Task.Entrypoint}} """], # space inside the quote is required
     name="{{  .Task.Name | replace "_" "-" }}",
     task_id={{ .Task.Name | quote}},
     get_logs=True,
@@ -193,8 +187,8 @@ hook_{{$hookName}} = SuperKubernetesPodOperator(
     image_pull_policy=IMAGE_PULL_POLICY,
     namespace=conf.get('kubernetes', 'namespace', fallback="default"),
     image="{{ $t.Image }}",
-    cmds=["/bin/sh"],
-    arguments=["-c", get_entrypoint_cmd("""{{ $t.Entrypoint }} """)],
+    cmds=[EXEC_CONTAINER_ENTRYPOINT],
+    arguments=["""{{ $t.Entrypoint }} """], # space inside the quote is required
     name="hook_{{ $t.Name | replace "_" "-" }}",
     task_id="hook_{{ $t.Name }}",
     get_logs=True,

diff --git a/ext/scheduler/airflow/dag/expected_dag.py b/ext/scheduler/airflow/dag/expected_dag.py
@@ -71,13 +71,7 @@
 IMAGE_PULL_POLICY = "IfNotPresent"
 INIT_CONTAINER_IMAGE = "odpf/optimus:dev"
 INIT_CONTAINER_ENTRYPOINT = "/opt/entrypoint_init_container.sh"
-
-def get_entrypoint_cmd(plugin_entrypoint):
-    path_config = JOB_DIR + "/in/.env"
-    path_secret = JOB_DIR + "/in/.secret"
-    entrypoint = "set -o allexport; source {path_config}; set +o allexport; cat {path_config}; ".format(path_config=path_config)
-    entrypoint += "set -o allexport; source {path_secret}; set +o allexport; ".format(path_secret=path_secret)
-    return entrypoint + plugin_entrypoint
+EXEC_CONTAINER_ENTRYPOINT = JOB_DIR + "/exec_entrypoint.sh"
 
 volume = k8s.V1Volume(
     name='asset-volume',
@@ -119,8 +113,8 @@ def get_entrypoint_cmd(plugin_entrypoint):
     image_pull_policy=IMAGE_PULL_POLICY,
     namespace=conf.get('kubernetes', 'namespace', fallback="default"),
     image="example.io/namespace/bq2bq-executor:latest",
-    cmds=["/bin/sh"],
-    arguments=["-c", get_entrypoint_cmd("""python3 /opt/bumblebee/main.py """)],
+    cmds=[EXEC_CONTAINER_ENTRYPOINT],
+    arguments=["""python3 /opt/bumblebee/main.py """], # space inside the quote is required
     name="bq-bq",
     task_id="bq-bq",
     get_logs=True,
@@ -156,8 +150,8 @@ def get_entrypoint_cmd(plugin_entrypoint):
     image_pull_policy=IMAGE_PULL_POLICY,
     namespace=conf.get('kubernetes', 'namespace', fallback="default"),
     image="example.io/namespace/transporter-executor:latest",
-    cmds=["/bin/sh"],
-    arguments=["-c", get_entrypoint_cmd("""java -cp /opt/transporter/transporter.jar:/opt/transporter/jolokia-jvm-agent.jar -javaagent:jolokia-jvm-agent.jar=port=7777,host=0.0.0.0 com.gojek.transporter.Main """)],
+    cmds=[EXEC_CONTAINER_ENTRYPOINT],
+    arguments=["""java -cp /opt/transporter/transporter.jar:/opt/transporter/jolokia-jvm-agent.jar -javaagent:jolokia-jvm-agent.jar=port=7777,host=0.0.0.0 com.gojek.transporter.Main """], # space inside the quote is required
     name="hook_transporter",
     task_id="hook_transporter",
     get_logs=True,
@@ -189,8 +183,8 @@ def get_entrypoint_cmd(plugin_entrypoint):
     image_pull_policy=IMAGE_PULL_POLICY,
     namespace=conf.get('kubernetes', 'namespace', fallback="default"),
     image="example.io/namespace/predator-image:latest",
-    cmds=["/bin/sh"],
-    arguments=["-c", get_entrypoint_cmd("""predator ${SUB_COMMAND} -s ${PREDATOR_URL} -u "${BQ_PROJECT}.${BQ_DATASET}.${BQ_TABLE}" """)],
+    cmds=[EXEC_CONTAINER_ENTRYPOINT],
+    arguments=["""predator ${SUB_COMMAND} -s ${PREDATOR_URL} -u "${BQ_PROJECT}.${BQ_DATASET}.${BQ_TABLE}" """], # space inside the quote is required
     name="hook_predator",
     task_id="hook_predator",
     get_logs=True,
@@ -222,8 +216,8 @@ def get_entrypoint_cmd(plugin_entrypoint):
     image_pull_policy=IMAGE_PULL_POLICY,
     namespace=conf.get('kubernetes', 'namespace', fallback="default"),
     image="example.io/namespace/failure-hook-image:latest",
-    cmds=["/bin/sh"],
-    arguments=["-c", get_entrypoint_cmd("""sleep 5 """)],
+    cmds=[EXEC_CONTAINER_ENTRYPOINT],
+    arguments=["""sleep 5 """], # space inside the quote is required
     name="hook_failureHook",
     task_id="hook_failureHook",
     get_logs=True,