From c377e7fcebcbf2b2752e937d9b797864329333ef Mon Sep 17 00:00:00 2001 From: Kalyan R Date: Tue, 8 Oct 2024 21:40:18 +0530 Subject: [PATCH] add warning log when task_key>100 (#42813) related to #41816 Adds a warning log to indicate failure if length of task_key>100. --- airflow/providers/databricks/operators/databricks.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/airflow/providers/databricks/operators/databricks.py b/airflow/providers/databricks/operators/databricks.py index d9256477f949..58ffaeaece58 100644 --- a/airflow/providers/databricks/operators/databricks.py +++ b/airflow/providers/databricks/operators/databricks.py @@ -1045,7 +1045,15 @@ def _get_hook(self, caller: str) -> DatabricksHook: def _get_databricks_task_id(self, task_id: str) -> str: """Get the databricks task ID using dag_id and task_id. Removes illegal characters.""" - return f"{self.dag_id}__{task_id.replace('.', '__')}" + task_id = f"{self.dag_id}__{task_id.replace('.', '__')}" + if len(task_id) > 100: + self.log.warning( + "The generated task_key '%s' exceeds 100 characters and will be truncated by the Databricks API. " + "This will cause failure when trying to monitor the task. task_key is generated by ", + "concatenating dag_id and task_id.", + task_id, + ) + return task_id @property def _databricks_workflow_task_group(self) -> DatabricksWorkflowTaskGroup | None: