Skip to content

Commit

Permalink
add warning log when task_key>100 (apache#42813)
Browse files Browse the repository at this point in the history
related to apache#41816

Adds a warning log to indicate failure if length of task_key>100.
  • Loading branch information
rawwar authored and pavansharma36 committed Oct 14, 2024
1 parent 1686c0c commit 0385d53
Showing 1 changed file with 9 additions and 1 deletion.
10 changes: 9 additions & 1 deletion airflow/providers/databricks/operators/databricks.py
Original file line number Diff line number Diff line change
Expand Up @@ -1045,7 +1045,15 @@ def _get_hook(self, caller: str) -> DatabricksHook:

def _get_databricks_task_id(self, task_id: str) -> str:
"""Get the databricks task ID using dag_id and task_id. Removes illegal characters."""
return f"{self.dag_id}__{task_id.replace('.', '__')}"
task_id = f"{self.dag_id}__{task_id.replace('.', '__')}"
if len(task_id) > 100:
self.log.warning(
"The generated task_key '%s' exceeds 100 characters and will be truncated by the Databricks API. "
"This will cause failure when trying to monitor the task. task_key is generated by ",
"concatenating dag_id and task_id.",
task_id,
)
return task_id

@property
def _databricks_workflow_task_group(self) -> DatabricksWorkflowTaskGroup | None:
Expand Down

0 comments on commit 0385d53

Please sign in to comment.