Skip to content

Commit

Permalink
try different params
Browse files Browse the repository at this point in the history
  • Loading branch information
eveleighoj committed Aug 14, 2024
1 parent 519518f commit 8263a57
Showing 1 changed file with 11 additions and 4 deletions.
15 changes: 11 additions & 4 deletions dags/collection_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
EcsRegisterTaskDefinitionOperator,
EcsRunTaskOperator,
)
from airflow.models.param import Param

#TO-DO generate name from env
cluster_name = "development-cluster"
Expand Down Expand Up @@ -36,21 +37,27 @@
default_args=default_args,
description=f"Collection task for the {collection} collection",
schedule=None,
params={"cpu": 8192, "memory": 32768,"timeout":480}
params={
"cpu": Param(8192, type="integer"),
"memory": Param(32768, type="integer"),
"timeout": Param(480, type="integer")
},
render_template_as_native_obj=True
) as dag:
print(f'running with params:{dag.params}')
EcsRunTaskOperator(
task_id=f"{collection}-collection",
dag=dag,
execution_timeout=timedelta(minutes= dag.params.get('timeout')),
execution_timeout=timedelta(minutes= "{{ params.my_int_param }}"),
cluster=cluster_name,
task_definition="development-mwaa-collection-task",
launch_type="FARGATE",
overrides={
"containerOverrides": [
{
"name": "development-mwaa-collection-task",
'cpu': dag.params.get('cpu'),
'memory': dag.params.get('memory'),
'cpu': "{{ params.cpu }}",
'memory': "{{ params.memory }}",
"environment": [
{"name": "COLLECTION_NAME", "value": collection}
],
Expand Down

0 comments on commit 8263a57

Please sign in to comment.