Upped timeout and machine spec.

digital-land · Jul 24, 2024 · 4848e67 · 4848e67
1 parent 49b9441
commit 4848e67
Showing 1 changed file with 3 additions and 45 deletions.
diff --git a/dags/dynamic_data_workflow_dag.py b/dags/dynamic_data_workflow_dag.py
@@ -32,9 +32,9 @@
         },
     ],
     register_task_kwargs={
-        "cpu": "256",
+        "cpu": "1024",
         "executionRoleArn": "arn:aws:iam::955696714113:role/development-mwaa-execution-role",
-        "memory": "512",
+        "memory": "8192",
         "networkMode": "awsvpc",
         "requiresCompatibilities": ["FARGATE"],
     },
@@ -68,7 +68,7 @@
         EcsRunTaskOperator(
             task_id="collection",
             dag=dag,
-            execution_timeout=timedelta(minutes=5),
+            execution_timeout=timedelta(minutes=10),
             # retries=3,
             # aws_conn_id="aws_default",
             cluster=cluster_name,
@@ -96,45 +96,3 @@
             awslogs_stream_prefix="ecs",
             # awslogs_fetch_interval=timedelta(seconds=5)
         )
-
-
-    """
-    with DAG(
-        "Fargate",
-        default_args=DEFAULT_ARGS,
-        description="A test DAG to try out functionality",
-        schedule=None,
-    ) as dag:
-        EcsRunTaskOperator(
-            task_id="fargate-test",
-            dag=dag,
-            execution_timeout=timedelta(minutes=5),
-            # retries=3,
-            # aws_conn_id="aws_default",
-            cluster=cluster_name,
-            task_definition=register_task.output,
-            launch_type="FARGATE",
-            overrides={
-                "containerOverrides": [
-                    {
-                        "name": "hello",
-                        "command": [
-                            "echo",
-                            "hello",
-                        ],  # python", "-c", "import time; for i in range(30): print(i); time.sleep(1)"],
-                    },
-                ]
-            },
-            network_configuration={
-                "awsvpcConfiguration": {
-                    "subnets": ["subnet-05a0d548ea8d901ab", "subnet-07252405b5369afd3"],
-                    "securityGroups": ["sg-0fe390dd951829c75"],
-                    "assignPublicIp": "ENABLED",
-                }
-            },
-            awslogs_group="airflow-development-mwaa-Task",
-            awslogs_region="eu-west-1",
-            # awslogs_stream_prefix=f"ecs/test",
-            awslogs_fetch_interval=timedelta(seconds=5),
-        )
-    """