diff --git a/dags/collection_generator.py b/dags/collection_generator.py index aff76d9..589d549 100644 --- a/dags/collection_generator.py +++ b/dags/collection_generator.py @@ -115,6 +115,16 @@ def configure_dag(**kwargs): dag=dag, ) + # listed-building / tree-preservation-order need a bigger Fargate TASK (not just the container) + task_size_override = ( + { + "cpu": '{{ task_instance.xcom_pull(task_ids="configure-dag", key="cpu") | string }}', + "memory": '{{ task_instance.xcom_pull(task_ids="configure-dag", key="memory") | string }}', + } + if collection in ("listed-building", "tree-preservation-order") + else {} + ) + collect_ecs_task = EcsRunTaskOperator( task_id=f"{collection}-collect", dag=dag, @@ -123,6 +133,7 @@ def configure_dag(**kwargs): task_definition=collection_task_name, launch_type="FARGATE", overrides={ + **task_size_override, "containerOverrides": [ { "name": collection_task_name, @@ -151,7 +162,7 @@ def configure_dag(**kwargs): {"name": "REPROCESS", "value": '\'{{ task_instance.xcom_pull(task_ids="configure-dag", key="force-reprocessing") }}\''}, ], }, - ] + ], }, network_configuration={"awsvpcConfiguration": '{{ task_instance.xcom_pull(task_ids="configure-dag", key="aws_vpc_config") }}'}, awslogs_group='{{ task_instance.xcom_pull(task_ids="configure-dag", key="collection-task-log-group") }}', @@ -198,6 +209,7 @@ def configure_dag(**kwargs): task_definition=collection_task_name, launch_type="FARGATE", overrides={ + **task_size_override, "containerOverrides": [ { "name": collection_task_name, @@ -226,7 +238,7 @@ def configure_dag(**kwargs): {"name": "REGENERATE_LOG_OVERRIDE", "value": '\'{{ task_instance.xcom_pull(task_ids="configure-dag", key="regenerate-log-override") | string }}\''}, ], }, - ] + ], }, network_configuration={"awsvpcConfiguration": '{{ task_instance.xcom_pull(task_ids="configure-dag", key="aws_vpc_config") }}'}, awslogs_group='{{ task_instance.xcom_pull(task_ids="configure-dag", key="collection-task-log-group") }}', diff --git a/dags/dag_triggers.py b/dags/dag_triggers.py index 650920f..ef2f3ca 100644 --- a/dags/dag_triggers.py +++ b/dags/dag_triggers.py @@ -56,8 +56,8 @@ def collection_selected(collection_name, configuration): if collection not in ["organisation"]: if collection_selected(collection, config): - # Set custom CPU for listed-building collection - conf = {"cpu": 16384, "transformed-jobs": 16} if collection in ("listed-building", "tree-preservation-order") else {} + # Set custom CPU and RAM for listed-building collection + conf = {"cpu": 16384, "memory": 65536, "transformed-jobs": 16} if collection in ("listed-building", "tree-preservation-order") else {} collection_dag = TriggerDagRunOperator( task_id=f"trigger-{collection}-collection-dag", @@ -98,8 +98,8 @@ def collection_selected(collection_name, configuration): for collection, datasets in collections.items(): if collection not in ["organisation"]: - # Set custom CPU for listed-building collection - conf = {"cpu": 16384, "transformed-jobs": 16} if collection in ("listed-building", "tree-preservation-order") else {} + # Set custom CPU and RAM for listed-building collection + conf = {"cpu": 16384, "memory": 65536, "transformed-jobs": 16} if collection in ("listed-building", "tree-preservation-order") else {} collection_dag = TriggerDagRunOperator(task_id=f"trigger-{collection}-collection-dag", trigger_dag_id=f"{collection}-collection", wait_for_completion=True, conf=conf) collection_tasks.append(collection_dag)