local setup now mirrors remote; adds new celery compose config and ma…

…ke commands

Makefile

@@ -1,10 +1,16 @@
 .DEFAULT_GOAL := help
 
-up: ## Brings up local docker airflow instance
+up: ## Brings up docker airflow instance using LocalExecutor
 	docker-compose up -d
 
-down: ## Shuts down local docker airflow instance
+down: ## Shuts down local docker airflow instance using LocalExecutor
 	docker-compose down
+
+up-celery: ## Brings up docker airflow instance using CeleryExecutor
+	docker-compose -f docker-compose_celery.yml up -d 
+
+down-celery: ## Shuts down local docker airflow instance using CeleryExecutor
+	docker-compose -f docker-compose_celery.yml down
 
 scale-up: ## Scales up CF airflow test deploy in current CF space
 	cf scale airflow-test-scheduler -i 2

docker-compose.yml

@@ -46,15 +46,13 @@ x-airflow-common: &airflow-common
     image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:2.7.3}
     # build: .
     environment: &airflow-common-env
-        AIRFLOW__CORE__EXECUTOR: CeleryExecutor
+        AIRFLOW__CORE__EXECUTOR: LocalExecutor
         AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow
         # For backward compatibility, with Airflow <2.3
         ## ORIGINAL
         # AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow
         ## ALTERED TO TEST
         AIRFLOW__CORE__SQL_ALCHEMY_CONN: sql_alchemy_conn = postgresql+psycopg2://postgres:1234@localhost:5432/test
-        AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@postgres/airflow
-        AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0
         AIRFLOW__CORE__FERNET_KEY: ''
         AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'false'
         AIRFLOW__API__AUTH_BACKENDS: 'airflow.api.auth.backend.basic_auth'
@@ -67,8 +65,6 @@ x-airflow-common: &airflow-common
 
     user: '${AIRFLOW_UID:-50000}:0'
     depends_on: &airflow-common-depends-on
-        redis:
-            condition: service_healthy
         postgres:
             condition: service_healthy
 
@@ -87,17 +83,6 @@ services:
             retries: 5
         restart: always
 
-    redis:
-        image: redis:latest
-        expose:
-            - 6379
-        healthcheck:
-            test: ['CMD', 'redis-cli', 'ping']
-            interval: 5s
-            timeout: 30s
-            retries: 50
-        restart: always
-
     airflow-webserver:
         <<: *airflow-common
         command: webserver
@@ -128,41 +113,6 @@ services:
             airflow-init:
                 condition: service_completed_successfully
 
-    airflow-worker:
-        <<: *airflow-common
-        command: celery worker
-        healthcheck:
-            test:
-                - 'CMD-SHELL'
-                - 'celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"'
-            interval: 10s
-            timeout: 10s
-            retries: 5
-        environment:
-            <<: *airflow-common-env
-            # Required to handle warm shutdown of the celery workers properly
-            # See https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation
-            DUMB_INIT_SETSID: '0'
-        restart: always
-        depends_on:
-            <<: *airflow-common-depends-on
-            airflow-init:
-                condition: service_completed_successfully
-
-    airflow-triggerer:
-        <<: *airflow-common
-        command: triggerer
-        healthcheck:
-            test: ['CMD-SHELL', 'airflow jobs check --job-type TriggererJob --hostname "$${HOSTNAME}"']
-            interval: 10s
-            timeout: 10s
-            retries: 5
-        restart: always
-        depends_on:
-            <<: *airflow-common-depends-on
-            airflow-init:
-                condition: service_completed_successfully
-
     airflow-init:
         <<: *airflow-common
         entrypoint: /bin/bash
@@ -253,21 +203,5 @@ services:
             - -c
             - airflow
 
-    flower:
-        <<: *airflow-common
-        command: celery flower
-        ports:
-            - 5555:5555
-        healthcheck:
-            test: ['CMD', 'curl', '--fail', 'http://localhost:5555/']
-            interval: 10s
-            timeout: 10s
-            retries: 5
-        restart: always
-        depends_on:
-            <<: *airflow-common-depends-on
-            airflow-init:
-                condition: service_completed_successfully
-
 volumes:
     postgres-db-volume: