update GHA for allowing staging

.github/workflows/deploy.yml

@@ -1,8 +1,8 @@
 name: Deploy dags to aws
 
 on:
-  # push:
-  #   branches: [main]
+  push:
+    branches: [main]
   workflow_dispatch:
     inputs:
       environment:
@@ -28,8 +28,13 @@ jobs:
 
   test:
     runs-on: ubuntu-latest
+    needs: [detect-environments]
+    strategy:
+      matrix:
+        environment: ${{ fromJSON(needs.detect-environments.outputs.environments) }}
+    environment: ${{ matrix.environment }}
     env:
-      ENVIRONMENT: development
+      ENVIRONMENT: ${{ matrix.environment }}
     steps:
       - uses: actions/checkout@v4
 
@@ -39,6 +44,11 @@ jobs:
 
       - run: |
           pip install -r requirements/dev-requirements.txt
+        if: ${{ matrix.environment == 'development' }}
+
+      - run: |
+          pip install -r requirements/requirements.txt
+        if: ${{ matrix.environment != 'development' }}
 
       - run: |
           make dags/config.json
@@ -65,6 +75,11 @@ jobs:
 
       - run: |
           pip install -r requirements/dev-requirements.txt
+        if: ${{ matrix.environment == 'development' }}
+
+      - run: |
+          pip install -r requirements/requirements.txt
+        if: ${{ matrix.environment != 'development' }}
 
       - run: |
           make dags/config.json

requirements/requirements.in

@@ -1,3 +1,6 @@
 apache-airflow==2.9.2
 apache-airflow-providers-amazon
-boto
+boto
+pydantic
+pytest
+moto

requirements/requirements.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.10
+# This file is autogenerated by pip-compile with Python 3.12
 # by the following command:
 #
 #    pip-compile --output-file=requirements/requirements.txt requirements/requirements.in
@@ -12,6 +12,8 @@ aiosignal==1.3.1
     # via aiohttp
 alembic==1.13.2
     # via apache-airflow
+annotated-types==0.7.0
+    # via pydantic
 anyio==4.4.0
     # via httpx
 apache-airflow==2.9.2
@@ -63,8 +65,6 @@ asgiref==3.8.1
     #   apache-airflow-providers-http
 asn1crypto==1.5.1
     # via scramp
-async-timeout==4.0.3
-    # via aiohttp
 attrs==24.2.0
     # via
     #   aiohttp
@@ -82,13 +82,15 @@ boto==2.49.0
 boto3==1.35.3
     # via
     #   apache-airflow-providers-amazon
+    #   moto
     #   pyathena
     #   redshift-connector
     #   watchtower
 botocore==1.35.3
     # via
     #   apache-airflow-providers-amazon
     #   boto3
+    #   moto
     #   pyathena
     #   redshift-connector
     #   s3transfer
@@ -125,7 +127,9 @@ cron-descriptor==1.4.3
 croniter==3.0.3
     # via apache-airflow
 cryptography==43.0.0
-    # via apache-airflow
+    # via
+    #   apache-airflow
+    #   moto
 deprecated==1.2.14
     # via
     #   apache-airflow
@@ -142,8 +146,6 @@ docutils==0.21.2
     # via python-daemon
 email-validator==2.2.0
     # via flask-appbuilder
-exceptiongroup==1.2.2
-    # via anyio
 flask==2.2.5
     # via
     #   apache-airflow
@@ -197,8 +199,6 @@ googleapis-common-protos==1.63.2
     # via
     #   opentelemetry-exporter-otlp-proto-grpc
     #   opentelemetry-exporter-otlp-proto-http
-greenlet==3.0.3
-    # via sqlalchemy
 grpcio==1.65.5
     # via opentelemetry-exporter-otlp-proto-grpc
 gunicorn==23.0.0
@@ -217,15 +217,15 @@ idna==3.7
     #   requests
     #   yarl
 importlib-metadata==8.0.0
-    # via
-    #   apache-airflow
-    #   opentelemetry-api
+    # via opentelemetry-api
 importlib-resources==6.4.4
     # via limits
 inflection==0.5.1
     # via
     #   apache-airflow-providers-amazon
     #   connexion
+iniconfig==2.0.0
+    # via pytest
 itsdangerous==2.2.0
     # via
     #   apache-airflow
@@ -237,6 +237,7 @@ jinja2==3.1.4
     #   apache-airflow
     #   flask
     #   flask-babel
+    #   moto
     #   python-nvd3
 jmespath==1.0.1
     # via
@@ -296,6 +297,8 @@ methodtools==0.4.7
     # via apache-airflow
 more-itertools==10.4.0
     # via apache-airflow-providers-common-sql
+moto==5.0.16
+    # via -r requirements/requirements.in
 multidict==6.0.5
     # via
     #   aiohttp
@@ -339,14 +342,17 @@ packaging==24.1
     #   limits
     #   marshmallow
     #   marshmallow-sqlalchemy
+    #   pytest
     #   redshift-connector
     #   sqlalchemy-redshift
 pathspec==0.12.1
     # via apache-airflow
 pendulum==3.0.0
     # via apache-airflow
 pluggy==1.5.0
-    # via apache-airflow
+    # via
+    #   apache-airflow
+    #   pytest
 ply==3.11
     # via jsonpath-ng
 prison==0.2.1
@@ -361,6 +367,10 @@ pyathena==3.9.0
     # via apache-airflow-providers-amazon
 pycparser==2.22
     # via cffi
+pydantic==2.9.2
+    # via -r requirements/requirements.in
+pydantic-core==2.23.4
+    # via pydantic
 pygments==2.18.0
     # via
     #   apache-airflow
@@ -370,6 +380,8 @@ pyjwt==2.9.0
     #   apache-airflow
     #   flask-appbuilder
     #   flask-jwt-extended
+pytest==8.3.3
+    # via -r requirements/requirements.in
 python-daemon==3.0.1
     # via apache-airflow
 python-dateutil==2.9.0.post0
@@ -378,6 +390,7 @@ python-dateutil==2.9.0.post0
     #   botocore
     #   croniter
     #   flask-appbuilder
+    #   moto
     #   pendulum
     #   pyathena
     #   time-machine
@@ -397,6 +410,7 @@ pyyaml==6.0.2
     #   apispec
     #   clickclick
     #   connexion
+    #   responses
 redshift-connector==2.1.3
     # via apache-airflow-providers-amazon
 referencing==0.35.1
@@ -408,11 +422,15 @@ requests==2.32.3
     #   apache-airflow
     #   apache-airflow-providers-http
     #   connexion
+    #   moto
     #   opentelemetry-exporter-otlp-proto-http
     #   redshift-connector
     #   requests-toolbelt
+    #   responses
 requests-toolbelt==1.0.0
     # via apache-airflow-providers-http
+responses==0.25.3
+    # via moto
 rfc3339-validator==0.1.4
     # via apache-airflow
 rich==13.7.1
@@ -477,11 +495,11 @@ time-machine==2.15.0
 typing-extensions==4.12.2
     # via
     #   alembic
-    #   anyio
-    #   asgiref
     #   flask-limiter
     #   limits
     #   opentelemetry-sdk
+    #   pydantic
+    #   pydantic-core
 tzdata==2024.1
     # via pendulum
 uc-micro-py==1.0.3
@@ -494,6 +512,7 @@ urllib3==2.2.2
     # via
     #   botocore
     #   requests
+    #   responses
 watchtower==3.3.1
     # via apache-airflow-providers-amazon
 werkzeug==2.2.3
@@ -504,6 +523,7 @@ werkzeug==2.2.3
     #   flask-appbuilder
     #   flask-jwt-extended
     #   flask-login
+    #   moto
 wirerope==0.4.7
     # via methodtools
 wrapt==1.16.0
@@ -512,6 +532,8 @@ wtforms==3.1.2
     # via
     #   flask-appbuilder
     #   flask-wtf
+xmltodict==0.13.0
+    # via moto
 yarl==1.9.4
     # via aiohttp
 zipp==3.20.0