Feature/1351 - elastic (#2160)

* install django-elasticsearch-dsl-drf * set up elastic infra * add search indexes app and wire up t1 submission type * fix elastic impl, configure remaining submission models * update settings * add docker dns env variable for elastic * fix elastic version locally * update example env * rm default elastic host * code cleanup and linting * fix test * signed auth headers for elastic requests in non-local environments * update elastic settings * update aws auth impl * update auth impl (inconsistent docs) * ws * unused import * pin deps * actually pin deps * update dsl for django 3 support * removed unused boto3 connection * add model mapping tests * pinned version note * rebuild search indexes on dev startup * damn linter * set env default * Update tdrs-backend/tdpservice/search_indexes/test/test_model_mapping.py Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> * fix pipfile, permissions conflicts * move index rebuild to all envs * reorganize tests * relock deps * remove extra document fields * update field lengths Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> Co-authored-by: Alex P <63075587+ADPennington@users.noreply.github.com>
raft-tech · Dec 1, 2022 · 014056a · 014056a
1 parent aa54f3f
commit 014056a
Show file tree

Hide file tree

Showing 20 changed files with 1,621 additions and 53 deletions.
diff --git a/tdrs-backend/.env.example b/tdrs-backend/.env.example
@@ -80,3 +80,6 @@ AWS_REGION_NAME=us-gov-west-1
 # Database startup script options
 POSTGRES_CHECK_TIMEOUT=30
 POSTGRES_CHECK_INTERVAL=1
+
+# Elastic stack
+ELASTIC_HOST=elastic:9200
diff --git a/tdrs-backend/Dockerfile b/tdrs-backend/Dockerfile
@@ -8,7 +8,7 @@ ARG gid=1000
 ENV DJANGO_SETTINGS_MODULE=tdpservice.settings.local
 ENV DJANGO_CONFIGURATION=Local
 # Allows docker to cache installed dependencies between builds
-COPY Pipfile /tdpapp/
+COPY Pipfile Pipfile.lock /tdpapp/
 WORKDIR /tdpapp/
 # Download latest listing of available packages:
 RUN apt-get -y update
@@ -17,7 +17,8 @@ RUN apt-get -y upgrade
 # Install a new package:
 RUN apt-get install -y gcc && apt-get install -y graphviz && apt-get install -y graphviz-dev
 # Install pipenv
-RUN pip install --upgrade pip pipenv && pipenv install --dev --system --deploy --skip-lock
+RUN pip install --upgrade pip pipenv
+RUN pipenv install --dev --system --deploy
 
 # Adds our application code to the image
 COPY . /tdpapp

diff --git a/tdrs-backend/Pipfile b/tdrs-backend/Pipfile
@@ -52,6 +52,10 @@ flower = "==1.1.0"
 django-celery-beat = "==2.2.1"
 paramiko = "==2.11.0"
 pytest_sftpserver = "==1.3.0"
+elasticsearch = "==7.13.4" # REQUIRED - v7.14.0 introduces breaking changes
+django-elasticsearch-dsl = "==7.1.1"
+django-elasticsearch-dsl-drf = "==0.22.5"
+requests-aws4auth = "==1.1.2"
 
 [requires]
 python_version = "3.10.8"
diff --git a/tdrs-backend/Pipfile.lock b/tdrs-backend/Pipfile.lock
diff --git a/tdrs-backend/docker-compose.local.yml b/tdrs-backend/docker-compose.local.yml
@@ -31,6 +31,18 @@ services:
       # Copy in the Localstack setup script to configure any buckets needed
       - ../scripts/localstack-setup.sh:/docker-entrypoint-initaws.d/localstack-setup.sh
 
+  elastic:
+    image: elasticsearch:7.17.6
+    environment:
+      - discovery.type=single-node
+      - xpack.security.enabled=false
+      - logger.discovery.level=debug
+    ports:
+      - 9200:9200
+      - 9300:9300
+    volumes:
+      - elastic_data:/usr/share/elasticsearch/data
+
   web:
     restart: always
     env_file:
@@ -88,3 +100,4 @@ services:
 volumes:
   localstack_data:
   postgres_data:
+  elastic_data:
diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml
@@ -45,6 +45,18 @@ services:
       # Copy in the Localstack setup script to configure any buckets needed
       - ../scripts/localstack-setup.sh:/docker-entrypoint-initaws.d/localstack-setup.sh
 
+  elastic:
+    image: elasticsearch:7.17.6
+    environment:
+      - discovery.type=single-node
+      - xpack.security.enabled=false
+      - logger.discovery.level=debug
+    ports:
+      - 9200:9200
+      - 9300:9300
+    volumes:
+      - elastic_data:/usr/share/elasticsearch/data
+
   web:
     restart: always
     environment:
@@ -100,3 +112,4 @@ services:
 volumes:
   localstack_data:
   postgres_data:
+  elastic_data:
diff --git a/tdrs-backend/gunicorn_start.sh b/tdrs-backend/gunicorn_start.sh
@@ -25,6 +25,9 @@ sleep 5
 # celery -A tdpservice.settings --broker=$REDIS_URI flower &
 celery -A tdpservice.settings beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler &
 
+# rebuild and repopulate elastic search indexes
+python manage.py search_index --rebuild -f
+
 echo "Starting Gunicorn"
 if [[ "$DJANGO_CONFIGURATION" = "Development" || "$DJANGO_CONFIGURATION" = "Local" ]]; then
     gunicorn_params="--bind 0.0.0.0:8080 --timeout 10 --workers 3 --reload --log-level $LOGGING_LEVEL"

diff --git a/tdrs-backend/tdpservice/search_indexes/__init__.py b/tdrs-backend/tdpservice/search_indexes/__init__.py