feature/opensearch (#1114)

README.md

@@ -173,3 +173,11 @@ Follow the instructions here https://django-q2.readthedocs.io/en/master/schedule
 3. func = `django.core.management.call_command`
 4. args = `"delete_expired_data"`
 5. save
+
+
+## Vector databases
+
+We are currently using ElasticSearch as our vector database. 
+
+We have also successfully deployed Redbox to OpenSearch Serverless but this support should be considered experimental 
+at this stage.

django_app/pyproject.toml

@@ -43,6 +43,7 @@ django-plotly-dash = "^2.3.1"
 django-adminplus = "^0.6"
 pandas = "^2.2.2"
 django-waffle = "^4.1.0"
+opensearch-py = "^2.7.1"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^8.3.2"

infrastructure/aws/data.tf

@@ -49,8 +49,7 @@ locals {
   }
 
   django_app_secrets = {
-    "ELASTIC__API_KEY" : var.elastic_api_key,
-    "ELASTIC__CLOUD_ID" : var.cloud_id,
+    "ELASTIC__COLLECTION_ENDPOINT": module.opensearch.collection_enpdoint,
 
     "AZURE_OPENAI_API_KEY": var.azure_openai_api_key,
     "AZURE_OPENAI_ENDPOINT" : var.azure_openai_endpoint,

infrastructure/aws/iam.tf

@@ -46,6 +46,13 @@ data "aws_iam_policy_document" "ecs_exec_role_policy" {
     ]
   }
 
+  # Add this for OpenSearchServerless access
+  # statement {
+  #  effect = "Allow"
+  #  actions = ["aoss:*"]
+  #  resources = ["*"] restrict this
+  # }
+
 }
 
 resource "aws_iam_policy" "redbox_policy" {

infrastructure/aws/opensearch.txt

@@ -0,0 +1,13 @@
+# rename this file to opensearch.tf to use opensearch
+
+module "opensearch" {
+  # checkov:skip=CKV_TF_1: We're using semantic versions instead of commit hash
+  # source = "../../../i-ai-core-infrastructure//modules/opensearch"  # For testing local changes
+  source          = "git::https://github.com/i-dot-ai/i-ai-core-infrastructure.git//modules/opensearch"
+  account_id      = var.account_id
+  collection_name = "${var.team_name}-${terraform.workspace}-${var.project_name}-collection"
+  type            = "SEARCH"
+  environment     = var.env
+  region          = var.region
+  state_bucket    = var.state_bucket
+}

redbox-core/pyproject.toml

@@ -25,6 +25,7 @@ pytest-dotenv = "^0.5.2"
 kneed = "^0.8.5"
 langgraph = "^0.2.15"
 langchain-aws = "^0.1.17"
+opensearch-py = "^2.7.1"
 
 
 [tool.poetry.group.dev.dependencies]

redbox-core/redbox/models/settings.py

@@ -7,11 +7,20 @@
 from elasticsearch import Elasticsearch
 from pydantic import BaseModel
 from pydantic_settings import BaseSettings, SettingsConfigDict
-
+from opensearchpy import OpenSearch, RequestsHttpConnection
 from redbox.models.chain import ChatLLMBackend
 
+
 logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO"))
-log = logging.getLogger()
+logger = logging.getLogger()
+
+
+class OpenSearchSettings(BaseModel):
+    """settings required for a aws/opensearch"""
+
+    model_config = SettingsConfigDict(frozen=True)
+
+    collection_enpdoint: str
 
 
 class ElasticLocalSettings(BaseModel):
@@ -66,7 +75,7 @@ class Settings(BaseSettings):
     partition_strategy: Literal["auto", "fast", "ocr_only", "hi_res"] = "fast"
     clustering_strategy: Literal["full"] | None = None
 
-    elastic: ElasticCloudSettings | ElasticLocalSettings = ElasticLocalSettings()
+    elastic: ElasticCloudSettings | ElasticLocalSettings | OpenSearchSettings = ElasticLocalSettings()
     elastic_root_index: str = "redbox-data"
     elastic_chunk_alias: str = "redbox-data-chunk-current"
 
@@ -131,6 +140,16 @@ def elasticsearch_client(self) -> Elasticsearch:
                 ],
                 basic_auth=(self.elastic.user, self.elastic.password),
             )
+
+        elif isinstance(self.elastic, OpenSearchSettings):
+            client = OpenSearch(
+                hosts=[{"host": self.collection_enpdoint, "port": 443}],
+                use_ssl=True,
+                verify_certs=True,
+                connection_class=RequestsHttpConnection,
+                pool_maxsize=100,
+            )
+
         else:
             client = Elasticsearch(cloud_id=self.elastic.cloud_id, api_key=self.elastic.api_key)