diff --git a/README.md b/README.md index 03a385df5..67fb374c3 100644 --- a/README.md +++ b/README.md @@ -173,3 +173,11 @@ Follow the instructions here https://django-q2.readthedocs.io/en/master/schedule 3. func = `django.core.management.call_command` 4. args = `"delete_expired_data"` 5. save + + +## Vector databases + +We are currently using ElasticSearch as our vector database. + +We have also successfully deployed Redbox to OpenSearch Serverless but this support should be considered experimental +at this stage. \ No newline at end of file diff --git a/django_app/poetry.lock b/django_app/poetry.lock index 2bac2d00c..fa8a462b7 100644 --- a/django_app/poetry.lock +++ b/django_app/poetry.lock @@ -1808,6 +1808,16 @@ pyarrow = ["pyarrow (>=1)"] requests = ["requests (>=2.4.0,!=2.32.2,<3.0.0)"] vectorstore-mmr = ["numpy (>=1)", "simsimd (>=3)"] +[[package]] +name = "events" +version = "0.5" +description = "Bringing the elegance of C# EventHandler to Python" +optional = false +python-versions = "*" +files = [ + {file = "Events-0.5-py3-none-any.whl", hash = "sha256:a7286af378ba3e46640ac9825156c93bdba7502174dd696090fdfcd4d80a1abd"}, +] + [[package]] name = "filelock" version = "3.16.1" @@ -1954,19 +1964,41 @@ optional = false python-versions = ">=3.9" files = [ {file = "gevent-24.10.2-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:562b66d8b061b9cfae1bc704b0cd5d2b255628d86c3639ddc16e4ffa3ebf6e7a"}, + {file = "gevent-24.10.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:44dd79cfefea24f9bb630844a25047c3807e02722436e826ef2aed3d646190c1"}, + {file = "gevent-24.10.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fa190663f964583c8dbbab06bc863966e6f7eceaac8aa67c3ac0fae0a0a73b80"}, + {file = "gevent-24.10.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aabffb8b86fb95cb1ee5dffa315c9bd68fe20a7fe7260c0328679723b0257b7c"}, {file = "gevent-24.10.2-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:a6a04df4732bb7fdf9969ddee9a16a829e7971692fefdcb5baca760976d23e04"}, + {file = "gevent-24.10.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:103097b39764a0a02f1a051225ea6b4c64a53dd37603424ca8a1e09be63a460b"}, {file = "gevent-24.10.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:539af6b66c6b9faca2cdd903f0a7564c85053f1faf95e9a37702df578ac37085"}, + {file = "gevent-24.10.2-cp310-cp310-win_amd64.whl", hash = "sha256:dd9c966e5fd8d7b0a54a130c5ad38ef581fd93ff4c44b6e73767519860da6ebe"}, {file = "gevent-24.10.2-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:a72a7cb67764adafbac7ddeeffe539a738309068e2b2ac89cbd2f498383ce537"}, + {file = "gevent-24.10.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b9da562d7d7707d5561ecf4a27a361fd9f4856f39b8491a0753c89d8f39674c"}, + {file = "gevent-24.10.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4e3fbaf484ee68437f0ec589bdb1dd6f1dccc01fd6b72eac707e858b407521fa"}, + {file = "gevent-24.10.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:22bc6efb0f9fbb1c2e005ef1b94374568945c711bfb92f85916f66a819a5e6d0"}, {file = "gevent-24.10.2-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:5d1db7bc758455e6f6406d66e8b276b80dda5645877392a100d1ed7dda6aa7ad"}, + {file = "gevent-24.10.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bc181db59d53e407650ebf44e63ff429c7bc25f9c346edddce1bdff1af436617"}, {file = "gevent-24.10.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:44174aa4dae4db158e6f11a4ea696f1991d43ccc1634aa0c189daf03a9ced5d7"}, + {file = "gevent-24.10.2-cp311-cp311-win_amd64.whl", hash = "sha256:f57b7a02e83d6e4a205cace6dd63e16b61a641a1da9366d9ec4f2b849430700f"}, {file = "gevent-24.10.2-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:ec800c25f09a7e031f2fbc3b17b4a4a0b54085c7532ac51b4c7ecef6d3ff8fc3"}, + {file = "gevent-24.10.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:013150cc0f00f0a06dd898463ad9ebc43bd9c70c7fe35555c77d83fe6f758225"}, + {file = "gevent-24.10.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4f0e6c49aac1c182be15a43d94e3b58c253d830c5b54dc93d6130e6987278611"}, + {file = "gevent-24.10.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:73b65ee9a73a35fb68d96899895162beef19d86c1bcbe6f8f92eb0bd18c1d891"}, {file = "gevent-24.10.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:618c4869e8140fd955b4620b10bc5a92ef1d62ae20aef38c1af7d892ee1bd996"}, + {file = "gevent-24.10.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:377c02d0ddae3ebf843d6f453943602102bb186b09f1c78a2247e5dbf0e07b1c"}, {file = "gevent-24.10.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:0814a5a7084e0bd357392e44e2a8bd72fc56fbdc3da0ff492ebb310c10fc95e6"}, + {file = "gevent-24.10.2-cp312-cp312-win_amd64.whl", hash = "sha256:f0d6cfff74be4efcafecd374e094a8fed9e0d68efe90109d374ef5d8f18aa21a"}, {file = "gevent-24.10.2-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:9f74faefea1acb398f057ed31ee9333e100bdae978b1e4c3b6a27d05df66e588"}, + {file = "gevent-24.10.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:16bf432b274795b360d88b38cbffe0a6410450c94bfa172548bf1f512cf448c2"}, + {file = "gevent-24.10.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7b5f10ac866d3432a829a3a4446489be1fa3648f3140f9373fe99440a2e05682"}, + {file = "gevent-24.10.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:90f9bc542f76efc56e5e76b420abaff42baf585db48a9fc0ac8edd6a16d9e60f"}, {file = "gevent-24.10.2-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:975699ac5701d7ec1c633f2067deecea8711dc2a8683530aed260dd641274791"}, + {file = "gevent-24.10.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:5bb80c88f572a11156f258333c0e7b1f80d0746a03784600017901a2f1aa584a"}, {file = "gevent-24.10.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:fa4cba4a8acbb71dd4215be8517879e4217c0746f7af2637330e7269694f53f2"}, + {file = "gevent-24.10.2-cp313-cp313-win_amd64.whl", hash = "sha256:fd9b670da1b7160e660cbba7f52e206892b97f61d8ff1872ce99dfaa9b475420"}, {file = "gevent-24.10.2-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:6a93f249a40bda8c42cbeefff9582b22bb1dd769da56b4cbb824038366c4202c"}, {file = "gevent-24.10.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a11db551555c58606ed3dfe359a9a502e44350ed3ecbd59cbe7b0093bd020418"}, + {file = "gevent-24.10.2-cp39-cp39-win32.whl", hash = "sha256:81b4915081d148a31b64ad0314d2f609920b8ae6a24d9a7e4ddaab7c1fe998e7"}, + {file = "gevent-24.10.2-cp39-cp39-win_amd64.whl", hash = "sha256:1a5012b7d047b16470063f0b8d003530e77362809f38cd7e601efb625c7ca71e"}, {file = "gevent-24.10.2-pp310-pypy310_pp73-macosx_11_0_universal2.whl", hash = "sha256:421cfeacae2555b11318c6ee11f34bc0a9517657068d8911c916d55a85362ce2"}, {file = "gevent-24.10.2.tar.gz", hash = "sha256:96e7bab9de56e0aca3858b8bc9c71f4eb0c0e12b7cf3cbfd170b62ce68cf71d7"}, ] @@ -3049,6 +3081,30 @@ typing-extensions = ">=4.11,<5" [package.extras] datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"] +[[package]] +name = "opensearch-py" +version = "2.7.1" +description = "Python client for OpenSearch" +optional = false +python-versions = "<4,>=3.8" +files = [ + {file = "opensearch_py-2.7.1-py3-none-any.whl", hash = "sha256:5417650eba98a1c7648e502207cebf3a12beab623ffe0ebbf55f9b1b4b6e44e9"}, + {file = "opensearch_py-2.7.1.tar.gz", hash = "sha256:67ab76e9373669bc71da417096df59827c08369ac3795d5438c9a8be21cbd759"}, +] + +[package.dependencies] +certifi = ">=2024.07.04" +Events = "*" +python-dateutil = "*" +requests = ">=2.32.0,<3.0.0" +urllib3 = {version = ">=1.26.19,<2.2.0 || >2.2.0,<2.2.1 || >2.2.1,<3", markers = "python_version >= \"3.10\""} + +[package.extras] +async = ["aiohttp (>=3.9.4,<4)"] +develop = ["black (>=24.3.0)", "botocore", "coverage (<8.0.0)", "jinja2", "myst-parser", "pytest (>=3.0.0)", "pytest-cov", "pytest-mock (<4.0.0)", "pytz", "pyyaml", "requests (>=2.0.0,<3.0.0)", "sphinx", "sphinx-copybutton", "sphinx-rtd-theme"] +docs = ["aiohttp (>=3.9.4,<4)", "myst-parser", "sphinx", "sphinx-copybutton", "sphinx-rtd-theme"] +kerberos = ["requests-kerberos"] + [[package]] name = "orjson" version = "3.10.7" @@ -5406,4 +5462,4 @@ testing = ["coverage[toml]", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = ">=3.12,<3.13" -content-hash = "e4a5860644b30910e2d3bec8c34d49c2b7cc1127baf2634b7c97ae5aedead111" +content-hash = "8cffd3cfd05cbbad32941b45a3a6e4c375709c78915802c9602d144364790bee" diff --git a/django_app/pyproject.toml b/django_app/pyproject.toml index 86853baf6..78c28b772 100644 --- a/django_app/pyproject.toml +++ b/django_app/pyproject.toml @@ -43,6 +43,7 @@ django-plotly-dash = "^2.3.1" django-adminplus = "^0.6" pandas = "^2.2.2" django-waffle = "^4.1.0" +opensearch-py = "^2.7.1" [tool.poetry.group.dev.dependencies] pytest = "^8.3.2" diff --git a/infrastructure/aws/data.tf b/infrastructure/aws/data.tf index db47fd300..8aa1624cc 100644 --- a/infrastructure/aws/data.tf +++ b/infrastructure/aws/data.tf @@ -49,8 +49,7 @@ locals { } django_app_secrets = { - "ELASTIC__API_KEY" : var.elastic_api_key, - "ELASTIC__CLOUD_ID" : var.cloud_id, + "ELASTIC__COLLECTION_ENDPOINT": module.opensearch.collection_enpdoint, "AZURE_OPENAI_API_KEY": var.azure_openai_api_key, "AZURE_OPENAI_ENDPOINT" : var.azure_openai_endpoint, diff --git a/infrastructure/aws/iam.tf b/infrastructure/aws/iam.tf index e79c561f7..fedc4c6d7 100644 --- a/infrastructure/aws/iam.tf +++ b/infrastructure/aws/iam.tf @@ -46,6 +46,13 @@ data "aws_iam_policy_document" "ecs_exec_role_policy" { ] } + # Add this for OpenSearchServerless access + # statement { + # effect = "Allow" + # actions = ["aoss:*"] + # resources = ["*"] restrict this + # } + } resource "aws_iam_policy" "redbox_policy" { diff --git a/infrastructure/aws/opensearch.txt b/infrastructure/aws/opensearch.txt new file mode 100644 index 000000000..f3421d13c --- /dev/null +++ b/infrastructure/aws/opensearch.txt @@ -0,0 +1,13 @@ +# rename this file to opensearch.tf to use opensearch + +module "opensearch" { + # checkov:skip=CKV_TF_1: We're using semantic versions instead of commit hash + # source = "../../../i-ai-core-infrastructure//modules/opensearch" # For testing local changes + source = "git::https://github.com/i-dot-ai/i-ai-core-infrastructure.git//modules/opensearch" + account_id = var.account_id + collection_name = "${var.team_name}-${terraform.workspace}-${var.project_name}-collection" + type = "SEARCH" + environment = var.env + region = var.region + state_bucket = var.state_bucket +} diff --git a/redbox-core/poetry.lock b/redbox-core/poetry.lock index 7e28e15d8..007bce30e 100644 --- a/redbox-core/poetry.lock +++ b/redbox-core/poetry.lock @@ -1207,6 +1207,16 @@ pyarrow = ["pyarrow (>=1)"] requests = ["requests (>=2.4.0,!=2.32.2,<3.0.0)"] vectorstore-mmr = ["numpy (>=1)", "simsimd (>=3)"] +[[package]] +name = "events" +version = "0.5" +description = "Bringing the elegance of C# EventHandler to Python" +optional = false +python-versions = "*" +files = [ + {file = "Events-0.5-py3-none-any.whl", hash = "sha256:a7286af378ba3e46640ac9825156c93bdba7502174dd696090fdfcd4d80a1abd"}, +] + [[package]] name = "execnet" version = "2.1.1" @@ -2437,6 +2447,30 @@ typing-extensions = ">=4.11,<5" [package.extras] datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"] +[[package]] +name = "opensearch-py" +version = "2.7.1" +description = "Python client for OpenSearch" +optional = false +python-versions = "<4,>=3.8" +files = [ + {file = "opensearch_py-2.7.1-py3-none-any.whl", hash = "sha256:5417650eba98a1c7648e502207cebf3a12beab623ffe0ebbf55f9b1b4b6e44e9"}, + {file = "opensearch_py-2.7.1.tar.gz", hash = "sha256:67ab76e9373669bc71da417096df59827c08369ac3795d5438c9a8be21cbd759"}, +] + +[package.dependencies] +certifi = ">=2024.07.04" +Events = "*" +python-dateutil = "*" +requests = ">=2.32.0,<3.0.0" +urllib3 = {version = ">=1.26.19,<2.2.0 || >2.2.0,<2.2.1 || >2.2.1,<3", markers = "python_version >= \"3.10\""} + +[package.extras] +async = ["aiohttp (>=3.9.4,<4)"] +develop = ["black (>=24.3.0)", "botocore", "coverage (<8.0.0)", "jinja2", "myst-parser", "pytest (>=3.0.0)", "pytest-cov", "pytest-mock (<4.0.0)", "pytz", "pyyaml", "requests (>=2.0.0,<3.0.0)", "sphinx", "sphinx-copybutton", "sphinx-rtd-theme"] +docs = ["aiohttp (>=3.9.4,<4)", "myst-parser", "sphinx", "sphinx-copybutton", "sphinx-rtd-theme"] +kerberos = ["requests-kerberos"] + [[package]] name = "opentelemetry-api" version = "1.24.0" @@ -4306,4 +4340,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.12,<3.13" -content-hash = "1a658f0bc7655ff0d11e46a385fc606d0e02d8b954ea30d249cb64c114c498ef" +content-hash = "85c58bcffd81b642728fe2a2d4578494738fac9890e77d3b904aedb3f7e9c217" diff --git a/redbox-core/pyproject.toml b/redbox-core/pyproject.toml index bdfb28292..446ef16c5 100644 --- a/redbox-core/pyproject.toml +++ b/redbox-core/pyproject.toml @@ -25,6 +25,7 @@ pytest-dotenv = "^0.5.2" kneed = "^0.8.5" langgraph = "^0.2.15" langchain-aws = "^0.1.17" +opensearch-py = "^2.7.1" [tool.poetry.group.dev.dependencies] diff --git a/redbox-core/redbox/models/settings.py b/redbox-core/redbox/models/settings.py index aa12c5200..ee25a955c 100644 --- a/redbox-core/redbox/models/settings.py +++ b/redbox-core/redbox/models/settings.py @@ -7,11 +7,20 @@ from elasticsearch import Elasticsearch from pydantic import BaseModel from pydantic_settings import BaseSettings, SettingsConfigDict - +from opensearchpy import OpenSearch, RequestsHttpConnection from redbox.models.chain import ChatLLMBackend + logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO")) -log = logging.getLogger() +logger = logging.getLogger() + + +class OpenSearchSettings(BaseModel): + """settings required for a aws/opensearch""" + + model_config = SettingsConfigDict(frozen=True) + + collection_enpdoint: str class ElasticLocalSettings(BaseModel): @@ -66,7 +75,7 @@ class Settings(BaseSettings): partition_strategy: Literal["auto", "fast", "ocr_only", "hi_res"] = "fast" clustering_strategy: Literal["full"] | None = None - elastic: ElasticCloudSettings | ElasticLocalSettings = ElasticLocalSettings() + elastic: ElasticCloudSettings | ElasticLocalSettings | OpenSearchSettings = ElasticLocalSettings() elastic_root_index: str = "redbox-data" elastic_chunk_alias: str = "redbox-data-chunk-current" @@ -131,6 +140,16 @@ def elasticsearch_client(self) -> Elasticsearch: ], basic_auth=(self.elastic.user, self.elastic.password), ) + + elif isinstance(self.elastic, OpenSearchSettings): + client = OpenSearch( + hosts=[{"host": self.collection_enpdoint, "port": 443}], + use_ssl=True, + verify_certs=True, + connection_class=RequestsHttpConnection, + pool_maxsize=100, + ) + else: client = Elasticsearch(cloud_id=self.elastic.cloud_id, api_key=self.elastic.api_key)