Feature/core api pyproject (#730)

.github/workflows/python-app.yml

@@ -7,7 +7,7 @@ on:
   push:
     paths:
       - redbox/**
-      - core_api/**
+      - core-api/**
       - django_app/**
       - worker/**
       - Makefile
@@ -51,8 +51,6 @@ jobs:
         cp .env.test .env
         docker compose up -d --wait elasticsearch
 
-        poetry install --no-root --no-ansi --with dev --without ai,api,worker
-
     - name: Add dependencies for worker
       run: |
         sudo apt update && sudo apt-get install -y \
@@ -67,6 +65,30 @@ jobs:
       run: |
         make test-core-api
 
+  test-redbox:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.11"
+
+    - name: Set up Poetry
+      uses: abatilo/actions-poetry@v2
+      with:
+        poetry-version: 1.7.0
+
+    - name: Build Containers
+      run: |
+        mkdir -p data/elastic/
+        chmod 777 data/elastic/
+        cp .env.test .env
+        docker compose up -d --wait elasticsearch
+
     - name: Test redbox with pytest
       run: |
         make test-redbox

Makefile

@@ -34,22 +34,23 @@ rebuild: stop prune ## Rebuild all images
 
 .PHONY: test-core-api
 test-core-api: ## Test core-api
-	poetry install --no-root --no-ansi --with api,dev,ai --without worker,docs
-	poetry run pytest core_api/tests -m "not ai" --cov=core_api/src -v --cov-report=term-missing --cov-fail-under=75
+	cp .env.test core-api/.env
+	cd core-api && poetry install --with dev && poetry run python -m pytest -m "not ai" --cov=core_api -v --cov-report=term-missing --cov-fail-under=80
 
 .PHONY: test-ai
 test-ai: ## Test code with live LLM
-	poetry install --no-root --no-ansi --with api,dev,ai --without worker,docs
-	poetry run pytest core_api/tests -m "ai" -vv
+	cp .env.test core-api/.env
+	cd core-api && poetry install --with dev && poetry run python -m pytest -m "ai" --cov=core_api -v --cov-report=term-missing --cov-fail-under=80
 
 .PHONY: test-redbox
 test-redbox: ## Test redbox
 	cp .env.test redbox-core/.env
 	cd redbox-core && poetry install && poetry run pytest --cov=redbox -v --cov-report=term-missing --cov-fail-under=80
 
+
 .PHONY: test-worker
 test-worker: ## Test worker
-	poetry install --no-root --no-ansi --with worker,dev --without ai,api,docs
+	poetry install --no-root --no-ansi --with worker,dev --without docs
 	poetry run pytest worker/tests --cov=worker -v --cov-report=term-missing --cov-fail-under=40
 
 .PHONY: test-django
@@ -62,7 +63,7 @@ test-integration: rebuild run test-integration-without-build ## Run all integrat
 
 .PHONY: test-integration-without-build
 test-integration-without-build : ## Run all integration tests without rebuilding
-	poetry install --no-root --no-ansi --with dev --without ai,api,worker,docs
+	poetry install --no-root --no-ansi --with dev --without worker,docs
 	poetry run pytest tests/
 
 .PHONY: collect-static

README.md

@@ -69,6 +69,7 @@ redbox-copilot/
 │  ├── src/
 │  │  └── app.py
 │  ├── tests/
+│  ├── pyproject.toml
 │  └── Dockerfile
 ├── redbox-core/
 │  ├── redbox

core-api/Dockerfile

@@ -0,0 +1,17 @@
+FROM python:3.11-buster as builder
+
+RUN apt-get update && apt-get install --yes build-essential curl > /dev/null
+
+# Add redbox python package and install it with poetry
+ADD redbox-core/ /app/redbox-core
+ADD core-api/ /app/core-api
+ADD README.md /app/README.md
+
+WORKDIR /app/core-api
+
+RUN pip install poetry
+RUN poetry install
+
+# Run FastAPI
+EXPOSE 5002
+CMD ["poetry", "run", "uvicorn", "core_api.app:app", "--host", "0.0.0.0", "--port", "5002"]

core_api/src/app.py → core-api/core_api/app.py

@@ -7,9 +7,9 @@
 from fastapi import Depends, FastAPI, Response
 from fastapi.responses import RedirectResponse
 
-from core_api.src.dependencies import get_elasticsearch_client
-from core_api.src.routes.chat import chat_app
-from core_api.src.routes.file import file_app
+from core_api.dependencies import get_elasticsearch_client
+from core_api.routes.chat import chat_app
+from core_api.routes.file import file_app
 from redbox import __version__ as redbox_version
 from redbox.models import Settings, StatusResponse
 

core_api/src/build_chains.py → core-api/core_api/build_chains.py

@@ -19,9 +19,9 @@
 from langchain_core.vectorstores import VectorStoreRetriever
 from tiktoken import Encoding
 
-from core_api.src import dependencies
-from core_api.src.format import format_documents
-from core_api.src.runnables import (
+from core_api import dependencies
+from core_api.format import format_documents
+from core_api.runnables import (
     make_chat_prompt_from_messages_runnable,
     resize_documents,
 )

core_api/src/dependencies.py → core-api/core_api/dependencies.py

@@ -13,8 +13,8 @@
 from langchain_elasticsearch import ApproxRetrievalStrategy, ElasticsearchStore
 from langchain_openai.embeddings import AzureOpenAIEmbeddings
 
-from core_api.src.callbacks import LoggerCallbackHandler
-from core_api.src.retriever import AllElasticsearchRetriever, ParameterisedElasticsearchRetriever
+from core_api.callbacks import LoggerCallbackHandler
+from core_api.retriever import AllElasticsearchRetriever, ParameterisedElasticsearchRetriever
 from redbox.models import Settings
 from redbox.storage import ElasticsearchStorageHandler
 

core_api/src/retriever/retrievers.py → core-api/core_api/retriever/retrievers.py

@@ -7,7 +7,7 @@
 from langchain_core.embeddings.embeddings import Embeddings
 from langchain_elasticsearch.retrievers import ElasticsearchRetriever
 
-from core_api.src.retriever.queries import ESParams, get_all, get_some
+from core_api.retriever.queries import ESParams, get_all, get_some
 
 
 def hit_to_doc(hit: dict[str, Any]) -> Document:

core_api/src/routes/chat.py → core-api/core_api/routes/chat.py

@@ -10,9 +10,9 @@
 from pydantic import BaseModel
 from semantic_router import RouteLayer
 
-from core_api.src.auth import get_user_uuid, get_ws_user_uuid
-from core_api.src.runnables import map_to_chat_response
-from core_api.src.semantic_routes import get_routable_chains, get_semantic_route_layer
+from core_api.auth import get_user_uuid, get_ws_user_uuid
+from core_api.runnables import map_to_chat_response
+from core_api.semantic_routes import get_routable_chains, get_semantic_route_layer
 from redbox.models.chain import ChainInput
 from redbox.models.chat import ChatRequest, ChatResponse, SourceDocument
 from redbox.models.errors import NoDocumentSelected, QuestionLengthError

core_api/src/routes/file.py → core-api/core_api/routes/file.py

@@ -9,8 +9,8 @@
 from faststream.redis.fastapi import RedisRouter
 from pydantic import BaseModel, Field
 
-from core_api.src.auth import get_user_uuid
-from core_api.src.publisher_handler import FilePublisher
+from core_api.auth import get_user_uuid
+from core_api.publisher_handler import FilePublisher
 from redbox.models import APIError404, Chunk, File, FileStatus, Settings
 from redbox.storage import ElasticsearchStorageHandler
 

core_api/src/runnables.py → core-api/core_api/runnables.py

@@ -5,7 +5,7 @@
 from langchain_core.runnables import RunnableLambda, RunnablePassthrough, chain
 from tiktoken import Encoding
 
-from core_api.src.format import reduce_chunks_by_tokens
+from core_api.format import reduce_chunks_by_tokens
 from redbox.models import ChatResponse
 from redbox.models.errors import QuestionLengthError
 from redbox.transform import map_document_to_source_document

core_api/src/semantic_routes.py → core-api/core_api/semantic_routes.py

@@ -6,13 +6,13 @@
 from semantic_router.encoders import AzureOpenAIEncoder, BaseEncoder
 from semantic_router.layer import RouteLayer
 
-from core_api.src.build_chains import (
+from core_api.build_chains import (
     build_condense_retrieval_chain,
     build_static_response_chain,
     build_summary_chain,
     build_vanilla_chain,
 )
-from core_api.src.dependencies import get_env
+from core_api.dependencies import get_env
 from redbox.models import Settings
 from redbox.models.chat import ChatRoute