Incorporate data snapshots (#31)

* Added python db package

* Make data-processor dumb

* Add 2 new schemas - snapshot and job

* Rename env var

* Save intermediate work

* Finalize changes

* Finish refactor

* Remove __pycache__

.env.example

@@ -40,6 +40,9 @@ SECRET_MANAGER_TYPE=file
 # in case of file/vault secret manager, we must specify the directory
 SECRET_MANAGER_DIRECTORY=/secrets
 
+# Knowledge base snapshots directory
+SNAPSHOTS_DIRECTORY=/snapshots
+
 # Collects telemetry to our PostHog. Change to false if you don't want to send telemetry.
 TELEMETRY_ENABLED=true
 
@@ -116,6 +119,10 @@ API_URL=${ENVOY_URL}/api
 DATA_PROCESSOR_PORT=3002
 DATA_PROCESSOR_URL=${ENVOY_URL}/data-processor
 
+# Doc indexer
+DOC_INDEXER_PORT=3005
+DOC_INDEXER_URL=${ENVOY_URL}/doc-indexer
+
 # Slackbot
 SLACKBOT_PORT=3003
 

.vscode/launch.json

@@ -81,6 +81,14 @@
       "cwd": "${workspaceFolder}/services/data-processor/",
       "justMyCode": false
     },
+    {
+      "type": "debugpy",
+      "name": "Doc Indexer: Debug",
+      "request": "launch",
+      "program": "src/main.py",
+      "cwd": "${workspaceFolder}/services/doc-indexer/",
+      "justMyCode": false
+    },
     {
       "type": "debugpy",
       "name": "Log Parser: Debug",

.vscode/settings.json

@@ -1,4 +1,5 @@
 {
   "typescript.tsdk": "node_modules\\typescript\\lib",
-  "python.envFile": ""
+  "python.envFile": "",
+  "python.defaultInterpreterPath": "/Users/dudulasry/Library/Caches/pypoetry/virtualenvs/data-processor-qzgB1a2g-py3.10/bin/python"
 }