Merge pull request #45 from mabel-dev/clash

clashing

requirements.txt

@@ -1,9 +1,7 @@
 uvicorn
 fastapi
 pydantic
-mabelbeta 
 orjson
-ujson
 requests
 google-cloud-storage
 google-cloud-tasks
@@ -15,3 +13,4 @@ pyarrow
 pyjwt
 opteryx
 pymemcache
+opteryx

src/internals/helpers/paths.py

@@ -5,14 +5,10 @@ def find_path(path):
     from here. Otherwise do a broader search which is a little slower.
     """
     import glob
-    from mabel.logging import get_logger
-
-    logger = get_logger()
 
     paths = glob.glob(f"/app/src/**/{path}", recursive=True)
     if len(paths) == 0:
         paths = glob.glob(f"**/{path}", recursive=True)
     for i in paths:
         if i.endswith(path):
-            logger.info(f"Found `{path}` at `{i}`")
             return i

src/internals/models/search.py

@@ -1,6 +1,7 @@
-from pydantic import BaseModel
-from typing import Optional, Union
 import datetime
+from typing import Optional, Union
+
+from pydantic import BaseModel
 
 
 class SearchModel(BaseModel):

src/main.py

@@ -1,29 +1,21 @@
 import os
 import sys
 
-sys.path.insert(0, os.path.join(sys.path[0], "../../mabel_/"))
+sys.path.insert(0, os.path.join(sys.path[0], "../../opteryx/"))
+#sys.path.insert(0, os.path.join(sys.path[0], "../../mabel@0.6/"))
 
 import uvicorn
-from fastapi.responses import HTMLResponse
 from fastapi import FastAPI, Request
-from mabel.logging import get_logger, set_log_name
-from mabel.utils.common import build_context
+from fastapi.responses import HTMLResponse
+#from mabel.logging import get_logger, set_log_name
 from internals.helpers.paths import find_path
+from routers import search, users
 
-from routers import search, download, users, stores
-
-context = build_context()
-set_log_name("QUERY")
-logger = get_logger()
-logger.setLevel(5)
-
-
-RESULT_BATCH = int(context.get("maximum_return", 2000)) or 5000
+RESULT_BATCH = 5000
 
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
 
-
 # set up API interface
 os.environ["TZ"] = "UTC"
 version = os.getenv("SHORT_SHA", "local")
@@ -43,11 +35,18 @@ def home(request: Request):
     """
     return templates.TemplateResponse("index.html", {"request": request})
 
+@application.get("/arrow", response_class=HTMLResponse)
+def home(request: Request):
+    """
+    This is a single page app, we deliver a single HTML page and interact
+    with the backend using APIs.
+    """
+    return templates.TemplateResponse("wow.html", {"request": request})
+
 
 application.include_router(search.router)
-application.include_router(download.router)
 application.include_router(users.router)
-application.include_router(stores.router)
+
 
 # tell the server to start
 if __name__ == "__main__":

src/routers/search.py

@@ -1,37 +1,39 @@
-import orjson
 import datetime
-from fastapi import APIRouter, HTTPException, Response, Request
-from mabel.logging import get_logger, set_log_name
-from mabel.errors import DataNotFoundError
+
+import orjson
+from fastapi import APIRouter, HTTPException, Request, Response
 from internals.models import SearchModel
-from internals.helpers.search import do_search
 
 
+class DataNotFoundError(Exception):
+    pass
+
 router = APIRouter()
-set_log_name("QUERY")
-logger = get_logger()
-logger.setLevel(5)
 
 
 RESULT_BATCH = 2000
 
 ##########################################################################
+
 def fix_dict(obj: dict) -> dict:
     def fix_fields(dt):
-        if isinstance(dt, (datetime.date, datetime.datetime)):
+        dt_type = type(dt)
+        if dt_type in (int, float, bool, str):
+            return dt
+        if dt_type in (datetime.date, datetime.datetime):
             return dt.isoformat()
-        if isinstance(dt, bytes):
+        if dt_type == dict:
+            return str(fix_dict(dt))
+        if dt_type == bytes:
             return dt.decode("UTF8")
-        if hasattr(dt, "mini"):
-            return dt.mini.decode("UTF8")
-        if isinstance(dt, dict):
-            return {k: fix_fields(v) for k, v in dt.items()}
         return str(dt)
 
     if not isinstance(obj, dict):
         return obj  # type:ignore
-    return {k: fix_fields(v) for k, v in obj.items()}
-
+
+    for key in obj.keys():
+        obj[key] = fix_fields(obj[key])
+    return obj
 
 ########################################################################
 
@@ -45,11 +47,8 @@ def serialize_response(response, max_records):
     for i, record in enumerate(response):
         if i > max_records:
             break
-        if i < max_records and hasattr(record, "mini"):
-            # we have a saved minified json string
-            yield record.mini + b"\n"
         elif i < max_records:
-            yield orjson.dumps(fix_dict(record)) + b"\n"
+            yield record.as_json + b"\n"
     if i == -1:
         # UNABLE TO SATISFY RANGE
         raise HTTPException(status_code=416)
@@ -60,14 +59,18 @@ def serialize_response(response, max_records):
 @router.post("/v1/search")
 def search(search: SearchModel, request: Request):
     try:
-        from internals.helpers.identity import get_jwt, get_identity
+        from internals.helpers.identity import get_identity, get_jwt
 
         encoded_jwt = get_jwt(request)
-        logger.info({**search.dict(), "user": get_identity(encoded_jwt)})
 
-        results = do_search(search, encoded_jwt)
+        import opteryx
+        conn = opteryx.connect(
+            partition_scheme=[]
+        )
+        cur = conn.cursor()
+        cur.execute(search.query)
 
-        body = b"\n".join(serialize_response(results, RESULT_BATCH))
+        body = b"\n".join(serialize_response(cur.fetchmany(100), RESULT_BATCH))
         response = Response(
             body,
             media_type="application/jsonlines",
@@ -85,13 +88,13 @@ def search(search: SearchModel, request: Request):
 
         trace = traceback.format_exc()
         error_message = {"error": type(err).__name__, "detail": str(err)}
-        logger.error(f"Error {type(err).__name__} - {err}:\n{trace}")
+        print(f"Error {type(err).__name__} - {err}:\n{trace}")
         # I'M A TEAPOT
         raise HTTPException(status_code=418, detail=error_message)
     except SystemExit as err:
         import traceback
 
         trace = traceback.format_exc()
-        logger.alert(f"Fatal Error {type(err).__name__} - {err}:\n{trace}")
+        print(f"Fatal Error {type(err).__name__} - {err}:\n{trace}")
         # ERROR
         raise HTTPException(status_code=500, detail=err)

src/routers/users.py

@@ -1,20 +1,15 @@
 from fastapi import APIRouter, Request
 from fastapi.responses import UJSONResponse
-from mabel.logging import get_logger, set_log_name
-
 
 router = APIRouter()
-set_log_name("QUERY")
-logger = get_logger()
-logger.setLevel(5)
 
 
 @router.get("/user", response_class=UJSONResponse)
 def get_user_informations(request: Request):
     """
     Get User information
     """
-    from internals.helpers.identity import get_jwt, get_identity
+    from internals.helpers.identity import get_identity, get_jwt
 
     encoded_jwt = get_jwt(request)
     return {"identity": get_identity(encoded_jwt), "saved_queries": []}

src/static/dist/js/colors.js

@@ -35,8 +35,14 @@ MACARONS = [
     '#59678c', '#c9ab00', '#7eb00a', '#6f5553', '#c14089'
 ]
 
+DRACULA = [
+    "#44475A", "#FF6E6E", "#282A36", "#21222C", "#A4FFFF",
+    "#BD93F9", "#8BE9FD", "#FF79C6", "#50FA7B", "#69FF94",
+    "#F1FA8C", "#6272A4", "#D6ACFF", "#FF92DF"
+]
+
 
-DEFAULT_COLORS = SPROUT
+DEFAULT_COLORS = DRACULA
 
 function interpolateColor(c0, c1, f) {
     c0 = c0.match(/.{1,2}/g).map((oct) => parseInt(oct, 16) * (1 - f))