HOUSEKEEPING

opteryx/__version__.py

@@ -1,4 +1,4 @@
-__build__ = 489
+__build__ = 492
 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

opteryx/connectors/disk_connector.py

@@ -105,6 +105,8 @@ async def async_read_blob(self, *, blob_name, pool, statistics, **kwargs):
         # DEBUG: log ("READ   ", blob_name)
         with open(blob_name, "rb") as file:
             data = file.read()
+            if len(data) > pool.size():
+                raise ValueError(f"File {blob_name} is too large for read buffer")
             ref = await pool.commit(data)
             while ref is None:
                 statistics.stalls_writing_to_read_buffer += 1

opteryx/managers/schemes/mabel_partitions.py

@@ -28,10 +28,10 @@
 class UnsupportedSegementationError(DataError):
     """Exception raised for unsupported segmentations."""
 
-    def __init__(self, dataset: str, segments: set = None):
+    def __init__(self, dataset: str, segment: str):
         self.dataset = dataset
-        self.segments = segments
-        message = f"'{dataset}' contains unsupported segmentation (`{'`, `'.join(segments)}`), only 'by_hour' segments are supported."
+        self.segment = segment
+        message = f"'{dataset}' contains unsupported segmentation (found `{segment}`), only 'by_hour' segments are supported."
         super().__init__(message)
 
 
@@ -124,14 +124,12 @@ def _inner(*, date, start, end):
                     if by_label in blob:
                         segment = extract_prefix(blob, "by_")
                         if segment != "by_hour":
-                            raise UnsupportedSegementationError(dataset=prefix, segments=[segment])
+                            raise UnsupportedSegementationError(dataset=prefix, segment=segment)
 
             if any(f"{OS_SEP}by_hour{OS_SEP}" in blob_name for blob_name in data_blobs):
 
-                if start > date:
-                    start = date
-                if end < date:
-                    end = date
+                start = min(start, date)
+                end = max(end, date)
 
                 selected_blobs = []
 

opteryx/operators/async_read_node.py

@@ -36,7 +36,7 @@
 from opteryx.utils.file_decoders import get_decoder
 
 CONCURRENT_READS = 4
-MAX_BUFFER_SIZE_MB = 256
+MAX_BUFFER_SIZE_MB = 512
 
 
 def normalize_morsel(schema: RelationSchema, morsel: pyarrow.Table) -> pyarrow.Table:

opteryx/shared/__init__.py

@@ -11,9 +11,9 @@
 # limitations under the License.
 
 from opteryx.compiled.structures import MemoryPool
+from opteryx.shared.async_memory_pool import AsyncMemoryPool
 from opteryx.shared.buffer_pool import BufferPool
 from opteryx.shared.materialized_datasets import MaterializedDatasets
-from opteryx.shared.memory_pool import AsyncMemoryPool
 from opteryx.shared.rolling_log import RollingLog
 
 __all__ = ("AsyncMemoryPool", "BufferPool", "MaterializedDatasets", "MemoryPool", "RollingLog")

opteryx/shared/memory_pool.py → opteryx/shared/async_memory_pool.py

@@ -39,3 +39,6 @@ async def read(self, ref_id: int) -> bytes:
     async def release(self, ref_id: int):
         async with self.lock:
             self.pool.release(ref_id)
+
+    def size(self):
+        return self.pool.size

opteryx/shared/buffer_pool.py

@@ -24,7 +24,7 @@
 """
 from typing import Optional
 
-from opteryx.shared.memory_pool import MemoryPool
+from opteryx.shared import MemoryPool
 from opteryx.utils.lru_2 import LRU2
 
 

pyproject.toml

@@ -26,3 +26,7 @@ skips = ["B101", "B105", "B324", "B608"]
 # B105 - Hardcoded passwords
 # B324 - Use of weak crypto
 # B608 - Hardcoded SQL
+
+[build-system]
+requires = ["setuptools>=42", "wheel", "Cython", "numpy"]
+build-backend = "setuptools.build_meta"

requirements.txt

@@ -4,4 +4,4 @@ orjson
 orso>=0.0.151
 pyarrow>=12.0.1
 typer==0.11.*
-
+aiohttp

setup.py

@@ -122,6 +122,9 @@ def rust_build(setup_kwargs: Dict[str, Any]) -> None:
     "entry_points": {
         "console_scripts": ["opteryx=opteryx.command:main"],
     },
+    "package_data": {
+        "": ["*.pyx", "*.pxd"],
+    },
 }
 
 rust_build(setup_config)

tests/requirements.txt

@@ -37,4 +37,5 @@ cassandra-driver
 
 setuptools_rust
 
-aiohttp
+aiohttp
+psutil