#1567

Makefile

@@ -29,7 +29,8 @@ test:
 
 mypy:
 	clear
-	mypy --ignore-missing-imports --python-version 3.10 --no-strict-optional --check-untyped-defs opteryx
+	python -m pip install --upgrade mypy
+	python -m mypy --ignore-missing-imports --python-version 3.10 --no-strict-optional --check-untyped-defs opteryx
 
 coverage:
 	clear

opteryx/__version__.py

@@ -1,4 +1,4 @@
-__build__ = 404
+__build__ = 410
 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,9 +27,9 @@ class VersionStatus(Enum):
 
 
 _major = 0
-_minor = 13
-_revision = 4
-_status = VersionStatus.ALPHA
+_minor = 14
+_revision = 0
+_status = VersionStatus.RELEASE
 
 __author__ = "@joocer"
 __version__ = f"{_major}.{_minor}.{_revision}" + (

opteryx/compiled/functions/__init__.py

@@ -3,6 +3,3 @@
 from hash_table import HashTable
 from hash_table import distinct
 from ip_address import ip_in_cidr
-from varchar_array import array_encode_utf8
-from varchar_array import pack_byte_array
-from varchar_array import unpack_byte_array

opteryx/components/binder/binder_visitor.py

@@ -226,7 +226,7 @@ def visit_node(self, node: Node, context: BindingContext) -> Tuple[Node, Binding
             Tuple[Node, Dict]
             The node and context after binding.
         """
-        node_type = node.node_type.name
+        node_type = node.node_type.name  # type:ignore
         visit_method_name = f"visit_{CAMEL_TO_SNAKE.sub('_', node_type).lower()}"
         visit_method = getattr(self, visit_method_name, None)
         if visit_method is None:

opteryx/connectors/cql_connector.py

@@ -39,7 +39,7 @@
 from opteryx.third_party.query_builder import Query
 
 
-def _handle_operand(operand: Node, parameters: list) -> Tuple[Any, dict]:
+def _handle_operand(operand: Node, parameters: list) -> Tuple[Any, list]:
     if operand.node_type == NodeType.IDENTIFIER:
         return f'"{operand.source_column}"', parameters
 
@@ -140,7 +140,7 @@ def read_dataset(  # type:ignore
                 col for col in self.schema.columns if f'"{col.name}"' in column_names  # type:ignore
             ]
         else:
-            query_builder.add("SELECT", f'"{self.single_column.name}"')
+            query_builder.add("SELECT", f'"{self.single_column.name}"')  # type:ignore
             self.schema.columns = [self.single_column]  # type:ignore
 
         # Update SQL if we've pushed predicates
@@ -211,6 +211,6 @@ def get_dataset_schema(self) -> RelationSchema:
             ],
         )
 
-        self.single_column = self.schema.columns[0]
+        self.single_column = self.schema.columns[0]  # type:ignore
 
         return self.schema

opteryx/functions/__init__.py

@@ -24,7 +24,6 @@
 from pyarrow import compute
 
 import opteryx
-from opteryx.compiled.functions import array_encode_utf8
 from opteryx.exceptions import FunctionNotFoundError
 from opteryx.exceptions import IncorrectTypeError
 from opteryx.exceptions import UnsupportedSyntaxError
@@ -35,6 +34,11 @@
 from opteryx.utils import dates
 
 
+def array_encode_utf8(arr):
+    # this is not the fastest way to do this, orso has a Cython method
+    return [None if s is None else s.encode() for s in arr]
+
+
 def _get(array, key):
     # Determine the type of the first element (assuming homogeneous array)
     first_element = next((item for item in array if item is not None), None)

opteryx/managers/expression/__init__.py

@@ -194,7 +194,7 @@ def prioritize_evaluation(expressions):
 
 
 def _inner_evaluate(root: Node, table: Table, context: ExecutionContext):
-    node_type = root.node_type
+    node_type = root.node_type  # type:ignore
 
     if node_type == NodeType.SUBQUERY:
         raise UnsupportedSyntaxError("IN (<subquery>) temporarily not supported.")
@@ -228,7 +228,7 @@ def _inner_evaluate(root: Node, table: Table, context: ExecutionContext):
         )  # type:ignore
 
     # BOOLEAN OPERATORS
-    if node_type & LOGICAL_TYPE == LOGICAL_TYPE:
+    if node_type & LOGICAL_TYPE == LOGICAL_TYPE:  # type:ignore
         if node_type == NodeType.OR:
             return short_cut_or(root, table, context)
         if node_type == NodeType.AND:
@@ -237,15 +237,15 @@ def _inner_evaluate(root: Node, table: Table, context: ExecutionContext):
         if node_type in LOGICAL_OPERATIONS:
             left = _inner_evaluate(root.left, table, context) if root.left else [None]
             right = _inner_evaluate(root.right, table, context) if root.right else [None]
-            return LOGICAL_OPERATIONS[node_type](left, right)
+            return LOGICAL_OPERATIONS[node_type](left, right)  # type:ignore
 
         if node_type == NodeType.NOT:
             centre = _inner_evaluate(root.centre, table, context) if root.centre else [None]
             centre = pyarrow.array(centre, type=pyarrow.bool_())
             return pyarrow.compute.invert(centre)
 
     # INTERAL IDENTIFIERS
-    if node_type & INTERNAL_TYPE == INTERNAL_TYPE:
+    if node_type & INTERNAL_TYPE == INTERNAL_TYPE:  # type:ignore
         if node_type == NodeType.FUNCTION:
             parameters = [_inner_evaluate(param, table, context) for param in root.parameters]
             # zero parameter functions get the number of rows as the parameter

setup.py

@@ -91,18 +91,6 @@ def rust_build(setup_kwargs: Dict[str, Any]) -> None:
         language="c++",
         extra_compile_args=COMPILE_FLAGS + ["-std=c++11"],
     ),
-    Extension(
-        name="bloom_filter",
-        sources=["opteryx/compiled/bloom_filter/bloom_filter.pyx"],
-        extra_compile_args=COMPILE_FLAGS,
-    ),
-    Extension(
-        name="varchar_array",
-        sources=["opteryx/compiled/functions/varchar_array.pyx"],
-        include_dirs=[numpy.get_include()],
-        language="c++",
-        extra_compile_args=COMPILE_FLAGS + ["-std=c++11"],
-    ),
 ]
 
 setup_config = {

tests/storage/test_sql_duckdb.py

@@ -32,7 +32,11 @@
 
 
 def test_duckdb_storage():
-    create_duck_db()
+    # We have some problems with creating duckdb, particularly in GitHub Actions
+    # we're going to brute force.
+    for i in range(5):
+        if create_duck_db() is None:
+            break
 
     opteryx.register_store(
         "duckdb",
@@ -55,7 +59,11 @@ def test_duckdb_storage():
 def test_duckdb_battery():
     from opteryx.utils.formatter import format_sql
 
-    create_duck_db()
+    # We have some problems with creating duckdb, particularly in GitHub Actions
+    # we're going to brute force.
+    for i in range(5):
+        if create_duck_db() is None:
+            break
 
     opteryx.register_store(
         "duckdb",

tests/tools.py

@@ -439,7 +439,7 @@ def create_duck_db():
     try:
         res = cur.execute(CREATE_DATABASE)
     except:
-        pass
+        return -1
     finally:
         if res is not None:
             res.commit()