Merge pull request #1637 from mabel-dev/dependabot/cargo/sqlparser-0.…

…46.0

Update sqlparser requirement from 0.45.0 to 0.46.0

Cargo.toml

@@ -16,5 +16,5 @@ pythonize = "0.21"
 serde = "1.0.171"
 
 [dependencies.sqlparser]
-version = "0.45.0"
+version = "0.46.0"
 features = ["serde", "visitor"]

opteryx/__version__.py

@@ -1,4 +1,4 @@
-__build__ = 482
+__build__ = 484
 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

opteryx/compiled/functions/__init__.py

@@ -1,4 +1,3 @@
-from .cython_functions import numpy_array_get_element
 from .ip_address import ip_in_cidr
 from .vectors import possible_match
 from .vectors import possible_match_indices

opteryx/compiled/list_ops/__init__.py

@@ -6,3 +6,6 @@
 from .cython_list_ops import cython_anyop_lt
 from .cython_list_ops import cython_anyop_lte
 from .cython_list_ops import cython_anyop_neq
+from .cython_list_ops import cython_arrow_op
+from .cython_list_ops import cython_get_element_op
+from .cython_list_ops import cython_long_arrow_op

opteryx/compiled/list_ops/cython_list_ops.pyx

@@ -140,3 +140,98 @@ cpdef cnp.ndarray[cnp.npy_bool, ndim=1] cython_anyop_gte(object literal, cnp.nda
                 break
 
     return result
+
+
+cpdef cnp.ndarray cython_arrow_op(cnp.ndarray arr, object key):
+    """
+    Fetch values from a list of dictionaries based on a specified key.
+
+    Parameters:
+        data: list
+            A list of dictionaries where each dictionary represents a structured record.
+        key: str
+            The key whose corresponding value is to be fetched from each dictionary.
+
+    Returns:
+        cnp.ndarray: An array containing the values associated with the key in each dictionary
+                     or None where the key does not exist.
+    """
+    # Determine the number of items in the input list
+    cdef int n = len(arr)
+    # Prepare an object array to store the results
+    cdef cnp.ndarray result = numpy.empty(n, dtype=object)
+
+    cdef int i
+    # Iterate over the list of dictionaries
+    for i in range(n):
+        # Check if the key exists in the dictionary
+        if key in arr[i]:
+            result[i] = arr[i][key]
+        else:
+            # Assign None if the key does not exist
+            result[i] = None
+
+    return result
+
+
+cpdef cnp.ndarray cython_long_arrow_op(cnp.ndarray arr, object key):
+    """
+    Fetch values from a list of dictionaries based on a specified key.
+
+    Parameters:
+        data: list
+            A list of dictionaries where each dictionary represents a structured record.
+        key: str
+            The key whose corresponding value is to be fetched from each dictionary.
+
+    Returns:
+        cnp.ndarray: An array containing the values associated with the key in each dictionary
+                     or None where the key does not exist.
+    """
+    # Determine the number of items in the input list
+    cdef int n = len(arr)
+    # Prepare an object array to store the results
+    cdef cnp.ndarray result = numpy.empty(n, dtype=object)
+
+    cdef int i
+    # Iterate over the list of dictionaries
+    for i in range(n):
+        # Check if the key exists in the dictionary
+        if key in arr[i]:
+            result[i] = str(arr[i][key])
+        else:
+            # Assign None if the key does not exist
+            result[i] = None
+
+    return result
+
+
+cpdef cython_get_element_op(cnp.ndarray[object, ndim=1] array, int key):
+    """
+    Fetches elements from each sub-array of a NumPy array at a given index.
+
+    Parameters:
+        array (numpy.ndarray): A 1D NumPy array of 1D NumPy arrays.
+        key (int): The index at which to retrieve the element from each sub-array.
+
+    Returns:
+        numpy.ndarray: A NumPy array containing the elements at the given index from each sub-array.
+    """
+
+    # Check if the array is empty
+    if array.size == 0:
+        return numpy.array([])
+
+    # Preallocate result array with the appropriate type
+    cdef cnp.ndarray result = numpy.empty(array.size, dtype=object)
+
+    # Iterate over the array using memory views for efficient access
+    cdef int i = 0
+    for sub_array in array:
+        if sub_array is not None and len(sub_array) > key:
+            result[i] = sub_array[key]
+        else:
+            result[i] = None
+        i += 1
+
+    return result

opteryx/functions/__init__.py

@@ -48,17 +48,19 @@ def _get(array, key):
     key = key[0]
     if isinstance(first_element, dict):
         # Handle dict type
-        return [item.get(key) for item in array]
+        from opteryx.compiled.list_ops import cython_arrow_op
+
+        return cython_arrow_op(array, key)
 
     try:
         index = int(key)
     except Exception:
         raise IncorrectTypeError("VARCHAR and ARRAY values must be subscripted with NUMERIC values")
     if isinstance(first_element, numpy.ndarray):
         # NumPy-specific optimization
-        from opteryx.compiled.functions import numpy_array_get_element
+        from opteryx.compiled.list_ops import cython_get_element_op
 
-        return numpy_array_get_element(array, key)
+        return cython_get_element_op(array, key)
 
     if isinstance(first_element, (list, str, pyarrow.ListScalar)):
         # Handle list type

opteryx/managers/expression/formatter.py

@@ -113,6 +113,8 @@ def format_expression(root, qualify: bool = False):
             "BitwiseOr": "|",
             "LtEq": "<=",
             "GtEq": ">=",
+            "Arrow": "->",
+            "LongArrow": "->>",
         }
         return f"{format_expression(root.left, qualify)} {_map.get(root.value, root.value).upper()} {format_expression(root.right, qualify)}"
     if node_type == NodeType.UNARY_OPERATOR:

opteryx/managers/expression/ops.py

@@ -180,4 +180,8 @@ def _inner_filter_operations(arr, operator, value):
         return list_ops.cython_allop_eq(arr[0], value)
     if operator == "AllOpNotEq":
         return list_ops.cython_allop_neq(arr[0], value)
+    if operator == "Arrow":
+        return list_ops.cython_arrow_op(arr, value[0])
+    if operator == "LongArrow":
+        return list_ops.cython_long_arrow_op(arr, value[0])
     raise NotImplementedError(f"Operator {operator} is not implemented!")  # pragma: no cover

opteryx/planner/binder/binder_visitor.py

@@ -55,11 +55,11 @@ def get_mismatched_condition_column_types(node: Node, relaxed: bool = False) ->
 
     elif node.node_type == NodeType.COMPARISON_OPERATOR:
         if (
-            node.value in ("InList", "NotInList")
+            node.value in ("InList", "NotInList", "Arrow", "LongArrow")
             or node.value.startswith("AllOp")
             or node.value.startswith("AnyOp")
         ):
-            return None  # ARRAY ops are meant to have different types
+            return None  # Some ops are meant to have different types
         left_type = node.left.schema_column.type if node.left.schema_column else None
         right_type = node.right.schema_column.type if node.right.schema_column else None
 

opteryx/planner/logical_planner/logical_planner.py

@@ -323,6 +323,9 @@ def inner_query_planner(ast_branch):
     if _selection:
         if len(_relations) == 0:
             raise UnsupportedSyntaxError("Statement has a WHERE clause but no FROM clause.")
+        all_ops = get_all_nodes_of_type(_selection, (NodeType.COMPARISON_OPERATOR,))
+        if any(op.value in ("Arrow", "LongArrow") for op in all_ops):
+            raise UnsupportedSyntaxError("JSON Accessors (->, ->>) cannot be used in filters.")
         selection_step = LogicalPlanNode(node_type=LogicalPlanStepType.Filter)
         selection_step.condition = _selection
         previous_step_id, step_id = step_id, random_string()