FIX/#459 Functions not run after the first page

docs/Release Notes/Change Log.md

@@ -15,14 +15,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 **Changed**
 
-- [[#457](https://github.com/mabel-dev/opteryx/issues/457)] Null values are removed before performing `INNER JOIN USING`. ([@joocer](https://github.com/joocer)) 
+- ⚠️**BREAKING** (correction) [[#457](https://github.com/mabel-dev/opteryx/issues/457)] Null values are removed before performing `INNER JOIN USING`. ([@joocer](https://github.com/joocer)) 
 
 **Fixed**
 
 - [[#448](https://github.com/mabel-dev/opteryx/issues/448)] `VERSION()` failed and missing from regression suite. ([@joocer](https://github.com/joocer)) 
 - [[#404](https://github.com/mabel-dev/opteryx/issues/404)] `COALESCE` fails for NaN values. ([@joocer](https://github.com/joocer))   
 - [[#453](https://github.com/mabel-dev/opteryx/issues/453)] PyArrow bug with long lists creating new columns. ([@joocer](https://github.com/joocer))   
 - [[#444](https://github.com/mabel-dev/opteryx/issues/444)] Very low cardinality `INNER JOINS` exceed memory allocation. ([@joocer](https://github.com/joocer))   
+- [[#459](https://github.com/mabel-dev/opteryx/issues/459)] Functions lose some detail on non-first page. ([@joocer](https://github.com/joocer))  
 
 ## [0.3.0] - 2022-08-28
 

opteryx/functions/__init__.py

@@ -72,6 +72,7 @@ def cast(_type):
     """cast a column to a specified type"""
     if _type in VECTORIZED_CASTERS:
         return lambda a: compute.cast(a, VECTORIZED_CASTERS[_type])
+
     if _type in ITERATIVE_CASTERS:
 
         def _inner(arr):

opteryx/managers/expression/__init__.py

@@ -31,6 +31,7 @@
 from opteryx.third_party.pyarrow_ops.ops import filter_operations
 
 
+# These are bit-masks
 LOGICAL_TYPE: int = int("0001", 2)
 INTERNAL_TYPE: int = int("0010", 2)
 LITERAL_TYPE: int = int("0100", 2)
@@ -55,14 +56,14 @@ def format_expression(root):
         if node_type == NodeType.WILDCARD:
             return "*"
         if node_type == NodeType.BINARY_OPERATOR:
-            MAP = {
+            _map = {
                 "StringConcat": "||",
                 "Plus": "+",
                 "Minus": "-",
                 "Multiply": "*",
                 "Divide": "/",
             }
-            return f"{format_expression(root.left)}{MAP.get(root.value, '?')}{format_expression(root.right)}"
+            return f"{format_expression(root.left)}{_map.get(root.value, '?')}{format_expression(root.right)}"
 
     return str(root.value)
 
@@ -356,4 +357,6 @@ def evaluate_and_append(expressions, table: Table):
 
         return_expressions.append(statement)
 
+        table = columns.apply(table)
+
     return columns, return_expressions, table

opteryx/operators/projection_node.py

@@ -95,9 +95,7 @@ def execute(self) -> Iterable:
 
             # If any of the columns are FUNCTIONs, we need to evaluate them
             start_time = time.time_ns()
-            _columns, self._expressions, page = evaluate_and_append(
-                self._expressions, page
-            )
+            _columns, _, page = evaluate_and_append(self._expressions, page)
             self._statistics.time_evaluating += time.time_ns() - start_time
 
             # first time round we're going work out what we need from the metadata

opteryx/version.py

@@ -16,4 +16,4 @@
 2) we can import it in setup.py for the same reason
 """
 
-__version__ = "0.4.0-alpha.1"
+__version__ = "0.4.0-alpha.2"