feat: more Aggregate functions

Signed-off-by: Andreas Reichel <andreas@manticore-projects.com>

src/main/java/com/manticore/transpiler/ExpressionTranspiler.java

@@ -17,17 +17,21 @@
  */
 package com.manticore.transpiler;
 
+import net.sf.jsqlparser.expression.AnalyticExpression;
 import net.sf.jsqlparser.expression.ArrayConstructor;
 import net.sf.jsqlparser.expression.CaseExpression;
 import net.sf.jsqlparser.expression.CastExpression;
 import net.sf.jsqlparser.expression.DateTimeLiteralExpression;
 import net.sf.jsqlparser.expression.Expression;
+import net.sf.jsqlparser.expression.ExpressionVisitor;
+import net.sf.jsqlparser.expression.ExpressionVisitorAdapter;
 import net.sf.jsqlparser.expression.ExtractExpression;
 import net.sf.jsqlparser.expression.Function;
 import net.sf.jsqlparser.expression.HexValue;
 import net.sf.jsqlparser.expression.IntervalExpression;
 import net.sf.jsqlparser.expression.LambdaExpression;
 import net.sf.jsqlparser.expression.LongValue;
+import net.sf.jsqlparser.expression.NullValue;
 import net.sf.jsqlparser.expression.OracleNamedFunctionParameter;
 import net.sf.jsqlparser.expression.Parenthesis;
 import net.sf.jsqlparser.expression.StringValue;
@@ -51,8 +55,11 @@
 import net.sf.jsqlparser.statement.select.SelectVisitor;
 import net.sf.jsqlparser.util.deparser.ExpressionDeParser;
 
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -89,7 +96,7 @@ enum TranspiledFunction {
 
     , SAFE_ADD, SAFE_DIVIDE, SAFE_MULTIPLY, SAFE_NEGATE, SAFE_SUBTRACT, TRUNC
 
-    , ARRAY_CONCAT_AGG
+    , ARRAY_CONCAT_AGG, COUNTIF
 
     , NVL, UNNEST;
     // @FORMATTER:ON
@@ -128,6 +135,10 @@ public static UnsupportedFunction from(String name) {
     public static UnsupportedFunction from(Function f) {
       return from(f.getName());
     }
+
+    public static UnsupportedFunction from(AnalyticExpression f) {
+      return from(f.getName());
+    }
   }
 
   public ExpressionTranspiler(SelectVisitor selectVisitor, StringBuilder buffer) {
@@ -810,6 +821,72 @@ THEN Instr( source_value, Regexp_Extract( source_value, reg_exp ) )
                   , new StringValue("ASC")
                   , new StringValue("NULLS FIRST")
           );
+          break;
+        case COUNTIF:
+          // COUNT(IF(x < 0, x, NULL))
+
+          final Set<Column> expressionColumns = new HashSet<>();
+          parameters.get(0).accept(new ExpressionVisitorAdapter() {
+            public void visit(Column column) {
+              expressionColumns.add(column);
+            }
+          });
+          // @todo: clarify if there can be only exactly 1 column
+          //        else, what do to on None or many?
+          assert expressionColumns.size()==1;
+          Column column = expressionColumns.toArray(new Column[0])[0];
+
+          warning("Different NULL handling");
+          function.setName("Count");
+          function.setParameters(
+                  new Function( "If", parameters.get(0), column , new NullValue())
+          );
+
+          break;
+      }
+    }
+    if (rewrittenExpression == null) {
+      super.visit(function);
+    }
+  }
+
+  public void visit(AnalyticExpression function) {
+    String functionName = function.getName();
+
+    if (UnsupportedFunction.from(function) != null) {
+      throw new RuntimeException(
+              "Unsupported: " + functionName + " is not supported by DuckDB (yet).");
+    } else if (functionName.endsWith("$$")) {
+      // work around for transpiling already transpiled functions twice
+      // @todo: figure out a better way to achieve that
+      function.setName(functionName.substring(0, functionName.length() - 2));
+      super.visit(function);
+      return;
+    }
+    Expression rewrittenExpression = null;
+    TranspiledFunction f = TranspiledFunction.from(functionName);
+    if (f != null) {
+      switch (f) {
+        case COUNTIF:
+          // COUNT(IF(x < 0, x, NULL))
+
+          final Set<Column> expressionColumns = new HashSet<>();
+          function.getExpression().accept(new ExpressionVisitorAdapter() {
+            public void visit(Column column) {
+              expressionColumns.add(column);
+            }
+          });
+          // @todo: clarify if there can be only exactly 1 column
+          //        else, what do to on None or many?
+          assert expressionColumns.size()==1;
+          Column column = expressionColumns.toArray(new Column[0])[0];
+
+          warning("Different NULL handling");
+          function.setName("Count");
+          function.setExpression(
+                  new Function( "If", function.getExpression(), column , new NullValue())
+          );
+
           break;
       }
     }

src/test/resources/com/manticore/transpiler/any/debug.sql

@@ -1,11 +1,26 @@
 -- provided
-SELECT SAFE_CONVERT_BYTES_TO_STRING(b'\x61') as safe_convert
-;
+SELECT
+  x,
+  COUNTIF(x<0) OVER (ORDER BY ABS(x) ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS num_negative
+FROM UNNEST([5, -2, 3, 6, -10, NULL, -7, 4, 0]) AS x
+order by 1 NULLS FIRST;
 
 -- expected
-SELECT DECODE(COALESCE(TRY_CAST('\x61' AS BLOB),ENCODE('\x61')))AS SAFE_CONVERT;
+SELECT
+  x,
+  /* Approximation: Different NULL handling */ COUNT(IF(x < 0, x, NULL)) OVER (ORDER BY ABS(x) ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS num_negative
+FROM (SELECT UNNEST([5, -2, 3, 6, -10, NULL, -7, 4, 0]) AS x) AS x
+order by 1 NULLS FIRST;
 
 -- result
-"safe_convert"
-"a"
+"x","num_negative"
+"","1"
+"-10","2"
+"-7","2"
+"-2","1"
+"0","1"
+"3","1"
+"4","0"
+"5","0"
+"6","1"
 

src/test/resources/com/manticore/transpiler/google_bigquery/aggregate_function.sql

@@ -137,3 +137,132 @@ FROM (SELECT UNNEST([61441, 161]) as x) as x;
 "bit_and"
 "1"
 
+
+-- provided
+SELECT BIT_OR(x) as bit_or FROM UNNEST([0xF001, 0x00A1]) as x;
+
+-- expected
+SELECT BIT_OR(x) as bit_or
+FROM (SELECT UNNEST([61441, 161]) as x) as x;
+
+-- result
+"bit_or"
+"61601"
+
+
+-- provided
+SELECT BIT_XOR(x) as bit_xor FROM UNNEST([0xF001, 0x00A1]) as x;
+
+-- expected
+SELECT BIT_XOR(x) as bit_xor
+FROM (SELECT UNNEST([61441, 161]) as x) as x;
+
+-- result
+"bit_xor"
+"61600"
+
+
+-- provided
+SELECT
+  x,
+  COUNT(*) OVER (PARTITION BY MOD(x, 3)) AS count_star,
+  COUNT(x) OVER (PARTITION BY MOD(x, 3)) AS count_x
+FROM UNNEST([1, 4, NULL, 4, 5]) AS x
+ORDER BY 1 NULLS FIRST;
+
+-- expected
+SELECT
+  x,
+  COUNT(*) OVER (PARTITION BY MOD(x, 3)) AS count_star,
+  COUNT(x) OVER (PARTITION BY MOD(x, 3)) AS count_x
+FROM (SELECT UNNEST([1, 4, NULL, 4, 5]) AS x) AS x
+ORDER BY 1 NULLS FIRST;
+
+-- result
+"x","count_star","count_x"
+"","1","0"
+"1","3","3"
+"4","3","3"
+"4","3","3"
+"5","1","1"
+
+
+-- provided
+SELECT COUNT(DISTINCT IF(x > 0, x, NULL)) AS distinct_positive
+FROM UNNEST([1, -2, 4, 1, -5, 4, 1, 3, -6, 1]) AS x;
+
+-- expected
+SELECT COUNT(DISTINCT IF(x > 0, x, NULL)) AS distinct_positive
+FROM (SELECT UNNEST([1, -2, 4, 1, -5, 4, 1, 3, -6, 1]) AS x) AS x;
+
+-- result
+"distinct_positive"
+"3"
+
+
+-- provided
+SELECT COUNTIF(x<0) AS num_negative, COUNTIF(x>0) AS num_positive
+FROM UNNEST([5, -2, 3, 6, -10, -7, 4, 0]) AS x;
+
+-- expected
+SELECT COUNT(IF(x < 0, x, NULL)) AS num_negative, COUNT(IF(x > 0, x, NULL)) AS num_positive
+FROM (SELECT UNNEST([5, -2, 3, 6, -10, -7, 4, 0]) AS x) AS x;
+
+-- result
+"num_negative","num_positive"
+"3","4"
+
+
+-- provided
+SELECT
+  x,
+  COUNTIF(x<0) OVER (ORDER BY ABS(x) ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS num_negative
+FROM UNNEST([5, -2, 3, 6, -10, NULL, -7, 4, 0]) AS x
+order by 1 NULLS FIRST;
+
+-- expected
+SELECT
+  x,
+  /* Approximation: Different NULL handling */ COUNT(IF(x < 0, x, NULL)) OVER (ORDER BY ABS(x) ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS num_negative
+FROM (SELECT UNNEST([5, -2, 3, 6, -10, NULL, -7, 4, 0]) AS x) AS x
+order by 1 NULLS FIRST;
+
+-- result
+"x","num_negative"
+"","1"
+"-10","2"
+"-7","2"
+"-2","1"
+"0","1"
+"3","1"
+"4","0"
+"5","0"
+"6","1"
+
+
+-- provided
+WITH
+  Products AS (
+    SELECT 'shirt' AS product_type, 't-shirt' AS product_name, 3 AS product_count UNION ALL
+    SELECT 'shirt', 't-shirt', 8 UNION ALL
+    SELECT 'shirt', 'polo', 25 UNION ALL
+    SELECT 'pants', 'jeans', 6
+  )
+SELECT
+  product_type,
+  product_name,
+  SUM(product_count) AS product_sum,
+  GROUPING(product_type) AS product_type_agg,
+  GROUPING(product_name) AS product_name_agg,
+FROM Products
+GROUP BY GROUPING SETS(product_type, product_name, ())
+ORDER BY product_name, product_type;
+
+-- result
+"product_type","product_name","product_sum","product_type_agg","product_name_agg"
+"","jeans","6.00","1","0"
+"","polo","25.00","1","0"
+"","t-shirt","11.00","1","0"
+"pants","","6.00","0","1"
+"shirt","","36.00","0","1"
+"","","42.00","1","1"