feat: support Insert, Update, Delete and Merge statements

Signed-off-by: Andreas Reichel <andreas@manticore-projects.com>

README.md

@@ -2,7 +2,11 @@
 
 A pure Java stand-alone SQL Transpiler for translating various large RDBMS SQL Dialects into a few smaller RDBMS Dialects for Unit Testing. Based on JSQLParser.
 
-Focus is on Queries (only) and work is on progress based on the [Feature Matrix](src/main/resources/doc/JSQLTranspiler.ods).
+Supports `SELECT` queries as well as `INSERT`, `UPDATE`, `DELETE` and `MERGE` statements.
+
+Internal Functions will be rewritten based on the actual meaning and purpose of the function (since DuckDB `Any()` function does not necessarily behave like the RDBMS specific `Any()`). Respecting different function arguments count, order and type.
+
+Rewrite of Window- and Aggregate-Functions.
 
 ## Dialects
 
@@ -57,6 +61,15 @@ String result = JSQLTranspiler.transpile(providedSQL, Dialect.AMAZON_REDSHIFT);
 assertEquals(expectedSQL, result);
 ```
 
+### Web API
+```shell
+curl -X 'POST'                                                                   \
+  'https://secure-api.starlake.ai/api/v1/transpiler/transpile?dialect=SNOWFLAKE' \
+  -H 'accept: text/plain'                                                        \
+  -H 'Content-Type: text/plain'                                                  \
+  -d 'SELECT Nvl(null, 1) a'
+```
+
 ### Java Command Line Interface
 ```text
 usage: java -jar JSQLTranspilerCLI.jar [-d <arg> | --any | --bigquery |

src/main/java/ai/starlake/transpiler/JSQLExpressionTranspiler.java

@@ -56,9 +56,7 @@
 import net.sf.jsqlparser.statement.select.PlainSelect;
 import net.sf.jsqlparser.statement.select.Select;
 import net.sf.jsqlparser.statement.select.SelectItem;
-import net.sf.jsqlparser.statement.select.SelectVisitor;
 import net.sf.jsqlparser.util.deparser.ExpressionDeParser;
-import net.sf.jsqlparser.util.deparser.OrderByDeParser;
 import net.sf.jsqlparser.util.deparser.SelectDeParser;
 
 import java.text.SimpleDateFormat;

src/main/java/ai/starlake/transpiler/JSQLSelectTranspiler.java

@@ -1,6 +1,5 @@
 package ai.starlake.transpiler;
 
-import net.sf.jsqlparser.expression.ExpressionVisitor;
 import net.sf.jsqlparser.parser.SimpleNode;
 import net.sf.jsqlparser.schema.Table;
 import net.sf.jsqlparser.statement.select.FromItem;

src/main/java/ai/starlake/transpiler/JSQLTranspiler.java

@@ -16,13 +16,9 @@
  */
 package ai.starlake.transpiler;
 
-import ai.starlake.transpiler.bigquery.BigQuerySelectTranspiler;
 import ai.starlake.transpiler.bigquery.BigQueryTranspiler;
-import ai.starlake.transpiler.databricks.DatabricksSelectTranspiler;
 import ai.starlake.transpiler.databricks.DatabricksTranspiler;
-import ai.starlake.transpiler.redshift.RedshiftSelectTranspiler;
 import ai.starlake.transpiler.redshift.RedshiftTranspiler;
-import ai.starlake.transpiler.snowflake.SnowflakeSelectTranspiler;
 import ai.starlake.transpiler.snowflake.SnowflakeTranspiler;
 import net.sf.jsqlparser.JSQLParserException;
 import net.sf.jsqlparser.parser.CCJSqlParser;
@@ -57,7 +53,7 @@
 import java.util.logging.Logger;
 
 /**
- * The type JSQLtranspiler.
+ * The type JSQLTranspiler.
  */
 public class JSQLTranspiler extends StatementDeParser {
   public static final Logger LOGGER = Logger.getLogger(JSQLTranspiler.class.getName());
@@ -115,24 +111,17 @@ public JSQLTranspiler() throws InvocationTargetException, NoSuchMethodException,
   public static String transpileQuery(String qryStr, Dialect dialect,
       ExecutorService executorService, Consumer<CCJSqlParser> consumer) throws JSQLParserException {
     Statement st = CCJSqlParserUtil.parse(qryStr, executorService, consumer);
-    if (st instanceof Select) {
-      Select select = (Select) st;
-
-      switch (dialect) {
-        case GOOGLE_BIG_QUERY:
-          return transpileGoogleBigQuery(select);
-        case DATABRICKS:
-          return transpileDatabricksQuery(select);
-        case SNOWFLAKE:
-          return transpileSnowflakeQuery(select);
-        case AMAZON_REDSHIFT:
-          return transpileAmazonRedshiftQuery(select);
-        default:
-          return transpile(select);
-      }
-    } else {
-      throw new RuntimeException("The " + st.getClass().getName()
-          + " is not supported yet. Only `PlainSelect` is supported right now.");
+    switch (dialect) {
+      case GOOGLE_BIG_QUERY:
+        return transpileBigQuery(st);
+      case DATABRICKS:
+        return transpileDatabricks(st);
+      case SNOWFLAKE:
+        return transpileSnowflake(st);
+      case AMAZON_REDSHIFT:
+        return transpileAmazonRedshift(st);
+      default:
+        return transpile(st);
     }
   }
 
@@ -143,7 +132,7 @@ public static String transpileQuery(String qryStr, Dialect dialect,
    * @param dialect the dialect of the query string
    * @return the transformed query string
    * @throws JSQLParserException a parser exception when the statement can't be parsed
-   * @throws InterruptedException a time out exception, when the statement can't be parsed within 6
+   * @throws InterruptedException a time-out exception, when the statement can't be parsed within 6
    *         seconds (hanging parser)
    */
   public static String transpileQuery(String qryStr, Dialect dialect)
@@ -170,6 +159,7 @@ public static String transpileQuery(String qryStr, Dialect dialect)
    * @param consumer the parser configuration to use for the parsing
    * @throws JSQLParserException a parser exception when the statement can't be parsed
    */
+  @SuppressWarnings({"PMD.CyclomaticComplexity"})
   public static void transpile(String sqlStr, File outputFile, ExecutorService executorService,
       Consumer<CCJSqlParser> consumer) throws JSQLParserException {
     try {
@@ -178,14 +168,8 @@ public static void transpile(String sqlStr, File outputFile, ExecutorService exe
       // @todo: we may need to split this manually to salvage any not parseable statements
       Statements statements = CCJSqlParserUtil.parseStatements(sqlStr, executorService, consumer);
       for (Statement st : statements) {
-        if (st instanceof Select) {
-          Select select = (Select) st;
-          select.accept(transpiler);
-
-          transpiler.getBuffer().append("\n;\n\n");
-        } else {
-          LOGGER.log(Level.SEVERE, st.getClass().getSimpleName() + " is not supported yet:\n" + st);
-        }
+        st.accept(transpiler);
+        transpiler.getBuffer().append("\n;\n\n");
       }
 
       String transpiledSqlStr = transpiler.getBuffer().toString();
@@ -223,7 +207,7 @@ public static void transpile(String sqlStr, File outputFile, ExecutorService exe
    * @param sqlStr the original query string
    * @param outputFile the output file, writing to STDOUT when not defined
    * @throws JSQLParserException a parser exception when the statement can't be parsed
-   * @throws InterruptedException a time out exception, when the statement can't be parsed within 6
+   * @throws InterruptedException a time-out exception, when the statement can't be parsed within 6
    *         seconds (hanging parser)
    */
   public static boolean transpile(String sqlStr, File outputFile)
@@ -315,7 +299,7 @@ public static Collection<String> getMacros()
 
     executorService.shutdown();
     boolean wasTerminated = executorService.awaitTermination(TIMEOUT, TimeUnit.SECONDS);
-    LOGGER.log(Level.FINE, "Exceutor Service terminated: " + wasTerminated);
+    LOGGER.log(Level.FINE, "Executor Service terminated: " + wasTerminated);
 
     return macroStrList;
   }
@@ -357,15 +341,15 @@ public static void createMacros(Connection conn)
   }
 
   /**
-   * Transpile string.
+   * Rewrite a given SQL Statement into a text representation.
    *
-   * @param select the select
+   * @param statement the statement
    * @return the string
    */
-  public static String transpile(Select select) {
+  public static String transpile(Statement statement) {
     try {
       JSQLTranspiler transpiler = new JSQLTranspiler();
-      select.accept(transpiler);
+      statement.accept(transpiler);
 
       return transpiler.getBuffer().toString();
     } catch (InvocationTargetException | NoSuchMethodException | InstantiationException
@@ -376,15 +360,15 @@ public static String transpile(Select select) {
   }
 
   /**
-   * Transpile google big query string.
+   * Rewrite a given BigQuery SQL Statement into a text representation.
    *
-   * @param select the select
+   * @param statement the statement
    * @return the string
    */
-  public static String transpileGoogleBigQuery(Select select) {
+  public static String transpileBigQuery(Statement statement) {
     try {
       BigQueryTranspiler transpiler = new BigQueryTranspiler();
-      select.accept(transpiler);
+      statement.accept(transpiler);
 
       return transpiler.getBuffer().toString();
     } catch (InvocationTargetException | NoSuchMethodException | InstantiationException
@@ -395,15 +379,15 @@ public static String transpileGoogleBigQuery(Select select) {
   }
 
   /**
-   * Transpile databricks query string.
+   * Rewrite a given DataBricks SQL Statement into a text representation.
    *
-   * @param select the select
+   * @param statement the statement
    * @return the string
    */
-  public static String transpileDatabricksQuery(Select select) {
+  public static String transpileDatabricks(Statement statement) {
     try {
       DatabricksTranspiler transpiler = new DatabricksTranspiler();
-      select.accept(transpiler);
+      statement.accept(transpiler);
 
       return transpiler.getBuffer().toString();
     } catch (InvocationTargetException | NoSuchMethodException | InstantiationException
@@ -414,15 +398,15 @@ public static String transpileDatabricksQuery(Select select) {
   }
 
   /**
-   * Transpile snowflake query string.
+   * Rewrite a given Snowflake SQL Statement into a text representation.
    *
-   * @param select the select
+   * @param statement the statement
    * @return the string
    */
-  public static String transpileSnowflakeQuery(Select select) {
+  public static String transpileSnowflake(Statement statement) {
     try {
       SnowflakeTranspiler transpiler = new SnowflakeTranspiler();
-      select.accept(transpiler);
+      statement.accept(transpiler);
 
       return transpiler.getBuffer().toString();
     } catch (InvocationTargetException | NoSuchMethodException | InstantiationException
@@ -433,15 +417,15 @@ public static String transpileSnowflakeQuery(Select select) {
   }
 
   /**
-   * Transpile amazon redshift query string.
+   * Rewrite a given Redshift SQL Statement into a text representation.
    *
-   * @param select the select
+   * @param statement the statement
    * @return the string
    */
-  public static String transpileAmazonRedshiftQuery(Select select) {
+  public static String transpileAmazonRedshift(Statement statement) {
     try {
       RedshiftTranspiler transpiler = new RedshiftTranspiler();
-      select.accept(transpiler);
+      statement.accept(transpiler);
 
       return transpiler.getBuffer().toString();
     } catch (InvocationTargetException | NoSuchMethodException | InstantiationException

src/main/java/ai/starlake/transpiler/databricks/DatabricksExpressionTranspiler.java

@@ -16,7 +16,6 @@
  */
 package ai.starlake.transpiler.databricks;
 
-import ai.starlake.transpiler.JSQLSelectTranspiler;
 import ai.starlake.transpiler.JSQLTranspiler;
 import ai.starlake.transpiler.redshift.RedshiftExpressionTranspiler;
 import net.sf.jsqlparser.expression.AnalyticExpression;
@@ -29,6 +28,7 @@
 import net.sf.jsqlparser.expression.Function;
 import net.sf.jsqlparser.expression.LambdaExpression;
 import net.sf.jsqlparser.expression.LongValue;
+import net.sf.jsqlparser.expression.NotExpression;
 import net.sf.jsqlparser.expression.StringValue;
 import net.sf.jsqlparser.expression.TimeKeyExpression;
 import net.sf.jsqlparser.expression.TimezoneExpression;
@@ -71,7 +71,7 @@ enum TranspiledFunction {
 
     , TRY_AVG, TRY_SUM, PERCENT_RANK
 
-    , ARRAY_APPEND, ARRAY_COMPACT
+    , ARRAY_APPEND, ARRAY_COMPACT, ARRAY_EXCEPT
 
     ;
     // @FORMATTER:ON
@@ -542,6 +542,21 @@ public void visit(Function function) {
                 new LambdaExpression("x", new IsNullExpression("x", true)));
           }
           break;
+        case ARRAY_EXCEPT:
+          if (paramCount == 2) {
+            // LIST_DISTINCT(LIST_FILTER([1,2,2,3],X->NOT
+            // ARRAY_CONTAINS(LIST_INTERSECT([1,2,2,3],[1,1,3,5]),X)))
+
+            NotExpression notExpression = new NotExpression(new Function("Array_Contains",
+                new Function("List_Intersect", parameters.get(0), parameters.get(1)),
+                new Column("x")));
+
+            function.setName("List_Distinct");
+            function.setParameters(new Function("List_Filter", parameters.get(0),
+                new LambdaExpression("x", notExpression)));
+          }
+          break;
+
       }
     }
     if (rewrittenExpression == null) {

src/main/java/ai/starlake/transpiler/redshift/RedshiftExpressionTranspiler.java

@@ -17,8 +17,6 @@
 package ai.starlake.transpiler.redshift;
 
 import ai.starlake.transpiler.JSQLExpressionTranspiler;
-import ai.starlake.transpiler.JSQLSelectTranspiler;
-import ai.starlake.transpiler.JSQLTranspiler;
 import net.sf.jsqlparser.expression.AnalyticExpression;
 import net.sf.jsqlparser.expression.ArrayConstructor;
 import net.sf.jsqlparser.expression.ArrayExpression;

src/main/java/ai/starlake/transpiler/snowflake/SnowflakeExpressionTranspiler.java

@@ -16,7 +16,6 @@
  */
 package ai.starlake.transpiler.snowflake;
 
-import ai.starlake.transpiler.JSQLSelectTranspiler;
 import ai.starlake.transpiler.JSQLTranspiler;
 import ai.starlake.transpiler.redshift.RedshiftExpressionTranspiler;
 import net.sf.jsqlparser.expression.AnalyticExpression;

src/main/java/ai/starlake/transpiler/snowflake/SnowflakeSelectTranspiler.java

@@ -16,9 +16,7 @@
  */
 package ai.starlake.transpiler.snowflake;
 
-import ai.starlake.transpiler.JSQLExpressionTranspiler;
 import ai.starlake.transpiler.JSQLSelectTranspiler;
-import ai.starlake.transpiler.JSQLTranspiler;
 import net.sf.jsqlparser.expression.Alias;
 import net.sf.jsqlparser.expression.ArrayConstructor;
 import net.sf.jsqlparser.expression.Expression;

src/site/sphinx/index.rst

@@ -1,6 +1,6 @@
 .. meta::
    :description: Java Software Library for rewriting Big RDBMS Queries into Duck DB compatible queries.
-   :keywords: java sql query transpiler DuckDB H2 BigQuery Snowflake Redshift
+   :keywords: java sql query transpiler DuckDB H2 BigQuery Snowflake Redshift Databricks
 
 ###########################
 Java SQL Transpiler Library
@@ -17,7 +17,11 @@ Java SQL Transpiler Library
 
 A pure Java stand-alone SQL Transpiler for translating various large RDBMS SQL Dialects into a few smaller RDBMS Dialects for Unit Testing. Based on JSQLParser.
 
-Focus is on Queries (only) and work is on progress based on the [Feature Matrix](src/main/resources/doc/JSQLTranspiler.ods).
+Supports `SELECT` queries as well as `INSERT`, `UPDATE`, `DELETE` and `MERGE` statements.
+
+Internal Functions will be rewritten based on the actual meaning and purpose of the function (since DuckDB `Any()` function does not necessarily behave like the RDBMS specific `Any()`). Respecting different function arguments count, order and type.
+
+Rewrite of Window- and Aggregate-Functions.
 
 Latest stable release: |JSQLTRANSPILER_STABLE_VERSION_LINK|
 
@@ -71,13 +75,18 @@ Features
     * Comprehensive support for Query statements:
         - ``SELECT ...``
         - RDBMS specific Functions, Predicates and Operators
-        - Date formatting parameter
+        - Date and Number formatting parameters
+        - `ARRAY` access based on different indices (DuckDB starts with 1)
+    * `INSERT` statements
+    * `DELETE` statements
+    * `UPDATE` statements
+    * `MERGE` statements
 
     * Nested Expressions (e.g. Sub-Selects)
     * ``WITH`` clauses
     * PostgreSQL implicit ``CAST ::``
     * SQL Parameters (e.g. ``?`` or ``:parameter``)
-    * Arrays and JSON/XML
+    * Internal Function rewrite based on the actual meaning and purpose of the function (since DuckDB `Any()` is not always RDBMS specific `Any()`)
 
 
 

src/site/sphinx/usage.rst

@@ -1,6 +1,6 @@
 .. meta::
    :description: Java Software Library for rewriting Big RDBMS Queries into Duck DB compatible queries.
-   :keywords: java sql query transpiler DuckDB H2 BigQuery Snowflake Redshift
+   :keywords: java sql query transpiler DuckDB H2 BigQuery Snowflake Redshift DataBricks
 
 *****************
 How to use it
@@ -14,6 +14,18 @@ How to use it
        --databricks | --snowflake | --redshift]      [-D <arg> | --duckdb]
        [-i <arg>] [-o <arg>] [-h]
 
+
+.. tab:: Web API Call
+
+  .. code:: shell
+
+    curl -X 'POST'                                                                   \
+      'https://secure-api.starlake.ai/api/v1/transpiler/transpile?dialect=SNOWFLAKE' \
+      -H 'accept: text/plain'                                                        \
+      -H 'Content-Type: text/plain'                                                  \
+      -d 'SELECT Nvl(null, 1) a'
+
+
 .. tab:: Java Library Call
 
   .. code:: java