From 6c8360ae3f73f056eceafcc7bdf5f739f5e07fa1 Mon Sep 17 00:00:00 2001 From: Andreas Reichel Date: Sat, 16 Mar 2024 08:20:21 +0700 Subject: [PATCH] feat: CLI - implement CLI for reading SQL from file or STDIN to transpile to file or STDOUT - provide unit test Signed-off-by: Andreas Reichel --- .../transpiler/ExpressionTranspiler.java | 17 ++ .../manticore/transpiler/JSQLTranspiler.java | 146 ++++++++++++++++-- .../AmazonRedshiftTranspilerTest.java | 17 ++ .../GoogleBigQueryTranspilerTest.java | 17 ++ .../transpiler/JSQLTranspilerTest.java | 55 ++++++- .../transpiler/JSQLTranspilerTest_MainIn.sql | 23 +++ .../transpiler/JSQLTranspilerTest_MainOut.sql | 23 +++ 7 files changed, 280 insertions(+), 18 deletions(-) create mode 100644 src/test/resources/com/manticore/transpiler/JSQLTranspilerTest_MainIn.sql create mode 100644 src/test/resources/com/manticore/transpiler/JSQLTranspilerTest_MainOut.sql diff --git a/src/main/java/com/manticore/transpiler/ExpressionTranspiler.java b/src/main/java/com/manticore/transpiler/ExpressionTranspiler.java index a1bc0c0..25a3df2 100644 --- a/src/main/java/com/manticore/transpiler/ExpressionTranspiler.java +++ b/src/main/java/com/manticore/transpiler/ExpressionTranspiler.java @@ -1,3 +1,20 @@ +/** + * Manticore Projects JSQLTranspiler is a multiple SQL Dialect to DuckDB Translation Software. + * Copyright (C) 2024 Andreas Reichel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ package com.manticore.transpiler; import net.sf.jsqlparser.expression.CastExpression; diff --git a/src/main/java/com/manticore/transpiler/JSQLTranspiler.java b/src/main/java/com/manticore/transpiler/JSQLTranspiler.java index 037d88f..69b0b72 100644 --- a/src/main/java/com/manticore/transpiler/JSQLTranspiler.java +++ b/src/main/java/com/manticore/transpiler/JSQLTranspiler.java @@ -1,8 +1,27 @@ +/** + * Manticore Projects JSQLTranspiler is a multiple SQL Dialect to DuckDB Translation Software. + * Copyright (C) 2024 Andreas Reichel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ package com.manticore.transpiler; +import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.parser.CCJSqlParserUtil; import net.sf.jsqlparser.parser.SimpleNode; import net.sf.jsqlparser.statement.Statement; +import net.sf.jsqlparser.statement.Statements; import net.sf.jsqlparser.statement.select.Limit; import net.sf.jsqlparser.statement.select.PlainSelect; import net.sf.jsqlparser.statement.select.Top; @@ -11,12 +30,15 @@ import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.DefaultParser; import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionGroup; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.io.IOUtils; import java.io.File; import java.io.FileInputStream; +import java.io.FileWriter; import java.io.IOException; import java.nio.charset.Charset; import java.nio.file.Path; @@ -31,7 +53,7 @@ public class JSQLTranspiler extends SelectDeParser { public final static Logger LOGGER = Logger.getLogger(JSQLTranspiler.class.getName()); public enum Dialect { - GOOGLE_BIG_QUERY, DATABRICKS, SNOWFLAKE, AMAZON_REDSHIFT, ANY + GOOGLE_BIG_QUERY, DATABRICKS, SNOWFLAKE, AMAZON_REDSHIFT, ANY, DUCK_DB } public static File getAbsoluteFile(String filename) { @@ -55,15 +77,40 @@ public static String getAbsoluteFileName(String filename) { return getAbsoluteFile(filename).getAbsolutePath(); } - @SuppressWarnings({"PMD.CyclomaticComplexity"}) + @SuppressWarnings({"PMD.CyclomaticComplexity", "PMD.ExcessiveMethodLength"}) public static void main(String[] args) { - System.out.println("Hello world"); - Options options = new Options(); - - options.addOption("i", "inputFile", true, "The input SQL file or folder."); - options.addOption("o", "outputFile", true, "The out SQL file for the formatted statements."); - + OptionGroup inputDialectOptions = new OptionGroup(); + inputDialectOptions.addOption(Option.builder("d").longOpt("input-dialect").hasArg() + .type(Dialect.class) + .desc( + "The SQL dialect to parse.\n[ANY*, GOOGLE_BIG_QUERY, DATABRICKS, SNOWFLAKE, AMAZON_REDSHIFT]") + .build()); + inputDialectOptions.addOption(Option.builder(null).longOpt("any").hasArg(false) + .desc("Interpret the SQL as Generic Dialect [DEFAULT].").build()); + inputDialectOptions.addOption(Option.builder(null).longOpt("bigquery").hasArg(false) + .desc("Interpret the SQL as Google BigQuery Dialect.").build()); + inputDialectOptions.addOption(Option.builder(null).longOpt("databricks").hasArg(false) + .desc("Interpret the SQL as DataBricks Dialect.").build()); + inputDialectOptions.addOption(Option.builder(null).longOpt("snowflake").hasArg(false) + .desc("Interpret the SQL as Snowflake Dialect.").build()); + inputDialectOptions.addOption(Option.builder(null).longOpt("redshift").hasArg(false) + .desc("Interpret the SQL as Amazon Snowflake Dialect.").build()); + + options.addOptionGroup(inputDialectOptions); + + + OptionGroup outputDialectOptions = new OptionGroup(); + outputDialectOptions.addOption(Option.builder("D").longOpt("output-dialect").hasArg() + .desc("The SQL dialect to write.\n[DUCKDB*]").build()); + outputDialectOptions.addOption(Option.builder(null).longOpt("duckdb").hasArg(false) + .desc("Write the SQL in the Duck DB Dialect [DEFAULT].").build()); + options.addOptionGroup(outputDialectOptions); + + options.addOption("i", "inputFile", true, + "The input SQL file or folder.\n - Read from STDIN when no input file provided."); + options.addOption("o", "outputFile", true, + "The out SQL file for the formatted statements.\n - Create new SQL file when folder provided.\n - Append when existing file provided.\n - Write to STDOUT when no output file provided."); // create the parser CommandLineParser parser = new DefaultParser(); @@ -83,6 +130,29 @@ public static void main(String[] args) { return; } + Dialect dialect = Dialect.ANY; + if (line.hasOption("d")) { + dialect = (Dialect) line.getParsedOptionValue("d"); + } else if (line.hasOption("bigquery")) { + dialect = Dialect.GOOGLE_BIG_QUERY; + } else if (line.hasOption("databricks")) { + dialect = Dialect.DATABRICKS; + } else if (line.hasOption("snowflake")) { + dialect = Dialect.SNOWFLAKE; + } else if (line.hasOption("redshift")) { + dialect = Dialect.AMAZON_REDSHIFT; + } + + File outputFile = null; + if (line.hasOption("outputFile")) { + outputFile = getAbsoluteFile(line.getOptionValue("outputFile")); + + // if an existing folder was provided, create a new file in it + if (outputFile.exists() && outputFile.isDirectory()) { + outputFile = File.createTempFile(dialect.name() + "_transpiled_", ".sql", outputFile); + } + } + File inputFile = null; if (line.hasOption("inputFile")) { inputFile = getAbsoluteFile(line.getOptionValue("inputFile")); @@ -94,21 +164,25 @@ public static void main(String[] args) { try (FileInputStream inputStream = new FileInputStream(inputFile)) { String sqlStr = IOUtils.toString(inputStream, Charset.defaultCharset()); - // @todo: do something useful here - System.out.println(sqlStr); + transpile(sqlStr, dialect, Dialect.DUCK_DB, outputFile); } catch (IOException ex) { throw new IOException( "Can't read the specified INPUT-FILE " + inputFile.getAbsolutePath(), ex); + } catch (JSQLParserException ex) { + throw new RuntimeException("Failed to parse the provided SQL.", ex); } } List argsList = line.getArgList(); - if (argsList.isEmpty() && !line.hasOption("input-file")) { + if (argsList.isEmpty() && !line.hasOption("inputFile")) { throw new IOException("No SQL statements provided for formatting."); } else { - for (String s : argsList) { - // @todo: do something useful here - System.out.println(s); + for (String sqlStr : argsList) { + try { + transpile(sqlStr, dialect, Dialect.DUCK_DB, outputFile); + } catch (JSQLParserException ex) { + throw new RuntimeException("Failed to parse the provided SQL.", ex); + } } } @@ -148,6 +222,50 @@ public static String transpileQuery(String qryStr, Dialect dialect) throws Excep } } + public static void transpile(String sqlStr, Dialect inputDialect, Dialect outputDialect, + File outputFile) throws JSQLParserException { + JSQLTranspiler transpiler = new JSQLTranspiler(); + + // @todo: we may need to split this manually to salvage any not parseable statements + Statements statements = CCJSqlParserUtil.parseStatements(sqlStr, parser -> { + parser.setErrorRecovery(true); + // parser.withTimeOut(60000); + // parser.withAllowComplexParsing(true); + }); + for (Statement st : statements) { + if (st instanceof PlainSelect) { + PlainSelect select = (PlainSelect) st; + transpiler.visit(select); + + transpiler.getResultBuilder().append("\n;\n\n"); + } else { + LOGGER.log(Level.SEVERE, + st.getClass().getSimpleName() + " is not supported yet:\n" + st.toString()); + } + } + + String transpiledSqlStr = transpiler.getResultBuilder().toString(); + LOGGER.fine("-- Transpiled SQL:\n" + transpiledSqlStr); + + // write to STDOUT when there is no OUTPUT File + if (outputFile == null) { + System.out.println(transpiledSqlStr); + } else { + if (!outputFile.exists() && outputFile.getParentFile() != null) { + boolean mkdirs = outputFile.getParentFile().mkdirs(); + if (mkdirs) { + LOGGER.fine("Created all the necessary folders."); + } + } + + try (FileWriter writer = new FileWriter(outputFile, Charset.defaultCharset(), true)) { + writer.write(transpiledSqlStr); + } catch (IOException e) { + LOGGER.log(Level.SEVERE, "Failed to write to " + outputFile.getAbsolutePath()); + } + } + } + public static String transpile(PlainSelect select) throws Exception { JSQLTranspiler transpiler = new JSQLTranspiler(); transpiler.visit(select); diff --git a/src/test/java/com/manticore/transpiler/AmazonRedshiftTranspilerTest.java b/src/test/java/com/manticore/transpiler/AmazonRedshiftTranspilerTest.java index 74f7e66..4605626 100644 --- a/src/test/java/com/manticore/transpiler/AmazonRedshiftTranspilerTest.java +++ b/src/test/java/com/manticore/transpiler/AmazonRedshiftTranspilerTest.java @@ -1,3 +1,20 @@ +/** + * Manticore Projects JSQLTranspiler is a multiple SQL Dialect to DuckDB Translation Software. + * Copyright (C) 2024 Andreas Reichel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ package com.manticore.transpiler; import org.junit.jupiter.params.ParameterizedTest; diff --git a/src/test/java/com/manticore/transpiler/GoogleBigQueryTranspilerTest.java b/src/test/java/com/manticore/transpiler/GoogleBigQueryTranspilerTest.java index 9a0e984..177c0c8 100644 --- a/src/test/java/com/manticore/transpiler/GoogleBigQueryTranspilerTest.java +++ b/src/test/java/com/manticore/transpiler/GoogleBigQueryTranspilerTest.java @@ -1,3 +1,20 @@ +/** + * Manticore Projects JSQLTranspiler is a multiple SQL Dialect to DuckDB Translation Software. + * Copyright (C) 2024 Andreas Reichel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ package com.manticore.transpiler; import org.junit.jupiter.params.ParameterizedTest; diff --git a/src/test/java/com/manticore/transpiler/JSQLTranspilerTest.java b/src/test/java/com/manticore/transpiler/JSQLTranspilerTest.java index 00a9f5e..79315d4 100644 --- a/src/test/java/com/manticore/transpiler/JSQLTranspilerTest.java +++ b/src/test/java/com/manticore/transpiler/JSQLTranspilerTest.java @@ -1,16 +1,33 @@ +/** + * Manticore Projects JSQLTranspiler is a multiple SQL Dialect to DuckDB Translation Software. + * Copyright (C) 2024 Andreas Reichel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ package com.manticore.transpiler; import com.opencsv.CSVWriter; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.parser.CCJSqlParserUtil; import net.sf.jsqlparser.statement.Statements; -import net.sf.jsqlparser.util.PerformanceTest; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; import org.apache.commons.io.IOUtils; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.function.Executable; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; @@ -60,7 +77,7 @@ class JSQLTranspilerTest { private static final Pattern SQL_SANITATION_PATTERN2 = Pattern.compile("\\s*([!/,()=+\\-*|\\]<>:])\\s*", Pattern.MULTILINE); - public final static String TEST_FOLDER_STR = "build/resources/test/com/manticore/transpiler/any"; + public final static String TEST_FOLDER_STR = "build/resources/test/com/manticore/transpiler"; public static final FilenameFilter FILENAME_FILTER = new FilenameFilter() { @Override @@ -69,6 +86,36 @@ public boolean accept(File dir, String name) { } }; + @Test + void main() throws IOException { + String providedSqlStr = IOUtils.resourceToString( + JSQLTranspilerTest.class.getCanonicalName().replaceAll("\\.", "/") + "_MainIn.sql", + Charset.defaultCharset(), JSQLTranspilerTest.class.getClassLoader()); + + String expectedSqlStr = IOUtils.resourceToString( + JSQLTranspilerTest.class.getCanonicalName().replaceAll("\\.", "/") + "_MainOut.sql", + Charset.defaultCharset(), JSQLTranspilerTest.class.getClassLoader()); + + String inputFileStr = TEST_FOLDER_STR + "/JSQLTranspilerTest_MainIn.sql"; + File outputFile = File.createTempFile("any_transpiled_", ".sql"); + + // Input file to Output file + String[] cmdLine = {"-i", inputFileStr, "--any", "-o", outputFile.getAbsolutePath()}; + JSQLTranspiler.main(cmdLine); + + // Input file to STDOUT + cmdLine = new String[]{"-i", inputFileStr, "--any"}; + JSQLTranspiler.main(cmdLine); + + // STDIN to STDOUT + cmdLine = new String[]{"--any", providedSqlStr}; + JSQLTranspiler.main(cmdLine); + + // STDIN to Output file + cmdLine = new String[]{"--any", "-o", outputFile.getAbsolutePath(), providedSqlStr}; + JSQLTranspiler.main(cmdLine); + } + static class SQLTest { String providedSqlStr; String expectedSqlStr; @@ -79,7 +126,7 @@ static class SQLTest { } static Stream> getSqlTestMap() { - return getSqlTestMap(new File(TEST_FOLDER_STR).listFiles(FILENAME_FILTER)); + return getSqlTestMap(new File(TEST_FOLDER_STR + "/any").listFiles(FILENAME_FILTER)); } static Stream> getSqlTestMap(File[] testFiles) { @@ -152,7 +199,7 @@ static void init() throws SQLException, IOException, JSQLParserException { if (!isInitialised) { String sqlStr = IOUtils.resourceToString( JSQLTranspilerTest.class.getCanonicalName().replaceAll("\\.", "/") + "_DDL.sql", - Charset.defaultCharset(), PerformanceTest.class.getClassLoader()); + Charset.defaultCharset(), JSQLTranspilerTest.class.getClassLoader()); Statements statements = CCJSqlParserUtil.parseStatements(sqlStr); LOGGER.info("Create the DuckDB Table with Indices"); diff --git a/src/test/resources/com/manticore/transpiler/JSQLTranspilerTest_MainIn.sql b/src/test/resources/com/manticore/transpiler/JSQLTranspilerTest_MainIn.sql new file mode 100644 index 0000000..888b149 --- /dev/null +++ b/src/test/resources/com/manticore/transpiler/JSQLTranspilerTest_MainIn.sql @@ -0,0 +1,23 @@ +SELECT Nvl( NULL, 1 ) a +; + +SELECT TOP 10 + qtysold + , sellerid +FROM sales +ORDER BY qtysold DESC + , sellerid +; + +SELECT TOP 10 + qtysold + , sellerid +FROM ( SELECT TOP 4 + qtysold + , sellerid + FROM sales + ORDER BY qtysold DESC + , sellerid ) a +ORDER BY qtysold DESC + , sellerid +; \ No newline at end of file diff --git a/src/test/resources/com/manticore/transpiler/JSQLTranspilerTest_MainOut.sql b/src/test/resources/com/manticore/transpiler/JSQLTranspilerTest_MainOut.sql new file mode 100644 index 0000000..e5f175f --- /dev/null +++ b/src/test/resources/com/manticore/transpiler/JSQLTranspilerTest_MainOut.sql @@ -0,0 +1,23 @@ +SELECT Coalesce( NULL, 1 ) a +; + +SELECT qtysold + , sellerid +FROM sales +ORDER BY qtysold DESC + , sellerid +LIMIT 10 +; + +SELECT qtysold + , sellerid +FROM ( SELECT qtysold + , sellerid + FROM sales + ORDER BY qtysold DESC + , sellerid + LIMIT 4 ) a +ORDER BY qtysold DESC + , sellerid +LIMIT 10 +; \ No newline at end of file