Skip to content

Commit

Permalink
feat: provide methods accepting prepared ExecutorService and `Consu…
Browse files Browse the repository at this point in the history 
…mer`

Signed-off-by: Andreas Reichel <andreas@manticore-projects.com>
  • Loading branch information
@manticore-projects
manticore-projects committed May 17, 2024
1 parent 480c82d commit 4cd03f6
Show file tree
Hide file tree
Showing 2 changed files with 65 additions and 26 deletions.
80 changes: 61 additions & 19 deletions src/main/java/ai/starlake/transpiler/JSQLTranspiler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,11 +61,22 @@
* The type JSQLtranspiler.
*/
public class JSQLTranspiler extends SelectDeParser {
public static final Logger LOGGER = Logger.getLogger(JSQLTranspiler.class.getName());

/**
* The constant parser TIMEOUT in seconds.
*/
public static final int TIMEOUT = 6;


/**
* The constant LOGGER.
* The Expression transpiler.
*/
public final static Logger LOGGER = Logger.getLogger(JSQLTranspiler.class.getName());
protected final JSQLExpressionTranspiler expressionTranspiler;

/**
* The builder holding the rewritten SQL statement text.
*/
protected StringBuilder resultBuilder;

/**
Expand All @@ -82,7 +93,7 @@ protected JSQLTranspiler(Class<? extends JSQLExpressionTranspiler> expressionTra
this.setExpressionVisitor(expressionTranspiler);
} catch (NoSuchMethodException | InvocationTargetException | InstantiationException
| IllegalAccessException e) {
// this can't happen
// this really can't happen
throw new RuntimeException(e);
}
}
Expand Down Expand Up @@ -144,12 +155,21 @@ public static String transpileQuery(String qryStr, Dialect dialect)
});

executorService.shutdown();
boolean wasTerminated = executorService.awaitTermination(6, TimeUnit.SECONDS);
boolean wasTerminated = executorService.awaitTermination(TIMEOUT, TimeUnit.SECONDS);

return result;
}


/**
* Transpile a query string from a file or STDIN and write the transformed query string into a
* file or STDOUT. Using the provided Executor Service for observing the parser.
*
* @param sqlStr the original query string
* @param outputFile the output file, writing to STDOUT when not defined
* @param executorService the ExecutorService to use for running and observing JSQLParser
* @param consumer the parser configuration to use for the parsing
* @throws JSQLParserException a parser exception when the statement can't be parsed
*/
public static void transpile(String sqlStr, File outputFile, ExecutorService executorService,
Consumer<CCJSqlParser> consumer) throws JSQLParserException {
JSQLTranspiler transpiler = new JSQLTranspiler();
Expand All @@ -163,8 +183,7 @@ public static void transpile(String sqlStr, File outputFile, ExecutorService exe

transpiler.getResultBuilder().append("\n;\n\n");
} else {
LOGGER.log(Level.SEVERE,
st.getClass().getSimpleName() + " is not supported yet:\n" + st.toString());
LOGGER.log(Level.SEVERE, st.getClass().getSimpleName() + " is not supported yet:\n" + st);
}
}

Expand Down Expand Up @@ -209,7 +228,7 @@ public static boolean transpile(String sqlStr, File outputFile)
.withUnsupportedStatements();
});
executorService.shutdown();
return executorService.awaitTermination(6, TimeUnit.SECONDS);
return executorService.awaitTermination(TIMEOUT, TimeUnit.SECONDS);
}

/**
Expand Down Expand Up @@ -247,37 +266,64 @@ public static String readResource(URL url) throws IOException {
public static String readResource(Class<?> clazz, String suffix) throws IOException {
URL url = JSQLTranspiler.class
.getResource("/" + clazz.getCanonicalName().replaceAll("\\.", "/") + suffix);
assert url != null;
return readResource(url);
}

/**
* Get the Macro `CREATE FUNCTION` statements as a list of text
*
* Get the Macro `CREATE FUNCTION` statements as a list of text, using the provided
* ExecutorService to monitor the parser
*
* @param executorService the ExecutorService to use for running and observing JSQLParser
* @param consumer the parser configuration to use for the parsing
* @return the list of statement texts
* @throws IOException when the Macro resource file can't be read
* @throws JSQLParserException when statements in the Macro resource file can't be parsed
*/
public static Collection<String> getMacros() throws IOException, JSQLParserException {
public static Collection<String> getMacros(ExecutorService executorService,
Consumer<CCJSqlParser> consumer) throws IOException, JSQLParserException {
ArrayList<String> macroStrList = new ArrayList<>();
String sqlStr = readResource(JSQLTranspiler.class, "Macro.sql");

Statements statements = CCJSqlParserUtil.parseStatements(sqlStr);
Statements statements = CCJSqlParserUtil.parseStatements(sqlStr, executorService, consumer);
for (net.sf.jsqlparser.statement.Statement statement : statements) {
macroStrList.add(statement.toString());
}
return macroStrList;
}

/**
* Get the Macro `CREATE FUNCTION` statements as a list of text
*
*
* @return the list of statement texts
* @throws IOException when the Macro resource file can't be read
* @throws JSQLParserException when statements in the Macro resource file can't be parsed
* @throws InterruptedException when the parser does not return a result with 6 seconds
*/
public static Collection<String> getMacros()
throws IOException, JSQLParserException, InterruptedException {
ExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
Collection<String> macroStrList = getMacros(executorService, parser -> {
});

executorService.shutdown();
boolean wasTerminated = executorService.awaitTermination(TIMEOUT, TimeUnit.SECONDS);

return macroStrList;
}

/**
* Get the Macro `CREATE FUNCTION` statements as an Array of text
*
*
* @return the array of statement texts
* @throws IOException when the Macro resource file can't be read
* @throws JSQLParserException when statements in the Macro resource file can't be parsed
* @throws InterruptedException when the parser does not return a result with 6 seconds
*/
public static String[] getMacroArray() throws IOException, JSQLParserException {
public static String[] getMacroArray()
throws IOException, JSQLParserException, InterruptedException {
return getMacros().toArray(getMacros().toArray(new String[0]));
}

Expand All @@ -289,9 +335,10 @@ public static String[] getMacroArray() throws IOException, JSQLParserException {
* @throws IOException when the Macro resource file can't be read
* @throws JSQLParserException when statements in the Macro resource file can't be parsed
* @throws SQLException when statements can't be executed
* @throws InterruptedException when the parser does not return a result with 6 seconds
*/
public static void createMacros(Connection conn)
throws SQLException, IOException, JSQLParserException {
throws SQLException, IOException, JSQLParserException, InterruptedException {

LOGGER.info("Create the DuckDB Macros");
try (java.sql.Statement st = conn.createStatement()) {
Expand All @@ -307,7 +354,6 @@ public static void createMacros(Connection conn)
*
* @param select the select
* @return the string
* @throws Exception the exception
*/
public static String transpile(Select select) {
JSQLTranspiler transpiler = new JSQLTranspiler();
Expand All @@ -321,7 +367,6 @@ public static String transpile(Select select) {
*
* @param select the select
* @return the string
* @throws Exception the exception
*/
public static String transpileGoogleBigQuery(Select select) {
BigQueryTranspiler transpiler = new BigQueryTranspiler();
Expand All @@ -335,7 +380,6 @@ public static String transpileGoogleBigQuery(Select select) {
*
* @param select the select
* @return the string
* @throws Exception the exception
*/
public static String transpileDatabricksQuery(Select select) {
DatabricksTranspiler transpiler = new DatabricksTranspiler();
Expand All @@ -349,7 +393,6 @@ public static String transpileDatabricksQuery(Select select) {
*
* @param select the select
* @return the string
* @throws Exception the exception
*/
public static String transpileSnowflakeQuery(Select select) {
SnowflakeTranspiler transpiler = new SnowflakeTranspiler();
Expand All @@ -363,7 +406,6 @@ public static String transpileSnowflakeQuery(Select select) {
*
* @param select the select
* @return the string
* @throws Exception the exception
*/
public static String transpileAmazonRedshiftQuery(Select select) {
RedshiftTranspiler transpiler = new RedshiftTranspiler();
Expand Down
11 changes: 4 additions & 7 deletions src/test/java/ai/starlake/transpiler/JSQLTranspilerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,8 @@ protected static Map<File, List<SQLTest>> getSqlTestMap(File[] testFiles,
@BeforeAll
// setting up a TEST Database according to
// https://docs.aws.amazon.com/redshift/latest/dg/c_sampledb.html
static synchronized void init() throws SQLException, IOException, JSQLParserException {
static synchronized void init()
throws SQLException, IOException, JSQLParserException, InterruptedException {
File extractionPathFolder = new File(EXTRACTION_PATH);
boolean mkdirs = extractionPathFolder.mkdirs();

Expand Down Expand Up @@ -374,11 +375,7 @@ private static String upperCaseExceptEnclosed(String input) {
result.append(c);
escaped = true;
} else if (c == '\'') {
if (inQuotes && !escaped) {
inQuotes = false;
} else if (!inQuotes) {
inQuotes = true;
}
inQuotes = !inQuotes;
result.append(c);
} else {
result.append(inQuotes ? c : Character.toUpperCase(c));
Expand Down Expand Up @@ -411,7 +408,7 @@ public static String sanitize(final String originalSql, boolean laxDeparsingChec
sanitizedSqlStr = sanitizedSqlStr.substring(0, sanitizedSqlStr.length() - 1).trim();
}
// @todo: "SELECT 1 AS ago" would trigger this wrongly, so we need to do this more
// sophisticated
// in a more sophisticated way
// else if (sanitizedSqlStr.endsWith("go")) {
// sanitizedSqlStr = sanitizedSqlStr.substring(0, sanitizedSqlStr.length() - 2).trim();
// }
Expand Down

0 comments on commit 4cd03f6

Please sign in to comment.