From f6b196ba1df4ebe0522b9d185dbd1ad706c4e2f4 Mon Sep 17 00:00:00 2001 From: snuyanzin Date: Thu, 18 Oct 2018 17:36:26 +0300 Subject: [PATCH] [SQLLINE-151] Add property 'escapeOutput' to escape control symbols --- src/docbkx/manual.xml | 8 ++ src/main/java/sqlline/BuiltInProperty.java | 1 + src/main/java/sqlline/JsonOutputFormat.java | 54 +---------- src/main/java/sqlline/Rows.java | 92 ++++++++++++++++++- src/main/java/sqlline/SqlLineOpts.java | 5 + src/main/resources/sqlline/SqlLine.properties | 2 + src/main/resources/sqlline/manual.txt | 10 +- src/test/java/sqlline/RowsTest.java | 60 ++++++++++++ src/test/java/sqlline/SqlLineArgsTest.java | 20 ++++ 9 files changed, 197 insertions(+), 55 deletions(-) create mode 100644 src/test/java/sqlline/RowsTest.java diff --git a/src/docbkx/manual.xml b/src/docbkx/manual.xml index 08ca9a47..5c438870 100644 --- a/src/docbkx/manual.xml +++ b/src/docbkx/manual.xml @@ -2777,6 +2777,14 @@ java.sql.SQLException: ORA-00942: table or view does not exist the setting "YYYY_MM_dd" yields values like "1970_01_01". + + escapeOutput + + When set to true, control symbols + in output will be escaped. + Otherwise they will be printed as is. Default is false.. + + fastconnect diff --git a/src/main/java/sqlline/BuiltInProperty.java b/src/main/java/sqlline/BuiltInProperty.java index 985fc8ef..f93e8864 100644 --- a/src/main/java/sqlline/BuiltInProperty.java +++ b/src/main/java/sqlline/BuiltInProperty.java @@ -31,6 +31,7 @@ public enum BuiltInProperty implements SqlLineProperty { CSV_QUOTE_CHARACTER("csvQuoteCharacter", Type.CHAR, '\''), DATE_FORMAT("dateFormat", Type.STRING, DEFAULT), + ESCAPE_OUTPUT("escapeOutput", Type.BOOLEAN, false), FAST_CONNECT("fastConnect", Type.BOOLEAN, true), FORCE("force", Type.BOOLEAN, false), diff --git a/src/main/java/sqlline/JsonOutputFormat.java b/src/main/java/sqlline/JsonOutputFormat.java index de61c5d4..adee235c 100644 --- a/src/main/java/sqlline/JsonOutputFormat.java +++ b/src/main/java/sqlline/JsonOutputFormat.java @@ -13,61 +13,11 @@ import java.sql.SQLException; import java.sql.Types; -import java.util.HashMap; -import java.util.Map; /** * Implementation of {@link OutputFormat} that formats rows as JSON. */ public class JsonOutputFormat extends AbstractOutputFormat { - private static final Map ESCAPING_MAP = new HashMap<>(); - static { - ESCAPING_MAP.put('\\', "\\\\"); - ESCAPING_MAP.put('\"', "\\\""); - ESCAPING_MAP.put('\b', "\\b"); - ESCAPING_MAP.put('\f', "\\f"); - ESCAPING_MAP.put('\n', "\\n"); - ESCAPING_MAP.put('\r', "\\r"); - ESCAPING_MAP.put('\t', "\\t"); - ESCAPING_MAP.put('/', "\\/"); - ESCAPING_MAP.put('\u0000', "\\u0000"); - ESCAPING_MAP.put('\u0001', "\\u0001"); - ESCAPING_MAP.put('\u0002', "\\u0002"); - ESCAPING_MAP.put('\u0003', "\\u0003"); - ESCAPING_MAP.put('\u0004', "\\u0004"); - ESCAPING_MAP.put('\u0005', "\\u0005"); - ESCAPING_MAP.put('\u0006', "\\u0006"); - ESCAPING_MAP.put('\u0007', "\\u0007"); - // ESCAPING_MAP.put('\u0008', "\\u0008"); - // covered by ESCAPING_MAP.put('\b', "\\b"); - // ESCAPING_MAP.put('\u0009', "\\u0009"); - // covered by ESCAPING_MAP.put('\t', "\\t"); - // ESCAPING_MAP.put((char) 10, "\\u000A"); - // covered by ESCAPING_MAP.put('\n', "\\n"); - ESCAPING_MAP.put('\u000B', "\\u000B"); - // ESCAPING_MAP.put('\u000C', "\\u000C"); - // covered by ESCAPING_MAP.put('\f', "\\f"); - // ESCAPING_MAP.put((char) 13, "\\u000D"); - // covered by ESCAPING_MAP.put('\r', "\\r"); - ESCAPING_MAP.put('\u000E', "\\u000E"); - ESCAPING_MAP.put('\u000F', "\\u000F"); - ESCAPING_MAP.put('\u0010', "\\u0010"); - ESCAPING_MAP.put('\u0011', "\\u0011"); - ESCAPING_MAP.put('\u0012', "\\u0012"); - ESCAPING_MAP.put('\u0013', "\\u0013"); - ESCAPING_MAP.put('\u0014', "\\u0014"); - ESCAPING_MAP.put('\u0015', "\\u0015"); - ESCAPING_MAP.put('\u0016', "\\u0016"); - ESCAPING_MAP.put('\u0017', "\\u0017"); - ESCAPING_MAP.put('\u0018', "\\u0018"); - ESCAPING_MAP.put('\u0019', "\\u0019"); - ESCAPING_MAP.put('\u001A', "\\u001A"); - ESCAPING_MAP.put('\u001B', "\\u001B"); - ESCAPING_MAP.put('\u001C', "\\u001C"); - ESCAPING_MAP.put('\u001D', "\\u001D"); - ESCAPING_MAP.put('\u001E', "\\u001E"); - ESCAPING_MAP.put('\u001F', "\\u001F"); - } private int[] columnTypes; public JsonOutputFormat(SqlLine sqlLine) { super(sqlLine); @@ -122,8 +72,8 @@ private void setJsonValue(StringBuilder sb, String value, int columnTypeId) { } sb.append("\""); for (int i = 0; i < value.length(); i++) { - if (ESCAPING_MAP.get(value.charAt(i)) != null) { - sb.append(ESCAPING_MAP.get(value.charAt(i))); + if (Rows.ESCAPING_MAP.get(value.charAt(i)) != null) { + sb.append(Rows.ESCAPING_MAP.get(value.charAt(i))); } else { sb.append(value.charAt(i)); } diff --git a/src/main/java/sqlline/Rows.java b/src/main/java/sqlline/Rows.java index 53a891c3..d13d2bf6 100644 --- a/src/main/java/sqlline/Rows.java +++ b/src/main/java/sqlline/Rows.java @@ -22,6 +22,7 @@ import java.text.Format; import java.text.NumberFormat; import java.text.SimpleDateFormat; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -34,6 +35,8 @@ * Abstract base class representing a set of rows to be displayed. */ abstract class Rows implements Iterator { + static final Map ESCAPING_MAP = createEscapeMap(); + protected final SqlLine sqlLine; final ResultSetMetaData rsMeta; final Boolean[] primaryKeys; @@ -43,6 +46,7 @@ abstract class Rows implements Iterator { final DateFormat timeFormat; final DateFormat timestampFormat; final String nullValue; + final boolean escapeOutput; Rows(SqlLine sqlLine, ResultSet rs) throws SQLException { this.sqlLine = sqlLine; @@ -93,6 +97,7 @@ abstract class Rows implements Iterator { } else { nullValue = String.valueOf(nullPropertyValue); } + escapeOutput = sqlLine.getOpts().getEscapeOutput(); } public void remove() { @@ -159,6 +164,56 @@ boolean isPrimaryKey(int col) { } } + private static Map createEscapeMap() { + final Map map = new HashMap<>(); + map.put('\\', "\\\\"); + map.put('\"', "\\\""); + map.put('\b', "\\b"); + map.put('\f', "\\f"); + map.put('\n', "\\n"); + map.put('\r', "\\r"); + map.put('\t', "\\t"); + map.put('/', "\\/"); + map.put('\u0000', "\\u0000"); + map.put('\u0001', "\\u0001"); + map.put('\u0002', "\\u0002"); + map.put('\u0003', "\\u0003"); + map.put('\u0004', "\\u0004"); + map.put('\u0005', "\\u0005"); + map.put('\u0006', "\\u0006"); + map.put('\u0007', "\\u0007"); + // ESCAPING_MAP.put('\u0008', "\\u0008"); + // covered by ESCAPING_MAP.put('\b', "\\b"); + // ESCAPING_MAP.put('\u0009', "\\u0009"); + // covered by ESCAPING_MAP.put('\t', "\\t"); + // ESCAPING_MAP.put((char) 10, "\\u000A"); + // covered by ESCAPING_MAP.put('\n', "\\n"); + map.put('\u000B', "\\u000B"); + // ESCAPING_MAP.put('\u000C', "\\u000C"); + // covered by ESCAPING_MAP.put('\f', "\\f"); + // ESCAPING_MAP.put((char) 13, "\\u000D"); + // covered by ESCAPING_MAP.put('\r', "\\r"); + map.put('\u000E', "\\u000E"); + map.put('\u000F', "\\u000F"); + map.put('\u0010', "\\u0010"); + map.put('\u0011', "\\u0011"); + map.put('\u0012', "\\u0012"); + map.put('\u0013', "\\u0013"); + map.put('\u0014', "\\u0014"); + map.put('\u0015', "\\u0015"); + map.put('\u0016', "\\u0016"); + map.put('\u0017', "\\u0017"); + map.put('\u0018', "\\u0018"); + map.put('\u0019', "\\u0019"); + map.put('\u001A', "\\u001A"); + map.put('\u001B', "\\u001B"); + map.put('\u001C', "\\u001C"); + map.put('\u001D', "\\u001D"); + map.put('\u001E', "\\u001E"); + map.put('\u001F', "\\u001F"); + return Collections.unmodifiableMap(map); + } + /** Row from a result set. */ class Row { final String[] values; @@ -204,7 +259,6 @@ class Row { } for (int i = 0; i < size; i++) { - final Object o; switch (rs.getMetaData().getColumnType(i + 1)) { case Types.TINYINT: case Types.SMALLINT: @@ -241,7 +295,11 @@ class Row { values[i] = rs.getString(i + 1); break; } - values[i] = values[i] == null ? nullValue : values[i]; + values[i] = values[i] == null + ? nullValue + : escapeOutput + ? escapeControlSymbols(values[i]) + : values[i]; sizes[i] = values[i] == null ? 1 : values[i].length(); } } @@ -257,6 +315,36 @@ private void setFormat(Object o, Format format, int i) { } } + /** + * Escapes control symbols (Character.getType(ch) == Character.CONTROL). + * + * @param value String to be escaped + * + * @return escaped string if input value contains control symbols + * otherwise returns the input value + */ + static String escapeControlSymbols(String value) { + if (value == null || value.isEmpty()) { + return value; + } + StringBuilder result = null; + for (int i = 0; i < value.length(); i++) { + char ch = value.charAt(i); + if (Character.getType(ch) == Character.CONTROL) { + if (result == null) { + result = new StringBuilder(); + if (i != 0) { + result.append(value, 0, i); + } + } + result.append(ESCAPING_MAP.get(ch)); + } else if (result != null) { + result.append(ch); + } + } + return result == null ? value : result.toString(); + } + /** * Load and cache a set of primary key column names given a table key * (i.e. catalog, schema and table name). The result cannot be considered diff --git a/src/main/java/sqlline/SqlLineOpts.java b/src/main/java/sqlline/SqlLineOpts.java index 28453717..5d9ddca5 100644 --- a/src/main/java/sqlline/SqlLineOpts.java +++ b/src/main/java/sqlline/SqlLineOpts.java @@ -36,6 +36,7 @@ import static sqlline.BuiltInProperty.CSV_QUOTE_CHARACTER; import static sqlline.BuiltInProperty.DATE_FORMAT; import static sqlline.BuiltInProperty.DEFAULT; +import static sqlline.BuiltInProperty.ESCAPE_OUTPUT; import static sqlline.BuiltInProperty.FAST_CONNECT; import static sqlline.BuiltInProperty.FORCE; import static sqlline.BuiltInProperty.HEADER_INTERVAL; @@ -375,6 +376,10 @@ public String getNumberFormat() { return get(NUMBER_FORMAT); } + public boolean getEscapeOutput() { + return getBoolean(ESCAPE_OUTPUT); + } + public void setNumberFormat(String numberFormat) { if (DEFAULT.equalsIgnoreCase(numberFormat)) { propertiesMap.put(NUMBER_FORMAT, NUMBER_FORMAT.defaultValue()); diff --git a/src/main/resources/sqlline/SqlLine.properties b/src/main/resources/sqlline/SqlLine.properties index b05db830..56c16d17 100644 --- a/src/main/resources/sqlline/SqlLine.properties +++ b/src/main/resources/sqlline/SqlLine.properties @@ -73,6 +73,7 @@ variables:\ \ncsvDelimiter String Delimiter in csv outputFormat\ \ncsvQuoteCharacter char Quote character in csv outputFormat\ \ndateFormat pattern Format dates using SimpleDateFormat pattern\ +\nescapeOutput true/false Escape control symbols in output\ \nfastConnect true/false Skip building table/column list for tab-completion\ \nforce true/false Continue running script even after errors\ \nheaderInterval integer The interval between which headers are displayed\ @@ -221,6 +222,7 @@ cmd-usage: Usage: java sqlline.SqlLine \n \ \ --color=[true/false] control whether color is used for display\n \ \ --csvDelimiter=[delimiter] Delimiter in csv outputFormat\n \ \ --csvQuoteCharacter=[char] Quote character in csv outputFormat\n \ +\ --escapeOutput=[true/false] escape control symbols in output\n \ \ --showHeader=[true/false] show column names in query results\n \ \ --headerInterval=ROWS the interval between which headers are displayed\n \ \ --fastConnect=[true/false] skip building table/column list for tab-completion\n \ diff --git a/src/main/resources/sqlline/manual.txt b/src/main/resources/sqlline/manual.txt index 6e7440f7..39267823 100644 --- a/src/main/resources/sqlline/manual.txt +++ b/src/main/resources/sqlline/manual.txt @@ -103,6 +103,7 @@ color csvDelimiter csvQuoteCharacter dateformat +escapeOutput fastconnect force headerinterval @@ -229,6 +230,7 @@ color csvDelimiter csvQuoteCharacter dateformat +escapeOutput fastconnect force headerinterval @@ -1076,7 +1078,7 @@ sqlline> !help !rehash Fetch table and column names for command completion !rollback Roll back the current transaction (if autocommit is off) !run Run a script from the specified file -!save Save the current variabes and aliases +!save Save the current variables and aliases !scan Scan for installed JDBC drivers !script Start saving a script to a file !set Set a sqlline variable @@ -1096,6 +1098,7 @@ color true/false Control whether color is used for display csvDelimiter String Delimiter in csv outputFormat csvQuoteCharacter char Quote character in csv outputFormat dateFormat pattern Format dates using SimpleDateFormat pattern +escapeOutput true/false Escape control symbols in output fastConnect true/false Skip building table/column list for tab-completion force true/false Continue running script even after errors headerInterval integer The interval between which headers are displayed @@ -2021,6 +2024,7 @@ color csvDelimiter csvQuoteCharacter dateformat +escapeOutput fastconnect force headerinterval @@ -2068,6 +2072,10 @@ dateformat The format for how date values are displayed. Setting to default causes date values to be fetched and rendered via ResultSet.getString. Any other setting results in fetch via ResultSet.getObject and rendering via java.text.SimpleDateFormat. For example, the setting "YYYY_MM_dd" yields values like "1970_01_01". +escapeOutput + +When set to true, control symbols in output will be escaped. Otherwise they will be printed as is. Default is false. + fastconnect When false, any new connection will cause SQLLine to access information about the available tables and columns in order to provide them as candidates for tab-completion. This can be a very slow operation for some databases, do by default it is off. Table and column information can always be explicitly retrieved using the rehash command. diff --git a/src/test/java/sqlline/RowsTest.java b/src/test/java/sqlline/RowsTest.java new file mode 100644 index 00000000..a88b983a --- /dev/null +++ b/src/test/java/sqlline/RowsTest.java @@ -0,0 +1,60 @@ +/* +// Licensed to Julian Hyde under one or more contributor license +// agreements. See the NOTICE file distributed with this work for +// additional information regarding copyright ownership. +// +// Julian Hyde licenses this file to you under the Modified BSD License +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at: +// +// http://opensource.org/licenses/BSD-3-Clause +*/ +package sqlline; + +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +/** + * Test cases for Rows. + */ +public class RowsTest { + @Test + public void testEscapeControlSymbols() { + // empty string + assertThat(Rows.escapeControlSymbols(""), is("")); + // one symbol + assertThat(Rows.escapeControlSymbols("\u0000"), is("\\u0000")); + assertThat(Rows.escapeControlSymbols("\u001F"), is("\\u001F")); + assertThat(Rows.escapeControlSymbols("\n"), is("\\n")); + assertThat(Rows.escapeControlSymbols("\t"), is("\\t")); + assertThat(Rows.escapeControlSymbols("\b"), is("\\b")); + assertThat(Rows.escapeControlSymbols("\f"), is("\\f")); + assertThat(Rows.escapeControlSymbols("\r"), is("\\r")); + + // several control symbols + assertThat(Rows.escapeControlSymbols("\n\n\n"), is("\\n\\n\\n")); + assertThat(Rows.escapeControlSymbols("\t\n\b\f\r"), is("\\t\\n\\b\\f\\r")); + assertThat(Rows.escapeControlSymbols("str\tstr2\nstr3\bstr4\fstr5\rstr6"), + is("str\\tstr2\\nstr3\\bstr4\\fstr5\\rstr6")); + + // including spaces + assertThat(Rows.escapeControlSymbols(" \n "), is(" \\n ")); + assertThat(Rows.escapeControlSymbols(" \n"), is(" \\n")); + assertThat(Rows.escapeControlSymbols("\b text \n"), is("\\b text \\n")); + assertThat(Rows.escapeControlSymbols("\t text"), is("\\t text")); + assertThat(Rows.escapeControlSymbols(" text \t"), is(" text \\t")); + assertThat( + Rows.escapeControlSymbols("str \tstr2 \nstr3\b str4\f str5\r str6 "), + is("str \\tstr2 \\nstr3\\b str4\\f str5\\r str6 ")); + + // non control symbols should not be escaped + assertThat(Rows.escapeControlSymbols("\""), is("\"")); + assertThat(Rows.escapeControlSymbols("\\\""), is("\\\"")); + assertThat(Rows.escapeControlSymbols("\\"), is("\\")); + assertThat(Rows.escapeControlSymbols("\\\\"), is("\\\\")); + } +} + +// End RowsTest.java diff --git a/src/test/java/sqlline/SqlLineArgsTest.java b/src/test/java/sqlline/SqlLineArgsTest.java index a17b4b6a..247c9cfc 100644 --- a/src/test/java/sqlline/SqlLineArgsTest.java +++ b/src/test/java/sqlline/SqlLineArgsTest.java @@ -1400,6 +1400,26 @@ public void testEmptyScript() { allOf(containsString(line), not(containsString("Exception")))); } + @Test + public void testEscapeSqlMultiline() { + // Set width so we don't inherit from the current terminal. + final String script = "!set maxwidth 80\n" + + "!set escapeOutput yes\n" + + "values \n" + + "('\n" + + "multiline\n" + + "value') \n" + + ";\n"; + final String line1 = "" + + "+-----------------+\n" + + "| C1 |\n" + + "+-----------------+\n" + + "| \\nmultiline \\nvalue |\n" + + "+-----------------+\n"; + checkScriptFile(script, true, equalTo(SqlLine.Status.OK), + containsString(line1)); + } + @Test public void testSqlMultiline() { // Set width so we don't inherit from the current terminal.