Skip to content

Commit

Permalink
[SQLLINE-151] Add property 'escapeOutput' to escape control symbols
Browse files Browse the repository at this point in the history
  • Loading branch information
snuyanzin authored and julianhyde committed Oct 27, 2018
1 parent 6815905 commit f6b196b
Show file tree
Hide file tree
Showing 9 changed files with 197 additions and 55 deletions.
8 changes: 8 additions & 0 deletions src/docbkx/manual.xml
Original file line number Diff line number Diff line change
Expand Up @@ -2777,6 +2777,14 @@ java.sql.SQLException: ORA-00942: table or view does not exist
the setting "YYYY_MM_dd" yields values like "1970_01_01".
</para>
</sect1>
<sect1 id="setting_escapeOutput">
<title>escapeOutput</title>
<para>
When set to true, control symbols
in output will be escaped.
Otherwise they will be printed as is. Default is false..
</para>
</sect1>
<sect1 id="setting_fastconnect">
<title>fastconnect</title>
<para>
Expand Down
1 change: 1 addition & 0 deletions src/main/java/sqlline/BuiltInProperty.java
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ public enum BuiltInProperty implements SqlLineProperty {
CSV_QUOTE_CHARACTER("csvQuoteCharacter", Type.CHAR, '\''),

DATE_FORMAT("dateFormat", Type.STRING, DEFAULT),
ESCAPE_OUTPUT("escapeOutput", Type.BOOLEAN, false),

FAST_CONNECT("fastConnect", Type.BOOLEAN, true),
FORCE("force", Type.BOOLEAN, false),
Expand Down
54 changes: 2 additions & 52 deletions src/main/java/sqlline/JsonOutputFormat.java
Original file line number Diff line number Diff line change
Expand Up @@ -13,61 +13,11 @@

import java.sql.SQLException;
import java.sql.Types;
import java.util.HashMap;
import java.util.Map;

/**
* Implementation of {@link OutputFormat} that formats rows as JSON.
*/
public class JsonOutputFormat extends AbstractOutputFormat {
private static final Map<Character, String> ESCAPING_MAP = new HashMap<>();
static {
ESCAPING_MAP.put('\\', "\\\\");
ESCAPING_MAP.put('\"', "\\\"");
ESCAPING_MAP.put('\b', "\\b");
ESCAPING_MAP.put('\f', "\\f");
ESCAPING_MAP.put('\n', "\\n");
ESCAPING_MAP.put('\r', "\\r");
ESCAPING_MAP.put('\t', "\\t");
ESCAPING_MAP.put('/', "\\/");
ESCAPING_MAP.put('\u0000', "\\u0000");
ESCAPING_MAP.put('\u0001', "\\u0001");
ESCAPING_MAP.put('\u0002', "\\u0002");
ESCAPING_MAP.put('\u0003', "\\u0003");
ESCAPING_MAP.put('\u0004', "\\u0004");
ESCAPING_MAP.put('\u0005', "\\u0005");
ESCAPING_MAP.put('\u0006', "\\u0006");
ESCAPING_MAP.put('\u0007', "\\u0007");
// ESCAPING_MAP.put('\u0008', "\\u0008");
// covered by ESCAPING_MAP.put('\b', "\\b");
// ESCAPING_MAP.put('\u0009', "\\u0009");
// covered by ESCAPING_MAP.put('\t', "\\t");
// ESCAPING_MAP.put((char) 10, "\\u000A");
// covered by ESCAPING_MAP.put('\n', "\\n");
ESCAPING_MAP.put('\u000B', "\\u000B");
// ESCAPING_MAP.put('\u000C', "\\u000C");
// covered by ESCAPING_MAP.put('\f', "\\f");
// ESCAPING_MAP.put((char) 13, "\\u000D");
// covered by ESCAPING_MAP.put('\r', "\\r");
ESCAPING_MAP.put('\u000E', "\\u000E");
ESCAPING_MAP.put('\u000F', "\\u000F");
ESCAPING_MAP.put('\u0010', "\\u0010");
ESCAPING_MAP.put('\u0011', "\\u0011");
ESCAPING_MAP.put('\u0012', "\\u0012");
ESCAPING_MAP.put('\u0013', "\\u0013");
ESCAPING_MAP.put('\u0014', "\\u0014");
ESCAPING_MAP.put('\u0015', "\\u0015");
ESCAPING_MAP.put('\u0016', "\\u0016");
ESCAPING_MAP.put('\u0017', "\\u0017");
ESCAPING_MAP.put('\u0018', "\\u0018");
ESCAPING_MAP.put('\u0019', "\\u0019");
ESCAPING_MAP.put('\u001A', "\\u001A");
ESCAPING_MAP.put('\u001B', "\\u001B");
ESCAPING_MAP.put('\u001C', "\\u001C");
ESCAPING_MAP.put('\u001D', "\\u001D");
ESCAPING_MAP.put('\u001E', "\\u001E");
ESCAPING_MAP.put('\u001F', "\\u001F");
}
private int[] columnTypes;
public JsonOutputFormat(SqlLine sqlLine) {
super(sqlLine);
Expand Down Expand Up @@ -122,8 +72,8 @@ private void setJsonValue(StringBuilder sb, String value, int columnTypeId) {
}
sb.append("\"");
for (int i = 0; i < value.length(); i++) {
if (ESCAPING_MAP.get(value.charAt(i)) != null) {
sb.append(ESCAPING_MAP.get(value.charAt(i)));
if (Rows.ESCAPING_MAP.get(value.charAt(i)) != null) {
sb.append(Rows.ESCAPING_MAP.get(value.charAt(i)));
} else {
sb.append(value.charAt(i));
}
Expand Down
92 changes: 90 additions & 2 deletions src/main/java/sqlline/Rows.java
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import java.text.Format;
import java.text.NumberFormat;
import java.text.SimpleDateFormat;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
Expand All @@ -34,6 +35,8 @@
* Abstract base class representing a set of rows to be displayed.
*/
abstract class Rows implements Iterator<Rows.Row> {
static final Map<Character, String> ESCAPING_MAP = createEscapeMap();

protected final SqlLine sqlLine;
final ResultSetMetaData rsMeta;
final Boolean[] primaryKeys;
Expand All @@ -43,6 +46,7 @@ abstract class Rows implements Iterator<Rows.Row> {
final DateFormat timeFormat;
final DateFormat timestampFormat;
final String nullValue;
final boolean escapeOutput;

Rows(SqlLine sqlLine, ResultSet rs) throws SQLException {
this.sqlLine = sqlLine;
Expand Down Expand Up @@ -93,6 +97,7 @@ abstract class Rows implements Iterator<Rows.Row> {
} else {
nullValue = String.valueOf(nullPropertyValue);
}
escapeOutput = sqlLine.getOpts().getEscapeOutput();
}

public void remove() {
Expand Down Expand Up @@ -159,6 +164,56 @@ boolean isPrimaryKey(int col) {
}
}

private static Map<Character, String> createEscapeMap() {
final Map<Character, String> map = new HashMap<>();
map.put('\\', "\\\\");
map.put('\"', "\\\"");
map.put('\b', "\\b");
map.put('\f', "\\f");
map.put('\n', "\\n");
map.put('\r', "\\r");
map.put('\t', "\\t");
map.put('/', "\\/");
map.put('\u0000', "\\u0000");
map.put('\u0001', "\\u0001");
map.put('\u0002', "\\u0002");
map.put('\u0003', "\\u0003");
map.put('\u0004', "\\u0004");
map.put('\u0005', "\\u0005");
map.put('\u0006', "\\u0006");
map.put('\u0007', "\\u0007");
// ESCAPING_MAP.put('\u0008', "\\u0008");
// covered by ESCAPING_MAP.put('\b', "\\b");
// ESCAPING_MAP.put('\u0009', "\\u0009");
// covered by ESCAPING_MAP.put('\t', "\\t");
// ESCAPING_MAP.put((char) 10, "\\u000A");
// covered by ESCAPING_MAP.put('\n', "\\n");
map.put('\u000B', "\\u000B");
// ESCAPING_MAP.put('\u000C', "\\u000C");
// covered by ESCAPING_MAP.put('\f', "\\f");
// ESCAPING_MAP.put((char) 13, "\\u000D");
// covered by ESCAPING_MAP.put('\r', "\\r");
map.put('\u000E', "\\u000E");
map.put('\u000F', "\\u000F");
map.put('\u0010', "\\u0010");
map.put('\u0011', "\\u0011");
map.put('\u0012', "\\u0012");
map.put('\u0013', "\\u0013");
map.put('\u0014', "\\u0014");
map.put('\u0015', "\\u0015");
map.put('\u0016', "\\u0016");
map.put('\u0017', "\\u0017");
map.put('\u0018', "\\u0018");
map.put('\u0019', "\\u0019");
map.put('\u001A', "\\u001A");
map.put('\u001B', "\\u001B");
map.put('\u001C', "\\u001C");
map.put('\u001D', "\\u001D");
map.put('\u001E', "\\u001E");
map.put('\u001F', "\\u001F");
return Collections.unmodifiableMap(map);
}

/** Row from a result set. */
class Row {
final String[] values;
Expand Down Expand Up @@ -204,7 +259,6 @@ class Row {
}

for (int i = 0; i < size; i++) {
final Object o;
switch (rs.getMetaData().getColumnType(i + 1)) {
case Types.TINYINT:
case Types.SMALLINT:
Expand Down Expand Up @@ -241,7 +295,11 @@ class Row {
values[i] = rs.getString(i + 1);
break;
}
values[i] = values[i] == null ? nullValue : values[i];
values[i] = values[i] == null
? nullValue
: escapeOutput
? escapeControlSymbols(values[i])
: values[i];
sizes[i] = values[i] == null ? 1 : values[i].length();
}
}
Expand All @@ -257,6 +315,36 @@ private void setFormat(Object o, Format format, int i) {
}
}

/**
* Escapes control symbols (Character.getType(ch) == Character.CONTROL).
*
* @param value String to be escaped
*
* @return escaped string if input value contains control symbols
* otherwise returns the input value
*/
static String escapeControlSymbols(String value) {
if (value == null || value.isEmpty()) {
return value;
}
StringBuilder result = null;
for (int i = 0; i < value.length(); i++) {
char ch = value.charAt(i);
if (Character.getType(ch) == Character.CONTROL) {
if (result == null) {
result = new StringBuilder();
if (i != 0) {
result.append(value, 0, i);
}
}
result.append(ESCAPING_MAP.get(ch));
} else if (result != null) {
result.append(ch);
}
}
return result == null ? value : result.toString();
}

/**
* Load and cache a set of primary key column names given a table key
* (i.e. catalog, schema and table name). The result cannot be considered
Expand Down
5 changes: 5 additions & 0 deletions src/main/java/sqlline/SqlLineOpts.java
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import static sqlline.BuiltInProperty.CSV_QUOTE_CHARACTER;
import static sqlline.BuiltInProperty.DATE_FORMAT;
import static sqlline.BuiltInProperty.DEFAULT;
import static sqlline.BuiltInProperty.ESCAPE_OUTPUT;
import static sqlline.BuiltInProperty.FAST_CONNECT;
import static sqlline.BuiltInProperty.FORCE;
import static sqlline.BuiltInProperty.HEADER_INTERVAL;
Expand Down Expand Up @@ -375,6 +376,10 @@ public String getNumberFormat() {
return get(NUMBER_FORMAT);
}

public boolean getEscapeOutput() {
return getBoolean(ESCAPE_OUTPUT);
}

public void setNumberFormat(String numberFormat) {
if (DEFAULT.equalsIgnoreCase(numberFormat)) {
propertiesMap.put(NUMBER_FORMAT, NUMBER_FORMAT.defaultValue());
Expand Down
2 changes: 2 additions & 0 deletions src/main/resources/sqlline/SqlLine.properties
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ variables:\
\ncsvDelimiter String Delimiter in csv outputFormat\
\ncsvQuoteCharacter char Quote character in csv outputFormat\
\ndateFormat pattern Format dates using SimpleDateFormat pattern\
\nescapeOutput true/false Escape control symbols in output\
\nfastConnect true/false Skip building table/column list for tab-completion\
\nforce true/false Continue running script even after errors\
\nheaderInterval integer The interval between which headers are displayed\
Expand Down Expand Up @@ -221,6 +222,7 @@ cmd-usage: Usage: java sqlline.SqlLine \n \
\ --color=[true/false] control whether color is used for display\n \
\ --csvDelimiter=[delimiter] Delimiter in csv outputFormat\n \
\ --csvQuoteCharacter=[char] Quote character in csv outputFormat\n \
\ --escapeOutput=[true/false] escape control symbols in output\n \
\ --showHeader=[true/false] show column names in query results\n \
\ --headerInterval=ROWS the interval between which headers are displayed\n \
\ --fastConnect=[true/false] skip building table/column list for tab-completion\n \
Expand Down
10 changes: 9 additions & 1 deletion src/main/resources/sqlline/manual.txt
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ color
csvDelimiter
csvQuoteCharacter
dateformat
escapeOutput
fastconnect
force
headerinterval
Expand Down Expand Up @@ -229,6 +230,7 @@ color
csvDelimiter
csvQuoteCharacter
dateformat
escapeOutput
fastconnect
force
headerinterval
Expand Down Expand Up @@ -1076,7 +1078,7 @@ sqlline> !help
!rehash Fetch table and column names for command completion
!rollback Roll back the current transaction (if autocommit is off)
!run Run a script from the specified file
!save Save the current variabes and aliases
!save Save the current variables and aliases
!scan Scan for installed JDBC drivers
!script Start saving a script to a file
!set Set a sqlline variable
Expand All @@ -1096,6 +1098,7 @@ color true/false Control whether color is used for display
csvDelimiter String Delimiter in csv outputFormat
csvQuoteCharacter char Quote character in csv outputFormat
dateFormat pattern Format dates using SimpleDateFormat pattern
escapeOutput true/false Escape control symbols in output
fastConnect true/false Skip building table/column list for tab-completion
force true/false Continue running script even after errors
headerInterval integer The interval between which headers are displayed
Expand Down Expand Up @@ -2021,6 +2024,7 @@ color
csvDelimiter
csvQuoteCharacter
dateformat
escapeOutput
fastconnect
force
headerinterval
Expand Down Expand Up @@ -2068,6 +2072,10 @@ dateformat

The format for how date values are displayed. Setting to default causes date values to be fetched and rendered via ResultSet.getString. Any other setting results in fetch via ResultSet.getObject and rendering via java.text.SimpleDateFormat. For example, the setting "YYYY_MM_dd" yields values like "1970_01_01".

escapeOutput

When set to true, control symbols in output will be escaped. Otherwise they will be printed as is. Default is false.

fastconnect

When false, any new connection will cause SQLLine to access information about the available tables and columns in order to provide them as candidates for tab-completion. This can be a very slow operation for some databases, do by default it is off. Table and column information can always be explicitly retrieved using the rehash command.
Expand Down
60 changes: 60 additions & 0 deletions src/test/java/sqlline/RowsTest.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
/*
// Licensed to Julian Hyde under one or more contributor license
// agreements. See the NOTICE file distributed with this work for
// additional information regarding copyright ownership.
//
// Julian Hyde licenses this file to you under the Modified BSD License
// (the "License"); you may not use this file except in compliance with
// the License. You may obtain a copy of the License at:
//
// http://opensource.org/licenses/BSD-3-Clause
*/
package sqlline;

import org.junit.Test;

import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertThat;

/**
* Test cases for Rows.
*/
public class RowsTest {
@Test
public void testEscapeControlSymbols() {
// empty string
assertThat(Rows.escapeControlSymbols(""), is(""));
// one symbol
assertThat(Rows.escapeControlSymbols("\u0000"), is("\\u0000"));
assertThat(Rows.escapeControlSymbols("\u001F"), is("\\u001F"));
assertThat(Rows.escapeControlSymbols("\n"), is("\\n"));
assertThat(Rows.escapeControlSymbols("\t"), is("\\t"));
assertThat(Rows.escapeControlSymbols("\b"), is("\\b"));
assertThat(Rows.escapeControlSymbols("\f"), is("\\f"));
assertThat(Rows.escapeControlSymbols("\r"), is("\\r"));

// several control symbols
assertThat(Rows.escapeControlSymbols("\n\n\n"), is("\\n\\n\\n"));
assertThat(Rows.escapeControlSymbols("\t\n\b\f\r"), is("\\t\\n\\b\\f\\r"));
assertThat(Rows.escapeControlSymbols("str\tstr2\nstr3\bstr4\fstr5\rstr6"),
is("str\\tstr2\\nstr3\\bstr4\\fstr5\\rstr6"));

// including spaces
assertThat(Rows.escapeControlSymbols(" \n "), is(" \\n "));
assertThat(Rows.escapeControlSymbols(" \n"), is(" \\n"));
assertThat(Rows.escapeControlSymbols("\b text \n"), is("\\b text \\n"));
assertThat(Rows.escapeControlSymbols("\t text"), is("\\t text"));
assertThat(Rows.escapeControlSymbols(" text \t"), is(" text \\t"));
assertThat(
Rows.escapeControlSymbols("str \tstr2 \nstr3\b str4\f str5\r str6 "),
is("str \\tstr2 \\nstr3\\b str4\\f str5\\r str6 "));

// non control symbols should not be escaped
assertThat(Rows.escapeControlSymbols("\""), is("\""));
assertThat(Rows.escapeControlSymbols("\\\""), is("\\\""));
assertThat(Rows.escapeControlSymbols("\\"), is("\\"));
assertThat(Rows.escapeControlSymbols("\\\\"), is("\\\\"));
}
}

// End RowsTest.java
Loading

0 comments on commit f6b196b

Please sign in to comment.