Fix special chars escaping in string literals, clarify error messages…

… (display \r\n instead of empty line)

fixes antlr#2281, antlr#2885

runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/LexerExecDescriptors.java

@@ -1098,8 +1098,6 @@ public static class StackoverflowDueToNotEscapedHyphen extends BaseLexerTestDesc
 		@CommentHasStringValue
 		public String output;
 
-		public String errors = null;
-		public String startRule = "";
 		public String grammarName = "L";
 
 		/**
@@ -1109,4 +1107,28 @@ public static class StackoverflowDueToNotEscapedHyphen extends BaseLexerTestDesc
 		@CommentHasStringValue
 		public String grammar;
 	}
+
+	public static class EscapedCharacters extends BaseLexerTestDescriptor {
+		public String input = "x\r\n";
+
+		/**
+		 [@0,0:0='x',<3>,1:0]
+		 [@1,1:1='\r',<1>,1:1]
+		 [@2,2:2='\n',<2>,1:2]
+		 [@3,3:2='<EOF>',<-1>,2:0]
+		 */
+		@CommentHasStringValue
+		public String output;
+
+		public String grammarName = "L";
+
+		/**
+		 lexer grammar L;
+		 CR : '\\u000D';
+		 LF : '\\u000A';
+		 X  : 'x';
+		 */
+		@CommentHasStringValue
+		public String grammar;
+	}
 }

runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/SemPredEvalParserDescriptors.java

@@ -283,8 +283,7 @@ public static class PredFromAltTestedInLoopBack_1 extends PredFromAltTestedInLoo
 		public String input = "s\n\n\nx\n";
 		public String output = "(file_ (para (paraContent s) \\n \\n) (para (paraContent \\n x \\n)) <EOF>)\n";
 		/**
-		line 5:0 mismatched input '<EOF>' expecting {'s', '
-		', 'x'}
+		line 5:0 mismatched input '<EOF>' expecting {'s', '\n', 'x'}
 		 */
 		@CommentHasStringValue
 		public String errors;

tool/src/org/antlr/v4/codegen/Target.java

@@ -201,15 +201,15 @@ public String getTargetStringLiteralFromString(String s) {
 	public String getTargetStringLiteralFromANTLRStringLiteral(
 		CodeGenerator generator,
 		String literal,
-		boolean addQuotes)
+		boolean addQuotes,
+		boolean escapeSpecial)
 	{
 		StringBuilder sb = new StringBuilder();
-		String is = literal;
 
 		if ( addQuotes ) sb.append('"');
 
-		for (int i = 1; i < is.length() -1; ) {
-			int codePoint = is.codePointAt(i);
+		for (int i = 1; i < literal.length() -1; ) {
+			int codePoint = literal.codePointAt(i);
 			int toAdvance = Character.charCount(codePoint);
 			if  (codePoint == '\\') {
 				// Anything escaped is what it is! We assume that
@@ -218,7 +218,7 @@ public String getTargetStringLiteralFromANTLRStringLiteral(
 				// is what the default implementation is dealing with and remove
 				// the escape. The C target does this for instance.
 				//
-				int escapedCodePoint = is.codePointAt(i+toAdvance);
+				int escapedCodePoint = literal.codePointAt(i+toAdvance);
 				toAdvance++;
 				switch (escapedCodePoint) {
 					// Pass through any escapes that Java also needs
@@ -230,29 +230,38 @@ public String getTargetStringLiteralFromANTLRStringLiteral(
 					case    'f':
 					case    '\\':
 						// Pass the escape through
+						if (escapeSpecial && escapedCodePoint != '\\') {
+							sb.append('\\');
+						}
 						sb.append('\\');
 						sb.appendCodePoint(escapedCodePoint);
 						break;
 
 					case    'u':    // Either unnnn or u{nnnnnn}
-						if (is.charAt(i+toAdvance) == '{') {
-							while (is.charAt(i+toAdvance) != '}') {
+						if (literal.charAt(i+toAdvance) == '{') {
+							while (literal.charAt(i+toAdvance) != '}') {
 								toAdvance++;
 							}
 							toAdvance++;
 						}
 						else {
 							toAdvance += 4;
 						}
-						if ( i+toAdvance <= is.length() ) { // we might have an invalid \\uAB or something
-							String fullEscape = is.substring(i, i+toAdvance);
+						if ( i+toAdvance <= literal.length() ) { // we might have an invalid \\uAB or something
+							String fullEscape = literal.substring(i, i+toAdvance);
+							if (escapeSpecial) {
+								sb.append('\\');
+							}
 							appendUnicodeEscapedCodePoint(
 								CharSupport.getCharValueFromCharInGrammarLiteral(fullEscape),
 								sb);
 						}
 						break;
 					default:
 						if (shouldUseUnicodeEscapeForCodePointInDoubleQuotedString(escapedCodePoint)) {
+							if (escapeSpecial) {
+								sb.append('\\');
+							}
 							appendUnicodeEscapedCodePoint(escapedCodePoint, sb);
 						}
 						else {
@@ -268,6 +277,9 @@ public String getTargetStringLiteralFromANTLRStringLiteral(
 					sb.append("\\\"");
 				}
 				else if (shouldUseUnicodeEscapeForCodePointInDoubleQuotedString(codePoint)) {
+					if (escapeSpecial) {
+						sb.append('\\');
+					}
 					appendUnicodeEscapedCodePoint(codePoint, sb);
 				}
 				else {

tool/src/org/antlr/v4/codegen/model/Recognizer.java

@@ -101,9 +101,8 @@ protected static String translateTokenStringToTarget(String tokenName, CodeGener
 		}
 
 		if (tokenName.charAt(0) == '\'') {
-			boolean addQuotes = false;
 			String targetString =
-				gen.getTarget().getTargetStringLiteralFromANTLRStringLiteral(gen, tokenName, addQuotes);
+				gen.getTarget().getTargetStringLiteralFromANTLRStringLiteral(gen, tokenName, false, true);
 			return "\"'" + targetString + "'\"";
 		}
 		else {

tool/src/org/antlr/v4/codegen/model/SemPred.java

@@ -67,6 +67,7 @@ public SemPred(OutputModelFactory factory, ActionAST ast) {
 		else {
 			msg = gen.getTarget().getTargetStringLiteralFromANTLRStringLiteral(gen,
 																		  failNode.getText(),
+																		  true,
 																		  true);
 		}
 	}

tool/src/org/antlr/v4/codegen/target/DartTarget.java

@@ -53,8 +53,9 @@ public DartTarget(CodeGenerator gen) {
 	}
 
 	@Override
-	public String getTargetStringLiteralFromANTLRStringLiteral(CodeGenerator generator, String literal, boolean addQuotes) {
-		return super.getTargetStringLiteralFromANTLRStringLiteral(generator, literal, addQuotes).replace("$", "\\$");
+	public String getTargetStringLiteralFromANTLRStringLiteral(CodeGenerator generator, String literal, boolean addQuotes,
+															   boolean escapeSpecial) {
+		return super.getTargetStringLiteralFromANTLRStringLiteral(generator, literal, addQuotes, escapeSpecial).replace("$", "\\$");
 	}
 
 	@Override

tool/src/org/antlr/v4/codegen/target/PHPTarget.java

@@ -104,10 +104,10 @@ protected void appendUnicodeEscapedCodePoint(int codePoint, StringBuilder sb) {
 	}
 
    @Override
-   public String getTargetStringLiteralFromANTLRStringLiteral(CodeGenerator generator, String literal, boolean addQuotes) {
-	   String targetStringLiteral = super.getTargetStringLiteralFromANTLRStringLiteral(generator, literal, addQuotes);
+   public String getTargetStringLiteralFromANTLRStringLiteral(CodeGenerator generator, String literal, boolean addQuotes,
+															  boolean escapeSpecial) {
+	   String targetStringLiteral = super.getTargetStringLiteralFromANTLRStringLiteral(generator, literal, addQuotes, escapeSpecial);
 	   targetStringLiteral = targetStringLiteral.replace("$", "\\$");
-
 	   return targetStringLiteral;
    }
 }

tool/src/org/antlr/v4/misc/CharSupport.java

@@ -17,11 +17,11 @@ public class CharSupport {
 	/** When converting ANTLR char and string literals, here is the
 	 *  value set of escape chars.
 	 */
-	public static int ANTLRLiteralEscapedCharValue[] = new int[255];
+	public static int[] ANTLRLiteralEscapedCharValue = new int[255];
 
 	/** Given a char, we need to be able to show as an ANTLR literal.
 	 */
-	public static String ANTLRLiteralCharValueEscape[] = new String[255];
+	public static String[] ANTLRLiteralCharValueEscape = new String[255];
 
 	static {
 		ANTLRLiteralEscapedCharValue['n'] = '\n';

tool/src/org/antlr/v4/tool/Grammar.java

@@ -793,8 +793,9 @@ public String[] getTokenLiteralNames() {
 		}
 
 		for (Map.Entry<String, Integer> entry : stringLiteralToTypeMap.entrySet()) {
-			if (entry.getValue() >= 0 && entry.getValue() < literalNames.length && literalNames[entry.getValue()] == null) {
-				literalNames[entry.getValue()] = entry.getKey();
+			int value = entry.getValue();
+			if (value >= 0 && value < literalNames.length && literalNames[value] == null) {
+				literalNames[value] = entry.getKey();
 			}
 		}