[SQLLINE-129] Allow multiline query parsing in shell mode and files

src/main/java/sqlline/Commands.java

@@ -1424,35 +1424,38 @@ public void run(String line, DispatchCallback callback) {
       try {
         // ### NOTE: fix for sf.net bug 879427
         StringBuilder cmd = null;
+        String waitingPattern = null;
         for (;;) {
           String scriptLine = reader.readLine();
 
           if (scriptLine == null) {
             break;
           }
 
-          String trimmedLine = scriptLine.trim();
-          if (sqlLine.getOpts().getTrimScripts()) {
-            scriptLine = trimmedLine;
-          }
-
           if (cmd != null) {
             // we're continuing an existing command
             cmd.append(" \n");
             cmd.append(scriptLine);
-            if (trimmedLine.endsWith(";")) {
+
+            waitingPattern =
+                sqlLine.getWaitingPattern(scriptLine, waitingPattern);
+            if (waitingPattern == null) {
               // this command has terminated
-              cmds.add(cmd.toString());
+              cmds.add(sqlLine.getOpts().getTrimScripts()
+                  ? cmd.toString().trim() : cmd.toString());
               cmd = null;
             }
           } else {
             // we're starting a new command
-            if (sqlLine.needsContinuation(scriptLine)) {
+            waitingPattern =
+                sqlLine.getWaitingPattern(scriptLine, waitingPattern);
+            if (waitingPattern != null) {
               // multi-line
               cmd = new StringBuilder(scriptLine);
             } else {
               // single-line
-              cmds.add(scriptLine);
+              cmds.add(sqlLine.getOpts().getTrimScripts()
+                  ? scriptLine.trim() : scriptLine);
             }
           }
         }

src/main/java/sqlline/SqlLine.java

@@ -678,38 +678,81 @@ void dispatch(String line, DispatchCallback callback) {
   }
 
   /**
-   * Test whether a line requires a continuation.
+   * Return waiting pattern or null depending on
+   * whether a line requires a continuation or not.
    *
-   * @param line the line to be tested
-   * @return true if continuation required
+   * @param line            the line to be tested
+   * @param waitingPattern  the waiting pattern before processing {@code line}
+   * @return waitingPattern if continuation is required, null otherwise
    */
-  boolean needsContinuation(String line) {
-    if (null == line) {
-      // happens when CTRL-C used to exit a malformed.
-      return false;
-    }
+  String getWaitingPattern(String line, String waitingPattern) {
+    if (waitingPattern == null) {
 
-    if (isHelpRequest(line)) {
-      return false;
+      if (isHelpRequest(line)) {
+        return null;
+      }
+
+      if (line.startsWith(COMMAND_PREFIX)
+          && !line.regionMatches(1, "sql", 0, "sql".length())
+          && !line.regionMatches(1, "all", 0, "all".length())) {
+        return null;
+      }
+      if (!line.trim().startsWith("--") && !line.trim().startsWith("#")) {
+        waitingPattern = ";";
+      }
     }
 
-    if (line.startsWith(COMMAND_PREFIX)
-        && !line.regionMatches(1, "sql", 0, "sql".length())
-        && !line.regionMatches(1, "all", 0, "all".length())) {
-      return false;
+    if (null == line) {
+      // happens when CTRL-C used to exit a malformed.
+      return waitingPattern;
     }
 
-    if (isComment(line)) {
-      return false;
+    int startPosition = -1;
+    if ("'".equals(waitingPattern)
+        || "\"".equals(waitingPattern)
+        || "*/".equals(waitingPattern)) {
+      startPosition = line.indexOf(waitingPattern);
+      waitingPattern = ";";
+      if (startPosition == -1) {
+        return waitingPattern;
+      }
     }
+    boolean commented = false;
+    for (int i = startPosition + 1; i < line.length(); i++) {
+      final char ch = line.charAt(i);
+      if (waitingPattern == null || ";".equals(waitingPattern)) {
+        if (ch == '\'' || ch == '"') {
+          waitingPattern = String.valueOf(ch);
+        } else if (ch == '/'
+            && i < line.length() - 1
+            && line.charAt(i + 1) == '*') {
+          waitingPattern = "*/";
+        } else if (ch == '#'
+            || (ch == '-'
+                && i < line.length() - 1
+                && line.charAt(i + 1) == '-')) {
+          commented = true;
+          break;
+        } else if (ch == ';') {
+          waitingPattern = null;
+        }
+      } else if ("'".equals(waitingPattern) || "\"".equals(waitingPattern)) {
+        if (ch == waitingPattern.charAt(0)) {
+          waitingPattern = ";";
+        }
+      } else if ("*/".equals(waitingPattern)) {
+        if (ch == '*' && i < line.length() - 1 && line.charAt(i + 1) == '/') {
+          waitingPattern = ";";
+        }
+      }
 
-    String trimmed = line.trim();
+    }
 
-    if (trimmed.length() == 0) {
-      return false;
+    if (!commented && ";".equals(waitingPattern) && line.trim().endsWith(";")) {
+      return null;
     }
 
-    return !trimmed.endsWith(";");
+    return waitingPattern;
   }
 
   /**

src/main/java/sqlline/SqlLineParser.java

@@ -20,7 +20,27 @@
 
 /**
  * SqlLineParser implements multiline
- * for sql, !sql, !all while its not ended with ';'.
+ * for sql, !sql, !all while its not ended with non commented ';'.
+ * The following table shows each of the prompts you may see and
+ * summarizes what they mean about the state that sqlline is in.
+ *
+ * +---------------+-----------------------------------------------------+
+ * |    Prompt     |                    Meaning                          |
+ * +---------------+-----------------------------------------------------+
+ * | sqlline>   |  Ready for a new query                              |
+ * +---------------+-----------------------------------------------------+
+ * | semicolon> |  Waiting for next line of multiple-line query,      |
+ * |               |  waiting for completion of query with semicolon (;) |
+ * +---------------+-----------------------------------------------------+
+ * | quote>     |  Waiting for next line, waiting for completion of   |
+ * |               |  a string that began with a single quote (')        |
+ * +---------------+-----------------------------------------------------+
+ * | dquote>    |  Waiting for next line, waiting for completion of   |
+ * |               |  a string that began with a double quote (")        |
+ * +---------------+-----------------------------------------------------+
+ * | *\/>       |  Waiting for next line, waiting for completion of   |
+ * |               |  a multiline comment that began with /*             |
+ * +---------------+-----------------------------------------------------+
  */
 public class SqlLineParser extends DefaultParser {
   private final SqlLine sqlLine;
@@ -34,9 +54,12 @@ public ParsedLine parse(final String line, final int cursor,
     List<String> words = new LinkedList<>();
     StringBuilder current = new StringBuilder();
 
+    boolean containsNonCommentData = false;
     int wordCursor = -1;
     int wordIndex = -1;
     int quoteStart = -1;
+    int oneLineCommentStart = -1;
+    int multiLineCommentStart = -1;
     int rawWordCursor = -1;
     int rawWordLength = -1;
     int rawWordStart = 0;
@@ -52,10 +75,12 @@ public ParsedLine parse(final String line, final int cursor,
         wordCursor = current.length();
         rawWordCursor = i - rawWordStart;
       }
-
-      if (quoteStart < 0 && isQuoteChar(line, i)) {
+      if (oneLineCommentStart == -1
+          && multiLineCommentStart == -1
+          && quoteStart < 0 && isQuoteChar(line, i)) {
         // Start a quote block
         quoteStart = i;
+        containsNonCommentData = true;
       } else if (quoteStart >= 0) {
         // In a quote block
         if (line.charAt(quoteStart) == line.charAt(i) && !isEscaped(line, i)) {
@@ -72,16 +97,42 @@ public ParsedLine parse(final String line, final int cursor,
             current.append(line.charAt(i));
           }
         }
+      } else if (oneLineCommentStart == -1 && isMultilineComment(line, i)) {
+        multiLineCommentStart = i;
+        rawWordLength = getRawWordLength(
+            words, current, rawWordCursor, rawWordLength, rawWordStart, i);
+        rawWordStart = i + 1;
+      } else if (multiLineCommentStart >= 0) {
+        if (line.charAt(i) == '/' && line.charAt(i - 1) == '*') {
+          // End the block; arg could be empty, but that's fine
+          words.add(current.toString());
+          current.setLength(0);
+          multiLineCommentStart = -1;
+          if (rawWordCursor >= 0 && rawWordLength < 0) {
+            rawWordLength = i - rawWordStart + 1;
+          }
+        }
+      } else if (oneLineCommentStart == -1 && isOneLineComment(line, i)) {
+        oneLineCommentStart = i;
+        rawWordLength = getRawWordLength(
+            words, current, rawWordCursor, rawWordLength, rawWordStart, i);
+        rawWordStart = i + 1;
+      } else if (oneLineCommentStart >= 0) {
+        if (line.charAt(i) == 13) {
+          // End the block; arg could be empty, but that's fine
+          oneLineCommentStart = -1;
+          rawWordLength = getRawWordLength(
+              words, current, rawWordCursor, rawWordLength, rawWordStart, i);
+          rawWordStart = i + 1;
+        } else {
+          current.append(line.charAt(i));
+        }
       } else {
-        // Not in a quote block
+        // Not in a quote or comment block
+        containsNonCommentData = true;
         if (isDelimiter(line, i)) {
-          if (current.length() > 0) {
-            words.add(current.toString());
-            current.setLength(0); // reset the arg
-            if (rawWordCursor >= 0 && rawWordLength < 0) {
-              rawWordLength = i - rawWordStart;
-            }
-          }
+          rawWordLength = getRawWordLength(
+              words, current, rawWordCursor, rawWordLength, rawWordStart, i);
           rawWordStart = i + 1;
         } else {
           if (!isEscapeChar(line, i)) {
@@ -109,31 +160,96 @@ public ParsedLine parse(final String line, final int cursor,
       throw new EOFError(
           -1, -1, "Escaped new line", getPaddedPrompt("newline"));
     }
+
     if (isEofOnUnclosedQuote() && quoteStart >= 0
         && context != ParseContext.COMPLETE) {
       throw new EOFError(-1, -1, "Missing closing quote",
           getPaddedPrompt(line.charAt(quoteStart) == '\''
               ? "quote" : "dquote"));
     }
 
-    if (isSql && !line.trim().endsWith(";")
-        && context != ParseContext.COMPLETE) {
+    if (isSql && context != ParseContext.COMPLETE
+        && multiLineCommentStart != -1) {
+      throw new EOFError(-1, -1, "Missing end of comment",
+          getPaddedPrompt("*/"));
+    }
+
+    if (isSql && containsNonCommentData && context != ParseContext.COMPLETE
+        && !isLineFinishedWithSemicolon(line)) {
       throw new EOFError(-1, -1, "Missing semicolon at the end",
           getPaddedPrompt("semicolon"));
     }
+
     String openingQuote = quoteStart >= 0
         ? line.substring(quoteStart, quoteStart + 1) : null;
     return new ArgumentList(line, words, wordIndex, wordCursor,
         cursor, openingQuote, rawWordCursor, rawWordLength);
   }
 
+  private int getRawWordLength(
+      List<String> words,
+      StringBuilder current,
+      int rawWordCursor,
+      int rawWordLength,
+      int rawWordStart,
+      int i) {
+    if (current.length() > 0) {
+      words.add(current.toString());
+      current.setLength(0); // reset the arg
+      if (rawWordCursor >= 0 && rawWordLength < 0) {
+        rawWordLength = i - rawWordStart;
+      }
+    }
+    return rawWordLength;
+  }
+
   private boolean isSql(String line, ParseContext context) {
     String trimmedLine = trimLeadingSpacesIfPossible(line, context);
     return  !trimmedLine.isEmpty() && (trimmedLine.charAt(0) != '!'
         || trimmedLine.regionMatches(0, "!sql", 0, "!sql".length())
         || trimmedLine.regionMatches(0, "!all", 0, "!all".length()));
   }
 
+  /**
+   * Checks if the line (trimmed) ends with semicolon which
+   * is not commented with one line comment
+   * ASSUMPTION: to have correct behavior should be
+   * called after quote and multiline check calls
+   * @param buffer input line to check for ending with ;
+   * @return true if the ends with non commented `;`
+   */
+  private boolean isLineFinishedWithSemicolon(final CharSequence buffer) {
+    final String line = buffer.toString().trim();
+    boolean result = false;
+    for (int i = line.length() - 1; i >= 0; i--) {
+      switch (line.charAt(i)) {
+      case ';' :
+        result = true;
+        break;
+      case '-' :
+        if (i > 0 && line.charAt(i - 1) == '-') {
+          return false;
+        }
+        break;
+      case '\n' :
+        return result;
+      }
+    }
+    return result;
+  }
+
+  private boolean isOneLineComment(final CharSequence buffer, final int pos) {
+    return pos < buffer.length() - 1
+        && buffer.charAt(pos) == '-'
+        && buffer.charAt(pos + 1) == '-';
+  }
+
+  private boolean isMultilineComment(final CharSequence buffer, final int pos) {
+    return pos < buffer.length() - 1
+        && buffer.charAt(pos) == '/'
+        && buffer.charAt(pos + 1) == '*';
+  }
+
   public static String trimLeadingSpacesIfPossible(
       String line, ParseContext context) {
     if (context != ParseContext.ACCEPT_LINE) {

src/test/java/sqlline/SqlLineArgsTest.java

@@ -175,16 +175,29 @@ private File createTempFile(String prefix, String suffix, File directory) {
 
   @Test
   public void testMultilineScriptWithComments() {
-    final String scriptText =
+    final String script1Text =
         "-- a comment  \n values\n--comment\n (\n1\n, ' ab'\n--comment\n)\n;\n";
 
-    checkScriptFile(scriptText, true,
+    checkScriptFile(script1Text, true,
         equalTo(SqlLine.Status.OK),
         containsString("+-------------+-----+\n"
             + "|     C1      | C2  |\n"
             + "+-------------+-----+\n"
             + "| 1           |  ab |\n"
             + "+-------------+-----+"));
+
+    final String script2Text =
+        "--comment \n values (';\n' /* comment */, '\"'"
+        + "/*multiline;\n ;\n comment*/)\n -- ; \n; -- comment";
+
+    checkScriptFile(script2Text, true,
+        equalTo(SqlLine.Status.OK),
+        containsString("+-----+----+\n"
+            + "| C1  | C2 |\n"
+            + "+-----+----+\n"
+            + "| ; \n"
+            + " | \"  |\n"
+            + "+-----+----+"));
   }
 
   /**