sys/shell: refactor tokenizer code.

sys/shell/shell.c

@@ -29,6 +29,7 @@
 #include <stdio.h>
 #include <stdint.h>
 #include <stdlib.h>
+#include <stdbool.h>
 #include "shell.h"
 #include "shell_commands.h"
 
@@ -45,171 +46,161 @@ static void _putchar(int c) {
 #endif
 #endif
 
+#ifdef MODULE_SHELL_COMMANDS
+#define _builtin_cmds _shell_command_list
+#else
+#define _builtin_cmds NULL
+#endif
+
 static void flush_if_needed(void)
 {
 #ifdef MODULE_NEWLIB
     fflush(stdout);
 #endif
 }
 
-static shell_command_handler_t find_handler(const shell_command_t *command_list, char *command)
+static shell_command_handler_t search_commands(const shell_command_t *entry,
+                                               char *command)
 {
-    const shell_command_t *command_lists[] = {
-        command_list,
-#ifdef MODULE_SHELL_COMMANDS
-        _shell_command_list,
-#endif
-    };
+    for (; entry->name != NULL; entry++) {
+        if (strcmp(entry->name, command) == 0) {
+            return entry->handler;
+        }
+    }
+    return NULL;
+}
 
-    /* iterating over command_lists */
-    for (unsigned int i = 0; i < ARRAY_SIZE(command_lists); i++) {
+static shell_command_handler_t find_handler(const shell_command_t *command_list, char *command)
+{
+    shell_command_handler_t handler = NULL;
 
-        const shell_command_t *entry;
+    if (command_list != NULL) {
+        handler = search_commands(command_list, command);
+    }
 
-        if ((entry = command_lists[i])) {
-            /* iterating over commands in command_lists entry */
-            while (entry->name != NULL) {
-                if (strcmp(entry->name, command) == 0) {
-                    return entry->handler;
-                }
-                else {
-                    entry++;
-                }
-            }
-        }
+    if (handler == NULL && _builtin_cmds != NULL) {
+        handler = search_commands(_builtin_cmds, command);
     }
 
-    return NULL;
+    return handler;
 }
 
-static void print_help(const shell_command_t *command_list)
+static void print_commands(const shell_command_t *entry)
 {
-    printf("%-20s %s\n", "Command", "Description");
-    puts("---------------------------------------");
-
-    const shell_command_t *command_lists[] = {
-        command_list,
-#ifdef MODULE_SHELL_COMMANDS
-        _shell_command_list,
-#endif
-    };
+    for (; entry->name != NULL; entry++) {
+        printf("%-20s %s\n", entry->name, entry->desc);
+    }
+}
 
-    /* iterating over command_lists */
-    for (unsigned int i = 0; i < ARRAY_SIZE(command_lists); i++) {
+static void print_help(const shell_command_t *command_list)
+{
+    puts("Command              Description\n---------------------------------------");
 
-        const shell_command_t *entry;
+    if (command_list != NULL) {
+        print_commands(command_list);
+    }
 
-        if ((entry = command_lists[i])) {
-            /* iterating over commands in command_lists entry */
-            while (entry->name != NULL) {
-                printf("%-20s %s\n", entry->name, entry->desc);
-                entry++;
-            }
-        }
+    if (_builtin_cmds != NULL) {
+        print_commands(_builtin_cmds);
     }
 }
 
+enum PARSE_STATE {
+    PARSE_SPACE,
+    PARSE_UNQUOTED,
+    PARSE_SINGLEQUOTE,
+    PARSE_DOUBLEQUOTE,
+    PARSE_ESCAPE_MASK,
+    PARSE_UNQUOTED_ESC,
+    PARSE_SINGLEQUOTE_ESC,
+    PARSE_DOUBLEQUOTE_ESC,
+};
+
+static enum PARSE_STATE escape_toggle(enum PARSE_STATE s)
+{
+    return s ^ PARSE_ESCAPE_MASK;
+}
+
+#define SQUOTE '\''
+#define DQUOTE '"'
+#define ESCAPECHAR '\\'
+#define BLANK ' '
+
 static void handle_input_line(const shell_command_t *command_list, char *line)
 {
     static const char *INCORRECT_QUOTING = "shell: incorrect quoting";
 
     /* first we need to calculate the number of arguments */
-    unsigned argc = 0;
-    char *pos = line;
-    int contains_esc_seq = 0;
-    while (1) {
-        if ((unsigned char) *pos > ' ') {
-            /* found an argument */
-            if (*pos == '"' || *pos == '\'') {
-                /* it's a quoted argument */
-                const char quote_char = *pos;
-                do {
-                    ++pos;
-                    if (!*pos) {
-                        puts(INCORRECT_QUOTING);
-                        return;
-                    }
-                    else if (*pos == '\\') {
-                        /* skip over the next character */
-                        ++contains_esc_seq;
-                        ++pos;
-                        if (!*pos) {
-                            puts(INCORRECT_QUOTING);
-                            return;
-                        }
-                        continue;
-                    }
-                } while (*pos != quote_char);
-                if ((unsigned char) pos[1] > ' ') {
-                    puts(INCORRECT_QUOTING);
-                    return;
+    int argc = 0;
+    char *readpos = line;
+    char *writepos = readpos;
+    enum PARSE_STATE pstate = PARSE_SPACE;
+
+    while (*readpos != '\0') {
+        char wordbreak;
+
+        switch (pstate) {
+            case PARSE_SPACE:
+                if (*readpos != BLANK) {
+                    argc++;
                 }
-            }
-            else {
-                /* it's an unquoted argument */
-                do {
-                    if (*pos == '\\') {
-                        /* skip over the next character */
-                        ++contains_esc_seq;
-                        ++pos;
-                        if (!*pos) {
-                            puts(INCORRECT_QUOTING);
-                            return;
-                        }
-                    }
-                    ++pos;
-                    if (*pos == '"') {
-                        puts(INCORRECT_QUOTING);
-                        return;
-                    }
-                } while ((unsigned char) *pos > ' ');
-            }
-
-            /* count the number of arguments we got */
-            ++argc;
+                if (*readpos == SQUOTE) {
+                    pstate = PARSE_SINGLEQUOTE;
+                } else if (*readpos == DQUOTE) {
+                    pstate = PARSE_DOUBLEQUOTE;
+                } else if (*readpos == ESCAPECHAR) {
+                    pstate = PARSE_UNQUOTED_ESC;
+                } else if (*readpos != BLANK) {
+                    pstate = PARSE_UNQUOTED;
+                    break;
+                }
+                goto parse_end;
+            case PARSE_UNQUOTED:
+                wordbreak = BLANK;
+                goto break_word;
+            case PARSE_SINGLEQUOTE:
+                wordbreak = SQUOTE;
+                goto break_word;
+            case PARSE_DOUBLEQUOTE:
+                wordbreak = DQUOTE;
+break_word:
+                if (*readpos == wordbreak) {
+                    pstate = PARSE_SPACE;
+                    *writepos++ = '\0';
+                } else if (*readpos == ESCAPECHAR) {
+                    pstate = escape_toggle(pstate);
+                } else {
+                    break;
+                }
+                goto parse_end;
+            default: /* QUOTED state */
+                pstate = escape_toggle(pstate);
+                break;
         }
+        *writepos++ = *readpos;
+parse_end:
+        readpos++;
+    }
 
-        /* zero out the current position (space or quotation mark) and advance */
-        if (*pos > 0) {
-            *pos = 0;
-            ++pos;
-        }
-        else {
-            break;
-        }
+    *writepos = '\0';
+
+    if (pstate != PARSE_SPACE && pstate != PARSE_UNQUOTED) {
+        puts(INCORRECT_QUOTING);
+        return;
     }
-    if (!argc) {
+
+    if (argc == 0) {
         return;
     }
 
     /* then we fill the argv array */
-    char *argv[argc + 1];
-    argv[argc] = NULL;
-    pos = line;
-    for (unsigned i = 0; i < argc; ++i) {
-        while (!*pos) {
-            ++pos;
-        }
-        if (*pos == '"' || *pos == '\'') {
-            ++pos;
-        }
-        argv[i] = pos;
-        while (*pos) {
-            ++pos;
-        }
-    }
-    for (char **arg = argv; contains_esc_seq && *arg; ++arg) {
-        for (char *c = *arg; *c; ++c) {
-            if (*c != '\\') {
-                continue;
-            }
-            for (char *d = c; *d; ++d) {
-                *d = d[1];
-            }
-            if (--contains_esc_seq == 0) {
-                break;
-            }
-        }
+    int collected;
+    char *argv[argc];
+
+    readpos = line;
+    for (collected = 0; collected < argc; collected++) {
+        argv[collected] = readpos;
+        readpos += strlen(readpos) + 1;
     }
 
     /* then we call the appropriate handler */

tests/shell/tests/01-run.py

@@ -49,6 +49,11 @@
     ('unknown_command', ('shell: command not found: unknown_command')),
     ('help', EXPECTED_HELP),
     ('echo a string', ('\"echo\"\"a\"\"string\"')),
+    ('echo "t\e st" "\\"" '"'\\'' a\ b", ('"echo""te st"""""\'""a b"')),
+    ('echo a\\', ('shell: incorrect quoting')),
+    ('echo "', ('shell: incorrect quoting')),
+    ("echo '", ('shell: incorrect quoting')),
+    ('echo "\'" \'"\'', ('"echo""\'""""')),
     ('ps', EXPECTED_PS),
     ('garbage1234'+CONTROL_C, ('>')),  # test cancelling a line
     ('help', EXPECTED_HELP),