From da3dcec8f337b073710c1e421bcfd99c4f47317a Mon Sep 17 00:00:00 2001 From: degory Date: Tue, 27 Feb 2024 23:13:49 +0100 Subject: [PATCH] String interpolation align and format support Enhancements: - Support alignment and format specifiers for interpolated expressions in string literals (#1071) - Improved error recovery and reporting for incomplete string interpolations --- .config/dotnet-tools.json | 2 +- Directory.Build.props | 2 +- .../err.expected | 2 +- .../test.ghul | 2 +- .../.vscode/tasks.json | 23 ++ .../err.expected | 1 + .../fail.expected | 1 + .../ghul.json | 6 + .../ghulflags | 1 + .../il.expected | 0 .../test.ghul | 9 + .../warn.expected | 0 .../.vscode/tasks.json | 23 ++ .../err.expected | 1 + .../fail.expected | 1 + .../ghul.json | 6 + .../ghulflags | 1 + .../il.expected | 0 .../test.ghul | 8 + .../warn.expected | 0 .../err.expected | 2 +- .../test.ghul | 3 +- .../.vscode/tasks.json | 23 ++ .../err.expected | 1 + .../fail.expected | 1 + .../ghul.json | 6 + .../ghulflags | 1 + .../il.expected | 0 .../test.ghul | 7 + .../warn.expected | 0 .../.vscode/tasks.json | 23 ++ .../err.expected | 1 + .../fail.expected | 1 + .../ghul.json | 6 + .../ghulflags | 1 + .../il.expected | 0 .../.vscode/tasks.json | 23 ++ .../err.expected | 1 + .../fail.expected | 1 + .../ghul.json | 6 + .../ghulflags | 1 + .../il.expected | 0 .../test.ghul | 8 + .../warn.expected | 0 .../test.ghul | 7 + .../warn.expected | 0 .../.vscode/tasks.json | 23 ++ .../err.expected | 1 + .../fail.expected | 1 + .../ghul.json | 6 + .../ghulflags | 1 + .../il.expected | 0 .../test.ghul | 7 + .../warn.expected | 0 .../.vscode/tasks.json | 23 ++ .../err.expected | 1 + .../fail.expected | 1 + .../ghul.json | 6 + .../ghulflags | 1 + .../il.expected | 0 .../test.ghul | 7 + .../warn.expected | 0 .../.vscode/tasks.json | 23 ++ .../err.expected | 1 + .../fail.expected | 1 + .../ghul.json | 6 + .../ghulflags | 1 + .../il.expected | 0 .../test.ghul | 7 + .../warn.expected | 0 .../.vscode/tasks.json | 23 ++ .../err.expected | 1 + .../fail.expected | 1 + .../ghul.json | 6 + .../ghulflags | 1 + .../il.expected | 0 .../test.ghul | 7 + .../warn.expected | 0 .../.vscode/tasks.json | 23 ++ .../err.expected | 1 + .../fail.expected | 1 + .../ghul.json | 6 + .../ghulflags | 1 + .../il.expected | 0 .../test.ghul | 7 + .../warn.expected | 0 src/lexical/token.ghul | 3 +- src/lexical/token_lookahead.ghul | 5 + src/lexical/token_names.ghul | 1 + src/lexical/tokenizer.ghul | 68 ++--- src/semantic/types/named.ghul | 2 - src/source/location.ghul | 14 + src/syntax/parsers/context.ghul | 4 + src/syntax/parsers/expressions/primary.ghul | 242 ++++++++++++++---- src/syntax/parsers/expressions/secondary.ghul | 13 + src/syntax/parsers/pragmas/pragma.ghul | 56 ++-- src/syntax/process/generate_il.ghul | 56 +++- src/syntax/process/printer/base.ghul | 20 +- .../expressions/string_interpolation.ghul | 29 ++- 99 files changed, 786 insertions(+), 134 deletions(-) create mode 100644 integration-tests/parse/incomplete-string-interpolation-10/.vscode/tasks.json create mode 100644 integration-tests/parse/incomplete-string-interpolation-10/err.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-10/fail.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-10/ghul.json create mode 100644 integration-tests/parse/incomplete-string-interpolation-10/ghulflags create mode 100644 integration-tests/parse/incomplete-string-interpolation-10/il.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-10/test.ghul create mode 100644 integration-tests/parse/incomplete-string-interpolation-10/warn.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-11/.vscode/tasks.json create mode 100644 integration-tests/parse/incomplete-string-interpolation-11/err.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-11/fail.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-11/ghul.json create mode 100644 integration-tests/parse/incomplete-string-interpolation-11/ghulflags create mode 100644 integration-tests/parse/incomplete-string-interpolation-11/il.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-11/test.ghul create mode 100644 integration-tests/parse/incomplete-string-interpolation-11/warn.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-3/.vscode/tasks.json create mode 100644 integration-tests/parse/incomplete-string-interpolation-3/err.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-3/fail.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-3/ghul.json create mode 100644 integration-tests/parse/incomplete-string-interpolation-3/ghulflags create mode 100644 integration-tests/parse/incomplete-string-interpolation-3/il.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-3/test.ghul create mode 100644 integration-tests/parse/incomplete-string-interpolation-3/warn.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-4/.vscode/tasks.json create mode 100644 integration-tests/parse/incomplete-string-interpolation-4/err.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-4/fail.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-4/ghul.json create mode 100644 integration-tests/parse/incomplete-string-interpolation-4/ghulflags create mode 100644 integration-tests/parse/incomplete-string-interpolation-4/il.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-4/incomplete-string-interpolation-2/.vscode/tasks.json create mode 100644 integration-tests/parse/incomplete-string-interpolation-4/incomplete-string-interpolation-2/err.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-4/incomplete-string-interpolation-2/fail.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-4/incomplete-string-interpolation-2/ghul.json create mode 100644 integration-tests/parse/incomplete-string-interpolation-4/incomplete-string-interpolation-2/ghulflags create mode 100644 integration-tests/parse/incomplete-string-interpolation-4/incomplete-string-interpolation-2/il.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-4/incomplete-string-interpolation-2/test.ghul create mode 100644 integration-tests/parse/incomplete-string-interpolation-4/incomplete-string-interpolation-2/warn.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-4/test.ghul create mode 100644 integration-tests/parse/incomplete-string-interpolation-4/warn.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-5/.vscode/tasks.json create mode 100644 integration-tests/parse/incomplete-string-interpolation-5/err.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-5/fail.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-5/ghul.json create mode 100644 integration-tests/parse/incomplete-string-interpolation-5/ghulflags create mode 100644 integration-tests/parse/incomplete-string-interpolation-5/il.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-5/test.ghul create mode 100644 integration-tests/parse/incomplete-string-interpolation-5/warn.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-6/.vscode/tasks.json create mode 100644 integration-tests/parse/incomplete-string-interpolation-6/err.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-6/fail.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-6/ghul.json create mode 100644 integration-tests/parse/incomplete-string-interpolation-6/ghulflags create mode 100644 integration-tests/parse/incomplete-string-interpolation-6/il.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-6/test.ghul create mode 100644 integration-tests/parse/incomplete-string-interpolation-6/warn.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-7/.vscode/tasks.json create mode 100644 integration-tests/parse/incomplete-string-interpolation-7/err.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-7/fail.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-7/ghul.json create mode 100644 integration-tests/parse/incomplete-string-interpolation-7/ghulflags create mode 100644 integration-tests/parse/incomplete-string-interpolation-7/il.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-7/test.ghul create mode 100644 integration-tests/parse/incomplete-string-interpolation-7/warn.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-8/.vscode/tasks.json create mode 100644 integration-tests/parse/incomplete-string-interpolation-8/err.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-8/fail.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-8/ghul.json create mode 100644 integration-tests/parse/incomplete-string-interpolation-8/ghulflags create mode 100644 integration-tests/parse/incomplete-string-interpolation-8/il.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-8/test.ghul create mode 100644 integration-tests/parse/incomplete-string-interpolation-8/warn.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-9/.vscode/tasks.json create mode 100644 integration-tests/parse/incomplete-string-interpolation-9/err.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-9/fail.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-9/ghul.json create mode 100644 integration-tests/parse/incomplete-string-interpolation-9/ghulflags create mode 100644 integration-tests/parse/incomplete-string-interpolation-9/il.expected create mode 100644 integration-tests/parse/incomplete-string-interpolation-9/test.ghul create mode 100644 integration-tests/parse/incomplete-string-interpolation-9/warn.expected diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index 5d9346cbd..5e7202263 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -15,7 +15,7 @@ ] }, "ghul.compiler": { - "version": "0.8.1", + "version": "0.8.2", "commands": [ "ghul-compiler" ] diff --git a/Directory.Build.props b/Directory.Build.props index 2c607c75b..ebbc094ce 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,6 +1,6 @@ - 0.8.2-alpha.127 + 0.8.3-alpha.56 $(NoWarn);NU1507 diff --git a/integration-tests/parse/incomplete-string-interpolation-1/err.expected b/integration-tests/parse/incomplete-string-interpolation-1/err.expected index 398daa44d..6800b07a9 100644 --- a/integration-tests/parse/incomplete-string-interpolation-1/err.expected +++ b/integration-tests/parse/incomplete-string-interpolation-1/err.expected @@ -1 +1 @@ -test.ghul: 6,5..7,1: error: newline in string literal +test.ghul: 6,17..6,18: error: newline in string literal diff --git a/integration-tests/parse/incomplete-string-interpolation-1/test.ghul b/integration-tests/parse/incomplete-string-interpolation-1/test.ghul index f602f11fe..15fcbaa07 100644 --- a/integration-tests/parse/incomplete-string-interpolation-1/test.ghul +++ b/integration-tests/parse/incomplete-string-interpolation-1/test.ghul @@ -2,7 +2,7 @@ entry() is let x = 10; let y = 20; - + "interpolate si diff --git a/integration-tests/parse/incomplete-string-interpolation-10/.vscode/tasks.json b/integration-tests/parse/incomplete-string-interpolation-10/.vscode/tasks.json new file mode 100644 index 000000000..49063613c --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-10/.vscode/tasks.json @@ -0,0 +1,23 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "Run test", + "command": "dotnet ghul-test \"${workspaceFolder}\"", + "type": "shell", + "group": { + "kind": "test", + "isDefault": true + } + }, + { + "label": "Capture test expectation", + "command": "../../../tasks/capture.sh \"${workspaceFolder}\"", + "type": "shell", + "group": { + "kind": "build", + "isDefault": true + } + } + ] +} \ No newline at end of file diff --git a/integration-tests/parse/incomplete-string-interpolation-10/err.expected b/integration-tests/parse/incomplete-string-interpolation-10/err.expected new file mode 100644 index 000000000..e00728923 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-10/err.expected @@ -0,0 +1 @@ +test.ghul: 8,5..8,6: error: newline in string literal diff --git a/integration-tests/parse/incomplete-string-interpolation-10/fail.expected b/integration-tests/parse/incomplete-string-interpolation-10/fail.expected new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-10/fail.expected @@ -0,0 +1 @@ + diff --git a/integration-tests/parse/incomplete-string-interpolation-10/ghul.json b/integration-tests/parse/incomplete-string-interpolation-10/ghul.json new file mode 100644 index 000000000..84539d122 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-10/ghul.json @@ -0,0 +1,6 @@ +{ + "compiler": "dotnet ../../../publish/ghul.dll", + "source": [ + "." + ] +} diff --git a/integration-tests/parse/incomplete-string-interpolation-10/ghulflags b/integration-tests/parse/incomplete-string-interpolation-10/ghulflags new file mode 100644 index 000000000..4d095b972 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-10/ghulflags @@ -0,0 +1 @@ +--type-check \ No newline at end of file diff --git a/integration-tests/parse/incomplete-string-interpolation-10/il.expected b/integration-tests/parse/incomplete-string-interpolation-10/il.expected new file mode 100644 index 000000000..e69de29bb diff --git a/integration-tests/parse/incomplete-string-interpolation-10/test.ghul b/integration-tests/parse/incomplete-string-interpolation-10/test.ghul new file mode 100644 index 000000000..949840369 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-10/test.ghul @@ -0,0 +1,9 @@ + +entry() is + let x = 10; + let y = 20; + + "interpolate {x,y:G} interpolate {x,1234: ABCDEF } blah blah"; + + " +si diff --git a/integration-tests/parse/incomplete-string-interpolation-10/warn.expected b/integration-tests/parse/incomplete-string-interpolation-10/warn.expected new file mode 100644 index 000000000..e69de29bb diff --git a/integration-tests/parse/incomplete-string-interpolation-11/.vscode/tasks.json b/integration-tests/parse/incomplete-string-interpolation-11/.vscode/tasks.json new file mode 100644 index 000000000..49063613c --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-11/.vscode/tasks.json @@ -0,0 +1,23 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "Run test", + "command": "dotnet ghul-test \"${workspaceFolder}\"", + "type": "shell", + "group": { + "kind": "test", + "isDefault": true + } + }, + { + "label": "Capture test expectation", + "command": "../../../tasks/capture.sh \"${workspaceFolder}\"", + "type": "shell", + "group": { + "kind": "build", + "isDefault": true + } + } + ] +} \ No newline at end of file diff --git a/integration-tests/parse/incomplete-string-interpolation-11/err.expected b/integration-tests/parse/incomplete-string-interpolation-11/err.expected new file mode 100644 index 000000000..3a2760dae --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-11/err.expected @@ -0,0 +1 @@ +test.ghul: 7,19..7,33: error: in primary expression: expected (, [, cast, char literal, false, float literal, identifier, if, int literal, isa, new, null, self, string literal, super, true, typeof or { but found } interpolate diff --git a/integration-tests/parse/incomplete-string-interpolation-11/fail.expected b/integration-tests/parse/incomplete-string-interpolation-11/fail.expected new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-11/fail.expected @@ -0,0 +1 @@ + diff --git a/integration-tests/parse/incomplete-string-interpolation-11/ghul.json b/integration-tests/parse/incomplete-string-interpolation-11/ghul.json new file mode 100644 index 000000000..84539d122 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-11/ghul.json @@ -0,0 +1,6 @@ +{ + "compiler": "dotnet ../../../publish/ghul.dll", + "source": [ + "." + ] +} diff --git a/integration-tests/parse/incomplete-string-interpolation-11/ghulflags b/integration-tests/parse/incomplete-string-interpolation-11/ghulflags new file mode 100644 index 000000000..4d095b972 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-11/ghulflags @@ -0,0 +1 @@ +--type-check \ No newline at end of file diff --git a/integration-tests/parse/incomplete-string-interpolation-11/il.expected b/integration-tests/parse/incomplete-string-interpolation-11/il.expected new file mode 100644 index 000000000..e69de29bb diff --git a/integration-tests/parse/incomplete-string-interpolation-11/test.ghul b/integration-tests/parse/incomplete-string-interpolation-11/test.ghul new file mode 100644 index 000000000..3937323c4 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-11/test.ghul @@ -0,0 +1,8 @@ + +entry() is + let x = 10; + let y = 20; + + // FIXME we need better error recovery here + "interpolate {} interpolate"; +si diff --git a/integration-tests/parse/incomplete-string-interpolation-11/warn.expected b/integration-tests/parse/incomplete-string-interpolation-11/warn.expected new file mode 100644 index 000000000..e69de29bb diff --git a/integration-tests/parse/incomplete-string-interpolation-2/err.expected b/integration-tests/parse/incomplete-string-interpolation-2/err.expected index 21542d488..89c7b01da 100644 --- a/integration-tests/parse/incomplete-string-interpolation-2/err.expected +++ b/integration-tests/parse/incomplete-string-interpolation-2/err.expected @@ -1 +1 @@ -test.ghul: 7,19..8,1: error: newline in string literal +test.ghul: 7,31..7,32: error: newline in string literal diff --git a/integration-tests/parse/incomplete-string-interpolation-2/test.ghul b/integration-tests/parse/incomplete-string-interpolation-2/test.ghul index 5fae893af..457afe736 100644 --- a/integration-tests/parse/incomplete-string-interpolation-2/test.ghul +++ b/integration-tests/parse/incomplete-string-interpolation-2/test.ghul @@ -4,6 +4,5 @@ entry() is let y = 20; // FIXME we need better error recovery here - "interpolate {} + "interpolate {} interpolate si - diff --git a/integration-tests/parse/incomplete-string-interpolation-3/.vscode/tasks.json b/integration-tests/parse/incomplete-string-interpolation-3/.vscode/tasks.json new file mode 100644 index 000000000..49063613c --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-3/.vscode/tasks.json @@ -0,0 +1,23 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "Run test", + "command": "dotnet ghul-test \"${workspaceFolder}\"", + "type": "shell", + "group": { + "kind": "test", + "isDefault": true + } + }, + { + "label": "Capture test expectation", + "command": "../../../tasks/capture.sh \"${workspaceFolder}\"", + "type": "shell", + "group": { + "kind": "build", + "isDefault": true + } + } + ] +} \ No newline at end of file diff --git a/integration-tests/parse/incomplete-string-interpolation-3/err.expected b/integration-tests/parse/incomplete-string-interpolation-3/err.expected new file mode 100644 index 000000000..262efc485 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-3/err.expected @@ -0,0 +1 @@ +test.ghul: 7,1..7,3: error: in primary expression: expected (, [, cast, char literal, false, float literal, identifier, if, int literal, isa, new, null, self, string literal, super, true, typeof or { but found si diff --git a/integration-tests/parse/incomplete-string-interpolation-3/fail.expected b/integration-tests/parse/incomplete-string-interpolation-3/fail.expected new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-3/fail.expected @@ -0,0 +1 @@ + diff --git a/integration-tests/parse/incomplete-string-interpolation-3/ghul.json b/integration-tests/parse/incomplete-string-interpolation-3/ghul.json new file mode 100644 index 000000000..84539d122 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-3/ghul.json @@ -0,0 +1,6 @@ +{ + "compiler": "dotnet ../../../publish/ghul.dll", + "source": [ + "." + ] +} diff --git a/integration-tests/parse/incomplete-string-interpolation-3/ghulflags b/integration-tests/parse/incomplete-string-interpolation-3/ghulflags new file mode 100644 index 000000000..4d095b972 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-3/ghulflags @@ -0,0 +1 @@ +--type-check \ No newline at end of file diff --git a/integration-tests/parse/incomplete-string-interpolation-3/il.expected b/integration-tests/parse/incomplete-string-interpolation-3/il.expected new file mode 100644 index 000000000..e69de29bb diff --git a/integration-tests/parse/incomplete-string-interpolation-3/test.ghul b/integration-tests/parse/incomplete-string-interpolation-3/test.ghul new file mode 100644 index 000000000..2a3ba1164 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-3/test.ghul @@ -0,0 +1,7 @@ + +entry() is + let x = 10; + let y = 20; + + "interpolate {x,y:G} interpolate { +si diff --git a/integration-tests/parse/incomplete-string-interpolation-3/warn.expected b/integration-tests/parse/incomplete-string-interpolation-3/warn.expected new file mode 100644 index 000000000..e69de29bb diff --git a/integration-tests/parse/incomplete-string-interpolation-4/.vscode/tasks.json b/integration-tests/parse/incomplete-string-interpolation-4/.vscode/tasks.json new file mode 100644 index 000000000..49063613c --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-4/.vscode/tasks.json @@ -0,0 +1,23 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "Run test", + "command": "dotnet ghul-test \"${workspaceFolder}\"", + "type": "shell", + "group": { + "kind": "test", + "isDefault": true + } + }, + { + "label": "Capture test expectation", + "command": "../../../tasks/capture.sh \"${workspaceFolder}\"", + "type": "shell", + "group": { + "kind": "build", + "isDefault": true + } + } + ] +} \ No newline at end of file diff --git a/integration-tests/parse/incomplete-string-interpolation-4/err.expected b/integration-tests/parse/incomplete-string-interpolation-4/err.expected new file mode 100644 index 000000000..e54786dd5 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-4/err.expected @@ -0,0 +1 @@ +test.ghul: 7,1..7,3: error: syntax error: expected } but found si diff --git a/integration-tests/parse/incomplete-string-interpolation-4/fail.expected b/integration-tests/parse/incomplete-string-interpolation-4/fail.expected new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-4/fail.expected @@ -0,0 +1 @@ + diff --git a/integration-tests/parse/incomplete-string-interpolation-4/ghul.json b/integration-tests/parse/incomplete-string-interpolation-4/ghul.json new file mode 100644 index 000000000..84539d122 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-4/ghul.json @@ -0,0 +1,6 @@ +{ + "compiler": "dotnet ../../../publish/ghul.dll", + "source": [ + "." + ] +} diff --git a/integration-tests/parse/incomplete-string-interpolation-4/ghulflags b/integration-tests/parse/incomplete-string-interpolation-4/ghulflags new file mode 100644 index 000000000..4d095b972 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-4/ghulflags @@ -0,0 +1 @@ +--type-check \ No newline at end of file diff --git a/integration-tests/parse/incomplete-string-interpolation-4/il.expected b/integration-tests/parse/incomplete-string-interpolation-4/il.expected new file mode 100644 index 000000000..e69de29bb diff --git a/integration-tests/parse/incomplete-string-interpolation-4/incomplete-string-interpolation-2/.vscode/tasks.json b/integration-tests/parse/incomplete-string-interpolation-4/incomplete-string-interpolation-2/.vscode/tasks.json new file mode 100644 index 000000000..49063613c --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-4/incomplete-string-interpolation-2/.vscode/tasks.json @@ -0,0 +1,23 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "Run test", + "command": "dotnet ghul-test \"${workspaceFolder}\"", + "type": "shell", + "group": { + "kind": "test", + "isDefault": true + } + }, + { + "label": "Capture test expectation", + "command": "../../../tasks/capture.sh \"${workspaceFolder}\"", + "type": "shell", + "group": { + "kind": "build", + "isDefault": true + } + } + ] +} \ No newline at end of file diff --git a/integration-tests/parse/incomplete-string-interpolation-4/incomplete-string-interpolation-2/err.expected b/integration-tests/parse/incomplete-string-interpolation-4/incomplete-string-interpolation-2/err.expected new file mode 100644 index 000000000..e3e83febb --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-4/incomplete-string-interpolation-2/err.expected @@ -0,0 +1 @@ +test.ghul: 7,19..7,20: error: newline in string literal diff --git a/integration-tests/parse/incomplete-string-interpolation-4/incomplete-string-interpolation-2/fail.expected b/integration-tests/parse/incomplete-string-interpolation-4/incomplete-string-interpolation-2/fail.expected new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-4/incomplete-string-interpolation-2/fail.expected @@ -0,0 +1 @@ + diff --git a/integration-tests/parse/incomplete-string-interpolation-4/incomplete-string-interpolation-2/ghul.json b/integration-tests/parse/incomplete-string-interpolation-4/incomplete-string-interpolation-2/ghul.json new file mode 100644 index 000000000..84539d122 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-4/incomplete-string-interpolation-2/ghul.json @@ -0,0 +1,6 @@ +{ + "compiler": "dotnet ../../../publish/ghul.dll", + "source": [ + "." + ] +} diff --git a/integration-tests/parse/incomplete-string-interpolation-4/incomplete-string-interpolation-2/ghulflags b/integration-tests/parse/incomplete-string-interpolation-4/incomplete-string-interpolation-2/ghulflags new file mode 100644 index 000000000..4d095b972 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-4/incomplete-string-interpolation-2/ghulflags @@ -0,0 +1 @@ +--type-check \ No newline at end of file diff --git a/integration-tests/parse/incomplete-string-interpolation-4/incomplete-string-interpolation-2/il.expected b/integration-tests/parse/incomplete-string-interpolation-4/incomplete-string-interpolation-2/il.expected new file mode 100644 index 000000000..e69de29bb diff --git a/integration-tests/parse/incomplete-string-interpolation-4/incomplete-string-interpolation-2/test.ghul b/integration-tests/parse/incomplete-string-interpolation-4/incomplete-string-interpolation-2/test.ghul new file mode 100644 index 000000000..914528671 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-4/incomplete-string-interpolation-2/test.ghul @@ -0,0 +1,8 @@ + +entry() is + let x = 10; + let y = 20; + + // FIXME we need better error recovery here + "interpolate {} +si diff --git a/integration-tests/parse/incomplete-string-interpolation-4/incomplete-string-interpolation-2/warn.expected b/integration-tests/parse/incomplete-string-interpolation-4/incomplete-string-interpolation-2/warn.expected new file mode 100644 index 000000000..e69de29bb diff --git a/integration-tests/parse/incomplete-string-interpolation-4/test.ghul b/integration-tests/parse/incomplete-string-interpolation-4/test.ghul new file mode 100644 index 000000000..74551438b --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-4/test.ghul @@ -0,0 +1,7 @@ + +entry() is + let x = 10; + let y = 20; + + "interpolate {x,y:G} interpolate {x +si diff --git a/integration-tests/parse/incomplete-string-interpolation-4/warn.expected b/integration-tests/parse/incomplete-string-interpolation-4/warn.expected new file mode 100644 index 000000000..e69de29bb diff --git a/integration-tests/parse/incomplete-string-interpolation-5/.vscode/tasks.json b/integration-tests/parse/incomplete-string-interpolation-5/.vscode/tasks.json new file mode 100644 index 000000000..49063613c --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-5/.vscode/tasks.json @@ -0,0 +1,23 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "Run test", + "command": "dotnet ghul-test \"${workspaceFolder}\"", + "type": "shell", + "group": { + "kind": "test", + "isDefault": true + } + }, + { + "label": "Capture test expectation", + "command": "../../../tasks/capture.sh \"${workspaceFolder}\"", + "type": "shell", + "group": { + "kind": "build", + "isDefault": true + } + } + ] +} \ No newline at end of file diff --git a/integration-tests/parse/incomplete-string-interpolation-5/err.expected b/integration-tests/parse/incomplete-string-interpolation-5/err.expected new file mode 100644 index 000000000..262efc485 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-5/err.expected @@ -0,0 +1 @@ +test.ghul: 7,1..7,3: error: in primary expression: expected (, [, cast, char literal, false, float literal, identifier, if, int literal, isa, new, null, self, string literal, super, true, typeof or { but found si diff --git a/integration-tests/parse/incomplete-string-interpolation-5/fail.expected b/integration-tests/parse/incomplete-string-interpolation-5/fail.expected new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-5/fail.expected @@ -0,0 +1 @@ + diff --git a/integration-tests/parse/incomplete-string-interpolation-5/ghul.json b/integration-tests/parse/incomplete-string-interpolation-5/ghul.json new file mode 100644 index 000000000..84539d122 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-5/ghul.json @@ -0,0 +1,6 @@ +{ + "compiler": "dotnet ../../../publish/ghul.dll", + "source": [ + "." + ] +} diff --git a/integration-tests/parse/incomplete-string-interpolation-5/ghulflags b/integration-tests/parse/incomplete-string-interpolation-5/ghulflags new file mode 100644 index 000000000..4d095b972 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-5/ghulflags @@ -0,0 +1 @@ +--type-check \ No newline at end of file diff --git a/integration-tests/parse/incomplete-string-interpolation-5/il.expected b/integration-tests/parse/incomplete-string-interpolation-5/il.expected new file mode 100644 index 000000000..e69de29bb diff --git a/integration-tests/parse/incomplete-string-interpolation-5/test.ghul b/integration-tests/parse/incomplete-string-interpolation-5/test.ghul new file mode 100644 index 000000000..b5e2f84d4 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-5/test.ghul @@ -0,0 +1,7 @@ + +entry() is + let x = 10; + let y = 20; + + "interpolate {x,y:G} interpolate {x, +si diff --git a/integration-tests/parse/incomplete-string-interpolation-5/warn.expected b/integration-tests/parse/incomplete-string-interpolation-5/warn.expected new file mode 100644 index 000000000..e69de29bb diff --git a/integration-tests/parse/incomplete-string-interpolation-6/.vscode/tasks.json b/integration-tests/parse/incomplete-string-interpolation-6/.vscode/tasks.json new file mode 100644 index 000000000..49063613c --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-6/.vscode/tasks.json @@ -0,0 +1,23 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "Run test", + "command": "dotnet ghul-test \"${workspaceFolder}\"", + "type": "shell", + "group": { + "kind": "test", + "isDefault": true + } + }, + { + "label": "Capture test expectation", + "command": "../../../tasks/capture.sh \"${workspaceFolder}\"", + "type": "shell", + "group": { + "kind": "build", + "isDefault": true + } + } + ] +} \ No newline at end of file diff --git a/integration-tests/parse/incomplete-string-interpolation-6/err.expected b/integration-tests/parse/incomplete-string-interpolation-6/err.expected new file mode 100644 index 000000000..e54786dd5 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-6/err.expected @@ -0,0 +1 @@ +test.ghul: 7,1..7,3: error: syntax error: expected } but found si diff --git a/integration-tests/parse/incomplete-string-interpolation-6/fail.expected b/integration-tests/parse/incomplete-string-interpolation-6/fail.expected new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-6/fail.expected @@ -0,0 +1 @@ + diff --git a/integration-tests/parse/incomplete-string-interpolation-6/ghul.json b/integration-tests/parse/incomplete-string-interpolation-6/ghul.json new file mode 100644 index 000000000..84539d122 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-6/ghul.json @@ -0,0 +1,6 @@ +{ + "compiler": "dotnet ../../../publish/ghul.dll", + "source": [ + "." + ] +} diff --git a/integration-tests/parse/incomplete-string-interpolation-6/ghulflags b/integration-tests/parse/incomplete-string-interpolation-6/ghulflags new file mode 100644 index 000000000..4d095b972 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-6/ghulflags @@ -0,0 +1 @@ +--type-check \ No newline at end of file diff --git a/integration-tests/parse/incomplete-string-interpolation-6/il.expected b/integration-tests/parse/incomplete-string-interpolation-6/il.expected new file mode 100644 index 000000000..e69de29bb diff --git a/integration-tests/parse/incomplete-string-interpolation-6/test.ghul b/integration-tests/parse/incomplete-string-interpolation-6/test.ghul new file mode 100644 index 000000000..d32c1683e --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-6/test.ghul @@ -0,0 +1,7 @@ + +entry() is + let x = 10; + let y = 20; + + "interpolate {x,y:G} interpolate {x,1234 +si diff --git a/integration-tests/parse/incomplete-string-interpolation-6/warn.expected b/integration-tests/parse/incomplete-string-interpolation-6/warn.expected new file mode 100644 index 000000000..e69de29bb diff --git a/integration-tests/parse/incomplete-string-interpolation-7/.vscode/tasks.json b/integration-tests/parse/incomplete-string-interpolation-7/.vscode/tasks.json new file mode 100644 index 000000000..49063613c --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-7/.vscode/tasks.json @@ -0,0 +1,23 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "Run test", + "command": "dotnet ghul-test \"${workspaceFolder}\"", + "type": "shell", + "group": { + "kind": "test", + "isDefault": true + } + }, + { + "label": "Capture test expectation", + "command": "../../../tasks/capture.sh \"${workspaceFolder}\"", + "type": "shell", + "group": { + "kind": "build", + "isDefault": true + } + } + ] +} \ No newline at end of file diff --git a/integration-tests/parse/incomplete-string-interpolation-7/err.expected b/integration-tests/parse/incomplete-string-interpolation-7/err.expected new file mode 100644 index 000000000..5069f553a --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-7/err.expected @@ -0,0 +1 @@ +test.ghul: 6,53..6,54: error: newline in interpolation format string diff --git a/integration-tests/parse/incomplete-string-interpolation-7/fail.expected b/integration-tests/parse/incomplete-string-interpolation-7/fail.expected new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-7/fail.expected @@ -0,0 +1 @@ + diff --git a/integration-tests/parse/incomplete-string-interpolation-7/ghul.json b/integration-tests/parse/incomplete-string-interpolation-7/ghul.json new file mode 100644 index 000000000..84539d122 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-7/ghul.json @@ -0,0 +1,6 @@ +{ + "compiler": "dotnet ../../../publish/ghul.dll", + "source": [ + "." + ] +} diff --git a/integration-tests/parse/incomplete-string-interpolation-7/ghulflags b/integration-tests/parse/incomplete-string-interpolation-7/ghulflags new file mode 100644 index 000000000..4d095b972 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-7/ghulflags @@ -0,0 +1 @@ +--type-check \ No newline at end of file diff --git a/integration-tests/parse/incomplete-string-interpolation-7/il.expected b/integration-tests/parse/incomplete-string-interpolation-7/il.expected new file mode 100644 index 000000000..e69de29bb diff --git a/integration-tests/parse/incomplete-string-interpolation-7/test.ghul b/integration-tests/parse/incomplete-string-interpolation-7/test.ghul new file mode 100644 index 000000000..359407af9 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-7/test.ghul @@ -0,0 +1,7 @@ + +entry() is + let x = 10; + let y = 20; + + "interpolate {x,y:G} interpolate {x,1234: ABCDEF +si diff --git a/integration-tests/parse/incomplete-string-interpolation-7/warn.expected b/integration-tests/parse/incomplete-string-interpolation-7/warn.expected new file mode 100644 index 000000000..e69de29bb diff --git a/integration-tests/parse/incomplete-string-interpolation-8/.vscode/tasks.json b/integration-tests/parse/incomplete-string-interpolation-8/.vscode/tasks.json new file mode 100644 index 000000000..49063613c --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-8/.vscode/tasks.json @@ -0,0 +1,23 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "Run test", + "command": "dotnet ghul-test \"${workspaceFolder}\"", + "type": "shell", + "group": { + "kind": "test", + "isDefault": true + } + }, + { + "label": "Capture test expectation", + "command": "../../../tasks/capture.sh \"${workspaceFolder}\"", + "type": "shell", + "group": { + "kind": "build", + "isDefault": true + } + } + ] +} \ No newline at end of file diff --git a/integration-tests/parse/incomplete-string-interpolation-8/err.expected b/integration-tests/parse/incomplete-string-interpolation-8/err.expected new file mode 100644 index 000000000..2a9edab6f --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-8/err.expected @@ -0,0 +1 @@ +test.ghul: 6,54..6,55: error: newline in string literal diff --git a/integration-tests/parse/incomplete-string-interpolation-8/fail.expected b/integration-tests/parse/incomplete-string-interpolation-8/fail.expected new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-8/fail.expected @@ -0,0 +1 @@ + diff --git a/integration-tests/parse/incomplete-string-interpolation-8/ghul.json b/integration-tests/parse/incomplete-string-interpolation-8/ghul.json new file mode 100644 index 000000000..84539d122 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-8/ghul.json @@ -0,0 +1,6 @@ +{ + "compiler": "dotnet ../../../publish/ghul.dll", + "source": [ + "." + ] +} diff --git a/integration-tests/parse/incomplete-string-interpolation-8/ghulflags b/integration-tests/parse/incomplete-string-interpolation-8/ghulflags new file mode 100644 index 000000000..4d095b972 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-8/ghulflags @@ -0,0 +1 @@ +--type-check \ No newline at end of file diff --git a/integration-tests/parse/incomplete-string-interpolation-8/il.expected b/integration-tests/parse/incomplete-string-interpolation-8/il.expected new file mode 100644 index 000000000..e69de29bb diff --git a/integration-tests/parse/incomplete-string-interpolation-8/test.ghul b/integration-tests/parse/incomplete-string-interpolation-8/test.ghul new file mode 100644 index 000000000..f016097af --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-8/test.ghul @@ -0,0 +1,7 @@ + +entry() is + let x = 10; + let y = 20; + + "interpolate {x,y:G} interpolate {x,1234: ABCDEF } +si diff --git a/integration-tests/parse/incomplete-string-interpolation-8/warn.expected b/integration-tests/parse/incomplete-string-interpolation-8/warn.expected new file mode 100644 index 000000000..e69de29bb diff --git a/integration-tests/parse/incomplete-string-interpolation-9/.vscode/tasks.json b/integration-tests/parse/incomplete-string-interpolation-9/.vscode/tasks.json new file mode 100644 index 000000000..49063613c --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-9/.vscode/tasks.json @@ -0,0 +1,23 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "Run test", + "command": "dotnet ghul-test \"${workspaceFolder}\"", + "type": "shell", + "group": { + "kind": "test", + "isDefault": true + } + }, + { + "label": "Capture test expectation", + "command": "../../../tasks/capture.sh \"${workspaceFolder}\"", + "type": "shell", + "group": { + "kind": "build", + "isDefault": true + } + } + ] +} \ No newline at end of file diff --git a/integration-tests/parse/incomplete-string-interpolation-9/err.expected b/integration-tests/parse/incomplete-string-interpolation-9/err.expected new file mode 100644 index 000000000..d77dfdca9 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-9/err.expected @@ -0,0 +1 @@ +test.ghul: 6,60..6,61: error: newline in string literal diff --git a/integration-tests/parse/incomplete-string-interpolation-9/fail.expected b/integration-tests/parse/incomplete-string-interpolation-9/fail.expected new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-9/fail.expected @@ -0,0 +1 @@ + diff --git a/integration-tests/parse/incomplete-string-interpolation-9/ghul.json b/integration-tests/parse/incomplete-string-interpolation-9/ghul.json new file mode 100644 index 000000000..84539d122 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-9/ghul.json @@ -0,0 +1,6 @@ +{ + "compiler": "dotnet ../../../publish/ghul.dll", + "source": [ + "." + ] +} diff --git a/integration-tests/parse/incomplete-string-interpolation-9/ghulflags b/integration-tests/parse/incomplete-string-interpolation-9/ghulflags new file mode 100644 index 000000000..4d095b972 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-9/ghulflags @@ -0,0 +1 @@ +--type-check \ No newline at end of file diff --git a/integration-tests/parse/incomplete-string-interpolation-9/il.expected b/integration-tests/parse/incomplete-string-interpolation-9/il.expected new file mode 100644 index 000000000..e69de29bb diff --git a/integration-tests/parse/incomplete-string-interpolation-9/test.ghul b/integration-tests/parse/incomplete-string-interpolation-9/test.ghul new file mode 100644 index 000000000..4d1d16f62 --- /dev/null +++ b/integration-tests/parse/incomplete-string-interpolation-9/test.ghul @@ -0,0 +1,7 @@ + +entry() is + let x = 10; + let y = 20; + + "interpolate {x,y:G} interpolate {x,1234: ABCDEF } blah +si diff --git a/integration-tests/parse/incomplete-string-interpolation-9/warn.expected b/integration-tests/parse/incomplete-string-interpolation-9/warn.expected new file mode 100644 index 000000000..e69de29bb diff --git a/src/lexical/token.ghul b/src/lexical/token.ghul index 6b771804b..a13c32f3c 100644 --- a/src/lexical/token.ghul +++ b/src/lexical/token.ghul @@ -88,6 +88,7 @@ namespace Lexical is ENTER_STRING, CONTINUE_STRING, EXIT_STRING, - CANCEL_STRING + CANCEL_STRING, + FORMAT_STRING si si diff --git a/src/lexical/token_lookahead.ghul b/src/lexical/token_lookahead.ghul index 47f700c9c..1f289f682 100644 --- a/src/lexical/token_lookahead.ghul +++ b/src/lexical/token_lookahead.ghul @@ -5,6 +5,7 @@ namespace Lexical is trait TokenSource is read_token() -> TOKEN_PAIR; + expect_format_specifier(); si class TOKEN_LOOKAHEAD is @@ -65,6 +66,10 @@ namespace Lexical is fi si + expect_format_specifier() is + _tokenizer.expect_format_specifier(); + si + read_token() -> TOKEN_PAIR is if !_queue.avail then let result = _tokenizer.read_token(); diff --git a/src/lexical/token_names.ghul b/src/lexical/token_names.ghul index 0004966fc..0634623b8 100644 --- a/src/lexical/token_names.ghul +++ b/src/lexical/token_names.ghul @@ -94,6 +94,7 @@ namespace Lexical is _names[TOKEN.EXIT_STRING] = "}}"; _names[TOKEN.CONTINUE_STRING] = "string fragment"; _names[TOKEN.CANCEL_STRING] = "cancel string"; + _names[TOKEN.FORMAT_STRING] = ":"; si instance: TOKEN_NAMES private static is diff --git a/src/lexical/tokenizer.ghul b/src/lexical/tokenizer.ghul index 9cfcf0f0b..fa5330968 100644 --- a/src/lexical/tokenizer.ghul +++ b/src/lexical/tokenizer.ghul @@ -85,6 +85,7 @@ namespace Lexical is _token_name: string[]; _interpolation_depth: int; + _expect_format_string: bool; static_init() static is if _symbol_tokens == null then @@ -172,13 +173,13 @@ namespace Lexical is fi _token_pair = new TOKEN_PAIR(TOKEN.UNKNOWN, location, null); - - // debug_enter(); si is_end_of_file: bool => _end_of_file; location: LOCATION => _cursor.location; + + character_location: LOCATION => _cursor.character_location; advance_cursor(c: char) is if c == cast char(13) then @@ -273,6 +274,10 @@ namespace Lexical is od return c; si + + expect_format_specifier() is + _expect_format_string = true; + si read_token() -> TOKEN_PAIR is let r: TOKEN; @@ -287,7 +292,31 @@ namespace Lexical is let _buffer: System.Text.StringBuilder = null; - if c >= '0' /\ c <= '9' then + if _expect_format_string then + _expect_format_string = false; + + _buffer = new System.Text.StringBuilder(); + + while c != '\n' /\ c != '}' /\ c != '"' do + _buffer.append(c); + + c = next_char(); + od + + if c == '}' then + prev_char(c); + elif c == '\n' then + prev_char(c); + _logger.error(character_location, "newline in interpolation format string"); + return new TOKEN_PAIR(TOKEN.CANCEL_STRING, location, _buffer.to_string()); + elif c == '"' then + _logger.error(character_location, "expected '}}' after format string"); + return new TOKEN_PAIR(TOKEN.CANCEL_STRING, location, _buffer.to_string()); + fi + + return new TOKEN_PAIR(TOKEN.FORMAT_STRING, location, _buffer.to_string()); + + elif c >= '0' /\ c <= '9' then let is_float = false; _buffer = new System.Text.StringBuilder(); @@ -467,8 +496,6 @@ namespace Lexical is prev_char(c); return read_operator('/'); fi - elif _interpolation_depth > 0 /\ c == ':' then - return interpolation_exit(c); elif _operator_chars.contains(c) \/ (cast int(c) > 0x7E /\ char.is_symbol(c)) then return read_operator(c); fi @@ -599,9 +626,9 @@ namespace Lexical is fragment.append(c); c = next_char(); else - prev_char(c); - c = '}'; - break; + _logger.error(character_location, "unmatched '}}' in string interpolation"); + + fragment.append('}'); fi else fragment.append(c); @@ -610,7 +637,7 @@ namespace Lexical is od if c == '\n' then - _logger.error(location, "newline in string literal"); + _logger.error(character_location, "newline in string literal"); fi return (fragment.to_string(), c); @@ -632,12 +659,6 @@ namespace Lexical is fi od - if c == '\n' then - _logger.error(location, "newline in interpolation format string"); - elif c == '"' then - _logger.error(location, "expected '}}' after format string"); - fi - return (format.to_string(), c); si @@ -658,7 +679,7 @@ namespace Lexical is return new TOKEN_PAIR(TOKEN.ENTER_STRING, location, fragment); else - _logger.error(location, "unexpected character '{c}' in string interpolation"); + _logger.error(character_location, "unexpected character '{c}' in string interpolation"); return new TOKEN_PAIR(TOKEN.STRING_LITERAL, location, fragment); fi @@ -670,22 +691,11 @@ namespace Lexical is interpolation_exit(c: char) -> TOKEN_PAIR is if _interpolation_depth == 0 then - _logger.error(location, "unmatched '\"'"); + _logger.error(character_location, "unmatched '\"'"); next_char(); return read_token(); fi - let format: string = null; - - if c == ':' then - let start = location; - let format_c = read_format_string(); - format = format_c.format; - - _logger.warn(start::location, "format string '{format}' ignored"); - c = format_c.c; - fi - let fragment = ""; if c == '}' then @@ -705,7 +715,7 @@ namespace Lexical is elif c == '{' then return new TOKEN_PAIR(TOKEN.CONTINUE_STRING, location, fragment); else - _logger.error(location, "unexpected character '{c}' in string interpolation"); + _logger.error(character_location, "unexpected character '{c}' in string interpolation"); return new TOKEN_PAIR(TOKEN.STRING_LITERAL, location, fragment); fi diff --git a/src/semantic/types/named.ghul b/src/semantic/types/named.ghul index 11412e15b..104185444 100644 --- a/src/semantic/types/named.ghul +++ b/src/semantic/types/named.ghul @@ -1,8 +1,6 @@ namespace Semantic.Types is use IO.Std; - use Logging.debug; - class NAMED: Type is _compare_count: int static; _same_count: int static; diff --git a/src/source/location.ghul b/src/source/location.ghul index ba8f1d40e..db91860a1 100644 --- a/src/source/location.ghul +++ b/src/source/location.ghul @@ -226,6 +226,20 @@ namespace Source is _current_line, _current_column ); + + character_location: LOCATION => + new LOCATION( + _file_name, + + // in practice if we report a character location, we + // want to report the previous character because we + // have one character lookahead + _previous_line, + _previous_column, + + _previous_line, + _previous_column + ); si class INTERNAL_LOCATION_CURSOR: LOCATION_CURSOR is diff --git a/src/syntax/parsers/context.ghul b/src/syntax/parsers/context.ghul index 34d9f7a78..3cf9adbee 100644 --- a/src/syntax/parsers/context.ghul +++ b/src/syntax/parsers/context.ghul @@ -81,6 +81,10 @@ namespace Syntax.Parsers is backtrack() is current = tokenizer.backtrack(); si + + expect_format_specifier() is + tokenizer.expect_format_specifier(); + si next_token() -> bool is current = tokenizer.read_token(); diff --git a/src/syntax/parsers/expressions/primary.ghul b/src/syntax/parsers/expressions/primary.ghul index 1d5ced534..fa7e2ec5e 100644 --- a/src/syntax/parsers/expressions/primary.ghul +++ b/src/syntax/parsers/expressions/primary.ghul @@ -3,6 +3,8 @@ namespace Syntax.Parsers.Expressions is use Source; + use Logging; + class PRIMARY: Base[Trees.Expressions.Expression] is identifier_parser: Parser[Trees.Identifiers.Identifier]; type_parser: Parser[Trees.TypeExpressions.TypeExpression]; @@ -211,7 +213,7 @@ namespace Syntax.Parsers.Expressions is ); add_parser( - (context: CONTEXT) => parse_string_interpolation(context), + (context: CONTEXT) => parse_string_with_interpolations(context), Lexical.TOKEN.ENTER_STRING ); @@ -260,79 +262,227 @@ namespace Syntax.Parsers.Expressions is ); si - parse_string_interpolation(context: CONTEXT) -> Trees.Expressions.Expression is + parse_string_with_interpolations(context: CONTEXT) -> Trees.Expressions.Expression is + // ENTER_STRING expression (':' FORMAT_STRING)? (CONTINUE_STRING expression (':' FORMAT_STRING)? )* (EXIT_STRING | CANCEL_STRING) + // ^ we're here + let start = context.location; - let expressions = new Collections.LIST[Trees.Expressions.Expression](); + let fragments = new Collections.LIST[Trees.Expressions.INTERPOLATION_FRAGMENT](); - let fragment = new Trees.Expressions.Literals.STRING(context.location, context.current_string); + let is_first = true; + let success = false; + let line = start.start_line; - let total_fragment_length = context.current_string.length; - let expression_count = 0; + while + context.current_token != Lexical.TOKEN.EXIT_STRING + do + // a newline in the string part will result in a CANCEL_STRING token + // we should stop parsing the string and return immediately + // TODO: return the interpolation that we've assembled so far + + if context.current_token == Lexical.TOKEN.CANCEL_STRING then + // TODO might want to skip to end of line, as the input could be garbled + context.next_token(); + return new Trees.Expressions.Literals.STRING(start::context.location, ""); + fi - expressions.add(fragment); + // we've either just entered a string interpolation, or we've just passed a closing curly bracket + // either way we're expecting a string fragment - context.next_token(); + line = context.location.start_line; - let end = context.location; - - let expression = expression_parser.parse(context); + let success_line = parse_single_interpolated_string_fragment(context, is_first, fragments); + success = success_line.success; + line = success_line.line; - if !expression? \/ expression.is_poisoned then - // even if the expression is garbled, we expect the tokenizer to - // provide the EXIT_STRING token - if context.current_token == Lexical.TOKEN.EXIT_STRING then + if !success then + skip_to_end_of_string_with_interpolations(context, line); + + return new Trees.Expressions.Literals.STRING(start::context.location, ""); + fi + + if context.current_token == Lexical.TOKEN.CANCEL_STRING then + // TODO might want to skip to end of line, as the input could be garbled context.next_token(); + return new Trees.Expressions.Literals.STRING(start::context.location, ""); fi + + success_line = parse_single_interpolated_expression(context, fragments); + success = success_line.success; + line = success_line.line; + + if !success then + skip_to_end_of_string_with_interpolations(context, line); + + return new Trees.Expressions.Literals.STRING(start::context.location, ""); + fi + + is_first = false; + od - return fragment; + if context.current_token == Lexical.TOKEN.CANCEL_STRING then + context.next_token(); + elif context.current_token != Lexical.TOKEN.EXIT_STRING then + context.error(context.location, "unexpected token in string interpolation"); + + if context.location.start_line == start.start_line then + skip_to_end_of_string_with_interpolations(context, line); + fi + else + parse_single_interpolated_string_fragment(context, is_first, fragments); fi - expressions.add(expression); - expression_count = expression_count + 1; + let end = context.location; - while context.current.token == Lexical.TOKEN.CONTINUE_STRING do - fragment = new Trees.Expressions.Literals.STRING(context.location, context.current_string); - total_fragment_length = total_fragment_length + context.current_string.length; - expressions.add(fragment); + let expression_count = 0; + let should_poison = false; + let total_fragment_length = 0; + + for e in fragments do + if e.is_expression then + expression_count = expression_count + 1; + if e.expression.is_poisoned then + should_poison = true; + fi + else + let fragment = cast Trees.Expressions.Literals.STRING(e.expression); + let fragment_length = fragment.value_string.length; + + total_fragment_length = total_fragment_length + fragment_length; + fi + od + + let result = new Trees.Expressions.STRING_INTERPOLATION(start::end, fragments, total_fragment_length, expression_count); + + return result; + si + + skip_to_end_of_string_with_interpolations(context: CONTEXT, line: int) is + let retries = 50; + while + context.current_token != Lexical.TOKEN.EXIT_STRING /\ + context.current_token != Lexical.TOKEN.CANCEL_STRING /\ + context.location.start_line == line /\ + !context.is_end_of_file /\ + retries > 0 + do context.next_token(); + retries = retries - 1; + od + si + + parse_single_interpolated_string_fragment( + context: CONTEXT, + is_first: bool, + into: Collections.MutableList[Trees.Expressions.INTERPOLATION_FRAGMENT] + ) -> + (success: bool, line: int) + is + // ENTER_STRING expression (':' FORMAT_STRING)? (CONTINUE_STRING expression (':' FORMAT_STRING)? )* (EXIT_STRING | CANCEL_STRING) + // ^ we're here ^ or here ^ or here - expression = expression_parser.parse(context); + // we've either just entered a string interpolation, or we've just passed a closing curly bracket + // either way we're expecting a string fragment - if !expression? \/ expression.is_poisoned then - end = context.location; - // even if the expression is garbled, we expect the tokenizer to - // provide the EXIT_STRING token - if context.current_token == Lexical.TOKEN.EXIT_STRING then - context.next_token(); - fi + if is_first then + if !context.expect_token(Lexical.TOKEN.ENTER_STRING) then + return (false, 0); + fi + else + if + context.current_token != Lexical.TOKEN.CONTINUE_STRING /\ + context.current_token != Lexical.TOKEN.EXIT_STRING + then + context.logger.error(context.location, "syntax error: expected }} but found {context.current_token_name}"); - return new Trees.Expressions.STRING_INTERPOLATION(start::end, expressions, total_fragment_length, expression_count); + return (false, 0); fi + fi - expressions.add(expression); - expression_count = expression_count + 1; + into.add( + new Trees.Expressions.INTERPOLATION_FRAGMENT( + false, + new Trees.Expressions.Literals.STRING(context.location, context.current_string), + null, + null + ) + ); - end = expression.location; - od + let line = context.location.start_line; + + context.next_token(); + + return (true, line); + si + + parse_single_interpolated_expression(context: CONTEXT, into: Collections.MutableList[Trees.Expressions.INTERPOLATION_FRAGMENT]) -> (success: bool, line: int) is + // ENTER_STRING expression (':' FORMAT_STRING)? (CONTINUE_STRING expression (':' FORMAT_STRING)? )* (EXIT_STRING | CANCEL_STRING) + // ^ we're here ^ or here - if context.current_token == Lexical.TOKEN.EXIT_STRING then - end = context.location; - fragment = new Trees.Expressions.Literals.STRING(context.location, context.current_string); - total_fragment_length = total_fragment_length + context.current_string.length; - expressions.add(fragment); + let start = context.location; + let expression: Trees.Expressions.Expression; + + if context.current_token != Lexical.TOKEN.CONTINUE_STRING then + expression = expression_parser.parse(context); + else + expression = new Trees.Expressions.Literals.STRING(context.location, ""); + context.logger.error(context.location, "expected an expression"); context.next_token(); - elif context.current_token == Lexical.TOKEN.CANCEL_STRING then + fi + + if !expression? \/ expression.is_poisoned then + if + context.current_token != Lexical.TOKEN.COMMA /\ + context.current_token != Lexical.TOKEN.COLON /\ + context.current_token != Lexical.TOKEN.CONTINUE_STRING /\ + context.current_token != Lexical.TOKEN.EXIT_STRING + then + return (false, 0); + fi + fi + + let alignment: Trees.Expressions.Expression = null; + let format: string = null; + + let line = 0; + + if context.current_token == Lexical.TOKEN.COMMA then context.next_token(); - else - context.error(context.location, "unexpected token in string interpolation"); + + alignment = expression_parser.parse(context); + + if !alignment? \/ alignment.is_poisoned then + if + context.current_token != Lexical.TOKEN.COLON /\ + context.current_token != Lexical.TOKEN.CONTINUE_STRING /\ + context.current_token != Lexical.TOKEN.EXIT_STRING + then + return (false, 0); + fi + fi + + line = alignment.location.end_line; fi - - let result = new Trees.Expressions.STRING_INTERPOLATION(start::end, expressions, total_fragment_length, expression_count); - return result; + if context.current_token == Lexical.TOKEN.COLON then + context.expect_format_specifier(); + + context.next_token(); + + if context.expect_token(Lexical.TOKEN.FORMAT_STRING) then + format = context.current_string; + line = context.location.end_line; + + context.next_token(); + fi + fi + + into.add(new Trees.Expressions.INTERPOLATION_FRAGMENT(true, expression, alignment, format)); + + return (true, line); si other_token(context: CONTEXT) -> Trees.Expressions.Expression is diff --git a/src/syntax/parsers/expressions/secondary.ghul b/src/syntax/parsers/expressions/secondary.ghul index f3bd69518..d62f36fed 100644 --- a/src/syntax/parsers/expressions/secondary.ghul +++ b/src/syntax/parsers/expressions/secondary.ghul @@ -90,6 +90,7 @@ namespace Syntax.Parsers.Expressions is let is_nested_function_definition = false; if result.could_be_nested_function_definition then + // TODO need same logic but for nested within a global function if context.in_member then if result.location.start_column > context.member_indent then // probably an attempt at nesting a named function definition @@ -105,6 +106,18 @@ namespace Syntax.Parsers.Expressions is throw new UNWIND_TO_MEMBER_EXCEPTION(null); fi + elif context.in_global_function then + if result.location.start_column > context.global_indent then + // probably an attempt at nesting a named function definition + is_nested_function_definition = true; + else + // probably a missing `si` in a preceding member definition + context.error(result.location, "expected 'si' after member definition"); + + commit = false; + + throw new UNWIND_TO_GLOBAL_EXCEPTION(null); + fi elif context.in_classy then // not clear how we might have ended up here commit = false; diff --git a/src/syntax/parsers/pragmas/pragma.ghul b/src/syntax/parsers/pragmas/pragma.ghul index 841b67a7c..962bb5eba 100644 --- a/src/syntax/parsers/pragmas/pragma.ghul +++ b/src/syntax/parsers/pragmas/pragma.ghul @@ -19,45 +19,35 @@ namespace Syntax.Parsers.Pragmas is si parse(context: CONTEXT) -> Trees.Pragmas.PRAGMA is - try - // debug_always(">>pragma"); + let fail = false; + let expect_semicolon = false; + + let start = context.location; - // debug_enter(); + if !context.next_token(Lexical.TOKEN.AT) then + return null; + fi - let fail = false; - let expect_semicolon = false; - - let start = context.location; - - if !context.next_token(Lexical.TOKEN.AT) then - return null; - fi + let name = qualified_identifier_parser.parse(context); - let name = qualified_identifier_parser.parse(context); - - if !name? then - return null; - fi - - if context.next_token(Lexical.TOKEN.PAREN_OPEN) then - let arguments: Trees.Expressions.LIST; - - if context.current.token != Lexical.TOKEN.PAREN_CLOSE then - arguments = expression_list_parser.parse(context); - fi - - context.next_token(Lexical.TOKEN.PAREN_CLOSE); - - return new Trees.Pragmas.PRAGMA(start::context.location, name, arguments); + if !name? then + return null; + fi + + if context.next_token(Lexical.TOKEN.PAREN_OPEN) then + let arguments: Trees.Expressions.LIST; + + if context.current.token != Lexical.TOKEN.PAREN_CLOSE then + arguments = expression_list_parser.parse(context); fi - // FIXME: should this be an error? - return new Trees.Pragmas.PRAGMA(start::context.location, name, new Trees.Expressions.LIST(context.location, new Collections.LIST[Trees.Expressions.Expression]())); - finally - // debug_exit(); + context.next_token(Lexical.TOKEN.PAREN_CLOSE); + + return new Trees.Pragmas.PRAGMA(start::context.location, name, arguments); + fi - // debug_always("< f.is_generic /\ f.arguments.count == 1) .only(); + // there are two overloads of append_formatted with 2 arguments, we want the one where the second argument is an int: + _append_formatted_generic_alignment = cast Semantic.Symbols.FUNCTION_GROUP(_interpolation_handler.find_member("append_formatted")).functions | + .filter(f => f.is_generic /\ f.arguments.count == 2 /\ f.arguments[1].type.compare(int_type) == Semantic.Types.MATCH.SAME) + .only(); + + // there are two overloads of append_formatted with 2 arguments, we want the one where the second argument is a string: + _append_formatted_generic_format = cast Semantic.Symbols.FUNCTION_GROUP(_interpolation_handler.find_member("append_formatted")).functions | + .filter(f => f.is_generic /\ f.arguments.count == 2 /\ f.arguments[1].type.compare(string_type) == Semantic.Types.MATCH.SAME) + .only(); + + // there is only one overload of append_formatted with 3 arguments: + _append_formatted_generic_alignment_format = cast Semantic.Symbols.FUNCTION_GROUP(_interpolation_handler.find_member("append_formatted")).functions | + .filter(f => f.is_generic /\ f.arguments.count == 3) + .only(); + // there is only one overload of to_string_and_clear: _to_string_and_clear = cast Semantic.Symbols.FUNCTION_GROUP(_interpolation_handler.find_member("to_string_and_clear")).functions[0]; si @@ -1162,13 +1181,17 @@ namespace Syntax.Process is ) ); - for e in interpolation.values do + for eaf in interpolation.values do + let expression = eaf.expression; + let alignment = eaf.alignment; + let format = eaf.format; + let args = new Collections.LIST[Value](1); - args.add(e.value); + args.add(expression.value); - if isa Expressions.Literals.STRING(e) then - if cast Expressions.Literals.STRING(e).value_string.length == 0 then + if isa Expressions.Literals.STRING(expression) then + if cast Expressions.Literals.STRING(expression).value_string.length == 0 then continue; fi @@ -1176,7 +1199,28 @@ namespace Syntax.Process is gen(call); else - let append_formatted = _append_formatted_generic.specialize([e.value.type]); + let append_formatted: Semantic.Symbols.Symbol; + let args = new Collections.LIST[Value](3); + + args.add(expression.value); + + if format? /\ alignment? then + args.add(alignment.value); + args.add(new Literal.STRING(format, string_type)); + + append_formatted = _append_formatted_generic_alignment_format.specialize([expression.value.type]); + + elif format? then + args.add(new Literal.STRING(format, string_type)); + + append_formatted = _append_formatted_generic_format.specialize([expression.value.type]); + elif alignment? then + args.add(alignment.value); + + append_formatted = _append_formatted_generic_alignment.specialize([expression.value.type]); + else + append_formatted = _append_formatted_generic.specialize([expression.value.type]); + fi let call = append_formatted.call(interpolation.location, interpolator.load(), args, null, _function_caller); diff --git a/src/syntax/process/printer/base.ghul b/src/syntax/process/printer/base.ghul index 3584f1202..8b0ef559c 100644 --- a/src/syntax/process/printer/base.ghul +++ b/src/syntax/process/printer/base.ghul @@ -414,22 +414,28 @@ namespace Syntax.Process.Printer is let in_expression = false; - // TODO proper quoting - write("{{"); + write("\""); for e in interpolation.values do if in_expression then write("{{"); - fi - e.accept(self); + e.expression.accept(self); - if in_expression then - write("}}"); + if e.format? then + write(":{e.format}"); + fi + + if in_expression then + write("}}"); + fi + else + // TODO: no quotes around string literals + e.expression.accept(self); fi in_expression = !in_expression; od - write("}}"); + write("\""); si visit(integer: Expressions.Literals.INTEGER) is diff --git a/src/syntax/trees/expressions/string_interpolation.ghul b/src/syntax/trees/expressions/string_interpolation.ghul index 88034705f..90bf73459 100644 --- a/src/syntax/trees/expressions/string_interpolation.ghul +++ b/src/syntax/trees/expressions/string_interpolation.ghul @@ -3,13 +3,13 @@ namespace Syntax.Trees.Expressions is use Source; class STRING_INTERPOLATION: Expression is - values: Collections.List[Expression]; + values: Collections.List[INTERPOLATION_FRAGMENT]; literal_length: int; expression_count: int; init( location: LOCATION, - values: Collections.List[Expression], + values: Collections.List[INTERPOLATION_FRAGMENT], literal_length: int, expression_count: int ) is @@ -27,11 +27,34 @@ namespace Syntax.Trees.Expressions is walk(visitor: Visitor) is if !visitor.pre(self) then for v in values do - v.walk(visitor); + v.expression.walk(visitor); + + if v.alignment? then + v.alignment.walk(visitor); + fi od fi accept(visitor); si si + + struct INTERPOLATION_FRAGMENT is + is_expression: bool; + expression: Expression; + alignment: Expression; + format: string; + + init( + is_expression: bool, + expression: Expression, + alignment: Expression, + format: string + ) is + self.is_expression = is_expression; + self.expression = expression; + self.alignment = alignment; + self.format = format; + si + si si