Parser recovery improvements

Enhancements:
- Better parser error recovery for incomplete and garbled class, method and property definitions (see #1086)

Technical:
- Rewrite the tokenizer lookahead mechanism so it reliably supports multiple levels of speculation
- Replace use of string concatenation with interpolation throughout the compiler source

.config/dotnet-tools.json

@@ -15,7 +15,7 @@
       ]
     },
     "ghul.compiler": {
-      "version": "0.7.13",
+      "version": "0.8.1",
       "commands": [
         "ghul-compiler"
       ]

.gitattributes

@@ -25,4 +25,6 @@
 
 *.png binary
 *.exe binary
-*.dll binary
+*.dll binary
+
+integration-tests/parse/carriage-returns/test.ghul eol=crlf

.github/workflows/dependabot-manual-trigger.yml

@@ -1,17 +1,17 @@
-name: Trigger Dependabot
-
-on:
-  workflow_dispatch:
-
-jobs:
-  trigger-dependabot:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Trigger Dependabot
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          curl -X POST -H "Authorization: token $GITHUB_TOKEN" \
-          -H "Accept: application/vnd.github.v3+json" \
-          https://api.github.com/repos/${{ github.repository }}/dispatches \
-          -d '{"event_type": "trigger-dependabot"}'
+name: Trigger Dependabot
+
+on:
+  workflow_dispatch:
+
+jobs:
+  trigger-dependabot:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Trigger Dependabot
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          curl -X POST -H "Authorization: token $GITHUB_TOKEN" \
+          -H "Accept: application/vnd.github.v3+json" \
+          https://api.github.com/repos/${{ github.repository }}/dispatches \
+          -d '{"event_type": "trigger-dependabot"}'

Directory.Build.props

@@ -1,10 +1,10 @@
 <Project>
   <PropertyGroup>
-    <Version>0.7.13-alpha.2</Version>
+    <Version>0.8.2-alpha.127</Version>
     <NoWarn>$(NoWarn);NU1507</NoWarn>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="ghul.runtime" />
   </ItemGroup>
-</Project>
+</Project>

integration-tests/parse/carriage-returns/test.ghul

@@ -1,5 +1,5 @@
-namespace Test is
-    use Std = IO.Std;
-
-    class Main is
-        entry() static is
+namespace Test is
+    use Std = IO.Std;
+
+    class Main is
+        entry() static is

integration-tests/parse/function-nesting-error-1/.vscode/tasks.json

@@ -0,0 +1,23 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "Run test",
+            "command": "dotnet ghul-test \"${workspaceFolder}\"",
+            "type": "shell",
+            "group": {
+                "kind": "test",
+                "isDefault": true
+            }
+        },
+        {
+            "label": "Capture test expectation",
+            "command": "../../../tasks/capture.sh \"${workspaceFolder}\"",
+            "type": "shell",
+            "group": {
+                "kind": "build",
+                "isDefault": true
+            }
+        }
+    ]
+}

integration-tests/parse/function-nesting-error-1/err.expected

@@ -0,0 +1,2 @@
+test.ghul: 13,13..13,28: error: expected function literal formal arguments
+test.ghul: 15,24..15,29: error: cannot return value of type Ghul.int from function of type Test.IncompleteFunction.Test

integration-tests/parse/function-nesting-error-1/fail.expected

@@ -0,0 +1 @@
+

integration-tests/parse/function-nesting-error-1/ghul.json

@@ -0,0 +1,6 @@
+{
+    "compiler": "dotnet ../../../publish/ghul.dll",
+    "source": [
+        "."
+    ]
+}

integration-tests/parse/function-nesting-error-1/ghulflags

@@ -0,0 +1 @@
+--type-check 

integration-tests/parse/function-nesting-error-1/test.ghul

@@ -0,0 +1,38 @@
+namespace Test.IncompleteFunction is
+    class Test is
+        count: int;
+
+        f(i: int) -> int is
+            return 0
+        si
+
+        g(i:int) -> int is
+            count = count - 1;
+            return 0;
+
+            "asdf" (i: int) -> Test is
+                count = count - 1;
+                return count;
+            si
+
+        si
+
+        i(i: int) -> Test is
+            let x = if count > 0 then
+                "count is {count}"
+            else
+                "count is 0"
+            fi;
+
+            if count > 0 then
+                return null
+            else
+                return null
+            fi
+        si
+
+        j(i: int) -> int is
+            return count
+        si
+    si
+si

integration-tests/parse/incomplete-class-14/err.expected

@@ -1,3 +1 @@
-test.ghul: 38,15..38,17: error: in definition: expected @, [, class, enum, identifier, namespace, operator, struct, trait or use but found is
-test.ghul: 38,15..38,17: error: in member: expected :, ;, = or => but found is
 test.ghul: 38,5..38,14: error: syntax error: expected is but found identifier ICS_STATE

integration-tests/parse/incomplete-class-14/test.ghul

@@ -71,4 +71,8 @@ namespace Test is
             return diagnostics_for_path;
         si
     si
+
+    class CHECK is
+
+    si
 si

integration-tests/parse/incomplete-function-10/.vscode/tasks.json

@@ -0,0 +1,23 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "Run test",
+            "command": "dotnet ghul-test \"${workspaceFolder}\"",
+            "type": "shell",
+            "group": {
+                "kind": "test",
+                "isDefault": true
+            }
+        },
+        {
+            "label": "Capture test expectation",
+            "command": "../../../tasks/capture.sh \"${workspaceFolder}\"",
+            "type": "shell",
+            "group": {
+                "kind": "build",
+                "isDefault": true
+            }
+        }
+    ]
+}

integration-tests/parse/incomplete-function-10/err.expected

@@ -0,0 +1 @@
+test.ghul: 13,13..13,22: error: nested function definition

integration-tests/parse/incomplete-function-10/fail.expected

@@ -0,0 +1 @@
+

integration-tests/parse/incomplete-function-10/ghul.json

@@ -0,0 +1,6 @@
+{
+    "compiler": "dotnet ../../../publish/ghul.dll",
+    "source": [
+        "."
+    ]
+}

integration-tests/parse/incomplete-function-10/ghulflags

@@ -0,0 +1 @@
+--type-check 

integration-tests/parse/incomplete-function-10/test.ghul

@@ -0,0 +1,47 @@
+namespace Test.IncompleteFunction is
+    class Test is
+        count: int;
+
+        f(i: int) -> int is
+            return 0
+        si
+
+        g(i:int) -> int is
+            count = count - 1;
+            return 0;
+
+            h(i: int) -> Test is
+                // let x = if count > 0 then
+                //     "count is {count}"
+                // else
+                //     "count is 0"
+                // fi;
+
+                if count > 0 then
+                    return null
+                else
+                    return null
+                fi
+            si
+
+        si
+
+        i(i: int) -> Test is
+            let x = if count > 0 then
+                "count is {count}"
+            else
+                "count is 0"
+            fi;
+
+            if count > 0 then
+                return null
+            else
+                return null
+            fi
+        si
+
+        j(i: int) -> int is
+            return count
+        si
+    si
+si

integration-tests/parse/incomplete-function-9/.vscode/tasks.json

@@ -0,0 +1,23 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "Run test",
+            "command": "dotnet ghul-test \"${workspaceFolder}\"",
+            "type": "shell",
+            "group": {
+                "kind": "test",
+                "isDefault": true
+            }
+        },
+        {
+            "label": "Capture test expectation",
+            "command": "../../../tasks/capture.sh \"${workspaceFolder}\"",
+            "type": "shell",
+            "group": {
+                "kind": "build",
+                "isDefault": true
+            }
+        }
+    ]
+}

integration-tests/parse/incomplete-function-9/err.expected

@@ -0,0 +1 @@
+test.ghul: 13,9..13,18: error: expected 'si' after member definition

integration-tests/parse/incomplete-function-9/fail.expected

@@ -0,0 +1 @@
+

integration-tests/parse/incomplete-function-9/ghul.json

@@ -0,0 +1,6 @@
+{
+    "compiler": "dotnet ../../../publish/ghul.dll",
+    "source": [
+        "."
+    ]
+}

integration-tests/parse/incomplete-function-9/ghulflags

@@ -0,0 +1 @@
+--type-check 

integration-tests/parse/incomplete-function-9/test.ghul

@@ -0,0 +1,45 @@
+namespace Test.IncompleteFunction is
+    class Test is
+        count: int;
+
+        f(i: int) -> int is
+            return 0
+        si
+
+        g(i:int) -> int is
+            count = count - 1;
+            return 0;
+
+        h(i: int) -> Test is
+            // let x = if count > 0 then
+            //     "count is {count}"
+            // else
+            //     "count is 0"
+            // fi;
+
+            if count > 0 then
+                return null
+            else
+                return null
+            fi
+        si
+
+        i(i: int) -> Test is
+            let x = if count > 0 then
+                "count is {count}"
+            else
+                "count is 0"
+            fi;
+
+            if count > 0 then
+                return null
+            else
+                return null
+            fi
+        si
+
+        j(i: int) -> int is
+            return count
+        si
+    si
+si