Include ilasm errors in testable build output

.gitignore

@@ -14,6 +14,7 @@ forward*.h
 **/binary
 **/junit.xml
 **/out.il
+**/runtimeconfig.json
 tester/tester
 tests/junit.xml
 tests-legacy/junit.xml

src/driver/main.ghul

@@ -266,12 +266,25 @@ namespace Driver is
             to_build.add("-output:" + output_file);
 
             let ilasm = new Util.Process();
+
+            let ilasm_stdout = IO.File.openCreateStream("ilasm.out");
+
+            native.dup2(ilasm_stdout.Handle, 1);
+            native.close(ilasm_stdout.Handle);
+
             let result = ilasm.run(to_run, to_build.Array, Arguments.ProgramEnvironment, true);
 
             if result != 0 then
+                IO.Std.err.println("ilasm failed");
+                IO.Std.err.println(IO.File.openRead("ilasm.out").readAll());
+
+                IO.File.deleteQuiet("ilasm.out");
+
                 return result;
             fi
 
+            IO.File.deleteQuiet("ilasm.out");
+
             let runtime_config_file: String;
 
             if output_file.lastIndexOf('.') == output_file.Length - 4 then

tasks/test.sh

@@ -11,9 +11,10 @@ if [ ! -f $CASE/ghulflags ] ; then
     exit 1;
 fi
 
-# docker run --name "test-`date +'%s'`" --rm --env PATH='/home/dev/source:/bin:/usr/bin:/usr/local/bin' -v `pwd`/../../..:/home/dev/source/ -w /home/dev/source/tests -u `id -u`:`id -g` ghul/compiler:stable ../tester/tester ${@:1}
-pwd
 cd ../..
-pwd
+
+pushd ..>/dev/null
+PATH=`pwd`:${PATH}
+popd
 ../tester/tester ${@:1}
 

tester/src/test_runner.ghul

@@ -23,7 +23,7 @@ namespace Tester is
 
             return stat.stat(path);
         si
-
+        
         files_matching(path: String, extension: String) -> Iterable[String] is
             let result = new Vector[String]();
 
@@ -73,13 +73,14 @@ namespace Tester is
             let p = new Util.Process();
 
             let binary_path = "./binary";
+            let runtime_config_json_path = "./binary.runtimeconfig.json";
 
             let errors_expected_path = "err.expected";
             let warnings_expected_path = "warn.expected";
             let il_expected_path = "il.expected";
             let run_expected_path = "run.expected";
 
-            let build_out_path = "compiler.out";
+            let build_err_path = "compiler.out";
             let run_out_path = "run.out";
 
             let errors_out_path = "err.out";
@@ -100,7 +101,7 @@ namespace Tester is
 
             var std_out: String;
             var std_err: String;
-
+            
             let want_mono = false;
 
             let ghul_flags = get_ghul_options();
@@ -126,7 +127,7 @@ namespace Tester is
                 // IO.Std.err.println(ghul_args.toString().replace(',', ' '));
 
                 let out = IO.File.openWriteStream("/dev/null");
-                let err = IO.File.openCreateStream(build_out_path);
+                let err = IO.File.openCreateStream(build_err_path);
 
                 dup2(out.Handle, 1);
                 close(out.Handle);
@@ -140,32 +141,28 @@ namespace Tester is
 
             let build_wait_result = p.wait();
 
-            std_err = read_all(build_out_path);
+            std_err = read_all(build_err_path);
 
             if wait_failed(build_wait_result) then
-                if !file_exists(build_out_path) then
+                if !file_exists(build_err_path) then
                     IO.Std.err.println("no compiler output present: giving up");
 
                     failures.add(
                         new FAILURE("build failed with no output", "build")
                     );
                 elif !fail_expected then
-                    failures.add(
-                        new FAILURE("unexpected build failure", "build")
-                    );
+                    add_failure(failures, "unexpected build failure", "build", std_err);
                 fi
             elif !file_exists(binary_path) then
-                if !fail_expected then
+                if !fail_expected then                    
                     IO.Std.err.println("no binary: " + binary_path);
-                    failures.add(
-                        new FAILURE("unexpected build failure", "build")
-                    );
+
+                    add_failure(failures, "no binary found", "build",  std_err);
                 fi
             elif fail_expected then
-                IO.Std.err.println("unexpected binary: " + binary_path);
-                failures.add(
-                    new FAILURE("unexpected build success", "build")
-                );
+                IO.Std.err.println("unexpected build success: " + binary_path);
+
+                add_failure(failures, "unexpected build success", "build", std_err);
             else
                 if p.fork() then
                     let out = IO.File.openCreateStream(run_out_path);
@@ -203,8 +200,8 @@ namespace Tester is
                 fi
             fi
 
-            grep(build_out_path, errors_out_path, "error:");
-            grep(build_out_path, warnings_out_path, "warn:");
+            grep(build_err_path, errors_out_path, "[Ee]rror\\s?:");
+            grep(build_err_path, warnings_out_path, "warn:");
 
             sort(errors_out_path, errors_sort_path);
             sort(warnings_out_path, warnings_sort_path);
@@ -238,7 +235,7 @@ namespace Tester is
             );
 
             if failures.Length == 0 then
-                IO.File.deleteQuiet(build_out_path);
+                IO.File.deleteQuiet(build_err_path);
                 IO.File.deleteQuiet(run_out_path);
 
                 IO.File.deleteQuiet(errors_out_path);
@@ -258,10 +255,11 @@ namespace Tester is
                 IO.File.deleteQuiet(binary_path);
                 IO.File.deleteQuiet(binary_path + ".bc");
                 IO.File.deleteQuiet(binary_path + ".lh");
-                else
+
+                IO.File.deleteQuiet(runtime_config_json_path);
+            else
                 IO.File.openCreateStream(failed_marker_path).close();
             fi
-
 
             let elapsed = timer.elapsed_milli_seconds;
 
@@ -276,6 +274,30 @@ namespace Tester is
                 );                    
         si
 
+        add_failure(
+            list: List[FAILURE],
+            prefix: String,
+            type: String,
+            message: String
+        )
+        is
+            if message? && message.Length > 0 then
+                list.add(
+                    new FAILURE(
+                        prefix + ": " + message,
+                        type
+                    )
+                );
+            else
+                list.add(
+                    new FAILURE(
+                        prefix,
+                        type
+                    )
+                );                
+            fi
+        si
+
         add_diff_failure(
             list: List[FAILURE],
             is_failed: bool,
@@ -296,13 +318,13 @@ namespace Tester is
                 else
                     list.add(
                         new FAILURE(
-                            "no diff file present",
+                            "no file present: " + path,
                             type
                         )
                     );                
                 fi
             fi
-        si        
+        si
 
         grep(in_path: String, out_path: String, pattern: String) -> bool is
             let p = new Util.Process();
@@ -313,7 +335,7 @@ namespace Tester is
                 dup2(err.Handle, 1);
                 close(err.Handle);
 
-                p.spawn("grep", ["grep", pattern, in_path], System.Arguments.ProgramEnvironment);                                
+                p.spawn("egrep", ["egrep", pattern, in_path], System.Arguments.ProgramEnvironment);                                
             else
                 let result = p.wait();
 

tests/cases/integration-hello-world/.vscode/tasks.json

@@ -0,0 +1,24 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "Run test",
+            "command": "../../../tasks/test.sh ${workspaceFolder}",
+            "type": "shell",
+            "group": {
+                "kind": "test",
+                "isDefault": true
+            },
+            "problemMatcher": "$ghul"
+        },
+        {
+            "label": "Capture test expectation",
+            "command": "../../../tasks/capture.sh ${workspaceFolder}",
+            "type": "shell",
+            "group": {
+                "kind": "build",
+                "isDefault": true
+            }
+        }
+    ]
+}

tests/cases/integration-hello-world/ghul.json

@@ -0,0 +1,7 @@
+{
+    "ghul_compiler": "../../../ghul",
+    "ghul_source": [
+        ".", "../../../lib"
+    ],
+    "other_flags": ""
+}

tests/cases/integration-hello-world/ghulflags

@@ -0,0 +1 @@
+-N ../../../lib/ghul.ghul

tests/cases/integration-hello-world/run.expected

@@ -0,0 +1 @@
+hello, world

tests/cases/integration-hello-world/test.ghul

@@ -0,0 +1,19 @@
+namespace Test is
+    class Main is
+        init() static is
+            @IL.entrypoint()
+
+            System.Console.write_line("hello, world");
+        si
+    si    
+si
+
+
+@IL.stub()
+namespace System is
+    @IL.name("[mscorlib]System.Console")
+    class Console is
+        @IL.name("WriteLine")
+        write_line(text: String) static;
+    si
+si

tests/cases/integration-ilasm-failure/.vscode/tasks.json

@@ -0,0 +1,24 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "Run test",
+            "command": "../../../tasks/test.sh ${workspaceFolder}",
+            "type": "shell",
+            "group": {
+                "kind": "test",
+                "isDefault": true
+            },
+            "problemMatcher": "$ghul"
+        },
+        {
+            "label": "Capture test expectation",
+            "command": "../../../tasks/capture.sh ${workspaceFolder}",
+            "type": "shell",
+            "group": {
+                "kind": "build",
+                "isDefault": true
+            }
+        }
+    ]
+}

tests/cases/integration-ilasm-failure/err.expected

@@ -0,0 +1 @@
+out.il : Error : No entry point found.

tests/cases/integration-ilasm-failure/fail.expected

@@ -0,0 +1 @@
+

tests/cases/integration-ilasm-failure/ghul.json

@@ -0,0 +1,7 @@
+{
+    "ghul_compiler": "../../../ghul",
+    "ghul_source": [
+        ".", "../../../lib"
+    ],
+    "other_flags": ""
+}

tests/cases/integration-ilasm-failure/ghulflags

@@ -0,0 +1 @@
+-N ../../../lib/ghul.ghul

tests/cases/integration-ilasm-failure/test.ghul

@@ -0,0 +1,7 @@
+namespace Test is
+    class Main is
+        init() is
+            // test code here
+        si
+    si
+si

tests/template/ghulflags

@@ -1 +1 @@
--G ../../../lib/ghul.ghul
+-N ../../../lib/ghul.ghul

tests/test.sh

@@ -1,7 +1,10 @@
 #!/bin/bash
-pushd `dirname "$0"`
+
+PATH=`pwd`:$PATH
+
+pushd `dirname "$0"` >/dev/null
 
 ../tester/tester
 
-popd
+popd >/dev/null