Skip to content

Commit

Permalink
Added support for setting run_all_in_same_sim and fail_on_warning att…
Browse files Browse the repository at this point in the history
…ributes from Python
  • Loading branch information
LarsAsplund committed Aug 15, 2024
1 parent a27f0c7 commit 11e15f9
Show file tree
Hide file tree
Showing 21 changed files with 446 additions and 55 deletions.
1 change: 1 addition & 0 deletions docs/news.d/1053.feature.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Added support for setting ``run_all_in_same_sim`` and ``fail_on_warning attributes`` from Python.
21 changes: 21 additions & 0 deletions docs/run/img/tb_fail_on_warning.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
<div class="highlight"><pre><span></span><span class="n">test_runner</span><span class="w"> </span><span class="o">:</span><span class="w"> </span><span class="k">process</span>
<span class="w"> </span><span class="k">variable</span><span class="w"> </span><span class="n">my_vector</span><span class="w"> </span><span class="o">:</span><span class="w"> </span><span class="n">integer_vector</span><span class="p">(</span><span class="mi">1</span><span class="w"> </span><span class="k">to</span><span class="w"> </span><span class="mi">17</span><span class="p">);</span>
<span class="k">begin</span>
<span class="w"> </span><span class="n">test_runner_setup</span><span class="p">(</span><span class="n">runner</span><span class="p">,</span><span class="w"> </span><span class="n">runner_cfg</span><span class="p">);</span>

<span class="w"> </span><span class="c1">-- vunit: fail_on_warning</span>
<span class="w"> </span><span class="k">while</span><span class="w"> </span><span class="n">test_suite</span><span class="w"> </span><span class="k">loop</span>
<span class="w"> </span><span class="k">if</span><span class="w"> </span><span class="n">run</span><span class="p">(</span><span class="s">&quot;Test that fails on an assert&quot;</span><span class="p">)</span><span class="w"> </span><span class="k">then</span>
<span class="w"> </span><span class="k">assert</span><span class="w"> </span><span class="n">false</span><span class="p">;</span>
<span class="w"> </span><span class="k">elsif</span><span class="w"> </span><span class="n">run</span><span class="p">(</span><span class="s">&quot;Test that crashes on boundary problems&quot;</span><span class="p">)</span><span class="w"> </span><span class="k">then</span>
<span class="w"> </span><span class="n">report</span><span class="w"> </span><span class="n">to_string</span><span class="p">(</span><span class="n">my_vector</span><span class="p">(</span><span class="n">runner_cfg</span><span class="na">&#39;length</span><span class="p">));</span>
<span class="w"> </span><span class="k">elsif</span><span class="w"> </span><span class="n">run</span><span class="p">(</span><span class="s">&quot;Test that fails on VUnit check procedure&quot;</span><span class="p">)</span><span class="w"> </span><span class="k">then</span>
<span class="w"> </span><span class="n">check_equal</span><span class="p">(</span><span class="mi">17</span><span class="p">,</span><span class="w"> </span><span class="mi">18</span><span class="p">);</span>
<span class="w"> </span><span class="k">elsif</span><span class="w"> </span><span class="n">run</span><span class="p">(</span><span class="s">&quot;Test that a warning passes&quot;</span><span class="p">)</span><span class="w"> </span><span class="k">then</span>
<span class="w"> </span><span class="k">assert</span><span class="w"> </span><span class="n">false</span><span class="w"> </span><span class="k">severity</span><span class="w"> </span><span class="n">warning</span><span class="p">;</span>
<span class="w"> </span><span class="k">end</span><span class="w"> </span><span class="k">if</span><span class="p">;</span>
<span class="w"> </span><span class="k">end</span><span class="w"> </span><span class="k">loop</span><span class="p">;</span>

<span class="w"> </span><span class="n">test_runner_cleanup</span><span class="p">(</span><span class="n">runner</span><span class="p">);</span>
<span class="k">end</span><span class="w"> </span><span class="k">process</span><span class="p">;</span>
</pre></div>
5 changes: 2 additions & 3 deletions docs/run/img/tb_run_all_in_same_sim.html
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
<div class="highlight"><pre><span></span><span class="c1">-- vunit: run_all_in_same_sim</span>

<span class="k">library</span><span class="w"> </span><span class="nn">vunit_lib</span><span class="p">;</span>
<div class="highlight"><pre><span></span><span class="k">library</span><span class="w"> </span><span class="nn">vunit_lib</span><span class="p">;</span>
<span class="n">context</span><span class="w"> </span><span class="n">vunit_lib</span><span class="p">.</span><span class="n">vunit_context</span><span class="p">;</span>

<span class="k">entity</span><span class="w"> </span><span class="nc">tb_run_all_in_same_sim</span><span class="w"> </span><span class="k">is</span>
Expand All @@ -13,6 +11,7 @@
<span class="w"> </span><span class="k">begin</span>
<span class="w"> </span><span class="n">test_runner_setup</span><span class="p">(</span><span class="n">runner</span><span class="p">,</span><span class="w"> </span><span class="n">runner_cfg</span><span class="p">);</span>

<span class="w"> </span><span class="c1">-- vunit: run_all_in_same_sim</span>
<span class="w"> </span><span class="k">while</span><span class="w"> </span><span class="n">test_suite</span><span class="w"> </span><span class="k">loop</span>
<span class="w"> </span><span class="k">if</span><span class="w"> </span><span class="n">run</span><span class="p">(</span><span class="s">&quot;Test to_string for integer again&quot;</span><span class="p">)</span><span class="w"> </span><span class="k">then</span>
<span class="w"> </span><span class="n">check_equal</span><span class="p">(</span><span class="n">to_string</span><span class="p">(</span><span class="mi">17</span><span class="p">),</span><span class="w"> </span><span class="s">&quot;17&quot;</span><span class="p">);</span>
Expand Down
2 changes: 2 additions & 0 deletions docs/run/img/tb_stopping_failure.html
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
<span class="w"> </span><span class="n">report</span><span class="w"> </span><span class="n">to_string</span><span class="p">(</span><span class="n">my_vector</span><span class="p">(</span><span class="n">runner_cfg</span><span class="na">&#39;length</span><span class="p">));</span>
<span class="w"> </span><span class="k">elsif</span><span class="w"> </span><span class="n">run</span><span class="p">(</span><span class="s">&quot;Test that fails on VUnit check procedure&quot;</span><span class="p">)</span><span class="w"> </span><span class="k">then</span>
<span class="w"> </span><span class="n">check_equal</span><span class="p">(</span><span class="mi">17</span><span class="p">,</span><span class="w"> </span><span class="mi">18</span><span class="p">);</span>
<span class="w"> </span><span class="k">elsif</span><span class="w"> </span><span class="n">run</span><span class="p">(</span><span class="s">&quot;Test that a warning passes&quot;</span><span class="p">)</span><span class="w"> </span><span class="k">then</span>
<span class="w"> </span><span class="k">assert</span><span class="w"> </span><span class="n">false</span><span class="w"> </span><span class="k">severity</span><span class="w"> </span><span class="n">warning</span><span class="p">;</span>
<span class="w"> </span><span class="k">end</span><span class="w"> </span><span class="k">if</span><span class="p">;</span>
<span class="w"> </span><span class="k">end</span><span class="w"> </span><span class="k">loop</span><span class="p">;</span>

Expand Down
62 changes: 28 additions & 34 deletions docs/run/img/tb_stopping_failure_stdout.html
Original file line number Diff line number Diff line change
@@ -1,46 +1,40 @@
<div class="highlight" style="background: #002b36; color: #eee8d5;"><pre style="line-height: 125%; background: #002b36; color: #eee8d5;">> python run.py
Starting lib.tb_stopping_failure.Test that fails on an assert
Output file: C:\github\vunit\docs\run\src\vunit_out\test_output\lib.tb_stopping_failure.Test_that_fails_on_an_assert_f53b930e2c7649bc33253af52f8ea89a9c05f07b\output.txt
C:\github\vunit\docs\run\src\tb_stopping_failure.vhd:24:9:@0ms:(assertion error): Assertion violation
C:\ghdl\bin\ghdl.exe:error: assertion failed
in process .tb_stopping_failure(tb).test_runner
C:\ghdl\bin\ghdl.exe:error: simulation failed
<span style="color: #cb4b16; ">fail</span> (P=0 S=0 F=1 T=3) lib.tb_stopping_failure.Test that fails on an assert (0.5 seconds)
Output file: C:\repos\vunit\docs\run\src\vunit_out\test_output\lib.tb_stopping_failure.Test_that_fails_on_an_assert_f53b930e2c7649bc33253af52f8ea89a9c05f07b\output.txt
C:\repos\vunit\docs\run\src\tb_stopping_failure.vhd:24:9:@0ms:(assertion error): Assertion violation
ghdl:error: assertion failed
ghdl:error: simulation failed
<span style="color: #cb4b16; ">fail</span> (P=0 S=0 F=1 T=4) lib.tb_stopping_failure.Test that fails on an assert (0.5 s)

Starting lib.tb_stopping_failure.Test that crashes on boundary problems
Output file: C:\github\vunit\docs\run\src\vunit_out\test_output\lib.tb_stopping_failure.Test_that_crashes_on_boundary_problems_b53105615efefaa16d0cf9ee1bad37b5d3369e95\output.txt
C:\ghdl\bin\ghdl.exe:error: index (316) out of bounds (1 to 17) at C:\github\vunit\docs\run\src\tb_stopping_failure.vhd:26
in process .tb_stopping_failure(tb).test_runner
C:\ghdl\bin\ghdl.exe:error: simulation failed
<span style="color: #cb4b16; ">fail</span> (P=0 S=0 F=2 T=3) lib.tb_stopping_failure.Test that crashes on boundary problems (0.5 seconds)
(11:35:31) Starting lib.tb_stopping_failure.Test that crashes on boundary problems
Output file: C:\repos\vunit\docs\run\src\vunit_out\test_output\lib.tb_stopping_failure.Test_that_crashes_on_boundary_problems_b53105615efefaa16d0cf9ee1bad37b5d3369e95\output.txt
ghdl:error: index (314) out of bounds (1 to 17) at C:\repos\vunit\docs\run\src\tb_stopping_failure.vhd:26
ghdl:error: simulation failed
<span style="color: #cb4b16; ">fail</span> (P=0 S=0 F=2 T=4) lib.tb_stopping_failure.Test that crashes on boundary problems (0.5 s)

Starting lib.tb_stopping_failure.Test that fails on VUnit check procedure
Output file: C:\github\vunit\docs\run\src\vunit_out\test_output\lib.tb_stopping_failure.Test_that_fails_on_VUnit_check_procedure_717a6f8ff044e3d5fa7d7d3ec5a32971d74864dd\output.txt
(11:35:31) Starting lib.tb_stopping_failure.Test that fails on VUnit check procedure
Output file: C:\repos\vunit\docs\run\src\vunit_out\test_output\lib.tb_stopping_failure.Test_that_fails_on_VUnit_check_procedure_717a6f8ff044e3d5fa7d7d3ec5a32971d74864dd\output.txt
0 fs - check - ERROR - Equality check failed - Got 17. Expected 18.
C:\github\vunit\vunit\vhdl\core\src\core_pkg.vhd:84:7:@0ms:(report failure): Stop simulation on log level error
C:\ghdl\bin\ghdl.exe:error: report failed
in process .tb_stopping_failure(tb).test_runner
from: vunit_lib.logger_pkg.decrease_stop_count at logger_pkg-body.vhd:736
from: vunit_lib.logger_pkg.decrease_stop_count at logger_pkg-body.vhd:731
from: vunit_lib.logger_pkg.log at logger_pkg-body.vhd:910
from: vunit_lib.checker_pkg.log_failing_check at checker_pkg-body.vhd:258
from: vunit_lib.checker_pkg.failing_check at checker_pkg-body.vhd:275
from: vunit_lib.check_pkg.check_equal at check.vhd:4358
from: vunit_lib.check_pkg.check_equal at check.vhd:4313
from: process lib.tb_stopping_failure(tb).test_runner at tb_stopping_failure.vhd:28
C:\ghdl\bin\ghdl.exe:error: simulation failed
<span style="color: #cb4b16; ">fail</span> (P=0 S=0 F=3 T=3) lib.tb_stopping_failure.Test that fails on VUnit check procedure (0.5 seconds)
C:\repos\vunit\vunit\vhdl\core\src\core_pkg.vhd:85:7:@0ms:(report failure): Stop simulation on log level error
ghdl:error: report failed
ghdl:error: simulation failed
<span style="color: #cb4b16; ">fail</span> (P=0 S=0 F=3 T=4) lib.tb_stopping_failure.Test that fails on VUnit check procedure (0.5 s)

(11:35:32) Starting lib.tb_stopping_failure.Test that a warning passes
Output file: C:\repos\vunit\docs\run\src\vunit_out\test_output\lib.tb_stopping_failure.Test_that_a_warning_passes_7db91f3b27aea5f89e74e39ea51ce6d61558674e\output.txt
<span style="color: #008000; ">pass</span> (P=1 S=0 F=3 T=4) lib.tb_stopping_failure.Test that a warning passes (0.4 s)

==== Summary ============================================================================
<span style="color: #cb4b16; ">fail</span> lib.tb_stopping_failure.Test that fails on an assert (0.5 seconds)
<span style="color: #cb4b16; ">fail</span> lib.tb_stopping_failure.Test that crashes on boundary problems (0.5 seconds)
<span style="color: #cb4b16; ">fail</span> lib.tb_stopping_failure.Test that fails on VUnit check procedure (0.5 seconds)
<span style="color: #008000; ">pass</span> lib.tb_stopping_failure.Test that a warning passes (0.4 s)
<span style="color: #cb4b16; ">fail</span> lib.tb_stopping_failure.Test that fails on an assert (0.5 s)
<span style="color: #cb4b16; ">fail</span> lib.tb_stopping_failure.Test that crashes on boundary problems (0.5 s)
<span style="color: #cb4b16; ">fail</span> lib.tb_stopping_failure.Test that fails on VUnit check procedure (0.5 s)
=========================================================================================
<span style="color: #008000; ">pass</span> 0 of 3
<span style="color: #cb4b16; ">fail</span> 3 of 3
<span style="color: #008000; ">pass</span> 1 of 4
<span style="color: #cb4b16; ">fail</span> 3 of 4
=========================================================================================
Total time was 1.6 seconds
Elapsed time was 1.6 seconds
Total time was 1.8 s
Elapsed time was 1.8 s
=========================================================================================
<span style="color: #cb4b16; ">Some failed!</span>
</pre></div>
11 changes: 10 additions & 1 deletion docs/run/src/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,15 @@ def extract_snippets():
snippet,
)

for snippet in [
"tb_fail_on_warning",
]:
highlight_code(
root / "tb_fail_on_warning.vhd",
root / ".." / "img" / f"{snippet}.html",
snippet,
)

for snippet in [
"tb_stop_level",
]:
Expand Down Expand Up @@ -164,7 +173,7 @@ def _post_run(results):
args.verbose = True
options += " -v"

vu = VUnit.from_args(args=args, compile_builtins=False)
vu = VUnit.from_args(args=args)
vu.add_vhdl_builtins()

lib = vu.add_library("lib")
Expand Down
38 changes: 38 additions & 0 deletions docs/run/src/tb_fail_on_warning.vhd
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
-- This Source Code Form is subject to the terms of the Mozilla Public
-- License, v. 2.0. If a copy of the MPL was not distributed with this file,
-- You can obtain one at http://mozilla.org/MPL/2.0/.
--
-- Copyright (c) 2014-2024, Lars Asplund lars.anders.asplund@gmail.com

library vunit_lib;
context vunit_lib.vunit_context;

entity tb_fail_on_warning is
generic(runner_cfg : string);
end entity;

architecture tb of tb_fail_on_warning is
begin
-- start_snippet tb_fail_on_warning
test_runner : process
variable my_vector : integer_vector(1 to 17);
begin
test_runner_setup(runner, runner_cfg);

-- vunit: fail_on_warning
while test_suite loop
if run("Test that fails on an assert") then
assert false;
elsif run("Test that crashes on boundary problems") then
report to_string(my_vector(runner_cfg'length));
elsif run("Test that fails on VUnit check procedure") then
check_equal(17, 18);
elsif run("Test that a warning passes") then
assert false severity warning;
end if;
end loop;

test_runner_cleanup(runner);
end process;
-- end_snippet tb_fail_on_warning
end;
3 changes: 1 addition & 2 deletions docs/run/src/tb_run_all_in_same_sim.vhd
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,6 @@
-- Copyright (c) 2014-2024, Lars Asplund lars.anders.asplund@gmail.com

-- start_snippet tb_run_all_in_same_sim
-- vunit: run_all_in_same_sim

library vunit_lib;
context vunit_lib.vunit_context;

Expand All @@ -20,6 +18,7 @@ begin
begin
test_runner_setup(runner, runner_cfg);

-- vunit: run_all_in_same_sim
while test_suite loop
if run("Test to_string for integer again") then
check_equal(to_string(17), "17");
Expand Down
2 changes: 2 additions & 0 deletions docs/run/src/tb_stopping_failure.vhd
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ begin
report to_string(my_vector(runner_cfg'length));
elsif run("Test that fails on VUnit check procedure") then
check_equal(17, 18);
elsif run("Test that a warning passes") then
assert false severity warning;
end if;
end loop;

Expand Down
22 changes: 20 additions & 2 deletions docs/run/user_guide.rst
Original file line number Diff line number Diff line change
Expand Up @@ -163,12 +163,18 @@ good reasons for this
Possible drawbacks to this approach are that test cases have to be independent and the overhead
of starting a new simulation for each test case (this is typically less than one second per test case). If that
is the case you can force all test cases of a testbench to be run in the same simulation. This is done by adding
the ``run_all_in_same_sim`` attribute.
the ``run_all_in_same_sim`` attribute (``-- vunit: run_all_in_same_sim``) before the test suite.

.. raw:: html
:file: img/tb_run_all_in_same_sim.html


The ``run_all_in_same_sim`` attribute can also be set from the run script, see :class:`vunit.ui.testbench.TestBench`.

.. important::
When setting ``run_all_in_same_sim`` from the run script, the setting must be identical for all configurations
of the testbench.

The VUnit Watchdog
------------------

Expand Down Expand Up @@ -196,11 +202,23 @@ test.
.. raw:: html
:file: img/tb_stopping_failure.html

All these test cases will fail
All but the last of these test cases will fail

.. raw:: html
:file: img/tb_stopping_failure_stdout.html

By setting the VUnit ``fail_on_warning`` attribute (``-- vunit: fail_on_warning``) before the test suite,
the last test case will also fail.

.. raw:: html
:file: img/tb_fail_on_warning.html

The ``fail_on_warning`` attribute can also be set from the run script, see :class:`vunit.ui.testbench.TestBench`.

.. important::
When setting ``fail_on_warning`` from the run script, the setting must be identical for all configurations
of the testbench.

Counting Errors with VUnit Logging/Check Libraries
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Expand Down
7 changes: 5 additions & 2 deletions tests/acceptance/artificial/verilog/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,10 +63,13 @@ def post_check(output_path):

module = ui.library("lib").module("tb_same_sim_all_pass")
module.add_config("cfg", post_check=post_check)


module = ui.library("lib").module("tb_same_sim_from_python_all_pass")
module.add_config("cfg", post_check=post_check, attributes=dict(run_all_in_same_sim=True))

configure_tb_with_parameter_config()
configure_tb_same_sim_all_pass(vu)
lib.module("tb_same_sim_from_python_some_fail").set_attribute("run_all_in_same_sim", True)
lib.module("tb_other_file_tests").scan_tests_from_file(str(root / "other_file_tests.sv"))
lib.module("tb_fail_on_warning_from_python").set_attribute("fail_on_warning", True)

vu.main()
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
//
// Copyright (c) 2014-2024, Lars Asplund lars.anders.asplund@gmail.com

`include "vunit_defines.svh"

module tb_fail_on_warning_from_python;
`TEST_SUITE begin
`TEST_CASE("fail") begin
$warning("A warning");
end
end;
endmodule
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
//
// Copyright (c) 2014-2024, Lars Asplund lars.anders.asplund@gmail.com

`include "vunit_defines.svh"

module tb_same_sim_from_python_all_pass;

parameter string output_path = "";

integer counter = 1;

`TEST_SUITE begin

`TEST_CASE("Test 1") begin
$info("Test 1");
`CHECK_EQUAL(counter, 1);
counter = counter + 1;
end

`TEST_CASE("Test 2") begin
$info("Test 2");
`CHECK_EQUAL(counter, 2);
counter = counter + 1;
end

`TEST_CASE("Test 3") begin
int fd;
$info("Test 3");
`CHECK_EQUAL(counter, 3);
counter = counter + 1;
fd = $fopen({output_path, "post_check.txt"});
$fwrite(fd, "Test 3 was here");
$fclose(fd);
end
end;

`WATCHDOG(1ns);
endmodule
Loading

0 comments on commit 11e15f9

Please sign in to comment.