Save the subresults for tmt-report-result

All the results generated by tmt-report-results become objects of
`tmt.result.SubResult`.

docs/releases.rst

@@ -4,7 +4,14 @@
     Releases
 ======================
 
-tmt-1.37
+tmt-1.38.0
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Each execution of ``tmt-report-result`` command inside a test will now create a
+tmt subresult which gets assigned to the test.
+
+
+tmt-1.37.0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The :ref:`/plugins/report/junit` report plugin now validates all the XML
@@ -20,10 +27,6 @@ new ``--flavor`` argument. Using this argument the user can choose between a
 ``custom`` flavor where user must provide a custom template using a
 ``--template-path`` argument.
 
-
-tmt-1.37.0
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
 tmt will now put SSH master control socket into ``ssh-socket``
 subdirectory of a workdir. Originally, sockets were stored in
 ``/run/user/$UID`` directory, but this path led to conflicts when

spec/plans/results.fmf

@@ -141,6 +141,9 @@ description: |
           check:
             ...
 
+          # String, path to /data directory storing possible test artifacts
+          data-path: path/to/test/data
+
     .. _/spec/plans/results/outcomes:
 
     The ``result`` key can have the following values:

stories/features/report-result.fmf

@@ -12,14 +12,12 @@ description: |
     guest and overwrite any existing scripts with the same name.
 
     The command can be called multiple times for a single test,
-    the final result will be the most severe rating. Available
-    values ordered by severity are SKIP, PASS, WARN and FAIL.
-    Note, that the only value currently processed by ``tmt`` is
-    the test result, other options are silently ignored.
+    all these calls will be saved as tmt subresults. The final result
+    will be the most severe rating. Available values ordered by severity are
+    SKIP, PASS, WARN and FAIL.
 
     When ``tmt-report-result`` is called, the return value of the
-    test itself is ignored, and only result saved by
-    ``tmt-report-result`` is consumed by tmt.
+    test itself is saved as a tmt subresult.
 
     __ https://restraint.readthedocs.io/en/latest/commands.html#rstrnt-report-result
 

tests/execute/result/main.fmf

@@ -7,3 +7,6 @@
 /repeated:
     summary: Repeated test should provide multiple results
     test: ./repeated.sh
+/subresults:
+    summary: Multiple calls to tmt-report-result should generate tmt subresults
+    test: ./subresults.sh

tests/execute/result/subresults.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+# vim: dict+=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k
+. /usr/share/beakerlib/beakerlib.sh || exit 1
+
+
+rlJournalStart
+    rlPhaseStartSetup
+        rlRun "run_dir=\$(mktemp -d)" 0 "Create run directory"
+        rlRun "pushd subresults"
+        rlRun "set -o pipefail"
+    rlPhaseEnd
+
+    rlPhaseStartTest "Test the subresults were generated into results.yaml"
+        rlRun "tmt run --id $run_dir --scratch -v 2>&1 >/dev/null | tee output" 1
+
+        # Parent beaker test should fail because one subresult should fail
+        rlAssertGrep "fail /test/beakerlib (on default-0)" "output"
+        rlAssertGrep "fail /test/fail (on default-0)" "output"
+        rlAssertGrep "pass /test/pass (on default-0)" "output"
+        rlAssertGrep "total: 1 test passed and 2 tests failed" "output"
+
+        rlRun "results_file=${run_dir}/plan/execute/results.yaml"
+
+        rlRun "yq -ey '.[] | select(.name == \"/test/beakerlib\") | .subresult' ${results_file} > subresults_beakerlib.yaml"
+        rlAssertGrep "name: /test/beakerlib/phase-setup" "subresults_beakerlib.yaml"
+        rlAssertGrep "name: /test/beakerlib/phase-test-pass" "subresults_beakerlib.yaml"
+        rlAssertGrep "name: /test/beakerlib/phase-test-fail" "subresults_beakerlib.yaml"
+        rlAssertGrep "name: /test/beakerlib/phase-cleanup" "subresults_beakerlib.yaml"
+
+        rlRun "yq -ey '.[] | select(.name == \"/test/fail\") | .subresult' ${results_file} > subresults_fail.yaml"
+        rlAssertGrep "name: /test/fail/subtest/good" "subresults_fail.yaml"
+        rlAssertGrep "name: /test/fail/subtest/fail" "subresults_fail.yaml"
+        rlAssertGrep "name: /test/fail/subtest/weird" "subresults_fail.yaml"
+
+        rlRun "yq -ey '.[] | select(.name == \"/test/pass\") | .subresult' ${results_file} > subresults_pass.yaml"
+        rlAssertGrep "name: /test/pass/subtest/good0" "subresults_pass.yaml"
+        rlAssertGrep "name: /test/pass/subtest/good1" "subresults_pass.yaml"
+        rlAssertGrep "name: /test/pass/subtest/good2" "subresults_pass.yaml"
+    rlPhaseEnd
+
+    rlPhaseStartCleanup
+        rlRun "rm output"
+        rlRun "rm subresults_{beakerlib,fail,pass}.yaml"
+        rlRun "popd"
+        rlRun "rm -rf $run_dir" 0 "Remove run directory"
+    rlPhaseEnd
+rlJournalEnd

tests/execute/result/subresults/.fmf/version

@@ -0,0 +1 @@
+1

tests/execute/result/subresults/beaker-phases-subresults.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+. /usr/share/beakerlib/beakerlib.sh || exit 1
+
+
+rlJournalStart
+    rlPhaseStartSetup "phase-setup"
+        rlRun "tmp=\$(mktemp -d)" 0 "Create tmp directory"
+        rlRun "pushd $tmp"
+        rlRun "set -o pipefail"
+    rlPhaseEnd
+
+    rlPhaseStartTest "phase-test pass"
+        rlRun "echo mytest-pass | tee output" 0 "Check output"
+        rlAssertGrep "mytest-pass" "output"
+    rlPhaseEnd
+
+    rlPhaseStartTest "phase-test fail"
+        rlRun "echo mytest-fail | tee output" 0 "Check output"
+        rlAssertGrep "asdf-asdf" "output"
+    rlPhaseEnd
+
+    rlPhaseStartCleanup "phase-cleanup"
+        rlRun "popd"
+        rlRun "rm -r $tmp" 0 "Remove tmp directory"
+    rlPhaseEnd
+rlJournalEnd

tests/execute/result/subresults/main.fmf

@@ -0,0 +1,12 @@
+/plan:
+    discover:
+        how: fmf
+    provision:
+        # TODO:
+        # For some reason the tests are not working on my local machine,
+        # try to test in CI. Container works fine.
+        #how: local
+
+        how: container
+    execute:
+        how: tmt

tests/execute/result/subresults/test.fmf

@@ -0,0 +1,32 @@
+/pass:
+    summary: Basic test of subresults
+    test: |
+        tmt-report-result /subtest/good0 PASS
+        tmt-report-result /subtest/good1 PASS
+        tmt-report-result /subtest/good2 PASS
+
+/fail:
+    summary: Reduced outcome of subresults must be fail
+    test: |
+        tmt-report-result /subtest/good PASS
+        tmt-report-result /subtest/fail FAIL
+        tmt-report-result /subtest/weird WARN
+
+/beakerlib:
+    summary: Beakerlib rlPhaseEnd as tmt subresult
+
+    # Explicitly set the TESTID to non-empty value. Also, set the
+    # BEAKERLIB_COMMAND_REPORT_RESULT to `tmt-report-result` explicitly. The
+    # command in this variable gets called with every `rlPhaseEnd`.
+    #
+    # Refs:
+    # - https://github.com/teemtee/tmt/issues/2826#issue-2225993479
+    # - https://github.com/teemtee/tmt/issues/2826#issuecomment-2085014960
+    environment:
+        TESTID: 12345678
+        BEAKERLIB_COMMAND_REPORT_RESULT: tmt-report-result
+
+    # Also, explicitly set the framework
+    framework: beakerlib
+
+    test: ./beaker-phases-subresults.sh

tmt/result.py

@@ -161,6 +161,12 @@ class SubResult(BaseResult):
             SubCheckResult.from_serialized(check) for check in serialized]
         )
 
+    data_path: Optional[Path] = field(
+        default=None,
+        serialize=lambda path: None if path is None else str(path),
+        unserialize=lambda value: None if value is None else Path(value)
+        )
+
 
 @dataclasses.dataclass
 class PhaseResult(BaseResult):
@@ -218,7 +224,9 @@ def from_test_invocation(
             result: ResultOutcome,
             note: Optional[str] = None,
             ids: Optional[ResultIds] = None,
-            log: Optional[list[Path]] = None) -> 'Result':
+            log: Optional[list[Path]] = None,
+            subresult: Optional[list[SubResult]] = None,
+            ) -> 'Result':
         """
         Create a result from a test invocation.
 
@@ -268,7 +276,9 @@ def from_test_invocation(
             ids=ids,
             log=log or [],
             guest=guest_data,
-            data_path=invocation.relative_test_data_path)
+            data_path=invocation.relative_test_data_path,
+            subresult=subresult or [],
+            )
 
         return _result.interpret_result(ResultInterpret(
             invocation.test.result) if invocation.test.result else ResultInterpret.RESPECT)

tmt/schemas/results.yaml

@@ -81,6 +81,10 @@ definitions:
       check:
         $ref: "#/definitions/check_results"
 
+      # TODO: Fixme?
+      data-path:
+        type: string
+
     required:
       - name
       - result
@@ -135,6 +139,12 @@ items:
     duration:
       $ref: "#/definitions/duration"
 
+    # TODO: Fixme? e.g. using ref. otherwise I am getting the error:
+    # warn: Result format violation: 0 - Additional properties are not allowed
+    # ('data-path' was unexpected)
+    data-path:
+      type: string
+
     ids:
       type: object
       patternProperties: