Add check-result key

docs/releases.rst

@@ -73,6 +73,12 @@ now wrapped in double quotes, and any double quotes within the
 values are escaped to ensure that the resulting file is always
 valid YAML.
 
++A new ``check-result`` key has been added to the test specification. This key
++allows users to configure whether "check" failure will affect the test result.
++The key accepts two options: 'respect' (default) and 'skip'. When set to
++'respect', check failures will affect the test result as before. When set to
++'skip', check failures will not affect the overall test result.
+
 
 tmt-1.36.1
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

spec/tests/result.fmf

@@ -32,6 +32,15 @@ description: |
 
         .. versionadded:: 1.35
 
+    Additionally, the `check-result` attribute can be used to specify
+    how check results should be interpreted:
+
+    respect
+        check results are respected (test fails if any check fails) - default value
+    skip
+        check results are ignored (test result is not affected by check failures)
+
+        .. versionadded:: 1.37
 
 example:
   - |
@@ -42,6 +51,10 @@ example:
     # Look for $TMT_TEST_DATA/results.yaml (or results.json) with custom results
     result: custom
 
+  - |
+    # Ignore check failures
+    check-result: skip
+
 link:
   - implemented-by: /tmt/base.py
   - verified-by: /tests/execute/result

tests/execute/result/check_results.sh

@@ -14,15 +14,22 @@ rlJournalStart
         rlAssertGrep "pass /test/check-pass" $rlRun_LOG
         rlAssertGrep "pass dmesg (before-test check)" $rlRun_LOG
         rlAssertGrep "pass dmesg (after-test check)" $rlRun_LOG
+        echo $rlRun_LOG
 
-        rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-fail execute -vv report -vv 2>&1 >/dev/null" 1
-        rlAssertGrep "fail /test/check-fail" $rlRun_LOG
+        rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-fail-respect execute -vv report -vv 2>&1 >/dev/null" 1
+        rlAssertGrep "fail /test/check-fail-respect" $rlRun_LOG
         rlAssertGrep "pass dmesg (before-test check)" $rlRun_LOG
         rlAssertGrep "fail dmesg (after-test check)" $rlRun_LOG
         rlAssertGrep "check failed" $rlRun_LOG
 
-        rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-ignore execute -vv report -vv 2>&1 >/dev/null" 0
-        rlAssertGrep "pass /test/check-ignore" $rlRun_LOG
+        rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-fail-skip execute -vv report -vv 2>&1 >/dev/null" 0
+        rlAssertGrep "pass /test/check-fail-skip" $rlRun_LOG
+        rlAssertGrep "pass dmesg (before-test check)" $rlRun_LOG
+        rlAssertGrep "fail dmesg (after-test check)" $rlRun_LOG
+        rlAssertNotGrep "check failed" $rlRun_LOG
+
+        rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-skip execute -vv report -vv 2>&1 >/dev/null" 0
+        rlAssertGrep "pass /test/check-skip" $rlRun_LOG
         rlAssertGrep "pass dmesg (before-test check)" $rlRun_LOG
         rlAssertGrep "fail dmesg (after-test check)" $rlRun_LOG
         rlAssertNotGrep "check failed" $rlRun_LOG
@@ -42,9 +49,10 @@ rlJournalStart
         rlAssertGrep "- name: dmesg" $rlRun_LOG
         rlAssertGrep "  result: pass" $rlRun_LOG
         rlAssertGrep "  event: after-test" $rlRun_LOG
+        rlAssertGrep "check_result: respect" $rlRun_LOG
 
         rlRun -s "yq e '.[1]' $results_file"
-        rlAssertGrep "name: /test/check-fail" $rlRun_LOG
+        rlAssertGrep "name: /test/check-fail-respect" $rlRun_LOG
         rlAssertGrep "result: fail" $rlRun_LOG
         rlAssertGrep "note: check failed" $rlRun_LOG
         rlAssertGrep "check:" $rlRun_LOG
@@ -54,9 +62,23 @@ rlJournalStart
         rlAssertGrep "- name: dmesg" $rlRun_LOG
         rlAssertGrep "  result: fail" $rlRun_LOG
         rlAssertGrep "  event: after-test" $rlRun_LOG
+        rlAssertGrep "check_result: respect" $rlRun_LOG
 
         rlRun -s "yq e '.[2]' $results_file"
-        rlAssertGrep "name: /test/check-ignore" $rlRun_LOG
+        rlAssertGrep "name: /test/check-fail-skip" $rlRun_LOG
+        rlAssertGrep "result: pass" $rlRun_LOG
+        rlAssertNotGrep "note: check failed" $rlRun_LOG
+        rlAssertGrep "check:" $rlRun_LOG
+        rlAssertGrep "- name: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: pass" $rlRun_LOG
+        rlAssertGrep "  event: before-test" $rlRun_LOG
+        rlAssertGrep "- name: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: fail" $rlRun_LOG
+        rlAssertGrep "  event: after-test" $rlRun_LOG
+        rlAssertGrep "check_result: skip" $rlRun_LOG
+
+        rlRun -s "yq e '.[3]' $results_file"
+        rlAssertGrep "name: /test/check-skip" $rlRun_LOG
         rlAssertGrep "result: pass" $rlRun_LOG
         rlAssertNotGrep "note: check failed" $rlRun_LOG
         rlAssertGrep "check:" $rlRun_LOG
@@ -66,6 +88,7 @@ rlJournalStart
         rlAssertGrep "- name: dmesg" $rlRun_LOG
         rlAssertGrep "  result: fail" $rlRun_LOG
         rlAssertGrep "  event: after-test" $rlRun_LOG
+        rlAssertGrep "check_result: respect" $rlRun_LOG
     rlPhaseEnd
 
     rlPhaseStartCleanup

tests/execute/result/check_results/main.fmf

@@ -8,19 +8,35 @@ description: Verify that check results, including after-test checks, are correct
     duration: 1m
     check:
         - how: dmesg
+    check-result: respect
+    # Expected outcome: PASS (test passes, check passes)
 
-/test/check-fail:
-    summary: Test with failing dmesg check
+/test/check-fail-respect:
+    summary: Test with failing dmesg check (respect)
     test: |
         echo "Test passed"
         echo "Call Trace:" >> /dev/kmsg
     framework: shell
     duration: 1m
     check:
         - how: dmesg
+    check-result: respect
+    # Expected outcome: FAIL (test passes, but check fails and is respected)
 
-/test/check-ignore:
-    summary: Test with failing dmesg check but ignored
+/test/check-fail-skip:
+    summary: Test with failing dmesg check (skip)
+    test: |
+        echo "Test passed"
+        echo "Call Trace:" >> /dev/kmsg
+    framework: shell
+    duration: 1m
+    check:
+        - how: dmesg
+    check-result: skip
+    # Expected outcome: PASS (test passes, check fails but is ignored)
+
+/test/check-skip:
+    summary: Test with failing dmesg check but ignored due to result interpretation
     test: |
         echo "Test passed"
         echo "Call Trace:" >> /dev/kmsg
@@ -29,3 +45,5 @@ description: Verify that check results, including after-test checks, are correct
     result: pass
     check:
         - how: dmesg
+    check-result: respect
+    # Expected outcome: PASS (test passes, check fails but is overridden by 'result: pass')

tmt/base.py

@@ -1126,6 +1126,7 @@ class Test(
         'order',
         'result',
         'check',
+        'check-result',
         'restart_on_exit_code',
         'restart_max_count',
         'restart_with_reboot',

tmt/result.py

@@ -261,11 +261,6 @@ class Result(BaseResult):
         serialize=lambda path: None if path is None else str(path),
         unserialize=lambda value: None if value is None else Path(value)
         )
-    respect_checks: bool = field(
-        default=True,
-        serialize=lambda value: value,
-        unserialize=lambda value: value
-        )
 
     @classmethod
     def from_test_invocation(
@@ -275,8 +270,7 @@ def from_test_invocation(
             result: ResultOutcome,
             note: Optional[str] = None,
             ids: Optional[ResultIds] = None,
-            log: Optional[list[Path]] = None,
-            respect_checks: bool = True) -> 'Result':
+            log: Optional[list[Path]] = None) -> 'Result':
         """
         Create a result from a test invocation.
 
@@ -293,7 +287,6 @@ def from_test_invocation(
         :param ids: additional test IDs. They will be added to IDs extracted
             from the test.
         :param log: optional list of test logs.
-        :param respect_checks: whether to respect or ignore check results.
         """
 
         # Saving identifiable information for each test case so we can match them
@@ -326,22 +319,25 @@ def from_test_invocation(
             log=log or [],
             guest=ResultGuestData.from_test_invocation(invocation=invocation),
             data_path=invocation.relative_test_data_path,
-            respect_checks=respect_checks,
             check=invocation.check_results)
 
         for check in _result.check:
             print(f"  - Name: {check.name}, Result: {check.result}, Event: {check.event}")
 
         return _result.interpret_result(invocation.test.result)
 
-    def interpret_result(self, interpret: ResultInterpret) -> 'Result':
+    def interpret_result(
+            self,
+            interpret: ResultInterpret,
+            check_interpret: ResultInterpret) -> 'Result':
         """
         Interpret result according to a given interpretation instruction.
 
         Inspect and possibly modify :py:attr:`result` and :py:attr:`note`
         attributes, following the ``interpret`` value.
 
         :param interpret: how to interpret current result.
+        :param check_interpret: how to interpret check results.
         :returns: :py:class:`Result` instance containing the updated result.
         """
 
@@ -352,7 +348,7 @@ def interpret_result(self, interpret: ResultInterpret) -> 'Result':
         checks_failed = any(check.result == ResultOutcome.FAIL for check in self.check)
 
         if interpret == ResultInterpret.RESPECT:
-            if self.respect_checks and checks_failed:
+            if check_interpret == ResultInterpret.RESPECT and checks_failed:
                 self.result = ResultOutcome.FAIL
                 if self.note:
                     self.note += ', check failed'

tmt/schemas/test.yaml

@@ -61,6 +61,14 @@ properties:
             - type: string
             - $ref: "/schemas/common#/definitions/check"
 
+  # https://tmt.readthedocs.io/en/stable/spec/tests.html#check-result
+  check-result:
+    type: string
+    enum:
+      - respect
+      - skip
+    default: respect
+
   # https://tmt.readthedocs.io/en/stable/spec/core.html#id
   id:
     $ref: "/schemas/core#/definitions/id"