Let's go with test.check.result

Which is what I initially wanted and now it's 8pm, send help.

docs/releases.rst

@@ -49,12 +49,13 @@ execution. Use ``tmt run discover -vvv`` to see the details.
 ``tmt try`` now supports :ref:`/stories/cli/try/option/epel` option
 backed by :ref:`prepare/feature</plugins/prepare/feature>` plugin.
 
-+A new ``check-result`` key has been added to the test specification. This key
-+allows users to configure whether "check" failure will affect the test result.
-+The key accepts two options: 'respect' (default) and 'skip'. When set to
-+'respect', check failures will affect the test result as before. When set to
-+'skip', check failures will not affect the overall test result.
-
+A new ``result`` key has been added to the test check specification. This key
+allows users to configure how check results are interpreted. The key accepts
+three options: 'respect' (default), 'skip', and 'xfail'. When set to 'respect',
+check failure will affect the test result. When set to 'skip', check
+results will not affect the overall test result. When set to 'xfail', the check
+is expected to fail, and the result will be inverted (fail becomes pass, and
+pass becomes fail).
 
 tmt-1.36.1
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

spec/tests/check.fmf

@@ -16,6 +16,13 @@ description: |
 
     See :ref:`/plugins/test-checks` for the list of available checks.
 
+    The `result` key can be used to specify how the check result should be
+    interpreted. It accepts three options:
+    - 'respect' (default): The check result affects the overall test result.
+    - 'skip': The check result does not affect the overall test result.
+    - 'xfail': The check is expected to fail. If it passes, the test fails,
+      and if it fails, the test passes.
+
 example:
   - |
     # Enable a single check, AVC denial detection.
@@ -37,6 +44,22 @@ example:
       - how: test-inspector
         enable: false
 
+  - |
+    # Enable a check with a specific result interpretation
+    check:
+      - how: dmesg
+        result: xfail
+
+  - |
+    # Enable multiple checks with different result interpretations
+    check:
+      - how: avc
+        result: respect
+      - how: dmesg
+        result: skip
+      - how: kernel-panic
+        result: xfail
+
 link:
   - implemented-by: /tmt/checks
   - verified-by: /tests/test/check/avc

tests/execute/result/check_results.sh

@@ -14,7 +14,6 @@ rlJournalStart
         rlAssertGrep "pass /test/check-pass" $rlRun_LOG
         rlAssertGrep "pass dmesg (before-test check)" $rlRun_LOG
         rlAssertGrep "pass dmesg (after-test check)" $rlRun_LOG
-        echo $rlRun_LOG
 
         rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-fail-respect execute -vv report -vv 2>&1 >/dev/null" 1
         rlAssertGrep "fail /test/check-fail-respect" $rlRun_LOG
@@ -28,8 +27,28 @@ rlJournalStart
         rlAssertGrep "fail dmesg (after-test check)" $rlRun_LOG
         rlAssertNotGrep "check failed" $rlRun_LOG
 
-        rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-skip execute -vv report -vv 2>&1 >/dev/null" 0
-        rlAssertGrep "pass /test/check-skip" $rlRun_LOG
+        rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-xfail-pass execute -vv report -vv 2>&1 >/dev/null" 1
+        rlAssertGrep "fail /test/check-xfail-pass" $rlRun_LOG
+        rlAssertGrep "pass dmesg (before-test check)" $rlRun_LOG
+        rlAssertGrep "pass dmesg (after-test check)" $rlRun_LOG
+        rlAssertGrep "unexpected pass" $rlRun_LOG
+
+        rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-xfail-fail execute -vv report -vv 2>&1 >/dev/null" 0
+        rlAssertGrep "pass /test/check-xfail-fail" $rlRun_LOG
+        rlAssertGrep "pass dmesg (before-test check)" $rlRun_LOG
+        rlAssertGrep "fail dmesg (after-test check)" $rlRun_LOG
+        rlAssertGrep "expected failure" $rlRun_LOG
+
+        rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-multiple execute -vv report -vv 2>&1 >/dev/null" 1
+        rlAssertGrep "fail /test/check-multiple" $rlRun_LOG
+        rlAssertGrep "fail dmesg (after-test check)" $rlRun_LOG
+        rlAssertGrep "pass dmesg (before-test check)" $rlRun_LOG
+        rlAssertGrep "fail dmesg (before-test check)" $rlRun_LOG
+        rlAssertGrep "check failed" $rlRun_LOG
+        rlAssertGrep "unexpected pass" $rlRun_LOG
+
+        rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-override execute -vv report -vv 2>&1 >/dev/null" 0
+        rlAssertGrep "pass /test/check-override" $rlRun_LOG
         rlAssertGrep "pass dmesg (before-test check)" $rlRun_LOG
         rlAssertGrep "fail dmesg (after-test check)" $rlRun_LOG
         rlAssertNotGrep "check failed" $rlRun_LOG
@@ -43,52 +62,84 @@ rlJournalStart
         rlAssertGrep "name: /test/check-pass" $rlRun_LOG
         rlAssertGrep "result: pass" $rlRun_LOG
         rlAssertGrep "check:" $rlRun_LOG
-        rlAssertGrep "- name: dmesg" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
         rlAssertGrep "  result: pass" $rlRun_LOG
         rlAssertGrep "  event: before-test" $rlRun_LOG
-        rlAssertGrep "- name: dmesg" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
         rlAssertGrep "  result: pass" $rlRun_LOG
         rlAssertGrep "  event: after-test" $rlRun_LOG
-        rlAssertGrep "check_result: respect" $rlRun_LOG
 
         rlRun -s "yq e '.[1]' $results_file"
         rlAssertGrep "name: /test/check-fail-respect" $rlRun_LOG
         rlAssertGrep "result: fail" $rlRun_LOG
         rlAssertGrep "note: check failed" $rlRun_LOG
         rlAssertGrep "check:" $rlRun_LOG
-        rlAssertGrep "- name: dmesg" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
         rlAssertGrep "  result: pass" $rlRun_LOG
         rlAssertGrep "  event: before-test" $rlRun_LOG
-        rlAssertGrep "- name: dmesg" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
         rlAssertGrep "  result: fail" $rlRun_LOG
         rlAssertGrep "  event: after-test" $rlRun_LOG
-        rlAssertGrep "check_result: respect" $rlRun_LOG
 
         rlRun -s "yq e '.[2]' $results_file"
         rlAssertGrep "name: /test/check-fail-skip" $rlRun_LOG
         rlAssertGrep "result: pass" $rlRun_LOG
         rlAssertNotGrep "note: check failed" $rlRun_LOG
         rlAssertGrep "check:" $rlRun_LOG
-        rlAssertGrep "- name: dmesg" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
         rlAssertGrep "  result: pass" $rlRun_LOG
         rlAssertGrep "  event: before-test" $rlRun_LOG
-        rlAssertGrep "- name: dmesg" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
         rlAssertGrep "  result: fail" $rlRun_LOG
         rlAssertGrep "  event: after-test" $rlRun_LOG
-        rlAssertGrep "check_result: skip" $rlRun_LOG
 
         rlRun -s "yq e '.[3]' $results_file"
-        rlAssertGrep "name: /test/check-skip" $rlRun_LOG
+        rlAssertGrep "name: /test/check-xfail-pass" $rlRun_LOG
+        rlAssertGrep "result: fail" $rlRun_LOG
+        rlAssertGrep "note: unexpected pass" $rlRun_LOG
+        rlAssertGrep "check:" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: pass" $rlRun_LOG
+        rlAssertGrep "  event: before-test" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: pass" $rlRun_LOG
+        rlAssertGrep "  event: after-test" $rlRun_LOG
+
+        rlRun -s "yq e '.[4]' $results_file"
+        rlAssertGrep "name: /test/check-xfail-fail" $rlRun_LOG
+        rlAssertGrep "result: pass" $rlRun_LOG
+        rlAssertGrep "note: expected failure" $rlRun_LOG
+        rlAssertGrep "check:" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: pass" $rlRun_LOG
+        rlAssertGrep "  event: before-test" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: fail" $rlRun_LOG
+        rlAssertGrep "  event: after-test" $rlRun_LOG
+
+        rlRun -s "yq e '.[5]' $results_file"
+        rlAssertGrep "name: /test/check-multiple" $rlRun_LOG
+        rlAssertGrep "result: fail" $rlRun_LOG
+        rlAssertGrep "note: check failed, unexpected pass" $rlRun_LOG
+        rlAssertGrep "check:" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: fail" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: pass" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: fail" $rlRun_LOG
+
+        rlRun -s "yq e '.[6]' $results_file"
+        rlAssertGrep "name: /test/check-override" $rlRun_LOG
         rlAssertGrep "result: pass" $rlRun_LOG
         rlAssertNotGrep "note: check failed" $rlRun_LOG
         rlAssertGrep "check:" $rlRun_LOG
-        rlAssertGrep "- name: dmesg" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
         rlAssertGrep "  result: pass" $rlRun_LOG
         rlAssertGrep "  event: before-test" $rlRun_LOG
-        rlAssertGrep "- name: dmesg" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
         rlAssertGrep "  result: fail" $rlRun_LOG
         rlAssertGrep "  event: after-test" $rlRun_LOG
-        rlAssertGrep "check_result: respect" $rlRun_LOG
     rlPhaseEnd
 
     rlPhaseStartCleanup

tests/execute/result/check_results/main.fmf

@@ -8,42 +8,76 @@ description: Verify that check results, including after-test checks, are correct
     duration: 1m
     check:
         - how: dmesg
-    check-result: respect
+          result: respect
     # Expected outcome: PASS (test passes, check passes)
 
 /test/check-fail-respect:
     summary: Test with failing dmesg check (respect)
-    test: |
-        echo "Test passed"
-        echo "Call Trace:" >> /dev/kmsg
+    test: echo "Test passed"
     framework: shell
     duration: 1m
     check:
         - how: dmesg
-    check-result: respect
+          failure-pattern: '.*'
+          result: respect
     # Expected outcome: FAIL (test passes, but check fails and is respected)
 
 /test/check-fail-skip:
     summary: Test with failing dmesg check (skip)
-    test: |
-        echo "Test passed"
-        echo "Call Trace:" >> /dev/kmsg
+    test: echo "Test passed"
     framework: shell
     duration: 1m
     check:
         - how: dmesg
-    check-result: skip
+          failure-pattern: '.*'
+          result: skip
     # Expected outcome: PASS (test passes, check fails but is ignored)
 
-/test/check-skip:
-    summary: Test with failing dmesg check but ignored due to result interpretation
-    test: |
-        echo "Test passed"
-        echo "Call Trace:" >> /dev/kmsg
+/test/check-xfail-pass:
+    summary: Test with passing dmesg check (xfail)
+    test: echo "Test passed"
+    framework: shell
+    duration: 1m
+    check:
+        - how: dmesg
+          result: xfail
+    # Expected outcome: FAIL (test passes, check passes but xfail expects it to fail)
+
+/test/check-xfail-fail:
+    summary: Test with failing dmesg check (xfail)
+    test: echo "Test passed"
+    framework: shell
+    duration: 1m
+    check:
+        - how: dmesg
+          failure-pattern: '.*'
+          result: xfail
+    # Expected outcome: PASS (test passes, check fails but xfail expects it to fail)
+
+/test/check-multiple:
+    summary: Test with multiple checks with different result interpretations
+    test: echo "Test passed"
+    framework: shell
+    duration: 1m
+    check:
+        - how: dmesg
+          failure-pattern: '.*'
+          result: respect
+        - how: dmesg
+          result: xfail
+        - how: dmesg
+          failure-pattern: '.*'
+          result: skip
+    # Expected outcome: FAIL (first dmesg check fails and is respected, second dmesg check passes but xfail expects it to fail, third failing dmesg check is skipped)
+
+/test/check-override:
+    summary: Test with failing dmesg check but overridden by test result
+    test: echo "Test passed"
     framework: shell
     duration: 1m
     result: pass
     check:
         - how: dmesg
-    check-result: respect
+          failure-pattern: '.*'
+          result: respect
     # Expected outcome: PASS (test passes, check fails but is overridden by 'result: pass')

tmt/base.py

@@ -1074,7 +1074,6 @@ class Test(
 
     duration: str = DEFAULT_TEST_DURATION_L1
     result: str = 'respect'
-    check_result: str = 'respect'
 
     where: list[str] = field(default_factory=list)
 
@@ -1121,7 +1120,6 @@ class Test(
         'order',
         'result',
         'check',
-        'check-result',
         'restart_on_exit_code',
         'restart_max_count',
         'restart_with_reboot',
@@ -1346,9 +1344,6 @@ def show(self) -> None:
                     [check.to_spec() for check in cast(list[Check], value)]
                     ))
                 continue
-            if key == 'check_result':
-                echo(tmt.utils.format(key, value))
-                continue
             if value not in [None, [], {}]:
                 echo(tmt.utils.format(key, value))
         if self.verbosity_level:

tmt/checks/__init__.py

@@ -67,6 +67,7 @@ def find_plugin(name: str) -> 'CheckPluginClass':
 class _RawCheck(TypedDict):
     how: str
     enabled: bool
+    result: str
 
 
 class CheckEvent(enum.Enum):
@@ -100,6 +101,11 @@ class Check(
         default=True,
         is_flag=True,
         help='Whether the check is enabled or not.')
+    result: Any = field(
+        default='respect',
+        serialize=lambda result: result.value if isinstance(result, enum.Enum) else str(result),
+        unserialize=lambda spec: spec,
+        help='How to interpret the check result.')
 
     @functools.cached_property
     def plugin(self) -> 'CheckPluginClass':
@@ -228,14 +234,16 @@ def normalize_test_check(
     if isinstance(raw_test_check, str):
         try:
             return CheckPlugin.delegate(
-                raw_data={'how': raw_test_check, 'enabled': True},
+                raw_data={'how': raw_test_check, 'enabled': True, 'result': 'respect'},
                 logger=logger)
 
         except Exception as exc:
             raise tmt.utils.SpecificationError(
                 f"Cannot instantiate check from '{key_address}'.") from exc
 
     if isinstance(raw_test_check, dict):
+        if 'result' not in raw_test_check:
+            raw_test_check['result'] = 'respect'
         try:
             return CheckPlugin.delegate(
                 raw_data=cast(_RawCheck, raw_test_check),