Refactor to test.check.result

docs/releases.rst

@@ -4,6 +4,14 @@
     Releases
 ======================
 
+tmt-1.38.0
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A new ``result`` key has been added to the :ref:`test check</spec/tests/check>`
+specification. This key allows users to configure how check results are
+interpreted.
+
+
 tmt-1.37.0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -73,12 +81,6 @@ now wrapped in double quotes, and any double quotes within the
 values are escaped to ensure that the resulting file is always
 valid YAML.
 
-+A new ``check-result`` key has been added to the test specification. This key
-+allows users to configure whether "check" failure will affect the test result.
-+The key accepts two options: 'respect' (default) and 'skip'. When set to
-+'respect', check failures will affect the test result as before. When set to
-+'skip', check failures will not affect the overall test result.
-
 
 tmt-1.36.1
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

spec/tests/check.fmf

@@ -16,6 +16,15 @@ description: |
 
     See :ref:`/plugins/test-checks` for the list of available checks.
 
+    The `result` key can be used to specify how the check result should be
+    interpreted. It accepts the following options:
+    - 'respect' (default): The check result affects the overall test result.
+    - 'ignore': The check result does not affect the overall test result.
+    - 'xfail': The check is expected to fail. If it passes, the test fails,
+      and if it fails, the test passes.
+    - 'pass', 'fail', 'info', 'warn', 'error': The check result is interpreted
+      as the specified outcome, regardless of the actual check result.
+
 example:
   - |
     # Enable a single check, AVC denial detection.
@@ -37,6 +46,26 @@ example:
       - how: test-inspector
         enable: false
 
+  - |
+    # Enable a check with a specific result interpretation
+    check:
+      - how: dmesg
+        result: xfail
+
+  - |
+    # Enable multiple checks with different result interpretations
+    check:
+      - how: dmesg
+        result: ignore
+      - how: avc
+        result: xfail
+
+  - |
+    # Enable a check with a specific outcome interpretation
+    check:
+      - how: dmesg
+        result: warn
+
 link:
   - implemented-by: /tmt/checks
   - verified-by: /tests/test/check/avc

spec/tests/result.fmf

@@ -32,16 +32,6 @@ description: |
 
         .. versionadded:: 1.35
 
-    Additionally, the `check-result` attribute can be used to specify
-    how check results should be interpreted:
-
-    respect
-        check results are respected (test fails if any check fails) - default value
-    skip
-        check results are ignored (test result is not affected by check failures)
-
-        .. versionadded:: 1.37
-
 example:
   - |
     # Plain swapping fail to pass and pass to fail result
@@ -51,9 +41,6 @@ example:
     # Look for $TMT_TEST_DATA/results.yaml (or results.json) with custom results
     result: custom
 
-  - |
-    # Ignore check failures
-    check-result: skip
 
 link:
   - implemented-by: /tmt/base.py

tests/execute/result/check_results.sh

@@ -14,25 +14,52 @@ rlJournalStart
         rlAssertGrep "pass /test/check-pass" $rlRun_LOG
         rlAssertGrep "pass dmesg (before-test check)" $rlRun_LOG
         rlAssertGrep "pass dmesg (after-test check)" $rlRun_LOG
-        echo $rlRun_LOG
 
         rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-fail-respect execute -vv report -vv 2>&1 >/dev/null" 1
         rlAssertGrep "fail /test/check-fail-respect" $rlRun_LOG
         rlAssertGrep "pass dmesg (before-test check)" $rlRun_LOG
         rlAssertGrep "fail dmesg (after-test check)" $rlRun_LOG
         rlAssertGrep "check failed" $rlRun_LOG
 
-        rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-fail-skip execute -vv report -vv 2>&1 >/dev/null" 0
-        rlAssertGrep "pass /test/check-fail-skip" $rlRun_LOG
+        rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-fail-ignore execute -vv report -vv 2>&1 >/dev/null" 0
+        rlAssertGrep "pass /test/check-fail-ignore" $rlRun_LOG
         rlAssertGrep "pass dmesg (before-test check)" $rlRun_LOG
         rlAssertGrep "fail dmesg (after-test check)" $rlRun_LOG
         rlAssertNotGrep "check failed" $rlRun_LOG
 
-        rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-skip execute -vv report -vv 2>&1 >/dev/null" 0
-        rlAssertGrep "pass /test/check-skip" $rlRun_LOG
+        rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-xfail-pass execute -vv report -vv 2>&1 >/dev/null" 1
+        rlAssertGrep "fail /test/check-xfail-pass" $rlRun_LOG
+        rlAssertGrep "pass dmesg (before-test check)" $rlRun_LOG
+        rlAssertGrep "pass dmesg (after-test check)" $rlRun_LOG
+        rlAssertGrep "unexpected pass" $rlRun_LOG
+
+        rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-xfail-fail execute -vv report -vv 2>&1 >/dev/null" 0
+        rlAssertGrep "pass /test/check-xfail-fail" $rlRun_LOG
+        rlAssertGrep "pass dmesg (before-test check)" $rlRun_LOG
+        rlAssertGrep "fail dmesg (after-test check)" $rlRun_LOG
+        rlAssertGrep "expected failure" $rlRun_LOG
+
+        rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-multiple execute -vv report -vv 2>&1 >/dev/null" 1
+        rlAssertGrep "fail /test/check-multiple" $rlRun_LOG
+        rlAssertGrep "fail dmesg (after-test check)" $rlRun_LOG
+        rlAssertGrep "pass dmesg (before-test check)" $rlRun_LOG
+        rlAssertGrep "fail dmesg (before-test check)" $rlRun_LOG
+        rlAssertGrep "check failed" $rlRun_LOG
+        rlAssertGrep "unexpected pass" $rlRun_LOG
+
+        rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-override execute -vv report -vv 2>&1 >/dev/null" 0
+        rlAssertGrep "pass /test/check-override" $rlRun_LOG
         rlAssertGrep "pass dmesg (before-test check)" $rlRun_LOG
         rlAssertGrep "fail dmesg (after-test check)" $rlRun_LOG
         rlAssertNotGrep "check failed" $rlRun_LOG
+
+        rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-warn execute -vv report -vv 2>&1 >/dev/null" 1
+        rlAssertGrep "warn /test/check-warn" $rlRun_LOG
+        rlAssertGrep "warn dmesg (after-test check)" $rlRun_LOG
+
+        rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-info execute -vv report -vv 2>&1 >/dev/null" 0
+        rlAssertGrep "info /test/check-info" $rlRun_LOG
+        rlAssertGrep "info dmesg (after-test check)" $rlRun_LOG
     rlPhaseEnd
 
     rlPhaseStartTest "Verify results.yaml content"
@@ -43,52 +70,100 @@ rlJournalStart
         rlAssertGrep "name: /test/check-pass" $rlRun_LOG
         rlAssertGrep "result: pass" $rlRun_LOG
         rlAssertGrep "check:" $rlRun_LOG
-        rlAssertGrep "- name: dmesg" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
         rlAssertGrep "  result: pass" $rlRun_LOG
         rlAssertGrep "  event: before-test" $rlRun_LOG
-        rlAssertGrep "- name: dmesg" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
         rlAssertGrep "  result: pass" $rlRun_LOG
         rlAssertGrep "  event: after-test" $rlRun_LOG
-        rlAssertGrep "check_result: respect" $rlRun_LOG
 
         rlRun -s "yq e '.[1]' $results_file"
         rlAssertGrep "name: /test/check-fail-respect" $rlRun_LOG
         rlAssertGrep "result: fail" $rlRun_LOG
         rlAssertGrep "note: check failed" $rlRun_LOG
         rlAssertGrep "check:" $rlRun_LOG
-        rlAssertGrep "- name: dmesg" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
         rlAssertGrep "  result: pass" $rlRun_LOG
         rlAssertGrep "  event: before-test" $rlRun_LOG
-        rlAssertGrep "- name: dmesg" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
         rlAssertGrep "  result: fail" $rlRun_LOG
         rlAssertGrep "  event: after-test" $rlRun_LOG
-        rlAssertGrep "check_result: respect" $rlRun_LOG
 
         rlRun -s "yq e '.[2]' $results_file"
-        rlAssertGrep "name: /test/check-fail-skip" $rlRun_LOG
+        rlAssertGrep "name: /test/check-fail-ignore" $rlRun_LOG
         rlAssertGrep "result: pass" $rlRun_LOG
         rlAssertNotGrep "note: check failed" $rlRun_LOG
         rlAssertGrep "check:" $rlRun_LOG
-        rlAssertGrep "- name: dmesg" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
         rlAssertGrep "  result: pass" $rlRun_LOG
         rlAssertGrep "  event: before-test" $rlRun_LOG
-        rlAssertGrep "- name: dmesg" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
         rlAssertGrep "  result: fail" $rlRun_LOG
         rlAssertGrep "  event: after-test" $rlRun_LOG
-        rlAssertGrep "check_result: skip" $rlRun_LOG
 
         rlRun -s "yq e '.[3]' $results_file"
-        rlAssertGrep "name: /test/check-skip" $rlRun_LOG
+        rlAssertGrep "name: /test/check-xfail-pass" $rlRun_LOG
+        rlAssertGrep "result: fail" $rlRun_LOG
+        rlAssertGrep "note: unexpected pass" $rlRun_LOG
+        rlAssertGrep "check:" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: pass" $rlRun_LOG
+        rlAssertGrep "  event: before-test" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: pass" $rlRun_LOG
+        rlAssertGrep "  event: after-test" $rlRun_LOG
+
+        rlRun -s "yq e '.[4]' $results_file"
+        rlAssertGrep "name: /test/check-xfail-fail" $rlRun_LOG
+        rlAssertGrep "result: pass" $rlRun_LOG
+        rlAssertGrep "note: expected failure" $rlRun_LOG
+        rlAssertGrep "check:" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: pass" $rlRun_LOG
+        rlAssertGrep "  event: before-test" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: fail" $rlRun_LOG
+        rlAssertGrep "  event: after-test" $rlRun_LOG
+
+        rlRun -s "yq e '.[5]' $results_file"
+        rlAssertGrep "name: /test/check-multiple" $rlRun_LOG
+        rlAssertGrep "result: fail" $rlRun_LOG
+        rlAssertGrep "note: check failed, unexpected pass" $rlRun_LOG
+        rlAssertGrep "check:" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: fail" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: pass" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: fail" $rlRun_LOG
+
+        rlRun -s "yq e '.[6]' $results_file"
+        rlAssertGrep "name: /test/check-override" $rlRun_LOG
         rlAssertGrep "result: pass" $rlRun_LOG
         rlAssertNotGrep "note: check failed" $rlRun_LOG
         rlAssertGrep "check:" $rlRun_LOG
-        rlAssertGrep "- name: dmesg" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
         rlAssertGrep "  result: pass" $rlRun_LOG
         rlAssertGrep "  event: before-test" $rlRun_LOG
-        rlAssertGrep "- name: dmesg" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
         rlAssertGrep "  result: fail" $rlRun_LOG
         rlAssertGrep "  event: after-test" $rlRun_LOG
-        rlAssertGrep "check_result: respect" $rlRun_LOG
+
+        rlRun -s "yq e '.[7]' $results_file"
+        rlAssertGrep "name: /test/check-warn" $rlRun_LOG
+        rlAssertGrep "result: warn" $rlRun_LOG
+        rlAssertGrep "check:" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: warn" $rlRun_LOG
+        rlAssertGrep "  event: after-test" $rlRun_LOG
+
+        rlRun -s "yq e '.[8]' $results_file"
+        rlAssertGrep "name: /test/check-info" $rlRun_LOG
+        rlAssertGrep "result: info" $rlRun_LOG
+        rlAssertGrep "check:" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: info" $rlRun_LOG
+        rlAssertGrep "  event: after-test" $rlRun_LOG
     rlPhaseEnd
 
     rlPhaseStartCleanup

tests/execute/result/check_results/main.fmf

@@ -1,4 +1,4 @@
-summary: Tests for check results behavior
+summary: Tests for check results behaviour
 description: Verify that check results, including after-test checks, are correctly handled
 
 /test/check-pass:
@@ -8,42 +8,76 @@ description: Verify that check results, including after-test checks, are correct
     duration: 1m
     check:
         - how: dmesg
-    check-result: respect
+          result: respect
     # Expected outcome: PASS (test passes, check passes)
 
 /test/check-fail-respect:
     summary: Test with failing dmesg check (respect)
-    test: |
-        echo "Test passed"
-        echo "Call Trace:" >> /dev/kmsg
+    test: echo "Test passed"
     framework: shell
     duration: 1m
     check:
         - how: dmesg
-    check-result: respect
+          failure-pattern: '.*'
+          result: respect
     # Expected outcome: FAIL (test passes, but check fails and is respected)
 
-/test/check-fail-skip:
-    summary: Test with failing dmesg check (skip)
-    test: |
-        echo "Test passed"
-        echo "Call Trace:" >> /dev/kmsg
+/test/check-fail-ignore:
+    summary: Test with failing dmesg check (ignore)
+    test: echo "Test passed"
     framework: shell
     duration: 1m
     check:
         - how: dmesg
-    check-result: skip
+          failure-pattern: '.*'
+          result: ignore
     # Expected outcome: PASS (test passes, check fails but is ignored)
 
-/test/check-skip:
-    summary: Test with failing dmesg check but ignored due to result interpretation
-    test: |
-        echo "Test passed"
-        echo "Call Trace:" >> /dev/kmsg
+/test/check-xfail-pass:
+    summary: Test with passing dmesg check (xfail)
+    test: echo "Test passed"
+    framework: shell
+    duration: 1m
+    check:
+        - how: dmesg
+          result: xfail
+    # Expected outcome: FAIL (test passes, check passes but xfail expects it to fail)
+
+/test/check-xfail-fail:
+    summary: Test with failing dmesg check (xfail)
+    test: echo "Test passed"
+    framework: shell
+    duration: 1m
+    check:
+        - how: dmesg
+          failure-pattern: '.*'
+          result: xfail
+    # Expected outcome: PASS (test passes, check fails but xfail expects it to fail)
+
+/test/check-multiple:
+    summary: Test with multiple checks with different result interpretations
+    test: echo "Test passed"
+    framework: shell
+    duration: 1m
+    check:
+        - how: dmesg
+          failure-pattern: '.*'
+          result: respect
+        - how: dmesg
+          result: xfail
+        - how: dmesg
+          failure-pattern: '.*'
+          result: ignore
+    # Expected outcome: FAIL (first dmesg check fails and is respected, second dmesg check passes but xfail expects it to fail, third failing dmesg check is ignored)
+
+/test/check-override:
+    summary: Test with failing dmesg check but overridden by test result
+    test: echo "Test passed"
     framework: shell
     duration: 1m
     result: pass
     check:
         - how: dmesg
-    check-result: respect
+          failure-pattern: '.*'
+          result: respect
     # Expected outcome: PASS (test passes, check fails but is overridden by 'result: pass')

tests/execute/result/main.fmf

@@ -11,5 +11,5 @@
     summary: Test special characters generated to tmt-report-results.yaml
     test: ./special.sh
 /check_results:
-    summary: Test behavior of check results, including after-test checks
+    summary: Test behaviour of check results, including after-test checks
     test: ./check_results.sh

tmt/base.py

@@ -1126,7 +1126,6 @@ class Test(
         'order',
         'result',
         'check',
-        'check-result',
         'restart_on_exit_code',
         'restart_max_count',
         'restart_with_reboot',