diff --git a/docs/releases.rst b/docs/releases.rst index 7722125402..ef19904fc1 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -49,12 +49,13 @@ execution. Use ``tmt run discover -vvv`` to see the details. ``tmt try`` now supports :ref:`/stories/cli/try/option/epel` option backed by :ref:`prepare/feature` plugin. -+A new ``check-result`` key has been added to the test specification. This key -+allows users to configure whether "check" failure will affect the test result. -+The key accepts two options: 'respect' (default) and 'skip'. When set to -+'respect', check failures will affect the test result as before. When set to -+'skip', check failures will not affect the overall test result. - +A new ``result`` key has been added to the test check specification. This key +allows users to configure how check results are interpreted. The key accepts +three options: 'respect' (default), 'skip', and 'xfail'. When set to 'respect', +check failure will affect the test result. When set to 'skip', check +results will not affect the overall test result. When set to 'xfail', the check +is expected to fail, and the result will be inverted (fail becomes pass, and +pass becomes fail). tmt-1.36.1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/spec/tests/check.fmf b/spec/tests/check.fmf index 102eb4771c..8f50a6ef3f 100644 --- a/spec/tests/check.fmf +++ b/spec/tests/check.fmf @@ -16,6 +16,13 @@ description: | See :ref:`/plugins/test-checks` for the list of available checks. + The `result` key can be used to specify how the check result should be + interpreted. It accepts three options: + - 'respect' (default): The check result affects the overall test result. + - 'skip': The check result does not affect the overall test result. + - 'xfail': The check is expected to fail. If it passes, the test fails, + and if it fails, the test passes. + example: - | # Enable a single check, AVC denial detection. @@ -37,6 +44,22 @@ example: - how: test-inspector enable: false + - | + # Enable a check with a specific result interpretation + check: + - how: dmesg + result: xfail + + - | + # Enable multiple checks with different result interpretations + check: + - how: avc + result: respect + - how: dmesg + result: skip + - how: kernel-panic + result: xfail + link: - implemented-by: /tmt/checks - verified-by: /tests/test/check/avc diff --git a/tests/execute/result/check_results.sh b/tests/execute/result/check_results.sh index 8e51ef81ca..aa946a03f8 100644 --- a/tests/execute/result/check_results.sh +++ b/tests/execute/result/check_results.sh @@ -14,7 +14,6 @@ rlJournalStart rlAssertGrep "pass /test/check-pass" $rlRun_LOG rlAssertGrep "pass dmesg (before-test check)" $rlRun_LOG rlAssertGrep "pass dmesg (after-test check)" $rlRun_LOG - echo $rlRun_LOG rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-fail-respect execute -vv report -vv 2>&1 >/dev/null" 1 rlAssertGrep "fail /test/check-fail-respect" $rlRun_LOG @@ -28,8 +27,28 @@ rlJournalStart rlAssertGrep "fail dmesg (after-test check)" $rlRun_LOG rlAssertNotGrep "check failed" $rlRun_LOG - rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-skip execute -vv report -vv 2>&1 >/dev/null" 0 - rlAssertGrep "pass /test/check-skip" $rlRun_LOG + rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-xfail-pass execute -vv report -vv 2>&1 >/dev/null" 1 + rlAssertGrep "fail /test/check-xfail-pass" $rlRun_LOG + rlAssertGrep "pass dmesg (before-test check)" $rlRun_LOG + rlAssertGrep "pass dmesg (after-test check)" $rlRun_LOG + rlAssertGrep "unexpected pass" $rlRun_LOG + + rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-xfail-fail execute -vv report -vv 2>&1 >/dev/null" 0 + rlAssertGrep "pass /test/check-xfail-fail" $rlRun_LOG + rlAssertGrep "pass dmesg (before-test check)" $rlRun_LOG + rlAssertGrep "fail dmesg (after-test check)" $rlRun_LOG + rlAssertGrep "expected failure" $rlRun_LOG + + rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-multiple execute -vv report -vv 2>&1 >/dev/null" 1 + rlAssertGrep "fail /test/check-multiple" $rlRun_LOG + rlAssertGrep "fail dmesg (after-test check)" $rlRun_LOG + rlAssertGrep "pass dmesg (before-test check)" $rlRun_LOG + rlAssertGrep "fail dmesg (before-test check)" $rlRun_LOG + rlAssertGrep "check failed" $rlRun_LOG + rlAssertGrep "unexpected pass" $rlRun_LOG + + rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-override execute -vv report -vv 2>&1 >/dev/null" 0 + rlAssertGrep "pass /test/check-override" $rlRun_LOG rlAssertGrep "pass dmesg (before-test check)" $rlRun_LOG rlAssertGrep "fail dmesg (after-test check)" $rlRun_LOG rlAssertNotGrep "check failed" $rlRun_LOG @@ -43,52 +62,84 @@ rlJournalStart rlAssertGrep "name: /test/check-pass" $rlRun_LOG rlAssertGrep "result: pass" $rlRun_LOG rlAssertGrep "check:" $rlRun_LOG - rlAssertGrep "- name: dmesg" $rlRun_LOG + rlAssertGrep "- how: dmesg" $rlRun_LOG rlAssertGrep " result: pass" $rlRun_LOG rlAssertGrep " event: before-test" $rlRun_LOG - rlAssertGrep "- name: dmesg" $rlRun_LOG + rlAssertGrep "- how: dmesg" $rlRun_LOG rlAssertGrep " result: pass" $rlRun_LOG rlAssertGrep " event: after-test" $rlRun_LOG - rlAssertGrep "check_result: respect" $rlRun_LOG rlRun -s "yq e '.[1]' $results_file" rlAssertGrep "name: /test/check-fail-respect" $rlRun_LOG rlAssertGrep "result: fail" $rlRun_LOG rlAssertGrep "note: check failed" $rlRun_LOG rlAssertGrep "check:" $rlRun_LOG - rlAssertGrep "- name: dmesg" $rlRun_LOG + rlAssertGrep "- how: dmesg" $rlRun_LOG rlAssertGrep " result: pass" $rlRun_LOG rlAssertGrep " event: before-test" $rlRun_LOG - rlAssertGrep "- name: dmesg" $rlRun_LOG + rlAssertGrep "- how: dmesg" $rlRun_LOG rlAssertGrep " result: fail" $rlRun_LOG rlAssertGrep " event: after-test" $rlRun_LOG - rlAssertGrep "check_result: respect" $rlRun_LOG rlRun -s "yq e '.[2]' $results_file" rlAssertGrep "name: /test/check-fail-skip" $rlRun_LOG rlAssertGrep "result: pass" $rlRun_LOG rlAssertNotGrep "note: check failed" $rlRun_LOG rlAssertGrep "check:" $rlRun_LOG - rlAssertGrep "- name: dmesg" $rlRun_LOG + rlAssertGrep "- how: dmesg" $rlRun_LOG rlAssertGrep " result: pass" $rlRun_LOG rlAssertGrep " event: before-test" $rlRun_LOG - rlAssertGrep "- name: dmesg" $rlRun_LOG + rlAssertGrep "- how: dmesg" $rlRun_LOG rlAssertGrep " result: fail" $rlRun_LOG rlAssertGrep " event: after-test" $rlRun_LOG - rlAssertGrep "check_result: skip" $rlRun_LOG rlRun -s "yq e '.[3]' $results_file" - rlAssertGrep "name: /test/check-skip" $rlRun_LOG + rlAssertGrep "name: /test/check-xfail-pass" $rlRun_LOG + rlAssertGrep "result: fail" $rlRun_LOG + rlAssertGrep "note: unexpected pass" $rlRun_LOG + rlAssertGrep "check:" $rlRun_LOG + rlAssertGrep "- how: dmesg" $rlRun_LOG + rlAssertGrep " result: pass" $rlRun_LOG + rlAssertGrep " event: before-test" $rlRun_LOG + rlAssertGrep "- how: dmesg" $rlRun_LOG + rlAssertGrep " result: pass" $rlRun_LOG + rlAssertGrep " event: after-test" $rlRun_LOG + + rlRun -s "yq e '.[4]' $results_file" + rlAssertGrep "name: /test/check-xfail-fail" $rlRun_LOG + rlAssertGrep "result: pass" $rlRun_LOG + rlAssertGrep "note: expected failure" $rlRun_LOG + rlAssertGrep "check:" $rlRun_LOG + rlAssertGrep "- how: dmesg" $rlRun_LOG + rlAssertGrep " result: pass" $rlRun_LOG + rlAssertGrep " event: before-test" $rlRun_LOG + rlAssertGrep "- how: dmesg" $rlRun_LOG + rlAssertGrep " result: fail" $rlRun_LOG + rlAssertGrep " event: after-test" $rlRun_LOG + + rlRun -s "yq e '.[5]' $results_file" + rlAssertGrep "name: /test/check-multiple" $rlRun_LOG + rlAssertGrep "result: fail" $rlRun_LOG + rlAssertGrep "note: check failed, unexpected pass" $rlRun_LOG + rlAssertGrep "check:" $rlRun_LOG + rlAssertGrep "- how: dmesg" $rlRun_LOG + rlAssertGrep " result: fail" $rlRun_LOG + rlAssertGrep "- how: dmesg" $rlRun_LOG + rlAssertGrep " result: pass" $rlRun_LOG + rlAssertGrep "- how: dmesg" $rlRun_LOG + rlAssertGrep " result: fail" $rlRun_LOG + + rlRun -s "yq e '.[6]' $results_file" + rlAssertGrep "name: /test/check-override" $rlRun_LOG rlAssertGrep "result: pass" $rlRun_LOG rlAssertNotGrep "note: check failed" $rlRun_LOG rlAssertGrep "check:" $rlRun_LOG - rlAssertGrep "- name: dmesg" $rlRun_LOG + rlAssertGrep "- how: dmesg" $rlRun_LOG rlAssertGrep " result: pass" $rlRun_LOG rlAssertGrep " event: before-test" $rlRun_LOG - rlAssertGrep "- name: dmesg" $rlRun_LOG + rlAssertGrep "- how: dmesg" $rlRun_LOG rlAssertGrep " result: fail" $rlRun_LOG rlAssertGrep " event: after-test" $rlRun_LOG - rlAssertGrep "check_result: respect" $rlRun_LOG rlPhaseEnd rlPhaseStartCleanup diff --git a/tests/execute/result/check_results/main.fmf b/tests/execute/result/check_results/main.fmf index c2c4d9117a..8a1dea7e07 100644 --- a/tests/execute/result/check_results/main.fmf +++ b/tests/execute/result/check_results/main.fmf @@ -8,42 +8,76 @@ description: Verify that check results, including after-test checks, are correct duration: 1m check: - how: dmesg - check-result: respect + result: respect # Expected outcome: PASS (test passes, check passes) /test/check-fail-respect: summary: Test with failing dmesg check (respect) - test: | - echo "Test passed" - echo "Call Trace:" >> /dev/kmsg + test: echo "Test passed" framework: shell duration: 1m check: - how: dmesg - check-result: respect + failure-pattern: '.*' + result: respect # Expected outcome: FAIL (test passes, but check fails and is respected) /test/check-fail-skip: summary: Test with failing dmesg check (skip) - test: | - echo "Test passed" - echo "Call Trace:" >> /dev/kmsg + test: echo "Test passed" framework: shell duration: 1m check: - how: dmesg - check-result: skip + failure-pattern: '.*' + result: skip # Expected outcome: PASS (test passes, check fails but is ignored) -/test/check-skip: - summary: Test with failing dmesg check but ignored due to result interpretation - test: | - echo "Test passed" - echo "Call Trace:" >> /dev/kmsg +/test/check-xfail-pass: + summary: Test with passing dmesg check (xfail) + test: echo "Test passed" + framework: shell + duration: 1m + check: + - how: dmesg + result: xfail + # Expected outcome: FAIL (test passes, check passes but xfail expects it to fail) + +/test/check-xfail-fail: + summary: Test with failing dmesg check (xfail) + test: echo "Test passed" + framework: shell + duration: 1m + check: + - how: dmesg + failure-pattern: '.*' + result: xfail + # Expected outcome: PASS (test passes, check fails but xfail expects it to fail) + +/test/check-multiple: + summary: Test with multiple checks with different result interpretations + test: echo "Test passed" + framework: shell + duration: 1m + check: + - how: dmesg + failure-pattern: '.*' + result: respect + - how: dmesg + result: xfail + - how: dmesg + failure-pattern: '.*' + result: skip + # Expected outcome: FAIL (first dmesg check fails and is respected, second dmesg check passes but xfail expects it to fail, third failing dmesg check is skipped) + +/test/check-override: + summary: Test with failing dmesg check but overridden by test result + test: echo "Test passed" framework: shell duration: 1m result: pass check: - how: dmesg - check-result: respect + failure-pattern: '.*' + result: respect # Expected outcome: PASS (test passes, check fails but is overridden by 'result: pass') diff --git a/tmt/base.py b/tmt/base.py index 7e511003c9..7aa07407c1 100644 --- a/tmt/base.py +++ b/tmt/base.py @@ -1074,7 +1074,6 @@ class Test( duration: str = DEFAULT_TEST_DURATION_L1 result: str = 'respect' - check_result: str = 'respect' where: list[str] = field(default_factory=list) @@ -1121,7 +1120,6 @@ class Test( 'order', 'result', 'check', - 'check-result', 'restart_on_exit_code', 'restart_max_count', 'restart_with_reboot', @@ -1346,9 +1344,6 @@ def show(self) -> None: [check.to_spec() for check in cast(list[Check], value)] )) continue - if key == 'check_result': - echo(tmt.utils.format(key, value)) - continue if value not in [None, [], {}]: echo(tmt.utils.format(key, value)) if self.verbosity_level: diff --git a/tmt/checks/__init__.py b/tmt/checks/__init__.py index 905c19f72f..dd93ea052b 100644 --- a/tmt/checks/__init__.py +++ b/tmt/checks/__init__.py @@ -67,6 +67,7 @@ def find_plugin(name: str) -> 'CheckPluginClass': class _RawCheck(TypedDict): how: str enabled: bool + result: str class CheckEvent(enum.Enum): @@ -100,6 +101,11 @@ class Check( default=True, is_flag=True, help='Whether the check is enabled or not.') + result: Any = field( + default='respect', + serialize=lambda result: result.value if isinstance(result, enum.Enum) else str(result), + unserialize=lambda spec: spec, + help='How to interpret the check result.') @functools.cached_property def plugin(self) -> 'CheckPluginClass': @@ -228,7 +234,7 @@ def normalize_test_check( if isinstance(raw_test_check, str): try: return CheckPlugin.delegate( - raw_data={'how': raw_test_check, 'enabled': True}, + raw_data={'how': raw_test_check, 'enabled': True, 'result': 'respect'}, logger=logger) except Exception as exc: @@ -236,6 +242,8 @@ def normalize_test_check( f"Cannot instantiate check from '{key_address}'.") from exc if isinstance(raw_test_check, dict): + if 'result' not in raw_test_check: + raw_test_check['result'] = 'respect' try: return CheckPlugin.delegate( raw_data=cast(_RawCheck, raw_test_check), diff --git a/tmt/result.py b/tmt/result.py index 496ad3a40f..0e1c42bc40 100644 --- a/tmt/result.py +++ b/tmt/result.py @@ -302,15 +302,12 @@ def from_test_invocation( return _result.interpret_result( ResultInterpret(invocation.test.result) - if invocation.test.result else ResultInterpret.RESPECT, - ResultInterpret(invocation.test.check_result) - if invocation.test.check_result else ResultInterpret.RESPECT + if invocation.test.result else ResultInterpret.RESPECT ) def interpret_result( self, - interpret: ResultInterpret, - check_interpret: ResultInterpret) -> 'Result': + interpret: ResultInterpret) -> 'Result': """ Interpret result according to a given interpretation instruction. @@ -318,7 +315,6 @@ def interpret_result( attributes, following the ``interpret`` value. :param interpret: how to interpret current result. - :param check_interpret: how to interpret check results. :returns: :py:class:`Result` instance containing the updated result. """ @@ -329,7 +325,7 @@ def interpret_result( checks_failed = any(check.result == ResultOutcome.FAIL for check in self.check) if interpret == ResultInterpret.RESPECT: - if check_interpret == ResultInterpret.RESPECT and checks_failed: + if checks_failed: self.result = ResultOutcome.FAIL if self.note: self.note += ', check failed' @@ -337,7 +333,7 @@ def interpret_result( self.note = 'check failed' return self - # Handle XFAIL + # Handle XFAIL for the overall test result if interpret == ResultInterpret.XFAIL: if self.result == ResultOutcome.PASS and checks_failed: self.result = ResultOutcome.FAIL @@ -359,6 +355,45 @@ def interpret_result( self.note = 'unexpected pass' return self + # Handle individual check results + for check in self.check: + check_result = ResultInterpret(check.result) + + if check_result == ResultInterpret.RESPECT: + if check.result == ResultOutcome.FAIL: + self.result = ResultOutcome.FAIL + if self.note: + self.note += ', check failed' + else: + self.note = 'check failed' + + elif check_result == ResultInterpret.XFAIL: + if check.result == ResultOutcome.PASS: + self.result = ResultOutcome.FAIL + if self.note: + self.note += ', unexpected pass in check' + else: + self.note = 'unexpected pass in check' + elif check.result == ResultOutcome.FAIL: + # Check failed as expected, don't change the overall result + if self.note: + self.note += ', expected failure in check' + else: + self.note = 'expected failure in check' + + elif check_result == ResultInterpret.SKIP: + continue + + elif ResultInterpret.is_result_outcome(check_result): + self.result = ResultOutcome(check_result.value) + if self.note: + self.note += f', check result: {check.result.value}' + else: + self.note = f'check result: {check.result.value}' + else: + raise tmt.utils.SpecificationError( + f"Invalid result '{check_result.value}' in check for test '{self.name}'.") + # Extend existing note or set a new one if self.note: self.note += f', original result: {self.result.value}' diff --git a/tmt/schemas/test.yaml b/tmt/schemas/test.yaml index 2214a9e6d7..ddc6f8d776 100644 --- a/tmt/schemas/test.yaml +++ b/tmt/schemas/test.yaml @@ -60,6 +60,16 @@ properties: anyOf: - type: string - $ref: "/schemas/common#/definitions/check" + - type: object + properties: + how: + type: string + enabled: + type: boolean + result: + $ref: "/schemas/common#/definitions/result_interpret" + required: + - how # https://tmt.readthedocs.io/en/stable/spec/tests.html#check-result check-result: @@ -67,6 +77,7 @@ properties: enum: - respect - skip + - xfail default: respect # https://tmt.readthedocs.io/en/stable/spec/core.html#id