From 284f2d29da14c7314cea7665098f29801a84b49e Mon Sep 17 00:00:00 2001 From: Martin Hoyer Date: Mon, 30 Sep 2024 17:36:38 +0200 Subject: [PATCH] Refactor to ResultInterpret --- docs/releases.rst | 4 ++-- spec/tests/result.fmf | 4 ++-- tests/execute/result/check_results.sh | 14 +++++------ tests/execute/result/check_results/main.fmf | 8 +++---- tmt/base.py | 3 +++ tmt/result.py | 26 ++++++++++----------- tmt/schemas/test.yaml | 2 +- 7 files changed, 32 insertions(+), 29 deletions(-) diff --git a/docs/releases.rst b/docs/releases.rst index 2fe06702ba..7722125402 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -51,9 +51,9 @@ backed by :ref:`prepare/feature` plugin. +A new ``check-result`` key has been added to the test specification. This key +allows users to configure whether "check" failure will affect the test result. -+The key accepts two options: 'respect' (default) and 'ignore'. When set to ++The key accepts two options: 'respect' (default) and 'skip'. When set to +'respect', check failures will affect the test result as before. When set to -+'ignore', check failures will not affect the overall test result. ++'skip', check failures will not affect the overall test result. tmt-1.36.1 diff --git a/spec/tests/result.fmf b/spec/tests/result.fmf index d73240d425..af147fceff 100644 --- a/spec/tests/result.fmf +++ b/spec/tests/result.fmf @@ -37,7 +37,7 @@ description: | respect check results are respected (test fails if any check fails) - default value - ignore + skip check results are ignored (test result is not affected by check failures) .. versionadded:: 1.37 @@ -53,7 +53,7 @@ example: - | # Ignore check failures - check-result: ignore + check-result: skip link: - implemented-by: /tmt/base.py diff --git a/tests/execute/result/check_results.sh b/tests/execute/result/check_results.sh index 85629a9cbc..8e51ef81ca 100644 --- a/tests/execute/result/check_results.sh +++ b/tests/execute/result/check_results.sh @@ -22,14 +22,14 @@ rlJournalStart rlAssertGrep "fail dmesg (after-test check)" $rlRun_LOG rlAssertGrep "check failed" $rlRun_LOG - rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-fail-ignore execute -vv report -vv 2>&1 >/dev/null" 0 - rlAssertGrep "pass /test/check-fail-ignore" $rlRun_LOG + rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-fail-skip execute -vv report -vv 2>&1 >/dev/null" 0 + rlAssertGrep "pass /test/check-fail-skip" $rlRun_LOG rlAssertGrep "pass dmesg (before-test check)" $rlRun_LOG rlAssertGrep "fail dmesg (after-test check)" $rlRun_LOG rlAssertNotGrep "check failed" $rlRun_LOG - rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-ignore execute -vv report -vv 2>&1 >/dev/null" 0 - rlAssertGrep "pass /test/check-ignore" $rlRun_LOG + rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-skip execute -vv report -vv 2>&1 >/dev/null" 0 + rlAssertGrep "pass /test/check-skip" $rlRun_LOG rlAssertGrep "pass dmesg (before-test check)" $rlRun_LOG rlAssertGrep "fail dmesg (after-test check)" $rlRun_LOG rlAssertNotGrep "check failed" $rlRun_LOG @@ -65,7 +65,7 @@ rlJournalStart rlAssertGrep "check_result: respect" $rlRun_LOG rlRun -s "yq e '.[2]' $results_file" - rlAssertGrep "name: /test/check-fail-ignore" $rlRun_LOG + rlAssertGrep "name: /test/check-fail-skip" $rlRun_LOG rlAssertGrep "result: pass" $rlRun_LOG rlAssertNotGrep "note: check failed" $rlRun_LOG rlAssertGrep "check:" $rlRun_LOG @@ -75,10 +75,10 @@ rlJournalStart rlAssertGrep "- name: dmesg" $rlRun_LOG rlAssertGrep " result: fail" $rlRun_LOG rlAssertGrep " event: after-test" $rlRun_LOG - rlAssertGrep "check_result: ignore" $rlRun_LOG + rlAssertGrep "check_result: skip" $rlRun_LOG rlRun -s "yq e '.[3]' $results_file" - rlAssertGrep "name: /test/check-ignore" $rlRun_LOG + rlAssertGrep "name: /test/check-skip" $rlRun_LOG rlAssertGrep "result: pass" $rlRun_LOG rlAssertNotGrep "note: check failed" $rlRun_LOG rlAssertGrep "check:" $rlRun_LOG diff --git a/tests/execute/result/check_results/main.fmf b/tests/execute/result/check_results/main.fmf index a2718bddc7..c2c4d9117a 100644 --- a/tests/execute/result/check_results/main.fmf +++ b/tests/execute/result/check_results/main.fmf @@ -23,8 +23,8 @@ description: Verify that check results, including after-test checks, are correct check-result: respect # Expected outcome: FAIL (test passes, but check fails and is respected) -/test/check-fail-ignore: - summary: Test with failing dmesg check (ignore) +/test/check-fail-skip: + summary: Test with failing dmesg check (skip) test: | echo "Test passed" echo "Call Trace:" >> /dev/kmsg @@ -32,10 +32,10 @@ description: Verify that check results, including after-test checks, are correct duration: 1m check: - how: dmesg - check-result: ignore + check-result: skip # Expected outcome: PASS (test passes, check fails but is ignored) -/test/check-ignore: +/test/check-skip: summary: Test with failing dmesg check but ignored due to result interpretation test: | echo "Test passed" diff --git a/tmt/base.py b/tmt/base.py index 40579a7b15..7e511003c9 100644 --- a/tmt/base.py +++ b/tmt/base.py @@ -1346,6 +1346,9 @@ def show(self) -> None: [check.to_spec() for check in cast(list[Check], value)] )) continue + if key == 'check_result': + echo(tmt.utils.format(key, value)) + continue if value not in [None, [], {}]: echo(tmt.utils.format(key, value)) if self.verbosity_level: diff --git a/tmt/result.py b/tmt/result.py index 7e172a47c2..496ad3a40f 100644 --- a/tmt/result.py +++ b/tmt/result.py @@ -237,11 +237,6 @@ class Result(BaseResult): serialize=lambda path: None if path is None else str(path), unserialize=lambda value: None if value is None else Path(value) ) - check_result: str = field( - default='respect', - serialize=lambda value: value, - unserialize=lambda value: value - ) @classmethod def from_test_invocation( @@ -286,8 +281,6 @@ def from_test_invocation( default_ids.update(ids) ids = default_ids - check_result = cast(str, invocation.test.node.get('check-result', 'respect')) - _result = Result( name=invocation.test.name, serial_number=invocation.test.serial_number, @@ -302,16 +295,22 @@ def from_test_invocation( log=log or [], guest=ResultGuestData.from_test_invocation(invocation=invocation), data_path=invocation.relative_test_data_path, - check_result=check_result, check=invocation.check_results) for check in _result.check: print(f" - Name: {check.name}, Result: {check.result}, Event: {check.event}") - return _result.interpret_result(ResultInterpret( - invocation.test.result) if invocation.test.result else ResultInterpret.RESPECT) - - def interpret_result(self, interpret: ResultInterpret) -> 'Result': + return _result.interpret_result( + ResultInterpret(invocation.test.result) + if invocation.test.result else ResultInterpret.RESPECT, + ResultInterpret(invocation.test.check_result) + if invocation.test.check_result else ResultInterpret.RESPECT + ) + + def interpret_result( + self, + interpret: ResultInterpret, + check_interpret: ResultInterpret) -> 'Result': """ Interpret result according to a given interpretation instruction. @@ -319,6 +318,7 @@ def interpret_result(self, interpret: ResultInterpret) -> 'Result': attributes, following the ``interpret`` value. :param interpret: how to interpret current result. + :param check_interpret: how to interpret check results. :returns: :py:class:`Result` instance containing the updated result. """ @@ -329,7 +329,7 @@ def interpret_result(self, interpret: ResultInterpret) -> 'Result': checks_failed = any(check.result == ResultOutcome.FAIL for check in self.check) if interpret == ResultInterpret.RESPECT: - if self.check_result == 'respect' and checks_failed: + if check_interpret == ResultInterpret.RESPECT and checks_failed: self.result = ResultOutcome.FAIL if self.note: self.note += ', check failed' diff --git a/tmt/schemas/test.yaml b/tmt/schemas/test.yaml index 8b9156f808..2214a9e6d7 100644 --- a/tmt/schemas/test.yaml +++ b/tmt/schemas/test.yaml @@ -66,7 +66,7 @@ properties: type: string enum: - respect - - ignore + - skip default: respect # https://tmt.readthedocs.io/en/stable/spec/core.html#id