diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index 425dc44ebf..c13c6a979a 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -328,7 +328,7 @@ test_pytest_qemu: -m qemu --embedded-services idf,qemu --junitxml=XUNIT_RESULT.xml - --known-failure-cases-file known_failure_cases/known_failure_cases.txt + --ignore-result-files known_failure_cases/known_failure_cases.txt --app-info-filepattern \"list_job_*.txt\" test_pytest_linux: @@ -356,5 +356,5 @@ test_pytest_linux: --target linux -m host_test --junitxml=XUNIT_RESULT.xml - --known-failure-cases-file known_failure_cases/known_failure_cases.txt + --ignore-result-files known_failure_cases/known_failure_cases.txt --app-info-filepattern \"list_job_*.txt\" diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml index ee5c598733..85265869e6 100644 --- a/.gitlab/ci/target-test.yml +++ b/.gitlab/ci/target-test.yml @@ -35,7 +35,7 @@ - run_cmd pytest $TEST_DIR -m \"${markers}\" --junitxml=XUNIT_RESULT.xml - --known-failure-cases-file known_failure_cases/known_failure_cases.txt + --ignore-result-files known_failure_cases/known_failure_cases.txt --parallel-count ${CI_NODE_TOTAL:-1} --parallel-index ${CI_NODE_INDEX:-1} ${PYTEST_EXTRA_FLAGS} diff --git a/conftest.py b/conftest.py index 9502ed5d90..6e97d8d578 100644 --- a/conftest.py +++ b/conftest.py @@ -251,7 +251,6 @@ def pytest_addoption(parser: pytest.Parser) -> None: '--sdkconfig', help='sdkconfig postfix, like sdkconfig.ci.. (Default: None, which would build all found apps)', ) - idf_group.addoption('--known-failure-cases-file', help='known failure cases file path') idf_group.addoption( '--dev-user', help='user name associated with some specific device/service used during the test execution', @@ -313,7 +312,6 @@ def pytest_configure(config: Config) -> None: config.stash[IDF_PYTEST_EMBEDDED_KEY] = IdfPytestEmbedded( target=target, sdkconfig=config.getoption('sdkconfig'), - known_failure_cases_file=config.getoption('known_failure_cases_file'), apps_list=apps_list, ) config.pluginmanager.register(config.stash[IDF_PYTEST_EMBEDDED_KEY]) diff --git a/tools/ci/idf_pytest/plugin.py b/tools/ci/idf_pytest/plugin.py index 12a440411d..b9dd1ed9a8 100644 --- a/tools/ci/idf_pytest/plugin.py +++ b/tools/ci/idf_pytest/plugin.py @@ -4,18 +4,17 @@ import logging import os import typing as t -from fnmatch import fnmatch from xml.etree import ElementTree as ET import pytest from _pytest.config import ExitCode from _pytest.main import Session from _pytest.python import Function -from _pytest.reports import TestReport from _pytest.runner import CallInfo -from _pytest.terminal import TerminalReporter +from pytest_embedded import Dut from pytest_embedded.plugin import parse_multi_dut_args from pytest_embedded.utils import find_by_suffix, to_list +from pytest_ignore_test_results.ignore_results import ChildCase, ChildCasesStashKey from .constants import DEFAULT_SDKCONFIG, PREVIEW_TARGETS, SUPPORTED_TARGETS, PytestApp, PytestCase from .utils import format_case_id, merge_junit_files @@ -26,55 +25,25 @@ ITEM_FAILED_KEY = pytest.StashKey[bool]() class IdfPytestEmbedded: + UNITY_RESULT_MAPPINGS = { + 'PASS': 'passed', + 'FAIL': 'failed', + 'IGNORE': 'skipped', + } + def __init__( self, target: str, sdkconfig: t.Optional[str] = None, - known_failure_cases_file: t.Optional[str] = None, apps_list: t.Optional[t.List[str]] = None, ): # CLI options to filter the test cases self.target = target.lower() self.sdkconfig = sdkconfig - self.known_failure_patterns = self._parse_known_failure_cases_file(known_failure_cases_file) self.apps_list = apps_list self.cases: t.List[PytestCase] = [] - self._failed_cases: t.List[t.Tuple[str, bool, bool]] = [] # (test_case_name, is_known_failure_cases, is_xfail) - - @property - def failed_cases(self) -> t.List[str]: - return [case for case, is_known, is_xfail in self._failed_cases if not is_known and not is_xfail] - - @property - def known_failure_cases(self) -> t.List[str]: - return [case for case, is_known, _ in self._failed_cases if is_known] - - @property - def xfail_cases(self) -> t.List[str]: - return [case for case, _, is_xfail in self._failed_cases if is_xfail] - - @staticmethod - def _parse_known_failure_cases_file( - known_failure_cases_file: t.Optional[str] = None, - ) -> t.List[str]: - if not known_failure_cases_file or not os.path.isfile(known_failure_cases_file): - return [] - - patterns = [] - with open(known_failure_cases_file) as fr: - for line in fr.readlines(): - if not line: - continue - if not line.strip(): - continue - without_comments = line.split('#')[0].strip() - if without_comments: - patterns.append(without_comments) - - return patterns - @staticmethod def get_param(item: Function, key: str, default: t.Any = None) -> t.Any: # implement like this since this is a limitation of pytest, couldn't get fixture values while collecting @@ -214,54 +183,46 @@ class IdfPytestEmbedded: for item in items: self.cases.append(self.item_to_pytest_case(item)) - def pytest_runtest_makereport(self, item: Function, call: CallInfo[None]) -> t.Optional[TestReport]: - report = TestReport.from_item_and_call(item, call) - if item.stash.get(ITEM_FAILED_KEY, None) is None: - item.stash[ITEM_FAILED_KEY] = False + def pytest_custom_test_case_name(self, item: Function) -> str: + return item.funcargs.get('test_case_name', item.nodeid) # type: ignore - if report.outcome == 'failed': - # Mark the failed test cases - # - # This hook function would be called in 3 phases, setup, call, teardown. - # the report.outcome is the outcome of the single call of current phase, which is independent - # the call phase outcome is the test result - item.stash[ITEM_FAILED_KEY] = True + def pytest_runtest_makereport(self, item: Function, call: CallInfo[None]) -> None: + if call.when == 'call': + target = item.funcargs['target'] + config = item.funcargs['config'] + is_qemu = item.get_closest_marker('qemu') is not None - if call.when == 'teardown': - item_failed = item.stash[ITEM_FAILED_KEY] - if item_failed: - # unity real test cases - failed_sub_cases = item.stash.get(ITEM_FAILED_CASES_KEY, []) - if failed_sub_cases: - for test_case_name in failed_sub_cases: - self._failed_cases.append((test_case_name, self._is_known_failure(test_case_name), False)) - else: # the case iteself is failing - test_case_name = item.funcargs.get('test_case_name', '') - if test_case_name: - self._failed_cases.append( - ( - test_case_name, - self._is_known_failure(test_case_name), - report.keywords.get('xfail', False), + dut: t.Union[Dut, t.Tuple[Dut]] = item.funcargs['dut'] # type: ignore + if isinstance(dut, (list, tuple)): + res = [] + for i, _dut in enumerate(dut): + res.extend( + [ + ChildCase( + format_case_id(target, config, case.name + f' {i}', is_qemu=is_qemu), + self.UNITY_RESULT_MAPPINGS[case.result], ) + for case in _dut.testsuite.testcases + ] + ) + item.config.stash[ChildCasesStashKey] = {item.nodeid: res} + else: + item.config.stash[ChildCasesStashKey] = { + item.nodeid: [ + ChildCase( + format_case_id(target, config, case.name, is_qemu=is_qemu), + self.UNITY_RESULT_MAPPINGS[case.result], ) - - return report - - def _is_known_failure(self, case_id: str) -> bool: - for pattern in self.known_failure_patterns: - if case_id == pattern: - return True - if fnmatch(case_id, pattern): - return True - return False + for case in dut.testsuite.testcases + ] + } @pytest.hookimpl(trylast=True) def pytest_runtest_teardown(self, item: Function) -> None: """ - Format the test case generated junit reports + Modify the junit reports. Format the unity c test case names. """ - tempdir = item.funcargs.get('test_case_tempdir') + tempdir: t.Optional[str] = item.funcargs.get('test_case_tempdir') # type: ignore if not tempdir: return @@ -273,8 +234,8 @@ class IdfPytestEmbedded: merge_junit_files(junits, os.path.join(tempdir, 'dut.xml')) junits = [os.path.join(tempdir, 'dut.xml')] + # unity cases is_qemu = item.get_closest_marker('qemu') is not None - failed_sub_cases = [] target = item.funcargs['target'] config = item.funcargs['config'] for junit in junits: @@ -287,30 +248,9 @@ class IdfPytestEmbedded: if 'file' in case.attrib: case.attrib['file'] = case.attrib['file'].replace('/IDF/', '') # our unity test framework - # collect real failure cases - if case.find('failure') is not None: - failed_sub_cases.append(new_case_name) - xml.write(junit) - item.stash[ITEM_FAILED_CASES_KEY] = failed_sub_cases - def pytest_sessionfinish(self, session: Session, exitstatus: int) -> None: if exitstatus != 0: if exitstatus == ExitCode.NO_TESTS_COLLECTED: session.exitstatus = 0 - elif self.known_failure_cases and not self.failed_cases: - session.exitstatus = 0 - - def pytest_terminal_summary(self, terminalreporter: TerminalReporter) -> None: - if self.known_failure_cases: - terminalreporter.section('Known failure cases', bold=True, yellow=True) - terminalreporter.line('\n'.join(self.known_failure_cases)) - - if self.xfail_cases: - terminalreporter.section('xfail cases', bold=True, yellow=True) - terminalreporter.line('\n'.join(self.xfail_cases)) - - if self.failed_cases: - terminalreporter.section('Failed cases', bold=True, red=True) - terminalreporter.line('\n'.join(self.failed_cases)) diff --git a/tools/requirements/requirements.pytest.txt b/tools/requirements/requirements.pytest.txt index 11b1ad1729..09dcfd6246 100644 --- a/tools/requirements/requirements.pytest.txt +++ b/tools/requirements/requirements.pytest.txt @@ -7,6 +7,7 @@ pytest-embedded-jtag pytest-embedded-qemu pytest-rerunfailures pytest-timeout +pytest-ignore-test-results # build idf-build-apps