Merge branch 'bugfix/fix_multi_dut_testcases_report' into 'master'

ci(pytest): Add functionality to merge JUnit files and collect real failure cases...

Closes RDT-495

See merge request espressif/esp-idf!24632

conftest.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2021-2023 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 
 # pylint: disable=W0621  # redefined-outer-name
@@ -22,7 +22,7 @@ import sys
 import xml.etree.ElementTree as ET
 from datetime import datetime
 from fnmatch import fnmatch
-from typing import Callable, List, Optional, Tuple
+from typing import Callable, Dict, List, Optional, Tuple
 
 import pytest
 from _pytest.config import Config, ExitCode
@@ -139,6 +139,8 @@ ENV_MARKERS = {
     'sdio_master_slave': 'Test sdio multi board.',
 }
 
+SUB_JUNIT_FILENAME = 'dut.xml'
+
 
 ##################
 # Help Functions #
@@ -215,6 +217,49 @@ def get_target_marker_from_expr(markexpr: str) -> str:
         raise ValueError('Please specify one target marker via "--target [TARGET]" or via "-m [TARGET]"')
 
 
+def merge_junit_files(junit_files: List[str], target_path: str) -> Optional[ET.Element]:
+    merged_testsuite: ET.Element = ET.Element('testsuite')
+    testcases: Dict[str, ET.Element] = {}
+
+    if len(junit_files) == 0:
+        return None
+
+    if len(junit_files) == 1:
+        return ET.parse(junit_files[0]).getroot()
+
+    for junit in junit_files:
+        logging.info(f'Merging {junit} to {target_path}')
+        tree: ET.ElementTree = ET.parse(junit)
+        testsuite: ET.Element = tree.getroot()
+
+        for testcase in testsuite.findall('testcase'):
+            name: str = testcase.get('name') if testcase.get('name') else ''  # type: ignore
+
+            if name not in testcases:
+                testcases[name] = testcase
+                merged_testsuite.append(testcase)
+                continue
+
+            existing_testcase = testcases[name]
+            for element_name in ['failure', 'error']:
+                for element in testcase.findall(element_name):
+                    existing_element = existing_testcase.find(element_name)
+                    if existing_element is None:
+                        existing_testcase.append(element)
+                    else:
+                        existing_element.attrib.setdefault('message', '')  # type: ignore
+                        existing_element.attrib['message'] += '. ' + element.get('message', '')  # type: ignore
+
+        os.remove(junit)
+
+    merged_testsuite.set('tests', str(len(merged_testsuite.findall('testcase'))))
+    merged_testsuite.set('failures', str(len(merged_testsuite.findall('.//testcase/failure'))))
+    merged_testsuite.set('errors', str(len(merged_testsuite.findall('.//testcase/error'))))
+    merged_testsuite.set('skipped', str(len(merged_testsuite.findall('.//testcase/skipped'))))
+
+    return merged_testsuite
+
+
 ############
 # Fixtures #
 ############
@@ -448,13 +493,13 @@ def pytest_addoption(parser: pytest.Parser) -> None:
         '--app-info-basedir',
         default=IDF_PATH,
         help='app info base directory. specify this value when you\'re building under a '
-        'different IDF_PATH. (Default: $IDF_PATH)',
+             'different IDF_PATH. (Default: $IDF_PATH)',
     )
     idf_group.addoption(
         '--app-info-filepattern',
         help='glob pattern to specify the files that include built app info generated by '
-        '`idf-build-apps --collect-app-info ...`. will not raise ValueError when binary '
-        'paths not exist in local file system if not listed recorded in the app info.',
+             '`idf-build-apps --collect-app-info ...`. will not raise ValueError when binary '
+             'paths not exist in local file system if not listed recorded in the app info.',
     )
 
 
@@ -688,22 +733,23 @@ class IdfPytestEmbedded:
         failed_sub_cases = []
         target = item.funcargs['target']
         config = item.funcargs['config']
-        for junit in junits:
-            xml = ET.parse(junit)
-            testcases = xml.findall('.//testcase')
-            for case in testcases:
-                # modify the junit files
-                new_case_name = format_case_id(target, config, case.attrib['name'])
-                case.attrib['name'] = new_case_name
-                if 'file' in case.attrib:
-                    case.attrib['file'] = case.attrib['file'].replace('/IDF/', '')  # our unity test framework
+        merged_dut_junit_filepath = os.path.join(tempdir, SUB_JUNIT_FILENAME)
+        merged_testsuite = merge_junit_files(junit_files=junits, target_path=merged_dut_junit_filepath)
 
-                # collect real failure cases
-                if case.find('failure') is not None:
-                    failed_sub_cases.append(new_case_name)
+        if merged_testsuite is None:
+            return
 
-            xml.write(junit)
+        for testcase in merged_testsuite.findall('testcase'):
+            new_case_name: str = format_case_id(target, config, testcase.attrib['name'])
+            testcase.attrib['name'] = new_case_name
+            if 'file' in testcase.attrib:
+                testcase.attrib['file'] = testcase.attrib['file'].replace('/IDF/', '')  # Our unity test framework
+            # Collect real failure cases
+            if testcase.find('failure') is not None:
+                failed_sub_cases.append(new_case_name)
 
+        merged_tree: ET.ElementTree = ET.ElementTree(merged_testsuite)
+        merged_tree.write(merged_dut_junit_filepath)
         item.stash[_item_failed_cases_key] = failed_sub_cases
 
     def pytest_sessionfinish(self, session: Session, exitstatus: int) -> None: