Merge branch 'feat/elf_unit_test_parser' into 'master'

ci: extract ElfUnitTestParser allowing resolve elf offline

See merge request espressif/esp-idf!18205

tools/unit-test-app/README.md

@@ -13,6 +13,7 @@ ESP-IDF unit tests are run using Unit Test App. The app can be built with the un
 * `idf.py -T <component> -T <component> ... build` with `component` set to names of the components to be included in the test app. Or `idf.py -T all build` to build the test app with all the tests for components having `test` subdirectory.
 * Follow the printed instructions to flash, or run `idf.py -p PORT flash`.
 * Unit test have a few preset sdkconfigs. It provides command `idf.py ut-clean-config_name` and `idf.py ut-build-config_name` (where `config_name` is the file name under `unit-test-app/configs` folder) to build with preset configs. For example, you can use `idf.py -T all ut-build-default` to build with config file `unit-test-app/configs/default`. Built binary for this config will be copied to `unit-test-app/output/config_name` folder.
+* You may extract the test cases presented in the built elf file by calling `ElfUnitTestParser.py <your_elf>`.
 
 # Flash Size
 
@@ -38,7 +39,7 @@ Unit test uses 3 stages in CI: `build`, `assign_test`, `unit_test`.
 
 ### Build Stage:
 
-`build_esp_idf_tests` job will build all UT configs and parse test cases form built elf files. Built binary (`tools/unit-test-app/output`) and parsed cases (`components/idf_test/unit_test/TestCaseAll.yml`) will be saved as artifacts.
+`build_esp_idf_tests` job will build all UT configs and run script `UnitTestParser.py` to parse test cases form built elf files. Built binary (`tools/unit-test-app/output`) and parsed cases (`components/idf_test/unit_test/TestCaseAll.yml`) will be saved as artifacts.
 
 When we add new test case, it will construct a structure to save case data during build. We'll parse the test case from this structure. The description (defined in test case: `TEST_CASE("name", "description")`) is used to extend test case definition. The format of test description is a list of tags:
 
@@ -117,6 +118,7 @@ If you want to reproduce locally, you need to:
         * You can refer to [unit test document](https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/unit-tests.html#running-unit-tests) to run test manually.
         * Or, you can use `tools/unit-test-app/unit_test.py` to run the test cases (see below)
 
+# Testing and debugging on local machine
 ## Running unit tests on local machine by `unit_test.py`
 
 First, install Python dependencies and export the Python path where the IDF CI Python modules are found:

tools/unit-test-app/tools/ElfUnitTestParser.py

@@ -0,0 +1,83 @@
+# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+import argparse
+import os
+import subprocess
+import sys
+from typing import Dict, List
+
+import yaml
+
+try:
+    import CreateSectionTable
+except ImportError:
+    sys.path.append(os.path.expandvars(os.path.join('$IDF_PATH', 'tools', 'unit-test-app', 'tools')))
+    import CreateSectionTable
+
+
+def get_target_objdump(idf_target: str) -> str:
+    toolchain_for_target = {
+        'esp32': 'xtensa-esp32-elf-',
+        'esp32s2': 'xtensa-esp32s2-elf-',
+        'esp32s3': 'xtensa-esp32s3-elf-',
+        'esp32c2': 'riscv32-esp-elf-',
+        'esp32c3': 'riscv32-esp-elf-',
+    }
+    return toolchain_for_target.get(idf_target, '') + 'objdump'
+
+
+def parse_elf_test_cases(elf_file: str, idf_target: str) -> List[Dict]:
+    objdump = get_target_objdump(idf_target)
+
+    try:
+        subprocess.check_output('{} -s {} > section_table.tmp'.format(objdump, elf_file), shell=True)
+        table = CreateSectionTable.SectionTable('section_table.tmp')
+    except subprocess.CalledProcessError:
+        raise Exception('Can\'t resolve elf file. File not found.')
+    finally:
+        os.remove('section_table.tmp')
+
+    bin_test_cases = []
+    try:
+        subprocess.check_output('{} -t {} | grep test_desc > case_address.tmp'.format(objdump, elf_file),
+                                shell=True)
+
+        with open('case_address.tmp', 'rb') as input_f:
+            for line in input_f:
+                # process symbol table like: "3ffb4310 l     O .dram0.data	00000018 test_desc_33$5010"
+                sections = line.split()
+                test_addr = int(sections[0], 16)
+                section = sections[3]
+
+                name_addr = table.get_unsigned_int(section, test_addr, 4)
+                desc_addr = table.get_unsigned_int(section, test_addr + 4, 4)
+                tc = {
+                    'name': table.get_string('any', name_addr),
+                    'desc': table.get_string('any', desc_addr),
+                    'function_count': table.get_unsigned_int(section, test_addr + 20, 4),
+                }
+                bin_test_cases.append(tc)
+    except subprocess.CalledProcessError:
+        raise Exception('Test cases not found')
+    finally:
+        os.remove('case_address.tmp')
+
+    return bin_test_cases
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('elf_file', help='Elf file to parse')
+    parser.add_argument('-t', '--idf_target',
+                        type=str, default=os.environ.get('IDF_TARGET', ''),
+                        help='Target of the elf, e.g. esp32s2')
+    parser.add_argument('-o', '--output_file',
+                        type=str, default='elf_test_cases.yml',
+                        help='Target of the elf, e.g. esp32s2')
+    args = parser.parse_args()
+
+    assert args.idf_target
+
+    test_cases = parse_elf_test_cases(args.elf_file, args.idf_target)
+    with open(args.output_file, 'w') as out_file:
+        yaml.dump(test_cases, out_file, default_flow_style=False)

tools/unit-test-app/tools/UnitTestParser.py

@@ -4,10 +4,9 @@ import argparse
 import os
 import re
 import shutil
-import subprocess
+import sys
 from copy import deepcopy
 
-import CreateSectionTable
 import yaml
 
 try:
@@ -15,6 +14,13 @@ try:
 except ImportError:
     from yaml import Loader as Loader  # type: ignore
 
+try:
+    from ElfUnitTestParser import parse_elf_test_cases
+except ImportError:
+    sys.path.append(os.path.expandvars(os.path.join('$IDF_PATH', 'tools', 'unit-test-app', 'tools')))
+    from ElfUnitTestParser import parse_elf_test_cases
+
+
 TEST_CASE_PATTERN = {
     'initial condition': 'UTINIT1',
     'chip_target': 'esp32',
@@ -50,12 +56,6 @@ class Parser(object):
     ELF_FILE = 'unit-test-app.elf'
     SDKCONFIG_FILE = 'sdkconfig'
     STRIP_CONFIG_PATTERN = re.compile(r'(.+?)(_\d+)?$')
-    TOOLCHAIN_FOR_TARGET = {
-        'esp32': 'xtensa-esp32-elf-',
-        'esp32s2': 'xtensa-esp32s2-elf-',
-        'esp32s3': 'xtensa-esp32s3-elf-',
-        'esp32c3': 'riscv32-esp-elf-',
-    }
 
     def __init__(self, binary_folder, node_index):
         idf_path = os.getenv('IDF_PATH')
@@ -67,7 +67,6 @@ class Parser(object):
         self.idf_target = idf_target
         self.node_index = node_index
         self.ut_bin_folder = binary_folder
-        self.objdump = Parser.TOOLCHAIN_FOR_TARGET.get(idf_target, '') + 'objdump'
         self.tag_def = yaml.load(open(os.path.join(idf_path, self.TAG_DEF_FILE), 'r'), Loader=Loader)
         self.module_map = yaml.load(open(os.path.join(idf_path, self.MODULE_DEF_FILE), 'r'), Loader=Loader)
         self.config_dependencies = yaml.load(open(os.path.join(idf_path, self.CONFIG_DEPENDENCY_FILE), 'r'),
@@ -89,62 +88,43 @@ class Parser(object):
         test_groups = self.get_test_groups(os.path.join(configs_folder, config_name))
 
         elf_file = os.path.join(config_output_folder, self.ELF_FILE)
-        subprocess.check_output('{} -t {} | grep test_desc > case_address.tmp'.format(self.objdump, elf_file),
+        bin_test_cases = parse_elf_test_cases(elf_file, self.idf_target)
-                                shell=True)
-        subprocess.check_output('{} -s {} > section_table.tmp'.format(self.objdump, elf_file), shell=True)
 
-        table = CreateSectionTable.SectionTable('section_table.tmp')
         test_cases = []
+        for bin_tc in bin_test_cases:
+            # we could split cases of same config into multiple binaries as we have limited rom space
+            # we should regard those configs like `default` and `default_2` as the same config
+            match = self.STRIP_CONFIG_PATTERN.match(config_name)
+            stripped_config_name = match.group(1)
 
-        # we could split cases of same config into multiple binaries as we have limited rom space
+            tc = self.parse_one_test_case(bin_tc['name'], bin_tc['desc'], config_name, stripped_config_name, tags)
-        # we should regard those configs like `default` and `default_2` as the same config
-        match = self.STRIP_CONFIG_PATTERN.match(config_name)
-        stripped_config_name = match.group(1)
 
-        with open('case_address.tmp', 'rb') as f:
+            # check if duplicated case names
-            for line in f:
+            # we need to use it to select case,
-                # process symbol table like: "3ffb4310 l     O .dram0.data	00000018 test_desc_33$5010"
+            # if duplicated IDs, Unity could select incorrect case to run
-                line = line.split()
+            # and we need to check all cases no matter if it's going te be executed by CI
-                test_addr = int(line[0], 16)
+            # also add app_name here, we allow same case for different apps
-                section = line[3]
+            if (tc['summary'] + stripped_config_name) in self.test_case_names:
+                self.parsing_errors.append('{} ({}): duplicated test case ID: {}'.format(stripped_config_name, config_name, tc['summary']))
+            else:
+                self.test_case_names.add(tc['summary'] + stripped_config_name)
 
-                name_addr = table.get_unsigned_int(section, test_addr, 4)
+            test_group_included = True
-                desc_addr = table.get_unsigned_int(section, test_addr + 4, 4)
+            if test_groups is not None and tc['group'] not in test_groups:
-                function_count = table.get_unsigned_int(section, test_addr + 20, 4)
+                test_group_included = False
-                name = table.get_string('any', name_addr)
-                desc = table.get_string('any', desc_addr)
 
-                tc = self.parse_one_test_case(name, desc, config_name, stripped_config_name, tags)
+            if tc['CI ready'] == 'Yes' and test_group_included:
-
+                # update test env list and the cases of same env list
-                # check if duplicated case names
+                if tc['test environment'] in self.test_env_tags:
-                # we need to use it to select case,
+                    self.test_env_tags[tc['test environment']].append(tc['ID'])
-                # if duplicated IDs, Unity could select incorrect case to run
-                # and we need to check all cases no matter if it's going te be executed by CI
-                # also add app_name here, we allow same case for different apps
-                if (tc['summary'] + stripped_config_name) in self.test_case_names:
-                    self.parsing_errors.append('{} ({}): duplicated test case ID: {}'.format(stripped_config_name, config_name, tc['summary']))
                 else:
-                    self.test_case_names.add(tc['summary'] + stripped_config_name)
+                    self.test_env_tags.update({tc['test environment']: [tc['ID']]})
 
-                test_group_included = True
+                if bin_tc['function_count'] > 1:
-                if test_groups is not None and tc['group'] not in test_groups:
+                    tc.update({'child case num': bin_tc['function_count']})
-                    test_group_included = False
 
-                if tc['CI ready'] == 'Yes' and test_group_included:
+                # only add  cases need to be executed
-                    # update test env list and the cases of same env list
+                test_cases.append(tc)
-                    if tc['test environment'] in self.test_env_tags:
-                        self.test_env_tags[tc['test environment']].append(tc['ID'])
-                    else:
-                        self.test_env_tags.update({tc['test environment']: [tc['ID']]})
-
-                    if function_count > 1:
-                        tc.update({'child case num': function_count})
-
-                    # only add  cases need to be executed
-                    test_cases.append(tc)
-
-        os.remove('section_table.tmp')
-        os.remove('case_address.tmp')
 
         return test_cases