Merge branch 'ci/migrate_panic_test_to_pytest_embedded' into 'master'

CI: migrate panic test to pytest embedded

See merge request espressif/esp-idf!17119

.gitlab-ci.yml

@@ -73,7 +73,7 @@ variables:
   TEST_ENV_CONFIG_REPO: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/ci-test-runner-configs.git"
   CI_AUTO_TEST_SCRIPT_REPO_URL: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/auto_test_script.git"
   CI_AUTO_TEST_SCRIPT_REPO_BRANCH: "ci/v4.1"
-  PYTEST_EMBEDDED_TAG: "v0.5.1"
+  PYTEST_EMBEDDED_TAG: "v0.6.0rc0"
 
   # cache python dependencies
   PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"

.gitlab/ci/build.yml

@@ -93,6 +93,20 @@ build_pytest_components_esp32c3:
   script:
     - run_cmd python tools/ci/build_pytest_apps.py components --target esp32c3 --size-info $SIZE_INFO_LOCATION -vv
 
+build_pytest_test_apps_esp32:
+  extends:
+    - .build_pytest_template
+    - .rules:build:custom_test-esp32
+  script:
+    - run_cmd python tools/ci/build_pytest_apps.py tools/test_apps --target esp32 --size-info $SIZE_INFO_LOCATION -vv
+
+build_pytest_test_apps_esp32s2:
+  extends:
+    - .build_pytest_template
+    - .rules:build:custom_test-esp32s2
+  script:
+    - run_cmd python tools/ci/build_pytest_apps.py tools/test_apps --target esp32s2 --size-info $SIZE_INFO_LOCATION -vv
+
 build_non_test_components_apps:
   extends:
     - .build_template

.gitlab/ci/dependencies/dependencies.yml

@@ -63,6 +63,7 @@
   patterns:
     - build_components
     - build_system
+    - build_target_test
   included_in:
     - "build:{0}"
     - build:target_test

.gitlab/ci/rules.yml

@@ -26,12 +26,6 @@
   - "tools/ci/python_packages/tiny_test_fw/**/*"
   - "tools/ci/python_packages/ttfw_idf/**/*"
 
-  - "tools/ci/find_apps_build_apps.sh"
-  - "tools/ci/build_pytest_apps.py"
-  - "tools/build_apps.py"
-  - "tools/find_apps.py"
-  - "tools/find_build_apps/**/*"
-
   - "tools/esp_prov/**/*"
   - "examples/**/*"
 
@@ -43,6 +37,14 @@
   - "components/**/*"
   - "examples/cxx/experimental/experimental_cpp_component/*"
 
+.patterns-build_target_test: &patterns-build_target_test
+  - "tools/ci/find_apps_build_apps.sh"
+  - "tools/build_apps.py"
+  - "tools/find_apps.py"
+  - "tools/find_build_apps/**/*"
+
+  - "tools/ci/build_pytest_apps.py"
+
 .patterns-build_system: &patterns-build_system
   - "tools/cmake/**/*"
   - "tools/kconfig_new/**/*"
@@ -437,6 +439,8 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-component_ut
 
@@ -455,6 +459,8 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-component_ut
 
@@ -473,6 +479,8 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-component_ut
 
@@ -491,6 +499,8 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-component_ut
 
@@ -509,6 +519,8 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-component_ut
 
@@ -527,6 +539,8 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-component_ut
 
@@ -545,6 +559,8 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-component_ut
 
@@ -567,6 +583,8 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-custom_test
 
@@ -584,6 +602,8 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-custom_test
 
@@ -600,6 +620,8 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-custom_test
 
@@ -616,6 +638,8 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-custom_test
 
@@ -632,6 +656,8 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-custom_test
 
@@ -648,6 +674,8 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-custom_test
 
@@ -664,6 +692,8 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-custom_test
 
@@ -698,6 +728,8 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-example_test
 
@@ -717,6 +749,8 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-example_test
 
@@ -735,6 +769,8 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-example_test
 
@@ -753,6 +789,8 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-example_test
 
@@ -771,6 +809,8 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-example_test
 
@@ -789,6 +829,8 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-example_test
 
@@ -807,6 +849,8 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-example_test
 
@@ -880,6 +924,8 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-component_ut
     - <<: *if-dev-push
@@ -909,6 +955,8 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-unit_test
 
@@ -925,6 +973,8 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-unit_test
 
@@ -941,6 +991,8 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-unit_test
 
@@ -957,6 +1009,8 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-unit_test
 
@@ -973,6 +1027,8 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-unit_test
 
@@ -989,6 +1045,8 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-unit_test
 
@@ -1005,6 +1063,8 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-unit_test
 

.gitlab/ci/target-test.yml

@@ -167,6 +167,39 @@ component_ut_pytest_esp32c3_generic:
     - ESP32C3
     - COMPONENT_UT_GENERIC
 
+.pytest_test_apps_dir_template:
+  extends: .pytest_template
+  variables:
+    TEST_DIR: tools/test_apps
+
+test_app_test_pytest_esp32_generic:
+  extends:
+    - .pytest_test_apps_dir_template
+    - .rules:test:custom_test-esp32
+  needs:
+    - build_pytest_test_apps_esp32
+  variables:
+    TARGET: esp32
+    ENV_MARKER: generic
+    SETUP_TOOLS: "1"  # need gdb
+  tags:
+    - ESP32
+    - Example_GENERIC
+
+test_app_test_pytest_esp32s2_generic:
+  extends:
+    - .pytest_test_apps_dir_template
+    - .rules:test:custom_test-esp32s2
+  needs:
+    - build_pytest_test_apps_esp32s2
+  variables:
+    TARGET: esp32s2
+    ENV_MARKER: generic
+    SETUP_TOOLS: "1"  # need gdb
+  tags:
+    - ESP32S2
+    - Example_GENERIC
+
 # for parallel jobs, CI_JOB_NAME will be "job_name index/total" (for example, "IT_001 1/2")
 # we need to convert to pattern "job_name_index.yml"
 .define_config_file_name: &define_config_file_name |
@@ -538,12 +571,9 @@ test_app_test_005:
 
 test_app_test_esp32_generic:
   extends: .test_app_esp32_template
-  parallel: 5
   tags:
     - ESP32
     - Example_GENERIC
-  variables:
-    SETUP_TOOLS: "1"
 
 test_app_test_flash_psram_f4r4:
   extends: .test_app_esp32s3_template

conftest.py

@@ -29,6 +29,7 @@ from pytest_embedded.utils import find_by_suffix
 
 SUPPORTED_TARGETS = ['esp32', 'esp32s2', 'esp32c3', 'esp32s3']
 PREVIEW_TARGETS = ['linux', 'esp32h2', 'esp32c2']
+DEFAULT_SDKCONFIG = 'default'
 
 
 ##################
@@ -57,7 +58,12 @@ def item_marker_names(item: Item) -> List[str]:
 ############
 @pytest.fixture
 def config(request: FixtureRequest) -> str:
-    return getattr(request, 'param', None) or request.config.getoption('config', 'default')  # type: ignore
+    return getattr(request, 'param', None) or DEFAULT_SDKCONFIG
+
+
+@pytest.fixture
+def test_func_name(request: FixtureRequest) -> str:
+    return request.node.function.__name__  # type: ignore
 
 
 @pytest.fixture
@@ -67,7 +73,9 @@ def test_case_name(request: FixtureRequest, target: str, config: str) -> str:
 
 @pytest.fixture
 @parse_configuration
-def build_dir(request: FixtureRequest, app_path: str, target: Optional[str], config: Optional[str]) -> str:
+def build_dir(
+    request: FixtureRequest, app_path: str, target: Optional[str], config: Optional[str]
+) -> str:
     """
     Check local build dir with the following priority:
 
@@ -85,8 +93,10 @@ def build_dir(request: FixtureRequest, app_path: str, target: Optional[str], con
     Returns:
         valid build directory
     """
-    param_or_cli: str = getattr(request, 'param', None) or request.config.option.__dict__.get('build_dir')
+    param_or_cli: str = getattr(
-    if param_or_cli is not None:  # respect the parametrize and the cli
+        request, 'param', None
+    ) or request.config.option.__dict__.get('build_dir')
+    if param_or_cli is not None:  # respect the param and the cli
         return param_or_cli
 
     check_dirs = []
@@ -104,16 +114,21 @@ def build_dir(request: FixtureRequest, app_path: str, target: Optional[str], con
             logging.info(f'find valid binary path: {binary_path}')
             return check_dir
 
-        logging.warning(f'checking binary path: {binary_path}... missing... try another place')
+        logging.warning(
+            'checking binary path: %s... missing... try another place', binary_path
+        )
 
     recommend_place = check_dirs[0]
     logging.error(
-        f'no build dir valid. Please build the binary via "idf.py -B {recommend_place} build" and run pytest again')
+        f'no build dir valid. Please build the binary via "idf.py -B {recommend_place} build" and run pytest again'
+    )
     sys.exit(1)
 
 
 @pytest.fixture(autouse=True)
-def junit_properties(test_case_name: str, record_xml_attribute: Callable[[str, object], None]) -> None:
+def junit_properties(
+    test_case_name: str, record_xml_attribute: Callable[[str, object], None]
+) -> None:
     """
     This fixture is autoused and will modify the junit report test case name to <target>.<config>.<case_name>
     """
@@ -123,12 +138,30 @@ def junit_properties(test_case_name: str, record_xml_attribute: Callable[[str, o
 ##################
 # Hook functions #
 ##################
+def pytest_addoption(parser: pytest.Parser) -> None:
+    base_group = parser.getgroup('idf')
+    base_group.addoption(
+        '--sdkconfig',
+        help='sdkconfig postfix, like sdkconfig.ci.<config>. (Default: None, which would build all found apps)',
+    )
+
+
 @pytest.hookimpl(tryfirst=True)
-def pytest_collection_modifyitems(config: Config, items: List[Item]) -> None:
+def pytest_collection_modifyitems(config: Config, items: List[Function]) -> None:
     target = config.getoption('target', None)  # use the `build` dir
     if not target:
         return
 
+    # sort by file path and callspec.config
+    # implement like this since this is a limitation of pytest, couldn't get fixture values while collecting
+    # https://github.com/pytest-dev/pytest/discussions/9689
+    def _get_param_config(_item: Function) -> str:
+        if hasattr(_item, 'callspec'):
+            return _item.callspec.params.get('config', DEFAULT_SDKCONFIG)  # type: ignore
+        return DEFAULT_SDKCONFIG
+
+    items.sort(key=lambda x: (os.path.dirname(x.path), _get_param_config(x)))
+
     # add markers for special markers
     for item in items:
         if 'supported_targets' in item_marker_names(item):
@@ -144,6 +177,14 @@ def pytest_collection_modifyitems(config: Config, items: List[Item]) -> None:
     # filter all the test cases with "--target"
     items[:] = [item for item in items if target in item_marker_names(item)]
 
+    # filter all the test cases with cli option "config"
+    if config.getoption('sdkconfig'):
+        items[:] = [
+            item
+            for item in items
+            if _get_param_config(item) == config.getoption('sdkconfig')
+        ]
+
 
 @pytest.hookimpl(trylast=True)
 def pytest_runtest_teardown(item: Function) -> None:
@@ -166,5 +207,7 @@ def pytest_runtest_teardown(item: Function) -> None:
         for case in testcases:
             case.attrib['name'] = format_case_id(target, config, case.attrib['name'])
             if 'file' in case.attrib:
-                case.attrib['file'] = case.attrib['file'].replace('/IDF/', '')  # our unity test framework
+                case.attrib['file'] = case.attrib['file'].replace(
+                    '/IDF/', ''
+                )  # our unity test framework
         xml.write(junit)

tools/ci/build_pytest_apps.py

@@ -6,6 +6,7 @@ This file is used to generate binary files for the given path.
 """
 
 import argparse
+import copy
 import logging
 import os
 import sys
@@ -58,17 +59,25 @@ def main(args: argparse.Namespace) -> None:
             build_system='cmake',
             config_rules=config_rules,
         )
-    logging.info(f'Found {len(build_items)} builds')
-    build_items.sort(key=lambda x: x.build_path)  # type: ignore
 
+    modified_build_items = []
     # auto clean up the binaries if no flag --preserve-all
-    if args.preserve_all is False:
+    for item in build_items:
-        for item in build_items:
+        is_test_related = item.config_name in app_configs[item.app_dir]
-            if item.config_name not in app_configs[item.app_dir]:
+        if args.test_only and not is_test_related:
-                item.preserve = False
+            logging.info(f'Skipping non-test app: {item}')
+            continue
+
+        copied_item = copy.deepcopy(item)
+        if not args.preserve_all and not is_test_related:
+            copied_item.preserve = False
+        modified_build_items.append(copied_item)
+
+    logging.info(f'Found {len(modified_build_items)} builds')
+    modified_build_items.sort(key=lambda x: x.build_path)  # type: ignore
 
     build_apps(
-        build_items=build_items,
+        build_items=modified_build_items,
         parallel_count=args.parallel_count,
         parallel_index=args.parallel_index,
         dry_run=False,
@@ -128,7 +137,12 @@ if __name__ == '__main__':
     parser.add_argument(
         '--preserve-all',
         action='store_true',
-        help='add this flag to preserve the binaries for all apps',
+        help='Preserve the binaries for all apps when specified.',
+    )
+    parser.add_argument(
+        '--test-only',
+        action='store_true',
+        help='Build only test related app when specified.',
     )
     arguments = parser.parse_args()
     setup_logging(arguments)

tools/ci/check_copyright_config.yaml

@@ -52,6 +52,7 @@ examples_and_unit_tests:
    - 'examples/'
    - 'components/**/test/**'
    - 'components/**/test_apps/**'
+   - 'tools/test_apps/**'
   allowed_licenses:
   - Apache-2.0
   - Unlicense

tools/test_apps/system/panic/app_test.py

@@ -1,345 +0,0 @@
-#!/usr/bin/env python
-import sys
-
-import panic_tests as test
-from test_panic_util.test_panic_util import panic_test, run_all
-
-
-# test_task_wdt
-@panic_test(target=['ESP32', 'ESP32S2'])
-def test_panic_task_wdt(env, _extra_data):
-    test.task_wdt_inner(env, 'panic')
-
-
-@panic_test()
-def test_coredump_task_wdt_uart_elf_crc(env, _extra_data):
-    test.task_wdt_inner(env, 'coredump_uart_elf_crc')
-
-
-@panic_test()
-def test_coredump_task_wdt_uart_bin_crc(env, _extra_data):
-    test.task_wdt_inner(env, 'coredump_uart_bin_crc')
-
-
-@panic_test()
-def test_coredump_task_wdt_flash_elf_sha(env, _extra_data):
-    test.task_wdt_inner(env, 'coredump_flash_elf_sha')
-
-
-@panic_test()
-def test_coredump_task_wdt_flash_bin_crc(env, _extra_data):
-    test.task_wdt_inner(env, 'coredump_flash_bin_crc')
-
-
-@panic_test()
-def test_gdbstub_task_wdt(env, _extra_data):
-    test.task_wdt_inner(env, 'gdbstub')
-
-
-# test_int_wdt
-
-@panic_test()
-def test_panic_int_wdt(env, _extra_data):
-    test.int_wdt_inner(env, 'panic')
-
-
-@panic_test()
-def test_coredump_int_wdt_uart_elf_crc(env, _extra_data):
-    test.int_wdt_inner(env, 'coredump_uart_elf_crc')
-
-
-@panic_test()
-def test_coredump_int_wdt_uart_bin_crc(env, _extra_data):
-    test.int_wdt_inner(env, 'coredump_uart_bin_crc')
-
-
-@panic_test()
-def test_coredump_int_wdt_flash_elf_sha(env, _extra_data):
-    test.int_wdt_inner(env, 'coredump_flash_elf_sha')
-
-
-@panic_test()
-def test_coredump_int_wdt_flash_bin_crc(env, _extra_data):
-    test.int_wdt_inner(env, 'coredump_flash_bin_crc')
-
-
-@panic_test()
-def test_gdbstub_int_wdt(env, _extra_data):
-    test.int_wdt_inner(env, 'gdbstub')
-
-
-# test_int_wdt_cache_disabled
-
-@panic_test()
-def test_panic_int_wdt_cache_disabled(env, _extra_data):
-    test.int_wdt_cache_disabled_inner(env, 'panic')
-
-
-@panic_test()
-def test_coredump_int_wdt_cache_disabled_uart_elf_crc(env, _extra_data):
-    test.int_wdt_cache_disabled_inner(env, 'coredump_uart_elf_crc')
-
-
-@panic_test()
-def test_coredump_int_wdt_cache_disabled_uart_bin_crc(env, _extra_data):
-    test.int_wdt_cache_disabled_inner(env, 'coredump_uart_bin_crc')
-
-
-@panic_test()
-def test_coredump_int_wdt_cache_disabled_flash_elf_sha(env, _extra_data):
-    test.int_wdt_cache_disabled_inner(env, 'coredump_flash_elf_sha')
-
-
-@panic_test()
-def test_coredump_int_wdt_cache_disabled_flash_bin_crc(env, _extra_data):
-    test.int_wdt_cache_disabled_inner(env, 'coredump_flash_bin_crc')
-
-
-@panic_test()
-def test_gdbstub_int_wdt_cache_disabled(env, _extra_data):
-    test.int_wdt_cache_disabled_inner(env, 'gdbstub')
-
-
-# test_cache_error
-
-@panic_test()
-def test_panic_cache_error(env, _extra_data):
-    test.cache_error_inner(env, 'panic')
-
-
-@panic_test()
-def test_coredump_cache_error_uart_elf_crc(env, _extra_data):
-    test.cache_error_inner(env, 'coredump_uart_elf_crc')
-
-
-@panic_test()
-def test_coredump_cache_error_uart_bin_crc(env, _extra_data):
-    test.cache_error_inner(env, 'coredump_uart_bin_crc')
-
-
-@panic_test()
-def test_coredump_cache_error_flash_elf_sha(env, _extra_data):
-    test.cache_error_inner(env, 'coredump_flash_elf_sha')
-
-
-@panic_test()
-def test_coredump_cache_error_flash_bin_crc(env, _extra_data):
-    test.cache_error_inner(env, 'coredump_flash_bin_crc')
-
-
-@panic_test()
-def test_gdbstub_cache_error(env, _extra_data):
-    test.cache_error_inner(env, 'gdbstub')
-
-
-# test_stack_overflow
-
-@panic_test(target=['ESP32', 'ESP32S2'])
-def test_panic_stack_overflow(env, _extra_data):
-    test.stack_overflow_inner(env, 'panic')
-
-
-@panic_test()
-def test_coredump_stack_overflow_uart_elf_crc(env, _extra_data):
-    test.stack_overflow_inner(env, 'coredump_uart_elf_crc')
-
-
-@panic_test()
-def test_coredump_stack_overflow_uart_bin_crc(env, _extra_data):
-    test.stack_overflow_inner(env, 'coredump_uart_bin_crc')
-
-
-@panic_test()
-def test_coredump_stack_overflow_flash_elf_sha(env, _extra_data):
-    test.stack_overflow_inner(env, 'coredump_flash_elf_sha')
-
-
-@panic_test()
-def test_coredump_stack_overflow_flash_bin_crc(env, _extra_data):
-    test.stack_overflow_inner(env, 'coredump_flash_bin_crc')
-
-
-@panic_test()
-def test_gdbstub_stack_overflow(env, _extra_data):
-    test.stack_overflow_inner(env, 'gdbstub')
-
-
-# test_instr_fetch_prohibited
-
-@panic_test(target=['ESP32', 'ESP32S2'])
-def test_panic_instr_fetch_prohibited(env, _extra_data):
-    test.instr_fetch_prohibited_inner(env, 'panic')
-
-
-@panic_test()
-def test_coredump_instr_fetch_prohibited_uart_elf_crc(env, _extra_data):
-    test.instr_fetch_prohibited_inner(env, 'coredump_uart_elf_crc')
-
-
-@panic_test()
-def test_coredump_instr_fetch_prohibited_uart_bin_crc(env, _extra_data):
-    test.instr_fetch_prohibited_inner(env, 'coredump_uart_bin_crc')
-
-
-@panic_test()
-def test_coredump_instr_fetch_prohibited_flash_elf_sha(env, _extra_data):
-    test.instr_fetch_prohibited_inner(env, 'coredump_flash_elf_sha')
-
-
-@panic_test()
-def test_coredump_instr_fetch_prohibited_flash_bin_crc(env, _extra_data):
-    test.instr_fetch_prohibited_inner(env, 'coredump_flash_bin_crc')
-
-
-@panic_test()
-def test_gdbstub_instr_fetch_prohibited(env, _extra_data):
-    test.instr_fetch_prohibited_inner(env, 'gdbstub')
-
-
-# test_illegal_instruction
-
-@panic_test(target=['ESP32', 'ESP32S2'])
-def test_panic_illegal_instruction(env, _extra_data):
-    test.illegal_instruction_inner(env, 'panic')
-
-
-@panic_test()
-def test_coredump_illegal_instruction_uart_elf_crc(env, _extra_data):
-    test.illegal_instruction_inner(env, 'coredump_uart_elf_crc')
-
-
-@panic_test()
-def test_coredump_illegal_instruction_uart_bin_crc(env, _extra_data):
-    test.illegal_instruction_inner(env, 'coredump_uart_bin_crc')
-
-
-@panic_test()
-def test_coredump_illegal_instruction_flash_elf_sha(env, _extra_data):
-    test.illegal_instruction_inner(env, 'coredump_flash_elf_sha')
-
-
-@panic_test()
-def test_coredump_illegal_instruction_flash_bin_crc(env, _extra_data):
-    test.illegal_instruction_inner(env, 'coredump_flash_bin_crc')
-
-
-@panic_test()
-def test_gdbstub_illegal_instruction(env, _extra_data):
-    test.illegal_instruction_inner(env, 'gdbstub')
-
-
-# test_storeprohibited
-
-@panic_test(target=['ESP32', 'ESP32S2'])
-def test_panic_storeprohibited(env, _extra_data):
-    test.storeprohibited_inner(env, 'panic')
-
-
-@panic_test()
-def test_coredump_storeprohibited_uart_elf_crc(env, _extra_data):
-    test.storeprohibited_inner(env, 'coredump_uart_elf_crc')
-
-
-@panic_test()
-def test_coredump_storeprohibited_uart_bin_crc(env, _extra_data):
-    test.storeprohibited_inner(env, 'coredump_uart_bin_crc')
-
-
-@panic_test()
-def test_coredump_storeprohibited_flash_elf_sha(env, _extra_data):
-    test.storeprohibited_inner(env, 'coredump_flash_elf_sha')
-
-
-@panic_test()
-def test_coredump_storeprohibited_flash_bin_crc(env, _extra_data):
-    test.storeprohibited_inner(env, 'coredump_flash_bin_crc')
-
-
-@panic_test()
-def test_gdbstub_storeprohibited(env, _extra_data):
-    test.storeprohibited_inner(env, 'gdbstub')
-
-
-# test_abort
-
-@panic_test(target=['ESP32', 'ESP32S2'])
-def test_panic_abort(env, _extra_data):
-    test.abort_inner(env, 'panic')
-
-
-@panic_test(target=['ESP32'])
-def test_panic_abort_cache_disabled(env, _extra_data):
-    test.abort_cached_disabled_inner(env, 'panic')
-
-
-@panic_test()
-def test_coredump_abort_uart_elf_crc(env, _extra_data):
-    test.abort_inner(env, 'coredump_uart_elf_crc')
-
-
-@panic_test()
-def test_coredump_abort_uart_bin_crc(env, _extra_data):
-    test.abort_inner(env, 'coredump_uart_bin_crc')
-
-
-@panic_test()
-def test_coredump_abort_flash_elf_sha(env, _extra_data):
-    test.abort_inner(env, 'coredump_flash_elf_sha')
-
-
-@panic_test()
-def test_coredump_abort_flash_bin_crc(env, _extra_data):
-    test.abort_inner(env, 'coredump_flash_bin_crc')
-
-
-@panic_test()
-def test_gdbstub_abort(env, _extra_data):
-    test.abort_inner(env, 'gdbstub')
-
-
-# test_assert
-
-@panic_test(target=['ESP32', 'ESP32S2'])
-def test_panic_assert(env, _extra_data):
-    test.assert_inner(env, 'panic')
-
-
-@panic_test(target=['ESP32'])
-def test_panic_assert_cache_disabled(env, _extra_data):
-    test.assert_cached_disabled_inner(env, 'panic')
-
-
-# test_ub
-
-@panic_test()
-def test_panic_ub(env, _extra_data):
-    test.ub_inner(env, 'panic')
-
-
-@panic_test()
-def test_coredump_ub_uart_elf_crc(env, _extra_data):
-    test.ub_inner(env, 'coredump_uart_elf_crc')
-
-
-@panic_test()
-def test_coredump_ub_uart_bin_crc(env, _extra_data):
-    test.ub_inner(env, 'coredump_uart_bin_crc')
-
-
-@panic_test()
-def test_coredump_ub_flash_elf_sha(env, _extra_data):
-    test.ub_inner(env, 'coredump_flash_elf_sha')
-
-
-@panic_test()
-def test_coredump_ub_flash_bin_crc(env, _extra_data):
-    test.ub_inner(env, 'coredump_flash_bin_crc')
-
-
-@panic_test()
-def test_gdbstub_ub(env, _extra_data):
-    test.ub_inner(env, 'gdbstub')
-
-
-if __name__ == '__main__':
-    run_all(__file__, sys.argv[1:])

tools/test_apps/system/panic/conftest.py

@@ -0,0 +1,279 @@
+# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+
+# pylint: disable=W0621  # redefined-outer-name
+
+import hashlib
+import logging
+import os
+import subprocess
+import sys
+from typing import Any, Dict, List, TextIO
+
+import pexpect
+import pytest
+from _pytest.fixtures import FixtureRequest
+from _pytest.monkeypatch import MonkeyPatch
+from pygdbmi.gdbcontroller import GdbController, GdbTimeoutError, NoGdbProcessError
+from pytest_embedded_idf.app import IdfApp
+from pytest_embedded_idf.dut import IdfDut
+from pytest_embedded_idf.serial import IdfSerial
+
+
+def sha256(file: str) -> str:
+    res = hashlib.sha256()
+    with open(file, 'rb') as fr:
+        res.update(fr.read())
+    return res.hexdigest()
+
+
+class PanicTestDut(IdfDut):
+    BOOT_CMD_ADDR = 0x9000
+    BOOT_CMD_SIZE = 0x1000
+    DEFAULT_EXPECT_TIMEOUT = 10
+    COREDUMP_UART_START = '================= CORE DUMP START ================='
+    COREDUMP_UART_END = '================= CORE DUMP END ================='
+
+    app: IdfApp
+    serial: IdfSerial
+
+    def __init__(self, *args, **kwargs) -> None:  # type: ignore
+        super().__init__(*args, **kwargs)
+
+        self.gdb: GdbController = None  # type: ignore
+        # record this since pygdbmi is using logging.debug to generate some single character mess
+        self.log_level = logging.getLogger().level
+        # pygdbmi is using logging.debug to generate some single character mess
+        if self.log_level <= logging.DEBUG:
+            logging.getLogger().setLevel(logging.INFO)
+
+        self.coredump_output: TextIO = None  # type: ignore
+
+    def close(self) -> None:
+        if self.gdb:
+            self.gdb.exit()
+
+        super().close()
+
+    def revert_log_level(self) -> None:
+        logging.getLogger().setLevel(self.log_level)
+
+    def expect_test_func_name(self, test_func_name: str) -> None:
+        self.expect_exact('Enter test name:')
+        self.write(test_func_name)
+        self.expect_exact('Got test name: ' + test_func_name)
+
+    def expect_none(self, pattern, **kwargs) -> None:  # type: ignore
+        """like dut.expect_all, but with an inverse logic"""
+        if 'timeout' not in kwargs:
+            kwargs['timeout'] = 1
+
+        try:
+            res = self.expect(pattern, **kwargs)
+            raise AssertionError(f'Unexpected: {res.group().decode("utf8")}')
+        except pexpect.TIMEOUT:
+            pass
+
+    def expect_backtrace(self) -> None:
+        self.expect_exact('Backtrace:')
+        self.expect_none('CORRUPTED')
+
+    def expect_gme(self, reason: str) -> None:
+        """Expect method for Guru Meditation Errors"""
+        self.expect_exact(f"Guru Meditation Error: Core  0 panic'ed ({reason})")
+
+    def expect_reg_dump(self, core: int = 0) -> None:
+        """Expect method for the register dump"""
+        self.expect(r'Core\s+%d register dump:' % core)
+
+    def expect_elf_sha256(self) -> None:
+        """Expect method for ELF SHA256 line"""
+        elf_sha256 = sha256(self.app.elf_file)
+        elf_sha256_len = int(
+            self.app.sdkconfig.get('CONFIG_APP_RETRIEVE_LEN_ELF_SHA', '16')
+        )
+        self.expect_exact('ELF file SHA256: ' + elf_sha256[0:elf_sha256_len])
+
+    def _call_espcoredump(
+        self, extra_args: List[str], coredump_file_name: str, output_file_name: str
+    ) -> None:
+        # no "with" here, since we need the file to be open for later inspection by the test case
+        if not self.coredump_output:
+            self.coredump_output = open(output_file_name, 'w')
+
+        espcoredump_script = os.path.join(
+            os.environ['IDF_PATH'], 'components', 'espcoredump', 'espcoredump.py'
+        )
+        espcoredump_args = [
+            sys.executable,
+            espcoredump_script,
+            'info_corefile',
+            '--core',
+            coredump_file_name,
+        ]
+        espcoredump_args += extra_args
+        espcoredump_args.append(self.app.elf_file)
+        logging.info('Running %s', ' '.join(espcoredump_args))
+        logging.info('espcoredump output is written to %s', self.coredump_output.name)
+
+        subprocess.check_call(espcoredump_args, stdout=self.coredump_output)
+        self.coredump_output.flush()
+        self.coredump_output.seek(0)
+
+    def process_coredump_uart(self) -> None:
+        """Extract the core dump from UART output of the test, run espcoredump on it"""
+        self.expect(self.COREDUMP_UART_START)
+        res = self.expect('(.+)' + self.COREDUMP_UART_END)
+        coredump_base64 = res.group(1).decode('utf8')
+        with open(os.path.join(self.logdir, 'coredump_data.b64'), 'w') as coredump_file:
+            logging.info('Writing UART base64 core dump to %s', coredump_file.name)
+            coredump_file.write(coredump_base64)
+
+        output_file_name = os.path.join(self.logdir, 'coredump_uart_result.txt')
+        self._call_espcoredump(
+            ['--core-format', 'b64'], coredump_file.name, output_file_name
+        )
+
+    def process_coredump_flash(self) -> None:
+        """Extract the core dump from flash, run espcoredump on it"""
+        coredump_file_name = os.path.join(self.logdir, 'coredump_data.bin')
+        logging.info('Writing flash binary core dump to %s', coredump_file_name)
+        self.serial.dump_flash(coredump_file_name, partition='coredump')
+
+        output_file_name = os.path.join(self.logdir, 'coredump_flash_result.txt')
+        self._call_espcoredump(
+            ['--core-format', 'raw'], coredump_file_name, output_file_name
+        )
+
+    def gdb_write(self, command: str) -> Any:
+        """
+        Wrapper to write to gdb with a longer timeout, as test runner
+        host can be slow sometimes
+        """
+        return self.gdb.write(command, timeout_sec=10)
+
+    def start_gdb(self) -> None:
+        """
+        Runs GDB and connects it to the "serial" port of the DUT.
+        After this, the DUT expect methods can no longer be used to capture output.
+        """
+        self.gdb = GdbController(gdb_path=self.toolchain_prefix + 'gdb')
+
+        # pygdbmi logs to console by default, make it log to a file instead
+        pygdbmi_log_file_name = os.path.join(self.logdir, 'pygdbmi_log.txt')
+        pygdbmi_logger = self.gdb.logger
+        pygdbmi_logger.setLevel(logging.DEBUG)
+        while pygdbmi_logger.hasHandlers():
+            pygdbmi_logger.removeHandler(pygdbmi_logger.handlers[0])
+        log_handler = logging.FileHandler(pygdbmi_log_file_name)
+        log_handler.setFormatter(
+            logging.Formatter('%(asctime)s %(levelname)s: %(message)s')
+        )
+        pygdbmi_logger.addHandler(log_handler)
+
+        logging.info('Running command: %s', self.gdb.get_subprocess_cmd())
+        for _ in range(10):
+            try:
+                # GdbController creates a process with subprocess.Popen(). Is it really running? It is probable that
+                # an RPI under high load will get non-responsive during creating a lot of processes.
+                resp = self.gdb.get_gdb_response(
+                    timeout_sec=10
+                )  # calls verify_valid_gdb_subprocess() internally
+                # it will be interesting to look up this response if the next GDB command fails (times out)
+                logging.info('GDB response: %s', resp)
+                break  # success
+            except GdbTimeoutError:
+                logging.warning(
+                    'GDB internal error: cannot get response from the subprocess'
+                )
+            except NoGdbProcessError:
+                logging.error('GDB internal error: process is not running')
+                break  # failure - TODO: create another GdbController
+            except ValueError:
+                logging.error(
+                    'GDB internal error: select() returned an unexpected file number'
+                )
+
+        # Set up logging for GDB remote protocol
+        gdb_remotelog_file_name = os.path.join(self.logdir, 'gdb_remote_log.txt')
+        self.gdb_write('-gdb-set remotelogfile ' + gdb_remotelog_file_name)
+
+        # Load the ELF file
+        self.gdb_write('-file-exec-and-symbols {}'.format(self.app.elf_file))
+
+        # Connect GDB to UART
+        self.serial.proc.close()
+        logging.info('Connecting to GDB Stub...')
+        self.gdb_write('-gdb-set serial baud 115200')
+        responses = self.gdb_write('-target-select remote ' + self.serial.port)
+
+        # Make sure we get the 'stopped' notification
+        stop_response = self.find_gdb_response('stopped', 'notify', responses)
+        if not stop_response:
+            responses = self.gdb_write('-exec-interrupt')
+            stop_response = self.find_gdb_response('stopped', 'notify', responses)
+            assert stop_response
+        frame = stop_response['payload']['frame']
+        if 'file' not in frame:
+            frame['file'] = '?'
+        if 'line' not in frame:
+            frame['line'] = '?'
+        logging.info('Stopped in {func} at {addr} ({file}:{line})'.format(**frame))
+
+        # Drain remaining responses
+        self.gdb.get_gdb_response(raise_error_on_timeout=False)
+
+    def gdb_backtrace(self) -> Any:
+        """
+        Returns the list of stack frames for the current thread.
+        Each frame is a dictionary, refer to pygdbmi docs for the format.
+        """
+        assert self.gdb
+
+        responses = self.gdb_write('-stack-list-frames')
+        return self.find_gdb_response('done', 'result', responses)['payload']['stack']
+
+    @staticmethod
+    def match_backtrace(
+        gdb_backtrace: List[Any], expected_functions_list: List[Any]
+    ) -> bool:
+        """
+        Returns True if the function names listed in expected_functions_list match the backtrace
+        given by gdb_backtrace argument. The latter is in the same format as returned by gdb_backtrace()
+        function.
+        """
+        return all(
+            [
+                frame['func'] == expected_functions_list[i]
+                for i, frame in enumerate(gdb_backtrace)
+            ]
+        )
+
+    @staticmethod
+    def find_gdb_response(
+        message: str, response_type: str, responses: List[Any]
+    ) -> Any:
+        """
+        Helper function which extracts one response from an array of GDB responses, filtering
+        by message and type. Returned message is a dictionary, refer to pygdbmi docs for the format.
+        """
+
+        def match_response(response: Dict[str, Any]) -> bool:
+            return response['message'] == message and response['type'] == response_type  # type: ignore
+
+        filtered_responses = [r for r in responses if match_response(r)]
+        if not filtered_responses:
+            return None
+        return filtered_responses[0]
+
+
+@pytest.fixture(scope='module')
+def monkeypatch_module(request: FixtureRequest) -> MonkeyPatch:
+    mp = MonkeyPatch()
+    request.addfinalizer(mp.undo)
+    return mp
+
+
+@pytest.fixture(scope='module', autouse=True)
+def replace_dut_class(monkeypatch_module: MonkeyPatch) -> None:
+    monkeypatch_module.setattr('pytest_embedded_idf.dut.IdfDut', PanicTestDut)

tools/test_apps/system/panic/panic_tests.py

@@ -1,196 +0,0 @@
-#!/usr/bin/env python
-import re
-from pprint import pformat
-
-from test_panic_util.test_panic_util import get_dut
-
-
-def get_default_backtrace(test_name):
-    return [
-        test_name,
-        'app_main',
-        'main_task',
-        'vPortTaskWrapper'
-    ]
-
-
-def test_common(dut, test_name, expected_backtrace=None):
-    if expected_backtrace is None:
-        expected_backtrace = get_default_backtrace(dut.test_name)
-
-    if 'gdbstub' in test_name:
-        dut.expect('Entering gdb stub now.')
-        dut.start_gdb()
-        frames = dut.gdb_backtrace()
-        if not dut.match_backtrace(frames, expected_backtrace):
-            raise AssertionError('Unexpected backtrace in test {}:\n{}'.format(test_name, pformat(frames)))
-        return
-
-    if 'uart' in test_name:
-        dut.expect(dut.COREDUMP_UART_END)
-
-    dut.expect('Rebooting...')
-
-    if 'uart' in test_name:
-        dut.process_coredump_uart()
-        # TODO: check backtrace
-    elif 'flash' in test_name:
-        dut.process_coredump_flash()
-        # TODO: check backtrace
-    elif 'panic' in test_name:
-        # TODO: check backtrace
-        pass
-
-
-def task_wdt_inner(env, test_name):
-    with get_dut(env, test_name, 'test_task_wdt', qemu_wdt_enable=True) as dut:
-        dut.expect('Task watchdog got triggered. The following tasks did not reset the watchdog in time:')
-        dut.expect('CPU 0: main')
-        dut.expect(re.compile(r'abort\(\) was called at PC [0-9xa-f]+ on core 0'))
-        dut.expect_none('register dump:')
-        dut.expect_backtrace()
-        dut.expect_elf_sha256()
-        dut.expect_none('Guru Meditation')
-        if ('gdbstub' in test_name):
-            test_common(dut, test_name, expected_backtrace=[
-                # Backtrace interrupted when abort is called, IDF-842
-                'panic_abort', 'esp_system_abort'
-            ])
-        else:
-            test_common(dut, test_name)
-
-
-def int_wdt_inner(env, test_name):
-    with get_dut(env, test_name, 'test_int_wdt', qemu_wdt_enable=True) as dut:
-        dut.expect_gme('Interrupt wdt timeout on CPU0')
-        dut.expect_reg_dump(0)
-        dut.expect_backtrace()
-        dut.expect_none('Guru Meditation')
-        dut.expect_reg_dump(1)
-        dut.expect_backtrace()
-        dut.expect_elf_sha256()
-        dut.expect_none('Guru Meditation')
-        test_common(dut, test_name)
-
-
-def int_wdt_cache_disabled_inner(env, test_name):
-    with get_dut(env, test_name, 'test_int_wdt_cache_disabled', qemu_wdt_enable=True) as dut:
-        dut.expect_gme('Interrupt wdt timeout on CPU0')
-        dut.expect_reg_dump(0)
-        dut.expect('Backtrace:')
-        dut.expect_none('Guru Meditation')
-        dut.expect_reg_dump(1)
-        dut.expect_backtrace()
-        dut.expect_elf_sha256()
-        dut.expect_none('Guru Meditation')
-        test_common(dut, test_name)
-
-
-def cache_error_inner(env, test_name):
-    with get_dut(env, test_name, 'test_cache_error') as dut:
-        dut.expect_gme('Cache disabled but cached memory region accessed')
-        dut.expect_reg_dump(0)
-        dut.expect_backtrace()
-        dut.expect_elf_sha256()
-        dut.expect_none('Guru Meditation')
-        test_common(dut, test_name,
-                    expected_backtrace=['die'] + get_default_backtrace(dut.test_name))
-
-
-def abort_inner(env, test_name):
-    with get_dut(env, test_name, 'test_abort') as dut:
-        dut.expect(re.compile(r'abort\(\) was called at PC [0-9xa-f]+ on core 0'))
-        dut.expect_backtrace()
-        dut.expect_elf_sha256()
-        dut.expect_none('Guru Meditation', 'Re-entered core dump')
-        if ('gdbstub' in test_name):
-            test_common(dut, test_name, expected_backtrace=[
-                # Backtrace interrupted when abort is called, IDF-842
-                'panic_abort', 'esp_system_abort'
-            ])
-        else:
-            test_common(dut, test_name)
-
-
-def abort_cached_disabled_inner(env, test_name):
-    with get_dut(env, test_name, 'test_abort_cache_disabled') as dut:
-        dut.expect(re.compile(r'abort\(\) was called at PC [0-9xa-f]+ on core 0'))
-        dut.expect_backtrace()
-        dut.expect_elf_sha256()
-        dut.expect_none('Guru Meditation', 'Re-entered core dump')
-        test_common(dut, test_name)
-
-
-def assert_inner(env, test_name):
-    with get_dut(env, test_name, 'test_assert') as dut:
-        dut.expect(re.compile(r'(assert failed:[\s\w\(\)]*?\s[\.\w\/]*\.(?:c|cpp|h|hpp):\d*.*)'))
-        dut.expect_backtrace()
-        dut.expect_elf_sha256()
-        dut.expect_none('Guru Meditation', 'Re-entered core dump')
-        test_common(dut, test_name)
-
-
-def assert_cached_disabled_inner(env, test_name):
-    with get_dut(env, test_name, 'test_assert_cache_disabled') as dut:
-        dut.expect(re.compile(r'(assert failed: [0-9xa-fA-F]+.*)'))
-        dut.expect_backtrace()
-        dut.expect_elf_sha256()
-        dut.expect_none('Guru Meditation', 'Re-entered core dump')
-        test_common(dut, test_name)
-
-
-def storeprohibited_inner(env, test_name):
-    with get_dut(env, test_name, 'test_storeprohibited') as dut:
-        dut.expect_gme('StoreProhibited')
-        dut.expect_reg_dump(0)
-        dut.expect_backtrace()
-        dut.expect_elf_sha256()
-        dut.expect_none('Guru Meditation')
-        test_common(dut, test_name)
-
-
-def stack_overflow_inner(env, test_name):
-    with get_dut(env, test_name, 'test_stack_overflow') as dut:
-        dut.expect_gme('Unhandled debug exception')
-        dut.expect('Stack canary watchpoint triggered (main)')
-        dut.expect_reg_dump(0)
-        dut.expect_backtrace()
-        dut.expect_elf_sha256()
-        dut.expect_none('Guru Meditation')
-        test_common(dut, test_name)
-
-
-def illegal_instruction_inner(env, test_name):
-    with get_dut(env, test_name, 'test_illegal_instruction') as dut:
-        dut.expect_gme('IllegalInstruction')
-        dut.expect_reg_dump(0)
-        dut.expect_backtrace()
-        dut.expect_elf_sha256()
-        dut.expect_none('Guru Meditation')
-        test_common(dut, test_name)
-
-
-def instr_fetch_prohibited_inner(env, test_name):
-    with get_dut(env, test_name, 'test_instr_fetch_prohibited') as dut:
-        dut.expect_gme('InstrFetchProhibited')
-        dut.expect_reg_dump(0)
-        dut.expect_backtrace()
-        dut.expect_elf_sha256()
-        dut.expect_none('Guru Meditation')
-        test_common(dut, test_name,
-                    expected_backtrace=['_init'] + get_default_backtrace(dut.test_name))
-
-
-def ub_inner(env, test_name):
-    with get_dut(env, test_name, 'test_ub') as dut:
-        dut.expect(re.compile(r'Undefined behavior of type out_of_bounds'))
-        dut.expect_backtrace()
-        dut.expect_elf_sha256()
-        dut.expect_none('Guru Meditation', 'Re-entered core dump')
-        if ('gdbstub' in test_name):
-            test_common(dut, test_name, expected_backtrace=[
-                # Backtrace interrupted when abort is called, IDF-842
-                'panic_abort', 'esp_system_abort'
-            ])
-        else:
-            test_common(dut, test_name)

tools/test_apps/system/panic/pytest_panic.py

@@ -0,0 +1,287 @@
+# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: CC0-1.0
+
+import re
+from pprint import pformat
+from typing import List, Optional
+
+import pytest
+
+from conftest import PanicTestDut
+
+CONFIGS = [
+    pytest.param('coredump_flash_bin_crc', marks=[pytest.mark.esp32, pytest.mark.esp32s2]),
+    pytest.param('coredump_flash_elf_sha', marks=[pytest.mark.esp32]),  # sha256 only supported on esp32
+    pytest.param('coredump_uart_bin_crc', marks=[pytest.mark.esp32, pytest.mark.esp32s2]),
+    pytest.param('coredump_uart_elf_crc', marks=[pytest.mark.esp32, pytest.mark.esp32s2]),
+    pytest.param('gdbstub', marks=[pytest.mark.esp32, pytest.mark.esp32s2]),
+    pytest.param('panic', marks=[pytest.mark.esp32, pytest.mark.esp32s2]),
+]
+
+
+def get_default_backtrace(config: str) -> List[str]:
+    return [config, 'app_main', 'main_task', 'vPortTaskWrapper']
+
+
+def common_test(dut: PanicTestDut, config: str, expected_backtrace: Optional[List[str]] = None) -> None:
+    if 'gdbstub' in config:
+        dut.expect_exact('Entering gdb stub now.')
+        dut.start_gdb()
+        frames = dut.gdb_backtrace()
+        if not dut.match_backtrace(frames, expected_backtrace):
+            raise AssertionError(
+                'Unexpected backtrace in test {}:\n{}'.format(config, pformat(frames))
+            )
+        dut.revert_log_level()
+        return
+
+    if 'uart' in config:
+        dut.process_coredump_uart()
+    elif 'flash' in config:
+        dut.process_coredump_flash()
+    elif 'panic' in config:
+        pass
+
+    dut.expect('Rebooting...')
+
+
+@pytest.mark.parametrize('config', CONFIGS, indirect=True)
+@pytest.mark.generic
+def test_task_wdt(dut: PanicTestDut, config: str, test_func_name: str) -> None:
+    dut.expect_test_func_name(test_func_name)
+    dut.expect_exact(
+        'Task watchdog got triggered. The following tasks did not reset the watchdog in time:'
+    )
+    dut.expect_exact('CPU 0: main')
+    dut.expect(r'abort\(\) was called at PC [0-9xa-f]+ on core 0')
+    dut.expect_none('register dump:')
+    dut.expect_backtrace()
+    dut.expect_elf_sha256()
+    dut.expect_none('Guru Meditation')
+
+    if config == 'gdbstub':
+        common_test(
+            dut,
+            config,
+            expected_backtrace=[
+                # Backtrace interrupted when abort is called, IDF-842
+                'panic_abort',
+                'esp_system_abort',
+            ],
+        )
+    else:
+        common_test(dut, config)
+
+
+@pytest.mark.parametrize('config', CONFIGS, indirect=True)
+@pytest.mark.generic
+def test_int_wdt(
+    dut: PanicTestDut, target: str, config: str, test_func_name: str
+) -> None:
+    dut.expect_test_func_name(test_func_name)
+    dut.expect_gme('Interrupt wdt timeout on CPU0')
+    dut.expect_reg_dump(0)
+    dut.expect_backtrace()
+    if target == 'esp32s2':
+        dut.expect_elf_sha256()
+    dut.expect_none('Guru Meditation')
+
+    if target != 'esp32s2':  # esp32s2 is single-core
+        dut.expect_reg_dump(1)
+        dut.expect_backtrace()
+        dut.expect_elf_sha256()
+        dut.expect_none('Guru Meditation')
+
+    common_test(dut, config, expected_backtrace=get_default_backtrace(test_func_name))
+
+
+@pytest.mark.parametrize('config', CONFIGS, indirect=True)
+@pytest.mark.generic
+def test_int_wdt_cache_disabled(
+    dut: PanicTestDut, target: str, config: str, test_func_name: str
+) -> None:
+    dut.expect_test_func_name(test_func_name)
+    dut.expect_gme('Interrupt wdt timeout on CPU0')
+    dut.expect_reg_dump(0)
+    dut.expect_backtrace()
+    if target == 'esp32s2':
+        dut.expect_elf_sha256()
+    dut.expect_none('Guru Meditation')
+
+    if target != 'esp32s2':  # esp32s2 is single-core
+        dut.expect_reg_dump(1)
+        dut.expect_backtrace()
+        dut.expect_elf_sha256()
+        dut.expect_none('Guru Meditation')
+
+    common_test(dut, config, expected_backtrace=get_default_backtrace(test_func_name))
+
+
+@pytest.mark.parametrize('config', CONFIGS, indirect=True)
+@pytest.mark.xfail('config.getvalue("target") == "esp32s2"', reason='raised IllegalInstruction instead')
+@pytest.mark.generic
+def test_cache_error(dut: PanicTestDut, config: str, test_func_name: str) -> None:
+    dut.expect_test_func_name(test_func_name)
+    dut.expect_gme('Cache disabled but cached memory region accessed')
+    dut.expect_reg_dump(0)
+    dut.expect_backtrace()
+    dut.expect_elf_sha256()
+    dut.expect_none('Guru Meditation')
+    common_test(
+        dut, config, expected_backtrace=['die'] + get_default_backtrace(test_func_name)
+    )
+
+
+@pytest.mark.parametrize('config', CONFIGS, indirect=True)
+@pytest.mark.generic
+def test_stack_overflow(dut: PanicTestDut, config: str, test_func_name: str) -> None:
+    dut.expect_test_func_name(test_func_name)
+    dut.expect_gme('Unhandled debug exception')
+    dut.expect_exact('Stack canary watchpoint triggered (main)')
+    dut.expect_reg_dump(0)
+    dut.expect_backtrace()
+    dut.expect_elf_sha256()
+    dut.expect_none('Guru Meditation')
+    common_test(dut, config, expected_backtrace=get_default_backtrace(test_func_name))
+
+
+@pytest.mark.parametrize('config', CONFIGS, indirect=True)
+@pytest.mark.generic
+def test_instr_fetch_prohibited(
+    dut: PanicTestDut, config: str, test_func_name: str
+) -> None:
+    dut.expect_test_func_name(test_func_name)
+    dut.expect_gme('InstrFetchProhibited')
+    dut.expect_reg_dump(0)
+    dut.expect_backtrace()
+    dut.expect_elf_sha256()
+    dut.expect_none('Guru Meditation')
+    common_test(
+        dut,
+        config,
+        expected_backtrace=['_init'] + get_default_backtrace(test_func_name),
+    )
+
+
+@pytest.mark.parametrize('config', CONFIGS, indirect=True)
+@pytest.mark.generic
+def test_illegal_instruction(
+    dut: PanicTestDut, config: str, test_func_name: str
+) -> None:
+    dut.expect_test_func_name(test_func_name)
+    dut.expect_gme('IllegalInstruction')
+    dut.expect_reg_dump(0)
+    dut.expect_backtrace()
+    dut.expect_elf_sha256()
+    dut.expect_none('Guru Meditation')
+    common_test(dut, config, expected_backtrace=get_default_backtrace(test_func_name))
+
+
+@pytest.mark.parametrize('config', CONFIGS, indirect=True)
+@pytest.mark.generic
+def test_storeprohibited(dut: PanicTestDut, config: str, test_func_name: str) -> None:
+    dut.expect_test_func_name(test_func_name)
+    dut.expect_gme('StoreProhibited')
+    dut.expect_reg_dump(0)
+    dut.expect_backtrace()
+    dut.expect_elf_sha256()
+    dut.expect_none('Guru Meditation')
+    common_test(dut, config, expected_backtrace=get_default_backtrace(test_func_name))
+
+
+@pytest.mark.parametrize('config', CONFIGS, indirect=True)
+@pytest.mark.generic
+def test_abort(dut: PanicTestDut, config: str, test_func_name: str) -> None:
+    dut.expect_test_func_name(test_func_name)
+    dut.expect(r'abort\(\) was called at PC [0-9xa-f]+ on core 0')
+    dut.expect_backtrace()
+    dut.expect_elf_sha256()
+    dut.expect_none(['Guru Meditation', 'Re-entered core dump'])
+
+    if config == 'gdbstub':
+        common_test(
+            dut,
+            config,
+            expected_backtrace=[
+                # Backtrace interrupted when abort is called, IDF-842
+                'panic_abort',
+                'esp_system_abort',
+            ],
+        )
+    else:
+        common_test(dut, config)
+
+
+@pytest.mark.parametrize('config', CONFIGS, indirect=True)
+@pytest.mark.generic
+def test_ub(dut: PanicTestDut, config: str, test_func_name: str) -> None:
+    dut.expect_test_func_name(test_func_name)
+    dut.expect('Undefined behavior of type out_of_bounds')
+    dut.expect_backtrace()
+    dut.expect_elf_sha256()
+    dut.expect_none(['Guru Meditation', 'Re-entered core dump'])
+
+    if config == 'gdbstub':
+        common_test(
+            dut,
+            config,
+            expected_backtrace=[
+                # Backtrace interrupted when abort is called, IDF-842
+                'panic_abort',
+                'esp_system_abort',
+            ],
+        )
+    else:
+        common_test(dut, config)
+
+
+#########################
+# for config panic only #
+#########################
+@pytest.mark.esp32
+@pytest.mark.esp32s2
+@pytest.mark.xfail('config.getvalue("target") == "esp32s2"', reason='raised IllegalInstruction instead')
+@pytest.mark.parametrize('config', ['panic'], indirect=True)
+@pytest.mark.generic
+def test_abort_cache_disabled(
+    dut: PanicTestDut, config: str, test_func_name: str
+) -> None:
+    dut.expect_test_func_name(test_func_name)
+    dut.expect(r'abort\(\) was called at PC [0-9xa-f]+ on core 0')
+    dut.expect_backtrace()
+    dut.expect_elf_sha256()
+    dut.expect_none(['Guru Meditation', 'Re-entered core dump'])
+    common_test(dut, config, expected_backtrace=get_default_backtrace(test_func_name))
+
+
+@pytest.mark.esp32
+@pytest.mark.esp32s2
+@pytest.mark.parametrize('config', ['panic'], indirect=True)
+@pytest.mark.generic
+def test_assert(dut: PanicTestDut, config: str, test_func_name: str) -> None:
+    dut.expect_test_func_name(test_func_name)
+    dut.expect(
+        re.compile(
+            rb'assert failed:[\s\w()]*?\s[.\w/]*\.(?:c|cpp|h|hpp):\d.*$', re.MULTILINE
+        )
+    )
+    dut.expect_backtrace()
+    dut.expect_elf_sha256()
+    dut.expect_none(['Guru Meditation', 'Re-entered core dump'])
+    common_test(dut, config, expected_backtrace=get_default_backtrace(test_func_name))
+
+
+@pytest.mark.esp32
+@pytest.mark.esp32s2
+@pytest.mark.xfail('config.getvalue("target") == "esp32s2"', reason='raised IllegalInstruction instead')
+@pytest.mark.parametrize('config', ['panic'], indirect=True)
+@pytest.mark.generic
+def test_assert_cache_disabled(
+    dut: PanicTestDut, config: str, test_func_name: str
+) -> None:
+    dut.expect_test_func_name(test_func_name)
+    dut.expect(re.compile(rb'assert failed: [0-9xa-fA-F]+.*$', re.MULTILINE))
+    dut.expect_backtrace()
+    dut.expect_elf_sha256()
+    dut.expect_none(['Guru Meditation', 'Re-entered core dump'])
+    common_test(dut, config, expected_backtrace=get_default_backtrace(test_func_name))

tools/test_apps/system/panic/test_panic_util/test_panic_util.py

@@ -1,319 +0,0 @@
-import logging
-import os
-import re
-import subprocess
-import sys
-
-import ttfw_idf
-from pygdbmi.gdbcontroller import GdbController, GdbTimeoutError, NoGdbProcessError
-from tiny_test_fw import DUT, TinyFW, Utility
-from tiny_test_fw.Utility import CaseConfig, SearchCases
-
-# hard-coded to the path one level above - only intended to be used from the panic test app
-TEST_PATH = os.path.relpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'), os.getenv('IDF_PATH'))
-TEST_SUITE = 'Panic'
-
-
-def ok(data):
-    """ Helper function used with dut.expect_any """
-    pass
-
-
-def unexpected(data):
-    """ Helper function used with dut.expect_any """
-    raise AssertionError('Unexpected: {}'.format(data))
-
-
-class PanicTestApp(ttfw_idf.TestApp):
-    pass
-
-
-class PanicTestMixin(object):
-    """ Provides custom functionality for the panic test DUT """
-    BOOT_CMD_ADDR = 0x9000
-    BOOT_CMD_SIZE = 0x1000
-    DEFAULT_EXPECT_TIMEOUT = 10
-    COREDUMP_UART_START = '================= CORE DUMP START ================='
-    COREDUMP_UART_END = '================= CORE DUMP END ================='
-
-    def start_test(self, test_name):
-        """ Starts the app and sends it the test name """
-        self.test_name = test_name
-        # Start the app and verify that it has started up correctly
-        self.start_capture_raw_data()
-        self.start_app()
-        self.expect('Enter test name: ')
-        Utility.console_log('Setting boot command: ' + test_name)
-        self.write(test_name)
-        self.expect('Got test name: ' + test_name)
-
-    def expect_none(self, *patterns, **timeout_args):
-        """ like dut.expect_all, but with an inverse logic """
-        found_data = []
-        if 'timeout' not in timeout_args:
-            timeout_args['timeout'] = 1
-
-        def found(data):
-            raise AssertionError('Unexpected: {}'.format(data))
-            found_data.append(data)
-        try:
-            expect_items = [(pattern, found) for pattern in patterns]
-            self.expect_any(*expect_items, **timeout_args)
-            raise AssertionError('Unexpected: {}'.format(found_data))
-        except DUT.ExpectTimeout:
-            return True
-
-    def expect_gme(self, reason):
-        """ Expect method for Guru Meditation Errors """
-        self.expect(r"Guru Meditation Error: Core  0 panic'ed (%s)" % reason)
-
-    def expect_reg_dump(self, core=0):
-        """ Expect method for the register dump """
-        self.expect(re.compile(r'Core\s+%d register dump:' % core))
-
-    def expect_elf_sha256(self):
-        """ Expect method for ELF SHA256 line """
-        elf_sha256 = self.app.get_elf_sha256()
-        sdkconfig = self.app.get_sdkconfig()
-        elf_sha256_len = int(sdkconfig.get('CONFIG_APP_RETRIEVE_LEN_ELF_SHA', '16'))
-        self.expect('ELF file SHA256: ' + elf_sha256[0:elf_sha256_len])
-
-    def expect_backtrace(self):
-        self.expect('Backtrace:')
-        self.expect_none('CORRUPTED')
-
-    def __enter__(self):
-        self._raw_data = None
-        self.gdb = None
-        return self
-
-    def __exit__(self, type, value, traceback):
-        log_folder = self.app.get_log_folder(TEST_SUITE)
-        with open(os.path.join(log_folder, 'log_' + self.test_name + '.txt'), 'w') as log_file:
-            Utility.console_log('Writing output of {} to {}'.format(self.test_name, log_file.name))
-            log_file.write(self.get_raw_data())
-        if self.gdb:
-            self.gdb.exit()
-        self.close()
-
-    def get_raw_data(self):
-        if not self._raw_data:
-            self._raw_data = self.stop_capture_raw_data()
-        return self._raw_data
-
-    def _call_espcoredump(self, extra_args, coredump_file_name, output_file_name):
-        # no "with" here, since we need the file to be open for later inspection by the test case
-        self.coredump_output = open(output_file_name, 'w')
-        espcoredump_script = os.path.join(os.environ['IDF_PATH'], 'components', 'espcoredump', 'espcoredump.py')
-        espcoredump_args = [
-            sys.executable,
-            espcoredump_script,
-            'info_corefile',
-            '--core', coredump_file_name,
-        ]
-        espcoredump_args += extra_args
-        espcoredump_args.append(self.app.elf_file)
-        Utility.console_log('Running ' + ' '.join(espcoredump_args))
-        Utility.console_log('espcoredump output is written to ' + self.coredump_output.name)
-
-        subprocess.check_call(espcoredump_args, stdout=self.coredump_output)
-        self.coredump_output.flush()
-        self.coredump_output.seek(0)
-
-    def process_coredump_uart(self):
-        """ Extract the core dump from UART output of the test, run espcoredump on it """
-        log_folder = self.app.get_log_folder(TEST_SUITE)
-        data = self.get_raw_data()
-        coredump_start = data.find(self.COREDUMP_UART_START)
-        coredump_end = data.find(self.COREDUMP_UART_END)
-        coredump_base64 = data[coredump_start + len(self.COREDUMP_UART_START):coredump_end]
-        with open(os.path.join(log_folder, 'coredump_data_' + self.test_name + '.b64'), 'w') as coredump_file:
-            Utility.console_log('Writing UART base64 core dump to ' + coredump_file.name)
-            coredump_file.write(coredump_base64)
-
-        output_file_name = os.path.join(log_folder, 'coredump_uart_result_' + self.test_name + '.txt')
-        self._call_espcoredump(['--core-format', 'b64'], coredump_file.name, output_file_name)
-
-    def process_coredump_flash(self):
-        """ Extract the core dump from flash, run espcoredump on it """
-        log_folder = self.app.get_log_folder(TEST_SUITE)
-        coredump_file_name = os.path.join(log_folder, 'coredump_data_' + self.test_name + '.bin')
-        Utility.console_log('Writing flash binary core dump to ' + coredump_file_name)
-        self.dump_flash(coredump_file_name, partition='coredump')
-
-        output_file_name = os.path.join(log_folder, 'coredump_flash_result_' + self.test_name + '.txt')
-        self._call_espcoredump(['--core-format', 'raw'], coredump_file_name, output_file_name)
-
-    def _gdb_write(self, command):
-        """
-        Wrapper to write to gdb with a longer timeout, as test runner
-        host can be slow sometimes
-        """
-        return self.gdb.write(command, timeout_sec=10)
-
-    def start_gdb(self):
-        """
-        Runs GDB and connects it to the "serial" port of the DUT.
-        After this, the DUT expect methods can no longer be used to capture output.
-        """
-        self.stop_receive()
-        self._port_close()
-
-        Utility.console_log('Starting GDB...', 'orange')
-        self.gdb = GdbController(gdb_path=self.TOOLCHAIN_PREFIX + 'gdb')
-        Utility.console_log('Running command: {}'.format(self.gdb.get_subprocess_cmd()), 'orange')
-
-        for _ in range(10):
-            try:
-                # GdbController creates a process with subprocess.Popen(). Is it really running? It is probable that
-                # an RPI under high load will get non-responsive during creating a lot of processes.
-                resp = self.gdb.get_gdb_response(timeout_sec=10)  # calls verify_valid_gdb_subprocess() internally
-                # it will be interesting to look up this response if the next GDB command fails (times out)
-                Utility.console_log('GDB response: {}'.format(resp), 'orange')
-                break  # success
-            except GdbTimeoutError:
-                Utility.console_log('GDB internal error: cannot get response from the subprocess', 'orange')
-            except NoGdbProcessError:
-                Utility.console_log('GDB internal error: process is not running', 'red')
-                break  # failure - TODO: create another GdbController
-            except ValueError:
-                Utility.console_log('GDB internal error: select() returned an unexpected file number', 'red')
-
-        # pygdbmi logs to console by default, make it log to a file instead
-        log_folder = self.app.get_log_folder(TEST_SUITE)
-        pygdbmi_log_file_name = os.path.join(log_folder, 'pygdbmi_log_' + self.test_name + '.txt')
-        pygdbmi_logger = self.gdb.logger
-        pygdbmi_logger.setLevel(logging.DEBUG)
-        while pygdbmi_logger.hasHandlers():
-            pygdbmi_logger.removeHandler(pygdbmi_logger.handlers[0])
-        log_handler = logging.FileHandler(pygdbmi_log_file_name)
-        log_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s'))
-        pygdbmi_logger.addHandler(log_handler)
-
-        # Set up logging for GDB remote protocol
-        gdb_remotelog_file_name = os.path.join(log_folder, 'gdb_remote_log_' + self.test_name + '.txt')
-        self._gdb_write('-gdb-set remotelogfile ' + gdb_remotelog_file_name)
-
-        # Load the ELF file
-        self._gdb_write('-file-exec-and-symbols {}'.format(self.app.elf_file))
-
-        # Connect GDB to UART
-        Utility.console_log('Connecting to GDB Stub...', 'orange')
-        self._gdb_write('-gdb-set serial baud 115200')
-        responses = self._gdb_write('-target-select remote ' + self.get_gdb_remote())
-
-        # Make sure we get the 'stopped' notification
-        stop_response = self.find_gdb_response('stopped', 'notify', responses)
-        if not stop_response:
-            responses = self._gdb_write('-exec-interrupt')
-            stop_response = self.find_gdb_response('stopped', 'notify', responses)
-            assert stop_response
-        frame = stop_response['payload']['frame']
-        if 'file' not in frame:
-            frame['file'] = '?'
-        if 'line' not in frame:
-            frame['line'] = '?'
-        Utility.console_log('Stopped in {func} at {addr} ({file}:{line})'.format(**frame), 'orange')
-
-        # Drain remaining responses
-        self.gdb.get_gdb_response(raise_error_on_timeout=False)
-
-    def gdb_backtrace(self):
-        """
-        Returns the list of stack frames for the current thread.
-        Each frame is a dictionary, refer to pygdbmi docs for the format.
-        """
-        assert self.gdb
-
-        responses = self._gdb_write('-stack-list-frames')
-        return self.find_gdb_response('done', 'result', responses)['payload']['stack']
-
-    @staticmethod
-    def match_backtrace(gdb_backtrace, expected_functions_list):
-        """
-        Returns True if the function names listed in expected_functions_list match the backtrace
-        given by gdb_backtrace argument. The latter is in the same format as returned by gdb_backtrace()
-        function.
-        """
-        return all([frame['func'] == expected_functions_list[i] for i, frame in enumerate(gdb_backtrace)])
-
-    @staticmethod
-    def find_gdb_response(message, response_type, responses):
-        """
-        Helper function which extracts one response from an array of GDB responses, filtering
-        by message and type. Returned message is a dictionary, refer to pygdbmi docs for the format.
-        """
-        def match_response(response):
-            return (response['message'] == message and
-                    response['type'] == response_type)
-
-        filtered_responses = [r for r in responses if match_response(r)]
-        if not filtered_responses:
-            return None
-        return filtered_responses[0]
-
-
-class ESP32PanicTestDUT(ttfw_idf.ESP32DUT, PanicTestMixin):
-    def get_gdb_remote(self):
-        return self.port
-
-
-class ESP32S2PanicTestDUT(ttfw_idf.ESP32S2DUT, PanicTestMixin):
-    def get_gdb_remote(self):
-        return self.port
-
-
-PANIC_TEST_DUT_DICT = {
-    'ESP32': ESP32PanicTestDUT,
-    'ESP32S2': ESP32S2PanicTestDUT
-}
-
-
-def panic_test(**kwargs):
-    """ Decorator for the panic tests, sets correct App and DUT classes """
-    if 'target' not in kwargs:
-        kwargs['target'] = ['ESP32']
-
-    if 'additional_duts' not in kwargs:
-        kwargs['additional_duts'] = PANIC_TEST_DUT_DICT
-    return ttfw_idf.idf_custom_test(app=PanicTestApp, env_tag='Example_GENERIC', **kwargs)
-
-
-def get_dut(env, app_config_name, test_name, qemu_wdt_enable=False):
-    dut = env.get_dut('panic', TEST_PATH, app_config_name=app_config_name, allow_dut_exception=True)
-    dut.qemu_wdt_enable = qemu_wdt_enable
-    """ Wrapper for getting the DUT and starting the test """
-    dut.start_test(test_name)
-    return dut
-
-
-def run_all(filename, case_filter=[]):
-    """ Helper function to run test cases defined in a file; to be called from __main__.
-        case_filter is an optional list of case names to run.
-        If not specified, all test cases are run.
-    """
-    TinyFW.set_default_config(env_config_file=None, test_suite_name=TEST_SUITE)
-    test_methods = SearchCases.Search.search_test_cases(filename)
-    test_methods = filter(lambda m: not m.case_info['ignore'], test_methods)
-    test_cases = CaseConfig.Parser.apply_config(test_methods, None)
-    tests_failed = []
-    for case in test_cases:
-        test_name = case.test_method.__name__
-        if case_filter:
-            if case_filter[0].endswith('*'):
-                if not test_name.startswith(case_filter[0][:-1]):
-                    continue
-            else:
-                if test_name not in case_filter:
-                    continue
-        result = case.run()
-        if not result:
-            tests_failed.append(case)
-
-    if tests_failed:
-        print('The following tests have failed:')
-        for case in tests_failed:
-            print(' - ' + case.test_method.__name__)
-        raise SystemExit(1)
-
-    print('Tests pass')