ci: replace all component ut with pytest-embedded

.gitlab/ci/assign-test.yml

@@ -17,8 +17,6 @@ assign_test:
     EXAMPLE_TEST_DIR: "${CI_PROJECT_DIR}/examples"
     CUSTOM_TEST_DIR: "${CI_PROJECT_DIR}/tools/test_apps"
     UNIT_TEST_DIR: "${CI_PROJECT_DIR}/components/idf_test/unit_test"
-    # COMPONENT_UT_DIRS is set by `set_component_ut_vars` in `utils.sh`
-    COMPONENT_UT_OUTPUT_DIR: "${CI_PROJECT_DIR}/component_ut"
     INTEGRATION_CONFIG_OUTPUT_PATH: "${CI_PROJECT_DIR}/components/idf_test/integration_test/CIConfigs"
     INTEGRATION_TEST_CASE_PATH: "${CI_PROJECT_DIR}/auto_test_script/TestCaseFiles"
     ASSIGN_TEST_CASE_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIAssignTestCases.py"
@@ -30,17 +28,13 @@ assign_test:
       - components/idf_test/*/CIConfigs
       - $EXAMPLE_TEST_DIR/test_configs
       - $CUSTOM_TEST_DIR/test_configs
-      - $COMPONENT_UT_OUTPUT_DIR/test_configs
       - build_examples/artifact_index.json
       - build_test_apps/artifact_index.json
-      - build_component_ut/artifact_index.json
       - tools/unit-test-app/builds/artifact_index.json
     expire_in: 1 week
   script:
-    - set_component_ut_vars
     - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py example_test $EXAMPLE_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $EXAMPLE_TEST_DIR/test_configs
     - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py custom_test $CUSTOM_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $CUSTOM_TEST_DIR/test_configs
-    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py component_ut $COMPONENT_UT_DIRS -c $CI_TARGET_TEST_CONFIG_FILE -o $COMPONENT_UT_OUTPUT_DIR/test_configs
     - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py unit_test $UNIT_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $UNIT_TEST_DIR/CIConfigs
     # clone test script to assign tests
     # can not retry if downing git lfs files failed, so using empty_branch first.

.gitlab/ci/build.yml

@@ -51,6 +51,13 @@ build_pytest_examples_esp32s2:
   script:
     - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir examples --target esp32s2 --size-info $SIZE_INFO_LOCATION -vv
 
+build_pytest_examples_esp32s3:
+  extends:
+    - .build_pytest_template
+    - .rules:build:example_test-esp32s3
+  script:
+    - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir examples --target esp32s3 --size-info $SIZE_INFO_LOCATION -vv
+
 build_pytest_examples_esp32c3:
   extends:
     - .build_pytest_template
@@ -58,6 +65,41 @@ build_pytest_examples_esp32c3:
   script:
     - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir examples --target esp32c3 --size-info $SIZE_INFO_LOCATION -vv
 
+build_pytest_components_esp32:
+  extends:
+    - .build_pytest_template
+    - .rules:build:component_ut-esp32
+  script:
+    - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir components --target esp32 --size-info $SIZE_INFO_LOCATION -vv
+
+build_pytest_components_esp32s2:
+  extends:
+    - .build_pytest_template
+    - .rules:build:component_ut-esp32s2
+  script:
+    - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir components --target esp32s2 --size-info $SIZE_INFO_LOCATION -vv
+
+build_pytest_components_esp32s3:
+  extends:
+    - .build_pytest_template
+    - .rules:build:component_ut-esp32s3
+  script:
+    - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir components --target esp32s3 --size-info $SIZE_INFO_LOCATION -vv
+
+build_pytest_components_esp32c2:
+  extends:
+    - .build_pytest_template
+    - .rules:build:component_ut-esp32c2
+  script:
+    - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir components --target esp32c2 --size-info $SIZE_INFO_LOCATION -vv
+
+build_pytest_components_esp32c3:
+  extends:
+    - .build_pytest_template
+    - .rules:build:component_ut-esp32c3
+  script:
+    - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir components --target esp32c3 --size-info $SIZE_INFO_LOCATION -vv
+
 .build_template_app_template:
   extends: .build_template
   variables:
@@ -331,47 +373,6 @@ build_test_apps_esp32c2:
   variables:
     IDF_TARGET: esp32c2
 
-.build_component_ut_template:
-  extends: .build_test_apps_template
-  variables:
-    TEST_PREFIX: component_ut
-    TEST_RELATIVE_DIR: component_ut
-
-build_component_ut_esp32:
-  extends:
-    - .build_component_ut_template
-    - .rules:build:component_ut-esp32
-  variables:
-    IDF_TARGET: esp32
-
-build_component_ut_esp32s2:
-  extends:
-    - .build_component_ut_template
-    - .rules:build:component_ut-esp32s2
-  variables:
-    IDF_TARGET: esp32s2
-
-build_component_ut_esp32s3:
-  extends:
-    - .build_component_ut_template
-    - .rules:build:component_ut-esp32s3
-  variables:
-    IDF_TARGET: esp32s3
-
-build_component_ut_esp32c3:
-  extends:
-    - .build_component_ut_template
-    - .rules:build:component_ut-esp32c3
-  variables:
-    IDF_TARGET: esp32c3
-
-build_component_ut_esp32c2:
-  extends:
-    - .build_component_ut_template
-    - .rules:build:component_ut-esp32c2
-  variables:
-    IDF_TARGET: esp32c2
-
 .test_build_system_template:
   stage: host_test
   extends:

.gitlab/ci/pre_check.yml

@@ -150,23 +150,19 @@ scan_tests:
     paths:
       - $EXAMPLE_TEST_OUTPUT_DIR
       - $TEST_APPS_OUTPUT_DIR
-      - $COMPONENT_UT_OUTPUT_DIR
   variables:
     EXAMPLE_TEST_DIR: ${CI_PROJECT_DIR}/examples
     EXAMPLE_TEST_OUTPUT_DIR: ${CI_PROJECT_DIR}/examples/test_configs
     TEST_APPS_TEST_DIR: ${CI_PROJECT_DIR}/tools/test_apps
     TEST_APPS_OUTPUT_DIR: ${CI_PROJECT_DIR}/tools/test_apps/test_configs
-    COMPONENT_UT_OUTPUT_DIR: ${CI_PROJECT_DIR}/component_ut/test_configs
     CI_SCAN_TESTS_PY: ${CI_PROJECT_DIR}/tools/ci/python_packages/ttfw_idf/CIScanTests.py
     EXTRA_TEST_DIRS: >-
       examples/bluetooth/esp_ble_mesh/ble_mesh_console
       examples/bluetooth/hci/controller_hci_uart_esp32
       examples/wifi/iperf
   script:
-    - set_component_ut_vars
     - run_cmd python $CI_SCAN_TESTS_PY example_test $EXAMPLE_TEST_DIR -b cmake --exclude examples/build_system/idf_as_lib -c $CI_TARGET_TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR --extra_test_dirs $EXTRA_TEST_DIRS
     - run_cmd python $CI_SCAN_TESTS_PY test_apps $TEST_APPS_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_APPS_OUTPUT_DIR
-    - run_cmd python $CI_SCAN_TESTS_PY component_ut $COMPONENT_UT_DIRS --exclude $COMPONENT_UT_EXCLUDES -c $CI_TARGET_TEST_CONFIG_FILE -o $COMPONENT_UT_OUTPUT_DIR
 
 # For release tag pipelines only, make sure the tag was created with 'git tag -a' so it will update
 # the version returned by 'git describe'

.gitlab/ci/target-test.yml

@@ -5,6 +5,7 @@
     when: always
     paths:
       - XUNIT_RESULT.xml
+      - /tmp/pytest-embedded/
     reports:
       junit: XUNIT_RESULT.xml
   script:
@@ -41,6 +42,19 @@ example_test_pytest_esp32s2_generic:
     - ESP32S2
     - Example_GENERIC
 
+example_test_pytest_esp32s3_generic:
+  extends:
+    - .pytest_examples_dir_template
+    - .rules:test:example_test-esp32s3
+  needs:
+    - build_pytest_examples_esp32s3
+  variables:
+    TARGET: esp32s3
+    ENV_MARKER: generic
+  tags:
+    - ESP32S3
+    - Example_GENERIC
+
 example_test_pytest_esp32c3_generic:
   extends:
     - .pytest_examples_dir_template
@@ -67,6 +81,89 @@ example_test_pytest_esp32c3_flash_suspend:
     - ESP32C3_IDF
     - UT_T1_Flash_Suspend
 
+.pytest_components_dir_template:
+  extends: .pytest_template
+  variables:
+    TEST_DIR: components
+
+component_ut_pytest_esp32_generic:
+  extends:
+    - .pytest_components_dir_template
+    - .rules:test:component_ut-esp32
+  needs:
+    - build_pytest_components_esp32
+  variables:
+    TARGET: esp32
+    ENV_MARKER: generic
+  tags:
+    - ESP32
+    - COMPONENT_UT_GENERIC
+
+component_ut_pytest_esp32_ip101:
+  extends:
+    - .pytest_components_dir_template
+    - .rules:test:component_ut-esp32
+  needs:
+    - build_pytest_components_esp32
+  variables:
+    TARGET: esp32
+    ENV_MARKER: ip101
+  tags:
+    - ESP32
+    - COMPONENT_UT_IP101
+
+component_ut_pytest_esp32_lan8720:
+  extends:
+    - .pytest_components_dir_template
+    - .rules:test:component_ut-esp32
+  needs:
+    - build_pytest_components_esp32
+  variables:
+    TARGET: esp32
+    ENV_MARKER: lan8720
+  tags:
+    - ESP32
+    - COMPONENT_UT_LAN8720
+
+component_ut_pytest_esp32s2_generic:
+  extends:
+    - .pytest_components_dir_template
+    - .rules:test:component_ut-esp32s2
+  needs:
+    - build_pytest_components_esp32s2
+  variables:
+    TARGET: esp32s2
+    ENV_MARKER: generic
+  tags:
+    - ESP32S2
+    - COMPONENT_UT_GENERIC
+
+component_ut_pytest_esp32s3_generic:
+  extends:
+    - .pytest_components_dir_template
+    - .rules:test:component_ut-esp32s3
+  needs:
+    - build_pytest_components_esp32s3
+  variables:
+    TARGET: esp32s3
+    ENV_MARKER: generic
+  tags:
+    - ESP32S3
+    - COMPONENT_UT_GENERIC
+
+component_ut_pytest_esp32c3_generic:
+  extends:
+    - .pytest_components_dir_template
+    - .rules:test:component_ut-esp32c3
+  needs:
+    - build_pytest_components_esp32c3
+  variables:
+    TARGET: esp32c3
+    ENV_MARKER: generic
+  tags:
+    - ESP32C3
+    - COMPONENT_UT_GENERIC
+
 # for parallel jobs, CI_JOB_NAME will be "job_name index/total" (for example, "IT_001 1/2")
 # we need to convert to pattern "job_name_index.yml"
 .define_config_file_name: &define_config_file_name |
@@ -459,68 +556,6 @@ test_app_test_flash_psram_f8r8:
     - ESP32S3
     - MSPI_F8R8
 
-.component_ut_template:
-  extends: .target_test_job_template
-  variables:
-    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/component_ut/test_configs"
-  script:
-    - *define_config_file_name
-    # first test if config file exists, if not exist, exit 0
-    - test -e $CONFIG_FILE || exit 0
-    - set_component_ut_vars
-    # clone test env configs
-    - retry_failed git clone $TEST_ENV_CONFIG_REPO
-    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
-    # git clone the known failure cases repo, run test
-    - retry_failed git clone $KNOWN_FAILURE_CASES_REPO known_failure_cases
-    # run test
-    - cd tools/ci/python_packages/tiny_test_fw/bin
-    - run_cmd python Runner.py $COMPONENT_UT_DIRS -c $CONFIG_FILE -e $ENV_FILE --known_failure_cases_file $CI_PROJECT_DIR/known_failure_cases/known_failure_cases.txt
-
-.component_ut_esp32_template:
-  extends:
-    - .component_ut_template
-    - .rules:test:component_ut-esp32
-
-.component_ut_esp32s2_template:
-  extends:
-    - .component_ut_template
-    - .rules:test:component_ut-esp32s2
-
-.component_ut_esp32s3_template:
-  extends:
-    - .component_ut_template
-    - .rules:test:component_ut-esp32s3
-
-.component_ut_esp32c3_template:
-  extends:
-    - .component_ut_template
-    - .rules:test:component_ut-esp32c3
-
-component_ut_test_001:
-  extends: .component_ut_esp32_template
-  tags:
-    - ESP32
-    - COMPONENT_UT_GENERIC
-
-component_ut_test_esp32s2:
-  extends: .component_ut_esp32s2_template
-  tags:
-    - ESP32S2
-    - COMPONENT_UT_GENERIC
-
-component_ut_test_esp32s3:
-  extends: .component_ut_esp32s3_template
-  tags:
-    - ESP32S3
-    - COMPONENT_UT_GENERIC
-
-component_ut_test_esp32c3:
-  extends: .component_ut_esp32c3_template
-  tags:
-    - ESP32C3
-    - COMPONENT_UT_GENERIC
-
 .unit_test_template:
   extends: .target_test_job_template
   variables:
@@ -825,18 +860,6 @@ UT_S3_FLASH:
     - ESP32S3_IDF
     - UT_T1_ESP_FLASH
 
-component_ut_test_ip101:
-  extends: .component_ut_esp32_template
-  tags:
-    - ESP32
-    - COMPONENT_UT_IP101
-
-component_ut_test_lan8720:
-  extends: .component_ut_esp32_template
-  tags:
-    - ESP32
-    - COMPONENT_UT_LAN8720
-
 .integration_test_template:
   extends:
     - .target_test_job_template

components/driver/test_apps/gptimer/app_test.py

@@ -1,30 +0,0 @@
-# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
-# SPDX-License-Identifier: Apache-2.0
-import glob
-import os
-
-import ttfw_idf
-from tiny_test_fw import Utility
-
-
-@ttfw_idf.idf_component_unit_test(env_tag='COMPONENT_UT_GENERIC', target=['esp32', 'esp32s2', 'esp32s3', 'esp32c3'])
-def test_component_ut_gptimer(env, _):  # type: (ttfw_idf.TinyFW.Env, None) -> None
-    # Get the names of all configs (sdkconfig.ci.* files)
-    config_files = glob.glob(os.path.join(os.path.dirname(__file__), 'sdkconfig.ci.*'))
-    config_names = [os.path.basename(s).replace('sdkconfig.ci.', '') for s in config_files]
-
-    # Run test once with binaries built for each config
-    for name in config_names:
-        Utility.console_log(f'Checking config "{name}"... ', end='')
-        dut = env.get_dut('gptimer', 'components/driver/test_apps/gptimer', app_config_name=name)
-        dut.start_app()
-        stdout = dut.expect('Press ENTER to see the list of tests', full_stdout=True)
-        dut.write('*')
-        stdout = dut.expect("Enter next test, or 'enter' to see menu", full_stdout=True, timeout=30)
-        ttfw_idf.ComponentUTResult.parse_result(stdout,ttfw_idf.TestFormat.UNITY_BASIC)
-        env.close_dut(dut.name)
-        Utility.console_log(f'Test config "{name}" done')
-
-
-if __name__ == '__main__':
-    test_component_ut_gptimer()

components/driver/test_apps/gptimer/pytest_gptimer.py

@@ -0,0 +1,17 @@
+# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: CC0-1.0
+
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.supported_targets
+@pytest.mark.generic
+@pytest.mark.parametrize('config', [
+    'iram_safe',
+    'release',
+], indirect=True)
+def test_gptimer(dut: Dut) -> None:
+    dut.expect('Press ENTER to see the list of tests')
+    dut.write('*')
+    dut.expect_unity_test_output()

components/driver/test_apps/legacy_timer_driver/app_test.py

@@ -1,30 +0,0 @@
-# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
-# SPDX-License-Identifier: Apache-2.0
-import glob
-import os
-
-import ttfw_idf
-from tiny_test_fw import Utility
-
-
-@ttfw_idf.idf_component_unit_test(env_tag='COMPONENT_UT_GENERIC', target=['esp32', 'esp32s2', 'esp32s3', 'esp32c3'])
-def test_component_ut_legacy_timer_driver(env, _):  # type: (ttfw_idf.TinyFW.Env, None) -> None
-    # Get the names of all configs (sdkconfig.ci.* files)
-    config_files = glob.glob(os.path.join(os.path.dirname(__file__), 'sdkconfig.ci.*'))
-    config_names = [os.path.basename(s).replace('sdkconfig.ci.', '') for s in config_files]
-
-    # Run test once with binaries built for each config
-    for name in config_names:
-        Utility.console_log(f'Checking config "{name}"... ', end='')
-        dut = env.get_dut('gptimer', 'components/driver/test_apps/legacy_timer_driver', app_config_name=name)
-        dut.start_app()
-        stdout = dut.expect('Press ENTER to see the list of tests', full_stdout=True)
-        dut.write('*')
-        stdout = dut.expect("Enter next test, or 'enter' to see menu", full_stdout=True, timeout=80)
-        ttfw_idf.ComponentUTResult.parse_result(stdout,ttfw_idf.TestFormat.UNITY_BASIC)
-        env.close_dut(dut.name)
-        Utility.console_log(f'Test config "{name}" done')
-
-
-if __name__ == '__main__':
-    test_component_ut_legacy_timer_driver()

components/driver/test_apps/legacy_timer_driver/pytest_legacy_timer_driver.py

@@ -0,0 +1,16 @@
+# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: CC0-1.0
+
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.supported_targets
+@pytest.mark.generic
+@pytest.mark.parametrize('config', [
+    'release',
+], indirect=True)
+def test_legacy_timer_driver(dut: Dut) -> None:
+    dut.expect('Press ENTER to see the list of tests')
+    dut.write('*')
+    dut.expect_unity_test_output(timeout=120)

components/esp_eth/test_apps/component_ut_test.py

@@ -1,114 +0,0 @@
-import os
-import re
-import socket
-
-import tiny_test_fw
-import ttfw_idf
-from tiny_test_fw import Utility
-from ttfw_idf import TestFormat
-
-try:
-    import typing  # noqa: F401 # pylint: disable=unused-import
-except ImportError:
-    pass
-
-
-def configure_eth_if(func):         # type: (typing.Any) -> typing.Any
-    def inner(*args, **kwargs):     # type: (typing.Any, typing.Any) -> typing.Any
-        # try to determine which interface to use
-        netifs = os.listdir('/sys/class/net/')
-        target_if = ''
-        Utility.console_log('detected interfaces: ' + str(netifs))
-        for netif in netifs:
-            if netif.find('eth') == 0 or netif.find('enp') == 0 or netif.find('eno') == 0:
-                target_if = netif
-                break
-        if target_if == '':
-            raise Exception('no network interface found')
-        Utility.console_log('Use ' + target_if + ' for testing')
-        so = socket.socket(socket.AF_PACKET, socket.SOCK_RAW, 0x2222)
-        so.bind((target_if, 0))
-
-        func(so, *args, **kwargs)
-
-        so.close()
-
-    return inner
-
-
-@configure_eth_if
-def check_eth_recv_packet(so, before_recv=None):    # type: (socket.socket, typing.Any) -> None
-    so.settimeout(10)
-    if before_recv is not None:
-        before_recv()   # If configured, execute user function just before sock recv
-    try:
-        pkt = so.recv(1024)
-        for i in range(128, 1024):
-            if pkt[i] != i & 0xff:
-                raise Exception('Packet content mismatch')
-    except Exception as e:
-        raise e
-
-
-@configure_eth_if
-def send_eth_packet(so, mac):    # type: (socket.socket, bytes) -> None
-    so.settimeout(10)
-    pkt = bytearray()
-    pkt += mac                      # dest
-    pkt += so.getsockname()[4]      # src
-    pkt += bytes.fromhex('2222')    # proto
-    pkt += bytes(1010)              # padding to 1024
-    for i in range(128, 1024):
-        pkt[i] = i & 0xff
-    try:
-        so.send(pkt)
-    except Exception as e:
-        raise e
-
-
-def test_component_ut_esp_eth(env, appname):  # type: (tiny_test_fw.Env, str) -> None
-    dut = env.get_dut('esp_eth', 'components/esp_eth/test_apps', app_config_name=appname)
-    dut.start_app()
-    stdout = dut.expect('Press ENTER to see the list of tests', full_stdout=True)
-
-    Utility.console_log('Running test case: start_and_stop')
-    dut.write('"start_and_stop"')
-    stdout += dut.expect("Enter next test, or 'enter' to see menu", full_stdout=True)
-    ttfw_idf.ComponentUTResult.parse_result(stdout, test_format=TestFormat.UNITY_BASIC)
-
-    Utility.console_log('Running test case: get_set_mac')
-    dut.write('"get_set_mac"')
-    stdout = dut.expect("Enter next test, or 'enter' to see menu", full_stdout=True)
-    ttfw_idf.ComponentUTResult.parse_result(stdout, test_format=TestFormat.UNITY_BASIC)
-
-    Utility.console_log('Running test case: ethernet_broadcast_transmit')
-    check_eth_recv_packet(dut.write('"ethernet_broadcast_transmit"'))  # Need to start the test after the socket is bound
-    stdout = dut.expect("Enter next test, or 'enter' to see menu", full_stdout=True)
-    ttfw_idf.ComponentUTResult.parse_result(stdout, test_format=TestFormat.UNITY_BASIC)
-
-    Utility.console_log('Running test case: recv_pkt')
-    dut.write('"recv_pkt"')
-    expect_result = dut.expect(re.compile(r'([\s\S]*)DUT MAC: ([0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2})'),
-                               timeout=10)
-    stdout = expect_result[0]
-    Utility.console_log('DUTs MAC address: {}'.format(expect_result[1]))
-    send_eth_packet(bytes.fromhex('ffffffffffff'))      # broadcast frame
-    send_eth_packet(bytes.fromhex('010000000000'))      # multicast frame
-    send_eth_packet(bytes.fromhex(expect_result[1].replace(':', '')))   # unicast frame
-    stdout += dut.expect("Enter next test, or 'enter' to see menu", full_stdout=True)
-    ttfw_idf.ComponentUTResult.parse_result(stdout, test_format=TestFormat.UNITY_BASIC)
-
-
-@ttfw_idf.idf_component_unit_test(env_tag='COMPONENT_UT_IP101', target=['esp32'])
-def test_component_ut_esp_eth_ip101(env, _):  # type: (tiny_test_fw.Env, typing.Any) -> None
-    test_component_ut_esp_eth(env, 'ip101')
-
-
-@ttfw_idf.idf_component_unit_test(env_tag='COMPONENT_UT_LAN8720', target=['esp32'])
-def test_component_ut_esp_eth_lan8720(env, _):  # type: (tiny_test_fw.Env, typing.Any) -> None
-    test_component_ut_esp_eth(env, 'lan8720')
-
-
-if __name__ == '__main__':
-    test_component_ut_esp_eth_ip101()
-    test_component_ut_esp_eth_lan8720()

components/esp_eth/test_apps/pytest_esp_eth.py

@@ -0,0 +1,103 @@
+# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: CC0-1.0
+
+import contextlib
+import logging
+import os
+import socket
+from typing import Iterator
+
+import pytest
+from pytest_embedded import Dut
+
+
+@contextlib.contextmanager
+def configure_eth_if() -> Iterator[socket.socket]:
+    # try to determine which interface to use
+    netifs = os.listdir('/sys/class/net/')
+    logging.info('detected interfaces: %s', str(netifs))
+
+    target_if = ''
+    for netif in netifs:
+        if netif.find('eth') == 0 or netif.find('enp') == 0 or netif.find('eno') == 0:
+            target_if = netif
+            break
+    if target_if == '':
+        raise Exception('no network interface found')
+    logging.info('Use %s for testing', target_if)
+
+    so = socket.socket(socket.AF_PACKET, socket.SOCK_RAW, 0x2222)
+    so.bind((target_if, 0))
+
+    try:
+        yield so
+    finally:
+        so.close()
+
+
+def send_eth_packet(mac: bytes) -> None:
+    with configure_eth_if() as so:
+        so.settimeout(10)
+        pkt = bytearray()
+        pkt += mac  # dest
+        pkt += so.getsockname()[4]  # src
+        pkt += bytes.fromhex('2222')  # proto
+        pkt += bytes(1010)  # padding to 1024
+        for i in range(128, 1024):
+            pkt[i] = i & 0xff
+        try:
+            so.send(pkt)
+        except Exception as e:
+            raise e
+
+
+def actual_test(dut: Dut) -> None:
+    dut.expect_exact('Press ENTER to see the list of tests')
+    dut.write('\n')
+
+    dut.expect_exact('Enter test for running.')
+    dut.write('"start_and_stop"')
+    dut.expect_unity_test_output()
+
+    dut.expect_exact("Enter next test, or 'enter' to see menu")
+    dut.write('"get_set_mac"')
+    dut.expect_unity_test_output()
+
+    dut.expect_exact("Enter next test, or 'enter' to see menu")
+    with configure_eth_if() as so:
+        so.settimeout(30)
+        dut.write('"ethernet_broadcast_transmit"')
+        pkt = so.recv(1024)
+        for i in range(128, 1024):
+            if pkt[i] != i & 0xff:
+                raise Exception('Packet content mismatch')
+    dut.expect_unity_test_output()
+
+    dut.expect_exact("Enter next test, or 'enter' to see menu")
+    dut.write('"recv_pkt"')
+    res = dut.expect(
+        r'([\s\S]*)'
+        r'DUT MAC: ([0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2})'
+    )
+    send_eth_packet(bytes.fromhex('ffffffffffff'))  # broadcast frame # pylint: disable=no-value-for-parameter
+    send_eth_packet(bytes.fromhex('010000000000'))  # multicast frame # pylint: disable=no-value-for-parameter
+    send_eth_packet(bytes.fromhex(res.group(2).decode('utf-8').replace(':', '')))  # unicast fram  # pylint: disable=no-value-for-parameter, line-too-long # noqa
+    dut.expect_unity_test_output(extra_before=res.group(1))
+
+
+@pytest.mark.esp32
+@pytest.mark.ip101
+@pytest.mark.parametrize('config', [
+    'ip101',
+], indirect=True)
+def test_esp_eth_ip101(dut: Dut) -> None:
+    actual_test(dut)
+
+
+@pytest.mark.esp32
+@pytest.mark.lan8720
+@pytest.mark.parametrize('config', [
+    'lan8720',
+], indirect=True)
+def test_esp_eth_lan8720(dut: Dut) -> None:
+    actual_test(dut)

components/esp_netif/test_apps/component_ut_test.py

@@ -1,15 +0,0 @@
-from __future__ import print_function
-
-import ttfw_idf
-
-
-@ttfw_idf.idf_component_unit_test(env_tag='COMPONENT_UT_GENERIC')
-def test_component_ut_esp_netif(env, extra_data):
-    dut = env.get_dut('esp_netif', 'components/esp_netif/test_apps')
-    dut.start_app()
-    stdout = dut.expect('Tests finished', full_stdout=True)
-    ttfw_idf.ComponentUTResult.parse_result(stdout)
-
-
-if __name__ == '__main__':
-    test_component_ut_esp_netif()

components/esp_netif/test_apps/pytest_esp_netif.py

@@ -0,0 +1,11 @@
+# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: CC0-1.0
+
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.esp32s2
+@pytest.mark.generic
+def test_esp_netif(dut: Dut) -> None:
+    dut.expect_unity_test_output()

components/newlib/test_apps/app_test.py

@@ -1,19 +0,0 @@
-import tiny_test_fw  # noqa: F401 # pylint: disable=unused-import
-import ttfw_idf
-
-try:
-    import typing  # noqa: F401 # pylint: disable=unused-import
-except ImportError:
-    pass
-
-
-@ttfw_idf.idf_component_unit_test(env_tag='COMPONENT_UT_GENERIC', target=['esp32', 'esp32s2', 'esp32s3', 'esp32c3'])
-def test_component_ut_newlib(env, _):  # type: (tiny_test_fw.Env, typing.Any) -> None
-    dut = env.get_dut('newlib', 'components/newlib/test_apps')
-    dut.start_app()
-    stdout = dut.expect('Tests finished, rc=0', full_stdout=True)
-    ttfw_idf.ComponentUTResult.parse_result(stdout)
-
-
-if __name__ == '__main__':
-    test_component_ut_newlib()

components/newlib/test_apps/pytest_newlib.py

@@ -0,0 +1,11 @@
+# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: CC0-1.0
+
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.supported_targets
+@pytest.mark.generic
+def test_newlib(dut: Dut) -> None:
+    dut.expect_unity_test_output()

components/wear_levelling/test_apps/component_ut_test.py

@@ -1,28 +0,0 @@
-# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
-# SPDX-License-Identifier: Apache-2.0
-import glob
-import os
-
-import ttfw_idf
-from tiny_test_fw import Utility
-
-
-@ttfw_idf.idf_component_unit_test(env_tag='COMPONENT_UT_GENERIC', target=['esp32', 'esp32c3'])
-def test_component_ut_wear_levelling(env, _):   # type: (ttfw_idf.TinyFW.Env, None) -> None
-    # Get the names of all configs (sdkconfig.ci.* files)
-    config_files = glob.glob(os.path.join(os.path.dirname(__file__), 'sdkconfig.ci.*'))
-    config_names = [os.path.basename(s).replace('sdkconfig.ci.', '') for s in config_files]
-
-    # Run test once with binaries built for each config
-    for name in config_names:
-        Utility.console_log("Checking config \"{}\"... ".format(name), end='')
-        dut = env.get_dut('wear_levelling', 'components/wear_levelling/test_apps', app_config_name=name)
-        dut.start_app()
-        stdout = dut.expect('Tests finished', full_stdout=True, timeout=30)
-        ttfw_idf.ComponentUTResult.parse_result(stdout)
-        env.close_dut(dut.name)
-        Utility.console_log('done')
-
-
-if __name__ == '__main__':
-    test_component_ut_wear_levelling()

components/wear_levelling/test_apps/pytest_wear_levelling.py

@@ -0,0 +1,18 @@
+# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: CC0-1.0
+
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.esp32
+@pytest.mark.esp32c3
+@pytest.mark.generic
+@pytest.mark.parametrize('config', [
+    '4k',
+    '512perf',
+    '512safe',
+    'release',
+], indirect=True)
+def test_wear_levelling(dut: Dut) -> None:
+    dut.expect_unity_test_output()

conftest.py

@@ -16,17 +16,19 @@
 import logging
 import os
 import sys
+import xml.etree.ElementTree as ET
 from typing import Callable, List, Optional
 
 import pytest
 from _pytest.config import Config
 from _pytest.fixtures import FixtureRequest
 from _pytest.nodes import Item
+from _pytest.python import Function
 from pytest_embedded.plugin import parse_configuration
-from pytest_embedded_idf.app import IdfApp
+from pytest_embedded.utils import find_by_suffix
 
 SUPPORTED_TARGETS = ['esp32', 'esp32s2', 'esp32c3', 'esp32s3']
-PREVIEW_TARGETS = ['linux', 'esp32h2', 'esp8684']
+PREVIEW_TARGETS = ['linux', 'esp32h2', 'esp32c2']
 
 
 ##################
@@ -42,7 +44,7 @@ def is_target_marker(marker: str) -> bool:
     return False
 
 
-def format_case_id(target: str, config: str, case: str) -> str:
+def format_case_id(target: Optional[str], config: Optional[str], case: str) -> str:
     return f'{target}.{config}.{case}'
 
 
@@ -58,6 +60,11 @@ def config(request: FixtureRequest) -> str:
     return getattr(request, 'param', None) or request.config.getoption('config', 'default')  # type: ignore
 
 
+@pytest.fixture
+def test_case_name(request: FixtureRequest, target: str, config: str) -> str:
+    return format_case_id(target, config, request.node.originalname)
+
+
 @pytest.fixture
 @parse_configuration
 def build_dir(request: FixtureRequest, app_path: str, target: Optional[str], config: Optional[str]) -> str:
@@ -106,34 +113,58 @@ def build_dir(request: FixtureRequest, app_path: str, target: Optional[str], con
 
 
 @pytest.fixture(autouse=True)
-def junit_properties(app: IdfApp, config: str, test_case_name: str,
+def junit_properties(test_case_name: str, record_xml_attribute: Callable[[str, object], None]) -> None:
-                     record_xml_attribute: Callable[[str, object], None]) -> None:
     """
     This fixture is autoused and will modify the junit report test case name to <target>.<config>.<case_name>
     """
-    record_xml_attribute('name', format_case_id(app.target, config, test_case_name))
+    record_xml_attribute('name', test_case_name)
 
 
 ##################
 # Hook functions #
 ##################
-@pytest.hookimpl(trylast=True)
+@pytest.hookimpl(tryfirst=True)
 def pytest_collection_modifyitems(config: Config, items: List[Item]) -> None:
-    target = config.getoption('target', None)
+    target = config.getoption('target', None)  # use the `build` dir
     if not target:
         return
 
     # add markers for special markers
     for item in items:
         if 'supported_targets' in item_marker_names(item):
-            for target in SUPPORTED_TARGETS:
+            for _target in SUPPORTED_TARGETS:
-                item.add_marker(target)
+                item.add_marker(_target)
         if 'preview_targets' in item_marker_names(item):
-            for target in PREVIEW_TARGETS:
+            for _target in PREVIEW_TARGETS:
-                item.add_marker(target)
+                item.add_marker(_target)
         if 'all_targets' in item_marker_names(item):
-            for target in [*SUPPORTED_TARGETS, *PREVIEW_TARGETS]:
+            for _target in [*SUPPORTED_TARGETS, *PREVIEW_TARGETS]:
-                item.add_marker(target)
+                item.add_marker(_target)
 
     # filter all the test cases with "--target"
     items[:] = [item for item in items if target in item_marker_names(item)]
+
+
+@pytest.hookimpl(trylast=True)
+def pytest_runtest_teardown(item: Function) -> None:
+    """
+    Format the test case generated junit reports
+    """
+    tempdir = item.funcargs.get('test_case_tempdir')
+    if not tempdir:
+        return
+
+    junits = find_by_suffix('.xml', tempdir)
+    if not junits:
+        return
+
+    target = item.funcargs['target']
+    config = item.funcargs['config']
+    for junit in junits:
+        xml = ET.parse(junit)
+        testcases = xml.findall('.//testcase')
+        for case in testcases:
+            case.attrib['name'] = format_case_id(target, config, case.attrib['name'])
+            if 'file' in case.attrib:
+                case.attrib['file'] = case.attrib['file'].replace('/IDF/', '')  # our unity test framework
+        xml.write(junit)

pytest.ini

@@ -13,11 +13,15 @@ markers =
   esp32s2: support esp32s2 target
   esp32s3: support esp32s3 target
   esp32c3: support esp32c3 target
+  supported_targets: support all supported targets ('esp32', 'esp32s2', 'esp32c3', 'esp32s3')
+  preview_targets: support all preview targets ('linux', 'esp32h2', 'esp32c2')
+  all_targets: support all targets, including supported ones and preview ones
+
+  # env markers
   generic: tests should be run on generic runners
   flash_suspend: support flash suspend feature
-  supported_targets: support all supported targets ('esp32', 'esp32s2', 'esp32c3', 'esp32s3')
+  ip101: connected via wired 10/100M ethernet
-  preview_targets: support all preview targets ('linux', 'esp32h2', 'esp8684')
+  lan8720: connected via LAN8720 ethernet transceiver
-  all_targets: support all targets, including supported ones and preview ones
 
 # log related
 log_cli = True

tools/ci/build_pytest_apps.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 
 """
@@ -27,7 +27,7 @@ except ImportError:
 
 def main(args: argparse.Namespace) -> None:
     if args.all_pytest_apps:
-        paths = get_pytest_dirs(IDF_PATH, args.under_dir)
+        paths = get_pytest_dirs(args.under_dir)
         args.recursive = True
     elif args.paths is None:
         paths = [os.getcwd()]

tools/ci/idf_ci_utils.py

@@ -8,7 +8,7 @@ import logging
 import os
 import subprocess
 import sys
-from typing import List, Optional
+from typing import List
 
 IDF_PATH = os.path.abspath(os.getenv('IDF_PATH', os.path.join(os.path.dirname(__file__), '..', '..')))
 
@@ -86,7 +86,7 @@ def is_in_directory(file_path: str, folder: str) -> bool:
     return os.path.realpath(file_path).startswith(os.path.realpath(folder) + os.sep)
 
 
-def get_pytest_dirs(folder: str, under_dir: Optional[str] = None) -> List[str]:
+def get_pytest_dirs(folder: str) -> List[str]:
     from io import StringIO
 
     import pytest
@@ -109,7 +109,4 @@ def get_pytest_dirs(folder: str, under_dir: Optional[str] = None) -> List[str]:
 
     test_file_paths = set(node.fspath for node in collector.nodes)
 
-    if under_dir:
-        return [os.path.dirname(file) for file in test_file_paths if is_in_directory(file, under_dir)]
-
     return [os.path.dirname(file) for file in test_file_paths]

tools/ci/utils.sh

@@ -42,13 +42,6 @@ function get_all_submodules() {
   git config --file .gitmodules --get-regexp path | awk '{ print $2 }' | sed -e 's|$|/**|' | xargs | sed -e 's/ /,/g'
 }
 
-function set_component_ut_vars() {
-  local exclude_list_fp="${IDF_PATH}/tools/ci/component_ut_excludes.txt"
-  export COMPONENT_UT_DIRS=$(find components/ -name test_apps -type d)
-  export COMPONENT_UT_EXCLUDES=$([ -r $exclude_list_fp ] && cat $exclude_list_fp | xargs)
-  echo "COMPONENT_UT_DIRS, COMPONENT_UT_EXCLUDES written into export"
-}
-
 function error() {
   printf "\033[0;31m%s\n\033[0m" "${1}" >&2
 }