From a801555299da7049dc74265728caa5c2a288ee4b Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Wed, 19 Jan 2022 12:12:15 +0800 Subject: [PATCH] ci: replace all component ut with pytest-embedded --- .gitlab/ci/assign-test.yml | 6 - .gitlab/ci/build.yml | 83 ++++----- .gitlab/ci/pre_check.yml | 4 - .gitlab/ci/target-test.yml | 171 ++++++++++-------- .../driver/test_apps/gptimer/app_test.py | 30 --- .../test_apps/gptimer/pytest_gptimer.py | 17 ++ .../test_apps/legacy_timer_driver/app_test.py | 30 --- .../pytest_legacy_timer_driver.py | 16 ++ .../esp_eth/test_apps/component_ut_test.py | 114 ------------ .../esp_eth/test_apps/pytest_esp_eth.py | 103 +++++++++++ .../esp_netif/test_apps/component_ut_test.py | 15 -- .../esp_netif/test_apps/pytest_esp_netif.py | 11 ++ components/newlib/test_apps/app_test.py | 19 -- components/newlib/test_apps/pytest_newlib.py | 11 ++ .../test_apps/component_ut_test.py | 28 --- .../test_apps/pytest_wear_levelling.py | 18 ++ conftest.py | 59 ++++-- ...t_gptimer.py => pytest_gptimer_example.py} | 0 pytest.ini | 10 +- tools/ci/build_pytest_apps.py | 4 +- tools/ci/idf_ci_utils.py | 7 +- tools/ci/utils.sh | 7 - 22 files changed, 371 insertions(+), 392 deletions(-) delete mode 100644 components/driver/test_apps/gptimer/app_test.py create mode 100644 components/driver/test_apps/gptimer/pytest_gptimer.py delete mode 100644 components/driver/test_apps/legacy_timer_driver/app_test.py create mode 100644 components/driver/test_apps/legacy_timer_driver/pytest_legacy_timer_driver.py delete mode 100644 components/esp_eth/test_apps/component_ut_test.py create mode 100644 components/esp_eth/test_apps/pytest_esp_eth.py delete mode 100644 components/esp_netif/test_apps/component_ut_test.py create mode 100644 components/esp_netif/test_apps/pytest_esp_netif.py delete mode 100644 components/newlib/test_apps/app_test.py create mode 100644 components/newlib/test_apps/pytest_newlib.py delete mode 100644 components/wear_levelling/test_apps/component_ut_test.py create mode 100644 components/wear_levelling/test_apps/pytest_wear_levelling.py rename examples/peripherals/timer_group/gptimer/{pytest_gptimer.py => pytest_gptimer_example.py} (100%) diff --git a/.gitlab/ci/assign-test.yml b/.gitlab/ci/assign-test.yml index c03e2cdadd..1471fb7156 100644 --- a/.gitlab/ci/assign-test.yml +++ b/.gitlab/ci/assign-test.yml @@ -17,8 +17,6 @@ assign_test: EXAMPLE_TEST_DIR: "${CI_PROJECT_DIR}/examples" CUSTOM_TEST_DIR: "${CI_PROJECT_DIR}/tools/test_apps" UNIT_TEST_DIR: "${CI_PROJECT_DIR}/components/idf_test/unit_test" - # COMPONENT_UT_DIRS is set by `set_component_ut_vars` in `utils.sh` - COMPONENT_UT_OUTPUT_DIR: "${CI_PROJECT_DIR}/component_ut" INTEGRATION_CONFIG_OUTPUT_PATH: "${CI_PROJECT_DIR}/components/idf_test/integration_test/CIConfigs" INTEGRATION_TEST_CASE_PATH: "${CI_PROJECT_DIR}/auto_test_script/TestCaseFiles" ASSIGN_TEST_CASE_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIAssignTestCases.py" @@ -30,17 +28,13 @@ assign_test: - components/idf_test/*/CIConfigs - $EXAMPLE_TEST_DIR/test_configs - $CUSTOM_TEST_DIR/test_configs - - $COMPONENT_UT_OUTPUT_DIR/test_configs - build_examples/artifact_index.json - build_test_apps/artifact_index.json - - build_component_ut/artifact_index.json - tools/unit-test-app/builds/artifact_index.json expire_in: 1 week script: - - set_component_ut_vars - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py example_test $EXAMPLE_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $EXAMPLE_TEST_DIR/test_configs - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py custom_test $CUSTOM_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $CUSTOM_TEST_DIR/test_configs - - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py component_ut $COMPONENT_UT_DIRS -c $CI_TARGET_TEST_CONFIG_FILE -o $COMPONENT_UT_OUTPUT_DIR/test_configs - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py unit_test $UNIT_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $UNIT_TEST_DIR/CIConfigs # clone test script to assign tests # can not retry if downing git lfs files failed, so using empty_branch first. diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index 4da8f6e6f9..8e814b5755 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -51,6 +51,13 @@ build_pytest_examples_esp32s2: script: - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir examples --target esp32s2 --size-info $SIZE_INFO_LOCATION -vv +build_pytest_examples_esp32s3: + extends: + - .build_pytest_template + - .rules:build:example_test-esp32s3 + script: + - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir examples --target esp32s3 --size-info $SIZE_INFO_LOCATION -vv + build_pytest_examples_esp32c3: extends: - .build_pytest_template @@ -58,6 +65,41 @@ build_pytest_examples_esp32c3: script: - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir examples --target esp32c3 --size-info $SIZE_INFO_LOCATION -vv +build_pytest_components_esp32: + extends: + - .build_pytest_template + - .rules:build:component_ut-esp32 + script: + - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir components --target esp32 --size-info $SIZE_INFO_LOCATION -vv + +build_pytest_components_esp32s2: + extends: + - .build_pytest_template + - .rules:build:component_ut-esp32s2 + script: + - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir components --target esp32s2 --size-info $SIZE_INFO_LOCATION -vv + +build_pytest_components_esp32s3: + extends: + - .build_pytest_template + - .rules:build:component_ut-esp32s3 + script: + - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir components --target esp32s3 --size-info $SIZE_INFO_LOCATION -vv + +build_pytest_components_esp32c2: + extends: + - .build_pytest_template + - .rules:build:component_ut-esp32c2 + script: + - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir components --target esp32c2 --size-info $SIZE_INFO_LOCATION -vv + +build_pytest_components_esp32c3: + extends: + - .build_pytest_template + - .rules:build:component_ut-esp32c3 + script: + - python tools/ci/build_pytest_apps.py --all-pytest-apps --under-dir components --target esp32c3 --size-info $SIZE_INFO_LOCATION -vv + .build_template_app_template: extends: .build_template variables: @@ -331,47 +373,6 @@ build_test_apps_esp32c2: variables: IDF_TARGET: esp32c2 -.build_component_ut_template: - extends: .build_test_apps_template - variables: - TEST_PREFIX: component_ut - TEST_RELATIVE_DIR: component_ut - -build_component_ut_esp32: - extends: - - .build_component_ut_template - - .rules:build:component_ut-esp32 - variables: - IDF_TARGET: esp32 - -build_component_ut_esp32s2: - extends: - - .build_component_ut_template - - .rules:build:component_ut-esp32s2 - variables: - IDF_TARGET: esp32s2 - -build_component_ut_esp32s3: - extends: - - .build_component_ut_template - - .rules:build:component_ut-esp32s3 - variables: - IDF_TARGET: esp32s3 - -build_component_ut_esp32c3: - extends: - - .build_component_ut_template - - .rules:build:component_ut-esp32c3 - variables: - IDF_TARGET: esp32c3 - -build_component_ut_esp32c2: - extends: - - .build_component_ut_template - - .rules:build:component_ut-esp32c2 - variables: - IDF_TARGET: esp32c2 - .test_build_system_template: stage: host_test extends: diff --git a/.gitlab/ci/pre_check.yml b/.gitlab/ci/pre_check.yml index e6e2e22fd9..14c9c6a4d0 100644 --- a/.gitlab/ci/pre_check.yml +++ b/.gitlab/ci/pre_check.yml @@ -150,23 +150,19 @@ scan_tests: paths: - $EXAMPLE_TEST_OUTPUT_DIR - $TEST_APPS_OUTPUT_DIR - - $COMPONENT_UT_OUTPUT_DIR variables: EXAMPLE_TEST_DIR: ${CI_PROJECT_DIR}/examples EXAMPLE_TEST_OUTPUT_DIR: ${CI_PROJECT_DIR}/examples/test_configs TEST_APPS_TEST_DIR: ${CI_PROJECT_DIR}/tools/test_apps TEST_APPS_OUTPUT_DIR: ${CI_PROJECT_DIR}/tools/test_apps/test_configs - COMPONENT_UT_OUTPUT_DIR: ${CI_PROJECT_DIR}/component_ut/test_configs CI_SCAN_TESTS_PY: ${CI_PROJECT_DIR}/tools/ci/python_packages/ttfw_idf/CIScanTests.py EXTRA_TEST_DIRS: >- examples/bluetooth/esp_ble_mesh/ble_mesh_console examples/bluetooth/hci/controller_hci_uart_esp32 examples/wifi/iperf script: - - set_component_ut_vars - run_cmd python $CI_SCAN_TESTS_PY example_test $EXAMPLE_TEST_DIR -b cmake --exclude examples/build_system/idf_as_lib -c $CI_TARGET_TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR --extra_test_dirs $EXTRA_TEST_DIRS - run_cmd python $CI_SCAN_TESTS_PY test_apps $TEST_APPS_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_APPS_OUTPUT_DIR - - run_cmd python $CI_SCAN_TESTS_PY component_ut $COMPONENT_UT_DIRS --exclude $COMPONENT_UT_EXCLUDES -c $CI_TARGET_TEST_CONFIG_FILE -o $COMPONENT_UT_OUTPUT_DIR # For release tag pipelines only, make sure the tag was created with 'git tag -a' so it will update # the version returned by 'git describe' diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml index db369a6a45..5955974564 100644 --- a/.gitlab/ci/target-test.yml +++ b/.gitlab/ci/target-test.yml @@ -5,6 +5,7 @@ when: always paths: - XUNIT_RESULT.xml + - /tmp/pytest-embedded/ reports: junit: XUNIT_RESULT.xml script: @@ -41,6 +42,19 @@ example_test_pytest_esp32s2_generic: - ESP32S2 - Example_GENERIC +example_test_pytest_esp32s3_generic: + extends: + - .pytest_examples_dir_template + - .rules:test:example_test-esp32s3 + needs: + - build_pytest_examples_esp32s3 + variables: + TARGET: esp32s3 + ENV_MARKER: generic + tags: + - ESP32S3 + - Example_GENERIC + example_test_pytest_esp32c3_generic: extends: - .pytest_examples_dir_template @@ -67,6 +81,89 @@ example_test_pytest_esp32c3_flash_suspend: - ESP32C3_IDF - UT_T1_Flash_Suspend +.pytest_components_dir_template: + extends: .pytest_template + variables: + TEST_DIR: components + +component_ut_pytest_esp32_generic: + extends: + - .pytest_components_dir_template + - .rules:test:component_ut-esp32 + needs: + - build_pytest_components_esp32 + variables: + TARGET: esp32 + ENV_MARKER: generic + tags: + - ESP32 + - COMPONENT_UT_GENERIC + +component_ut_pytest_esp32_ip101: + extends: + - .pytest_components_dir_template + - .rules:test:component_ut-esp32 + needs: + - build_pytest_components_esp32 + variables: + TARGET: esp32 + ENV_MARKER: ip101 + tags: + - ESP32 + - COMPONENT_UT_IP101 + +component_ut_pytest_esp32_lan8720: + extends: + - .pytest_components_dir_template + - .rules:test:component_ut-esp32 + needs: + - build_pytest_components_esp32 + variables: + TARGET: esp32 + ENV_MARKER: lan8720 + tags: + - ESP32 + - COMPONENT_UT_LAN8720 + +component_ut_pytest_esp32s2_generic: + extends: + - .pytest_components_dir_template + - .rules:test:component_ut-esp32s2 + needs: + - build_pytest_components_esp32s2 + variables: + TARGET: esp32s2 + ENV_MARKER: generic + tags: + - ESP32S2 + - COMPONENT_UT_GENERIC + +component_ut_pytest_esp32s3_generic: + extends: + - .pytest_components_dir_template + - .rules:test:component_ut-esp32s3 + needs: + - build_pytest_components_esp32s3 + variables: + TARGET: esp32s3 + ENV_MARKER: generic + tags: + - ESP32S3 + - COMPONENT_UT_GENERIC + +component_ut_pytest_esp32c3_generic: + extends: + - .pytest_components_dir_template + - .rules:test:component_ut-esp32c3 + needs: + - build_pytest_components_esp32c3 + variables: + TARGET: esp32c3 + ENV_MARKER: generic + tags: + - ESP32C3 + - COMPONENT_UT_GENERIC + # for parallel jobs, CI_JOB_NAME will be "job_name index/total" (for example, "IT_001 1/2") # we need to convert to pattern "job_name_index.yml" .define_config_file_name: &define_config_file_name | @@ -459,68 +556,6 @@ test_app_test_flash_psram_f8r8: - ESP32S3 - MSPI_F8R8 -.component_ut_template: - extends: .target_test_job_template - variables: - CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/component_ut/test_configs" - script: - - *define_config_file_name - # first test if config file exists, if not exist, exit 0 - - test -e $CONFIG_FILE || exit 0 - - set_component_ut_vars - # clone test env configs - - retry_failed git clone $TEST_ENV_CONFIG_REPO - - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs - # git clone the known failure cases repo, run test - - retry_failed git clone $KNOWN_FAILURE_CASES_REPO known_failure_cases - # run test - - cd tools/ci/python_packages/tiny_test_fw/bin - - run_cmd python Runner.py $COMPONENT_UT_DIRS -c $CONFIG_FILE -e $ENV_FILE --known_failure_cases_file $CI_PROJECT_DIR/known_failure_cases/known_failure_cases.txt - -.component_ut_esp32_template: - extends: - - .component_ut_template - - .rules:test:component_ut-esp32 - -.component_ut_esp32s2_template: - extends: - - .component_ut_template - - .rules:test:component_ut-esp32s2 - -.component_ut_esp32s3_template: - extends: - - .component_ut_template - - .rules:test:component_ut-esp32s3 - -.component_ut_esp32c3_template: - extends: - - .component_ut_template - - .rules:test:component_ut-esp32c3 - -component_ut_test_001: - extends: .component_ut_esp32_template - tags: - - ESP32 - - COMPONENT_UT_GENERIC - -component_ut_test_esp32s2: - extends: .component_ut_esp32s2_template - tags: - - ESP32S2 - - COMPONENT_UT_GENERIC - -component_ut_test_esp32s3: - extends: .component_ut_esp32s3_template - tags: - - ESP32S3 - - COMPONENT_UT_GENERIC - -component_ut_test_esp32c3: - extends: .component_ut_esp32c3_template - tags: - - ESP32C3 - - COMPONENT_UT_GENERIC - .unit_test_template: extends: .target_test_job_template variables: @@ -825,18 +860,6 @@ UT_S3_FLASH: - ESP32S3_IDF - UT_T1_ESP_FLASH -component_ut_test_ip101: - extends: .component_ut_esp32_template - tags: - - ESP32 - - COMPONENT_UT_IP101 - -component_ut_test_lan8720: - extends: .component_ut_esp32_template - tags: - - ESP32 - - COMPONENT_UT_LAN8720 - .integration_test_template: extends: - .target_test_job_template diff --git a/components/driver/test_apps/gptimer/app_test.py b/components/driver/test_apps/gptimer/app_test.py deleted file mode 100644 index 649eac8830..0000000000 --- a/components/driver/test_apps/gptimer/app_test.py +++ /dev/null @@ -1,30 +0,0 @@ -# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import glob -import os - -import ttfw_idf -from tiny_test_fw import Utility - - -@ttfw_idf.idf_component_unit_test(env_tag='COMPONENT_UT_GENERIC', target=['esp32', 'esp32s2', 'esp32s3', 'esp32c3']) -def test_component_ut_gptimer(env, _): # type: (ttfw_idf.TinyFW.Env, None) -> None - # Get the names of all configs (sdkconfig.ci.* files) - config_files = glob.glob(os.path.join(os.path.dirname(__file__), 'sdkconfig.ci.*')) - config_names = [os.path.basename(s).replace('sdkconfig.ci.', '') for s in config_files] - - # Run test once with binaries built for each config - for name in config_names: - Utility.console_log(f'Checking config "{name}"... ', end='') - dut = env.get_dut('gptimer', 'components/driver/test_apps/gptimer', app_config_name=name) - dut.start_app() - stdout = dut.expect('Press ENTER to see the list of tests', full_stdout=True) - dut.write('*') - stdout = dut.expect("Enter next test, or 'enter' to see menu", full_stdout=True, timeout=30) - ttfw_idf.ComponentUTResult.parse_result(stdout,ttfw_idf.TestFormat.UNITY_BASIC) - env.close_dut(dut.name) - Utility.console_log(f'Test config "{name}" done') - - -if __name__ == '__main__': - test_component_ut_gptimer() diff --git a/components/driver/test_apps/gptimer/pytest_gptimer.py b/components/driver/test_apps/gptimer/pytest_gptimer.py new file mode 100644 index 0000000000..a72a78e308 --- /dev/null +++ b/components/driver/test_apps/gptimer/pytest_gptimer.py @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: CC0-1.0 + +import pytest +from pytest_embedded import Dut + + +@pytest.mark.supported_targets +@pytest.mark.generic +@pytest.mark.parametrize('config', [ + 'iram_safe', + 'release', +], indirect=True) +def test_gptimer(dut: Dut) -> None: + dut.expect('Press ENTER to see the list of tests') + dut.write('*') + dut.expect_unity_test_output() diff --git a/components/driver/test_apps/legacy_timer_driver/app_test.py b/components/driver/test_apps/legacy_timer_driver/app_test.py deleted file mode 100644 index af6e79ba47..0000000000 --- a/components/driver/test_apps/legacy_timer_driver/app_test.py +++ /dev/null @@ -1,30 +0,0 @@ -# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import glob -import os - -import ttfw_idf -from tiny_test_fw import Utility - - -@ttfw_idf.idf_component_unit_test(env_tag='COMPONENT_UT_GENERIC', target=['esp32', 'esp32s2', 'esp32s3', 'esp32c3']) -def test_component_ut_legacy_timer_driver(env, _): # type: (ttfw_idf.TinyFW.Env, None) -> None - # Get the names of all configs (sdkconfig.ci.* files) - config_files = glob.glob(os.path.join(os.path.dirname(__file__), 'sdkconfig.ci.*')) - config_names = [os.path.basename(s).replace('sdkconfig.ci.', '') for s in config_files] - - # Run test once with binaries built for each config - for name in config_names: - Utility.console_log(f'Checking config "{name}"... ', end='') - dut = env.get_dut('gptimer', 'components/driver/test_apps/legacy_timer_driver', app_config_name=name) - dut.start_app() - stdout = dut.expect('Press ENTER to see the list of tests', full_stdout=True) - dut.write('*') - stdout = dut.expect("Enter next test, or 'enter' to see menu", full_stdout=True, timeout=80) - ttfw_idf.ComponentUTResult.parse_result(stdout,ttfw_idf.TestFormat.UNITY_BASIC) - env.close_dut(dut.name) - Utility.console_log(f'Test config "{name}" done') - - -if __name__ == '__main__': - test_component_ut_legacy_timer_driver() diff --git a/components/driver/test_apps/legacy_timer_driver/pytest_legacy_timer_driver.py b/components/driver/test_apps/legacy_timer_driver/pytest_legacy_timer_driver.py new file mode 100644 index 0000000000..45f6df99f5 --- /dev/null +++ b/components/driver/test_apps/legacy_timer_driver/pytest_legacy_timer_driver.py @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: CC0-1.0 + +import pytest +from pytest_embedded import Dut + + +@pytest.mark.supported_targets +@pytest.mark.generic +@pytest.mark.parametrize('config', [ + 'release', +], indirect=True) +def test_legacy_timer_driver(dut: Dut) -> None: + dut.expect('Press ENTER to see the list of tests') + dut.write('*') + dut.expect_unity_test_output(timeout=120) diff --git a/components/esp_eth/test_apps/component_ut_test.py b/components/esp_eth/test_apps/component_ut_test.py deleted file mode 100644 index 06a42ed6df..0000000000 --- a/components/esp_eth/test_apps/component_ut_test.py +++ /dev/null @@ -1,114 +0,0 @@ -import os -import re -import socket - -import tiny_test_fw -import ttfw_idf -from tiny_test_fw import Utility -from ttfw_idf import TestFormat - -try: - import typing # noqa: F401 # pylint: disable=unused-import -except ImportError: - pass - - -def configure_eth_if(func): # type: (typing.Any) -> typing.Any - def inner(*args, **kwargs): # type: (typing.Any, typing.Any) -> typing.Any - # try to determine which interface to use - netifs = os.listdir('/sys/class/net/') - target_if = '' - Utility.console_log('detected interfaces: ' + str(netifs)) - for netif in netifs: - if netif.find('eth') == 0 or netif.find('enp') == 0 or netif.find('eno') == 0: - target_if = netif - break - if target_if == '': - raise Exception('no network interface found') - Utility.console_log('Use ' + target_if + ' for testing') - so = socket.socket(socket.AF_PACKET, socket.SOCK_RAW, 0x2222) - so.bind((target_if, 0)) - - func(so, *args, **kwargs) - - so.close() - - return inner - - -@configure_eth_if -def check_eth_recv_packet(so, before_recv=None): # type: (socket.socket, typing.Any) -> None - so.settimeout(10) - if before_recv is not None: - before_recv() # If configured, execute user function just before sock recv - try: - pkt = so.recv(1024) - for i in range(128, 1024): - if pkt[i] != i & 0xff: - raise Exception('Packet content mismatch') - except Exception as e: - raise e - - -@configure_eth_if -def send_eth_packet(so, mac): # type: (socket.socket, bytes) -> None - so.settimeout(10) - pkt = bytearray() - pkt += mac # dest - pkt += so.getsockname()[4] # src - pkt += bytes.fromhex('2222') # proto - pkt += bytes(1010) # padding to 1024 - for i in range(128, 1024): - pkt[i] = i & 0xff - try: - so.send(pkt) - except Exception as e: - raise e - - -def test_component_ut_esp_eth(env, appname): # type: (tiny_test_fw.Env, str) -> None - dut = env.get_dut('esp_eth', 'components/esp_eth/test_apps', app_config_name=appname) - dut.start_app() - stdout = dut.expect('Press ENTER to see the list of tests', full_stdout=True) - - Utility.console_log('Running test case: start_and_stop') - dut.write('"start_and_stop"') - stdout += dut.expect("Enter next test, or 'enter' to see menu", full_stdout=True) - ttfw_idf.ComponentUTResult.parse_result(stdout, test_format=TestFormat.UNITY_BASIC) - - Utility.console_log('Running test case: get_set_mac') - dut.write('"get_set_mac"') - stdout = dut.expect("Enter next test, or 'enter' to see menu", full_stdout=True) - ttfw_idf.ComponentUTResult.parse_result(stdout, test_format=TestFormat.UNITY_BASIC) - - Utility.console_log('Running test case: ethernet_broadcast_transmit') - check_eth_recv_packet(dut.write('"ethernet_broadcast_transmit"')) # Need to start the test after the socket is bound - stdout = dut.expect("Enter next test, or 'enter' to see menu", full_stdout=True) - ttfw_idf.ComponentUTResult.parse_result(stdout, test_format=TestFormat.UNITY_BASIC) - - Utility.console_log('Running test case: recv_pkt') - dut.write('"recv_pkt"') - expect_result = dut.expect(re.compile(r'([\s\S]*)DUT MAC: ([0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2})'), - timeout=10) - stdout = expect_result[0] - Utility.console_log('DUTs MAC address: {}'.format(expect_result[1])) - send_eth_packet(bytes.fromhex('ffffffffffff')) # broadcast frame - send_eth_packet(bytes.fromhex('010000000000')) # multicast frame - send_eth_packet(bytes.fromhex(expect_result[1].replace(':', ''))) # unicast frame - stdout += dut.expect("Enter next test, or 'enter' to see menu", full_stdout=True) - ttfw_idf.ComponentUTResult.parse_result(stdout, test_format=TestFormat.UNITY_BASIC) - - -@ttfw_idf.idf_component_unit_test(env_tag='COMPONENT_UT_IP101', target=['esp32']) -def test_component_ut_esp_eth_ip101(env, _): # type: (tiny_test_fw.Env, typing.Any) -> None - test_component_ut_esp_eth(env, 'ip101') - - -@ttfw_idf.idf_component_unit_test(env_tag='COMPONENT_UT_LAN8720', target=['esp32']) -def test_component_ut_esp_eth_lan8720(env, _): # type: (tiny_test_fw.Env, typing.Any) -> None - test_component_ut_esp_eth(env, 'lan8720') - - -if __name__ == '__main__': - test_component_ut_esp_eth_ip101() - test_component_ut_esp_eth_lan8720() diff --git a/components/esp_eth/test_apps/pytest_esp_eth.py b/components/esp_eth/test_apps/pytest_esp_eth.py new file mode 100644 index 0000000000..85ce618a20 --- /dev/null +++ b/components/esp_eth/test_apps/pytest_esp_eth.py @@ -0,0 +1,103 @@ +# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: CC0-1.0 + +import contextlib +import logging +import os +import socket +from typing import Iterator + +import pytest +from pytest_embedded import Dut + + +@contextlib.contextmanager +def configure_eth_if() -> Iterator[socket.socket]: + # try to determine which interface to use + netifs = os.listdir('/sys/class/net/') + logging.info('detected interfaces: %s', str(netifs)) + + target_if = '' + for netif in netifs: + if netif.find('eth') == 0 or netif.find('enp') == 0 or netif.find('eno') == 0: + target_if = netif + break + if target_if == '': + raise Exception('no network interface found') + logging.info('Use %s for testing', target_if) + + so = socket.socket(socket.AF_PACKET, socket.SOCK_RAW, 0x2222) + so.bind((target_if, 0)) + + try: + yield so + finally: + so.close() + + +def send_eth_packet(mac: bytes) -> None: + with configure_eth_if() as so: + so.settimeout(10) + pkt = bytearray() + pkt += mac # dest + pkt += so.getsockname()[4] # src + pkt += bytes.fromhex('2222') # proto + pkt += bytes(1010) # padding to 1024 + for i in range(128, 1024): + pkt[i] = i & 0xff + try: + so.send(pkt) + except Exception as e: + raise e + + +def actual_test(dut: Dut) -> None: + dut.expect_exact('Press ENTER to see the list of tests') + dut.write('\n') + + dut.expect_exact('Enter test for running.') + dut.write('"start_and_stop"') + dut.expect_unity_test_output() + + dut.expect_exact("Enter next test, or 'enter' to see menu") + dut.write('"get_set_mac"') + dut.expect_unity_test_output() + + dut.expect_exact("Enter next test, or 'enter' to see menu") + with configure_eth_if() as so: + so.settimeout(30) + dut.write('"ethernet_broadcast_transmit"') + pkt = so.recv(1024) + for i in range(128, 1024): + if pkt[i] != i & 0xff: + raise Exception('Packet content mismatch') + dut.expect_unity_test_output() + + dut.expect_exact("Enter next test, or 'enter' to see menu") + dut.write('"recv_pkt"') + res = dut.expect( + r'([\s\S]*)' + r'DUT MAC: ([0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2})' + ) + send_eth_packet(bytes.fromhex('ffffffffffff')) # broadcast frame # pylint: disable=no-value-for-parameter + send_eth_packet(bytes.fromhex('010000000000')) # multicast frame # pylint: disable=no-value-for-parameter + send_eth_packet(bytes.fromhex(res.group(2).decode('utf-8').replace(':', ''))) # unicast fram # pylint: disable=no-value-for-parameter, line-too-long # noqa + dut.expect_unity_test_output(extra_before=res.group(1)) + + +@pytest.mark.esp32 +@pytest.mark.ip101 +@pytest.mark.parametrize('config', [ + 'ip101', +], indirect=True) +def test_esp_eth_ip101(dut: Dut) -> None: + actual_test(dut) + + +@pytest.mark.esp32 +@pytest.mark.lan8720 +@pytest.mark.parametrize('config', [ + 'lan8720', +], indirect=True) +def test_esp_eth_lan8720(dut: Dut) -> None: + actual_test(dut) diff --git a/components/esp_netif/test_apps/component_ut_test.py b/components/esp_netif/test_apps/component_ut_test.py deleted file mode 100644 index 5673a09b4c..0000000000 --- a/components/esp_netif/test_apps/component_ut_test.py +++ /dev/null @@ -1,15 +0,0 @@ -from __future__ import print_function - -import ttfw_idf - - -@ttfw_idf.idf_component_unit_test(env_tag='COMPONENT_UT_GENERIC') -def test_component_ut_esp_netif(env, extra_data): - dut = env.get_dut('esp_netif', 'components/esp_netif/test_apps') - dut.start_app() - stdout = dut.expect('Tests finished', full_stdout=True) - ttfw_idf.ComponentUTResult.parse_result(stdout) - - -if __name__ == '__main__': - test_component_ut_esp_netif() diff --git a/components/esp_netif/test_apps/pytest_esp_netif.py b/components/esp_netif/test_apps/pytest_esp_netif.py new file mode 100644 index 0000000000..bdc35cc9d7 --- /dev/null +++ b/components/esp_netif/test_apps/pytest_esp_netif.py @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: CC0-1.0 + +import pytest +from pytest_embedded import Dut + + +@pytest.mark.esp32s2 +@pytest.mark.generic +def test_esp_netif(dut: Dut) -> None: + dut.expect_unity_test_output() diff --git a/components/newlib/test_apps/app_test.py b/components/newlib/test_apps/app_test.py deleted file mode 100644 index 7ea8e5428d..0000000000 --- a/components/newlib/test_apps/app_test.py +++ /dev/null @@ -1,19 +0,0 @@ -import tiny_test_fw # noqa: F401 # pylint: disable=unused-import -import ttfw_idf - -try: - import typing # noqa: F401 # pylint: disable=unused-import -except ImportError: - pass - - -@ttfw_idf.idf_component_unit_test(env_tag='COMPONENT_UT_GENERIC', target=['esp32', 'esp32s2', 'esp32s3', 'esp32c3']) -def test_component_ut_newlib(env, _): # type: (tiny_test_fw.Env, typing.Any) -> None - dut = env.get_dut('newlib', 'components/newlib/test_apps') - dut.start_app() - stdout = dut.expect('Tests finished, rc=0', full_stdout=True) - ttfw_idf.ComponentUTResult.parse_result(stdout) - - -if __name__ == '__main__': - test_component_ut_newlib() diff --git a/components/newlib/test_apps/pytest_newlib.py b/components/newlib/test_apps/pytest_newlib.py new file mode 100644 index 0000000000..81a2f579b4 --- /dev/null +++ b/components/newlib/test_apps/pytest_newlib.py @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: CC0-1.0 + +import pytest +from pytest_embedded import Dut + + +@pytest.mark.supported_targets +@pytest.mark.generic +def test_newlib(dut: Dut) -> None: + dut.expect_unity_test_output() diff --git a/components/wear_levelling/test_apps/component_ut_test.py b/components/wear_levelling/test_apps/component_ut_test.py deleted file mode 100644 index 899869887e..0000000000 --- a/components/wear_levelling/test_apps/component_ut_test.py +++ /dev/null @@ -1,28 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import glob -import os - -import ttfw_idf -from tiny_test_fw import Utility - - -@ttfw_idf.idf_component_unit_test(env_tag='COMPONENT_UT_GENERIC', target=['esp32', 'esp32c3']) -def test_component_ut_wear_levelling(env, _): # type: (ttfw_idf.TinyFW.Env, None) -> None - # Get the names of all configs (sdkconfig.ci.* files) - config_files = glob.glob(os.path.join(os.path.dirname(__file__), 'sdkconfig.ci.*')) - config_names = [os.path.basename(s).replace('sdkconfig.ci.', '') for s in config_files] - - # Run test once with binaries built for each config - for name in config_names: - Utility.console_log("Checking config \"{}\"... ".format(name), end='') - dut = env.get_dut('wear_levelling', 'components/wear_levelling/test_apps', app_config_name=name) - dut.start_app() - stdout = dut.expect('Tests finished', full_stdout=True, timeout=30) - ttfw_idf.ComponentUTResult.parse_result(stdout) - env.close_dut(dut.name) - Utility.console_log('done') - - -if __name__ == '__main__': - test_component_ut_wear_levelling() diff --git a/components/wear_levelling/test_apps/pytest_wear_levelling.py b/components/wear_levelling/test_apps/pytest_wear_levelling.py new file mode 100644 index 0000000000..0261cd412e --- /dev/null +++ b/components/wear_levelling/test_apps/pytest_wear_levelling.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: CC0-1.0 + +import pytest +from pytest_embedded import Dut + + +@pytest.mark.esp32 +@pytest.mark.esp32c3 +@pytest.mark.generic +@pytest.mark.parametrize('config', [ + '4k', + '512perf', + '512safe', + 'release', +], indirect=True) +def test_wear_levelling(dut: Dut) -> None: + dut.expect_unity_test_output() diff --git a/conftest.py b/conftest.py index 9823995042..fe107e7ffa 100644 --- a/conftest.py +++ b/conftest.py @@ -16,17 +16,19 @@ import logging import os import sys +import xml.etree.ElementTree as ET from typing import Callable, List, Optional import pytest from _pytest.config import Config from _pytest.fixtures import FixtureRequest from _pytest.nodes import Item +from _pytest.python import Function from pytest_embedded.plugin import parse_configuration -from pytest_embedded_idf.app import IdfApp +from pytest_embedded.utils import find_by_suffix SUPPORTED_TARGETS = ['esp32', 'esp32s2', 'esp32c3', 'esp32s3'] -PREVIEW_TARGETS = ['linux', 'esp32h2', 'esp8684'] +PREVIEW_TARGETS = ['linux', 'esp32h2', 'esp32c2'] ################## @@ -42,7 +44,7 @@ def is_target_marker(marker: str) -> bool: return False -def format_case_id(target: str, config: str, case: str) -> str: +def format_case_id(target: Optional[str], config: Optional[str], case: str) -> str: return f'{target}.{config}.{case}' @@ -58,6 +60,11 @@ def config(request: FixtureRequest) -> str: return getattr(request, 'param', None) or request.config.getoption('config', 'default') # type: ignore +@pytest.fixture +def test_case_name(request: FixtureRequest, target: str, config: str) -> str: + return format_case_id(target, config, request.node.originalname) + + @pytest.fixture @parse_configuration def build_dir(request: FixtureRequest, app_path: str, target: Optional[str], config: Optional[str]) -> str: @@ -106,34 +113,58 @@ def build_dir(request: FixtureRequest, app_path: str, target: Optional[str], con @pytest.fixture(autouse=True) -def junit_properties(app: IdfApp, config: str, test_case_name: str, - record_xml_attribute: Callable[[str, object], None]) -> None: +def junit_properties(test_case_name: str, record_xml_attribute: Callable[[str, object], None]) -> None: """ This fixture is autoused and will modify the junit report test case name to .. """ - record_xml_attribute('name', format_case_id(app.target, config, test_case_name)) + record_xml_attribute('name', test_case_name) ################## # Hook functions # ################## -@pytest.hookimpl(trylast=True) +@pytest.hookimpl(tryfirst=True) def pytest_collection_modifyitems(config: Config, items: List[Item]) -> None: - target = config.getoption('target', None) + target = config.getoption('target', None) # use the `build` dir if not target: return # add markers for special markers for item in items: if 'supported_targets' in item_marker_names(item): - for target in SUPPORTED_TARGETS: - item.add_marker(target) + for _target in SUPPORTED_TARGETS: + item.add_marker(_target) if 'preview_targets' in item_marker_names(item): - for target in PREVIEW_TARGETS: - item.add_marker(target) + for _target in PREVIEW_TARGETS: + item.add_marker(_target) if 'all_targets' in item_marker_names(item): - for target in [*SUPPORTED_TARGETS, *PREVIEW_TARGETS]: - item.add_marker(target) + for _target in [*SUPPORTED_TARGETS, *PREVIEW_TARGETS]: + item.add_marker(_target) # filter all the test cases with "--target" items[:] = [item for item in items if target in item_marker_names(item)] + + +@pytest.hookimpl(trylast=True) +def pytest_runtest_teardown(item: Function) -> None: + """ + Format the test case generated junit reports + """ + tempdir = item.funcargs.get('test_case_tempdir') + if not tempdir: + return + + junits = find_by_suffix('.xml', tempdir) + if not junits: + return + + target = item.funcargs['target'] + config = item.funcargs['config'] + for junit in junits: + xml = ET.parse(junit) + testcases = xml.findall('.//testcase') + for case in testcases: + case.attrib['name'] = format_case_id(target, config, case.attrib['name']) + if 'file' in case.attrib: + case.attrib['file'] = case.attrib['file'].replace('/IDF/', '') # our unity test framework + xml.write(junit) diff --git a/examples/peripherals/timer_group/gptimer/pytest_gptimer.py b/examples/peripherals/timer_group/gptimer/pytest_gptimer_example.py similarity index 100% rename from examples/peripherals/timer_group/gptimer/pytest_gptimer.py rename to examples/peripherals/timer_group/gptimer/pytest_gptimer_example.py diff --git a/pytest.ini b/pytest.ini index 4ae83452d8..7386f2fba8 100644 --- a/pytest.ini +++ b/pytest.ini @@ -13,11 +13,15 @@ markers = esp32s2: support esp32s2 target esp32s3: support esp32s3 target esp32c3: support esp32c3 target + supported_targets: support all supported targets ('esp32', 'esp32s2', 'esp32c3', 'esp32s3') + preview_targets: support all preview targets ('linux', 'esp32h2', 'esp32c2') + all_targets: support all targets, including supported ones and preview ones + + # env markers generic: tests should be run on generic runners flash_suspend: support flash suspend feature - supported_targets: support all supported targets ('esp32', 'esp32s2', 'esp32c3', 'esp32s3') - preview_targets: support all preview targets ('linux', 'esp32h2', 'esp8684') - all_targets: support all targets, including supported ones and preview ones + ip101: connected via wired 10/100M ethernet + lan8720: connected via LAN8720 ethernet transceiver # log related log_cli = True diff --git a/tools/ci/build_pytest_apps.py b/tools/ci/build_pytest_apps.py index a7595bb443..f26e92fdb6 100644 --- a/tools/ci/build_pytest_apps.py +++ b/tools/ci/build_pytest_apps.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 """ @@ -27,7 +27,7 @@ except ImportError: def main(args: argparse.Namespace) -> None: if args.all_pytest_apps: - paths = get_pytest_dirs(IDF_PATH, args.under_dir) + paths = get_pytest_dirs(args.under_dir) args.recursive = True elif args.paths is None: paths = [os.getcwd()] diff --git a/tools/ci/idf_ci_utils.py b/tools/ci/idf_ci_utils.py index f915509366..42e1673acd 100644 --- a/tools/ci/idf_ci_utils.py +++ b/tools/ci/idf_ci_utils.py @@ -8,7 +8,7 @@ import logging import os import subprocess import sys -from typing import List, Optional +from typing import List IDF_PATH = os.path.abspath(os.getenv('IDF_PATH', os.path.join(os.path.dirname(__file__), '..', '..'))) @@ -86,7 +86,7 @@ def is_in_directory(file_path: str, folder: str) -> bool: return os.path.realpath(file_path).startswith(os.path.realpath(folder) + os.sep) -def get_pytest_dirs(folder: str, under_dir: Optional[str] = None) -> List[str]: +def get_pytest_dirs(folder: str) -> List[str]: from io import StringIO import pytest @@ -109,7 +109,4 @@ def get_pytest_dirs(folder: str, under_dir: Optional[str] = None) -> List[str]: test_file_paths = set(node.fspath for node in collector.nodes) - if under_dir: - return [os.path.dirname(file) for file in test_file_paths if is_in_directory(file, under_dir)] - return [os.path.dirname(file) for file in test_file_paths] diff --git a/tools/ci/utils.sh b/tools/ci/utils.sh index 3db3537224..42b2a83373 100644 --- a/tools/ci/utils.sh +++ b/tools/ci/utils.sh @@ -42,13 +42,6 @@ function get_all_submodules() { git config --file .gitmodules --get-regexp path | awk '{ print $2 }' | sed -e 's|$|/**|' | xargs | sed -e 's/ /,/g' } -function set_component_ut_vars() { - local exclude_list_fp="${IDF_PATH}/tools/ci/component_ut_excludes.txt" - export COMPONENT_UT_DIRS=$(find components/ -name test_apps -type d) - export COMPONENT_UT_EXCLUDES=$([ -r $exclude_list_fp ] && cat $exclude_list_fp | xargs) - echo "COMPONENT_UT_DIRS, COMPONENT_UT_EXCLUDES written into export" -} - function error() { printf "\033[0;31m%s\n\033[0m" "${1}" >&2 }