diff --git a/.gitlab/ci/assign-test.yml b/.gitlab/ci/assign-test.yml index 3da76262d2..d470655708 100644 --- a/.gitlab/ci/assign-test.yml +++ b/.gitlab/ci/assign-test.yml @@ -44,36 +44,6 @@ assign_example_test: TEST_TYPE: example_test TEST_DIR: examples -assign_custom_test: - extends: - - .assign_test_template - - .rules:build:custom_test - needs: - - job: build_test_apps_esp32 - artifacts: false - optional: true - - job: build_test_apps_esp32s2 - artifacts: false - optional: true - - job: build_test_apps_esp32c2 - artifacts: false - optional: true - - job: build_test_apps_esp32c3 - artifacts: false - optional: true - - job: build_test_apps_esp32c6 - artifacts: false - optional: true - - job: build_test_apps_esp32h2 - artifacts: false - optional: true - - job: build_test_apps_esp32s3 - artifacts: false - optional: true - variables: - TEST_TYPE: custom_test - TEST_DIR: tools/test_apps - assign_unit_test: extends: - .assign_test_template diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 534bb91a00..e4021f1efb 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -867,38 +867,6 @@ - <<: *if-dev-push changes: *patterns-target_test-wifi -.rules:build:custom_test: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-label-custom_test - - <<: *if-label-custom_test_esp32 - - <<: *if-label-custom_test_esp32c2 - - <<: *if-label-custom_test_esp32c3 - - <<: *if-label-custom_test_esp32c6 - - <<: *if-label-custom_test_esp32h2 - - <<: *if-label-custom_test_esp32h4 - - <<: *if-label-custom_test_esp32s2 - - <<: *if-label-custom_test_esp32s3 - - <<: *if-label-target_test - - <<: *if-label-weekend_test - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-custom_test - - <<: *if-dev-push - changes: *patterns-downloadable-tools - - <<: *if-dev-push - changes: *patterns-target_test-adc - - <<: *if-dev-push - changes: *patterns-target_test-i154 - - <<: *if-dev-push - changes: *patterns-target_test-wifi - .rules:build:custom_test-esp32: rules: - <<: *if-revert-branch @@ -1847,12 +1815,6 @@ when: never - <<: *if-label-nvs_coverage -.rules:labels:weekend_test: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-label-weekend_test - .rules:test:component_ut-esp32: rules: - <<: *if-revert-branch diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml index ff4fbf86dc..fecebf84e4 100644 --- a/.gitlab/ci/target-test.yml +++ b/.gitlab/ci/target-test.yml @@ -1074,19 +1074,6 @@ test_app_test_pytest_esp32s3_mspi_f4r4: TEST_CASE_PATH: "$CI_PROJECT_DIR/examples" CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/examples/test_configs" -test_weekend_mqtt: - extends: - - .test_app_esp32_template - - .rules:labels:weekend_test - tags: - - ESP32 - - ethernet_router - script: - - export MQTT_PUBLISH_TEST=1 - - export TEST_PATH=$CI_PROJECT_DIR/tools/test_apps/protocols/mqtt/publish_connect_test - - cd $IDF_PATH/tools/ci/python_packages/tiny_test_fw/bin - - run_cmd python Runner.py $TEST_PATH -c $TEST_PATH/publish_connect_mqtt_.yml - .example_test_esp32_template: extends: - .example_test_template @@ -1192,50 +1179,6 @@ example_test_H2_GENERIC: - ESP32H2 - Example_GENERIC -.test_app_template: - extends: .target_test_job_template - needs: - - assign_custom_test - variables: - GIT_DEPTH: 1 - TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/test_apps" - CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/tools/test_apps/test_configs" - -.test_app_esp32_template: - extends: - - .test_app_template - - .rules:test:custom_test-esp32 - -.test_app_esp32s2_template: - extends: - - .test_app_template - - .rules:test:custom_test-esp32s2 - -.test_app_esp32c2_template: - extends: - - .test_app_template - - .rules:test:custom_test-esp32c2 - -.test_app_esp32c3_template: - extends: - - .test_app_template - - .rules:test:custom_test-esp32c3 - -.test_app_esp32c6_template: - extends: - - .test_app_template - - .rules:test:custom_test-esp32c6 - -.test_app_esp32h2_template: - extends: - - .test_app_template - - .rules:test:custom_test-esp32h2 - -.test_app_esp32s3_template: - extends: - - .test_app_template - - .rules:test:custom_test-esp32s3 - .unit_test_template: extends: .target_test_job_template needs: # the assign already needs all the build jobs diff --git a/tools/test_apps/README.md b/tools/test_apps/README.md index 0fbe6be84b..40e7698a1f 100644 --- a/tools/test_apps/README.md +++ b/tools/test_apps/README.md @@ -12,9 +12,7 @@ Test applications are treated the same way as ESP-IDF examples, so each project - Build recipe in cmake and the main component with app sources - Configuration files, `sdkconfig.ci` and similar (see below) -- Test executor in `ttfw_idf` format if the project is intended to also run tests (otherwise the example is build only) - - test file in the project dir must end with `_test.py`, by should be named `app_test.py` - - test cases shall be decorated with `@ttfw_idf.idf_custom_test(env_tag="...")` +- Python test scripts, `pytest_....py` (optional) ## Test Apps layout @@ -39,52 +37,6 @@ The CI system expects to see at least a "default" config, so add `sdkconfig.ci` - By default, every CI configurations is built for every target SoC (an `m * n` configuration matrix). However if any `sdkconfig.ci.*` file contains a line of the form `CONFIG_IDF_TARGET="targetname"` then that CI config is only built for that one target. This only works in `sdkconfig.ci.CONFIG`, not in the default `sdkconfig.ci`. - Each configuration is also built with the contents of any `sdkconfig.defaults` file or a file named `sdkconfig.defaults.` appended. (Same as a normal ESP-IDF project build.) -### Test Execution - -If an example test or test app test supports more targets than just `ESP32`, then the `app_test.py` file needs to specify the list of supported targets in the test decorator. For example: - -```python -@ttfw_idf.idf_example_test(env_tag='Example_GENERIC', target=['esp32', 'esp32s2']) -def test_app_xyz(env, extra_data): -``` - -If the app test supports multiple targets but you only want some of these targets to be run automatically in CI, the list can be further filtered down by adding the `ci_target` list: - -```python -@ttfw_idf.idf_example_test(env_tag='Example_GENERIC', target=['esp32', 'esp32s2'], ci_target=['esp32']) -def test_app_xyz(env, extra_data): -``` - -(If no ci_target list is specified, all supported targets will be tested in CI.) - -## Test Apps local execution (ttfw) - -All the following instructions are general. Part of them may be complemented by more particular instructions in the corresponding app's README. - -### Requirements - -Install Python dependencies and export the Python path where the IDF CI Python modules are found with the following commands: - -```bash -bash install.sh --enable-ttfw -source export.sh -export PYTHONPATH=$IDF_PATH/tools/ci/python_packages:$PYTHONPATH -``` -You should also set the port via the environment variable `ESPPORT` to prevent the tools from looking and iterating over all serial ports. The latter causes much trouble, currently: - -``` -export ESPPORT=/dev/ttyUSB -``` - -### Execution - -- Create an sdkconfig file from the relevant `sdkconfig.ci.` and `sdkconfig.defaults`: `cat sdkconfig.defaults sdkconfig.ci. > sdkconfig` -- Run `idf.py menuconfig` to configure local project attributes -- Run `idf.py build` to build the test app -- Run `python app_test.py` to run the test locally - -## Test Apps local execution (pytest) +## Test Apps local execution Some of the examples have `pytest_....py` scripts that are using the `pytest` as the test framework. For detailed information, please refer to the "Run the Tests Locally" Section under [ESP-IDF tests in Pytest documentation](../../docs/en/contribute/esp-idf-tests-with-pytest.rst) - -Using `pytest` is the recommended way to write new tests. We will migrate all the test apps scripts to this new framework soon. diff --git a/tools/test_apps/protocols/mqtt/publish_connect_test/publish_connect_mqtt_.yml b/tools/test_apps/protocols/mqtt/publish_connect_test/publish_connect_mqtt_.yml deleted file mode 100644 index 70e13ef44e..0000000000 --- a/tools/test_apps/protocols/mqtt/publish_connect_test/publish_connect_mqtt_.yml +++ /dev/null @@ -1,2 +0,0 @@ -CaseConfig: -- name: test_app_protocol_mqtt_publish_connect diff --git a/tools/test_apps/protocols/mqtt/publish_connect_test/publish_connect_mqtt_qemu.yml b/tools/test_apps/protocols/mqtt/publish_connect_test/publish_connect_mqtt_qemu.yml deleted file mode 100644 index 06f1235238..0000000000 --- a/tools/test_apps/protocols/mqtt/publish_connect_test/publish_connect_mqtt_qemu.yml +++ /dev/null @@ -1,6 +0,0 @@ -CaseConfig: -- name: test_app_protocol_mqtt_publish_connect - overwrite: - dut: - class: ESP32QEMUDUT - package: ttfw_idf diff --git a/tools/test_apps/security/secure_boot/README.md b/tools/test_apps/security/secure_boot/README.md index 2d5c3f025c..70190ea2ff 100644 --- a/tools/test_apps/security/secure_boot/README.md +++ b/tools/test_apps/security/secure_boot/README.md @@ -95,21 +95,22 @@ Under `Security features` - Set UART ROM download mode to ENABLED (Required for the script to read the EFUSE) -- Install and export TTFW requirements -``` -python -m pip install -r $IDF_PATH/tools/ci/python_packages/ttfw_idf/requirements.txt +- Install pytest requirements -export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci/python_packages" -``` + ``` + bash $IDF_PATH/install.sh --enable-pytest + ``` ### Build and test - Build the example -``` -idf.py build -``` + + ``` + idf.py build + ``` - Run the example test -``` -python example_test.py -``` + + ``` + pytest --target esp32c3 + ```