From 8132f8b5e17efcefe208345e125a76c41f17bb63 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Tue, 26 Jul 2022 16:00:43 +0800 Subject: [PATCH 1/7] ci: add esp32h2 marker to pytest.ini --- pytest.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/pytest.ini b/pytest.ini index 497169b9f0..bc2221aeb9 100644 --- a/pytest.ini +++ b/pytest.ini @@ -23,6 +23,7 @@ markers = esp32s3: support esp32s3 target esp32c3: support esp32c3 target esp32c2: support esp32c2 target + esp32h2: support esp32h2 target supported_targets: support all supported targets ('esp32', 'esp32s2', 'esp32c3', 'esp32s3', 'esp32c2') preview_targets: support all preview targets ('linux', 'esp32h2') all_targets: support all targets, including supported ones and preview ones From 742663dd9b585e1918ccca4deb23e68253f9f5d7 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Tue, 26 Jul 2022 15:58:50 +0800 Subject: [PATCH 2/7] feat: add requirements.docs.txt --- docs/requirements.txt | 1 - tools/requirements.json | 6 ++++++ tools/requirements/requirements.docs.txt | 4 ++++ 3 files changed, 10 insertions(+), 1 deletion(-) delete mode 100644 docs/requirements.txt create mode 100644 tools/requirements/requirements.docs.txt diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index 2e0bc6cdc9..0000000000 --- a/docs/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -esp-docs~=1.1.0 diff --git a/tools/requirements.json b/tools/requirements.json index df1993c7b3..bfc09a428c 100644 --- a/tools/requirements.json +++ b/tools/requirements.json @@ -18,6 +18,12 @@ "description": "Packages for CI with pytest", "optional": true, "requirement_path": "tools/requirements/requirements.pytest.txt" + }, + { + "name": "docs", + "description": "Packages for building ESP-IDF documentation", + "optional": true, + "requirement_path": "tools/requirements/requirements.docs.txt" } ] } diff --git a/tools/requirements/requirements.docs.txt b/tools/requirements/requirements.docs.txt new file mode 100644 index 0000000000..17534adfe3 --- /dev/null +++ b/tools/requirements/requirements.docs.txt @@ -0,0 +1,4 @@ +# Python package requirements for building the documentation of ESP-IDF. +# This feature can be enabled by running "install.{sh,bat,ps1,fish} --enable-docs" + +esp-docs From 9c6e636c0a96fe21c00f90926a789f4660b7e8ca Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Fri, 29 Jul 2022 14:42:08 +0800 Subject: [PATCH 3/7] ci: `--preserve-all` is mandatory to run locally for unit-test apps --- .gitlab/ci/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index 0c56fc5024..80b3ae9fec 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -300,11 +300,11 @@ build_ssc_esp32s3: -t $IDF_TARGET --config "configs/*=" --copy-sdkconfig + --preserve-all --collect-size-info $SIZE_INFO_LOCATION --collect-app-info list_job_${CI_NODE_INDEX:-1}.json --parallel-count ${CI_NODE_TOTAL:-1} --parallel-index ${CI_NODE_INDEX:-1} - --preserve-all - run_cmd python tools/unit-test-app/tools/UnitTestParser.py tools/unit-test-app ${CI_NODE_INDEX:-1} build_esp_idf_tests_cmake_esp32: From 01baaf33ed5ceae6c3ce3b9d50f18f79aed94e15 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Fri, 29 Jul 2022 17:49:27 +0800 Subject: [PATCH 4/7] feat: add requirements.ci.txt --- tools/requirements.json | 6 ++++++ tools/requirements/requirements.ci.txt | 9 +++++++++ tools/requirements/requirements.pytest.txt | 8 ++++++++ 3 files changed, 23 insertions(+) create mode 100644 tools/requirements/requirements.ci.txt diff --git a/tools/requirements.json b/tools/requirements.json index bfc09a428c..ac5ced813a 100644 --- a/tools/requirements.json +++ b/tools/requirements.json @@ -19,6 +19,12 @@ "optional": true, "requirement_path": "tools/requirements/requirements.pytest.txt" }, + { + "name": "ci", + "description": "Packages for ESP-IDF CI scripts", + "optional": true, + "requirement_path": "tools/requirements/requirements.ci.txt" + }, { "name": "docs", "description": "Packages for building ESP-IDF documentation", diff --git a/tools/requirements/requirements.ci.txt b/tools/requirements/requirements.ci.txt new file mode 100644 index 0000000000..b183644fec --- /dev/null +++ b/tools/requirements/requirements.ci.txt @@ -0,0 +1,9 @@ +# Python package requirements for CI in ESP-IDF. +# This feature can be enabled by running "install.{sh,bat,ps1,fish} --enable-ci" + +# ci +coverage +idf-build-apps +junit_xml +python-gitlab +pyyaml diff --git a/tools/requirements/requirements.pytest.txt b/tools/requirements/requirements.pytest.txt index fbf994b1cb..fb84dc624f 100644 --- a/tools/requirements/requirements.pytest.txt +++ b/tools/requirements/requirements.pytest.txt @@ -1,7 +1,15 @@ +# Python package requirements for pytest in ESP-IDF. +# This feature can be enabled by running "install.{sh,bat,ps1,fish} --enable-pytest" + pytest-embedded-serial-esp pytest-embedded-idf pytest-embedded-qemu pytest-rerunfailures + +# build +idf-build-apps + +# dependencies in pytest test scripts scapy websocket-client netifaces From d0a2849b02e82f86d2f55c63c95cec7022b30879 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Fri, 29 Jul 2022 17:48:29 +0800 Subject: [PATCH 5/7] feat: add requirements.ttfw.txt --- tools/ble/requirements.txt | 3 -- .../idf_iperf_test_util/requirements.txt | 1 - .../tiny_test_fw/requirements.txt | 4 --- .../python_packages/ttfw_idf/requirements.txt | 4 --- tools/esp_prov/requirements.txt | 4 --- tools/requirements.json | 6 ++++ tools/requirements/requirements.ttfw.txt | 35 +++++++++++++++++++ 7 files changed, 41 insertions(+), 16 deletions(-) delete mode 100644 tools/ble/requirements.txt delete mode 100644 tools/ci/python_packages/idf_iperf_test_util/requirements.txt delete mode 100644 tools/ci/python_packages/tiny_test_fw/requirements.txt delete mode 100644 tools/ci/python_packages/ttfw_idf/requirements.txt delete mode 100644 tools/esp_prov/requirements.txt create mode 100644 tools/requirements/requirements.ttfw.txt diff --git a/tools/ble/requirements.txt b/tools/ble/requirements.txt deleted file mode 100644 index 8049fd2f9b..0000000000 --- a/tools/ble/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -future -dbus-python -pygobject diff --git a/tools/ci/python_packages/idf_iperf_test_util/requirements.txt b/tools/ci/python_packages/idf_iperf_test_util/requirements.txt deleted file mode 100644 index 51f02f030f..0000000000 --- a/tools/ci/python_packages/idf_iperf_test_util/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -pyecharts diff --git a/tools/ci/python_packages/tiny_test_fw/requirements.txt b/tools/ci/python_packages/tiny_test_fw/requirements.txt deleted file mode 100644 index 2a742540e2..0000000000 --- a/tools/ci/python_packages/tiny_test_fw/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -pyserial -pyyaml -junit_xml -netifaces diff --git a/tools/ci/python_packages/ttfw_idf/requirements.txt b/tools/ci/python_packages/ttfw_idf/requirements.txt deleted file mode 100644 index facf202666..0000000000 --- a/tools/ci/python_packages/ttfw_idf/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ --r ../tiny_test_fw/requirements.txt -pexpect -python-gitlab -pygdbmi>=0.9.0.0 diff --git a/tools/esp_prov/requirements.txt b/tools/esp_prov/requirements.txt deleted file mode 100644 index 4c5fb27240..0000000000 --- a/tools/esp_prov/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -bleak -future -cryptography -protobuf diff --git a/tools/requirements.json b/tools/requirements.json index ac5ced813a..50a1b31efb 100644 --- a/tools/requirements.json +++ b/tools/requirements.json @@ -19,6 +19,12 @@ "optional": true, "requirement_path": "tools/requirements/requirements.pytest.txt" }, + { + "name": "ttfw", + "description": "Packages for CI with ttfw", + "optional": true, + "requirement_path": "tools/requirements/requirements.ttfw.txt" + }, { "name": "ci", "description": "Packages for ESP-IDF CI scripts", diff --git a/tools/requirements/requirements.ttfw.txt b/tools/requirements/requirements.ttfw.txt new file mode 100644 index 0000000000..ed7cb2187f --- /dev/null +++ b/tools/requirements/requirements.ttfw.txt @@ -0,0 +1,35 @@ +# Python package requirements for CI in ESP-IDF. +# This feature can be enabled by running "install.{sh,bat,ps1,fish} --enable-ttfw" + +# build +idf-build-apps + +# ttfw +pyserial +pyyaml +junit_xml +netifaces + +# ttfw-idf +pexpect +python-gitlab +pygdbmi + +# ble +future +dbus-python; sys_platform != 'win32' +pygobject; sys_platform != 'win32' + +# iperf_test_util +pyecharts + +# esp_prov +bleak +# future # addressed before under ble +protobuf + +# tools/test_apps/system/monitor_ide_integration +SimpleWebSocketServer + +# py_debug_backend +debug_backend From 17e312f87b8beb5e3584ea7f11cdded5b07e4a47 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Fri, 29 Jul 2022 14:42:54 +0800 Subject: [PATCH 6/7] docs: improve the installation instructions --- examples/README.md | 13 +++++----- .../ble_l2cap_coc/coc_blecent/README.md | 2 +- .../ble_l2cap_coc/coc_bleprph/README.md | 4 +-- .../nimble/ble_spp/spp_client/README.md | 2 +- .../nimble/ble_spp/spp_server/README.md | 2 +- examples/bluetooth/nimble/blecent/README.md | 2 +- examples/bluetooth/nimble/blehr/README.md | 2 +- examples/bluetooth/nimble/bleprph/README.md | 2 +- examples/provisioning/wifi_prov_mgr/README.md | 2 +- tools/esp_prov/README.md | 9 ++++--- tools/test_apps/README.md | 12 +++++---- tools/unit-test-app/README.md | 25 +++++++++++++++---- 12 files changed, 48 insertions(+), 29 deletions(-) diff --git a/examples/README.md b/examples/README.md index ac20d232f3..2bc0387e29 100644 --- a/examples/README.md +++ b/examples/README.md @@ -42,14 +42,15 @@ Building an example is the same as building any other project: ## Running Test Python Script (ttfw) -Some of the examples have `..._test.py` scripts that are used to test that the example works as expected. These scripts run automatically in the internal test queue. They are not intended to be run by ESP-IDF users but sometimes you may want to run them locally. The following requirements must be met in the IDF python virtual environment. +Some of the examples have `..._test.py` scripts that are used to test that the example works as expected. These scripts run automatically in the internal test queue. They are not intended to be run by ESP-IDF users but sometimes you may want to run them locally. -- ttfw needs to be in the `PYTHONPATH`. Add it like this: `export PYTHONPATH=$PYTHONPATH:$IDF_PATH/tools/ci/python_packages` -- Install all requirements from `tools/ci/python_packages/ttfw_idf/requirements.txt`: `python -m pip install -r $IDF_PATH/tools/ci/python_packages/ttfw_idf/requirements.txt` +Install Python dependencies and export the Python path where the IDF CI Python modules are found with the following commands: -These commands help solve the issue with `ModuleNotFoundError: No module named 'ttfw_idf'` and `ModuleNotFoundError: No module named 'tiny_test_fw'`. - -Some examples might fail due to other missing packages. You might need to install them manually: `pip install websocket`. +```bash +bash install.sh --enable-ttfw +source export.sh +export PYTHONPATH=$IDF_PATH/tools/ci/python_packages:$IDF_PATH/tools:$PYTHONPATH +``` ## Running Test Python Script (pytest) diff --git a/examples/bluetooth/nimble/ble_l2cap_coc/coc_blecent/README.md b/examples/bluetooth/nimble/ble_l2cap_coc/coc_blecent/README.md index 0a78f3e167..9c8d94c6cc 100644 --- a/examples/bluetooth/nimble/ble_l2cap_coc/coc_blecent/README.md +++ b/examples/bluetooth/nimble/ble_l2cap_coc/coc_blecent/README.md @@ -21,7 +21,7 @@ Maximum data of 512 bytes can be transferred over L2CAP when MTU is set to 512 a Note : -* Make sure to run `python -m pip install --user -r $IDF_PATH/requirements.txt -r $IDF_PATH/tools/ble/requirements.txt` to install the dependency packages needed. +* To install the dependency packages needed, please refer to the top level [README file](../../../../README.md#running-test-python-script-ttfw). * Currently this Python utility is only supported on Linux (BLE communication is via BLuez + DBus). ## How to Use Example diff --git a/examples/bluetooth/nimble/ble_l2cap_coc/coc_bleprph/README.md b/examples/bluetooth/nimble/ble_l2cap_coc/coc_bleprph/README.md index 25290954d2..6750175027 100644 --- a/examples/bluetooth/nimble/ble_l2cap_coc/coc_bleprph/README.md +++ b/examples/bluetooth/nimble/ble_l2cap_coc/coc_bleprph/README.md @@ -21,8 +21,8 @@ Maximum data of 512 bytes can be transferred over L2CAP when MTU is set to 512 a Note : -* Make sure to run `python -m pip install --user -r $IDF_PATH/requirements.txt -r $IDF_PATH/tools/ble/requirements.txt` to install the dependency packages needed. -* Currently this Python utility is only supported on Linux (BLE communication is via BLuez + DBus). +* To install the dependency packages needed, please refer to the top level [README file](../../../../README.md#running-test-python-script-ttfw). +* * Currently this Python utility is only supported on Linux (BLE communication is via BLuez + DBus). ## How to Use Example diff --git a/examples/bluetooth/nimble/ble_spp/spp_client/README.md b/examples/bluetooth/nimble/ble_spp/spp_client/README.md index 8ca9d3fbe8..9ad80d1d4d 100644 --- a/examples/bluetooth/nimble/ble_spp/spp_client/README.md +++ b/examples/bluetooth/nimble/ble_spp/spp_client/README.md @@ -76,7 +76,7 @@ It performs three GATT operations against the specified peer: Note : -* Make sure to run `python -m pip install --user -r $IDF_PATH/requirements.txt -r $IDF_PATH/tools/ble/requirements.txt` to install the dependency packages needed. +* To install the dependency packages needed, please refer to the top level [README file](../../../../README.md#running-test-python-script-ttfw). * Currently this Python utility is only supported on Linux (BLE communication is via BLuez + DBus). ## How to use example diff --git a/examples/bluetooth/nimble/ble_spp/spp_server/README.md b/examples/bluetooth/nimble/ble_spp/spp_server/README.md index e5cc166610..a1d7490eb4 100644 --- a/examples/bluetooth/nimble/ble_spp/spp_server/README.md +++ b/examples/bluetooth/nimble/ble_spp/spp_server/README.md @@ -76,7 +76,7 @@ It performs three GATT operations against the specified peer: Note : -* Make sure to run `python -m pip install --user -r $IDF_PATH/requirements.txt -r $IDF_PATH/tools/ble/requirements.txt` to install the dependency packages needed. +* To install the dependency packages needed, please refer to the top level [README file](../../../../README.md#running-test-python-script-ttfw). * Currently this Python utility is only supported on Linux (BLE communication is via BLuez + DBus). ## How to use example diff --git a/examples/bluetooth/nimble/blecent/README.md b/examples/bluetooth/nimble/blecent/README.md index a65de3ae83..8e20b3e3a3 100644 --- a/examples/bluetooth/nimble/blecent/README.md +++ b/examples/bluetooth/nimble/blecent/README.md @@ -27,7 +27,7 @@ A Python based utility `blecent_test.py` is also provided (which will run as a B Note : -* Make sure to run `python -m pip install --user -r $IDF_PATH/requirements.txt -r $IDF_PATH/tools/ble/requirements.txt` to install the dependency packages needed. +* To install the dependency packages needed, please refer to the top level [README file](../../../README.md#running-test-python-script-ttfw). * Currently this Python utility is only supported on Linux (BLE communication is via BLuez + DBus). ## How to Use Example diff --git a/examples/bluetooth/nimble/blehr/README.md b/examples/bluetooth/nimble/blehr/README.md index 42e19bef31..54990d94e8 100644 --- a/examples/bluetooth/nimble/blehr/README.md +++ b/examples/bluetooth/nimble/blehr/README.md @@ -17,7 +17,7 @@ A Python based utility `blehr_test.py` is also provided (which will run as a BLE Note : -* Make sure to run `python -m pip install --user -r $IDF_PATH/requirements.txt -r $IDF_PATH/tools/ble/requirements.txt` to install the dependency packages needed. +* To install the dependency packages needed, please refer to the top level [README file](../../../README.md#running-test-python-script-ttfw). * Currently this Python utility is only supported on Linux (BLE communication is via BLuez + DBus). ## How to Use Example diff --git a/examples/bluetooth/nimble/bleprph/README.md b/examples/bluetooth/nimble/bleprph/README.md index 22c3a8a662..80fa1b5d79 100644 --- a/examples/bluetooth/nimble/bleprph/README.md +++ b/examples/bluetooth/nimble/bleprph/README.md @@ -21,7 +21,7 @@ A Python based utility `bleprph_test.py` is also provided (which will run as a B Note : -* Make sure to run `python -m pip install --user -r $IDF_PATH/requirements.txt -r $IDF_PATH/tools/ble/requirements.txt` to install the dependency packages needed. +* To install the dependency packages needed, please refer to the top level [README file](../../../README.md#running-test-python-script-ttfw). * Currently this Python utility is only supported on Linux (BLE communication is via BLuez + DBus). ## How to Use Example diff --git a/examples/provisioning/wifi_prov_mgr/README.md b/examples/provisioning/wifi_prov_mgr/README.md index 21abba602a..b0410bf7b1 100644 --- a/examples/provisioning/wifi_prov_mgr/README.md +++ b/examples/provisioning/wifi_prov_mgr/README.md @@ -45,7 +45,7 @@ For iOS, a provisioning application along with source code is available on GitHu #### Platform : Linux / Windows / macOS -To provision the device running this example, the `esp_prov.py` script needs to be run (found under `$IDF_PATH/tools/esp_prov`). Make sure to satisfy all the dependencies prior to running the script (check out the `requirements.txt` file in `esp_prov` directory). +To install the dependency packages needed, please refer to the top level [README file](../../README.md#running-test-python-script-ttfw). `esp_prov` supports BLE and SoftAP transport for Linux, MacOS and Windows platforms. For BLE, however, if dependencies are not met, the script falls back to console mode and requires another application through which the communication can take place. The `esp_prov` console will guide you through the provisioning process of locating the correct BLE GATT services and characteristics, the values to write, and input read values. diff --git a/tools/esp_prov/README.md b/tools/esp_prov/README.md index deb0f878fd..3f251194f0 100644 --- a/tools/esp_prov/README.md +++ b/tools/esp_prov/README.md @@ -90,16 +90,17 @@ For Android, a provisioning tool along with source code is available [here](http ## Dependencies -This requires the following python libraries to run (included in requirements.txt): +This requires the following python libraries to run: * `bleak` * `future` * `protobuf` * `cryptography` -Run `pip install -r $IDF_PATH/tools/esp_prov/requirements.txt` +To install the dependency packages needed, please run the following command: -Note : -* The packages listed in requirements.txt are limited only to the ones needed AFTER fully satisfying the requirements of ESP-IDF +```shell +bash install.sh --enable-ttfw +``` # EXAMPLE USAGE diff --git a/tools/test_apps/README.md b/tools/test_apps/README.md index 6606544c3e..0fbe6be84b 100644 --- a/tools/test_apps/README.md +++ b/tools/test_apps/README.md @@ -63,12 +63,14 @@ All the following instructions are general. Part of them may be complemented by ### Requirements -The following requirements need to be satisfied in the IDF python virtual environment. +Install Python dependencies and export the Python path where the IDF CI Python modules are found with the following commands: -- ttfw needs to be in the `PYTHONPATH`. Add it like this: `export PYTHONPATH=$PYTHONPATH:$IDF_PATH/tools/ci/python_packages` -- Install all requirements from `tools/ci/python_packages/ttfw_idf/requirements.txt`: `pip install -r $IDF_PATH/tools/ci/python_packages/ttfw_idf/requirements.txt` - -You should also set the port via the environment variable ESPPORT to prevent the tools from looking and iterating over all serial ports. The latter causes much trouble, currently: +```bash +bash install.sh --enable-ttfw +source export.sh +export PYTHONPATH=$IDF_PATH/tools/ci/python_packages:$PYTHONPATH +``` +You should also set the port via the environment variable `ESPPORT` to prevent the tools from looking and iterating over all serial ports. The latter causes much trouble, currently: ``` export ESPPORT=/dev/ttyUSB diff --git a/tools/unit-test-app/README.md b/tools/unit-test-app/README.md index a1999fd698..590581c43c 100644 --- a/tools/unit-test-app/README.md +++ b/tools/unit-test-app/README.md @@ -39,7 +39,7 @@ Unit test uses 3 stages in CI: `build`, `assign_test`, `unit_test`. ### Build Stage: -`build_esp_idf_tests` job will build all UT configs and run script `UnitTestParser.py` to parse test cases form built elf files. Built binary (`tools/unit-test-app/output`) and parsed cases (`components/idf_test/unit_test/TestCaseAll.yml`) will be saved as artifacts. +`build_esp_idf_tests` job will build all UT configs and run script `UnitTestParser.py` to parse test cases form built elf files. Built binary (`tools/unit-test-app/build__`) and parsed cases (`components/idf_test/unit_test/TestCaseAll.yml`) will be saved as artifacts. When we add new test case, it will construct a structure to save case data during build. We'll parse the test case from this structure. The description (defined in test case: `TEST_CASE("name", "description")`) is used to extend test case definition. The format of test description is a list of tags: @@ -63,7 +63,7 @@ We will build unit-test-app with different sdkconfigs. Some config items require ### Assign Test Stage: -`assign_test` job will try to assign all cases to test jobs defined in `.gitlab-ci.yml`, according to test environment and tags. For each job, one config file with same name of test job will be generated in `components/idf_test/unit_test/CIConfigs/`(this folder will be passed to test jobs as artifacts). These config files will tell test jobs which cases it need to run, and pass some extra configs (like if the case will reset) of test case to runner. +`assign_unit_test` job will try to assign all cases to test jobs defined in `.gitlab-ci.yml`, according to test environment and tags. For each job, one config file with same name of test job will be generated in `components/idf_test/unit_test/CIConfigs/`(this folder will be passed to test jobs as artifacts). These config files will tell test jobs which cases it need to run, and pass some extra configs (like if the case will reset) of test case to runner. Please check related document in tiny-test-fw for details. @@ -110,7 +110,7 @@ First you can check the logs. It's saved as unit test job artifacts. You can dow If you want to reproduce locally, you need to: -1. Download artifacts of `build_esp_idf_tests`. The built binary is in `tools/unit-test-app/output` folder. +1. Download artifacts of `build_esp_idf_tests`. The built binary is in `tools/unit-test-app/build__` folder. * Built binary in CI could be slightly different from locally built binary with the same revision, some cases might only fails with CI built binary. 2. Check the following print in CI job to get the config name: `Running unit test for config: config_name`. Then flash the binary of this config to your board. 3. Run the failed case on your board (refer to Running Unit Tests section). @@ -124,8 +124,9 @@ If you want to reproduce locally, you need to: First, install Python dependencies and export the Python path where the IDF CI Python modules are found: ```bash -pip install -r $IDF_PATH/tools/ci/python_packages/tiny_test_fw/requirements.txt -export PYTHONPATH=$IDF_PATH/tools/ci/python_packages +bash install.sh --enable-ttfw +source export.sh +export PYTHONPATH=$IDF_PATH/tools/ci/python_packages:$PYTHONPATH ``` Change to the unit test app directory, configure the app as needed and build it in the default "build" directory. For example: @@ -138,6 +139,20 @@ idf.py build -T vfs (Instead of these steps, you can do whatever is needed to configure & build a unit test app with the tests and config that you need.) +If you want to build exactly the same binary files under the same location as they are in CI pipelines, you may run: + +```bash +cd $IDF_PATH +python tools/ci/ci_build_apps.py tools/unit-test-app -v -t $IDF_TARGET --config "configs/*=" --copy-sdkconfig --preserve-all +``` + +This would build all configs. if you want to build only one config (let's take `psram` as an example), you may use: + +```bash +cd $IDF_PATH +python tools/ci/ci_build_apps.py tools/unit-test-app -v -t $IDF_TARGET --config "configs/psram=" --copy-sdkconfig --preserve-all +``` + ### run a single test case by name ```bash From b684730da29d9261fa31798c6f1f7becf19840ee Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Fri, 29 Jul 2022 16:12:22 +0800 Subject: [PATCH 7/7] ci: use the new requirement txt will be cleaned up in IDFCI-1347 --- .gitlab-ci.yml | 15 +++++---------- .gitlab/ci/docs.yml | 8 ++++---- .gitlab/ci/upload_cache.yml | 4 +--- 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 055687c0db..357d3dddbb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -211,7 +211,7 @@ before_script: # This adds tools (compilers) and the version-specific Python environment to PATH - *setup_tools_unless_target_test # Install packages required by CI scripts into IDF Python environment - - pip install -r $IDF_PATH/tools/ci/python_packages/ttfw_idf/requirements.txt + - pip install -r $IDF_PATH/tools/requirements/requirements.ci.txt - source tools/ci/configure_ci_environment.sh # Part of tools/ci/setup_python.sh; we don't use pyenv on macOS, so can't run the rest of the script. - export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH" @@ -228,9 +228,8 @@ before_script: - *setup_tools_unless_target_test - fetch_submodules - *download_test_python_contraint_file - - $IDF_PATH/tools/idf_tools.py install-python-env --features pytest - # TODO: remove this, IDFCI-1207 - - pip install esptool -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE} + # only need ci and pytest related packages, ttfw requirements would be mocked if not installed + - $IDF_PATH/tools/idf_tools.py install-python-env --features pytest,ci - eval "$($IDF_PATH/tools/idf_tools.py export)" # use idf venv instead .before_script_build_jobs: @@ -244,13 +243,9 @@ before_script: - *setup_tools_unless_target_test - fetch_submodules - *download_test_python_contraint_file - - $IDF_PATH/tools/idf_tools.py install-python-env --features pytest - # TODO: remove this, IDFCI-1207 - - pip install esptool -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE} + # only need ci and pytest related packages, ttfw requirements would be mocked if not installed + - $IDF_PATH/tools/idf_tools.py install-python-env --features pytest,ci - eval "$($IDF_PATH/tools/idf_tools.py export)" # use idf venv instead - # not only need pytest related packages, but also needs ttfw requirements - - internal_pip_install $IDF_BUILD_APPS_PROJ idf_build_apps - - pip install -r tools/ci/python_packages/ttfw_idf/requirements.txt -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE} - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS} - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS} diff --git a/.gitlab/ci/docs.yml b/.gitlab/ci/docs.yml index 9a17a99145..c781c77a52 100644 --- a/.gitlab/ci/docs.yml +++ b/.gitlab/ci/docs.yml @@ -62,7 +62,7 @@ check_docs_lang_sync: dependencies: [] script: - cd docs - - pip install -U -r requirements.txt + - pip install -U -r $IDF_PATH/tools/requirements/requirements.docs.txt - build-docs -t $DOCTGT -bs $DOC_BUILDERS -l $DOCLANG build parallel: matrix: @@ -76,7 +76,7 @@ check_docs_gh_links: - .doc-rules:build:docs script: - cd docs - - pip install -U -r requirements.txt + - pip install -U -r $IDF_PATH/tools/requirements/requirements.docs.txt - build-docs gh-linkcheck # stage: build_doc @@ -161,7 +161,7 @@ build_docs_pdf: script: - add_doc_server_ssh_keys $DOCS_DEPLOY_PRIVATEKEY $DOCS_DEPLOY_SERVER $DOCS_DEPLOY_SERVER_USER - export GIT_VER=$(git describe --always ${PIPELINE_COMMIT_SHA} --) - - pip install -U -r docs/requirements.txt + - pip install -U -r $IDF_PATH/tools/requirements/requirements.docs.txt - deploy-docs # stage: test_deploy @@ -227,5 +227,5 @@ check_doc_links: allow_failure: true script: - cd docs - - pip install -U -r requirements.txt + - pip install -U -r $IDF_PATH/tools/requirements/requirements.docs.txt - build-docs -t $DOCTGT -l $DOCLANG linkcheck diff --git a/.gitlab/ci/upload_cache.yml b/.gitlab/ci/upload_cache.yml index 0b2f166195..b2ddb81bc4 100644 --- a/.gitlab/ci/upload_cache.yml +++ b/.gitlab/ci/upload_cache.yml @@ -24,9 +24,7 @@ upload-pip-cache: - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS - source tools/ci/setup_python.sh - rm -rf .cache/pip # clear old packages - - $IDF_PATH/tools/idf_tools.py install-python-env --features pytest - # TODO: remove this, IDFCI-1207 - - pip install esptool -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE} + - $IDF_PATH/tools/idf_tools.py install-python-env --features pytest,ci parallel: matrix: - GEO: [ 'shiny', 'brew' ]