mirror of
https://github.com/espressif/esp-idf.git
synced 2024-10-05 20:47:46 -04:00
63bd3a18ad
Introduced changes: - refactor the cli script used for report generation - introduce failed jobs report generator - cover job report generation with tests - add job failure rate - add test cases failure rate - add current branch / other branches statistic for failed jobs / testcases
401 lines
13 KiB
YAML
401 lines
13 KiB
YAML
.host_test_template:
|
|
extends: .rules:test:host_test
|
|
stage: host_test
|
|
image: $ESP_ENV_IMAGE
|
|
tags:
|
|
- host_test
|
|
dependencies: # set dependencies to null to avoid missing artifacts issue
|
|
# run host_test jobs immediately, only after upload cache
|
|
needs:
|
|
- job: upload-pip-cache
|
|
optional: true
|
|
artifacts: false
|
|
- job: upload-submodules-cache
|
|
optional: true
|
|
artifacts: false
|
|
- pipeline_variables
|
|
artifacts:
|
|
expire_in: 1 week
|
|
when: always
|
|
|
|
check_public_headers:
|
|
extends:
|
|
- .host_test_template
|
|
- .rules:build:check
|
|
script:
|
|
- IDF_TARGET=esp32 python tools/ci/check_public_headers.py --jobs 4 --prefix xtensa-esp32-elf-
|
|
- IDF_TARGET=esp32s2 python tools/ci/check_public_headers.py --jobs 4 --prefix xtensa-esp32s2-elf-
|
|
- IDF_TARGET=esp32s3 python tools/ci/check_public_headers.py --jobs 4 --prefix xtensa-esp32s3-elf-
|
|
- IDF_TARGET=esp32c3 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf-
|
|
- IDF_TARGET=esp32c2 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf-
|
|
- IDF_TARGET=esp32c6 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf-
|
|
- IDF_TARGET=esp32c5 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf-
|
|
- IDF_TARGET=esp32h2 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf-
|
|
- IDF_TARGET=esp32p4 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf-
|
|
- IDF_TARGET=esp32c61 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf-
|
|
|
|
test_nvs_on_host:
|
|
extends: .host_test_template
|
|
script:
|
|
- cd components/nvs_flash/test_nvs_host
|
|
- make test
|
|
|
|
test_nvs_coverage:
|
|
extends:
|
|
- .host_test_template
|
|
- .rules:labels:nvs_coverage
|
|
artifacts:
|
|
paths:
|
|
- components/nvs_flash/test_nvs_host/coverage_report
|
|
script:
|
|
- cd components/nvs_flash/test_nvs_host
|
|
- make coverage_report
|
|
# the 'long' host tests take approx 11 hours on our current runners. Adding some margin here for possible CPU contention
|
|
timeout: 18 hours
|
|
|
|
test_partition_table_on_host:
|
|
extends: .host_test_template
|
|
script:
|
|
- cd components/partition_table/test_gen_esp32part_host
|
|
- ./gen_esp32part_tests.py
|
|
|
|
test_ldgen_on_host:
|
|
extends: .host_test_template
|
|
script:
|
|
- cd tools/ldgen/test
|
|
- export PYTHONPATH=$PYTHONPATH:..
|
|
- python -m unittest
|
|
variables:
|
|
LC_ALL: C.UTF-8
|
|
|
|
test_reproducible_build:
|
|
extends: .host_test_template
|
|
script:
|
|
- ./tools/ci/test_reproducible_build.sh
|
|
artifacts:
|
|
when: on_failure
|
|
paths:
|
|
- "**/sdkconfig"
|
|
- "**/build*/*.bin"
|
|
- "**/build*/*.elf"
|
|
- "**/build*/*.map"
|
|
- "**/build*/flasher_args.json"
|
|
- "**/build*/*.bin"
|
|
- "**/build*/bootloader/*.bin"
|
|
- "**/build*/partition_table/*.bin"
|
|
|
|
test_spiffs_on_host:
|
|
extends: .host_test_template
|
|
script:
|
|
- cd components/spiffs/test_spiffsgen/
|
|
- ./test_spiffsgen.py
|
|
|
|
test_fatfsgen_on_host:
|
|
extends: .host_test_template
|
|
script:
|
|
- cd components/fatfs/test_fatfsgen/
|
|
- ./test_fatfsgen.py
|
|
- ./test_wl_fatfsgen.py
|
|
- ./test_fatfsparse.py
|
|
|
|
test_multi_heap_on_host:
|
|
extends: .host_test_template
|
|
script:
|
|
- cd components/heap/test_multi_heap_host
|
|
- ./test_all_configs.sh
|
|
|
|
test_certificate_bundle_on_host:
|
|
extends: .host_test_template
|
|
script:
|
|
- cd components/mbedtls/esp_crt_bundle/test_gen_crt_bundle/
|
|
- ./test_gen_crt_bundle.py
|
|
|
|
test_gdbstub_on_host:
|
|
extends: .host_test_template
|
|
script:
|
|
- cd components/esp_gdbstub/test_gdbstub_host
|
|
- make test
|
|
|
|
# Test for create virtualenv. It must be invoked from Python, not from virtualenv.
|
|
# Use docker image system python without any extra dependencies
|
|
test_cli_installer:
|
|
extends:
|
|
- .host_test_template
|
|
- .before_script:minimal
|
|
artifacts:
|
|
when: on_failure
|
|
paths:
|
|
- tools/tools.new.json
|
|
- tools/test_idf_tools/test_python_env_logs.txt
|
|
image:
|
|
name: $ESP_ENV_IMAGE
|
|
entrypoint: [""] # use system python3. no extra pip package installed
|
|
script:
|
|
# Tools must be downloaded for testing
|
|
# We could use "idf_tools.py download all", but we don't want to install clang because of its huge size
|
|
- python3 ${IDF_PATH}/tools/idf_tools.py download required qemu-riscv32 qemu-xtensa cmake
|
|
- cd ${IDF_PATH}/tools/test_idf_tools
|
|
- python3 -m pip install jsonschema
|
|
- python3 ./test_idf_tools.py -v
|
|
- python3 ./test_idf_tools_python_env.py
|
|
# It runs at the end because it modifies dependencies
|
|
- IDF_TEST_MAY_BREAK_DEPENDENCIES=1 python3 ./test_idf_tools.py -v TestSystemDependencies.test_commands_when_nodeps
|
|
|
|
.test_efuse_table_on_host_template:
|
|
extends: .host_test_template
|
|
variables:
|
|
IDF_TARGET: "esp32"
|
|
artifacts:
|
|
when: on_failure
|
|
paths:
|
|
- components/efuse/${IDF_TARGET}/esp_efuse_table.c
|
|
script:
|
|
- cd ${IDF_PATH}/components/efuse/
|
|
- ./efuse_table_gen.py -t "${IDF_TARGET}" ${IDF_PATH}/components/efuse/${IDF_TARGET}/esp_efuse_table.csv
|
|
- git diff --exit-code -- ${IDF_TARGET}/esp_efuse_table.c || { echo 'Differences found for ${IDF_TARGET} target. Please run idf.py efuse-common-table and commit the changes.'; exit 1; }
|
|
- cd ${IDF_PATH}/components/efuse/test_efuse_host
|
|
- ./efuse_tests.py
|
|
|
|
test_efuse_table_on_host_esp32:
|
|
extends: .test_efuse_table_on_host_template
|
|
|
|
test_efuse_table_on_host_esp32s2:
|
|
extends: .test_efuse_table_on_host_template
|
|
variables:
|
|
IDF_TARGET: esp32s2
|
|
|
|
test_efuse_table_on_host_esp32s3:
|
|
extends: .test_efuse_table_on_host_template
|
|
variables:
|
|
IDF_TARGET: esp32s3
|
|
|
|
test_efuse_table_on_host_esp32c3:
|
|
extends: .test_efuse_table_on_host_template
|
|
variables:
|
|
IDF_TARGET: esp32c3
|
|
|
|
test_efuse_table_on_host_esp32h2:
|
|
extends: .test_efuse_table_on_host_template
|
|
variables:
|
|
IDF_TARGET: esp32h2
|
|
|
|
test_efuse_table_on_host_esp32c6:
|
|
extends: .test_efuse_table_on_host_template
|
|
variables:
|
|
IDF_TARGET: esp32c6
|
|
|
|
test_logtrace_proc:
|
|
extends: .host_test_template
|
|
artifacts:
|
|
when: on_failure
|
|
paths:
|
|
- tools/esp_app_trace/test/logtrace/output
|
|
- tools/esp_app_trace/test/logtrace/.coverage
|
|
script:
|
|
- cd ${IDF_PATH}/tools/esp_app_trace/test/logtrace
|
|
- ./test.sh
|
|
|
|
test_sysviewtrace_proc:
|
|
extends: .host_test_template
|
|
artifacts:
|
|
when: on_failure
|
|
paths:
|
|
- tools/esp_app_trace/test/sysview/output
|
|
- tools/esp_app_trace/test/sysview/.coverage
|
|
script:
|
|
- cd ${IDF_PATH}/tools/esp_app_trace/test/sysview
|
|
- ./test.sh
|
|
|
|
test_tools:
|
|
extends:
|
|
- .host_test_template
|
|
artifacts:
|
|
paths:
|
|
- ${IDF_PATH}/*.out
|
|
- ${IDF_PATH}/XUNIT_*.xml
|
|
reports:
|
|
junit: ${IDF_PATH}/XUNIT_*.xml
|
|
variables:
|
|
LC_ALL: C.UTF-8
|
|
INSTALL_QEMU: 1 # for test_idf_qemu.py
|
|
script:
|
|
- stat=0
|
|
- cd ${IDF_PATH}/tools/ci/test_autocomplete
|
|
- pytest --noconftest test_autocomplete.py --junitxml=${IDF_PATH}/XUNIT_AUTOCOMP.xml || stat=1
|
|
- cd ${IDF_PATH}/tools/test_idf_py
|
|
- pytest --noconftest test_idf_py.py --junitxml=${IDF_PATH}/XUNIT_IDF_PY.xml || stat=1
|
|
- pytest --noconftest test_hints.py --junitxml=${IDF_PATH}/XUNIT_HINTS.xml || stat=1
|
|
- pytest --noconftest test_idf_qemu.py --junitxml=${IDF_PATH}/XUNIT_IDF_PY_QEMU.xml || stat=1
|
|
- cd ${IDF_PATH}/tools/test_mkdfu
|
|
- pytest --noconftest test_mkdfu.py --junitxml=${IDF_PATH}/XUNIT_MKDFU.xml || stat=1
|
|
- cd ${IDF_PATH}/tools/test_idf_size
|
|
- pytest --noconftest test_idf_size.py --junitxml=${IDF_PATH}/XUNIT_IDF_SIZE.xml || stat=1
|
|
- cd ${IDF_PATH}
|
|
- shellcheck -s sh tools/detect_python.sh || stat=1
|
|
- shellcheck -s bash tools/detect_python.sh || stat=1
|
|
- shellcheck -s dash tools/detect_python.sh || stat=1
|
|
- "bash -c '. tools/detect_python.sh && echo Our Python: ${ESP_PYTHON?Python is not set}'"
|
|
- "dash -c '. tools/detect_python.sh && echo Our Python: ${ESP_PYTHON?Python is not set}'"
|
|
- "zsh -c '. tools/detect_python.sh && echo Our Python: ${ESP_PYTHON?Python is not set}'"
|
|
- "fish -c 'source tools/detect_python.fish && echo Our Python: $ESP_PYTHON'"
|
|
- exit "$stat"
|
|
|
|
test_split_path_by_spaces:
|
|
extends: .host_test_template
|
|
script:
|
|
- cd ${IDF_PATH}/tools
|
|
- python -m unittest split_paths_by_spaces.py
|
|
|
|
test_mqtt_on_host:
|
|
extends: .host_test_template
|
|
script:
|
|
- cd ${IDF_PATH}/components/mqtt/esp-mqtt/host_test
|
|
- idf.py build
|
|
- LSAN_OPTIONS=verbosity=1:log_threads=1 build/host_mqtt_client_test.elf
|
|
|
|
test_transport_on_host:
|
|
extends: .host_test_template
|
|
script:
|
|
- cd ${IDF_PATH}/components/tcp_transport/host_test
|
|
- idf.py build
|
|
- LSAN_OPTIONS=verbosity=1:log_threads=1 build/host_tcp_transport_test.elf
|
|
|
|
test_sockets_on_host:
|
|
extends: .host_test_template
|
|
script:
|
|
# test the tcp-client example with system sockets
|
|
- cd ${IDF_PATH}/examples/protocols/sockets/tcp_client
|
|
- echo 'CONFIG_EXAMPLE_IPV4_ADDR="127.0.0.1"' >> sdkconfig.defaults
|
|
- idf.py --preview set-target linux
|
|
- idf.py build
|
|
- timeout 5 ./build/tcp_client.elf >test.log || true
|
|
- grep "Socket unable to connect" test.log
|
|
# test the udp-client example with lwip sockets
|
|
- cd ${IDF_PATH}/examples/protocols/sockets/udp_client
|
|
- idf.py --preview set-target linux
|
|
- cat sdkconfig.ci.linux > sdkconfig
|
|
- idf.py build
|
|
- timeout 5 ./build/udp_client.elf >test.log || true
|
|
- grep "Message sent" test.log
|
|
|
|
test_eh_frame_parser:
|
|
extends: .host_test_template
|
|
script:
|
|
- cd ${IDF_PATH}/components/esp_system/test_eh_frame_parser
|
|
- make
|
|
- ./eh_frame_test
|
|
|
|
test_gen_soc_caps_kconfig:
|
|
extends: .host_test_template
|
|
script:
|
|
- cd ${IDF_PATH}/tools/gen_soc_caps_kconfig/
|
|
- ./test/test_gen_soc_caps_kconfig.py
|
|
|
|
test_pytest_qemu:
|
|
extends:
|
|
- .host_test_template
|
|
- .before_script:build
|
|
artifacts:
|
|
paths:
|
|
- XUNIT_RESULT.xml
|
|
- pytest-embedded/
|
|
reports:
|
|
junit: XUNIT_RESULT.xml
|
|
allow_failure: true # IDFCI-1752
|
|
parallel:
|
|
matrix:
|
|
- IDF_TARGET: [esp32, esp32c3]
|
|
variables:
|
|
INSTALL_QEMU: 1
|
|
script:
|
|
- run_cmd python tools/ci/ci_build_apps.py . -vv
|
|
--target $IDF_TARGET
|
|
--pytest-apps
|
|
-m qemu
|
|
--collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
|
|
--modified-components ${MR_MODIFIED_COMPONENTS}
|
|
--modified-files ${MR_MODIFIED_FILES}
|
|
- python tools/ci/get_known_failure_cases_file.py
|
|
- run_cmd pytest
|
|
--target $IDF_TARGET
|
|
-m qemu
|
|
--embedded-services idf,qemu
|
|
--junitxml=XUNIT_RESULT.xml
|
|
--ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
|
|
--app-info-filepattern \"list_job_*.txt\"
|
|
--qemu-extra-args \"-global driver=timer.$IDF_TARGET.timg,property=wdt_disable,value=true\"
|
|
|
|
test_pytest_linux:
|
|
extends:
|
|
- .host_test_template
|
|
- .before_script:build
|
|
artifacts:
|
|
paths:
|
|
- XUNIT_RESULT.xml
|
|
- pytest-embedded/
|
|
- "**/build*/build_log.txt"
|
|
reports:
|
|
junit: XUNIT_RESULT.xml
|
|
script:
|
|
- run_cmd python tools/ci/ci_build_apps.py components examples tools/test_apps -vv
|
|
--target linux
|
|
--pytest-apps
|
|
-m host_test
|
|
--collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
|
|
--modified-components ${MR_MODIFIED_COMPONENTS}
|
|
--modified-files ${MR_MODIFIED_FILES}
|
|
- python tools/ci/get_known_failure_cases_file.py
|
|
- run_cmd pytest
|
|
--target linux
|
|
-m host_test
|
|
--junitxml=XUNIT_RESULT.xml
|
|
--ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
|
|
--app-info-filepattern \"list_job_*.txt\"
|
|
|
|
test_pytest_macos:
|
|
extends:
|
|
- .host_test_template
|
|
- .before_script:build:macos
|
|
tags:
|
|
- macos_shell
|
|
artifacts:
|
|
paths:
|
|
- XUNIT_RESULT.xml
|
|
- pytest-embedded/
|
|
- "**/build*/build_log.txt"
|
|
reports:
|
|
junit: XUNIT_RESULT.xml
|
|
variables:
|
|
PYTEST_IGNORE_COLLECT_IMPORT_ERROR: "1"
|
|
script:
|
|
- run_cmd python tools/ci/ci_build_apps.py components examples tools/test_apps -vv
|
|
--target linux
|
|
--pytest-apps
|
|
-m \"host_test and macos_shell\"
|
|
--collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
|
|
--modified-components ${MR_MODIFIED_COMPONENTS}
|
|
--modified-files ${MR_MODIFIED_FILES}
|
|
- python tools/ci/get_known_failure_cases_file.py
|
|
- run_cmd pytest
|
|
--target linux
|
|
-m \"host_test and macos_shell\"
|
|
--junitxml=XUNIT_RESULT.xml
|
|
--ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
|
|
--app-info-filepattern \"list_job_*.txt\"
|
|
|
|
|
|
test_idf_pytest_plugin:
|
|
extends:
|
|
- .host_test_template
|
|
- .rules:patterns:idf-pytest-plugin
|
|
variables:
|
|
SUBMODULES_TO_FETCH: "none"
|
|
artifacts:
|
|
reports:
|
|
junit: XUNIT_RESULT.xml
|
|
script:
|
|
- cd ${IDF_PATH}/tools/ci/dynamic_pipelines/tests/test_report_generator
|
|
- python -m unittest test_report_generator.py
|
|
- cd ${IDF_PATH}/tools/ci/idf_pytest
|
|
- pytest --junitxml=${CI_PROJECT_DIR}/XUNIT_RESULT.xml
|