alex/esp-idf
alex/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf.git synced 2024-10-05 20:47:46 -04:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

CI: Refactor CI rules into rules.yml.

Browse Source 
- Add `utils.sh` for some widely used shell scripts
- Change `dependencies` into `needs` for strict dependency relationships
This commit is contained in:
Fu Hanxi 2020-09-27 11:20:53 +08:00
parent 653fa702b4
commit 7951152f7b
13 changed files with 689 additions and 768 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.editorconfig
View File

@ -37,3 +37,7 @@ max_line_length = 119
indent_style = space
indent_size = 4
max_line_length = 120
[{*.sh,*.yml}]
indent_style = space
indent_size = 2

134
.gitlab-ci.yml
View File

@ -49,13 +49,14 @@ variables:
IDF_PATH: "$CI_PROJECT_DIR"
BATCH_BUILD: "1"
V: "0"
APPLY_BOT_FILTER_SCRIPT: "$CI_PROJECT_DIR/tools/ci/apply_bot_filter.py"
CHECKOUT_REF_SCRIPT: "$CI_PROJECT_DIR/tools/ci/checkout_project_ref.py"
# Docker images
BOT_DOCKER_IMAGE_TAG: ":latest"
# target test config file, used by assign test job
CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/tools/ci/config/target-test.yml"
# target test repo parameters
TEST_ENV_CONFIG_REPO: "${GITLAB_SSH_SERVER}/qa/ci-test-runner-configs.git"
CI_AUTO_TEST_SCRIPT_REPO_URL: "${GITLAB_SSH_SERVER}/qa/auto_test_script.git"
@ -64,11 +65,6 @@ variables:
# Versioned esp-idf-doc env image to use for all document building jobs
ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env:v7"
# before each job, we need to check if this job is filtered by bot stage/job filter
.apply_bot_filter: &apply_bot_filter
python $APPLY_BOT_FILTER_SCRIPT || exit 0
.setup_tools_unless_target_test: &setup_tools_unless_target_test |
if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
export IDF_MIRROR_PREFIX_MAP=
@ -77,92 +73,41 @@ variables:
tools/idf_tools.py --non-interactive install && eval "$(tools/idf_tools.py --non-interactive export)" || exit 1
fi
.fetch_submodules: &fetch_submodules |
python $SUBMODULE_FETCH_TOOL -s $SUBMODULES_TO_FETCH
.add_ssh_keys: &add_ssh_keys |
mkdir -p ~/.ssh
chmod 700 ~/.ssh
echo -n $GITLAB_KEY > ~/.ssh/id_rsa_base64
base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
before_script:
- source tools/ci/utils.sh
- source tools/ci/setup_python.sh
# apply bot filter in before script
- *apply_bot_filter
# add gitlab ssh key
- *add_ssh_keys
# Set some options and environment for CI
- apply_bot_filter
- add_gitlab_ssh_keys
- source tools/ci/configure_ci_environment.sh
- *setup_tools_unless_target_test
- *fetch_submodules
# used for component-based unit test apps
.before_script_for_component_ut:
variables:
COMPONENT_UT_EXCLUDE_LIST_FP: ${CI_PROJECT_DIR}/tools/ci/component_ut_excludes.txt
before_script:
- source tools/ci/setup_python.sh
- *apply_bot_filter
- *add_ssh_keys
- source tools/ci/configure_ci_environment.sh
- *setup_tools_unless_target_test
- *fetch_submodules
- export COMPONENT_UT_DIRS=`find components/ -name test_apps -type d`
- export COMPONENT_UT_EXCLUDES=`[ -r $COMPONENT_UT_EXCLUDE_LIST_FP ] && cat $COMPONENT_UT_EXCLUDE_LIST_FP | xargs`
- fetch_submodules
# used for check scripts which we want to run unconditionally
.before_script_lesser_nofilter:
variables:
GIT_SUBMODULE_STRATEGY: none
before_script:
- echo "Not setting up GitLab key, not fetching submodules, not applying bot filter"
- source tools/ci/utils.sh
- source tools/ci/setup_python.sh
- source tools/ci/configure_ci_environment.sh
# used for everything else where we want to do no prep, except for bot filter
.before_script_lesser:
variables:
GIT_SUBMODULE_STRATEGY: none
before_script:
- echo "Not setting up GitLab key, not fetching submodules"
- source tools/ci/utils.sh
- source tools/ci/setup_python.sh
# apply bot filter in before script
- *apply_bot_filter
- apply_bot_filter
- source tools/ci/configure_ci_environment.sh
.check_job_template:
stage: pre_check
image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
tags:
- host_test
dependencies: []
extends: .before_script_lesser_nofilter
.check_job_template_with_filter:
stage: pre_check
image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
tags:
- host_test
dependencies: []
extends: .before_script_lesser
.python_lint_template:
stage: pre_check
image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
tags:
- host_test
dependencies: []
.macos_build_template:
stage: build
tags:
- macos_shell
dependencies: []
.before_script_slim:
before_script:
- *apply_bot_filter
- echo "Only load utils.sh inside"
- source tools/ci/utils.sh
.before_script_macos:
before_script:
- source tools/ci/utils.sh
- apply_bot_filter
- $IDF_PATH/tools/idf_tools.py install-python-env
# On macOS, these tools need to be installed
- $IDF_PATH/tools/idf_tools.py --non-interactive install cmake ninja
@ -173,51 +118,10 @@ before_script:
- source tools/ci/configure_ci_environment.sh
# Part of tools/ci/setup_python.sh; we don't use pyenv on macOS, so can't run the rest of the script.
- export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
- *fetch_submodules
.build_template_app_template:
stage: build
image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
tags:
- build
variables:
BATCH_BUILD: "1"
LOG_PATH: "${CI_PROJECT_DIR}/log_template_app"
BUILD_PATH: "${CI_PROJECT_DIR}/build_template_app"
SIZE_INFO_LOCATION: "$CI_PROJECT_DIR/size_info.txt"
BUILD_DIR: "@t/@w"
BUILD_LOG_MAKE: "${LOG_PATH}/make_@t_@w.txt"
BUILD_LOG_CMAKE: "${LOG_PATH}/cmake_@t_@w.txt"
BUILD_COMMAND_ARGS: ""
artifacts:
when: always
paths:
- log_template_app/*
- size_info.txt
- build_template_app/**/size.json
only:
variables:
- $BOT_TRIGGER_WITH_LABEL == null
- $BOT_LABEL_BUILD
- $BOT_LABEL_REGULAR_TEST
script:
# Set the variable for 'esp-idf-template' testing
- ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"}
- ./tools/ci/retry_failed.sh git clone ${ESP_IDF_TEMPLATE_GIT}
# Try to use the same branch name for esp-idf-template that we're
# using on esp-idf. If it doesn't exist then just stick to the default branch
- python $CHECKOUT_REF_SCRIPT esp-idf-template esp-idf-template
- export PATH="$IDF_PATH/tools:$PATH"
- export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
- export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
# Only do the default cmake build for each target, remaining part are done in the build_template_app job
- tools/ci/build_template_app.sh ${BUILD_COMMAND_ARGS}
# Check if there are any stray printf/ets_printf references in WiFi libs
- cd components/esp_wifi/lib
- for dir in esp32 esp32s2; do test $(xtensa-esp32-elf-nm $dir/*.a | grep -w printf | wc -l) -eq 0; done;
- for dir in esp32 esp32s2; do test $(xtensa-esp32-elf-nm $dir/*.a | grep -w ets_printf | wc -l) -eq 0; done;
- fetch_submodules
include:
- '/tools/ci/config/rules.yml'
- '/tools/ci/config/pre_check.yml'
- '/tools/ci/config/build.yml'
- '/tools/ci/config/assign-test.yml'

51
tools/ci/config/assign-test.yml
View File

@ -1,21 +1,23 @@
assign_test:
extends: .before_script_for_component_ut
extends:
- .rules:test:target_test-weekend_test
tags:
- assign_test
image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
stage: assign_test
# gitlab ci do not support match job with RegEx or wildcard now in dependencies.
# we have a lot build example jobs. now we don't use dependencies, just download all artifacts of build stage.
dependencies:
dependencies: # Here is not a hard dependency relationship, could be skipped. so we do not use "needs" here.
- build_ssc_esp32
- build_esp_idf_tests_cmake_esp32
- build_esp_idf_tests_cmake_esp32s2
variables:
SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
EXAMPLE_CONFIG_OUTPUT_PATH: "${CI_PROJECT_DIR}/examples/test_configs"
TEST_APP_CONFIG_OUTPUT_PATH: "${CI_PROJECT_DIR}/tools/test_apps/test_configs"
COMPONENT_UT_CONFIG_OUTPUT_PATH: "${CI_PROJECT_DIR}/component_ut/test_configs"
UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test"
EXAMPLE_TEST_DIR: "${CI_PROJECT_DIR}/examples"
CUSTOM_TEST_DIR: "${CI_PROJECT_DIR}/tools/test_apps"
UNIT_TEST_DIR: "${CI_PROJECT_DIR}/components/idf_test/unit_test"
# COMPONENT_UT_DIRS is set by `set_component_ut_vars` in `utils.sh`
COMPONENT_UT_OUTPUT_DIR: "${CI_PROJECT_DIR}/component_ut"
INTEGRATION_CONFIG_OUTPUT_PATH: "${CI_PROJECT_DIR}/components/idf_test/integration_test/CIConfigs"
INTEGRATION_TEST_CASE_PATH: "${CI_PROJECT_DIR}/auto_test_script/TestCaseFiles"
ASSIGN_TEST_CASE_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIAssignTestCases.py"
@ -32,42 +34,27 @@ assign_test:
- build_component_ut/artifact_index.json
- tools/unit-test-app/builds/artifact_index.json
expire_in: 1 week
only:
variables:
- $BOT_TRIGGER_WITH_LABEL == null
- $BOT_LABEL_UNIT_TEST
- $BOT_LABEL_UNIT_TEST_S2
- $BOT_LABEL_INTEGRATION_TEST
- $BOT_LABEL_EXAMPLE_TEST
- $BOT_LABEL_CUSTOM_TEST
- $BOT_LABEL_WEEKEND_TEST
script:
# assign example tests
- python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py example_test $IDF_PATH/examples -c $CI_TARGET_TEST_CONFIG_FILE -o $EXAMPLE_CONFIG_OUTPUT_PATH
# assign test apps
- python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py custom_test $IDF_PATH/tools/test_apps -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_APP_CONFIG_OUTPUT_PATH
# assign component ut
- python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py component_ut $COMPONENT_UT_DIRS -c $CI_TARGET_TEST_CONFIG_FILE -o $COMPONENT_UT_CONFIG_OUTPUT_PATH
# assign unit test cases
- python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py unit_test $UNIT_TEST_CASE_FILE -c $CI_TARGET_TEST_CONFIG_FILE -o $IDF_PATH/components/idf_test/unit_test/CIConfigs
- set_component_ut_vars
- python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py example_test $EXAMPLE_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $EXAMPLE_TEST_DIR/test_configs
- python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py custom_test $CUSTOM_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $CUSTOM_TEST_DIR/test_configs
- python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py component_ut $COMPONENT_UT_DIRS -c $CI_TARGET_TEST_CONFIG_FILE -o $COMPONENT_UT_OUTPUT_DIR/test_configs
- python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py unit_test $UNIT_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $UNIT_TEST_DIR/CIConfigs
# clone test script to assign tests
# can not retry if downing git lfs files failed, so using empty_branch first.
- ./tools/ci/retry_failed.sh git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
- ./tools/ci/retry_failed.sh git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
- retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
- retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
- python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
# assign integration test cases
- python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b $IDF_PATH/SSC/ssc_bin -o $INTEGRATION_CONFIG_OUTPUT_PATH
update_test_cases:
extends: .rules:master-schedule
stage: assign_test
image: $CI_DOCKER_REGISTRY/ubuntu-test-env
tags:
- deploy_test
only:
refs:
- master
- schedules
dependencies:
needs:
- build_esp_idf_tests_cmake_esp32
- build_esp_idf_tests_cmake_esp32s2
artifacts:
@ -77,7 +64,7 @@ update_test_cases:
expire_in: 1 week
variables:
SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
UNIT_TEST_CASE_DIR: "${CI_PROJECT_DIR}/components/idf_test/unit_test"
UNIT_TEST_DIR: "${CI_PROJECT_DIR}/components/idf_test/unit_test"
BOT_ACCOUNT_CONFIG_FILE: "${CI_PROJECT_DIR}/test-management/Config/Account.local.yml"
AUTO_TEST_SCRIPT_PATH: "${CI_PROJECT_DIR}/auto_test_script"
PYTHON_VER: 3.7.7
@ -88,7 +75,7 @@ update_test_cases:
- cd test-management
- echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE}
# update unit test cases
- export UNIT_TEST_CASE_FILES=$(find $UNIT_TEST_CASE_DIR -maxdepth 1 -name "*.yml" | xargs)
- export UNIT_TEST_CASE_FILES=$(find $UNIT_TEST_DIR -maxdepth 1 -name "*.yml" | xargs)
- python ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT unity -d $UNIT_TEST_CASE_FILES -r $GIT_SHA
# update example test cases
- python ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT tiny_test_fw -d ${CI_PROJECT_DIR}/examples -r $GIT_SHA

353
tools/ci/config/build.yml
View File

@ -4,25 +4,69 @@
tags:
- build
variables:
BATCH_BUILD: "1"
V: "0"
SIZE_INFO_LOCATION: "$CI_PROJECT_DIR/size_info.txt"
.build_template_app_template:
extends:
- .build_template
- .rules:labels:build
variables:
LOG_PATH: "${CI_PROJECT_DIR}/log_template_app"
BUILD_PATH: "${CI_PROJECT_DIR}/build_template_app"
BUILD_DIR: "@t/@w"
BUILD_LOG_MAKE: "${LOG_PATH}/make_@t_@w.txt"
BUILD_LOG_CMAKE: "${LOG_PATH}/cmake_@t_@w.txt"
BUILD_COMMAND_ARGS: ""
artifacts:
when: always
paths:
- log_template_app/*
- size_info.txt
- build_template_app/**/size.json
script:
# Set the variable for 'esp-idf-template' testing
- ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"}
- ./tools/ci/retry_failed.sh git clone ${ESP_IDF_TEMPLATE_GIT}
# Try to use the same branch name for esp-idf-template that we're
# using on esp-idf. If it doesn't exist then just stick to the default branch
- python $CHECKOUT_REF_SCRIPT esp-idf-template esp-idf-template
- export PATH="$IDF_PATH/tools:$PATH"
- export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
- export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
# Only do the default cmake build for each target, remaining part are done in the build_template_app job
- tools/ci/build_template_app.sh ${BUILD_COMMAND_ARGS}
# Check if there are any stray printf/ets_printf references in WiFi libs
- cd components/esp_wifi/lib
- for dir in esp32 esp32s2; do test $(xtensa-esp32-elf-nm $dir/*.a | grep -w printf | wc -l) -eq 0; done;
- for dir in esp32 esp32s2; do test $(xtensa-esp32-elf-nm $dir/*.a | grep -w ets_printf | wc -l) -eq 0; done;
# build-related-pre-check-jobs ------------------------------------------------
# Build at least one project for each target at earliest stage to reduce build cost for obvious failing commits
fast_template_app:
extends: .build_template_app_template
stage: pre_check
variables:
BUILD_COMMAND_ARGS: "-p"
check_docs_gh_links:
extends: .build_docs_template
stage: pre_check
variables:
SUBMODULES_TO_FETCH: "none"
script:
- cd docs
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py gh-linkcheck
#------------------------------------------------------------------------------
.build_ssc_template:
extends: .build_template
parallel: 3
extends:
- .build_template
- .rules:labels:build-integration_test
artifacts:
paths:
- SSC/ssc_bin
expire_in: 1 week
variables:
TARGET_NAME: "ESP32"
only:
variables:
- $BOT_TRIGGER_WITH_LABEL == null
- $BOT_LABEL_BUILD
- $BOT_LABEL_INTEGRATION_TEST
- $BOT_LABEL_REGULAR_TEST
script:
- ./tools/ci/retry_failed.sh git clone $SSC_REPOSITORY
- python $CHECKOUT_REF_SCRIPT SSC SSC
@ -31,6 +75,7 @@
build_ssc_esp32:
extends: .build_ssc_template
parallel: 3
variables:
TARGET_NAME: "ESP32"
@ -41,7 +86,9 @@ build_ssc_esp32s2:
TARGET_NAME: "ESP32S2"
.build_esp_idf_tests_cmake:
extends: .build_template
extends:
- .build_template
- .rules:labels:build-unit_test
artifacts:
paths:
- tools/unit-test-app/output/${IDF_TARGET}
@ -52,13 +99,6 @@ build_ssc_esp32s2:
- $SIZE_INFO_LOCATION
when: always
expire_in: 4 days
only:
variables:
- $BOT_TRIGGER_WITH_LABEL == null
- $BOT_LABEL_BUILD
- $BOT_LABEL_UNIT_TEST
- $BOT_LABEL_UNIT_TEST_S2
- $BOT_LABEL_REGULAR_TEST
variables:
LOG_PATH: "$CI_PROJECT_DIR/log_ut_cmake"
BUILD_PATH: ${CI_PROJECT_DIR}/tools/unit-test-app/builds
@ -89,17 +129,9 @@ build_esp_idf_tests_cmake_esp32s3:
IDF_TARGET: esp32s3
.build_examples_template:
extends: .build_template
artifacts:
when: always
expire_in: 4 days
only:
variables:
- $BOT_TRIGGER_WITH_LABEL == null
- $BOT_LABEL_BUILD
- $BOT_LABEL_EXAMPLE_TEST
- $BOT_LABEL_REGULAR_TEST
- $BOT_LABEL_WEEKEND_TEST
extends:
- .build_template
- .rules:labels:build-example_test-weekend_test
variables:
TEST_PREFIX: examples
TEST_RELATIVE_DIR: examples
@ -116,7 +148,8 @@ build_esp_idf_tests_cmake_esp32s3:
- ${IDF_PATH}/tools/ci/find_apps_build_apps.sh
build_examples_make:
extends: .build_examples_template
extends:
- .build_examples_template
# This is a workaround for a rarely encountered issue with building examples in CI.
# Probably related to building of Kconfig in 'make clean' stage
retry: 1
@ -126,23 +159,16 @@ build_examples_make:
- $LOG_PATH
- build_${TEST_PREFIX}/*/*/*/build/size.json
- $SIZE_INFO_LOCATION
when: always
expire_in: 4 days
variables:
BUILD_SYSTEM: make
IDF_TARGET: esp32 # currently we only support esp32
only:
refs:
- master
- /^release\/v/
- /^v\d+\.\d+(\.\d+)?($|-)/
- triggers
- schedules
- pipelines
- web
# same as above, but for CMake
.build_examples_cmake: &build_examples_cmake
.build_examples_cmake:
extends: .build_examples_template
dependencies:
needs:
- scan_tests
artifacts:
paths:
@ -174,18 +200,13 @@ build_examples_cmake_esp32s2:
IDF_TARGET: esp32s2
.build_test_apps:
extends: .build_examples_cmake
extends:
- .build_examples_cmake
- .rules:labels:build-custom_test-weekend_test
variables:
TEST_PREFIX: test_apps
TEST_RELATIVE_DIR: tools/test_apps
TEST_TYPE: custom_test
only:
variables:
- $BOT_TRIGGER_WITH_LABEL == null
- $BOT_LABEL_BUILD
- $BOT_LABEL_REGULAR_TEST
- $BOT_LABEL_CUSTOM_TEST
- $BOT_LABEL_WEEKEND_TEST
script:
- ${IDF_PATH}/tools/ci/find_apps_build_apps.sh
@ -202,17 +223,12 @@ build_test_apps_esp32s2:
IDF_TARGET: esp32s2
.build_component_ut:
extends: .build_test_apps
extends:
- .build_test_apps
- .rules:labels:build-unit_test
variables:
TEST_PREFIX: component_ut
TEST_RELATIVE_DIR: component_ut
only:
variables:
- $BOT_TRIGGER_WITH_LABEL == null
- $BOT_LABEL_BUILD
- $BOT_LABEL_REGULAR_TEST
- $BOT_LABEL_UNIT_TEST
- $BOT_LABEL_UNIT_TEST_S2
build_component_ut_esp32:
extends: .build_component_ut
@ -224,125 +240,81 @@ build_component_ut_esp32s2:
variables:
IDF_TARGET: esp32s2
# If you want to add new build example jobs, please add it into dependencies of `.example_test_template`
.build_docs_template: &build_docs_template
.build_docs_template:
stage: build
image: $ESP_IDF_DOC_ENV_IMAGE
tags:
- build_docs
script:
- cd docs
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py -bs $DOC_BUILDERS -l $DOCLANG -t $DOCTGT build
parallel:
matrix:
- DOCLANG: [ "en", "zh_CN" ]
DOCTGT: [ "esp32", "esp32s2" ]
build_docs_html:
extends:
- .build_docs_template
- .rules:labels:build_docs
artifacts:
when: always
paths:
- docs/_build/*/*/*.txt
- docs/_build/*/*/html/*
expire_in: 4 days
only:
variables:
- $BOT_TRIGGER_WITH_LABEL == null
- $BOT_LABEL_BUILD
- $BOT_LABEL_BUILD_DOCS
- $BOT_LABEL_REGULAR_TEST
dependencies: []
script:
- cd docs
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py -bs html -l $DOCLANG -t $DOCTGT build
build_docs_html:
extends: .build_docs_template
parallel:
matrix:
- DOCLANG: ["en", "zh_CN"]
DOCTGT: ["esp32", "esp32s2"]
variables:
DOC_BUILDERS: "html"
build_docs_pdf:
stage: build
image: $ESP_IDF_DOC_ENV_IMAGE
tags:
- build_docs
extends:
- .build_docs_template
- .rules:labels:build_docs-slim
artifacts:
when: always
paths:
- docs/_build/*/*/latex/*
expire_in: 4 days
rules:
- if: '$CI_COMMIT_REF_NAME == "master"'
- if: '$CI_COMMIT_REF_NAME =~ "/^release\/v/"'
- if: '$CI_COMMIT_REF_NAME =~ "/^v\d+\.\d+(\.\d+)?($|-)/"'
- if: $BOT_LABEL_BUILD_DOCS
dependencies: []
parallel:
matrix:
- DOCLANG: ["en", "zh_CN"]
DOCTGT: ["esp32", "esp32s2"]
variables:
DOC_BUILDERS: "latex"
.test_build_system_template:
extends:
- .build_template
- .rules:labels:build-weekend_test
script:
- cd docs
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py -bs latex -l $DOCLANG -t $DOCTGT build
- ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
- rm -rf test_build_system
- mkdir test_build_system
- cd test_build_system
- ${IDF_PATH}/tools/ci/${SHELL_TEST_SCRIPT}
test_build_system:
extends: .build_template
only:
variables:
- $BOT_TRIGGER_WITH_LABEL == null
- $BOT_LABEL_BUILD
- $BOT_LABEL_REGULAR_TEST
- $BOT_LABEL_WEEKEND_TEST
script:
- ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
- rm -rf test_build_system
- mkdir test_build_system
- cd test_build_system
- ${IDF_PATH}/tools/ci/test_build_system.sh
extends: .test_build_system_template
variables:
SHELL_TEST_SCRIPT: test_build_system.sh
test_build_system_cmake:
extends: .build_template
only:
variables:
- $BOT_TRIGGER_WITH_LABEL == null
- $BOT_LABEL_BUILD
- $BOT_LABEL_REGULAR_TEST
- $BOT_LABEL_WEEKEND_TEST
script:
- ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
- rm -rf test_build_system
- mkdir test_build_system
- cd test_build_system
- ${IDF_PATH}/tools/ci/test_build_system_cmake.sh
extends: .test_build_system_template
variables:
SHELL_TEST_SCRIPT: test_build_system_cmake.sh
test_build_system_cmake_macos:
extends: .macos_build_template
only:
refs:
- master
- /^release\/v/
- /^v\d+\.\d+(\.\d+)?($|-)/
- schedules
- triggers
- pipelines
- web
variables:
- $CI_PIPELINE_SOURCE != "push" && $BOT_LABEL_MACOS_TEST != null
- $CI_PIPELINE_SOURCE == "push"
script:
- ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
- rm -rf test_build_system
- mkdir test_build_system
- cd test_build_system
- ${IDF_PATH}/tools/ci/test_build_system_cmake.sh
extends:
- .test_build_system_template
- .before_script_macos
- .rules:os:mac_os
tags:
- macos_shell
build_docker:
extends:
- .rules:master-release-schedule
stage: build
image: espressif/docker-builder:1
tags:
- build_docker_amd64_brno
only:
refs:
- master
- /^release\/v/
- /^v\d+\.\d+(\.\d+)?($|-)/
- schedules
variables:
DOCKER_TMP_IMAGE_NAME: "idf_tmp_image"
before_script: []
@ -358,65 +330,46 @@ build_docker:
# Therefore, build a copy of the example located inside the container.
- docker run --rm --workdir /opt/esp/idf/examples/get-started/blink ${DOCKER_TMP_IMAGE_NAME} idf.py build
build_idf_exe:
.test-on-windows:
extends: .rules:master-release-schedule
stage: build
image: $CI_DOCKER_REGISTRY/esp32-toolchain-win-cross
tags:
- build
only:
refs:
- master
- /^release\/v/
- /^v\d+\.\d+(\.\d+)?($|-)/
- schedules
before_script: []
script:
- cd $TEST_DIR
- mkdir build
- cd build
- cmake -DCMAKE_TOOLCHAIN_FILE=../toolchain-i686-w64-mingw32.cmake -DCMAKE_BUILD_TYPE=Release ..
- cmake --build .
build_idf_exe:
extends: .test-on-windows
artifacts:
paths:
- tools/windows/idf_exe/build/idf-exe-v*.zip
expire_in: 4 days
script:
- cd tools/windows/idf_exe/
- mkdir build
- cd build
- cmake -DCMAKE_TOOLCHAIN_FILE=../toolchain-i686-w64-mingw32.cmake -DCMAKE_BUILD_TYPE=Release ..
- cmake --build .
variables:
TEST_DIR: tools/windows/idf_exe
build_cmdlinerunner:
stage: build
image: $CI_DOCKER_REGISTRY/esp32-toolchain-win-cross
tags:
- build
only:
refs:
- master
- /^release\/v/
- /^v\d+\.\d+(\.\d+)?($|-)/
- schedules
before_script: []
extends: .test-on-windows
artifacts:
paths:
- tools/windows/tool_setup/cmdlinerunner/build/cmdlinerunner.dll
expire_in: 4 days
script:
- cd tools/windows/tool_setup/cmdlinerunner
- mkdir build
- cd build
- cmake -DCMAKE_TOOLCHAIN_FILE=../toolchain-i686-w64-mingw32.cmake -DCMAKE_BUILD_TYPE=Release ..
- cmake --build .
variables:
TEST_DIR: tools/windows/tool_setup/cmdlinerunner
build_installer:
extends: .rules:master-release-schedule
# using a different stage here to be able to use artifacts from build_cmdlinerunner job
stage: host_test
image: $CI_DOCKER_REGISTRY/wine-innosetup:1
tags:
- build
only:
refs:
- master
- /^release\/v/
- /^v\d+\.\d+(\.\d+)?($|-)/
- schedules
dependencies:
needs:
- build_cmdlinerunner
before_script: []
script:
@ -444,14 +397,15 @@ build_template_app:
image:
name: $CI_DOCKER_REGISTRY/sonarqube-scanner:2
before_script:
- source tools/ci/utils.sh
- export PYTHONPATH="$CI_PROJECT_DIR/tools:$CI_PROJECT_DIR/tools/ci/python_packages:$PYTHONPATH"
- python $SUBMODULE_FETCH_TOOL
- fetch_submodules
# Exclude the submodules, all paths ends with /**
# get all submodules configs | get all paths | add /** as suffix | xargs | replace all <space> to <comma>
- export SUBMODULES=$(git config --file .gitmodules --get-regexp path | awk '{ print $2 }' | sed -e 's|$|/**|' | xargs | sed -e 's/ /,/g')
- export SUBMODULES=$(get_all_submodules)
# get all exclude paths specified in tools/ci/sonar_exclude_list.txt | ignore lines start with # | xargs | replace all <space> to <comma>
- export CUSTOM_EXCLUDES=$(cat $CI_PROJECT_DIR/tools/ci/sonar_exclude_list.txt | grep -v '^#' | xargs | sed -e 's/ /,/g')
- export EXCLUSIONS="$SUBMODULES,$CUSTOM_EXCLUDES,$REPORT_DIR/**,docs/_static/**,**/*.png,**/*.jpg"
# Exclude the report dir
- export EXCLUSIONS="$SUBMODULES,$REPORT_DIR/**,docs/_static/**,**/*.png,**/*.jpg"
- python $NORMALIZE_CLANGTIDY_PY $CI_PROJECT_DIR/$REPORT_DIR/warnings.txt $CI_PROJECT_DIR/$REPORT_DIR/clang_tidy_report.txt $CI_PROJECT_DIR
variables:
GIT_DEPTH: 0
@ -459,13 +413,13 @@ build_template_app:
REPORT_DIR: examples/get-started/hello_world/tidybuild/report
tags:
- host_test
needs:
- clang_tidy_check_regular
code_quality_check:
extends: .sonar_scan_template
dependencies:
- clang_tidy_check_regular
only:
- triggers
extends:
- .sonar_scan_template
- .rules:trigger
allow_failure: true
script:
- export CI_MERGE_REQUEST_IID=`python ${CI_PROJECT_DIR}/tools/ci/ci_get_latest_mr_iid.py ${CI_COMMIT_BRANCH} | xargs`
@ -491,14 +445,9 @@ code_quality_check:
-Dsonar.branch.name=$CI_COMMIT_REF_NAME
code_quality_report:
extends: .sonar_scan_template
only:
- master
- /^release\/v/
- /^v\d+\.\d+(\.\d+)?($|-)/
- schedules
dependencies:
- clang_tidy_check_regular
extends:
- .sonar_scan_template
- .rules:master-release-schedule
script:
- sonar-scanner
-Dsonar.host.url=$SONAR_HOST_URL

116
tools/ci/config/deploy.yml
View File

@ -1,16 +1,16 @@
.clang_tidy_deploy_template:
.deploy_job_template:
stage: deploy
image: $CI_DOCKER_REGISTRY/esp32-ci-env
image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
tags:
- deploy
.clang_tidy_deploy_template:
extends: .deploy_job_template
tags:
- deploy
- shiny
script:
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- echo -n $DOCS_DEPLOY_KEY > ~/.ssh/id_rsa_base64
- base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
- chmod 600 ~/.ssh/id_rsa
- echo -e "Host $DOCS_SERVER\n\tStrictHostKeyChecking no\n\tUser $DOCS_SERVER_USER\n" >> ~/.ssh/config
- add_doc_server_ssh_keys $DOCS_DEPLOY_KEY $DOCS_SERVER $DOCS_SERVER_USER
- export GIT_VER=$(git describe --always)
- cd $IDF_PATH/examples/get-started/hello_world/tidybuild
- mv report $GIT_VER
@ -27,89 +27,56 @@ clang_tidy_deploy:
extends: .clang_tidy_deploy_template
# Override default stage to happen before the post_check
stage: test_deploy
dependencies:
needs:
- clang_tidy_check
- clang_tidy_check_all
variables:
BOT_NEEDS_TRIGGER_BY_NAME: 1
clang_tidy_deploy_regular:
extends: .clang_tidy_deploy_template
dependencies:
extends:
- .clang_tidy_deploy_template
- .rules:labels:static_analysis
needs:
- clang_tidy_check_regular
only:
refs:
- master
- /^release\/v/
- /^v\d+\.\d+(\.\d+)?($|-)/
- triggers
- schedules
variables:
- $BOT_LABEL_STATIC_ANALYSIS
- $BOT_LABEL_STATIC_ANALYSIS_ALL
push_to_github:
stage: deploy
image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
tags:
- deploy
only:
refs:
- master
- /^release\/v/
- /^v\d+\.\d+(\.\d+)?($|-)/
variables:
- $BOT_TRIGGER_WITH_LABEL == null
when: on_success
dependencies: []
extends: .before_script_lesser
extends:
- .deploy_job_template
- .before_script_lesser
- .rules:deploy
script:
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- echo -n $GH_PUSH_KEY > ~/.ssh/id_rsa_base64
- base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
- chmod 600 ~/.ssh/id_rsa
- echo -e "Host github.com\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
- add_github_ssh_keys
- git remote remove github &>/dev/null || true
- git remote add github git@github.com:espressif/esp-idf.git
- tools/ci/push_to_github.sh
.deploy_docs_template:
extends: .before_script_lesser
stage: deploy
extends:
- .deploy_job_template
- .before_script_lesser
image: $ESP_IDF_DOC_ENV_IMAGE
tags:
- deploy
- shiny
dependencies:
needs:
- build_docs_html
- build_docs_pdf
variables:
DOCS_BUILD_DIR: "${IDF_PATH}/docs/_build/"
PYTHONUNBUFFERED: 1
script:
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- echo -n $DOCS_DEPLOY_PRIVATEKEY > ~/.ssh/id_rsa_base64
- base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
- chmod 600 ~/.ssh/id_rsa
- echo -e "Host $DOCS_DEPLOY_SERVER\n\tStrictHostKeyChecking no\n\tUser $DOCS_DEPLOY_SERVER_USER\n" >> ~/.ssh/config
- add_doc_server_ssh_keys $DOCS_DEPLOY_PRIVATEKEY $DOCS_DEPLOY_SERVER $DOCS_DEPLOY_SERVER_USER
- export GIT_VER=$(git describe --always)
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ${IDF_PATH}/tools/ci/deploy_docs.py
# deploys docs to CI_DOCKER_REGISTRY webserver, for internal review
deploy_docs_preview:
extends: .deploy_docs_template
extends:
- .deploy_docs_template
- .rules:labels:build_docs-slim
# Override default stage to happen before the post_check
stage: test_deploy
only:
refs:
- triggers
variables:
- $BOT_TRIGGER_WITH_LABEL == null
- $BOT_LABEL_BUILD_DOCS
variables:
TYPE: "preview"
# older branches use DOCS_DEPLOY_KEY, DOCS_SERVER, DOCS_SERVER_USER, DOCS_PATH for preview server so we keep these names for 'preview'
@ -121,15 +88,10 @@ deploy_docs_preview:
# deploy docs to production webserver
deploy_docs_production:
extends: .deploy_docs_template
only:
refs:
# The DOCS_PROD_* variables used by this job are "Protected" so these branches must all be marked "Protected" in Gitlab settings
- master
- /^release\/v/
- /^v\d+\.\d+(\.\d+)?($|-)/
variables:
- $BOT_TRIGGER_WITH_LABEL == null
# The DOCS_PROD_* variables used by this job are "Protected" so these branches must all be marked "Protected" in Gitlab settings
extends:
- .deploy_docs_template
- .rules:deploy
variables:
TYPE: "preview"
DOCS_DEPLOY_PRIVATEKEY: "$DOCS_PROD_DEPLOY_KEY"
@ -139,15 +101,13 @@ deploy_docs_production:
DOCS_DEPLOY_URL_BASE: "https://docs.espressif.com/projects/esp-idf"
deploy_test_result:
stage: deploy
extends:
- .deploy_job_template
- .before_script_slim
- .rules:master-schedule-always
image: $CI_DOCKER_REGISTRY/bot-env
tags:
- deploy_test
when: always
only:
refs:
- master
- schedules
artifacts:
when: always
paths:
@ -161,14 +121,8 @@ deploy_test_result:
BOT_ACCOUNT_CONFIG_FILE: "${CI_PROJECT_DIR}/test-management/Config/Account.local.yml"
TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
AUTO_TEST_SCRIPT_PATH: "${CI_PROJECT_DIR}/auto_test_script"
before_script:
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- echo -n $GITLAB_KEY > ~/.ssh/id_rsa_base64
- base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
- chmod 600 ~/.ssh/id_rsa
- echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
script:
- add_gitlab_ssh_keys
- export GIT_SHA=$(echo ${CI_COMMIT_SHA} | cut -c 1-8)
- export REV_COUNT=$(git rev-list --count HEAD)
- export SUMMARY="IDF CI test result for $GIT_SHA (r${REV_COUNT})"

46
tools/ci/config/host-test.yml
View File

@ -1,31 +1,22 @@
.host_test_template:
extends: .rules:labels:host_test
stage: host_test
image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
tags:
- host_test
dependencies: []
only:
variables:
- $BOT_TRIGGER_WITH_LABEL == null
- $BOT_LABEL_HOST_TEST
- $BOT_LABEL_REGULAR_TEST
.host_fuzzer_test_template:
stage: host_test
extends:
- .host_test_template
- .rules:labels:fuzzer_test-weekend_test
image: $CI_DOCKER_REGISTRY/afl-fuzzer-test
tags:
- host_test
dependencies: []
artifacts:
when: always
paths:
- ${FUZZER_TEST_DIR}/out/crashes
- ${FUZZER_TEST_DIR}/fuzz_output.log
expire_in: 1 week
only:
variables:
- $BOT_LABEL_FUZZER_TEST
- $BOT_LABEL_WEEKEND_TEST
script:
- export AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 && export AFL_SKIP_CPUFREQ=1
- cd ${FUZZER_TEST_DIR}
@ -42,16 +33,13 @@ test_nvs_on_host:
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh make test
test_nvs_coverage:
extends: .host_test_template
extends:
- .host_test_template
- .rules:labels:nvs_coverage
artifacts:
paths:
- components/nvs_flash/test_nvs_host/coverage_report
expire_in: 1 week
only:
refs:
- triggers
variables:
- $BOT_LABEL_NVS_COVERAGE
script:
- cd components/nvs_flash/test_nvs_host
- make coverage_report
@ -279,24 +267,16 @@ test_mkdfu:
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_mkdfu.py
test_docs:
stage: host_test
extends: .host_test_template
image: $ESP_IDF_DOC_ENV_IMAGE
tags:
- host_test
dependencies: []
only:
variables:
- $BOT_TRIGGER_WITH_LABEL == null
- $BOT_LABEL_HOST_TEST
- $BOT_LABEL_REGULAR_TEST
artifacts:
when: on_failure
paths:
- docs/test/_build/*/*/*/html/*
- docs/test/_build/*/*/*/html/*
expire_in: 1 week
script:
- cd ${IDF_PATH}/docs/test
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r ${IDF_PATH}/docs/requirements.txt
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./test_docs.py
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./test_sphinx_idf_extensions.py
- cd ${IDF_PATH}/docs/test
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r ${IDF_PATH}/docs/requirements.txt
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./test_docs.py
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./test_sphinx_idf_extensions.py

46
tools/ci/config/post_check.yml
View File

@ -2,9 +2,25 @@
.show_submodule_urls: &show_submodule_urls |
git config --get-regexp '^submodule\..*\.url$' || true
check_submodule_sync:
extends: .check_job_template
.post_check_base_template:
stage: post_check
image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
tags:
- host_test
dependencies: []
.post_check_job_template:
extends:
- .post_check_base_template
- .before_script_lesser_nofilter
.post_check_job_template_with_filter:
extends:
- .post_check_base_template
- .before_script_lesser
check_submodule_sync:
extends: .post_check_job_template
tags:
- github_sync
retry: 2
@ -13,7 +29,6 @@ check_submodule_sync:
SUBMODULES_TO_FETCH: "none"
PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git"
before_script: []
after_script: []
script:
- git submodule deinit --force .
# setting the default remote URL to the public one, to resolve relative location URLs
@ -26,40 +41,31 @@ check_submodule_sync:
allow_failure: true # remove this line when esp32s3 support in esptool got merged into master
check_ut_cmake_make:
extends: .check_job_template_with_filter
stage: post_check
image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
extends:
- .post_check_job_template_with_filter
- .rules:dev_push-trigger
tags:
- build
except:
- master
- /^release\/v/
- /^v\d+\.\d+(\.\d+)?($|-)/
dependencies: []
script:
- tools/ci/check_ut_cmake_make.sh
check_artifacts_expire_time:
extends: .check_job_template
stage: post_check
extends: .post_check_job_template
script:
# check if we have set expire time for all artifacts
- python tools/ci/check_artifacts_expire_time.py
check_pipeline_triggered_by_label:
extends: .check_job_template
stage: post_check
only:
variables:
- $BOT_TRIGGER_WITH_LABEL
extends:
- .post_check_job_template
- .rules:dev_push-trigger
script:
# If the pipeline is triggered with label, the pipeline will only succeeded if "regular_test" label is added.
# We want to make sure some jobs are always executed to detect regression.
- test "$BOT_LABEL_REGULAR_TEST" = "true" || { echo "CI can only pass if 'regular_test' label is included"; exit -1; }
check_commit_msg:
extends: .check_job_template
stage: post_check
extends: .post_check_job_template
script:
- git status
- git log -n10 --oneline

7
tools/ci/config/post_deploy.yml
View File

@ -1,11 +1,8 @@
.check_doc_links_template: &check_doc_links_template
.check_doc_links_template:
extends: .rules:master-release
stage: post_deploy
image: $ESP_IDF_DOC_ENV_IMAGE
tags: [ "build", "amd64", "internet" ]
only:
- master
- /^release\/v/
- /^v\d+\.\d+(\.\d+)?($|-)/
artifacts:
when: always
paths:

117
tools/ci/config/pre_check.yml
View File

@ -1,61 +1,63 @@
.pre_check_base_template:
stage: pre_check
image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
tags:
- host_test
dependencies: []
.pre_check_job_template:
extends:
- .pre_check_base_template
- .before_script_lesser_nofilter
.pre_check_job_template_with_filter:
extends:
- .pre_check_base_template
- .before_script_lesser
check_line_endings:
extends: .check_job_template
extends: .pre_check_job_template
script:
- tools/ci/check-line-endings.sh ${IDF_PATH}
check_permissions:
extends: .check_job_template
extends: .pre_check_job_template
script:
- tools/ci/check-executable.sh
check_docs_lang_sync:
extends: .check_job_template
stage: pre_check
extends: .pre_check_job_template
variables:
SUBMODULES_TO_FETCH: "none"
script:
- cd docs
- ./check_lang_folder_sync.sh
check_docs_gh_links:
extends: .build_docs_template
stage: pre_check
variables:
SUBMODULES_TO_FETCH: "none"
script:
- cd docs
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py gh-linkcheck
check_version:
extends: .check_job_template
# Don't run this for feature/bugfix branches, so that it is possible to modify
# esp_idf_version.h in a branch before tagging the next version.
only:
- master
- /^release\/v/
- /^v\d+\.\d+(\.\d+)?($|-)/
extends:
- .pre_check_job_template
- .rules:master-release
script:
- export IDF_PATH=$PWD
- tools/ci/check_idf_version.sh
check_examples_cmake_make:
extends: .check_job_template_with_filter
except:
- master
- /^release\/v/
- /^v\d+\.\d+(\.\d+)?($|-)/
extends:
- .pre_check_job_template_with_filter
- .rules:dev_push-trigger
script:
- python ${IDF_PATH}/tools/ci/check_examples_cmake_make.py
check_rom_api_header:
extends: .check_job_template_with_filter
extends: .pre_check_job_template_with_filter
script:
- tools/ci/check_examples_rom_header.sh
- tools/ci/check_rom_apis.sh
check_python_style:
extends: .python_lint_template
extends: .pre_check_base_template
artifacts:
when: on_failure
paths:
@ -65,7 +67,7 @@ check_python_style:
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh python -m flake8 --config=$IDF_PATH/.flake8 --output-file=flake8_output.txt --tee --benchmark $IDF_PATH
check_kconfigs:
extends: .check_job_template_with_filter
extends: .pre_check_job_template_with_filter
artifacts:
when: on_failure
paths:
@ -81,51 +83,31 @@ check_kconfigs:
- ${IDF_PATH}/tools/check_kconfigs.py
check_deprecated_kconfig_options:
extends: .check_job_template_with_filter
extends: .pre_check_job_template_with_filter
script:
- ${IDF_PATH}/tools/ci/check_deprecated_kconfigs.py
check_cmake_style:
extends: .check_job_template
only:
variables:
- $BOT_TRIGGER_WITH_LABEL == null
- $BOT_LABEL_BUILD
- $BOT_LABEL_REGULAR_TEST
extends: .pre_check_job_template
script:
tools/cmake/run_cmake_lint.sh
check_wifi_lib_md5:
stage: pre_check
image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
extends: .pre_check_base_template
tags:
- build
variables:
SUBMODULES_TO_FETCH: "components/esp_wifi/lib"
dependencies: []
script:
- IDF_TARGET=esp32 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
- IDF_TARGET=esp32s2 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
# Build at least one project for each target at earliest stage to reduce build cost for obvious failing commits
fast_template_app:
extends: .build_template_app_template
stage: pre_check
variables:
BUILD_COMMAND_ARGS: "-p"
check_public_headers:
stage: pre_check
image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
extends:
- .pre_check_base_template
- .rules:labels:build
tags:
- build
variables:
BATCH_BUILD: "1"
only:
variables:
- $BOT_TRIGGER_WITH_LABEL == null
- $BOT_LABEL_BUILD
- $BOT_LABEL_REGULAR_TEST
script:
- python tools/ci/check_public_headers.py --jobs 4 --prefix xtensa-esp32-elf-
@ -140,16 +122,8 @@ check_public_headers:
scan_tests:
extends:
- .before_script_for_component_ut
- .scan_build_tests
only:
variables:
- $BOT_TRIGGER_WITH_LABEL == null
- $BOT_LABEL_REGULAR_TEST
- $BOT_LABEL_EXAMPLE_TEST
- $BOT_LABEL_CUSTOM_TEST
- $BOT_LABEL_UNIT_TEST
- $BOT_LABEL_UNIT_TEST_S2
- .rules:tests:build-ttfw_tests
artifacts:
paths:
- $EXAMPLE_TEST_OUTPUT_DIR
@ -163,13 +137,14 @@ scan_tests:
COMPONENT_UT_OUTPUT_DIR: ${CI_PROJECT_DIR}/component_ut/test_configs
PYTHON_VER: 3
script:
- set_component_ut_vars
- python $CI_SCAN_TESTS_PY example_test $EXAMPLE_TEST_DIR -b make --exclude examples/build_system/idf_as_lib -c $TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR
- python $CI_SCAN_TESTS_PY example_test $EXAMPLE_TEST_DIR -b cmake --exclude examples/build_system/idf_as_lib -c $TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR
- python $CI_SCAN_TESTS_PY test_apps $TEST_APPS_TEST_DIR -c $TEST_CONFIG_FILE -o $TEST_APPS_OUTPUT_DIR
- python $CI_SCAN_TESTS_PY component_ut $COMPONENT_UT_DIRS --exclude $COMPONENT_UT_EXCLUDES -c $TEST_CONFIG_FILE -o $COMPONENT_UT_OUTPUT_DIR
check_readme_links:
extends: .check_job_template
extends: .pre_check_job_template
tags: [ "amd64", "deploy", "internet" ]
allow_failure: true
variables:
@ -178,11 +153,8 @@ check_readme_links:
- python ${IDF_PATH}/tools/ci/check_readme_links.py
.clang_tidy_check_template:
stage: pre_check
extends: .pre_check_base_template
image: ${CI_DOCKER_REGISTRY}/clang-static-analysis
tags:
- host_test
dependencies: []
artifacts:
reports:
junit: $IDF_PATH/output.xml
@ -212,18 +184,15 @@ clang_tidy_check_all:
BOT_LABEL_STATIC_ANALYSIS_ALL: 1
check_codeowners:
extends: .check_job_template
extends: .pre_check_job_template
script:
- tools/codeowners.py ci-check
# For release tag pipelines only, make sure the tag was created with 'git tag -a' so it will update
# the version returned by 'git describe'
check_version_tag:
extends: .check_job_template
only:
refs:
- /^v\d+\.\d+(\.\d+)?($|-)/
variables:
- $BOT_TRIGGER_WITH_LABEL == null
extends:
- .pre_check_job_template
- .rules:release-tag
script:
- (git cat-file -t $CI_COMMIT_REF_NAME | grep tag) || echo "ESP-IDF versions must be annotated tags." && exit 1

239
tools/ci/config/rules.yml Normal file
View File

@ -0,0 +1,239 @@
.if-master-refs: &if-master-refs
if: '$CI_COMMIT_REF_NAME == "master"'
.if-release-branch: &if-release-branch
if: '$CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/'
.if-release-tag-regular: &if-release-tag-regular
if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/ && $BOT_TRIGGER_WITH_LABEL == null'
.if-master-release-regular: &if-master-release-regular
if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && $BOT_TRIGGER_WITH_LABEL == null'
.if-dev-push: &if-dev-push
if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")'
.if-schedule: &if-schedule
if: '$CI_PIPELINE_SOURCE == "schedule"'
.if-trigger: &if-trigger
if: '$CI_PIPELINE_SOURCE == "trigger"'
.if-label-regular-test: &if-label-regular-test
if: '$BOT_LABEL_REGULAR_TEST'
.if-label-build: &if-label-build
if: '$BOT_LABEL_BUILD'
.if-label-build-docs: &if-label-build-docs
if: '$BOT_LABEL_BUILD_DOCS'
.if-label-integration-test: &if-label-integration-test
if: '$BOT_LABEL_INTEGRATION_TEST'
.if-label-unit-test: &if-label-unit-test
if: '$BOT_LABEL_UNIT_TEST'
.if-label-unit-test-s2: &if-label-unit-test-s2
if: '$BOT_LABEL_UNIT_TEST_S2'
.if-label-unit-test-all-target: &if-label-unit-test-all-target
if: '$BOT_LABEL_UNIT_TEST || $BOT_LABEL_UNIT_TEST_S2'
.if-label-weekend-test: &if-label-weekend-test
if: '$BOT_LABEL_WEEKEND_TEST'
.if-label-example-test: &if-label-example-test
if: '$BOT_LABEL_EXAMPLE_TEST'
.if-label-custom-test: &if-label-custom-test
if: '$BOT_LABEL_CUSTOM_TEST'
.if-label-host-test: &if-label-host-test
if: '$BOT_LABEL_HOST_TEST'
.if-label-fuzzer-test: &if-label-fuzzer-test
if: '$BOT_LABEL_FUZZER_TEST'
.if-label-nvs-coverage: &if-label-nvs-coverage
if: '$BOT_LABEL_NVS_COVERAGE'
.if-label-static-analysis: &if-label-static-analysis
if: '$BOT_LABEL_STATIC_ANALYSIS || $BOT_LABEL_STATIC_ANALYSIS_ALL'
.if-label-iperf-stress-test: &if-label-iperf-stress-test
if: '$BOT_LABEL_IPERF_STRESS_TEST'
.if-os-mac: &if-os-mac
if: '$BOT_LABEL_MACOS_TEST'
.rules:master-release:
rules:
- <<: *if-master-refs
- <<: *if-release-branch
.rules:release-tag:
rules:
- <<: *if-release-tag-regular
.rules:deploy:
rules:
- <<: *if-master-release-regular
.rules:master-schedule:
rules:
- <<: *if-master-refs
- <<: *if-schedule
.rules:master-schedule-always:
rules:
- <<: *if-master-refs
when: always
- <<: *if-schedule
when: always
.rules:master-release-schedule:
rules:
- <<: *if-master-refs
- <<: *if-release-branch
- <<: *if-schedule
.rules:trigger:
rules:
- <<: *if-trigger
.rules:dev_push-trigger:
rules:
- <<: *if-trigger
- <<: *if-dev-push
.rules:labels:static_analysis:
rules:
- <<: *if-label-static-analysis
.rules:labels:build:
rules:
- <<: *if-master-release-regular
- <<: *if-label-regular-test
- <<: *if-label-build
.rules:labels:build_docs:
rules:
- <<: *if-master-release-regular
- <<: *if-label-build
- <<: *if-label-regular-test
- <<: *if-label-build-docs
.rules:labels:build_docs-slim:
rules:
- <<: *if-master-release-regular
- <<: *if-label-build-docs
.rules:labels:host_test:
rules:
- <<: *if-master-release-regular
- <<: *if-label-regular-test
- <<: *if-label-host-test
.rules:labels:example_test:
rules:
- <<: *if-master-release-regular
- <<: *if-schedule
- <<: *if-label-example-test
.rules:labels:custom_test:
rules:
- <<: *if-master-release-regular
- <<: *if-schedule
- <<: *if-label-custom-test
.rules:labels:unit_test:
rules:
- <<: *if-master-release-regular
- <<: *if-label-unit-test
.rules:labels:unit_test_s2-only:
rules:
- <<: *if-label-unit-test-s2
.rules:labels:integration_test:
rules:
- <<: *if-master-release-regular
- <<: *if-label-integration-test
.rules:labels:weekend_test-only:
rules:
- <<: *if-label-weekend-test
.rules:labels:iperf_stress_test-only:
rules:
- <<: *if-label-iperf-stress-test
.rules:labels:build-integration_test:
rules:
- <<: *if-master-release-regular
- <<: *if-label-build
- <<: *if-label-regular-test
- <<: *if-label-integration-test
.rules:labels:build-weekend_test:
rules:
- <<: *if-master-release-regular
- <<: *if-label-build
- <<: *if-label-regular-test
- <<: *if-label-weekend-test
.rules:labels:build-unit_test:
rules:
- <<: *if-master-release-regular
- <<: *if-label-build
- <<: *if-label-regular-test
- <<: *if-label-unit-test-all-target
.rules:labels:build-example_test-weekend_test:
rules:
- <<: *if-master-release-regular
- <<: *if-label-build
- <<: *if-label-regular-test
- <<: *if-label-example-test
- <<: *if-label-weekend-test
.rules:labels:build-custom_test-weekend_test:
rules:
- <<: *if-master-release-regular
- <<: *if-label-build
- <<: *if-label-regular-test
- <<: *if-label-custom-test
- <<: *if-label-weekend-test
.rules:labels:fuzzer_test-weekend_test:
rules:
- <<: *if-label-fuzzer-test
- <<: *if-label-weekend-test
.rules:labels:nvs_coverage:
rules:
- <<: *if-label-nvs-coverage
.rules:test:target_test-weekend_test:
rules:
- <<: *if-master-release-regular
- <<: *if-label-example-test
- <<: *if-label-custom-test
- <<: *if-label-unit-test-all-target
- <<: *if-label-integration-test
- <<: *if-label-weekend-test
.rules:tests:build-ttfw_tests:
rules:
- <<: *if-master-release-regular
- <<: *if-label-build
- <<: *if-label-regular-test
- <<: *if-label-example-test
- <<: *if-label-custom-test
- <<: *if-label-unit-test-all-target
.rules:os:mac_os:
rules:
- <<: *if-master-release-regular
- <<: *if-os-mac

295
tools/ci/config/target-test.yml
View File

@ -1,24 +1,13 @@
# for parallel jobs, CI_JOB_NAME will be "job_name index/total" (for example, "IT_001 1/2")
# we need to convert to pattern "job_name_index.yml"
.define_config_file_name: &define_config_file_name |
JOB_NAME_PREFIX=$(echo ${CI_JOB_NAME} | awk '{print $1}')
JOB_FULL_NAME="${JOB_NAME_PREFIX}_${CI_NODE_INDEX}"
CONFIG_FILE="${CONFIG_FILE_PATH}/${JOB_FULL_NAME}.yml"
JOB_NAME_PREFIX=$(echo ${CI_JOB_NAME} | awk '{print $1}')
JOB_FULL_NAME="${JOB_NAME_PREFIX}_${CI_NODE_INDEX}"
CONFIG_FILE="${CONFIG_FILE_PATH}/${JOB_FULL_NAME}.yml"
.example_test_template:
.target_test_job_template:
stage: target_test
when: on_success
only:
refs:
- master
- /^release\/v/
- /^v\d+\.\d+(\.\d+)?($|-)/
- triggers
- schedules
variables:
- $BOT_TRIGGER_WITH_LABEL == null
- $BOT_LABEL_EXAMPLE_TEST
dependencies:
needs:
- assign_test
artifacts:
when: always
@ -26,10 +15,9 @@
- $LOG_PATH
expire_in: 1 week
reports:
junit: $LOG_PATH/*/XUNIT_RESULT.xml
junit: $LOG_PATH/*/XUNIT_RESULT.xml
variables:
TEST_CASE_PATH: "$CI_PROJECT_DIR/examples"
CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/examples/test_configs"
TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"
SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
@ -38,133 +26,70 @@
# first test if config file exists, if not exist, exit 0
- test -e $CONFIG_FILE || exit 0
# clone test env configs
- ./tools/ci/retry_failed.sh git clone $TEST_ENV_CONFIG_REPO
- retry_failed git clone $TEST_ENV_CONFIG_REPO
- python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
- cd tools/ci/python_packages/tiny_test_fw/bin
# run test
- python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE
.example_debug_template:
stage: target_test
when: on_success
only:
refs:
- master
- /^release\/v/
- /^v\d+\.\d+(\.\d+)?($|-)/
- triggers
- schedules
variables:
- $BOT_TRIGGER_WITH_LABEL == null
- $BOT_LABEL_EXAMPLE_TEST
dependencies:
- assign_test
artifacts:
when: always
paths:
- $LOG_PATH
expire_in: 1 week
reports:
junit: $LOG_PATH/*/XUNIT_RESULT.xml
.example_test_template:
extends:
- .target_test_job_template
- .rules:labels:example_test
variables:
TEST_CASE_PATH: "$CI_PROJECT_DIR/examples"
CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/examples/test_configs"
LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"
script:
- *define_config_file_name
# first test if config file exists, if not exist, exit 0
- test -e $CONFIG_FILE || exit 0
# clone test env configs
- ./tools/ci/retry_failed.sh git clone $TEST_ENV_CONFIG_REPO
- python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
- cd tools/ci/python_packages/tiny_test_fw/bin
# run test
- python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE
.example_debug_template:
extends: .example_test_template
variables:
SUBMODULES_TO_FETCH: "all"
.test_app_template:
extends: .example_test_template
only:
variables:
- $BOT_TRIGGER_WITH_LABEL == null
- $BOT_LABEL_CUSTOM_TEST
extends:
- .target_test_job_template
- .rules:labels:custom_test
variables:
TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/test_apps"
CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/tools/test_apps/test_configs"
LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"
.component_ut_template:
extends:
- .before_script_for_component_ut
- .example_test_template
only:
variables:
- $BOT_TRIGGER_WITH_LABEL == null
- $BOT_LABEL_UNIT_TEST
- .target_test_job_template
- .rules:labels:unit_test
variables:
TEST_CASE_PATH: "$COMPONENT_UT_DIRS"
CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/component_ut/test_configs"
PYTHON_VER: 3
script:
- *define_config_file_name
# first test if config file exists, if not exist, exit 0
- test -e $CONFIG_FILE || exit 0
- set_component_ut_vars
# clone test env configs
- ./tools/ci/retry_failed.sh git clone $TEST_ENV_CONFIG_REPO
- retry_failed git clone $TEST_ENV_CONFIG_REPO
- python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
- cd tools/ci/python_packages/tiny_test_fw/bin
# run test
- python Runner.py $COMPONENT_UT_DIRS -c $CONFIG_FILE -e $ENV_FILE
- python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE
.unit_test_template:
extends: .example_test_template
stage: target_test
dependencies:
- assign_test
only:
refs:
- master
- /^release\/v/
- /^v\d+\.\d+(\.\d+)?($|-)/
- triggers
- schedules
variables:
- $BOT_TRIGGER_WITH_LABEL == null
- $BOT_LABEL_UNIT_TEST
extends:
- .target_test_job_template
- .rules:labels:unit_test
variables:
TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/unit-test-app"
CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/unit_test/CIConfigs"
LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"
PYTHON_VER: 3
.test_template:
stage: target_test
when: on_success
only:
refs:
- master
- /^release\/v/
- /^v\d+\.\d+(\.\d+)?($|-)/
- triggers
- schedules
variables:
- $BOT_TRIGGER_WITH_LABEL == null
- $BOT_LABEL_INTEGRATION_TEST
dependencies:
.integration_test_template:
extends:
- .target_test_job_template
- .rules:labels:integration_test
needs:
- assign_test
- build_ssc_esp32
artifacts:
when: always
reports:
junit: $LOG_PATH/*/XUNIT_RESULT.xml
paths:
- $LOG_PATH
expire_in: 1 week
variables:
SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF"
LOG_PATH: "${CI_PROJECT_DIR}/${CI_COMMIT_SHA}"
TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/auto_test_script/TestCaseFiles"
@ -179,56 +104,48 @@
# first test if config file exists, if not exist, exit 0
- test -e $CONFIG_FILE || exit 0
# clone local test env configs
- ./tools/ci/retry_failed.sh git clone $TEST_ENV_CONFIG_REPO
- retry_failed git clone $TEST_ENV_CONFIG_REPO
- python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
# clone test bench
# can not retry if downing git lfs files failed, so using empty_branch first.
- ./tools/ci/retry_failed.sh git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
- ./tools/ci/retry_failed.sh git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
- retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
- retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
- python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
- cat ${KNOWN_ISSUE_FILE} >> ${TEST_CASE_FILE_PATH}/KnownIssues
# run test
- python ${CI_RUNNER_SCRIPT} -l "$LOG_PATH/$JOB_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH
test_weekend_mqtt:
extends: .example_test_template
stage: target_test
extends:
- .example_test_template
- .rules:labels:weekend_test-only
tags:
- ESP32
- Example_WIFI
only:
variables:
- $BOT_LABEL_WEEKEND_TEST
variables:
TEST_CASE_PATH: "$CI_PROJECT_DIR/components/mqtt/weekend_test"
LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
ENV_FILE: "$CI_PROJECT_DIR/components/mqtt/weekend_test/env.yml"
TEST_CASE_PATH: "$CI_PROJECT_DIR/components/mqtt/weekend_test"
CONFIG_FILE_PATH: "$CI_PROJECT_DIR/components/mqtt/weekend_test"
test_weekend_network:
extends: .example_test_template
stage: target_test
extends:
- .example_test_template
- .rules:labels:weekend_test-only
image: $CI_DOCKER_REGISTRY/rpi-net-suite$BOT_DOCKER_IMAGE_TAG
tags:
- ESP32
- Example_WIFI
only:
variables:
- $BOT_LABEL_WEEKEND_TEST
variables:
TEST_CASE_PATH: "$CI_PROJECT_DIR/components/lwip/weekend_test"
LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
ENV_FILE: "$CI_PROJECT_DIR/components/lwip/weekend_test/env.yml"
TEST_CASE_PATH: "$CI_PROJECT_DIR/components/lwip/weekend_test"
CONFIG_FILE_PATH: "$CI_PROJECT_DIR/components/lwip/weekend_test"
example_test_001A:
extends: .example_test_template
parallel: 4
artifacts:
when: always
paths:
- $CI_PROJECT_DIR/examples/*/*/*.log
expire_in: 1 week
paths:
- $CI_PROJECT_DIR/examples/*/*/*.log
tags:
- ESP32
- Example_WIFI
@ -252,17 +169,6 @@ example_test_002:
tags:
- ESP32
- Example_ShieldBox_Basic
script:
- *define_config_file_name
# first test if config file exists, if not exist, exit 0
- test -e $CONFIG_FILE || exit 0
# clone test env configs
- ./tools/ci/retry_failed.sh git clone $TEST_ENV_CONFIG_REPO
- python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
- cd tools/ci/python_packages/tiny_test_fw/bin
# run test
- python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE
.example_test_003:
extends: .example_test_template
@ -289,11 +195,10 @@ example_test_005:
- Example_WIFI_BT
example_test_006:
extends: .example_test_template
extends:
- .example_test_template
- .rules:labels:iperf_stress_test-only
image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
only:
variables:
- $BOT_LABEL_IPERF_STRESS_TEST
tags:
- ESP32
- Example_ShieldBox
@ -322,10 +227,8 @@ example_test_009:
- ESP32
- test_jtag_arm
artifacts:
when: always
paths:
- $CI_PROJECT_DIR/examples/*/*/*.log
expire_in: 1 week
paths:
- $CI_PROJECT_DIR/examples/*/*/*.log
variables:
SETUP_TOOLS: "1"
PYTHON_VER: 3
@ -342,11 +245,9 @@ example_test_011:
- ESP32
- Example_T2_RS485
artifacts:
when: always
expire_in: 1 week
paths:
- $CI_PROJECT_DIR/examples/protocols/modbus/serial/*.log
- $LOG_PATH
paths:
- $CI_PROJECT_DIR/examples/protocols/modbus/serial/*.log
- $LOG_PATH
variables:
SETUP_TOOLS: "1"
@ -374,11 +275,9 @@ example_test_015:
- ESP32
- Example_PPP
artifacts:
when: always
expire_in: 1 week
paths:
- $CI_PROJECT_DIR/examples/*/*/*.log
- $LOG_PATH
paths:
- $CI_PROJECT_DIR/examples/*/*/*.log
- $LOG_PATH
example_test_016:
extends: .example_test_template
@ -386,11 +285,9 @@ example_test_016:
- ESP32
- Example_Modbus_TCP
artifacts:
when: always
expire_in: 1 week
paths:
- $CI_PROJECT_DIR/examples/*/*/*.log
- $LOG_PATH
paths:
- $CI_PROJECT_DIR/examples/*/*/*.log
- $LOG_PATH
test_app_test_001:
extends: .test_app_template
@ -399,10 +296,8 @@ test_app_test_001:
- ESP32
- test_jtag_arm
artifacts:
when: always
paths:
- $CI_PROJECT_DIR/tools/test_apps/system/*/*.log
expire_in: 1 week
paths:
- $CI_PROJECT_DIR/tools/test_apps/system/*/*.log
variables:
SETUP_TOOLS: "1"
PYTHON_VER: 3
@ -578,21 +473,9 @@ UT_034:
- UT_T1_ESP_FLASH
.unit_test_s2_template:
extends: .unit_test_template
dependencies:
- assign_test
only:
refs:
# Due to lack of runners, the tests are only done by manual trigger
# - master
# - /^release\/v/
# - /^v\d+\.\d+(\.\d+)?($|-)/
- triggers
- schedules
variables:
# Due to lack of runners, S2 tests will only be triggered with label (@bot test will not trigger)
# - $BOT_TRIGGER_WITH_LABEL == null
- $BOT_LABEL_UNIT_TEST_S2
extends:
- .unit_test_template
- .rules:labels:unit_test_s2-only # due to the lack of runners, s2 tests will only be triggered by label
UT_035:
extends: .unit_test_s2_template
@ -610,7 +493,7 @@ UT_036:
# ToDo: re-enable this job when ESP32-S2 LEDC runner installed
# UT_037:
# extends: .unit_test_template
# extends: .unit_test_s2_template
# tags:
# - ESP32S2_IDF
# - UT_T1_LEDC
@ -648,7 +531,7 @@ UT_045:
- ESP32_IDF
- UT_SDIO
- psram
UT_046:
extends: .unit_test_template
tags:
@ -656,7 +539,7 @@ UT_046:
- UT_T1_GPIO
nvs_compatible_test:
extends: .test_template
extends: .integration_test_template
artifacts:
when: always
paths:
@ -685,129 +568,129 @@ nvs_compatible_test:
- python ${CI_RUNNER_SCRIPT} -l "$LOG_PATH/$JOB_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH
IT_001:
extends: .test_template
extends: .integration_test_template
parallel: 3
tags:
- ESP32_IDF
- SSC_T1_4
IT_002:
extends: .test_template
extends: .integration_test_template
tags:
- ESP32_IDF
- SSC_T1_2
IT_003:
extends: .test_template
extends: .integration_test_template
parallel: 14
tags:
- ESP32_IDF
- SSC_T2_5
IT_004:
extends: .test_template
extends: .integration_test_template
tags:
- ESP32_IDF
- SSC_T1_APC
IT_005:
extends: .test_template
extends: .integration_test_template
parallel: 2
tags:
- ESP32_IDF
- SSC_T1_5
IT_006:
extends: .test_template
extends: .integration_test_template
parallel: 12
tags:
- ESP32_IDF
- SSC_T1_6
IT_007:
extends: .test_template
extends: .integration_test_template
parallel: 3
tags:
- ESP32_IDF
- SSC_T1_7
IT_008:
extends: .test_template
extends: .integration_test_template
tags:
- ESP32_IDF
- SSC_T1_8
IT_009:
extends: .test_template
extends: .integration_test_template
tags:
- ESP32_IDF
- SSC_T1_3
IT_011:
extends: .test_template
extends: .integration_test_template
tags:
- ESP32_IDF
- SSC_T1_MESH1
IT_012:
extends: .test_template
extends: .integration_test_template
parallel: 2
tags:
- ESP32_IDF
- SSC_T2_MESH1
IT_013:
extends: .test_template
extends: .integration_test_template
tags:
- ESP32_IDF
- SSC_T3_MESH1
IT_014:
extends: .test_template
extends: .integration_test_template
tags:
- ESP32_IDF
- SSC_T6_MESH1
IT_015:
extends: .test_template
extends: .integration_test_template
tags:
- ESP32_IDF
- SSC_T12_MESH1
IT_016:
extends: .test_template
extends: .integration_test_template
allow_failure: true
tags:
- ESP32_IDF
- SSC_T50_MESH1
IT_017:
extends: .test_template
extends: .integration_test_template
tags:
- ESP32_IDF
- SSC_T1_MESH2
IT_018:
extends: .test_template
extends: .integration_test_template
tags:
- ESP32_IDF
- SSC_T1_9
IT_019:
extends: .test_template
extends: .integration_test_template
parallel: 2
tags:
- ESP32_IDF
- SSC_T2_2
IT_020:
extends: .test_template
extends: .integration_test_template
tags:
- ESP32_IDF
- SSC_T2_3
IT_021:
extends: .test_template
extends: .integration_test_template
tags:
- ESP32_IDF
- SSC_T2_4

1
tools/ci/executable-list.txt
View File

@ -62,6 +62,7 @@ tools/ci/retry_failed.sh
tools/ci/test_build_system.sh
tools/ci/test_build_system_cmake.sh
tools/ci/test_configure_ci_environment.sh
tools/ci/utils.sh
tools/cmake/convert_to_cmake.py
tools/cmake/run_cmake_lint.sh
tools/codeowners.py

48
tools/ci/utils.sh Executable file
View File

@ -0,0 +1,48 @@
# Modified from https://gitlab.com/gitlab-org/gitlab/-/blob/master/scripts/utils.sh
# before each job, we need to check if this job is filtered by bot stage/job filter
function apply_bot_filter() {
python "${IDF_PATH}"/tools/ci/apply_bot_filter.py || exit 0
}
function add_ssh_keys() {
local key_string="${1}"
mkdir -p ~/.ssh
chmod 700 ~/.ssh
echo -n "${key_string}" >~/.ssh/id_rsa_base64
base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 >~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
}
function add_gitlab_ssh_keys() {
add_ssh_keys "${GITLAB_KEY}"
echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >>~/.ssh/config
}
function add_github_ssh_keys() {
add_ssh_keys "${GH_PUSH_KEY}"
echo -e "Host github.com\n\tStrictHostKeyChecking no\n" >>~/.ssh/config
}
function add_doc_server_ssh_keys() {
local key_string="${1}"
local server_url="${2}"
local server_user="${3}"
add_ssh_keys "${key_string}"
echo -e "Host ${server_url}\n\tStrictHostKeyChecking no\n\tUser ${server_user}\n" >>~/.ssh/config
}
function fetch_submodules() {
python "${SUBMODULE_FETCH_TOOL}" -s "${SUBMODULES_TO_FETCH}"
}
function get_all_submodules() {
echo "$(git config --file .gitmodules --get-regexp path | awk '{ print $2 }' | sed -e 's|$|/**|' | xargs | sed -e 's/ /,/g')"
}
function set_component_ut_vars() {
local exclude_list_fp="${IDF_PATH}/tools/ci/component_ut_excludes.txt"
export COMPONENT_UT_DIRS=$(find components/ -name test_apps -type d)
export COMPONENT_UT_EXCLUDES=$([ -r $exclude_list_fp ] && cat $exclude_list_fp | xargs)
echo "COMPONENT_UT_DIRS, COMPONENT_UT_EXCLUDES written into export"
}