alex/esp-idf
alex/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf.git synced 2024-10-05 20:47:46 -04:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

ci: dynamic integration test child pipeline

Browse Source
This commit is contained in:
Chen Yudong 2023-11-13 22:38:20 +08:00 committed by BOT
parent 525f5a06ad
commit a60cea61a3
10 changed files with 66 additions and 416 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.gitlab-ci.yml
View File

@ -17,7 +17,7 @@ workflow:
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: never
- if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_PIPELINE_SOURCE == "push"'
- if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_COMMIT_TAG !~ /^qa-test/ && $CI_PIPELINE_SOURCE == "push"'
when: never
- when: always
@ -54,9 +54,6 @@ variables:
# Docker images
BOT_DOCKER_IMAGE_TAG: ":latest"
# target test config file, used by assign test job
CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/tools/ci/config/target-test.yml"
# target test repo parameters
TEST_ENV_CONFIG_REPO: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/ci-test-runner-configs.git"
@ -151,6 +148,7 @@ include:
- '/tools/ci/config/pre_check.yml'
- '/tools/ci/config/build.yml'
- '/tools/ci/config/assign-test.yml'
- '/tools/ci/config/integration_test.yml'
- '/tools/ci/config/host-test.yml'
- '/tools/ci/config/target-test.yml'
- '/tools/ci/config/post_check.yml'

57
tools/ci/config/assign-test.yml
View File

@ -1,6 +1,6 @@
assign_test:
extends:
- .rules:assign_test:target_test-integration_test-weekend_test
- .rules:assign_test:target_test-weekend_test
tags:
- assign_test
image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
@ -8,32 +8,23 @@ assign_test:
# gitlab ci do not support match job with RegEx or wildcard now in dependencies.
# we have a lot build example jobs. now we don't use dependencies, just download all artifacts of build stage.
dependencies: # Here is not a hard dependency relationship, could be skipped. so we do not use "needs" here.
- build_ssc_esp32
- build_ssc_esp32c3
- build_esp_idf_tests_cmake_esp32
- build_esp_idf_tests_cmake_esp32s2
- build_esp_idf_tests_cmake_esp32c3
variables:
CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/tools/ci/config/target-test.yml"
SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
EXAMPLE_TEST_DIR: "${CI_PROJECT_DIR}/examples"
CUSTOM_TEST_DIR: "${CI_PROJECT_DIR}/tools/test_apps"
UNIT_TEST_DIR: "${CI_PROJECT_DIR}/components/idf_test/unit_test"
# COMPONENT_UT_DIRS is set by `set_component_ut_vars` in `utils.sh`
COMPONENT_UT_OUTPUT_DIR: "${CI_PROJECT_DIR}/component_ut"
INTEGRATION_TEST_DIR: "${CI_PROJECT_DIR}/tools/ci/integration_test"
INTEGRATION_TEST_CASE_PATH: "${CI_PROJECT_DIR}/auto_test_script/TestCaseFiles"
ASSIGN_TEST_CASE_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIAssignTestCases.py"
PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages
# auto_test_script only supports python 3.7.x
PYTHON_VER: 3.7.7
GIT_LFS_SKIP_SMUDGE: 1
artifacts:
paths:
- components/idf_test/*/CIConfigs
- $EXAMPLE_TEST_DIR/test_configs
- $CUSTOM_TEST_DIR/test_configs
- $COMPONENT_UT_OUTPUT_DIR/test_configs
- $INTEGRATION_TEST_DIR/test_configs
- build_examples/artifact_index.json
- build_test_apps/artifact_index.json
- build_component_ut/artifact_index.json
@ -45,47 +36,3 @@ assign_test:
- python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py custom_test $CUSTOM_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $CUSTOM_TEST_DIR/test_configs
- python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py component_ut $COMPONENT_UT_DIRS -c $CI_TARGET_TEST_CONFIG_FILE -o $COMPONENT_UT_OUTPUT_DIR/test_configs
- python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py unit_test $UNIT_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $UNIT_TEST_DIR/CIConfigs
# clone test script to assign tests
- retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} auto_test_script
- python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
- cd auto_test_script
- ./tools/ci/setup_idfci.sh
# assign integration test cases
- cd ${CI_PROJECT_DIR}
- python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b $IDF_PATH/SSC/ssc_bin -o $INTEGRATION_TEST_DIR/test_configs
update_test_cases:
extends: .rules:ref:master-schedule
stage: assign_test
image: $CI_DOCKER_REGISTRY/ubuntu-test-env
tags:
- deploy_test
needs:
- build_esp_idf_tests_cmake_esp32
- build_esp_idf_tests_cmake_esp32s2
- build_esp_idf_tests_cmake_esp32c3
artifacts:
when: always
paths:
- ${CI_PROJECT_DIR}/test-management/*.log
expire_in: 1 week
variables:
SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
UNIT_TEST_DIR: "${CI_PROJECT_DIR}/components/idf_test/unit_test"
BOT_ACCOUNT_CONFIG_FILE: "${CI_PROJECT_DIR}/test-management/Config/Account.local.yml"
AUTO_TEST_SCRIPT_PATH: "${CI_PROJECT_DIR}/auto_test_script"
PYTHON_VER: 3.7.7
script:
- export GIT_SHA=$(echo ${CI_COMMIT_SHA} | cut -c 1-8)
- retry_failed git clone $TEST_MANAGEMENT_REPO
- python $CHECKOUT_REF_SCRIPT test-management test-management
- cd test-management
- echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE}
# update unit test cases
- export UNIT_TEST_CASE_FILES=$(find $UNIT_TEST_DIR -maxdepth 1 -name "*.yml" | xargs)
- python ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT unity -d $UNIT_TEST_CASE_FILES -r $GIT_SHA
# update example test cases
- python ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT tiny_test_fw -d ${CI_PROJECT_DIR}/examples -r $GIT_SHA
- python ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT tiny_test_fw -d ${CI_PROJECT_DIR}/tools/test_apps -r $GIT_SHA
# organize test cases
- python OrganizeTestCases.py $JIRA_TEST_MANAGEMENT_PROJECT

32
tools/ci/config/build.yml
View File

@ -64,38 +64,6 @@ check_docs_gh_links:
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py gh-linkcheck
#------------------------------------------------------------------------------
.build_ssc_template:
extends:
- .build_template
- .rules:build_tests:integration_test
artifacts:
paths:
- SSC/ssc_bin
expire_in: 1 week
script:
- retry_failed git clone $SSC_REPOSITORY
- python $CHECKOUT_REF_SCRIPT SSC SSC
- cd SSC
- MAKEFLAGS= ./ci_build_ssc.sh $TARGET_NAME
build_ssc_esp32:
extends: .build_ssc_template
parallel: 3
variables:
TARGET_NAME: "ESP32"
build_ssc_esp32s2:
extends: .build_ssc_template
parallel: 2
variables:
TARGET_NAME: "ESP32S2"
build_ssc_esp32c3:
extends: .build_ssc_template
parallel: 3
variables:
TARGET_NAME: "ESP32C3"
.build_esp_idf_tests_cmake:
extends:
- .build_template

35
tools/ci/config/deploy.yml
View File

@ -105,38 +105,3 @@ deploy_docs_production:
DOCS_DEPLOY_SERVER_USER: "$DOCS_PROD_SERVER_USER"
DOCS_DEPLOY_PATH: "$DOCS_PROD_PATH"
DOCS_DEPLOY_URL_BASE: "https://docs.espressif.com/projects/esp-idf"
deploy_test_result:
extends:
- .deploy_job_template
- .before_script_slim
- .rules:ref:master-always
image: $CI_DOCKER_REGISTRY/bot-env
tags:
- deploy_test
artifacts:
when: always
paths:
- ${CI_PROJECT_DIR}/test-management/*.log
# save all test logs as artifacts, make it easier to track errors
- ${CI_PROJECT_DIR}/TEST_LOGS
expire_in: 1 mos
variables:
UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test/TestCaseAll.yml"
BOT_ACCOUNT_CONFIG_FILE: "${CI_PROJECT_DIR}/test-management/Config/Account.local.yml"
TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
AUTO_TEST_SCRIPT_PATH: "${CI_PROJECT_DIR}/auto_test_script"
script:
- add_gitlab_ssh_keys
- export GIT_SHA=$(echo ${CI_COMMIT_SHA} | cut -c 1-8)
- export REV_COUNT=$(git rev-list --count HEAD)
- export SUMMARY="IDF CI test result for $GIT_SHA (r${REV_COUNT})"
# artifacts of job update_test_cases creates test-management folder
# we need to remove it so we can clone test-management folder again
- rm -rf test-management
- retry_failed git clone $TEST_MANAGEMENT_REPO
- python3 $CHECKOUT_REF_SCRIPT test-management test-management
- cd test-management
- echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE}
# update test results
- python3 ImportTestResult.py -r "$GIT_SHA (r${REV_COUNT})" -j $JIRA_TEST_MANAGEMENT_PROJECT -s "$SUMMARY" -l CI -p ${CI_PROJECT_DIR}/TEST_LOGS --pipeline_url ${CI_PIPELINE_URL}

61
tools/ci/config/integration_test.yml Normal file
View File

@ -0,0 +1,61 @@
# generate dynamic integration pipeline by `idf-integration-ci` project
.patterns-integration_test: &patterns-integration_test
# add all possible patterns to make sure `gen_integration_pipeline` can be triggered.
# fine-grained control will be done while generating the pipeline
# find `patterns` in `idf-integration-ci` project
- "components/**/*"
- "tools/**/*"
- ".gitlab-ci.yml"
- ".gitmodules"
- "CMakeLists.txt"
- "install.sh"
- "export.sh"
- "Kconfig"
- "sdkconfig.rename"
# Simplify the rules, mr pipeline not enabled
.integration_test_rules:
rules:
- when: on_success
gen_integration_pipeline:
extends:
- .before_script_lesser
- .integration_test_rules
image: ${CI_INTEGRATION_ASSIGN_ENV}
stage: assign_test
cache: []
tags:
- assign_test
variables:
SUBMODULES_TO_FETCH: "none"
GIT_LFS_SKIP_SMUDGE: 1
needs:
- job: check_pre_commit_MR
artifacts: false
optional: true
artifacts:
paths:
- idf-integration-ci/child_pipeline/
expire_in: 2 weeks
script:
- add_gitlab_ssh_keys
- retry_failed git clone ${CI_GEN_INTEGRATION_PIPELINE_REPO} idf-integration-ci
- python $CHECKOUT_REF_SCRIPT idf-integration-ci idf-integration-ci
- cd idf-integration-ci
- python tools/generate_child_pipeline.py -o child_pipeline/
child_integration_test_pipeline:
extends:
- .integration_test_rules
stage: assign_test
needs:
- gen_integration_pipeline
trigger:
include:
- artifact: idf-integration-ci/child_pipeline/pipeline.yml
job: gen_integration_pipeline
forward:
yaml_variables: false
strategy: depend

16
tools/ci/config/rules.yml
View File

@ -237,28 +237,14 @@
- <<: *if-label-unit_test
- <<: *if-label-unit_test-c3
.rules:tests:integration_test:
rules:
- <<: *if-protected-no_label
- <<: *if-label-integration_test
.rules:assign_test:target_test-integration_test-weekend_test:
.rules:assign_test:target_test-weekend_test:
rules:
- <<: *if-protected-no_label
- <<: *if-label-regular_test
- <<: *if-label-example_test
- <<: *if-label-custom_test
- <<: *if-label-unit_test-all_labels
- <<: *if-label-integration_test
- <<: *if-label-weekend_test
.rules:build_tests:integration_test:
rules:
- <<: *if-protected-no_label
- <<: *if-label-build
- <<: *if-label-regular_test
- <<: *if-label-integration_test
.rules:build_tests:weekend_test:
rules:
- <<: *if-protected-no_label

238
tools/ci/config/target-test.yml
View File

@ -115,50 +115,6 @@
- .unit_test_template
- .rules:tests:unit_test_c3 # due to the lack of runners, c3 tests will only be triggered by label
.integration_test_template:
extends:
- .target_test_job_template
- .rules:tests:integration_test
- .before_script_lesser
image: ${CI_INTEGRATION_TEST_ENV_IMAGE}
needs:
- assign_test
variables:
LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF"
LOG_PATH: "${CI_PROJECT_DIR}/TEST_LOGS"
TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/auto_test_script/TestCaseFiles"
INTEGRATION_TEST_DIR: "${CI_PROJECT_DIR}/tools/ci/integration_test"
CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/tools/ci/integration_test/test_configs"
PREPARE_TEST_BIN_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/tools/ci/idf_prepare_test_bins.py"
CI_RUNNER_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIRunner.py"
PYTHONPATH: "${CI_PROJECT_DIR}/auto_test_script/packages:${CI_PROJECT_DIR}/tools/ci/python_packages:${PYTHONPATH}"
INITIAL_CONDITION_RETRY_COUNT: "1"
GIT_LFS_SKIP_SMUDGE: 1
script:
- *define_config_file_name
# first test if config file exists, if not exist, exit 0
- test -e $CONFIG_FILE || exit 0
- add_gitlab_ssh_keys
# clone local test env configs
- retry_failed git clone $TEST_ENV_CONFIG_REPO
- python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
# clone test bench
- retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} auto_test_script
- python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
- cd auto_test_script
- ./tools/ci/setup_idfci.sh
# Merge known issues
- cat ${INTEGRATION_TEST_DIR}/KnownIssues >> ${TEST_CASE_FILE_PATH}/KnownIssues
# run test
- python ${PREPARE_TEST_BIN_SCRIPT} $CONFIG_FILE
- python ${CI_RUNNER_SCRIPT} -l "$LOG_PATH/$JOB_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH
.integration_test_esp32c3_template:
extends:
- .integration_test_template
variables:
LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32C3_IDF"
test_weekend_mqtt:
extends:
- .test_app_template
@ -643,197 +599,3 @@ UT_C3_I2C:
tags:
- ESP32C3_IDF
- UT_T2_I2C
nvs_compatible_test:
extends: .integration_test_template
artifacts:
when: always
paths:
- $LOG_PATH
- nvs_wifi.bin
expire_in: 1 mos
tags:
- ESP32_IDF
- NVS_Compatible
script:
- *define_config_file_name
# first test if config file exists, if not exist, exit 0
- test -e $CONFIG_FILE || exit 0
- add_gitlab_ssh_keys
# clone local test env configs
- retry_failed git clone $TEST_ENV_CONFIG_REPO
- python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
# clone test bench
- retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} auto_test_script
- python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
- cd auto_test_script
- ./tools/ci/setup_idfci.sh
# prepare nvs bins
- ./tools/prepare_nvs_bin.sh
# run test
- python ${PREPARE_TEST_BIN_SCRIPT} $CONFIG_FILE
- python ${CI_RUNNER_SCRIPT} -l "$LOG_PATH/$JOB_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH
IT_001:
extends: .integration_test_template
parallel: 2
tags:
- ESP32_IDF
- SSC_T1_4
IT_002:
extends: .integration_test_template
tags:
- ESP32_IDF
- SSC_T1_2
IT_003:
extends: .integration_test_template
parallel: 9
tags:
- ESP32_IDF
- SSC_T2_5
IT_T5_BT_Simple:
extends: .integration_test_template
parallel: 3
tags:
- ESP32_IDF
- SSC_T5_BT_Simple
IT_T2_BT_Simple:
extends: .integration_test_template
parallel: 6
tags:
- ESP32_IDF
- SSC_T2_BT_Simple
IT_T1_BT_Dongle:
extends: .integration_test_template
image: gitlab.espressif.cn:5050/qa/dockerfiles/integration-test-env-dbg:1
parallel: 2
tags:
- ESP32_IDF
- SSC_T1_BT_Dongle
IT_005:
extends: .integration_test_template
tags:
- ESP32_IDF
- SSC_T1_5
IT_006:
extends: .integration_test_template
parallel: 5
tags:
- ESP32_IDF
- SSC_T1_6
IT_007:
extends: .integration_test_template
tags:
- ESP32_IDF
- SSC_T1_7
IT_008:
extends: .integration_test_template
tags:
- ESP32_IDF
- SSC_T1_8
IT_011:
extends: .integration_test_template
tags:
- ESP32_IDF
- SSC_T1_MESH1
IT_012:
extends: .integration_test_template
parallel: 2
tags:
- ESP32_IDF
- SSC_T2_MESH1
IT_013:
extends: .integration_test_template
tags:
- ESP32_IDF
- SSC_T3_MESH1
IT_014:
extends: .integration_test_template
tags:
- ESP32_IDF
- SSC_T6_MESH1
IT_015:
extends: .integration_test_template
tags:
- ESP32_IDF
- SSC_T12_MESH1
IT_016:
extends: .integration_test_template
allow_failure: true
tags:
- ESP32_IDF
- SSC_T50_MESH1
IT_017:
extends: .integration_test_template
tags:
- ESP32_IDF
- SSC_T1_MESH2
IT_018:
extends: .integration_test_template
tags:
- ESP32_IDF
- SSC_T1_9
IT_019:
extends: .integration_test_template
tags:
- ESP32_IDF
- SSC_T2_2
IT_020:
extends: .integration_test_template
tags:
- ESP32_IDF
- SSC_T2_3
IT_022:
extends: .integration_test_template
tags:
- ESP32_IDF
- SSC_T3_2
IT_C3_001:
extends: .integration_test_esp32c3_template
parallel: 5
tags:
- ESP32C3_IDF
- SSC_T2_5
IT_C3_T2_BT_Simple:
extends: .integration_test_esp32c3_template
parallel: 11
tags:
- ESP32C3_IDF
- SSC_T2_BT_Simple
IT_C3_T5_BT_Simple:
extends: .integration_test_esp32c3_template
parallel: 5
tags:
- ESP32C3_IDF
- SSC_T5_BT_Simple
IT_C3_T1_BT_Dongle:
extends: .integration_test_esp32c3_template
image: gitlab.espressif.cn:5050/qa/dockerfiles/integration-test-env-dbg:1
parallel: 2
tags:
- ESP32C3_IDF
- SSC_T1_BT_Dongle

9
tools/ci/integration_test/KnownIssues
View File

@ -1,9 +0,0 @@
# MESH
ESP32.MESH_EST_2109
# CI
ESP32.BLUEDROID_GAP_03003
ESP32.NIMBLE_GAP_14007
# IDFCI-1067
ESP32C3.NIMBLE_GAP_14009
ESP32C3.NIMBLE_GAP_17021

17
tools/ci/integration_test/README.md
View File

@ -1,17 +0,0 @@
# Integration Test Description
## Case Lists
- WiFi Standard cases for only esp32.
- BLE Standard cases for esp32 and esp32c3.
## Trigger
- By bot:
- `@bot test with label: integration_test`
## Bot Filter
- A case filter could be used in `@bot` message:
- For wifi cases:
- `@bot test with label: integration_test; with filter module: WIFI MAC, TCPIP, Mesh, System`
- For bt/ble cases:
- `@bot test with label: integration_test; with filter module: BT Stack, BLUEDROID, BT Profile, NIMBLE`
- use `@bot help` for more details.

11
tools/ci/integration_test/test_configs/nvs_compatible_test_.yml
View File

@ -1,11 +0,0 @@
BinPath:
path: SSC/ssc_bin/ESP32/SSC_BLE_WIFI
test app: SSC_BLE_WIFI
DUT: [SSC1]
Filter:
- Add:
SDK: ESP32_IDF
Test App: SSC_BLE_WIFI
summary: 'use old NVS data WIFI function test'