From 7fecad1473571d993ac84726a58214dc44e39896 Mon Sep 17 00:00:00 2001 From: Chen Yudong Date: Thu, 10 Feb 2022 20:21:55 +0800 Subject: [PATCH 1/2] CI: support integration c3 test --- tools/ci/config/assign-test.yml | 1 + tools/ci/config/target-test.yml | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/tools/ci/config/assign-test.yml b/tools/ci/config/assign-test.yml index e603460e21..b83667ee5b 100644 --- a/tools/ci/config/assign-test.yml +++ b/tools/ci/config/assign-test.yml @@ -9,6 +9,7 @@ assign_test: # we have a lot build example jobs. now we don't use dependencies, just download all artifacts of build stage. dependencies: # Here is not a hard dependency relationship, could be skipped. so we do not use "needs" here. - build_ssc_esp32 + - build_ssc_esp32c3 - build_esp_idf_tests_cmake_esp32 - build_esp_idf_tests_cmake_esp32s2 - build_esp_idf_tests_cmake_esp32c3 diff --git a/tools/ci/config/target-test.yml b/tools/ci/config/target-test.yml index d8a4a35b8e..6832450c15 100644 --- a/tools/ci/config/target-test.yml +++ b/tools/ci/config/target-test.yml @@ -117,6 +117,7 @@ needs: - assign_test - build_ssc_esp32 + - build_ssc_esp32c3 variables: LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF" LOG_PATH: "${CI_PROJECT_DIR}/TEST_LOGS" @@ -746,3 +747,10 @@ IT_022: tags: - ESP32_IDF - SSC_T3_2 + +IT_C3_001: + extends: .integration_test_template + parallel: 5 + tags: + - ESP32C3_IDF + - SSC_T2_5 From ad6095ad80c74788145724e5943611b8f4940e39 Mon Sep 17 00:00:00 2001 From: Chen Yudong Date: Thu, 10 Feb 2022 21:13:37 +0800 Subject: [PATCH 2/2] CI: optimize integration test workflow --- .../idf_test/integration_test/KnownIssues | 0 tools/ci/config/assign-test.yml | 5 +- tools/ci/config/target-test.yml | 20 ++++-- tools/ci/integration_test/KnownIssues | 3 + tools/ci/integration_test/README.md | 17 +++++ .../ci/integration_test/prepare_test_bins.py | 67 +++++++++++++++++++ .../test_configs}/nvs_compatible_test_.yml | 0 7 files changed, 103 insertions(+), 9 deletions(-) delete mode 100644 components/idf_test/integration_test/KnownIssues create mode 100644 tools/ci/integration_test/KnownIssues create mode 100644 tools/ci/integration_test/README.md create mode 100644 tools/ci/integration_test/prepare_test_bins.py rename {components/idf_test/integration_test/CIConfigs => tools/ci/integration_test/test_configs}/nvs_compatible_test_.yml (100%) diff --git a/components/idf_test/integration_test/KnownIssues b/components/idf_test/integration_test/KnownIssues deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tools/ci/config/assign-test.yml b/tools/ci/config/assign-test.yml index b83667ee5b..40e3cfe875 100644 --- a/tools/ci/config/assign-test.yml +++ b/tools/ci/config/assign-test.yml @@ -20,7 +20,7 @@ assign_test: UNIT_TEST_DIR: "${CI_PROJECT_DIR}/components/idf_test/unit_test" # COMPONENT_UT_DIRS is set by `set_component_ut_vars` in `utils.sh` COMPONENT_UT_OUTPUT_DIR: "${CI_PROJECT_DIR}/component_ut" - INTEGRATION_CONFIG_OUTPUT_PATH: "${CI_PROJECT_DIR}/components/idf_test/integration_test/CIConfigs" + INTEGRATION_TEST_DIR: "${CI_PROJECT_DIR}/tools/ci/integration_test" INTEGRATION_TEST_CASE_PATH: "${CI_PROJECT_DIR}/auto_test_script/TestCaseFiles" ASSIGN_TEST_CASE_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIAssignTestCases.py" PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages @@ -32,6 +32,7 @@ assign_test: - $EXAMPLE_TEST_DIR/test_configs - $CUSTOM_TEST_DIR/test_configs - $COMPONENT_UT_OUTPUT_DIR/test_configs + - $INTEGRATION_TEST_DIR/test_configs - build_examples/artifact_index.json - build_test_apps/artifact_index.json - build_component_ut/artifact_index.json @@ -49,7 +50,7 @@ assign_test: - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH} - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only # assign integration test cases - - python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b $IDF_PATH/SSC/ssc_bin -o $INTEGRATION_CONFIG_OUTPUT_PATH + - python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b $IDF_PATH/SSC/ssc_bin -o $INTEGRATION_TEST_DIR/test_configs update_test_cases: extends: .rules:ref:master-schedule diff --git a/tools/ci/config/target-test.yml b/tools/ci/config/target-test.yml index 6832450c15..d370d8807c 100644 --- a/tools/ci/config/target-test.yml +++ b/tools/ci/config/target-test.yml @@ -116,15 +116,13 @@ - .rules:tests:integration_test needs: - assign_test - - build_ssc_esp32 - - build_ssc_esp32c3 variables: LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF" LOG_PATH: "${CI_PROJECT_DIR}/TEST_LOGS" TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/auto_test_script/TestCaseFiles" - MODULE_UPDATE_FILE: "$CI_PROJECT_DIR/components/idf_test/ModuleDefinition.yml" - CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/integration_test/CIConfigs" - KNOWN_ISSUE_FILE: "${CI_PROJECT_DIR}/components/idf_test/integration_test/KnownIssues" + INTEGRATION_TEST_DIR: "${CI_PROJECT_DIR}/tools/ci/integration_test" + CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/tools/ci/integration_test/test_configs" + PREPARE_TEST_BIN_SCRIPT: "${CI_PROJECT_DIR}/tools/ci/integration_test/prepare_test_bins.py" CI_RUNNER_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIRunner.py" PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages # auto_test_script only supports python 3.7.x @@ -141,10 +139,17 @@ - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH} - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only - - cat ${KNOWN_ISSUE_FILE} >> ${TEST_CASE_FILE_PATH}/KnownIssues + - cat ${INTEGRATION_TEST_DIR}/KnownIssues >> ${TEST_CASE_FILE_PATH}/KnownIssues # run test + - python ${PREPARE_TEST_BIN_SCRIPT} $CONFIG_FILE - python ${CI_RUNNER_SCRIPT} -l "$LOG_PATH/$JOB_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH +.integration_test_esp32c3_template: + extends: + - .integration_test_template + variables: + LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32C3_IDF" + test_weekend_mqtt: extends: - .test_app_template @@ -633,6 +638,7 @@ nvs_compatible_test: - cd auto_test_script - ./tools/prepare_nvs_bin.sh # run test + - python ${PREPARE_TEST_BIN_SCRIPT} $CONFIG_FILE - python ${CI_RUNNER_SCRIPT} -l "$LOG_PATH/$JOB_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH IT_001: @@ -749,7 +755,7 @@ IT_022: - SSC_T3_2 IT_C3_001: - extends: .integration_test_template + extends: .integration_test_esp32c3_template parallel: 5 tags: - ESP32C3_IDF diff --git a/tools/ci/integration_test/KnownIssues b/tools/ci/integration_test/KnownIssues new file mode 100644 index 0000000000..69141b0abd --- /dev/null +++ b/tools/ci/integration_test/KnownIssues @@ -0,0 +1,3 @@ + +# IDFCI-1067 +ESP32C3.NIMBLE_GAP_14009 diff --git a/tools/ci/integration_test/README.md b/tools/ci/integration_test/README.md new file mode 100644 index 0000000000..b4381d8923 --- /dev/null +++ b/tools/ci/integration_test/README.md @@ -0,0 +1,17 @@ +# Integration Test Description + +## Case Lists +- WiFi Standard cases for only esp32. +- BLE Standard cases for esp32 and esp32c3. + +## Trigger +- By bot: + - `@bot test with label: integration_test` + +## Bot Filter +- A case filter could be used in `@bot` message: +- For wifi cases: + - `@bot test with label: integration_test; with filter module: WIFI MAC, TCPIP, Mesh, System` +- For bt/ble cases: + - `@bot test with label: integration_test; with filter module: BT Stack, BLUEDROID, BT Profile, NIMBLE` +- use `@bot help` for more details. diff --git a/tools/ci/integration_test/prepare_test_bins.py b/tools/ci/integration_test/prepare_test_bins.py new file mode 100644 index 0000000000..fea2a11cd3 --- /dev/null +++ b/tools/ci/integration_test/prepare_test_bins.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python +# +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import os + +import gitlab +import gitlab_api +from AutoTestScript.RunnerConfigs.Config import Config + +SSC_BUILD_JOB_MAP = { + 'ESP32': 'build_ssc_esp32', + 'ESP32C3': 'build_ssc_esp32c3', +} +NEEDED_FILES = [ + 'flasher_args.json', + 'bootloader/bootloader.bin', + 'partition_table/partition-table.bin', + 'ssc.bin', + 'ssc.elf', +] +IDF_PATH = os.environ.get('IDF_PATH') + + +def try_to_download_artifacts(bin_path): + ''' + bin_path: "SSC/ssc_bin/ESP32[C3]/SSC[_APP]" + ''' + project_id = os.getenv('CI_PROJECT_ID') + pipeline_id = os.getenv('CI_PIPELINE_ID') + gitlab_inst = gitlab_api.Gitlab(project_id) + build_job_name = SSC_BUILD_JOB_MAP[bin_path.split('/')[-2]] + job_list = gitlab_inst.find_job_id(build_job_name, pipeline_id=pipeline_id) + files_to_download = [os.path.join(bin_path, f) for f in NEEDED_FILES] + for job_info in job_list: + try: + gitlab_inst.download_artifact(job_info['id'], files_to_download, IDF_PATH) + print('Downloaded {} from {}'.format(bin_path, job_info['id'])) + break + except gitlab.exceptions.GitlabError as e: + if e.response_code == 404: + continue + raise + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + 'test_config_file', + help='The test config file to be used.' + ) + args = parser.parse_args() + + configs = Config.parse(args.test_config_file) + test_bin_paths = configs.get_bin_paths() + + for _path in test_bin_paths: + if os.path.exists(_path): + continue + relative_path = os.path.relpath(_path, IDF_PATH) + try_to_download_artifacts(relative_path) + + +if __name__ == '__main__': + main() diff --git a/components/idf_test/integration_test/CIConfigs/nvs_compatible_test_.yml b/tools/ci/integration_test/test_configs/nvs_compatible_test_.yml similarity index 100% rename from components/idf_test/integration_test/CIConfigs/nvs_compatible_test_.yml rename to tools/ci/integration_test/test_configs/nvs_compatible_test_.yml