Merge branch 'ci/add_esp32c3_integration_test' into 'release/v4.3'

CI: support integration c3 test (4.3)

See merge request espressif/esp-idf!17108

tools/ci/config/assign-test.yml

@@ -9,6 +9,7 @@ assign_test:
   # we have a lot build example jobs. now we don't use dependencies, just download all artifacts of build stage.
   dependencies: # Here is not a hard dependency relationship, could be skipped. so we do not use "needs" here.
     - build_ssc_esp32
+    - build_ssc_esp32c3
     - build_esp_idf_tests_cmake_esp32
     - build_esp_idf_tests_cmake_esp32s2
     - build_esp_idf_tests_cmake_esp32c3
@@ -19,7 +20,7 @@ assign_test:
     UNIT_TEST_DIR: "${CI_PROJECT_DIR}/components/idf_test/unit_test"
     # COMPONENT_UT_DIRS is set by `set_component_ut_vars` in `utils.sh`
     COMPONENT_UT_OUTPUT_DIR: "${CI_PROJECT_DIR}/component_ut"
-    INTEGRATION_CONFIG_OUTPUT_PATH: "${CI_PROJECT_DIR}/components/idf_test/integration_test/CIConfigs"
+    INTEGRATION_TEST_DIR: "${CI_PROJECT_DIR}/tools/ci/integration_test"
     INTEGRATION_TEST_CASE_PATH: "${CI_PROJECT_DIR}/auto_test_script/TestCaseFiles"
     ASSIGN_TEST_CASE_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIAssignTestCases.py"
     PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages
@@ -31,6 +32,7 @@ assign_test:
       - $EXAMPLE_TEST_DIR/test_configs
       - $CUSTOM_TEST_DIR/test_configs
       - $COMPONENT_UT_OUTPUT_DIR/test_configs
+      - $INTEGRATION_TEST_DIR/test_configs
       - build_examples/artifact_index.json
       - build_test_apps/artifact_index.json
       - build_component_ut/artifact_index.json
@@ -48,7 +50,7 @@ assign_test:
     - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
     - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only
     # assign integration test cases
-    - python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b $IDF_PATH/SSC/ssc_bin -o $INTEGRATION_CONFIG_OUTPUT_PATH
+    - python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b $IDF_PATH/SSC/ssc_bin -o $INTEGRATION_TEST_DIR/test_configs
 
 update_test_cases:
   extends: .rules:ref:master-schedule

tools/ci/config/target-test.yml

@@ -116,14 +116,13 @@
     - .rules:tests:integration_test
   needs:
     - assign_test
-    - build_ssc_esp32
   variables:
     LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF"
     LOG_PATH: "${CI_PROJECT_DIR}/TEST_LOGS"
     TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/auto_test_script/TestCaseFiles"
-    MODULE_UPDATE_FILE: "$CI_PROJECT_DIR/components/idf_test/ModuleDefinition.yml"
-    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/integration_test/CIConfigs"
-    KNOWN_ISSUE_FILE: "${CI_PROJECT_DIR}/components/idf_test/integration_test/KnownIssues"
+    INTEGRATION_TEST_DIR: "${CI_PROJECT_DIR}/tools/ci/integration_test"
+    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/tools/ci/integration_test/test_configs"
+    PREPARE_TEST_BIN_SCRIPT: "${CI_PROJECT_DIR}/tools/ci/integration_test/prepare_test_bins.py"
     CI_RUNNER_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIRunner.py"
     PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages
     # auto_test_script only supports python 3.7.x
@@ -140,10 +139,17 @@
     - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
     - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
     - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only
-    - cat ${KNOWN_ISSUE_FILE} >> ${TEST_CASE_FILE_PATH}/KnownIssues
+    - cat ${INTEGRATION_TEST_DIR}/KnownIssues >> ${TEST_CASE_FILE_PATH}/KnownIssues
     # run test
+    - python ${PREPARE_TEST_BIN_SCRIPT} $CONFIG_FILE
     - python ${CI_RUNNER_SCRIPT} -l "$LOG_PATH/$JOB_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH
 
+.integration_test_esp32c3_template:
+  extends:
+    - .integration_test_template
+  variables:
+    LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32C3_IDF"
+
 test_weekend_mqtt:
   extends:
     - .test_app_template
@@ -632,6 +638,7 @@ nvs_compatible_test:
     - cd auto_test_script
     - ./tools/prepare_nvs_bin.sh
     # run test
+    - python ${PREPARE_TEST_BIN_SCRIPT} $CONFIG_FILE
     - python ${CI_RUNNER_SCRIPT} -l "$LOG_PATH/$JOB_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH
 
 IT_001:
@@ -746,3 +753,10 @@ IT_022:
   tags:
     - ESP32_IDF
     - SSC_T3_2
+
+IT_C3_001:
+  extends: .integration_test_esp32c3_template
+  parallel: 5
+  tags:
+    - ESP32C3_IDF
+    - SSC_T2_5

tools/ci/integration_test/KnownIssues

@@ -0,0 +1,3 @@
+
+# IDFCI-1067
+ESP32C3.NIMBLE_GAP_14009

tools/ci/integration_test/README.md

@@ -0,0 +1,17 @@
+# Integration Test Description
+
+## Case Lists
+- WiFi Standard cases for only esp32.
+- BLE Standard cases for esp32 and esp32c3.
+
+## Trigger
+- By bot:
+  - `@bot test with label: integration_test`
+
+## Bot Filter
+- A case filter could be used in `@bot` message:
+- For wifi cases:
+  - `@bot test with label: integration_test; with filter module: WIFI MAC, TCPIP, Mesh, System`
+- For bt/ble cases:
+  - `@bot test with label: integration_test; with filter module: BT Stack, BLUEDROID, BT Profile, NIMBLE`
+- use `@bot help` for more details.

tools/ci/integration_test/prepare_test_bins.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+#
+# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+
+import argparse
+import os
+
+import gitlab
+import gitlab_api
+from AutoTestScript.RunnerConfigs.Config import Config
+
+SSC_BUILD_JOB_MAP = {
+    'ESP32': 'build_ssc_esp32',
+    'ESP32C3': 'build_ssc_esp32c3',
+}
+NEEDED_FILES = [
+    'flasher_args.json',
+    'bootloader/bootloader.bin',
+    'partition_table/partition-table.bin',
+    'ssc.bin',
+    'ssc.elf',
+]
+IDF_PATH = os.environ.get('IDF_PATH')
+
+
+def try_to_download_artifacts(bin_path):
+    '''
+    bin_path: "SSC/ssc_bin/ESP32[C3]/SSC[_APP]"
+    '''
+    project_id = os.getenv('CI_PROJECT_ID')
+    pipeline_id = os.getenv('CI_PIPELINE_ID')
+    gitlab_inst = gitlab_api.Gitlab(project_id)
+    build_job_name = SSC_BUILD_JOB_MAP[bin_path.split('/')[-2]]
+    job_list = gitlab_inst.find_job_id(build_job_name, pipeline_id=pipeline_id)
+    files_to_download = [os.path.join(bin_path, f) for f in NEEDED_FILES]
+    for job_info in job_list:
+        try:
+            gitlab_inst.download_artifact(job_info['id'], files_to_download, IDF_PATH)
+            print('Downloaded {} from {}'.format(bin_path, job_info['id']))
+            break
+        except gitlab.exceptions.GitlabError as e:
+            if e.response_code == 404:
+                continue
+            raise
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        'test_config_file',
+        help='The test config file to be used.'
+    )
+    args = parser.parse_args()
+
+    configs = Config.parse(args.test_config_file)
+    test_bin_paths = configs.get_bin_paths()
+
+    for _path in test_bin_paths:
+        if os.path.exists(_path):
+            continue
+        relative_path = os.path.relpath(_path, IDF_PATH)
+        try_to_download_artifacts(relative_path)
+
+
+if __name__ == '__main__':
+    main()