From 5c211efaa89b791d32f9cc4604fd15045250ca17 Mon Sep 17 00:00:00 2001 From: Chen Yudong Date: Mon, 21 Feb 2022 18:56:56 +0800 Subject: [PATCH] CI: update test cases from test results --- .gitlab/ci/assign-test.yml | 37 ------------ .gitlab/ci/deploy.yml | 30 ++++----- tools/ci/exclude_check_tools_files.txt | 1 + tools/ci/get_all_test_results.py | 84 ++++++++++++++++++++++++++ 4 files changed, 101 insertions(+), 51 deletions(-) create mode 100644 tools/ci/get_all_test_results.py diff --git a/.gitlab/ci/assign-test.yml b/.gitlab/ci/assign-test.yml index df0951d575..4afa4a5ecf 100644 --- a/.gitlab/ci/assign-test.yml +++ b/.gitlab/ci/assign-test.yml @@ -104,40 +104,3 @@ assign_integration_test: - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only # assign integration test cases - python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b ${BUILD_DIR} -o $TEST_DIR/test_configs - -update_test_cases: - extends: .rules:ref:master-schedule - stage: assign_test - image: $TARGET_TEST_ENV_IMAGE - tags: - - deploy_test - needs: - - build_esp_idf_tests_cmake_esp32 - - build_esp_idf_tests_cmake_esp32s2 - - build_esp_idf_tests_cmake_esp32s3 - - build_esp_idf_tests_cmake_esp32c3 - artifacts: - when: always - paths: - - ${CI_PROJECT_DIR}/test-management/*.log - expire_in: 1 week - variables: - SUBMODULES_TO_FETCH: "components/esptool_py/esptool" - UNIT_TEST_DIR: "${CI_PROJECT_DIR}/components/idf_test/unit_test" - BOT_ACCOUNT_CONFIG_FILE: "${CI_PROJECT_DIR}/test-management/Config/Account.local.yml" - PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages - PYTHON_VER: 3.7.7 - script: - - export GIT_SHA=$(echo ${PIPELINE_COMMIT_SHA} | cut -c 1-8) - - retry_failed git clone $TEST_MANAGEMENT_REPO - - python $CHECKOUT_REF_SCRIPT test-management test-management - - cd test-management - - echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE} - # update unit test cases - - export UNIT_TEST_CASE_FILES=$(find $UNIT_TEST_DIR -maxdepth 1 -name "*.yml" | xargs) - - python ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT unity -d $UNIT_TEST_CASE_FILES -r $GIT_SHA - # update example test cases - - python ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT tiny_test_fw -d ${CI_PROJECT_DIR}/examples -r $GIT_SHA - - python ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT tiny_test_fw -d ${CI_PROJECT_DIR}/tools/test_apps -r $GIT_SHA - # organize test cases - - python OrganizeTestCases.py $JIRA_TEST_MANAGEMENT_PROJECT diff --git a/.gitlab/ci/deploy.yml b/.gitlab/ci/deploy.yml index f4e91d8fed..bb83218864 100644 --- a/.gitlab/ci/deploy.yml +++ b/.gitlab/ci/deploy.yml @@ -46,35 +46,37 @@ deploy_test_result: - .deploy_job_template - .before_script_minimal - .rules:ref:master-schedule-always - image: $CI_DOCKER_REGISTRY/bot-env + image: $CI_DOCKER_REGISTRY/bot-env:1 + dependencies: [] tags: - deploy_test artifacts: when: always paths: - ${CI_PROJECT_DIR}/test-management/*.log - # save all test logs as artifacts, make it easier to track errors - - ${CI_PROJECT_DIR}/TEST_LOGS - expire_in: 1 mos + expire_in: 1 week variables: - UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test/TestCaseAll.yml" BOT_ACCOUNT_CONFIG_FILE: "${CI_PROJECT_DIR}/test-management/Config/Account.local.yml" - TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw" - AUTO_TEST_SCRIPT_PATH: "${CI_PROJECT_DIR}/auto_test_script" + TEST_RESULTS_PATH: "${CI_PROJECT_DIR}/TEST_RESULTS" script: - add_gitlab_ssh_keys - - export GIT_SHA=$(echo ${PIPELINE_COMMIT_SHA} | cut -c 1-8) - - export REV_COUNT=$(git rev-list --count ${PIPELINE_COMMIT_SHA} --) + - export GIT_SHA=$(echo ${CI_COMMIT_SHA} | cut -c 1-8) + - export REV_COUNT=$(git rev-list --count ${GIT_SHA} --) - export SUMMARY="IDF CI test result for $GIT_SHA (r${REV_COUNT})" - # artifacts of job update_test_cases creates test-management folder - # we need to remove it so we can clone test-management folder again - - rm -rf test-management + # Download test result + - export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH" + - python3 ${IDF_PATH}/tools/ci/get_all_test_results.py --path ${TEST_RESULTS_PATH} --include_retried + - if [[ -z $(find ${TEST_RESULTS_PATH} -name "*.xml") ]]; then exit 0; fi + # Clone test-management repo - retry_failed git clone $TEST_MANAGEMENT_REPO - python3 $CHECKOUT_REF_SCRIPT test-management test-management - cd test-management - echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE} + # Make sure all requirements are installed + - pip3 install -r requirements.txt + # Update test cases + - python3 ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT from_xml -d ${TEST_RESULTS_PATH} -r $GIT_SHA -l IDFCI # update test results - - python3 ImportTestResult.py -r "$GIT_SHA (r${REV_COUNT})" -j $JIRA_TEST_MANAGEMENT_PROJECT -s "$SUMMARY" -l CI -p ${CI_PROJECT_DIR}/TEST_LOGS --pipeline_url ${CI_PIPELINE_URL} + - python3 ImportTestResult.py -r "$GIT_SHA (r${REV_COUNT})" -j $JIRA_TEST_MANAGEMENT_PROJECT -s "$SUMMARY" -l IDFCI -p ${TEST_RESULTS_PATH} --pipeline_url ${CI_PIPELINE_URL} # May need a long time to upload all test results. - retry: 2 timeout: 4 hours diff --git a/tools/ci/exclude_check_tools_files.txt b/tools/ci/exclude_check_tools_files.txt index 8db6df9411..147970942d 100644 --- a/tools/ci/exclude_check_tools_files.txt +++ b/tools/ci/exclude_check_tools_files.txt @@ -5,6 +5,7 @@ tools/ci/check_*.py tools/ci/check_*.txt tools/ci/check_*.sh tools/ci/check_copyright_config.yaml +tools/ci/get_all_test_results.py tools/gdb_panic_server.py tools/check_term.py tools/check_python_dependencies.py diff --git a/tools/ci/get_all_test_results.py b/tools/ci/get_all_test_results.py new file mode 100644 index 0000000000..7baa0832f2 --- /dev/null +++ b/tools/ci/get_all_test_results.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +# +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + + +''' +Download artifacts from all test jobs + +CI_PROJECT_DIR +└── TEST_RESULTS + ├── + │ └── XUNIT_RESULT.xml + ├── + │ └── XUNIT_RESULT.xml + ├── + │ └── TEST_LOGS + +''' + +import argparse +import json +import os +import shutil + +import gitlab_api + +CI_PROJECT_ID = int(os.getenv('CI_PROJECT_ID', '')) +CI_PIPELINE_ID = int(os.getenv('CI_PIPELINE_ID', '')) +IDF_PATH = os.getenv('IDF_PATH', '.') +TEST_RESULT_PATH = os.path.join(IDF_PATH, 'TEST_RESULTS') + + +def download_all_test_results(result_path: str, include_retried: bool = False) -> None: + if os.path.exists(result_path): + shutil.rmtree(result_path) + os.makedirs(result_path, exist_ok=True) + + gitlab_inst = gitlab_api.Gitlab(CI_PROJECT_ID) + pipelines = gitlab_inst.project.pipelines.get(CI_PIPELINE_ID) + _include_retried = 'true' if include_retried else 'false' + jobs = pipelines.jobs.list(all=True, per_page=100, include_retried=_include_retried) + + job_info_list = [] + for job in jobs: + if job.stage in ['target_test', 'host_test']: + log_path = '' + if job.status not in ['success', 'failed']: + print('Job {}({}) is not finished'.format(job.id, job.name)) + elif not hasattr(job, 'artifacts_file'): + print('Job {}({}) has no artifacts.'.format(job.id, job.name)) + else: + log_path = os.path.join(result_path, 'job_{}'.format(job.id)) + print('Downloading artifacts from: {}'.format(job.name)) + os.makedirs(log_path, exist_ok=True) + gitlab_inst.download_artifacts(job.id, log_path) + + job_info = { + 'id': job.id, + 'name': job.name, + 'tag_list': job.tag_list, + 'status': job.status, + 'stage': job.stage, + 'web_url': job.web_url, + 'commit_url': job.commit['web_url'], + 'log_path': log_path, + } + job_info_list.append(job_info) + + with open(os.path.join(result_path, 'index.json'), 'w') as f: + f.write(json.dumps({'jobs': job_info_list}, indent=1, sort_keys=True)) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('--path', '-p', default=TEST_RESULT_PATH, help='Path to save result files.') + parser.add_argument('--include_retried', action='store_true', help='Including retried jobs.') + args = parser.parse_args() + + download_all_test_results(args.path, args.include_retried) + + +if __name__ == '__main__': + main()