mirror of
https://github.com/espressif/esp-idf.git
synced 2024-10-05 20:47:46 -04:00
CI: update test cases from test results
This commit is contained in:
Chen Yudong
parent
229ed08484
commit
5c211efaa8
@ -104,40 +104,3 @@ assign_integration_test:
|
||||
- python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only
|
||||
# assign integration test cases
|
||||
- python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b ${BUILD_DIR} -o $TEST_DIR/test_configs
|
||||
|
||||
update_test_cases:
|
||||
extends: .rules:ref:master-schedule
|
||||
stage: assign_test
|
||||
image: $TARGET_TEST_ENV_IMAGE
|
||||
tags:
|
||||
- deploy_test
|
||||
needs:
|
||||
- build_esp_idf_tests_cmake_esp32
|
||||
- build_esp_idf_tests_cmake_esp32s2
|
||||
- build_esp_idf_tests_cmake_esp32s3
|
||||
- build_esp_idf_tests_cmake_esp32c3
|
||||
artifacts:
|
||||
when: always
|
||||
paths:
|
||||
- ${CI_PROJECT_DIR}/test-management/*.log
|
||||
expire_in: 1 week
|
||||
variables:
|
||||
SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
|
||||
UNIT_TEST_DIR: "${CI_PROJECT_DIR}/components/idf_test/unit_test"
|
||||
BOT_ACCOUNT_CONFIG_FILE: "${CI_PROJECT_DIR}/test-management/Config/Account.local.yml"
|
||||
PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages
|
||||
PYTHON_VER: 3.7.7
|
||||
script:
|
||||
- export GIT_SHA=$(echo ${PIPELINE_COMMIT_SHA} | cut -c 1-8)
|
||||
- retry_failed git clone $TEST_MANAGEMENT_REPO
|
||||
- python $CHECKOUT_REF_SCRIPT test-management test-management
|
||||
- cd test-management
|
||||
- echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE}
|
||||
# update unit test cases
|
||||
- export UNIT_TEST_CASE_FILES=$(find $UNIT_TEST_DIR -maxdepth 1 -name "*.yml" | xargs)
|
||||
- python ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT unity -d $UNIT_TEST_CASE_FILES -r $GIT_SHA
|
||||
# update example test cases
|
||||
- python ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT tiny_test_fw -d ${CI_PROJECT_DIR}/examples -r $GIT_SHA
|
||||
- python ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT tiny_test_fw -d ${CI_PROJECT_DIR}/tools/test_apps -r $GIT_SHA
|
||||
# organize test cases
|
||||
- python OrganizeTestCases.py $JIRA_TEST_MANAGEMENT_PROJECT
|
||||
|
||||
@ -46,35 +46,37 @@ deploy_test_result:
|
||||
- .deploy_job_template
|
||||
- .before_script_minimal
|
||||
- .rules:ref:master-schedule-always
|
||||
image: $CI_DOCKER_REGISTRY/bot-env
|
||||
image: $CI_DOCKER_REGISTRY/bot-env:1
|
||||
dependencies: []
|
||||
tags:
|
||||
- deploy_test
|
||||
artifacts:
|
||||
when: always
|
||||
paths:
|
||||
- ${CI_PROJECT_DIR}/test-management/*.log
|
||||
# save all test logs as artifacts, make it easier to track errors
|
||||
- ${CI_PROJECT_DIR}/TEST_LOGS
|
||||
expire_in: 1 mos
|
||||
expire_in: 1 week
|
||||
variables:
|
||||
UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test/TestCaseAll.yml"
|
||||
BOT_ACCOUNT_CONFIG_FILE: "${CI_PROJECT_DIR}/test-management/Config/Account.local.yml"
|
||||
TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
|
||||
AUTO_TEST_SCRIPT_PATH: "${CI_PROJECT_DIR}/auto_test_script"
|
||||
TEST_RESULTS_PATH: "${CI_PROJECT_DIR}/TEST_RESULTS"
|
||||
script:
|
||||
- add_gitlab_ssh_keys
|
||||
- export GIT_SHA=$(echo ${PIPELINE_COMMIT_SHA} | cut -c 1-8)
|
||||
- export REV_COUNT=$(git rev-list --count ${PIPELINE_COMMIT_SHA} --)
|
||||
- export GIT_SHA=$(echo ${CI_COMMIT_SHA} | cut -c 1-8)
|
||||
- export REV_COUNT=$(git rev-list --count ${GIT_SHA} --)
|
||||
- export SUMMARY="IDF CI test result for $GIT_SHA (r${REV_COUNT})"
|
||||
# artifacts of job update_test_cases creates test-management folder
|
||||
# we need to remove it so we can clone test-management folder again
|
||||
- rm -rf test-management
|
||||
# Download test result
|
||||
- export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
|
||||
- python3 ${IDF_PATH}/tools/ci/get_all_test_results.py --path ${TEST_RESULTS_PATH} --include_retried
|
||||
- if [[ -z $(find ${TEST_RESULTS_PATH} -name "*.xml") ]]; then exit 0; fi
|
||||
# Clone test-management repo
|
||||
- retry_failed git clone $TEST_MANAGEMENT_REPO
|
||||
- python3 $CHECKOUT_REF_SCRIPT test-management test-management
|
||||
- cd test-management
|
||||
- echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE}
|
||||
# Make sure all requirements are installed
|
||||
- pip3 install -r requirements.txt
|
||||
# Update test cases
|
||||
- python3 ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT from_xml -d ${TEST_RESULTS_PATH} -r $GIT_SHA -l IDFCI
|
||||
# update test results
|
||||
- python3 ImportTestResult.py -r "$GIT_SHA (r${REV_COUNT})" -j $JIRA_TEST_MANAGEMENT_PROJECT -s "$SUMMARY" -l CI -p ${CI_PROJECT_DIR}/TEST_LOGS --pipeline_url ${CI_PIPELINE_URL}
|
||||
- python3 ImportTestResult.py -r "$GIT_SHA (r${REV_COUNT})" -j $JIRA_TEST_MANAGEMENT_PROJECT -s "$SUMMARY" -l IDFCI -p ${TEST_RESULTS_PATH} --pipeline_url ${CI_PIPELINE_URL}
|
||||
# May need a long time to upload all test results.
|
||||
retry: 2
|
||||
timeout: 4 hours
|
||||
|
||||
@ -5,6 +5,7 @@ tools/ci/check_*.py
|
||||
tools/ci/check_*.txt
|
||||
tools/ci/check_*.sh
|
||||
tools/ci/check_copyright_config.yaml
|
||||
tools/ci/get_all_test_results.py
|
||||
tools/gdb_panic_server.py
|
||||
tools/check_term.py
|
||||
tools/check_python_dependencies.py
|
||||
|
||||
84
tools/ci/get_all_test_results.py
Normal file
84
tools/ci/get_all_test_results.py
Normal file
@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
|
||||
'''
|
||||
Download artifacts from all test jobs
|
||||
|
||||
CI_PROJECT_DIR
|
||||
└── TEST_RESULTS
|
||||
├── <job_id>
|
||||
│ └── XUNIT_RESULT.xml
|
||||
├── <job_id>
|
||||
│ └── XUNIT_RESULT.xml
|
||||
├── <job_id>
|
||||
│ └── TEST_LOGS
|
||||
|
||||
'''
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
|
||||
import gitlab_api
|
||||
|
||||
CI_PROJECT_ID = int(os.getenv('CI_PROJECT_ID', ''))
|
||||
CI_PIPELINE_ID = int(os.getenv('CI_PIPELINE_ID', ''))
|
||||
IDF_PATH = os.getenv('IDF_PATH', '.')
|
||||
TEST_RESULT_PATH = os.path.join(IDF_PATH, 'TEST_RESULTS')
|
||||
|
||||
|
||||
def download_all_test_results(result_path: str, include_retried: bool = False) -> None:
|
||||
if os.path.exists(result_path):
|
||||
shutil.rmtree(result_path)
|
||||
os.makedirs(result_path, exist_ok=True)
|
||||
|
||||
gitlab_inst = gitlab_api.Gitlab(CI_PROJECT_ID)
|
||||
pipelines = gitlab_inst.project.pipelines.get(CI_PIPELINE_ID)
|
||||
_include_retried = 'true' if include_retried else 'false'
|
||||
jobs = pipelines.jobs.list(all=True, per_page=100, include_retried=_include_retried)
|
||||
|
||||
job_info_list = []
|
||||
for job in jobs:
|
||||
if job.stage in ['target_test', 'host_test']:
|
||||
log_path = ''
|
||||
if job.status not in ['success', 'failed']:
|
||||
print('Job {}({}) is not finished'.format(job.id, job.name))
|
||||
elif not hasattr(job, 'artifacts_file'):
|
||||
print('Job {}({}) has no artifacts.'.format(job.id, job.name))
|
||||
else:
|
||||
log_path = os.path.join(result_path, 'job_{}'.format(job.id))
|
||||
print('Downloading artifacts from: {}'.format(job.name))
|
||||
os.makedirs(log_path, exist_ok=True)
|
||||
gitlab_inst.download_artifacts(job.id, log_path)
|
||||
|
||||
job_info = {
|
||||
'id': job.id,
|
||||
'name': job.name,
|
||||
'tag_list': job.tag_list,
|
||||
'status': job.status,
|
||||
'stage': job.stage,
|
||||
'web_url': job.web_url,
|
||||
'commit_url': job.commit['web_url'],
|
||||
'log_path': log_path,
|
||||
}
|
||||
job_info_list.append(job_info)
|
||||
|
||||
with open(os.path.join(result_path, 'index.json'), 'w') as f:
|
||||
f.write(json.dumps({'jobs': job_info_list}, indent=1, sort_keys=True))
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--path', '-p', default=TEST_RESULT_PATH, help='Path to save result files.')
|
||||
parser.add_argument('--include_retried', action='store_true', help='Including retried jobs.')
|
||||
args = parser.parse_args()
|
||||
|
||||
download_all_test_results(args.path, args.include_retried)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Loading…
Reference in New Issue
Block a user