mirror of
https://github.com/espressif/esp-idf.git
synced 2024-10-05 20:47:46 -04:00
Merge branch 'ci/optimize_ats_ci_flow_v4.2' into 'release/v4.2'
CI: optimize ATS ci flow (v4.2) See merge request espressif/esp-idf!20179
This commit is contained in:
Jiang Jiang Jian
commit
e81c5be7d8
@ -57,8 +57,6 @@ variables:
|
||||
CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/tools/ci/config/target-test.yml"
|
||||
# target test repo parameters
|
||||
TEST_ENV_CONFIG_REPO: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/ci-test-runner-configs.git"
|
||||
CI_AUTO_TEST_SCRIPT_REPO_URL: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/auto_test_script.git"
|
||||
CI_AUTO_TEST_SCRIPT_REPO_BRANCH: "ci/v4.1"
|
||||
|
||||
# Versioned esp-idf-doc env image to use for all document building jobs
|
||||
ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env:v7"
|
||||
|
||||
@ -10,8 +10,20 @@ import subprocess
|
||||
import re
|
||||
|
||||
|
||||
IDF_GIT_DESCRIBE_PATTERN = re.compile(r"^v(\d)\.(\d)")
|
||||
RETRY_COUNT = 3
|
||||
def _idf_version_from_cmake():
|
||||
version_path = os.path.join(os.environ['IDF_PATH'], 'tools/cmake/version.cmake')
|
||||
regex = re.compile(r'^\s*set\s*\(\s*IDF_VERSION_([A-Z]{5})\s+(\d+)')
|
||||
try:
|
||||
ver = {}
|
||||
with open(version_path) as f:
|
||||
for line in f:
|
||||
m = regex.match(line)
|
||||
if m:
|
||||
ver[m.group(1)] = m.group(2)
|
||||
return (int(ver['MAJOR']), int(ver['MINOR']))
|
||||
except (KeyError, OSError):
|
||||
print('WARNING: Cannot find ESP-IDF version in version.cmake')
|
||||
return (0, 0)
|
||||
|
||||
|
||||
def get_customized_project_revision(proj_name):
|
||||
@ -45,20 +57,12 @@ def target_branch_candidates(proj_name):
|
||||
candidates.insert(0, customized_candidate)
|
||||
|
||||
# branch name read from IDF
|
||||
try:
|
||||
git_describe = subprocess.check_output(["git", "describe", "HEAD"])
|
||||
match = IDF_GIT_DESCRIBE_PATTERN.search(git_describe.decode())
|
||||
if match:
|
||||
major_revision = match.group(1)
|
||||
minor_revision = match.group(2)
|
||||
# release branch
|
||||
candidates.append("release/v{}.{}".format(major_revision, minor_revision))
|
||||
# branch to match all major branches, like v3.x or v3
|
||||
candidates.append("release/v{}.x".format(major_revision))
|
||||
candidates.append("release/v{}".format(major_revision))
|
||||
except subprocess.CalledProcessError:
|
||||
# this should not happen as IDF should have describe message
|
||||
pass
|
||||
major_revision, minor_revision = _idf_version_from_cmake()
|
||||
# release branch
|
||||
candidates.append('release/v{}.{}'.format(major_revision, minor_revision))
|
||||
# branch to match all major branches, like v3.x or v3
|
||||
candidates.append('release/v{}.x'.format(major_revision))
|
||||
candidates.append('release/v{}'.format(major_revision))
|
||||
|
||||
return [c for c in candidates if c] # filter out null value
|
||||
|
||||
@ -85,23 +89,11 @@ if __name__ == "__main__":
|
||||
|
||||
ref_to_use = ""
|
||||
for candidate in candidate_branches:
|
||||
# check if candidate branch exists
|
||||
branch_match = subprocess.check_output(["git", "branch", "-a", "--list", "origin/" + candidate])
|
||||
if branch_match:
|
||||
ref_to_use = candidate
|
||||
try:
|
||||
subprocess.check_call(['git', 'checkout', '-f', candidate], stdout=subprocess.PIPE) # not print the stdout
|
||||
print('CI using ref {} for project {}'.format(candidate, args.project))
|
||||
break
|
||||
|
||||
if ref_to_use:
|
||||
for _ in range(RETRY_COUNT):
|
||||
# Add retry for projects with git-lfs
|
||||
try:
|
||||
subprocess.check_call(["git", "checkout", "-f", ref_to_use], stdout=subprocess.PIPE) # not print the stdout
|
||||
print("CI using ref {} for project {}".format(ref_to_use, args.project))
|
||||
break
|
||||
except subprocess.CalledProcessError:
|
||||
pass
|
||||
else:
|
||||
print("Failed to use ref {} for project {}".format(ref_to_use, args.project))
|
||||
exit(1)
|
||||
except subprocess.CalledProcessError:
|
||||
pass
|
||||
else:
|
||||
print("using default branch")
|
||||
|
||||
@ -19,6 +19,7 @@ assign_test:
|
||||
ASSIGN_TEST_CASE_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIAssignTestCases.py"
|
||||
PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages
|
||||
PYTHON_VER: 3.7.7
|
||||
GIT_LFS_SKIP_SMUDGE: 1
|
||||
artifacts:
|
||||
paths:
|
||||
- components/idf_test/*/CIConfigs
|
||||
@ -43,11 +44,12 @@ assign_test:
|
||||
# assign unit test cases
|
||||
- python tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py $UNIT_TEST_CASE_FILE $CI_TARGET_TEST_CONFIG_FILE $IDF_PATH/components/idf_test/unit_test/CIConfigs
|
||||
# clone test script to assign tests
|
||||
# can not retry if downloading git lfs files failed, so using empty_branch first.
|
||||
- ./tools/ci/retry_failed.sh git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
|
||||
- ./tools/ci/retry_failed.sh git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
|
||||
- python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only
|
||||
- ./tools/ci/retry_failed.sh git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} auto_test_script
|
||||
- python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
|
||||
- cd auto_test_script
|
||||
- ./tools/ci/setup_idfci.sh
|
||||
# assign integration test cases
|
||||
- cd ${CI_PROJECT_DIR}
|
||||
- python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b $IDF_PATH/SSC/ssc_bin -o $INTEGRATION_CONFIG_OUTPUT_PATH
|
||||
|
||||
update_test_cases:
|
||||
|
||||
@ -169,6 +169,8 @@
|
||||
CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/integration_test/CIConfigs"
|
||||
KNOWN_ISSUE_FILE: "${CI_PROJECT_DIR}/components/idf_test/integration_test/KnownIssues"
|
||||
PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages
|
||||
INITIAL_CONDITION_RETRY_COUNT: "1"
|
||||
GIT_LFS_SKIP_SMUDGE: 1
|
||||
PYTHON_VER: 3.7.7
|
||||
script:
|
||||
- *define_config_file_name
|
||||
@ -177,14 +179,14 @@
|
||||
# clone local test env configs
|
||||
- ./tools/ci/retry_failed.sh git clone $TEST_ENV_CONFIG_REPO
|
||||
- python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
|
||||
# clone auto test repo
|
||||
- ./tools/ci/retry_failed.sh git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
|
||||
- ./tools/ci/retry_failed.sh git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
|
||||
- python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only
|
||||
- export PYTHONPATH="${CI_PROJECT_DIR}/auto_test_script/packages:$PYTHONPATH"
|
||||
# clone test bench
|
||||
- ./tools/ci/retry_failed.sh git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} auto_test_script
|
||||
- python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
|
||||
- cd auto_test_script
|
||||
# run test
|
||||
- ./tools/ci/setup_idfci.sh
|
||||
# Merge known issues
|
||||
- cat ${KNOWN_ISSUE_FILE} >> ${TEST_CASE_FILE_PATH}/KnownIssues
|
||||
# run test
|
||||
- python bin/CIRunner.py -l "$LOG_PATH/$JOB_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH
|
||||
|
||||
test_weekend_mqtt:
|
||||
@ -615,11 +617,10 @@ nvs_compatible_test:
|
||||
- ./tools/ci/retry_failed.sh git clone $TEST_ENV_CONFIG_REPO
|
||||
- python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
|
||||
# clone test bench
|
||||
# can not retry if downing git lfs files failed, so using empty_branch first.
|
||||
- ./tools/ci/retry_failed.sh git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
|
||||
- ./tools/ci/retry_failed.sh git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
|
||||
- python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only
|
||||
- ./tools/ci/retry_failed.sh git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} auto_test_script
|
||||
- python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
|
||||
- cd auto_test_script
|
||||
- ./tools/ci/setup_idfci.sh
|
||||
# prepare nvs bins
|
||||
- ./tools/prepare_nvs_bin.sh
|
||||
# run test
|
||||
|
||||
Loading…
Reference in New Issue
Block a user