From 5ad0ccea461295a094b2f62f7559aaff7fbbc6f4 Mon Sep 17 00:00:00 2001 From: Chen Yudong Date: Wed, 14 Sep 2022 02:43:40 +0800 Subject: [PATCH] CI: optimize ATS ci flow --- .gitlab-ci.yml | 2 -- tools/ci/checkout_project_ref.py | 58 ++++++++++++++------------------ tools/ci/config/assign-test.yml | 10 +++--- tools/ci/config/target-test.yml | 20 ++++++----- 4 files changed, 42 insertions(+), 48 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 09ff4ee233..0da543f6c1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -59,8 +59,6 @@ variables: CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/tools/ci/config/target-test.yml" # target test repo parameters TEST_ENV_CONFIG_REPO: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/ci-test-runner-configs.git" - CI_AUTO_TEST_SCRIPT_REPO_URL: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/auto_test_script.git" - CI_AUTO_TEST_SCRIPT_REPO_BRANCH: "ci/v4.1" # Versioned esp-idf-doc env image to use for all document building jobs ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env:v7" diff --git a/tools/ci/checkout_project_ref.py b/tools/ci/checkout_project_ref.py index 175c6fe295..25feab70a4 100755 --- a/tools/ci/checkout_project_ref.py +++ b/tools/ci/checkout_project_ref.py @@ -10,8 +10,20 @@ import subprocess import re -IDF_GIT_DESCRIBE_PATTERN = re.compile(r"^v(\d)\.(\d)") -RETRY_COUNT = 3 +def _idf_version_from_cmake(): + version_path = os.path.join(os.environ['IDF_PATH'], 'tools/cmake/version.cmake') + regex = re.compile(r'^\s*set\s*\(\s*IDF_VERSION_([A-Z]{5})\s+(\d+)') + try: + ver = {} + with open(version_path) as f: + for line in f: + m = regex.match(line) + if m: + ver[m.group(1)] = m.group(2) + return (int(ver['MAJOR']), int(ver['MINOR'])) + except (KeyError, OSError): + print('WARNING: Cannot find ESP-IDF version in version.cmake') + return (0, 0) def get_customized_project_revision(proj_name): @@ -45,20 +57,12 @@ def target_branch_candidates(proj_name): candidates.insert(0, customized_candidate) # branch name read from IDF - try: - git_describe = subprocess.check_output(["git", "describe", "HEAD"]) - match = IDF_GIT_DESCRIBE_PATTERN.search(git_describe.decode()) - if match: - major_revision = match.group(1) - minor_revision = match.group(2) - # release branch - candidates.append("release/v{}.{}".format(major_revision, minor_revision)) - # branch to match all major branches, like v3.x or v3 - candidates.append("release/v{}.x".format(major_revision)) - candidates.append("release/v{}".format(major_revision)) - except subprocess.CalledProcessError: - # this should not happen as IDF should have describe message - pass + major_revision, minor_revision = _idf_version_from_cmake() + # release branch + candidates.append('release/v{}.{}'.format(major_revision, minor_revision)) + # branch to match all major branches, like v3.x or v3 + candidates.append('release/v{}.x'.format(major_revision)) + candidates.append('release/v{}'.format(major_revision)) return [c for c in candidates if c] # filter out null value @@ -85,23 +89,11 @@ if __name__ == "__main__": ref_to_use = "" for candidate in candidate_branches: - # check if candidate branch exists - branch_match = subprocess.check_output(["git", "branch", "-a", "--list", "origin/" + candidate]) - if branch_match: - ref_to_use = candidate + try: + subprocess.check_call(['git', 'checkout', '-f', candidate], stdout=subprocess.PIPE) # not print the stdout + print('CI using ref {} for project {}'.format(candidate, args.project)) break - - if ref_to_use: - for _ in range(RETRY_COUNT): - # Add retry for projects with git-lfs - try: - subprocess.check_call(["git", "checkout", "-f", ref_to_use], stdout=subprocess.PIPE) # not print the stdout - print("CI using ref {} for project {}".format(ref_to_use, args.project)) - break - except subprocess.CalledProcessError: - pass - else: - print("Failed to use ref {} for project {}".format(ref_to_use, args.project)) - exit(1) + except subprocess.CalledProcessError: + pass else: print("using default branch") diff --git a/tools/ci/config/assign-test.yml b/tools/ci/config/assign-test.yml index b4216aee4e..ec214ecf85 100644 --- a/tools/ci/config/assign-test.yml +++ b/tools/ci/config/assign-test.yml @@ -18,6 +18,7 @@ assign_test: ASSIGN_TEST_CASE_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIAssignTestCases.py" PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages PYTHON_VER: 3.7.7 + GIT_LFS_SKIP_SMUDGE: 1 artifacts: paths: - components/idf_test/*/CIConfigs @@ -36,11 +37,12 @@ assign_test: # assign unit test cases - python tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py $UNIT_TEST_CASE_FILE $CI_TARGET_TEST_CONFIG_FILE $IDF_PATH/components/idf_test/unit_test/CIConfigs # clone test script to assign tests - # can not retry if downing git lfs files failed, so using empty_branch first. - - ./tools/ci/retry_failed.sh git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch - - ./tools/ci/retry_failed.sh git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH} - - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only + - ./tools/ci/retry_failed.sh git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} auto_test_script + - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script + - cd auto_test_script + - ./tools/ci/setup_idfci.sh # assign integration test cases + - cd ${CI_PROJECT_DIR} - python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b $IDF_PATH/SSC/ssc_bin -o $INTEGRATION_CONFIG_OUTPUT_PATH update_test_cases: diff --git a/tools/ci/config/target-test.yml b/tools/ci/config/target-test.yml index 5ece5b8fcf..9071d9b425 100644 --- a/tools/ci/config/target-test.yml +++ b/tools/ci/config/target-test.yml @@ -149,6 +149,8 @@ KNOWN_ISSUE_FILE: "${CI_PROJECT_DIR}/components/idf_test/integration_test/KnownIssues" CI_RUNNER_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIRunner.py" PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages + INITIAL_CONDITION_RETRY_COUNT: "1" + GIT_LFS_SKIP_SMUDGE: 1 PYTHON_VER: 3.7.7 script: - *define_config_file_name @@ -158,10 +160,11 @@ - ./tools/ci/retry_failed.sh git clone $TEST_ENV_CONFIG_REPO - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs # clone test bench - # can not retry if downing git lfs files failed, so using empty_branch first. - - ./tools/ci/retry_failed.sh git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch - - ./tools/ci/retry_failed.sh git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH} - - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only + - ./tools/ci/retry_failed.sh git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} auto_test_script + - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script + - cd auto_test_script + - ./tools/ci/setup_idfci.sh + # Merge known issues - cat ${KNOWN_ISSUE_FILE} >> ${TEST_CASE_FILE_PATH}/KnownIssues # run test - python ${CI_RUNNER_SCRIPT} -l "$LOG_PATH/$JOB_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH @@ -523,12 +526,11 @@ nvs_compatible_test: - ./tools/ci/retry_failed.sh git clone $TEST_ENV_CONFIG_REPO - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs # clone test bench - # can not retry if downing git lfs files failed, so using empty_branch first. - - ./tools/ci/retry_failed.sh git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch - - ./tools/ci/retry_failed.sh git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH} - - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only - # prepare nvs bins + - ./tools/ci/retry_failed.sh git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} auto_test_script + - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script - cd auto_test_script + - ./tools/ci/setup_idfci.sh + # prepare nvs bins - ./tools/prepare_nvs_bin.sh # run test - python ${CI_RUNNER_SCRIPT} -l "$LOG_PATH/$JOB_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH