Merge branch 'ci/optimize_ats_ci_flow' into 'master'

CI: optimize ATS ci flow See merge request espressif/esp-idf!20131
2024-09-20 00:36:01 -04:00 · 2022-09-16 20:34:03 +08:00 · 2022-09-16 20:34:03 +08:00 · 40d13002d0
commit 40d13002d0
parent 952c0f7b23 bfa96bc597
3 changed files with 43 additions and 43 deletions
--- a/.gitlab/ci/assign-test.yml
+++ b/.gitlab/ci/assign-test.yml
@ -100,12 +100,13 @@ assign_integration_test:
    INTEGRATION_TEST_CASE_PATH: "${CI_PROJECT_DIR}/auto_test_script/TestCaseFiles"
    ASSIGN_TEST_CASE_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIAssignTestCases.py"
    PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages
+    GIT_LFS_SKIP_SMUDGE: 1
  script:
    - add_gitlab_ssh_keys
    # clone test script to assign tests
-    # can not retry if downing git lfs files failed, so using empty_branch first.
-    - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
-    - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
-    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only
+    - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} auto_test_script
+    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
+    - cd auto_test_script
+    - ./tools/ci/setup_idfci.sh
    # assign integration test cases
    - python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b ${BUILD_DIR} -o $TEST_DIR/test_configs
--- a/.gitlab/ci/target-test.yml
+++ b/.gitlab/ci/target-test.yml
@ -1169,8 +1169,8 @@ UT_S3_FLASH:
    CI_RUNNER_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIRunner.py"
    PREPARE_TEST_BIN_SCRIPT: "${CI_PROJECT_DIR}/tools/ci/integration_test/prepare_test_bins.py"
    PYTHONPATH: "${CI_PROJECT_DIR}/auto_test_script/packages:${CI_PROJECT_DIR}/tools/ci/python_packages:${PYTHONPATH}"
-    SETUP_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/tools/ci/setup_idfci.sh"
    INITIAL_CONDITION_RETRY_COUNT: "1"
+    GIT_LFS_SKIP_SMUDGE: 1
  script:
    - *define_config_file_name
    # first test if config file exists, if not exist, exit 0
@ -1180,11 +1180,11 @@ UT_S3_FLASH:
    - retry_failed git clone $TEST_ENV_CONFIG_REPO
    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
    # clone test bench
-    # can not retry if downing git lfs files failed, so using empty_branch first.
-    - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
-    - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
-    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only
-    - if [ -f ${SETUP_SCRIPT} ]; then ${SETUP_SCRIPT}; fi
+    - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} auto_test_script
+    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
+    - cd auto_test_script
+    - ./tools/ci/setup_idfci.sh
+    # Merge known issues
    - cat ${KNOWN_ISSUE_FILE} >> ${TEST_CASE_FILE_PATH}/KnownIssues
    # run test
    - python ${PREPARE_TEST_BIN_SCRIPT} $CONFIG_FILE
@ -1216,13 +1216,11 @@ nvs_compatible_test:
    - retry_failed git clone $TEST_ENV_CONFIG_REPO
    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
    # clone test bench
-    # can not retry if downing git lfs files failed, so using empty_branch first.
-    - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
-    - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
-    - if [ -f ${SETUP_SCRIPT} ]; then ${SETUP_SCRIPT}; fi
-    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only
-    # prepare nvs bins
+    - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} auto_test_script
+    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
    - cd auto_test_script
+    - ./tools/ci/setup_idfci.sh
+    # prepare nvs bins
    - ./tools/prepare_nvs_bin.sh
    # run test
    - python ${PREPARE_TEST_BIN_SCRIPT} $CONFIG_FILE
--- a/tools/ci/checkout_project_ref.py
+++ b/tools/ci/checkout_project_ref.py
@ -10,10 +10,25 @@ import json
 import os
 import re
 import subprocess
-from typing import List
+from typing import List, Tuple

 IDF_GIT_DESCRIBE_PATTERN = re.compile(r'^v(\d)\.(\d)')
-RETRY_COUNT = 3
+
+
+def _idf_version_from_cmake() -> Tuple[int, int]:
+    version_path = os.path.join(os.environ['IDF_PATH'], 'tools/cmake/version.cmake')
+    regex = re.compile(r'^\s*set\s*\(\s*IDF_VERSION_([A-Z]{5})\s+(\d+)')
+    try:
+        ver = {}
+        with open(version_path) as f:
+            for line in f:
+                m = regex.match(line)
+                if m:
+                    ver[m.group(1)] = m.group(2)
+        return (int(ver['MAJOR']), int(ver['MINOR']))
+    except (KeyError, OSError):
+        print('WARNING: Cannot find ESP-IDF version in version.cmake')
+        return (0, 0)


 def get_customized_project_revision(proj_name: str) -> str:
@ -47,20 +62,12 @@ def target_branch_candidates(proj_name: str) -> List:
        candidates.insert(0, customized_candidate)

    # branch name read from IDF
-    try:
-        git_describe = subprocess.check_output(['git', 'describe', 'HEAD'])
-        match = IDF_GIT_DESCRIBE_PATTERN.search(git_describe.decode())
-        if match:
-            major_revision = match.group(1)
-            minor_revision = match.group(2)
-            # release branch
-            candidates.append('release/v{}.{}'.format(major_revision, minor_revision))
-            # branch to match all major branches, like v3.x or v3
-            candidates.append('release/v{}.x'.format(major_revision))
-            candidates.append('release/v{}'.format(major_revision))
-    except subprocess.CalledProcessError:
-        # this should not happen as IDF should have describe message
-        pass
+    major_revision, minor_revision = _idf_version_from_cmake()
+    # release branch
+    candidates.append('release/v{}.{}'.format(major_revision, minor_revision))
+    # branch to match all major branches, like v3.x or v3
+    candidates.append('release/v{}.x'.format(major_revision))
+    candidates.append('release/v{}'.format(major_revision))

    return [c for c in candidates if c]  # filter out null value

@ -103,16 +110,10 @@ if __name__ == '__main__':
            continue

    if ref_to_use:
-        for _ in range(RETRY_COUNT):
-            # Add retry for projects with git-lfs
-            try:
-                subprocess.check_call(['git', 'checkout', '-f', ref_to_use], stdout=subprocess.PIPE)  # not print the stdout
-                print('CI using ref {} for project {}'.format(ref_to_use, args.project))
-                break
-            except subprocess.CalledProcessError:
-                pass
-        else:
-            print('Failed to use ref {} for project {}'.format(ref_to_use, args.project))
-            exit(1)
+        try:
+            subprocess.check_call(['git', 'checkout', '-f', ref_to_use], stdout=subprocess.PIPE)  # not print the stdout
+            print('CI using ref {} for project {}'.format(ref_to_use, args.project))
+        except subprocess.CalledProcessError:
+            pass
    else:
        print('using default branch')