Merge branch 'bugfix/auto_test_script_only_use_given_branch_v4.1' into 'release/v4.1'

CI: limit switching branches for auto_test_script (v4.1)

See merge request espressif/esp-idf!11147

tools/ci/checkout_project_ref.py

@@ -11,6 +11,22 @@ import re
 
 
 IDF_GIT_DESCRIBE_PATTERN = re.compile(r"^v(\d)\.(\d)")
+RETRY_COUNT = 3
+
+
+def get_customized_project_revision(proj_name):
+    """
+    get customized project revision defined in bot message
+    """
+    revision = ""
+    customized_project_revisions = os.getenv("BOT_CUSTOMIZED_REVISION")
+    if customized_project_revisions:
+        customized_project_revisions = json.loads(customized_project_revisions)
+    try:
+        revision = customized_project_revisions[proj_name.lower()]
+    except (KeyError, TypeError):
+        pass
+    return revision
 
 
 def target_branch_candidates(proj_name):
@@ -23,16 +39,11 @@ def target_branch_candidates(proj_name):
         # CI_MERGE_REQUEST_TARGET_BRANCH_NAME
         os.getenv("CI_MERGE_REQUEST_TARGET_BRANCH_NAME"),
     ]
-    # revision defined in bot message
-    customized_project_revisions = os.getenv("BOT_CUSTOMIZED_REVISION")
-    if customized_project_revisions:
-        customized_project_revisions = json.loads(customized_project_revisions)
-    try:
-        ref_to_use = customized_project_revisions[proj_name.lower()]
+    customized_candidate = get_customized_project_revision(proj_name)
+    if customized_candidate:
         # highest priority, insert to head of list
-        candidates.insert(0, ref_to_use)
-    except (KeyError, TypeError):
-        pass
+        candidates.insert(0, customized_candidate)
+
     # branch name read from IDF
     try:
         git_describe = subprocess.check_output(["git", "describe", "HEAD"])
@@ -58,20 +69,39 @@ if __name__ == "__main__":
                         help="the name of project")
     parser.add_argument("project_relative_path",
                         help="relative path of project to IDF repository directory")
+    parser.add_argument('--customized_only', action='store_true',
+                        help="Only to find customized revision")
 
     args = parser.parse_args()
 
-    candidate_branches = target_branch_candidates(args.project)
+    if args.customized_only:
+        customized_revision = get_customized_project_revision(args.project)
+        candidate_branches = [customized_revision] if customized_revision else []
+    else:
+        candidate_branches = target_branch_candidates(args.project)
 
     # change to project dir for checkout
     os.chdir(args.project_relative_path)
 
+    ref_to_use = ""
     for candidate in candidate_branches:
-        try:
-            subprocess.check_call(["git", "checkout", "-f", candidate], stdout=subprocess.PIPE, stderr=subprocess.PIPE)  # not print the stdout nor stderr
-            print("CI using ref {} for project {}".format(candidate, args.project))
+        # check if candidate branch exists
+        branch_match = subprocess.check_output(["git", "branch", "-a", "--list", "origin/" + candidate])
+        if branch_match:
+            ref_to_use = candidate
             break
-        except subprocess.CalledProcessError:
-            pass
+
+    if ref_to_use:
+        for _ in range(RETRY_COUNT):
+            # Add retry for projects with git-lfs
+            try:
+                subprocess.check_call(["git", "checkout", "-f", ref_to_use], stdout=subprocess.PIPE)  # not print the stdout
+                print("CI using ref {} for project {}".format(ref_to_use, args.project))
+                break
+            except subprocess.CalledProcessError:
+                pass
+        else:
+            print("Failed to use ref {} for project {}".format(ref_to_use, args.project))
+            exit(1)
     else:
         print("using default branch")

tools/ci/config/assign-test.yml

@@ -39,7 +39,7 @@ assign_test:
     # can not retry if downing git lfs files failed, so using empty_branch first.
     - ./tools/ci/retry_failed.sh git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
     - ./tools/ci/retry_failed.sh git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
-    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
+    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only
     # assign integration test cases
     - python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b $IDF_PATH/SSC/ssc_bin -o $INTEGRATION_CONFIG_OUTPUT_PATH
 

tools/ci/config/target-test.yml

@@ -153,7 +153,7 @@
     # can not retry if downing git lfs files failed, so using empty_branch first.
     - ./tools/ci/retry_failed.sh git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
     - ./tools/ci/retry_failed.sh git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
-    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
+    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only
     - cat ${KNOWN_ISSUE_FILE} >> ${TEST_CASE_FILE_PATH}/KnownIssues
     # run test
     - python ${CI_RUNNER_SCRIPT} -l "$LOG_PATH/$JOB_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH
@@ -521,7 +521,7 @@ nvs_compatible_test:
     # can not retry if downing git lfs files failed, so using empty_branch first.
     - ./tools/ci/retry_failed.sh git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
     - ./tools/ci/retry_failed.sh git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
-    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
+    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only
     # prepare nvs bins
     - cd auto_test_script
     - ./tools/prepare_nvs_bin.sh