CI: try to use the correct branch of other projects used in CI:

1. revision defined in bot message
2. branch name (or tag name) of current IDF
3. CI_MERGE_REQUEST_TARGET_BRANCH_NAME
4. branch name parsed from `git describe`
5. default branch

.gitlab-ci.yml

@@ -131,11 +131,11 @@ build_template_app:
     # Set the variable for 'esp-idf-template' testing
     - ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"}
     - git clone ${ESP_IDF_TEMPLATE_GIT}
+    - python $CHECKOUT_REF_SCRIPT esp-idf-template esp-idf-template
     - cd esp-idf-template
     # Try to use the same branch name for esp-idf-template that we're
     # using on esp-idf. If it doesn't exist then just stick to the default
     # branch
-    - python $CHECKOUT_REF_SCRIPT esp-idf-template
     - make defconfig
     # Test debug build (default)
     - make all V=1
@@ -175,8 +175,8 @@ build_ssc:
       - $BOT_LABEL_REGULAR_TEST
   script:
     - git clone $SSC_REPOSITORY
+    - python $CHECKOUT_REF_SCRIPT SSC SSC
     - cd SSC
-    - python $CHECKOUT_REF_SCRIPT SSC
     - MAKEFLAGS= ./ci_build_ssc.sh
 
 # If you want to add new build ssc jobs, please add it into dependencies of `assign_test` and `.test_template`
@@ -771,8 +771,8 @@ update_test_cases:
   script:
     - export GIT_SHA=$(echo ${CI_COMMIT_SHA} | cut -c 1-8)
     - git clone $TEST_MANAGEMENT_REPO
+    - python $CHECKOUT_REF_SCRIPT test-management test-management
     - cd test-management
-    - python $CHECKOUT_REF_SCRIPT test-management
     - echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE}
     # update unit test cases
     - python ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT unity -d $UNIT_TEST_CASE_FILE -r $GIT_SHA
@@ -821,8 +821,8 @@ deploy_test_result:
     # we need to remove it so we can clone test-management folder again
     - rm -r test-management
     - git clone $TEST_MANAGEMENT_REPO
+    - python3 $CHECKOUT_REF_SCRIPT test-management test-management
     - cd test-management
-    - python3 $CHECKOUT_REF_SCRIPT test-management
     - echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE}
     # update test results
     - python3 ImportTestResult.py -r "$GIT_SHA (r${REV_COUNT})" -j $JIRA_TEST_MANAGEMENT_PROJECT -s "$SUMMARY" -l CI -p ${CI_PROJECT_DIR}/TEST_LOGS ${CI_PROJECT_DIR}/${CI_COMMIT_SHA} --pipeline_url ${CI_PIPELINE_URL}
@@ -1005,8 +1005,8 @@ assign_test:
     - python $TEST_FW_PATH/CIAssignUnitTest.py $IDF_PATH/components/idf_test/unit_test/TestCaseAll.yml $IDF_PATH/.gitlab-ci.yml $IDF_PATH/components/idf_test/unit_test/CIConfigs
     # clone test script to assign tests
     - git clone $TEST_SCRIPT_REPOSITORY
+    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
     - cd auto_test_script
-    - python $CHECKOUT_REF_SCRIPT auto_test_script
     # assgin integration test cases
     - python CIAssignTestCases.py -t $IDF_PATH/components/idf_test/integration_test -c $IDF_PATH/.gitlab-ci.yml -b $IDF_PATH/SSC/ssc_bin
 
@@ -1045,8 +1045,7 @@ assign_test:
     - test -e $CONFIG_FILE || exit 0
     # clone test env configs
     - git clone $TEST_ENV_CONFIG_REPOSITORY
-    - cd ci-test-runner-configs
+    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
-    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs
     - cd $TEST_FW_PATH
     # run test
     - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE
@@ -1143,12 +1142,11 @@ test_weekend_network:
     - test -e $CONFIG_FILE || exit 0
     # clone local test env configs
     - git clone $TEST_ENV_CONFIG_REPOSITORY
-    - cd ci-test-runner-configs
+    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
-    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs
     # clone test bench
     - git clone $TEST_SCRIPT_REPOSITORY
+    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
     - cd auto_test_script
-    - python $CHECKOUT_REF_SCRIPT auto_test_script
     # run test
     - python CIRunner.py -l "$LOG_PATH/$CI_JOB_NAME_$CI_NODE_INDEX" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH -m $MODULE_UPDATE_FILE
 
@@ -1166,12 +1164,11 @@ nvs_compatible_test:
   script:
     # clone local test env configs
     - git clone $TEST_ENV_CONFIG_REPOSITORY
-    - cd ci-test-runner-configs
+    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
-    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs
     # clone test bench
     - git clone $TEST_SCRIPT_REPOSITORY
+    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
     - cd auto_test_script
-    - git checkout ${CI_COMMIT_REF_NAME} || echo "Using default branch..."
     # prepare nvs bins
     - ./Tools/prepare_nvs_bin.sh
     # run test

tools/ci/checkout_project_ref.py

@@ -7,32 +7,67 @@ import os
 import json
 import argparse
 import subprocess
+import re
 
 
-def checkout_branch(proj_name, customized_revision, default_ref_name):
+IDF_GIT_DESCRIBE_PATTERN = re.compile(r"^v(\d)\.(\d)")
+
+
+def target_branch_candidates(proj_name):
+    """
+    :return: a list of target branch candidates, from highest priority to lowest priority.
+    """
+    candidates = [
+        # branch name (or tag name) of current IDF
+        os.getenv("CI_COMMIT_REF_NAME"),
+        # CI_MERGE_REQUEST_TARGET_BRANCH_NAME
+        os.getenv("CI_MERGE_REQUEST_TARGET_BRANCH_NAME"),
+    ]
+    # revision defined in bot message
+    customized_project_revisions = os.getenv("BOT_CUSTOMIZED_REVISION")
+    if customized_project_revisions:
+        customized_project_revisions = json.loads(customized_project_revisions)
     try:
-        ref_to_use = customized_revision[proj_name.lower()]
+        ref_to_use = customized_project_revisions[proj_name.lower()]
+        # highest priority, insert to head of list
+        candidates.insert(0, ref_to_use)
     except (KeyError, TypeError):
-        ref_to_use = default_ref_name
+        pass
+    # branch name read from IDF
+    git_describe = subprocess.check_output(["git", "describe", "--tags", "HEAD"])
+    match = IDF_GIT_DESCRIBE_PATTERN.search(git_describe)
+    if match:
+        major_revision = match.group(1)
+        minor_revision = match.group(2)
+        # release branch
+        candidates.append("release/v{}.{}".format(major_revision, minor_revision))
+        # branch to match all major branches, like v3.x or v3
+        candidates.append("release/v{}.x".format(major_revision))
+        candidates.append("release/v{}".format(major_revision))
 
-    try:
+    return [c for c in candidates if c]  # filter out null value
-        subprocess.check_call(["git", "checkout", ref_to_use])
-        print("CI using ref {} for project {}".format(ref_to_use, proj_name))
-    except subprocess.CalledProcessError:
-        print("using default branch")
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("project",
                         help="the name of project")
+    parser.add_argument("project_relative_path",
+                        help="relative path of project to IDF repository directory")
 
     args = parser.parse_args()
-    project_name = args.project
 
-    customized_project_revisions = os.getenv("BOT_CUSTOMIZED_REVISION")
+    candidate_branches = target_branch_candidates(args.project)
-    if customized_project_revisions:
-        customized_project_revisions = json.loads(customized_project_revisions)
-    ci_ref_name = os.getenv("CI_COMMIT_REF_NAME")
 
-    checkout_branch(project_name, customized_project_revisions, ci_ref_name)
+    # change to project dir for checkout
+    os.chdir(args.project_relative_path)
+
+    for candidate in candidate_branches:
+        try:
+            subprocess.check_call(["git", "checkout", candidate])
+            print("CI using ref {} for project {}".format(candidate, args.project))
+            break
+        except subprocess.CalledProcessError:
+            pass
+    else:
+        print("using default branch")