ci: limit switching branches for auto_test_script

2024-10-05 20:47:46 -04:00 · 2020-10-30 14:39:42 +08:00 · 2020-10-30 14:39:42 +08:00 · 4620826c83
commit 4620826c83
parent 42ea9b7cb1
3 changed files with 55 additions and 22 deletions
--- a/tools/ci/checkout_project_ref.py
+++ b/tools/ci/checkout_project_ref.py
@ -11,6 +11,22 @@ import re


 IDF_GIT_DESCRIBE_PATTERN = re.compile(r"^v(\d)\.(\d)")
+RETRY_COUNT = 3
+
+
+def get_customized_project_revision(proj_name):
+    """
+    get customized project revision defined in bot message
+    """
+    revision = ""
+    customized_project_revisions = os.getenv("BOT_CUSTOMIZED_REVISION")
+    if customized_project_revisions:
+        customized_project_revisions = json.loads(customized_project_revisions)
+    try:
+        revision = customized_project_revisions[proj_name.lower()]
+    except (KeyError, TypeError):
+        pass
+    return revision


 def target_branch_candidates(proj_name):
@ -23,16 +39,11 @@ def target_branch_candidates(proj_name):
        # CI_MERGE_REQUEST_TARGET_BRANCH_NAME
        os.getenv("CI_MERGE_REQUEST_TARGET_BRANCH_NAME"),
    ]
-    # revision defined in bot message
-    customized_project_revisions = os.getenv("BOT_CUSTOMIZED_REVISION")
-    if customized_project_revisions:
-        customized_project_revisions = json.loads(customized_project_revisions)
-    try:
-        ref_to_use = customized_project_revisions[proj_name.lower()]
+    customized_candidate = get_customized_project_revision(proj_name)
+    if customized_candidate:
        # highest priority, insert to head of list
-        candidates.insert(0, ref_to_use)
-    except (KeyError, TypeError):
-        pass
+        candidates.insert(0, customized_candidate)
+
    # branch name read from IDF
    try:
        git_describe = subprocess.check_output(["git", "describe", "HEAD"])
@ -58,20 +69,39 @@ if __name__ == "__main__":
                        help="the name of project")
    parser.add_argument("project_relative_path",
                        help="relative path of project to IDF repository directory")
+    parser.add_argument('--customized_only', action='store_true',
+                        help="Only to find customized revision")

    args = parser.parse_args()

-    candidate_branches = target_branch_candidates(args.project)
+    if args.customized_only:
+        customized_revision = get_customized_project_revision(args.project)
+        candidate_branches = [customized_revision] if customized_revision else []
+    else:
+        candidate_branches = target_branch_candidates(args.project)

    # change to project dir for checkout
    os.chdir(args.project_relative_path)

+    ref_to_use = ""
    for candidate in candidate_branches:
-        try:
-            subprocess.check_call(["git", "checkout", "-f", candidate], stdout=subprocess.PIPE, stderr=subprocess.PIPE)  # not print the stdout nor stderr
-            print("CI using ref {} for project {}".format(candidate, args.project))
+        # check if candidate branch exists
+        branch_match = subprocess.check_output(["git", "branch", "-a", "--list", "origin/" + candidate])
+        if branch_match:
+            ref_to_use = candidate
            break
-        except subprocess.CalledProcessError:
-            pass
+
+    if ref_to_use:
+        for _ in range(RETRY_COUNT):
+            # Add retry for projects with git-lfs
+            try:
+                subprocess.check_call(["git", "checkout", "-f", ref_to_use], stdout=subprocess.PIPE)  # not print the stdout
+                print("CI using ref {} for project {}".format(ref_to_use, args.project))
+                break
+            except subprocess.CalledProcessError:
+                pass
+        else:
+            print("Failed to use ref {} for project {}".format(ref_to_use, args.project))
+            exit(1)
    else:
        print("using default branch")
--- a/tools/ci/config/assign-test.yml
+++ b/tools/ci/config/assign-test.yml
@ -15,6 +15,9 @@ assign_test:
    TEST_APP_CONFIG_OUTPUT_PATH: "$CI_PROJECT_DIR/tools/test_apps/test_configs"
    UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test"
    INTEGRATION_CONFIG_OUTPUT_PATH: "${CI_PROJECT_DIR}/components/idf_test/integration_test/CIConfigs"
+    INTEGRATION_TEST_CASE_PATH: "${CI_PROJECT_DIR}/auto_test_script/TestCaseFiles"
+    ASSIGN_TEST_CASE_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIAssignTestCases.py"
+    PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages
    PYTHON_VER: 3.7.7
  artifacts:
    paths:
@ -40,13 +43,12 @@ assign_test:
    # assign unit test cases
    - python tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py $UNIT_TEST_CASE_FILE $CI_TARGET_TEST_CONFIG_FILE $IDF_PATH/components/idf_test/unit_test/CIConfigs
    # clone test script to assign tests
+    # can not retry if downloading git lfs files failed, so using empty_branch first.
    - ./tools/ci/retry_failed.sh git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
    - ./tools/ci/retry_failed.sh git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
-    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
-    - export PYTHONPATH="${CI_PROJECT_DIR}/auto_test_script/packages:$PYTHONPATH"
-    - cd auto_test_script
+    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only
    # assign integration test cases
-    - python bin/CIAssignTestCases.py -t TestCaseFiles/ -c $CI_TARGET_TEST_CONFIG_FILE -b $IDF_PATH/SSC/ssc_bin -o $INTEGRATION_CONFIG_OUTPUT_PATH
+    - python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b $IDF_PATH/SSC/ssc_bin -o $INTEGRATION_CONFIG_OUTPUT_PATH

 update_test_cases:
  stage: assign_test
--- a/tools/ci/config/target-test.yml
+++ b/tools/ci/config/target-test.yml
@ -168,6 +168,7 @@
    MODULE_UPDATE_FILE: "$CI_PROJECT_DIR/components/idf_test/ModuleDefinition.yml"
    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/integration_test/CIConfigs"
    KNOWN_ISSUE_FILE: "${CI_PROJECT_DIR}/components/idf_test/integration_test/KnownIssues"
+    PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages
    PYTHON_VER: 3.7.7
  script:
    - *define_config_file_name
@ -179,7 +180,7 @@
    # clone auto test repo
    - ./tools/ci/retry_failed.sh git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
    - ./tools/ci/retry_failed.sh git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
-    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
+    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only
    - export PYTHONPATH="${CI_PROJECT_DIR}/auto_test_script/packages:$PYTHONPATH"
    - cd auto_test_script
    # run test
@ -614,10 +615,10 @@ nvs_compatible_test:
    - ./tools/ci/retry_failed.sh git clone $TEST_ENV_CONFIG_REPO
    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
    # clone test bench
+    # can not retry if downing git lfs files failed, so using empty_branch first.
    - ./tools/ci/retry_failed.sh git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
    - ./tools/ci/retry_failed.sh git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
-    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
-    - export PYTHONPATH="${CI_PROJECT_DIR}/auto_test_script/packages:$PYTHONPATH"
+    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only
    - cd auto_test_script
    # prepare nvs bins
    - ./tools/prepare_nvs_bin.sh