From f52519d7a2670cbb9ce68477d7b07f718990fc03 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Wed, 14 Jun 2017 17:52:33 +1000 Subject: [PATCH] CI: Use custom get_full_sources.sh script instead of gitlab-ci's strategies Work around problems caused by failed submodule clones corrupting the working directory on the build runner. Add 10 retries for cloning submodules. --- .gitlab-ci.yml | 47 ++++++++++++++--------------- tools/ci/get-full-sources.sh | 57 ++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 23 deletions(-) create mode 100755 tools/ci/get-full-sources.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a24d8654ae..bde5db9e09 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,6 +18,11 @@ variables: GET_SOURCES_ATTEMPTS: "10" ARTIFACT_DOWNLOAD_ATTEMPTS: "10" + # We use get_sources.sh script to fetch the submodules and/or re-fetch the repo + # if it was corrupted (if submodule update fails this can happen) + GIT_STRATEGY: fetch + GIT_SUBMODULE_STRATEGY: none + # IDF environment IDF_PATH: "$CI_PROJECT_DIR" @@ -32,12 +37,13 @@ before_script: - echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config # Set IS_PRIVATE or IS_PUBLIC depending on if our branch is public or not + # and adjust "our" submodule URLs if IS_PRIVATE # # (the same regular expressions are used to set these are used in 'only:' sections below - source tools/ci/configure_ci_environment.sh - # fetch all submodules - - git submodule update --init --recursive + # fetch the submodules (& if necessary re-fetch repo) from gitlab + - time ./tools/ci/get-full-sources.sh build_template_app: stage: build @@ -45,7 +51,6 @@ build_template_app: tags: - build variables: - GIT_STRATEGY: clone BATCH_BUILD: "1" IDF_CI_BUILD: "1" script: @@ -73,7 +78,6 @@ build_template_app: tags: - build variables: - GIT_STRATEGY: clone BATCH_BUILD: "1" V: "0" @@ -128,7 +132,6 @@ build_esp_idf_tests: expire_in: 1 week variables: IDF_CI_BUILD: "1" - GIT_STRATEGY: fetch script: # it's not possible to build 100% out-of-tree and have the "artifacts" # mechanism work, but this is the next best thing @@ -279,7 +282,6 @@ push_master_to_github: when: on_success dependencies: [] variables: - GIT_STRATEGY: clone GITHUB_PUSH_REFS: refs/remotes/origin/release refs/remotes/origin/master script: - mkdir -p ~/.ssh @@ -355,7 +357,7 @@ check_commit_msg: - /^v\d+\.\d+(\.\d+)?($|-)/ dependencies: [] before_script: - - echo "skip update submodule" + - echo "Not setting up GitLab key, not fetching submodules" script: - git status - git log -n10 --oneline @@ -373,7 +375,7 @@ check_submodule_sync: - /^v\d+\.\d+(\.\d+)?($|-)/ dependencies: [] before_script: - - echo "do not use gitlab submodule repository" + - echo "Not setting up GitLab key, not fetching submodules" script: # check if all submodules are correctly synced to public repostory - git submodule update --init --recursive @@ -390,6 +392,15 @@ assign_test: - components/idf_test/*/CIConfigs - components/idf_test/*/TC.sqlite expire_in: 1 mos + before_script: + - echo "Not fetching submodules" + # add gitlab ssh key + - mkdir -p ~/.ssh + - chmod 700 ~/.ssh + - echo -n $GITLAB_KEY > ~/.ssh/id_rsa_base64 + - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa + - chmod 600 ~/.ssh/id_rsa + - echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config script: # first move test bins together: test_bins/CHIP_SDK/TestApp/bin_files - mkdir -p test_bins/ESP32_IDF/UT @@ -405,8 +416,6 @@ assign_test: - python CIAssignTestCases.py -t $IDF_PATH/components/idf_test/integration_test -c $IDF_PATH/.gitlab-ci.yml -b $IDF_PATH/test_bins .test_template: &test_template - before_script: - - echo "Skip cloning submodule here" stage: test when: on_success only: @@ -423,16 +432,13 @@ assign_test: - $LOG_PATH expire_in: 6 mos variables: - # set git strategy to fetch so we can get esptool without update submodule - GIT_STRATEGY: fetch LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF" LOG_PATH: "$CI_PROJECT_DIR/$CI_COMMIT_SHA" TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test/integration_test" MODULE_UPDATE_FILE: "$CI_PROJECT_DIR/components/idf_test/ModuleDefinition.yml" CONFIG_FILE: "$CI_PROJECT_DIR/components/idf_test/integration_test/CIConfigs/$CI_JOB_NAME.yml" - script: - # first test if config file exists, if not exist, exit 0 - - test -e $CONFIG_FILE || exit 0 + before_script: + - echo "Not fetching submodules" # add gitlab ssh key - mkdir -p ~/.ssh - chmod 700 ~/.ssh @@ -440,6 +446,9 @@ assign_test: - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config + script: + # first test if config file exists, if not exist, exit 0 + - test -e $CONFIG_FILE || exit 0 # clone local test env configs - git clone $TEST_ENV_CONFIG_REPOSITORY # clone test bench @@ -455,7 +464,6 @@ assign_test: allow_failure: false stage: unit_test variables: - GIT_STRATEGY: fetch LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF" LOG_PATH: "$CI_PROJECT_DIR/$CI_COMMIT_SHA" TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test/unit_test" @@ -474,13 +482,6 @@ nvs_compatible_test: - ESP32_IDF - NVS_Compatible script: - # add gitlab ssh key - - mkdir -p ~/.ssh - - chmod 700 ~/.ssh - - echo -n $GITLAB_KEY > ~/.ssh/id_rsa_base64 - - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa - - chmod 600 ~/.ssh/id_rsa - - echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config # clone local test env configs - git clone $TEST_ENV_CONFIG_REPOSITORY # clone test bench diff --git a/tools/ci/get-full-sources.sh b/tools/ci/get-full-sources.sh new file mode 100755 index 0000000000..35d794e970 --- /dev/null +++ b/tools/ci/get-full-sources.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# +# Short script that is run as part of the CI environment +# in .gitlab-ci.yml +# +# Sets up submodules in the ESP-IDF source tree +# - Ideally, this just means doing a "git submodule update" +# - But if something goes wrong we re-clone the repo from scratch +# +# This is a "best of both worlds" for GIT_STRATEGY: fetch & GIT_STRATEGY: clone +# + +die() { + echo "${1:-"Unknown Error"}" 1>&2 + exit 1 +} + +[ -z ${CI_PROJECT_DIR} ] && die "This internal script should only be run by a Gitlab CI runner." + +DELETED_FILES=$(mktemp --tmpdir -d tmp_XXXX) +del_files() { + # if non-empty + [ "$(ls -A .)" ] && ( shopt -s dotglob; mv * "${DELETED_FILES}/" ) +} +del_files_confirm() { + rm -rf "${DELETED_FILES}" +} + +RETRIES=10 +# we're in gitlab-ci's build phase, so GET_SOURCES_ATTEMPTS doesn't apply here... + +# For the first time, we try the fastest way. +for try in `seq $RETRIES`; do + echo "Trying to add submodules to existing repo..." + git submodule update --init --recursive && + echo "Fetch strategy submodules succeeded" && + exit 0 +done + +# Then we use the clean way. +for try in `seq $RETRIES`; do + cd ${CI_PROJECT_DIR} # we are probably already here but pays to be certain + echo "Trying a clean clone of IDF..." + del_files + git clone ${CI_REPOSITORY_URL} . && + git checkout ${CI_COMMIT_SHA} && + git submodule update --init --recursive && + echo "Clone strategy succeeded" && + del_files_confirm && + exit 0 + + echo "Clean clone failed..." +done + +echo "Failed to clone repo & submodules together" + +exit 1