mirror of
https://github.com/espressif/esp-idf.git
synced 2024-10-05 20:47:46 -04:00
Merge branch 'feature/ci_relative_submodules' into 'master'
ci: Use relative submodules and a new cleaning working tree feature See merge request idf/esp-idf!4877
This commit is contained in:
Angus Gratton
commit
2b521bba1c
100
.gitlab-ci.yml
100
.gitlab-ci.yml
@ -19,10 +19,12 @@ variables:
|
||||
GET_SOURCES_ATTEMPTS: "10"
|
||||
ARTIFACT_DOWNLOAD_ATTEMPTS: "10"
|
||||
|
||||
# We use get-full-sources.sh script to fetch the submodules and/or re-fetch the repo
|
||||
# if it was corrupted (if submodule update fails this can happen)
|
||||
GIT_STRATEGY: fetch
|
||||
GIT_SUBMODULE_STRATEGY: none
|
||||
# GIT_STRATEGY is not defined here.
|
||||
# Use an option from "CI / CD Settings" - "General pipelines".
|
||||
|
||||
# "normal" strategy for fetching only top-level submodules since nothing requires the sub-submodules code for building IDF.
|
||||
# If the "recursive" strategy is used we have a problem with using relative URLs for sub-submodules.
|
||||
GIT_SUBMODULE_STRATEGY: normal
|
||||
|
||||
UNIT_TEST_BUILD_SYSTEM: make
|
||||
# IDF environment
|
||||
@ -37,19 +39,6 @@ variables:
|
||||
# Docker images
|
||||
BOT_DOCKER_IMAGE_TAG: ":latest"
|
||||
|
||||
# When 'fetch' strategy is used, Gitlab removes untracked files before checking out
|
||||
# new revision. However if the new revision doesn't include some of the submodules
|
||||
# which were present in the old revision, such submodule directories would not be
|
||||
# removed by the checkout. This extra step ensures that these stale submodules
|
||||
# are removed.
|
||||
.git_clean_stale_submodules: &git_clean_stale_submodules >
|
||||
find . -name '.git' -not -path './.git' -printf '%P\n'
|
||||
| sed 's|/.git||'
|
||||
| xargs -I {} sh -c '
|
||||
grep -q "path = {}" .gitmodules
|
||||
|| (echo "Removing {}, has .git directory but not in .gitmodules file"
|
||||
&& rm -rf {});'
|
||||
|
||||
# before each job, we need to check if this job is filtered by bot stage/job filter
|
||||
.apply_bot_filter: &apply_bot_filter
|
||||
python $APPLY_BOT_FILTER_SCRIPT || exit 0
|
||||
@ -78,9 +67,13 @@ variables:
|
||||
tools/idf_tools.py --non-interactive install && eval "$(tools/idf_tools.py --non-interactive export)" || exit 1
|
||||
fi
|
||||
|
||||
.show_submodule_urls: &show_submodule_urls |
|
||||
git config --get-regexp '^submodule\..*\.url$' || true
|
||||
|
||||
before_script:
|
||||
- echo "Running common script"
|
||||
- *show_submodule_urls
|
||||
- source tools/ci/setup_python.sh
|
||||
- *git_clean_stale_submodules
|
||||
# apply bot filter in before script
|
||||
- *apply_bot_filter
|
||||
# add gitlab ssh key
|
||||
@ -90,53 +83,34 @@ before_script:
|
||||
- base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
|
||||
- chmod 600 ~/.ssh/id_rsa
|
||||
- echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
|
||||
|
||||
# Set IS_PRIVATE or IS_PUBLIC depending on if our branch is public or not
|
||||
#
|
||||
# (the same regular expressions are used to set these are used in 'only:' sections below
|
||||
# Set some options and environment for CI
|
||||
- source tools/ci/configure_ci_environment.sh
|
||||
|
||||
- *setup_tools_unless_target_test
|
||||
|
||||
- *setup_custom_toolchain
|
||||
|
||||
# fetch the submodules (& if necessary re-fetch repo) from gitlab
|
||||
- time ./tools/ci/get-full-sources.sh
|
||||
|
||||
# used for check scripts which we want to run unconditionally
|
||||
.do_nothing_before_no_filter:
|
||||
before_script: &do_nothing_before_no_filter
|
||||
.before_script_lesser_nofilter: &before_script_lesser_nofilter
|
||||
variables:
|
||||
GIT_SUBMODULE_STRATEGY: none
|
||||
before_script:
|
||||
- echo "Not setting up GitLab key, not fetching submodules, not applying bot filter"
|
||||
- source tools/ci/setup_python.sh
|
||||
- *git_clean_stale_submodules
|
||||
- source tools/ci/configure_ci_environment.sh
|
||||
- *setup_custom_toolchain
|
||||
|
||||
# used for everything else where we want to do no prep, except for bot filter
|
||||
.do_nothing_before:
|
||||
before_script: &do_nothing_before
|
||||
- source tools/ci/setup_python.sh
|
||||
- *git_clean_stale_submodules
|
||||
# apply bot filter in before script
|
||||
- *apply_bot_filter
|
||||
.before_script_lesser: &before_script_lesser
|
||||
variables:
|
||||
GIT_SUBMODULE_STRATEGY: none
|
||||
before_script:
|
||||
- echo "Not setting up GitLab key, not fetching submodules"
|
||||
- source tools/ci/configure_ci_environment.sh
|
||||
- *setup_custom_toolchain
|
||||
|
||||
.add_gitlab_key_before:
|
||||
before_script: &add_gitlab_key_before
|
||||
- source tools/ci/setup_python.sh
|
||||
- *git_clean_stale_submodules
|
||||
# apply bot filter in before script
|
||||
- *apply_bot_filter
|
||||
- echo "Not fetching submodules"
|
||||
- source tools/ci/configure_ci_environment.sh
|
||||
- *setup_custom_toolchain
|
||||
# add gitlab ssh key
|
||||
- mkdir -p ~/.ssh
|
||||
- chmod 700 ~/.ssh
|
||||
- echo -n $GITLAB_KEY > ~/.ssh/id_rsa_base64
|
||||
- base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
|
||||
- chmod 600 ~/.ssh/id_rsa
|
||||
- echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
|
||||
|
||||
after_script:
|
||||
- *cleanup_custom_toolchain
|
||||
@ -423,7 +397,7 @@ build_docs:
|
||||
tags:
|
||||
- host_test
|
||||
dependencies: []
|
||||
before_script: *do_nothing_before_no_filter
|
||||
<<: *before_script_lesser_nofilter
|
||||
|
||||
verify_cmake_style:
|
||||
<<: *check_job_template
|
||||
@ -774,7 +748,7 @@ push_to_github:
|
||||
- /^v\d+\.\d+(\.\d+)?($|-)/
|
||||
when: on_success
|
||||
dependencies: []
|
||||
before_script: *do_nothing_before
|
||||
<<: *before_script_lesser
|
||||
script:
|
||||
- mkdir -p ~/.ssh
|
||||
- chmod 700 ~/.ssh
|
||||
@ -802,7 +776,7 @@ deploy_docs:
|
||||
- $BOT_LABEL_BUILD_DOCS
|
||||
dependencies:
|
||||
- build_docs
|
||||
before_script: *do_nothing_before
|
||||
<<: *before_script_lesser
|
||||
script:
|
||||
- mkdir -p ~/.ssh
|
||||
- chmod 700 ~/.ssh
|
||||
@ -969,7 +943,7 @@ check_examples_cmake_make:
|
||||
- master
|
||||
- /^release\/v/
|
||||
- /^v\d+\.\d+(\.\d+)?($|-)/
|
||||
before_script: *do_nothing_before
|
||||
<<: *before_script_lesser
|
||||
script:
|
||||
- tools/ci/check_examples_cmake_make.sh
|
||||
|
||||
@ -980,14 +954,13 @@ check_python_style:
|
||||
paths:
|
||||
- flake8_output.txt
|
||||
expire_in: 1 week
|
||||
before_script: *do_nothing_before
|
||||
<<: *before_script_lesser
|
||||
script:
|
||||
# run it only under Python 3 (it is very slow under Python 2)
|
||||
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 python -m flake8 --config=$IDF_PATH/.flake8 --output-file=flake8_output.txt --tee --benchmark $IDF_PATH
|
||||
|
||||
check_kconfigs:
|
||||
<<: *check_job_template
|
||||
before_script: *do_nothing_before
|
||||
artifacts:
|
||||
when: on_failure
|
||||
paths:
|
||||
@ -998,6 +971,7 @@ check_kconfigs:
|
||||
- tools/*/*/Kconfig*.new
|
||||
- tools/*/*/*/Kconfig*.new
|
||||
expire_in: 1 week
|
||||
<<: *before_script_lesser
|
||||
script:
|
||||
- ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ${IDF_PATH}/tools/test_check_kconfigs.py
|
||||
- ${IDF_PATH}/tools/check_kconfigs.py
|
||||
@ -1012,7 +986,7 @@ check_ut_cmake_make:
|
||||
- /^release\/v/
|
||||
- /^v\d+\.\d+(\.\d+)?($|-)/
|
||||
dependencies: []
|
||||
before_script: *do_nothing_before
|
||||
<<: *before_script_lesser
|
||||
script:
|
||||
- tools/ci/check_ut_cmake_make.sh
|
||||
|
||||
@ -1020,12 +994,22 @@ check_submodule_sync:
|
||||
<<: *check_job_template
|
||||
tags:
|
||||
- github_sync
|
||||
retry: 2
|
||||
variables:
|
||||
GIT_STRATEGY: clone
|
||||
retry: 2
|
||||
GIT_SUBMODULE_STRATEGY: none
|
||||
PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git"
|
||||
before_script: []
|
||||
after_script: []
|
||||
script:
|
||||
- git submodule deinit --force .
|
||||
# setting the default remote URL to the public one, to resolve relative location URLs
|
||||
- git config remote.origin.url ${PUBLIC_IDF_URL}
|
||||
# check if all submodules are correctly synced to public repostory
|
||||
- git submodule update --init --recursive
|
||||
- git submodule init
|
||||
- *show_submodule_urls
|
||||
- git submodule update --recursive
|
||||
- echo "IDF was cloned from ${PUBLIC_IDF_URL} completely"
|
||||
|
||||
check_artifacts_expire_time:
|
||||
<<: *check_job_template
|
||||
@ -1222,12 +1206,12 @@ test_weekend_network:
|
||||
- $LOG_PATH
|
||||
expire_in: 1 week
|
||||
variables:
|
||||
GIT_SUBMODULE_STRATEGY: none
|
||||
LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF"
|
||||
LOG_PATH: "$CI_PROJECT_DIR/$CI_COMMIT_SHA"
|
||||
TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test/integration_test"
|
||||
MODULE_UPDATE_FILE: "$CI_PROJECT_DIR/components/idf_test/ModuleDefinition.yml"
|
||||
CONFIG_FILE: "$CI_PROJECT_DIR/components/idf_test/integration_test/CIConfigs/$CI_JOB_NAME.yml"
|
||||
before_script: *add_gitlab_key_before
|
||||
script:
|
||||
# first test if config file exists, if not exist, exit 0
|
||||
- test -e $CONFIG_FILE || exit 0
|
||||
|
||||
40
.gitmodules
vendored
40
.gitmodules
vendored
@ -1,71 +1,75 @@
|
||||
#
|
||||
# All the relative URL paths are intended to be GitHub ones
|
||||
#
|
||||
|
||||
[submodule "components/esptool_py/esptool"]
|
||||
path = components/esptool_py/esptool
|
||||
url = https://github.com/espressif/esptool.git
|
||||
url = ../esptool.git
|
||||
|
||||
[submodule "components/bt/lib"]
|
||||
path = components/bt/lib
|
||||
url = https://github.com/espressif/esp32-bt-lib.git
|
||||
url = ../esp32-bt-lib.git
|
||||
|
||||
[submodule "components/bootloader/subproject/components/micro-ecc/micro-ecc"]
|
||||
path = components/bootloader/subproject/components/micro-ecc/micro-ecc
|
||||
url = https://github.com/kmackay/micro-ecc.git
|
||||
url = ../../kmackay/micro-ecc.git
|
||||
|
||||
[submodule "components/coap/libcoap"]
|
||||
path = components/coap/libcoap
|
||||
url = https://github.com/obgm/libcoap.git
|
||||
url = ../../obgm/libcoap.git
|
||||
|
||||
[submodule "components/nghttp/nghttp2"]
|
||||
path = components/nghttp/nghttp2
|
||||
url = https://github.com/nghttp2/nghttp2.git
|
||||
url = ../../nghttp2/nghttp2.git
|
||||
|
||||
[submodule "components/libsodium/libsodium"]
|
||||
path = components/libsodium/libsodium
|
||||
url = https://github.com/jedisct1/libsodium.git
|
||||
url = ../../jedisct1/libsodium.git
|
||||
|
||||
[submodule "components/spiffs/spiffs"]
|
||||
path = components/spiffs/spiffs
|
||||
url = https://github.com/pellepl/spiffs.git
|
||||
url = ../../pellepl/spiffs.git
|
||||
|
||||
[submodule "components/json/cJSON"]
|
||||
path = components/json/cJSON
|
||||
url = https://github.com/DaveGamble/cJSON.git
|
||||
url = ../../DaveGamble/cJSON.git
|
||||
|
||||
[submodule "components/mbedtls/mbedtls"]
|
||||
path = components/mbedtls/mbedtls
|
||||
url = https://github.com/espressif/mbedtls.git
|
||||
url = ../mbedtls.git
|
||||
|
||||
[submodule "components/asio/asio"]
|
||||
path = components/asio/asio
|
||||
url = https://github.com/espressif/asio.git
|
||||
url = ../asio.git
|
||||
|
||||
[submodule "components/expat/expat"]
|
||||
path = components/expat/expat
|
||||
url = https://github.com/libexpat/libexpat.git
|
||||
url = ../../libexpat/libexpat.git
|
||||
|
||||
[submodule "components/lwip/lwip"]
|
||||
path = components/lwip/lwip
|
||||
url = https://github.com/espressif/esp-lwip.git
|
||||
url = ../esp-lwip.git
|
||||
|
||||
[submodule "components/mqtt/esp-mqtt"]
|
||||
path = components/mqtt/esp-mqtt
|
||||
url = https://github.com/espressif/esp-mqtt.git
|
||||
url = ../esp-mqtt.git
|
||||
|
||||
[submodule "components/protobuf-c/protobuf-c"]
|
||||
path = components/protobuf-c/protobuf-c
|
||||
url = https://github.com/protobuf-c/protobuf-c
|
||||
url = ../../protobuf-c/protobuf-c
|
||||
|
||||
[submodule "components/unity/unity"]
|
||||
path = components/unity/unity
|
||||
url = https://github.com/ThrowTheSwitch/Unity
|
||||
url = ../../ThrowTheSwitch/Unity
|
||||
|
||||
[submodule "examples/build_system/cmake/import_lib/main/lib/tinyxml2"]
|
||||
path = examples/build_system/cmake/import_lib/main/lib/tinyxml2
|
||||
url = https://github.com/leethomason/tinyxml2
|
||||
url = ../../leethomason/tinyxml2
|
||||
|
||||
[submodule "components/esp_wifi/lib_esp32"]
|
||||
path = components/esp_wifi/lib_esp32
|
||||
url = https://github.com/espressif/esp32-wifi-lib.git
|
||||
url = ../esp32-wifi-lib.git
|
||||
|
||||
[submodule "components/nimble/nimble"]
|
||||
path = components/nimble/nimble
|
||||
url = https://github.com/espressif/esp-nimble.git
|
||||
url = ../esp-nimble.git
|
||||
|
||||
@ -7,30 +7,12 @@
|
||||
set -o errexit # Exit if command failed.
|
||||
set -o pipefail # Exit if pipe failed.
|
||||
|
||||
# we can use the appropriate secret variable for debugging
|
||||
[ ! -z $DEBUG_SHELL ] && set -x
|
||||
# We can use the appropriate CI variable for debugging
|
||||
DEBUG_SHELL=${DEBUG_SHELL:-"0"}
|
||||
[ "${DEBUG_SHELL}" = "1" ] && set -x
|
||||
|
||||
[ -z $CI_COMMIT_REF_NAME ] && echo "This internal script should only be run by a Gitlab CI runner." && exit 1
|
||||
|
||||
# Sets IS_PUBLIC and IS_PRIVATE based on branch type
|
||||
#
|
||||
# Public branches are:
|
||||
# release branches - start with release/
|
||||
# release tags - look like vXX.YY or vXX.YY.ZZ with an optional dash followed by anything on the end
|
||||
# master branch
|
||||
#
|
||||
# These POSIX REs are equivalent to the REs in some "only:" sections of the gitlab-ci.yml file
|
||||
#
|
||||
REF=$CI_COMMIT_REF_NAME
|
||||
if [[ $REF = "master" || $REF =~ ^release/v || $REF =~ ^v[0-9]+\.[0-9]+(\.[0-9]+)?(-|$) ]]; then
|
||||
export IS_PRIVATE=
|
||||
export IS_PUBLIC=1
|
||||
else
|
||||
export IS_PRIVATE=1
|
||||
export IS_PUBLIC=
|
||||
fi
|
||||
unset REF
|
||||
|
||||
# Compiler flags to thoroughly check the IDF code in some CI jobs
|
||||
# (Depends on default options '-Wno-error=XXX' used in the IDF build system)
|
||||
export PEDANTIC_CFLAGS="-DIDF_CI_BUILD -Werror -Werror=deprecated-declarations -Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function"
|
||||
|
||||
@ -38,9 +38,7 @@ tools/ci/check_idf_version.sh
|
||||
tools/ci/check_ut_cmake_make.sh
|
||||
tools/ci/checkout_project_ref.py
|
||||
tools/ci/envsubst.py
|
||||
tools/ci/get-full-sources.sh
|
||||
tools/ci/mirror-submodule-update.sh
|
||||
tools/ci/mirror-synchronize.sh
|
||||
tools/ci/multirun_with_pyenv.sh
|
||||
tools/ci/push_to_github.sh
|
||||
tools/ci/test_build_system.sh
|
||||
|
||||
@ -1,103 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Short script that is run as part of the CI environment
|
||||
# in .gitlab-ci.yml
|
||||
#
|
||||
# Sets up submodules in the ESP-IDF source tree
|
||||
# - Ideally, this just means doing a "git submodule update"
|
||||
# - But if something goes wrong we re-clone the repo from scratch
|
||||
#
|
||||
# This is a "best of both worlds" for GIT_STRATEGY: fetch & GIT_STRATEGY: clone
|
||||
#
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Common bash
|
||||
if [[ ! -z ${DEBUG_SHELL} ]]; then
|
||||
set -x # Activate the expand mode if DEBUG is anything but empty.
|
||||
fi
|
||||
|
||||
set -o errexit # Exit if command failed.
|
||||
set -o pipefail # Exit if pipe failed.
|
||||
set -o nounset # Exit if variable not set.
|
||||
|
||||
die() {
|
||||
echo "${1:-"Unknown Error"}" 1>&2
|
||||
exit ${2:-1}
|
||||
}
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
[ -z ${CI_PROJECT_DIR} ] && die "This internal script should only be run by a Gitlab CI runner."
|
||||
[ -z ${GITLAB_SSH_SERVER} ] && die "GITLAB_SSH_SERVER should be defined to run mirror-submodule-update.sh"
|
||||
[ -z ${CI_REPOSITORY_URL} ] && die "CI_REPOSITORY_URL should be defined to run mirror-submodule-update.sh"
|
||||
[ -z ${CI_COMMIT_SHA} ] && die "CI_COMMIT_SHA should be defined to run mirror-submodule-update.sh"
|
||||
DONT_USE_MIRROR=${DONT_USE_MIRROR:-"0"}
|
||||
|
||||
ERR_CANNOT_UPDATE=13
|
||||
|
||||
SCRIPT_DIR=$(dirname -- "${0}")
|
||||
update_submodules() {
|
||||
if [ "${DONT_USE_MIRROR}" = "1" ]; then
|
||||
git submodule update --init --recursive
|
||||
else
|
||||
${SCRIPT_DIR}/mirror-submodule-update.sh || return $?
|
||||
fi
|
||||
}
|
||||
|
||||
del_files() {
|
||||
DELETED_FILES=$(mktemp --tmpdir -d tmp_XXXX)
|
||||
# if non-empty
|
||||
[ "$(ls -A .)" ] && ( shopt -s dotglob; mv * "${DELETED_FILES}/" )
|
||||
trap 'del_files_rollback' ERR
|
||||
}
|
||||
del_files_confirm() {
|
||||
[ -d "${DELETED_FILES}" ] && rm -rf "${DELETED_FILES}"
|
||||
trap ERR
|
||||
}
|
||||
del_files_rollback() {
|
||||
[ "$(ls -A .)" ] && [ "$(ls -A ${DELETED_FILES}/)" ] && ( shopt -s dotglob; rm -rf * )
|
||||
[ "$(ls -A ${DELETED_FILES}/)" ] && ( shopt -s dotglob; mv "${DELETED_FILES}/"* . )
|
||||
[ -d "${DELETED_FILES}" ] && rmdir "${DELETED_FILES}"
|
||||
trap ERR
|
||||
}
|
||||
|
||||
RETRIES=10
|
||||
# we're in gitlab-ci's build phase, so GET_SOURCES_ATTEMPTS doesn't apply here...
|
||||
|
||||
# For the first time, we try the fastest way.
|
||||
for try in `seq $RETRIES`; do
|
||||
echo "Trying to add submodules to existing repo..."
|
||||
update_submodules &&
|
||||
echo "Fetch strategy submodules succeeded" &&
|
||||
exit 0
|
||||
|
||||
git submodule foreach --recursive "git reset --hard HEAD && git submodule deinit --force -- . || true"
|
||||
git reset --hard HEAD && git submodule deinit --force -- . || true
|
||||
done
|
||||
|
||||
# Then we use the clean way.
|
||||
for try in `seq $RETRIES`; do
|
||||
cd ${CI_PROJECT_DIR} # we are probably already here but pays to be certain
|
||||
echo "Trying a clean clone of IDF..."
|
||||
del_files
|
||||
git clone ${CI_REPOSITORY_URL} . &&
|
||||
git checkout ${CI_COMMIT_SHA} &&
|
||||
update_submodules &&
|
||||
echo "Clone strategy succeeded" &&
|
||||
del_files_confirm &&
|
||||
exit 0
|
||||
ERR_RES=$?
|
||||
del_files_rollback
|
||||
echo "Clean clone failed..."
|
||||
if [ $ERR_RES -eq $ERR_CANNOT_UPDATE ]; then
|
||||
echo "###"
|
||||
echo "### If you have updated one of the submodules,"
|
||||
echo "### you have to synchronize the local mirrors manually"
|
||||
echo "###"
|
||||
echo "### https://gitlab.espressif.cn:6688/idf/esp-idf/wikis/ci-use-guide#submodule-mirroring-for-private-branches"
|
||||
echo "###"
|
||||
|
||||
die "Failed to clone repo & submodules together" $ERR_RES
|
||||
fi
|
||||
done
|
||||
|
||||
die "Failed to clone repo & submodules together"
|
||||
@ -1,20 +0,0 @@
|
||||
components/esp_wifi/lib_esp32 @GENERAL_MIRROR_SERVER@/idf/esp32-wifi-lib.git
|
||||
components/bt/lib @GENERAL_MIRROR_SERVER@/idf/esp32-bt-lib.git
|
||||
components/esptool_py/esptool @GENERAL_MIRROR_SERVER@/idf/esptool.git
|
||||
components/coap/libcoap @GENERAL_MIRROR_SERVER@/idf/libcoap.git ALLOW_TO_SYNC_FROM_PUBLIC
|
||||
ext/tinydtls @GENERAL_MIRROR_SERVER@/idf/org.eclipse.tinydtls.git ALLOW_TO_SYNC_FROM_PUBLIC
|
||||
components/json/cJSON @GENERAL_MIRROR_SERVER@/idf/cJSON.git ALLOW_TO_SYNC_FROM_PUBLIC
|
||||
components/libsodium/libsodium @GENERAL_MIRROR_SERVER@/idf/libsodium.git ALLOW_TO_SYNC_FROM_PUBLIC
|
||||
components/mbedtls/mbedtls @GENERAL_MIRROR_SERVER@/idf/mbedtls.git ALLOW_TO_SYNC_FROM_PUBLIC
|
||||
components/expat/expat @GENERAL_MIRROR_SERVER@/idf/libexpat.git ALLOW_TO_SYNC_FROM_PUBLIC
|
||||
components/bootloader/subproject/components/micro-ecc/micro-ecc @GENERAL_MIRROR_SERVER@/idf/micro-ecc.git ALLOW_TO_SYNC_FROM_PUBLIC
|
||||
components/nghttp/nghttp2 @GENERAL_MIRROR_SERVER@/idf/nghttp2.git ALLOW_TO_SYNC_FROM_PUBLIC
|
||||
components/spiffs/spiffs @GENERAL_MIRROR_SERVER@/idf/spiffs.git ALLOW_TO_SYNC_FROM_PUBLIC
|
||||
components/asio/asio @GENERAL_MIRROR_SERVER@/idf/asio.git
|
||||
components/lwip/lwip @GENERAL_MIRROR_SERVER@/idf/esp-lwip.git
|
||||
third-party/mruby @GENERAL_MIRROR_SERVER@/idf/mruby.git ALLOW_TO_SYNC_FROM_PUBLIC
|
||||
third-party/neverbleed @GENERAL_MIRROR_SERVER@/idf/neverbleed.git ALLOW_TO_SYNC_FROM_PUBLIC
|
||||
components/mqtt/esp-mqtt @GENERAL_MIRROR_SERVER@/idf/esp-mqtt.git ALLOW_TO_SYNC_FROM_PUBLIC
|
||||
components/protobuf-c/protobuf-c @GENERAL_MIRROR_SERVER@/idf/protobuf-c.git ALLOW_TO_SYNC_FROM_PUBLIC
|
||||
components/unity/unity @GENERAL_MIRROR_SERVER@/idf/Unity.git ALLOW_TO_SYNC_FROM_PUBLIC
|
||||
examples/build_system/cmake/import_lib/main/lib/tinyxml2 @GENERAL_MIRROR_SERVER@/idf/tinyxml2.git ALLOW_TO_SYNC_FROM_PUBLIC
|
||||
@ -1,84 +1,5 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Redirects git submodules to the specified local mirrors and updates these recursively.
|
||||
#
|
||||
# To revert the changed URLs use 'git submodule deinit .'
|
||||
#
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Common bash
|
||||
# Now git submodules are redirected to mirror automatically according to relative URLs in .gitmodules
|
||||
|
||||
if [[ ! -z ${DEBUG_SHELL} ]]
|
||||
then
|
||||
set -x # Activate the expand mode if DEBUG is anything but empty.
|
||||
fi
|
||||
|
||||
set -o errexit # Exit if command failed.
|
||||
set -o pipefail # Exit if pipe failed.
|
||||
set -o nounset # Exit if variable not set.
|
||||
|
||||
die() {
|
||||
echo "${1:-"Unknown Error"}" 1>&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
[ -z ${GITLAB_SSH_SERVER:-} ] && die "Have to set up GITLAB_SSH_SERVER environment variable"
|
||||
|
||||
ERR_CANNOT_UPDATE=13
|
||||
|
||||
REPO_DIR=${1:-"${PWD}"}
|
||||
REPO_DIR=$(readlink -f -- "${REPO_DIR}")
|
||||
|
||||
SCRIPT_DIR=$(dirname -- "${0}")
|
||||
SCRIPT_DIR=$(readlink -f -- "${SCRIPT_DIR}")
|
||||
|
||||
SCRIPT_SH=$(readlink -f -- "${0}")
|
||||
|
||||
MIRRORLIST=${SCRIPT_DIR}/mirror-list.txt
|
||||
|
||||
[ -d "${REPO_DIR}" ] || die "${REPO_DIR} is not directory!"
|
||||
[ -f "${SCRIPT_SH}" ] || die "${SCRIPT_SH} does not exist!"
|
||||
[ -f "${MIRRORLIST}" ] || die "${MIRRORLIST} does not exist!"
|
||||
|
||||
pushd ${REPO_DIR} >/dev/null
|
||||
|
||||
# 0
|
||||
[ -f ".gitmodules" ] || exit 0
|
||||
|
||||
# 1
|
||||
git submodule init
|
||||
|
||||
|
||||
# 2
|
||||
# Replacing each submodule URL of the current repository
|
||||
# according to the one found in the MIRRORLIST
|
||||
#
|
||||
# Selecting paths among lines:
|
||||
# ...
|
||||
#submodule.components/esp32/lib.path
|
||||
#submodule.components/esp32/lib.url
|
||||
#submodule.components/esptool_py/esptool.path
|
||||
#submodule.components/esptool_py/esptool.url
|
||||
#...
|
||||
for SUBPATH in $(git config -f .gitmodules --list --name-only | grep "\.path" | sed 's/^submodule\.\([^ ]*\)\.path$/\1/')
|
||||
do
|
||||
SUBMIRROR=$(join -o"2.2" <(echo ${SUBPATH}) <(sort ${MIRRORLIST}))
|
||||
[ ${SUBMIRROR} ] || continue
|
||||
SUBMIRROR=${SUBMIRROR//@GENERAL_MIRROR_SERVER@/${GITLAB_SSH_SERVER}}
|
||||
echo -e "[switch mirror] $SUBPATH \tto\t $SUBMIRROR"
|
||||
|
||||
git config submodule.${SUBPATH}.url ${SUBMIRROR}
|
||||
done
|
||||
|
||||
# 3
|
||||
# Getting submodules of the current repository from the local mirrors
|
||||
git submodule update || exit $ERR_CANNOT_UPDATE
|
||||
|
||||
# 4
|
||||
# Replacing URLs for each sub-submodule.
|
||||
# The script runs recursively
|
||||
git submodule foreach "${SCRIPT_SH}" # No '--recursive'
|
||||
|
||||
popd >/dev/null
|
||||
echo "WARNING: Deprecated. This script is no longer required."
|
||||
|
||||
@ -1,87 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Script for automate mirroring
|
||||
# You can run this script manually
|
||||
#
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Common bash
|
||||
|
||||
if [[ ! -z ${DEBUG_SHELL} ]]
|
||||
then
|
||||
set -x # Activate the expand mode if DEBUG is anything but empty.
|
||||
fi
|
||||
|
||||
set -o errexit # Exit if command failed.
|
||||
set -o pipefail # Exit if pipe failed.
|
||||
set -o nounset # Exit if variable not set.
|
||||
|
||||
die() {
|
||||
echo "${1:-"Unknown Error"}" 1>&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
[ -z ${GITLAB_SSH_SERVER:-} ] && die "Have to set up GITLAB_SSH_SERVER environment variable"
|
||||
|
||||
REPO_DIR=${1:-"${PWD}"}
|
||||
REPO_DIR=$(readlink -f -- "${REPO_DIR}")
|
||||
|
||||
SCRIPT_DIR=$(dirname -- "${0}")
|
||||
SCRIPT_DIR=$(readlink -f -- "${SCRIPT_DIR}")
|
||||
|
||||
SCRIPT_SH=$(readlink -f -- "${0}")
|
||||
|
||||
MIRRORLIST=${SCRIPT_DIR}/mirror-list.txt
|
||||
|
||||
[ -d "${REPO_DIR}" ] || die "${REPO_DIR} is not directory!"
|
||||
[ -f "${SCRIPT_SH}" ] || die "${SCRIPT_SH} does not exist!"
|
||||
[ -f "${MIRRORLIST}" ] || die "${MIRRORLIST} does not exist!"
|
||||
|
||||
pushd ${REPO_DIR} >/dev/null
|
||||
|
||||
# 0
|
||||
[ -f ".gitmodules" ] || exit 0
|
||||
|
||||
# 1
|
||||
# Recursion
|
||||
git submodule update --init
|
||||
git submodule foreach "${SCRIPT_SH}" # No '--recursive'
|
||||
|
||||
# 2
|
||||
git submodule deinit --force .
|
||||
git submodule update --init
|
||||
|
||||
|
||||
# 3
|
||||
# Mirroring from original URLs to ones listed in the MIRRORLIST
|
||||
|
||||
# SED parses the strings like:
|
||||
#
|
||||
#-b991c67c1d91574ef22336cc3a5944d1e63230c9 roms/ipxe
|
||||
#b991c67c1d91574ef22336cc3a5944d1e63230c9 roms/ipxe (v1.0.0-2388-gb991c67)
|
||||
#
|
||||
for SUBPATH in $(git submodule status | sed -E 's/.*[[:space:]](.*)([[:space:]].*|$)/\1/')
|
||||
do
|
||||
SUBURL=$(git config -f .gitmodules --get submodule.$SUBPATH.url)
|
||||
SUBMIRROR=$(join -o"2.2" <(echo ${SUBPATH}) <(sort ${MIRRORLIST}))
|
||||
[ ${SUBMIRROR} ] || continue
|
||||
SUBMIRROR=${SUBMIRROR//@GENERAL_MIRROR_SERVER@/${GITLAB_SSH_SERVER}}
|
||||
|
||||
ALLOW_TO_SYNC=$(join -o"2.3" <(echo ${SUBPATH}) <(sort ${MIRRORLIST}))
|
||||
if [ "${ALLOW_TO_SYNC}" != "ALLOW_TO_SYNC_FROM_PUBLIC" ]
|
||||
then
|
||||
echo "[should not to sync mirror] ${SUBMIRROR}"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo -e "[mirror sync] ${SUBURL} \tto\t ${SUBMIRROR}"
|
||||
|
||||
pushd ${SUBPATH} >/dev/null
|
||||
git fetch --prune
|
||||
git push --prune ${SUBMIRROR} +refs/remotes/origin/*:refs/heads/* +refs/tags/*:refs/tags/*
|
||||
popd >/dev/null
|
||||
done
|
||||
|
||||
popd >/dev/null
|
||||
@ -5,34 +5,7 @@
|
||||
#
|
||||
cd $(dirname $0) # make dir
|
||||
|
||||
touch .gitmodules # dummy file
|
||||
|
||||
# $1 - branch name
|
||||
# $2 - 1 if public, empty if private
|
||||
function assert_branch_public()
|
||||
{
|
||||
(
|
||||
CI_COMMIT_REF_NAME=$1
|
||||
set -e
|
||||
source ./configure_ci_environment.sh
|
||||
[[ $IS_PUBLIC = $2 ]] || exit 1
|
||||
) || { echo "Expected $1 public=$2. Failing"; exit 1; }
|
||||
}
|
||||
|
||||
assert_branch_public master 1
|
||||
assert_branch_public release/v3.0 1
|
||||
assert_branch_public release/invalid
|
||||
assert_branch_public bugfix/invalid
|
||||
assert_branch_public v1.0 1
|
||||
assert_branch_public v1.0.0 1
|
||||
assert_branch_public v50.50.50 1
|
||||
assert_branch_public v1.2-rc77 1
|
||||
assert_branch_public v1.2.3-rc1 1
|
||||
assert_branch_public v1.2.3invalid
|
||||
|
||||
(
|
||||
. ./configure_ci_environment.sh
|
||||
[[ $PEDANTIC_CFLAGS ]] || { echo "PEDANTIC_CFLAGS is not defined"; exit 1; }
|
||||
) || { exit 1; }
|
||||
|
||||
rm -f .gitmodules
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user