alex/esp-idf
alex/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf.git synced 2024-10-05 20:47:46 -04:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity
8ff8b520f3
esp-idf/.gitlab/ci/common.yml

391 lines
14 KiB
YAML
Raw Blame History

#####################
# Default Variables #
#####################
stages:
- upload_cache
- pre_check
- build
- assign_test
- target_test
- host_test
- build_doc
- test_deploy
- deploy
- post_deploy
- retry_failed_jobs
variables:
# System environment
# Common parameters for the 'make' during CI tests
MAKEFLAGS: "-j5 --no-keep-going"
# GitLab-CI environment
# Thanks to pack-objects cache, clone strategy should behave faster than fetch
# so we pick "clone" as default git strategy
# Shiny runners by default remove the CI_PROJECT_DIR every time at the beginning of one job
# and clone with a --depth=1
# Brew runners will fetch from locally mirror first, and cache the local CI_PROJECT_DIR
# In conclusion
# - set GIT_STRATEGY: "clone" to shiny runners
# - set GIT_STRATEGY: "fetch" to brew runners
GIT_STRATEGY: clone
# we will download archive for each submodule instead of clone.
# we don't do "recursive" when fetch submodule as they're not used in CI now.
GIT_SUBMODULE_STRATEGY: none
# since we're using merged-result pipelines, the last commit should work for most cases
GIT_DEPTH: 1
# --no-recurse-submodules: we use cache for submodules
# --prune --prune-tags: in case remote branch or tag is force pushed
GIT_FETCH_EXTRA_FLAGS: "--no-recurse-submodules --prune --prune-tags"
# we're using .cache folder for caches
GIT_CLEAN_FLAGS: -ffdx -e .cache/
LATEST_GIT_TAG: v5.4-dev
SUBMODULE_FETCH_TOOL: "tools/ci/ci_fetch_submodule.py"
# by default we will fetch all submodules
# jobs can overwrite this variable to only fetch submodules they required
# set to "none" if don't need to fetch submodules
SUBMODULES_TO_FETCH: "all"
# tell build system do not check submodule update as we download archive instead of clone
IDF_SKIP_CHECK_SUBMODULES: 1
IDF_PATH: "$CI_PROJECT_DIR"
V: "0"
CHECKOUT_REF_SCRIPT: "$CI_PROJECT_DIR/tools/ci/checkout_project_ref.py"
# Docker images
ESP_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/esp-env-v5.4:1"
ESP_IDF_DOC_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/esp-idf-doc-env-v5.4:1-1"
TARGET_TEST_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/target-test-env-v5.4:1"
SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:5"
PRE_COMMIT_IMAGE: "${CI_DOCKER_REGISTRY}/esp-idf-pre-commit:1"
# target test repo parameters
TEST_ENV_CONFIG_REPO: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/ci-test-runner-configs.git"
# cache python dependencies
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
# Set this variable to the branch of idf-constraints repo in order to test a custom Python constraint file. The
# branch name must be without the remote part ("origin/"). Keep the variable empty in order to use the constraint
# file from https://dl.espressif.com/dl/esp-idf.
CI_PYTHON_CONSTRAINT_BRANCH: ""
# Update the filename for a specific ESP-IDF release. It is used only with CI_PYTHON_CONSTRAINT_BRANCH.
CI_PYTHON_CONSTRAINT_FILE: "espidf.constraints.v5.4.txt"
# Set this variable to repository name of a Python tool you wish to install and test in the context of ESP-IDF CI.
# Keep the variable empty when not used.
CI_PYTHON_TOOL_REPO: ""
# Set this variable to the branch of a Python tool repo specified in CI_PYTHON_TOOL_REPO. The
# branch name must be without the remote part ("origin/"). Keep the variable empty when not used.
# This is used only if CI_PYTHON_TOOL_REPO is not empty.
CI_PYTHON_TOOL_BRANCH: ""
# Set this variable to Clang toolchain distro URL to be used.
# NOTE: We have separate toolchains for Xtensa and RISCV, therefore jobs for one arch will fail.
# This is OK as far as we use CI_CLANG_DISTRO_URL for pre-release tests purposes only.
# Keep the variable empty when not used.
CI_CLANG_DISTRO_URL: ""
# Set this variable to specify the file name for the known failure cases.
KNOWN_FAILURE_CASES_FILE_NAME: "master.txt"
IDF_CI_BUILD: 1
# ccache settings
# some settings need to set in .gitlab-ci.yml as it takes effect while start-up the job
# https://ccache.dev/manual/latest.html#_configuring_ccache
# host mapping volume to share ccache between runner concurrent jobs
CCACHE_DIR: "/cache/idf_ccache"
CCACHE_MAXSIZE: "50G"
FF_USE_NEW_BASH_EVAL_STRATEGY: "true"
################################################
# `before_script` and `after_script` Templates #
################################################
.common_before_scripts: &common-before_scripts |
source tools/ci/utils.sh
is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
export IDF_MIRROR_PREFIX_MAP=
fi
if echo "$CI_MERGE_REQUEST_LABELS" | egrep "(^|,)include_nightly_run(,|$)"; then
export INCLUDE_NIGHTLY_RUN="1"
export NIGHTLY_RUN="1"
fi
# configure cmake related flags
source tools/ci/configure_ci_environment.sh
# add extra python packages
export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/esp_app_trace:$IDF_PATH/components/partition_table:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
.setup_tools_and_idf_python_venv: &setup_tools_and_idf_python_venv |
# must use after setup_tools_except_target_test
# otherwise the export.sh won't work properly
# download constraint file for dev
if [[ -n "$CI_PYTHON_CONSTRAINT_BRANCH" ]]; then
wget -O /tmp/constraint.txt --header="Authorization:Bearer ${ESPCI_TOKEN}" ${GITLAB_HTTP_SERVER}/api/v4/projects/2581/repository/files/${CI_PYTHON_CONSTRAINT_FILE}/raw?ref=${CI_PYTHON_CONSTRAINT_BRANCH}
mkdir -p ~/.espressif
mv /tmp/constraint.txt ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
fi
# Mirror
if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
export IDF_MIRROR_PREFIX_MAP=
fi
# install latest python packages
# target test jobs
if [[ "${CI_JOB_STAGE}" == "target_test" ]]; then
run_cmd bash install.sh --enable-ci --enable-pytest --enable-test-specific
elif [[ "${CI_JOB_STAGE}" == "build_doc" ]]; then
run_cmd bash install.sh --enable-ci --enable-docs
elif [[ "${CI_JOB_STAGE}" == "build" ]]; then
run_cmd bash install.sh --enable-ci
else
if ! echo "${CI_JOB_NAME}" | egrep ".*pytest.*"; then
run_cmd bash install.sh --enable-ci
else
run_cmd bash install.sh --enable-ci --enable-pytest --enable-test-specific
fi
fi
# Install esp-clang if necessary
if [[ "$IDF_TOOLCHAIN" == "clang" && -z "$CI_CLANG_DISTRO_URL" ]]; then
$IDF_PATH/tools/idf_tools.py --non-interactive install esp-clang
fi
# Install QEMU if necessary
if [[ ! -z "$INSTALL_QEMU" ]]; then
$IDF_PATH/tools/idf_tools.py --non-interactive install qemu-xtensa qemu-riscv32
fi
# Since the version 3.21 CMake passes source files and include dirs to ninja using absolute paths.
# Needed for pytest junit reports.
$IDF_PATH/tools/idf_tools.py --non-interactive install cmake
source ./export.sh
# Custom clang toolchain
if [[ ! -z "$CI_CLANG_DISTRO_URL" ]]; then
echo "Using custom clang from ${CI_CLANG_DISTRO_URL}"
wget $CI_CLANG_DISTRO_URL
ARCH_NAME=$(basename $CI_CLANG_DISTRO_URL)
tar -x -f $ARCH_NAME
export PATH=$PWD/esp-clang/bin:$PATH
fi
# Custom OpenOCD
if [[ ! -z "$OOCD_DISTRO_URL" && "$CI_JOB_STAGE" == "target_test" ]]; then
echo "Using custom OpenOCD from ${OOCD_DISTRO_URL}"
wget $OOCD_DISTRO_URL
ARCH_NAME=$(basename $OOCD_DISTRO_URL)
tar -x -f $ARCH_NAME
export OPENOCD_SCRIPTS=$PWD/openocd-esp32/share/openocd/scripts
export PATH=$PWD/openocd-esp32/bin:$PATH
fi
if [[ -n "$CI_PYTHON_TOOL_REPO" ]]; then
git clone --quiet --depth=1 -b ${CI_PYTHON_TOOL_BRANCH} https://gitlab-ci-token:${ESPCI_TOKEN}@${GITLAB_HTTPS_HOST}/espressif/${CI_PYTHON_TOOL_REPO}.git
pip install ./${CI_PYTHON_TOOL_REPO}
rm -rf ${CI_PYTHON_TOOL_REPO}
fi
.show_ccache_statistics: &show_ccache_statistics |
# Show ccache statistics if enabled globally
test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats -vv || true
.upload_failed_job_log_artifacts: &upload_failed_job_log_artifacts |
if [ $CI_JOB_STATUS = "failed" ]; then
python tools/ci/artifacts_handler.py upload --type logs
fi
.before_script:minimal:
before_script:
- *common-before_scripts
.before_script:build:macos:
before_script:
# macos is running shell executor, which means it would use
# the system installed /usr/local/bin/python3 by default.
# Ensure pyenv and PYENV_VERSION installed
- eval "$(pyenv init -)"
- *common-before_scripts
# On macOS, these tools need to be installed
- export IDF_TOOLS_PATH="${HOME}/.espressif_runner_${CI_RUNNER_ID}_${CI_CONCURRENT_ID}"
# remove idf-env.json, since it may contains enabled "features"
- rm -f $IDF_TOOLS_PATH/idf-env.json
- $IDF_PATH/tools/idf_tools.py --non-interactive install cmake ninja
# This adds tools (compilers) and the version-specific Python environment to PATH
- *setup_tools_and_idf_python_venv
- fetch_submodules
.after_script:build:macos:upload-when-fail:
after_script:
# macos is running shell executor, which means it would use
# the system installed /usr/local/bin/python3 by default.
# Ensure pyenv and PYENV_VERSION installed
- eval "$(pyenv init -)"
- *upload_failed_job_log_artifacts
.before_script:build:
before_script:
- *common-before_scripts
- *setup_tools_and_idf_python_venv
- add_gitlab_ssh_keys
- fetch_submodules
- export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
- export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
.after_script:build:ccache:
after_script:
- *show_ccache_statistics
.after_script:build:ccache:upload-when-fail:
after_script:
- *show_ccache_statistics
- *upload_failed_job_log_artifacts
##############################
# Git Strategy Job Templates #
##############################
.git_init: &git_init |
mkdir -p "${CI_PROJECT_DIR}"
cd "${CI_PROJECT_DIR}"
git init
.git_fetch_from_mirror_url_if_exists: &git_fetch_from_mirror_url_if_exists |
# check if set mirror
if [ -n "${LOCAL_GITLAB_HTTPS_HOST:-}" ] && [ -n "${ESPCI_TOKEN:-}" ]; then
MIRROR_REPO_URL="https://bot:${ESPCI_TOKEN}@${LOCAL_GITLAB_HTTPS_HOST}/${CI_PROJECT_PATH}"
elif [ -n "${LOCAL_GIT_MIRROR:-}" ]; then
MIRROR_REPO_URL="${LOCAL_GIT_MIRROR}/${CI_PROJECT_PATH}"
fi
# fetch from mirror first if set
if [ -n "${MIRROR_REPO_URL:-}" ]; then
if git remote -v | grep origin; then
git remote set-url origin "${MIRROR_REPO_URL}"
else
git remote add origin "${MIRROR_REPO_URL}"
fi
# mirror url may fail with authentication issue
git fetch origin --no-recurse-submodules || true
fi
# set remote url to CI_REPOSITORY_URL
if git remote -v | grep origin; then
git remote set-url origin "${CI_REPOSITORY_URL}"
else
git remote add origin "${CI_REPOSITORY_URL}"
fi
.git_checkout_ci_commit_sha: &git_checkout_ci_commit_sha |
git checkout $CI_COMMIT_SHA
git clean ${GIT_CLEAN_FLAGS}
# git diff requires two commits, with different CI env var
#
# By default, we use git strategy "clone" with depth 1 to speed up the clone process.
# But for jobs requires running `git diff`, we need to fetch more commits to get the correct diffs.
#
# Since there's no way to get the correct git_depth before the job starts,
# we can't set `GIT_DEPTH` in the job definition.
#
# Set git strategy to "none" and fetch manually instead.
.before_script:fetch:git_diff:
variables:
GIT_STRATEGY: none
before_script:
- *git_init
- *git_fetch_from_mirror_url_if_exists
- |
# merged results pipelines, by default
if [[ -n $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA ]]; then
git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
git fetch origin $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
export GIT_DIFF_OUTPUT=$(git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA)
git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
# merge request pipelines, when the mr got conflicts
elif [[ -n $CI_MERGE_REQUEST_DIFF_BASE_SHA ]]; then
git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
export GIT_DIFF_OUTPUT=$(git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_COMMIT_SHA)
# other pipelines, like the protected branches pipelines
elif [[ "$CI_COMMIT_BEFORE_SHA" != "0000000000000000000000000000000000000000" ]]; then
git fetch origin $CI_COMMIT_BEFORE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
export GIT_DIFF_OUTPUT=$(git diff --name-only $CI_COMMIT_BEFORE_SHA $CI_COMMIT_SHA)
else
# pipeline source could be web, scheduler, etc.
git fetch origin $CI_COMMIT_SHA --depth=2 ${GIT_FETCH_EXTRA_FLAGS}
export GIT_DIFF_OUTPUT=$(git diff --name-only $CI_COMMIT_SHA~1 $CI_COMMIT_SHA)
fi
- *git_checkout_ci_commit_sha
- *common-before_scripts
- *setup_tools_and_idf_python_venv
- add_gitlab_ssh_keys
# target test runners may locate in different places
# for runners set git mirror, we fetch from the mirror first, then fetch the HEAD commit
.before_script:fetch:target_test:
variables:
GIT_STRATEGY: none
before_script:
- *git_init
- *git_fetch_from_mirror_url_if_exists
- git fetch origin "${CI_COMMIT_SHA}" --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
- *git_checkout_ci_commit_sha
- *common-before_scripts
- *setup_tools_and_idf_python_venv
- add_gitlab_ssh_keys
# no submodules
#############
# `default` #
#############
default:
cache:
# pull only for most of the use cases since it's cache dir.
# Only set "push" policy for "upload_cache" stage jobs
- key: pip-cache-${LATEST_GIT_TAG}
fallback_keys:
- pip-cache
paths:
- .cache/pip
policy: pull
- key: submodule-cache-${LATEST_GIT_TAG}
fallback_keys:
- submodule-cache
paths:
- .cache/submodule_archives
policy: pull
before_script:
- *common-before_scripts
- *setup_tools_and_idf_python_venv
- add_gitlab_ssh_keys
- fetch_submodules
# gitlab bug, setting them here doesn't work
# - expire_in: https://gitlab.com/gitlab-org/gitlab/-/issues/404563
# - when: https://gitlab.com/gitlab-org/gitlab/-/issues/440672
# artifacts:
# expire_in: 1 week
# when: always
retry:
max: 2
when:
# In case of a runner failure we could hop to another one, or a network error could go away.
- runner_system_failure
# Job execution timeout may be caused by a network issue.
- job_execution_timeout
Reference in New Issue View Git Blame Copy Permalink