esp-idf/tools/ci/config/build.yml

.build_template:
  stage: build
  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
  tags:
    - build
  variables:
    BATCH_BUILD: "1"
    V: "0"
    SIZE_INFO_LOCATION: "$CI_PROJECT_DIR/size_info.txt"

.build_ssc_template:
  extends: .build_template
  parallel: 3
  artifacts:
    paths:
      - SSC/ssc_bin
    expire_in: 1 week
  variables:
    TARGET_NAME: "ESP32"
  only:
    variables:
      - $BOT_TRIGGER_WITH_LABEL == null
      - $BOT_LABEL_BUILD
      - $BOT_LABEL_INTEGRATION_TEST
      - $BOT_LABEL_REGULAR_TEST
  script:
    - ./tools/ci/retry_failed.sh git clone $SSC_REPOSITORY
    - python $CHECKOUT_REF_SCRIPT SSC SSC
    - cd SSC
    - MAKEFLAGS= ./ci_build_ssc.sh $TARGET_NAME

build_ssc_esp32:
  extends: .build_ssc_template
  variables:
    TARGET_NAME: "ESP32"

build_ssc_esp32s2:
  extends: .build_ssc_template
  parallel: 2
  variables:
    TARGET_NAME: "ESP32S2"

.build_esp_idf_tests_cmake:
  extends: .build_template
  artifacts:
    paths:
      - tools/unit-test-app/output/${IDF_TARGET}
      - tools/unit-test-app/builds/*.json
      - tools/unit-test-app/builds/${IDF_TARGET}/*/size.json
      - components/idf_test/unit_test/*.yml
      - $LOG_PATH
      - $SIZE_INFO_LOCATION
    when: always
    expire_in: 4 days
  only:
    variables:
      - $BOT_TRIGGER_WITH_LABEL == null
      - $BOT_LABEL_BUILD
      - $BOT_LABEL_UNIT_TEST
      - $BOT_LABEL_UNIT_TEST_S2
      - $BOT_LABEL_REGULAR_TEST
  variables:
    LOG_PATH: "$CI_PROJECT_DIR/log_ut_cmake"
    BUILD_PATH: ${CI_PROJECT_DIR}/tools/unit-test-app/builds
    OUTPUT_PATH: ${CI_PROJECT_DIR}/tools/unit-test-app/output
    BUILD_SYSTEM: "cmake"
    TEST_TYPE: "unit_test"
  script:
    # RISC-V toolchain is optional but ULP may need it, so install:
    - $IDF_PATH/tools/idf_tools.py install riscv-none-embed-gcc
    - . $IDF_PATH/export.sh
    - ${IDF_PATH}/tools/ci/find_apps_build_apps.sh
    - cd $CI_PROJECT_DIR/tools/unit-test-app
    - python tools/UnitTestParser.py ${BUILD_PATH}

build_esp_idf_tests_cmake_esp32:
  extends: .build_esp_idf_tests_cmake
  variables:
    IDF_TARGET: esp32

build_esp_idf_tests_cmake_esp32s2:
  extends: .build_esp_idf_tests_cmake
  variables:
    IDF_TARGET: esp32s2

.build_examples_template:
  extends: .build_template
  artifacts:
    when: always
    expire_in: 4 days
  only:
    variables:
      - $BOT_TRIGGER_WITH_LABEL == null
      - $BOT_LABEL_BUILD
      - $BOT_LABEL_EXAMPLE_TEST
      - $BOT_LABEL_REGULAR_TEST
      - $BOT_LABEL_WEEKEND_TEST
  variables:
    TEST_PREFIX: examples
    TEST_RELATIVE_DIR: examples
    SCAN_TEST_JSON: ${CI_PROJECT_DIR}/${TEST_RELATIVE_DIR}/test_configs/scan_${IDF_TARGET}_${BUILD_SYSTEM}.json
    TEST_TYPE: example_test
    LOG_PATH: ${CI_PROJECT_DIR}/log_${TEST_PREFIX}
    BUILD_PATH: ${CI_PROJECT_DIR}/build_${TEST_PREFIX}
  script:
    # RISC-V toolchain is optional but ULP may need it, so install:
    - $IDF_PATH/tools/idf_tools.py install riscv-none-embed-gcc
    - . $IDF_PATH/export.sh
    # it's not possible to build 100% out-of-tree and have the "artifacts"
    # mechanism work, but this is the next best thing
    - ${IDF_PATH}/tools/ci/find_apps_build_apps.sh

build_examples_make:
  extends: .build_examples_template
  # This is a workaround for a rarely encountered issue with building examples in CI.
  # Probably related to building of Kconfig in 'make clean' stage
  retry: 1
  parallel: 8
  artifacts:
    paths:
      - $LOG_PATH
      - build_${TEST_PREFIX}/*/*/*/build/size.json
      - $SIZE_INFO_LOCATION
  variables:
    BUILD_SYSTEM: make
    IDF_TARGET: esp32  # currently we only support esp32
  only:
    refs:
      - master
      - /^release\/v/
      - /^v\d+\.\d+(\.\d+)?($|-)/
      - triggers
      - schedules
      - pipelines
      - web

# same as above, but for CMake
.build_examples_cmake: &build_examples_cmake
  extends: .build_examples_template
  dependencies:
    - scan_tests
  artifacts:
    paths:
      - build_${TEST_PREFIX}/list.json
      - build_${TEST_PREFIX}/list_job_*.json
      - build_${TEST_PREFIX}/*/*/*/sdkconfig
      - build_${TEST_PREFIX}/*/*/*/build/size.json
      - build_${TEST_PREFIX}/*/*/*/build/*.bin
      - build_${TEST_PREFIX}/*/*/*/build/*.elf
      - build_${TEST_PREFIX}/*/*/*/build/*.map
      - build_${TEST_PREFIX}/*/*/*/build/flasher_args.json
      - build_${TEST_PREFIX}/*/*/*/build/bootloader/*.bin
      - build_${TEST_PREFIX}/*/*/*/build/partition_table/*.bin
      - $LOG_PATH
      - $SIZE_INFO_LOCATION
  variables:
    BUILD_SYSTEM: cmake

build_examples_cmake_esp32:
  extends: .build_examples_cmake
  parallel: 10
  variables:
    IDF_TARGET: esp32

build_examples_cmake_esp32s2:
  extends: .build_examples_cmake
  parallel: 8
  variables:
    IDF_TARGET: esp32s2

.build_test_apps:
  extends: .build_examples_cmake
  variables:
    TEST_PREFIX: test_apps
    TEST_RELATIVE_DIR: tools/test_apps
    TEST_TYPE: custom_test
  only:
    variables:
      - $BOT_TRIGGER_WITH_LABEL == null
      - $BOT_LABEL_BUILD
      - $BOT_LABEL_REGULAR_TEST
      - $BOT_LABEL_CUSTOM_TEST
      - $BOT_LABEL_WEEKEND_TEST
  script:
    - ${IDF_PATH}/tools/ci/find_apps_build_apps.sh

build_test_apps_esp32:
  extends: .build_test_apps
  parallel: 8
  variables:
    IDF_TARGET: esp32

build_test_apps_esp32s2:
  extends: .build_test_apps
  parallel: 8
  variables:
    IDF_TARGET: esp32s2

.build_component_ut:
  extends: .build_test_apps
  variables:
    TEST_PREFIX: component_ut
    TEST_RELATIVE_DIR: component_ut
  only:
    variables:
      - $BOT_TRIGGER_WITH_LABEL == null
      - $BOT_LABEL_BUILD
      - $BOT_LABEL_REGULAR_TEST
      - $BOT_LABEL_UNIT_TEST
      - $BOT_LABEL_UNIT_TEST_S2

build_component_ut_esp32:
  extends: .build_component_ut
  variables:
    IDF_TARGET: esp32

build_component_ut_esp32s2:
  extends: .build_component_ut
  variables:
    IDF_TARGET: esp32s2

# If you want to add new build example jobs, please add it into dependencies of `.example_test_template`

.build_docs_template: &build_docs_template
  stage: build
  image: $ESP_IDF_DOC_ENV_IMAGE
  tags:
    - build_docs
  artifacts:
    when: always
    paths:
      - docs/_build/*/*/*.txt
      - docs/_build/*/*/html/*
    expire_in: 4 days
  only:
    variables:
      - $BOT_TRIGGER_WITH_LABEL == null
      - $BOT_LABEL_BUILD
      - $BOT_LABEL_BUILD_DOCS
      - $BOT_LABEL_REGULAR_TEST
  dependencies: []
  script:
    - cd docs
    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py -bs html -l $DOCLANG -t $DOCTGT build

build_docs_html:
  extends: .build_docs_template
  parallel:
    matrix:
      - DOCLANG: ["en", "zh_CN"]
        DOCTGT: ["esp32", "esp32s2"]

build_docs_pdf:
  stage: build
  image: $ESP_IDF_DOC_ENV_IMAGE
  tags:
    - build_docs
  artifacts:
    when: always
    paths:
      - docs/_build/*/*/latex/*
    expire_in: 4 days
  rules:
    - if: '$CI_COMMIT_REF_NAME == "master"'
    - if: '$CI_COMMIT_REF_NAME =~ "/^release\/v/"'
    - if: '$CI_COMMIT_REF_NAME =~ "/^v\d+\.\d+(\.\d+)?($|-)/"'
    - if: $BOT_LABEL_BUILD_DOCS
  dependencies: []
  parallel:
    matrix:
      - DOCLANG: ["en", "zh_CN"]
        DOCTGT: ["esp32", "esp32s2"]
  script:
    - cd docs
    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py -bs latex -l $DOCLANG -t $DOCTGT build

test_build_system:
  extends: .build_template
  only:
    variables:
      - $BOT_TRIGGER_WITH_LABEL == null
      - $BOT_LABEL_BUILD
      - $BOT_LABEL_REGULAR_TEST
      - $BOT_LABEL_WEEKEND_TEST
  script:
    - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
    - rm -rf test_build_system
    - mkdir test_build_system
    - cd test_build_system
    - ${IDF_PATH}/tools/ci/test_build_system.sh

test_build_system_cmake:
  extends: .build_template
  only:
    variables:
      - $BOT_TRIGGER_WITH_LABEL == null
      - $BOT_LABEL_BUILD
      - $BOT_LABEL_REGULAR_TEST
      - $BOT_LABEL_WEEKEND_TEST
  script:
    - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
    - rm -rf test_build_system
    - mkdir test_build_system
    - cd test_build_system
    - ${IDF_PATH}/tools/ci/test_build_system_cmake.sh

test_build_system_cmake_macos:
  extends: .macos_build_template
  only:
    refs:
      - master
      - /^release\/v/
      - /^v\d+\.\d+(\.\d+)?($|-)/
      - schedules
      - triggers
      - pipelines
      - web
    variables:
      - $CI_PIPELINE_SOURCE != "push" && $BOT_LABEL_MACOS_TEST != null
      - $CI_PIPELINE_SOURCE == "push"
  script:
    - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
    - rm -rf test_build_system
    - mkdir test_build_system
    - cd test_build_system
    - ${IDF_PATH}/tools/ci/test_build_system_cmake.sh

build_docker:
  stage: build
  image: espressif/docker-builder:1
  tags:
    - build_docker_amd64_brno
  only:
    refs:
      - master
      - /^release\/v/
      - /^v\d+\.\d+(\.\d+)?($|-)/
      - schedules
  variables:
    DOCKER_TMP_IMAGE_NAME: "idf_tmp_image"
  before_script: []
  script:
    - export LOCAL_CI_REPOSITORY_URL=$CI_REPOSITORY_URL
    - if [ -n "$LOCAL_GITLAB_HTTPS_HOST" ]; then export LOCAL_CI_REPOSITORY_URL="https://gitlab-ci-token:${CI_JOB_TOKEN}@${LOCAL_GITLAB_HTTPS_HOST}/${CI_PROJECT_PATH}"; fi
    - echo "Using repository at $LOCAL_CI_REPOSITORY_URL"
    - export DOCKER_BUILD_ARGS="--build-arg IDF_CLONE_URL=${LOCAL_CI_REPOSITORY_URL} --build-arg IDF_CLONE_BRANCH_OR_TAG=${CI_COMMIT_REF_NAME} --build-arg IDF_CHECKOUT_REF=${CI_COMMIT_TAG:-$CI_COMMIT_SHA}"
    # Build
    - docker build --tag ${DOCKER_TMP_IMAGE_NAME} ${DOCKER_BUILD_ARGS} tools/docker/
    # We can't mount $PWD/examples/get-started/blink into the container, see https://gitlab.com/gitlab-org/gitlab-ce/issues/41227.
    # The workaround mentioned there works, but leaves around directories which need to be cleaned up manually.
    # Therefore, build a copy of the example located inside the container.
    - docker run --rm --workdir /opt/esp/idf/examples/get-started/blink ${DOCKER_TMP_IMAGE_NAME} idf.py build

build_idf_exe:
  stage: build
  image: $CI_DOCKER_REGISTRY/esp32-toolchain-win-cross
  tags:
    - build
  only:
    refs:
      - master
      - /^release\/v/
      - /^v\d+\.\d+(\.\d+)?($|-)/
      - schedules
  before_script: []
  artifacts:
    paths:
      - tools/windows/idf_exe/build/idf-exe-v*.zip
    expire_in: 4 days
  script:
    - cd tools/windows/idf_exe/
    - mkdir build
    - cd build
    - cmake -DCMAKE_TOOLCHAIN_FILE=../toolchain-i686-w64-mingw32.cmake -DCMAKE_BUILD_TYPE=Release ..
    - cmake --build .

build_cmdlinerunner:
  stage: build
  image: $CI_DOCKER_REGISTRY/esp32-toolchain-win-cross
  tags:
    - build
  only:
    refs:
      - master
      - /^release\/v/
      - /^v\d+\.\d+(\.\d+)?($|-)/
      - schedules
  before_script: []
  artifacts:
    paths:
      - tools/windows/tool_setup/cmdlinerunner/build/cmdlinerunner.dll
    expire_in: 4 days
  script:
    - cd tools/windows/tool_setup/cmdlinerunner
    - mkdir build
    - cd build
    - cmake -DCMAKE_TOOLCHAIN_FILE=../toolchain-i686-w64-mingw32.cmake -DCMAKE_BUILD_TYPE=Release ..
    - cmake --build .

build_installer:
  # using a different stage here to be able to use artifacts from build_cmdlinerunner job
  stage: host_test
  image: $CI_DOCKER_REGISTRY/wine-innosetup:1
  tags:
    - build
  only:
    refs:
      - master
      - /^release\/v/
      - /^v\d+\.\d+(\.\d+)?($|-)/
      - schedules
  dependencies:
    - build_cmdlinerunner
  before_script: []
  script:
    - cd tools/windows/tool_setup/
    - ./build_installer.sh

# This job builds template app with permutations of targets and optimization levels
build_template_app:
  extends: .build_template_app_template

# Sonarqube related jobs put here for this reason:
# Here we have two jobs. code_quality_check and code_quality_report.
#
# code_quality_check will analyze the code changes between your MR and
# code repo stored in sonarqube server. The analysis result is only shown in
# the comments under this MR and won't be transferred to the server.
#
# code_quality_report will analyze and transfer both of the newly added code
# and the analysis result to the server.
#
# Put in the front to ensure that the newly merged code can be stored in
# sonarqube server ASAP, in order to avoid reporting unrelated code issues
.sonar_scan_template:
  stage: build
  image:
    name: $CI_DOCKER_REGISTRY/sonarqube-scanner:1
  before_script:
    - export PYTHONPATH="$CI_PROJECT_DIR/tools:$CI_PROJECT_DIR/tools/ci/python_packages:$PYTHONPATH"
    - python $SUBMODULE_FETCH_TOOL
    # Exclude the submodules, all paths ends with /**
    # get all submodules configs | get all paths | add /** as suffix | xargs | replace all <space> to <comma>
    - export SUBMODULES=$(git config --file .gitmodules --get-regexp path | awk '{ print $2 }' | sed -e 's|$|/**|' | xargs | sed -e 's/ /,/g')
    # Exclude the report dir
    - export EXCLUSIONS="$SUBMODULES,$REPORT_DIR/**,docs/_static/**,**/*.png,**/*.jpg"
    - python $NORMALIZE_CLANGTIDY_PY $CI_PROJECT_DIR/$REPORT_DIR/warnings.txt $CI_PROJECT_DIR/$REPORT_DIR/clang_tidy_report.txt $CI_PROJECT_DIR
  variables:
    GIT_DEPTH: 0
    NORMALIZE_CLANGTIDY_PY: $CI_PROJECT_DIR/tools/ci/normalize_clangtidy_path.py
    REPORT_DIR: examples/get-started/hello_world/tidybuild/report
  tags:
    - host_test

code_quality_check:
  extends: .sonar_scan_template
  dependencies:
    - clang_tidy_check_regular
  only:
    - triggers
  allow_failure: true
  script:
    - export CI_MERGE_REQUEST_IID=`python ${CI_PROJECT_DIR}/tools/ci/ci_get_latest_mr_iid.py ${CI_COMMIT_BRANCH} | xargs`
    # test if this branch have merge request, if not, exit 0
    - test -n "$CI_MERGE_REQUEST_IID" || exit 0
    - sonar-scanner
      -Dsonar.analysis.mode=preview
      -Dsonar.host.url=$SONAR_HOST_URL
      -Dsonar.login=$SONAR_LOGIN
      -Dsonar.sources=$CI_PROJECT_DIR
      -Dsonar.sourceEncoding=UTF-8
      -Dsonar.projectKey=esp-idf
      -Dsonar.projectBaseDir=$CI_PROJECT_DIR
      -Dsonar.exclusions=$EXCLUSIONS
      -Dsonar.gitlab.project_id=$CI_PROJECT_ID
      -Dsonar.gitlab.commit_sha=$(git log --pretty=format:%H origin/master..origin/$CI_COMMIT_REF_NAME | tr '\n' ',')
      -Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME
      -Dsonar.cxx.clangtidy.reportPath=$REPORT_DIR/clang_tidy_report.txt
      -Dsonar.cxx.includeDirectories=components,/usr/include
      -Dsonar.python.pylint_config=.pylintrc
      -Dsonar.gitlab.ci_merge_request_iid=$CI_MERGE_REQUEST_IID
      -Dsonar.gitlab.merge_request_discussion=true
      -Dsonar.branch.name=$CI_COMMIT_REF_NAME

code_quality_report:
  extends: .sonar_scan_template
  only:
    - master
    - /^release\/v/
    - /^v\d+\.\d+(\.\d+)?($|-)/
    - schedules
  dependencies:
    - clang_tidy_check_regular
  script:
    - sonar-scanner
      -Dsonar.host.url=$SONAR_HOST_URL
      -Dsonar.login=$SONAR_LOGIN
      -Dsonar.sources=$CI_PROJECT_DIR
      -Dsonar.sourceEncoding=UTF-8
      -Dsonar.projectKey=esp-idf
      -Dsonar.projectBaseDir=$CI_PROJECT_DIR
      -Dsonar.exclusions=$EXCLUSIONS
      -Dsonar.gitlab.project_id=$CI_PROJECT_ID
      -Dsonar.gitlab.commit_sha=$CI_COMMIT_SHA
      -Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME
      -Dsonar.cxx.clangtidy.reportPath=$REPORT_DIR/clang_tidy_report.txt
      -Dsonar.cxx.includeDirectories=components,/usr/include
      -Dsonar.python.pylint_config=.pylintrc
      -Dsonar.branch.name=$CI_COMMIT_REF_NAME