esp-idf/.gitlab/ci/build.yml

.build_template:
  stage: build
  image: $ESP_ENV_IMAGE
  tags:
    - build
  variables:
    # Enable ccache for all build jobs. See configure_ci_environment.sh for more ccache related settings.
    IDF_CCACHE_ENABLE: "1"
  after_script:
    # Show ccache statistics if enabled globally
    - test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats || true
  dependencies: []

.build_cmake_template:
  extends:
    - .build_template
    - .before_script_build_jobs
  dependencies: # set dependencies to null to avoid missing artifacts issue
  needs:
    - job: fast_template_app
      artifacts: false
  tags:
    - build
    # build only on shiny servers since shiny storage server is at the same location
    - shiny
  artifacts:
    paths:
      - "**/build*/size.json"
      - "**/build*/build_log.txt"
      - "**/build*/*.bin"
      # upload to s3 server to save the artifacts size
      # - "**/build*/*.map"
      # ttfw tests require elf files
      - "**/build*/*.elf"
      - "**/build*/flasher_args.json"
      - "**/build*/flash_project_args"
      - "**/build*/config/sdkconfig.json"
      # ttfw tests require sdkconfig file
      - "**/build*/sdkconfig"
      - "**/build*/bootloader/*.bin"
      - "**/build*/partition_table/*.bin"
      - list_job_*.json
      - size_info.txt
      # unit test specific
      - components/idf_test/unit_test/*.yml
    when: always
    expire_in: 4 days
  after_script:
    # Show ccache statistics if enabled globally
    - test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats || true
    # upload the binary files to s3 server
    - echo -e "\e[0Ksection_start:`date +%s`:upload_binaries_to_s3_server[collapsed=true]\r\e[0KUploading binaries to s3 Server"
    - shopt -s globstar
    # use || true to bypass the no-file error
    - zip ${CI_JOB_ID}.zip **/build*/*.bin || true
    - zip ${CI_JOB_ID}.zip **/build*/*.elf || true
    - zip ${CI_JOB_ID}.zip **/build*/*.map || true
    - zip ${CI_JOB_ID}.zip **/build*/flasher_args.json || true
    - zip ${CI_JOB_ID}.zip **/build*/flash_project_args || true
    - zip ${CI_JOB_ID}.zip **/build*/config/sdkconfig.json || true
    - zip ${CI_JOB_ID}.zip **/build*/sdkconfig || true
    - zip ${CI_JOB_ID}.zip **/build*/bootloader/*.bin || true
    - zip ${CI_JOB_ID}.zip **/build*/partition_table/*.bin || true
    - mc alias set shiny-s3 ${SHINY_S3_SERVER} ${SHINY_S3_ACCESS_KEY} ${SHINY_S3_SECRET_KEY}
    - mc cp ${CI_JOB_ID}.zip shiny-s3/idf-artifacts/${CI_PIPELINE_ID}/${CI_JOB_ID}.zip
    - echo -e "\e[0Ksection_end:`date +%s`:upload_binaries_to_s3_server\r\e[0K"
    - echo "Please download the full binary files (including *.elf and *.map files) from the following share link"
    # would be clean up after 4 days
    - mc share download shiny-s3/idf-artifacts/${CI_PIPELINE_ID}/${CI_JOB_ID}.zip --expire=96h
  script:
    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
    - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
      -t $IDF_TARGET
      --copy-sdkconfig
      --collect-size-info size_info.txt
      --collect-app-info list_job_${CI_NODE_INDEX:-1}.json
      --parallel-count ${CI_NODE_TOTAL:-1}
      --parallel-index ${CI_NODE_INDEX:-1}
      --extra-preserve-dirs
        examples/bluetooth/esp_ble_mesh/ble_mesh_console
        examples/bluetooth/hci/controller_hci_uart_esp32
        examples/wifi/iperf

.build_pytest_template:
  extends:
    - .build_cmake_template
    - .before_script_build_jobs
  artifacts:
    paths:
      - "**/build*/size.json"
      - "**/build*/build_log.txt"
      - "**/build*/*.bin"
      # upload to s3 server to save the artifacts size
      # - "**/build*/*.map"
      # - "**/build*/*.elf"
      - "**/build*/flasher_args.json"
      - "**/build*/flash_project_args"
      - "**/build*/config/sdkconfig.json"
      - "**/build*/bootloader/*.bin"
      - "**/build*/bootloader/*.elf"
      - "**/build*/partition_table/*.bin"
      - "**/build*/project_description.json"
      - list_job_*.json
      - size_info.txt
    when: always
    expire_in: 4 days
  script:
    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
    - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
      -t $IDF_TARGET
      --pytest-apps
      --collect-size-info size_info.txt
      --parallel-count ${CI_NODE_TOTAL:-1}
      --parallel-index ${CI_NODE_INDEX:-1}

build_pytest_examples_esp32:
  extends:
    - .build_pytest_template
    - .rules:build:example_test-esp32
  parallel: 4
  variables:
    IDF_TARGET: esp32
    TEST_DIR: examples

build_pytest_examples_esp32s2:
  extends:
    - .build_pytest_template
    - .rules:build:example_test-esp32s2
  parallel: 3
  variables:
    IDF_TARGET: esp32s2
    TEST_DIR: examples

build_pytest_examples_esp32s3:
  extends:
    - .build_pytest_template
    - .rules:build:example_test-esp32s3
  parallel: 4
  variables:
    IDF_TARGET: esp32s3
    TEST_DIR: examples

build_pytest_examples_esp32c3:
  extends:
    - .build_pytest_template
    - .rules:build:example_test-esp32c3
  parallel: 3
  variables:
    IDF_TARGET: esp32c3
    TEST_DIR: examples

build_pytest_examples_esp32c2:
  extends:
    - .build_pytest_template
    - .rules:build:example_test-esp32c2
  variables:
    IDF_TARGET: esp32c2
    TEST_DIR: examples

build_pytest_components_esp32:
  extends:
    - .build_pytest_template
    - .rules:build:component_ut-esp32
  parallel: 2
  variables:
    IDF_TARGET: esp32
    TEST_DIR: components

build_pytest_components_esp32s2:
  extends:
    - .build_pytest_template
    - .rules:build:component_ut-esp32s2
  parallel: 2
  variables:
    IDF_TARGET: esp32s2
    TEST_DIR: components

build_pytest_components_esp32s3:
  extends:
    - .build_pytest_template
    - .rules:build:component_ut-esp32s3
  parallel: 2
  variables:
    IDF_TARGET: esp32s3
    TEST_DIR: components

build_pytest_components_esp32c3:
  extends:
    - .build_pytest_template
    - .rules:build:component_ut-esp32c3
  parallel: 2
  variables:
    IDF_TARGET: esp32c3
    TEST_DIR: components

build_pytest_components_esp32c2:
  extends:
    - .build_pytest_template
    - .rules:build:component_ut-esp32c2
  variables:
    IDF_TARGET: esp32c2
    TEST_DIR: components

build_non_test_components_apps:
  extends:
    - .build_cmake_template
    - .rules:build:component_ut
  script:
    - set_component_ut_vars
    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
    - run_cmd python tools/ci/ci_build_apps.py $COMPONENT_UT_DIRS -v
      -t all
      --collect-size-info size_info.txt
      --collect-app-info list_job_${CI_NODE_INDEX:-1}.json
      --parallel-count ${CI_NODE_TOTAL:-1}
      --parallel-index ${CI_NODE_INDEX:-1}

.build_pytest_test_apps_template:
  extends: .build_pytest_template
  artifacts:
    paths:
      - "**/build*/size.json"
      - "**/build*/build_log.txt"
      - "**/build*/*.bin"
      # upload to s3 server to save the artifacts size
      # - "**/build*/*.map"
      # pytest test apps requires elf files for coredump tests
      - "**/build*/*.elf"
      - "**/build*/flasher_args.json"
      - "**/build*/flash_project_args"
      - "**/build*/config/sdkconfig.json"
      - "**/build*/bootloader/*.bin"
      - "**/build*/partition_table/*.bin"
      - list_job_*.json
      - size_info.txt
    when: always
    expire_in: 4 days

build_pytest_test_apps_esp32:
  extends:
    - .build_pytest_test_apps_template
    - .rules:build:custom_test-esp32
  variables:
    IDF_TARGET: esp32
    TEST_DIR: tools/test_apps

build_pytest_test_apps_esp32s2:
  extends:
    - .build_pytest_test_apps_template
    - .rules:build:custom_test-esp32s2
  variables:
    IDF_TARGET: esp32s2
    TEST_DIR: tools/test_apps

build_pytest_test_apps_esp32s3:
  extends:
    - .build_pytest_test_apps_template
    - .rules:build:custom_test-esp32s3
  variables:
    IDF_TARGET: esp32s3
    TEST_DIR: tools/test_apps

build_pytest_test_apps_esp32c3:
  extends:
    - .build_pytest_test_apps_template
    - .rules:build:custom_test-esp32c3
  variables:
    IDF_TARGET: esp32c3
    TEST_DIR: tools/test_apps

build_pytest_test_apps_esp32c2:
  extends:
    - .build_pytest_test_apps_template
    - .rules:build:custom_test-esp32c2
  variables:
    IDF_TARGET: esp32c2
    TEST_DIR: tools/test_apps

.build_template_app_template:
  extends:
    - .build_template
    - .before_script_build_jobs
  variables:
    LOG_PATH: "${CI_PROJECT_DIR}/log_template_app"
    BUILD_PATH: "${CI_PROJECT_DIR}/build_template_app"
    BUILD_DIR: "@t/@w"
    BUILD_LOG_CMAKE: "${LOG_PATH}/cmake_@t_@w.txt"
    BUILD_COMMAND_ARGS: ""
  artifacts:
    when: always
    paths:
      - log_template_app/*
      - size_info.txt
      - build_template_app/**/size.json
    expire_in: 1 week
  script:
    # Set the variable for 'esp-idf-template' testing
    - ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"}
    - retry_failed git clone ${ESP_IDF_TEMPLATE_GIT}
    # Try to use the same branch name for esp-idf-template that we're
    # using on esp-idf. If it doesn't exist then just stick to the default branch
    - python $CHECKOUT_REF_SCRIPT esp-idf-template esp-idf-template
    - export PATH="$IDF_PATH/tools:$PATH"
    # Only do the default cmake build for each target, remaining part are done in the build_template_app job
    - tools/ci/build_template_app.sh ${BUILD_COMMAND_ARGS}

# build-related-pre-check-jobs ------------------------------------------------
# Build at least one project for each target at earliest stage to reduce build cost for obvious failing commits
fast_template_app:
  extends:
    - .build_template_app_template
    - .rules:build:target_test
  stage: pre_check
  variables:
    BUILD_COMMAND_ARGS: "-p"
#------------------------------------------------------------------------------

.build_esp_idf_tests_cmake_template:
  extends:
    - .build_cmake_template
    - .before_script_build_jobs
  artifacts:
    paths:
      - "**/build*/size.json"
      - "**/build*/build_log.txt"
      - "**/build*/*.bin"
      # upload to s3 server to save the artifacts size
      # - "**/build*/*.map"
      # ttfw tests require elf files
      - "**/build*/*.elf"
      - "**/build*/flasher_args.json"
      - "**/build*/flash_project_args"
      - "**/build*/config/sdkconfig.json"
      - "**/build*/sdkconfig"
      - "**/build*/bootloader/*.bin"
      - "**/build*/partition_table/*.bin"
      - list_job_*.json
      - size_info.txt
      - components/idf_test/unit_test/*.yml
    when: always
    expire_in: 4 days
  script:
    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
    - run_cmd python tools/ci/ci_build_apps.py tools/unit-test-app -v
      -t $IDF_TARGET
      --config "configs/*="
      --copy-sdkconfig
      --preserve-all
      --collect-size-info size_info.txt
      --collect-app-info list_job_${CI_NODE_INDEX:-1}.json
      --parallel-count ${CI_NODE_TOTAL:-1}
      --parallel-index ${CI_NODE_INDEX:-1}
    - run_cmd python tools/unit-test-app/tools/UnitTestParser.py tools/unit-test-app ${CI_NODE_INDEX:-1}

build_esp_idf_tests_cmake_esp32:
  extends:
    - .build_esp_idf_tests_cmake_template
    - .rules:build:unit_test-esp32
  parallel: 2
  variables:
    IDF_TARGET: esp32

build_esp_idf_tests_cmake_esp32s2:
  extends:
    - .build_esp_idf_tests_cmake_template
    - .rules:build:unit_test-esp32s2
  parallel: 2
  variables:
    IDF_TARGET: esp32s2

build_esp_idf_tests_cmake_esp32s3:
  extends:
    - .build_esp_idf_tests_cmake_template
    - .rules:build:unit_test-esp32s3
  variables:
    IDF_TARGET: esp32s3

build_esp_idf_tests_cmake_esp32c2:
  extends:
    - .build_esp_idf_tests_cmake_template
    - .rules:build:unit_test-esp32c2
  variables:
    IDF_TARGET: esp32c2

build_esp_idf_tests_cmake_esp32c3:
  extends:
    - .build_esp_idf_tests_cmake_template
    - .rules:build:unit_test-esp32c3
  variables:
    IDF_TARGET: esp32c3

build_examples_cmake_esp32:
  extends:
    - .build_cmake_template
    - .rules:build:example_test-esp32
  parallel: 9
  variables:
    IDF_TARGET: esp32
    TEST_DIR: examples

build_examples_cmake_esp32s2:
  extends:
    - .build_cmake_template
    - .rules:build:example_test-esp32s2
  parallel: 6
  variables:
    IDF_TARGET: esp32s2
    TEST_DIR: examples

build_examples_cmake_esp32s3:
  extends:
    - .build_cmake_template
    - .rules:build:example_test-esp32s3
  parallel: 7
  variables:
    IDF_TARGET: esp32s3
    TEST_DIR: examples

build_examples_cmake_esp32c2:
  extends:
    - .build_cmake_template
    - .rules:build:example_test-esp32c2
  parallel: 4
  variables:
    IDF_TARGET: esp32c2
    TEST_DIR: examples

build_examples_cmake_esp32c3:
  extends:
    - .build_cmake_template
    - .rules:build:example_test-esp32c3
  parallel: 6
  variables:
    IDF_TARGET: esp32c3
    TEST_DIR: examples

build_test_apps_esp32:
  extends:
    - .build_cmake_template
    - .rules:build:custom_test-esp32
  parallel: 2
  variables:
    IDF_TARGET: esp32
    TEST_DIR: tools/test_apps

build_test_apps_esp32s2:
  extends:
    - .build_cmake_template
    - .rules:build:custom_test-esp32s2
  parallel: 2
  variables:
    IDF_TARGET: esp32s2
    TEST_DIR: tools/test_apps

build_test_apps_esp32s3:
  extends:
    - .build_cmake_template
    - .rules:build:custom_test-esp32s3
  parallel: 2
  variables:
    IDF_TARGET: esp32s3
    TEST_DIR: tools/test_apps

build_test_apps_esp32c3:
  extends:
    - .build_cmake_template
    - .rules:build:custom_test-esp32c3
  parallel: 2
  variables:
    IDF_TARGET: esp32c3
    TEST_DIR: tools/test_apps

build_test_apps_esp32c2:
  extends:
    - .build_cmake_template
    - .rules:build:custom_test-esp32c2
  variables:
    IDF_TARGET: esp32c2
    TEST_DIR: tools/test_apps

.test_build_system_template:
  stage: host_test
  extends:
    - .build_template
    - .rules:build
  needs:
    - job: fast_template_app
      artifacts: false
      optional: true
  script:
    - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
    - rm -rf test_build_system
    - mkdir test_build_system
    - cd test_build_system
    - ${IDF_PATH}/tools/ci/${SHELL_TEST_SCRIPT}

test_build_system_cmake:
  extends: .test_build_system_template
  variables:
    SHELL_TEST_SCRIPT: test_build_system_cmake.sh

test_build_system_cmake_macos:
  extends:
    - .test_build_system_template
    - .before_script_macos
    - .rules:build:macos
  tags:
    - macos_shell
  variables:
    SHELL_TEST_SCRIPT: test_build_system_cmake.sh
    PYENV_VERSION: "3.7"
  script:
    - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
    - rm -rf test_build_system
    - mkdir test_build_system
    - cd test_build_system
    # copy-paste the script from .test_build_system_template
    # since `ESP_IDF_TEMPLATE_GIT` is a group variable and has higher precedence than job variable
    # export here to override the group variable
    #
    # Clone the template app from github for macos runners
    - export ESP_IDF_TEMPLATE_GIT="https://github.com/espressif/esp-idf-template.git"
    - ${IDF_PATH}/tools/ci/${SHELL_TEST_SCRIPT}

test_build_system_spaces:
  extends: .test_build_system_template
  variables:
    SHELL_TEST_SCRIPT: test_build_system_spaces.py

build_docker:
  extends:
    - .before_script_minimal
    - .rules:build:docker
  stage: host_test
  needs: []
  image: espressif/docker-builder:1
  tags:
    - build_docker_amd64_brno
  variables:
    DOCKER_TMP_IMAGE_NAME: "idf_tmp_image"
  script:
    - export LOCAL_CI_REPOSITORY_URL=$CI_REPOSITORY_URL
    - if [ -n "$LOCAL_GITLAB_HTTPS_HOST" ]; then export LOCAL_CI_REPOSITORY_URL="https://gitlab-ci-token:${CI_JOB_TOKEN}@${LOCAL_GITLAB_HTTPS_HOST}/${CI_PROJECT_PATH}"; fi
    - if [ -n "$LOCAL_GIT_MIRROR" ]; then export LOCAL_CI_REPOSITORY_URL="${LOCAL_GIT_MIRROR}/${CI_PROJECT_PATH}"; fi
    - echo "Using repository at $LOCAL_CI_REPOSITORY_URL"
    - export DOCKER_BUILD_ARGS="--build-arg IDF_CLONE_URL=${LOCAL_CI_REPOSITORY_URL} --build-arg IDF_CLONE_BRANCH_OR_TAG=${CI_COMMIT_REF_NAME} --build-arg IDF_CHECKOUT_REF=${CI_COMMIT_TAG:-$PIPELINE_COMMIT_SHA}"
    # Build
    - docker build --tag ${DOCKER_TMP_IMAGE_NAME} ${DOCKER_BUILD_ARGS} tools/docker/
    # We can't mount $PWD/examples/get-started/blink into the container, see https://gitlab.com/gitlab-org/gitlab-ce/issues/41227.
    # The workaround mentioned there works, but leaves around directories which need to be cleaned up manually.
    # Therefore, build a copy of the example located inside the container.
    - docker run --rm --workdir /opt/esp/idf/examples/get-started/blink ${DOCKER_TMP_IMAGE_NAME} idf.py build

# This job builds template app with permutations of targets and optimization levels
build_template_app:
  extends:
    - .build_template_app_template
    - .rules:build
  stage: host_test
  needs:
    - job: fast_template_app
      artifacts: false