Merge branch 'feature/docker_build_gh_actions_v4.1' into 'release/v4.1'

ci: move Docker image builds to Github, plus other improvements (v4.1)

See merge request espressif/esp-idf!18290

.github/workflows/docker.yml

@@ -0,0 +1,75 @@
+name: docker
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+
+on:
+  push:
+    branches:
+      - 'master'
+      - 'release/*'
+    tags:
+      - 'v*.*'
+
+env:
+  # Platforms to build the image for
+  BUILD_PLATFORMS: linux/amd64
+  DOCKERHUB_REPO: ${{ github.repository_owner }}/idf
+
+jobs:
+  docker:
+    # Disable the job in forks
+    if: ${{ github.repository_owner == 'espressif' }}
+
+    runs-on: ubuntu-latest
+    steps:
+      # Depending on the branch/tag, set CLONE_BRANCH_OR_TAG variable (used in the Dockerfile
+      # as a build arg) and TAG_NAME (used when tagging the image).
+      #
+      # The following 3 steps cover the alternatives (tag, release branch, master branch):
+      - name: Set variables (tags)
+        if: ${{ github.ref_type == 'tag' }}
+        run: |
+          echo "CLONE_BRANCH_OR_TAG=$GITHUB_REF_NAME" >> $GITHUB_ENV
+          echo "TAG_NAME=$GITHUB_REF_NAME" >> $GITHUB_ENV
+      - name: Set variables (release branches)
+        if: ${{ github.ref_type == 'branch' && startsWith(github.ref_name, 'release/') }}
+        run: |
+          echo "CLONE_BRANCH_OR_TAG=$GITHUB_REF_NAME" >> $GITHUB_ENV
+          echo "TAG_NAME=release-${GITHUB_REF_NAME##release/}" >> $GITHUB_ENV
+      - name: Set variables (main branch)
+        if: ${{ github.ref_type == 'branch' && github.ref_name == 'master' }}
+        run: |
+          echo "CLONE_BRANCH_OR_TAG=master" >> $GITHUB_ENV
+          echo "TAG_NAME=latest" >> $GITHUB_ENV
+
+      # Display the variables set above, just in case.
+      - name: Check variables
+        run: |
+          echo "CLONE_BRANCH_OR_TAG: $CLONE_BRANCH_OR_TAG"
+          echo "CHECKOUT_REF: $CHECKOUT_REF"
+          echo "TAG_NAME: $TAG_NAME"
+
+      # The following steps are the standard boilerplate from
+      # https://github.com/marketplace/actions/build-and-push-docker-images
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Set up QEMU for multiarch builds
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Build and push
+        uses: docker/build-push-action@v3
+        with:
+          context: tools/docker
+          push: true
+          tags: ${{ env.DOCKERHUB_REPO }}:${{ env.TAG_NAME }}
+          platforms: ${{ env.BUILD_PLATFORMS }}
+          build-args: |
+            IDF_CLONE_URL=${{ github.server_url }}/${{ github.repository }}.git
+            IDF_CLONE_BRANCH_OR_TAG=${{ env.CLONE_BRANCH_OR_TAG }}

docs/en/api-guides/tools/idf-docker-image.rst

@@ -59,11 +59,9 @@ The above command explained:
 
 To build with a specific docker image tag, specify it as ``espressif/idf:TAG``::
 
-    docker run --rm -v $PWD:/project -w /project espressif/idf:v4.0 idf.py build
+    docker run --rm -v $PWD:/project -w /project espressif/idf:release-v4.4 idf.py build
 
-.. note::
-
-    At the time of writing, v4.0 release of ESP-IDF does not exist, yet, so the above command will not work. You can check the up-to-date list of available tags at https://hub.docker.com/r/espressif/idf/tags.
+You can check the up-to-date list of available tags at https://hub.docker.com/r/espressif/idf/tags.
 
 
 Building a project with GNU Make
@@ -94,3 +92,21 @@ Then inside the container, use ``idf.py`` as usual::
 .. note::
 
     Commands which communicate with the development board, such as ``idf.py flash`` and ``idf.py monitor`` will not work in the container unless the serial port is passed through into the container. However currently this is not possible with Docker for Windows (https://github.com/docker/for-win/issues/1018) and Docker for Mac (https://github.com/docker/for-mac/issues/900).
+
+Building custom images
+======================
+
+The Dockerfile in ESP-IDF repository provides several build arguments which can be used to customize the Docker image:
+
+- ``IDF_CLONE_URL``: URL of the repository to clone ESP-IDF from. Can be set to a custom URL when working with a fork of ESP-IDF. Default is ``https://github.com/espressif/esp-idf.git``.
+- ``IDF_CLONE_BRANCH_OR_TAG``: Name of a git branch or tag use when cloning ESP-IDF. This value is passed to ``git clone`` command using the ``--branch`` argument. Default is ``master``.
+- ``IDF_CHECKOUT_REF``: If this argument is set to a non-empty value, ``git checkout $IDF_CHECKOUT_REF`` command will be performed after cloning. This argument can be set to the SHA of the specific commit to check out, for example if some specific commit on a release branch is desired.
+- ``IDF_CLONE_SHALLOW``: If this argument is set to a non-empty value, ``--depth=1 --shallow-submodules`` arguments will be used when performing ``git clone``. This significantly reduces the amount of data downloaded and the size of the resulting Docker image. However, if switching to a different branch in such a "shallow" repository is necessary, an additional ``git fetch origin <branch>`` command must be executed first.
+
+To use these arguments, pass them via the ``--build-arg`` command line option. For example, the following command will build a Docker image with a shallow clone of ESP-IDF v4.4.1 and tools for ESP32-C3, only::
+
+    docker build -t idf-custom:v4.4.1-esp32c3 \
+        --build-arg IDF_CLONE_BRANCH_OR_TAG=v4.4.1 \
+        --build-arg IDF_CLONE_SHALLOW=1 \
+        --build-arg IDF_INSTALL_TARGETS=esp32c3 \
+        tools/docker

tools/ci/executable-list.txt

@@ -57,7 +57,6 @@ tools/ci/test_configure_ci_environment.sh
 tools/cmake/convert_to_cmake.py
 tools/cmake/run_cmake_lint.sh
 tools/docker/entrypoint.sh
-tools/docker/hooks/build
 tools/elf_to_ld.sh
 tools/esp_app_trace/logtrace_proc.py
 tools/esp_app_trace/sysviewtrace_proc.py

tools/docker/Dockerfile

@@ -36,20 +36,26 @@ RUN python -m pip install --upgrade pip virtualenv
 # It is possibe to combine both, e.g.:
 #   IDF_CLONE_BRANCH_OR_TAG=release/vX.Y
 #   IDF_CHECKOUT_REF=<some commit on release/vX.Y branch>.
+# Use IDF_CLONE_SHALLOW=1 to peform shallow clone (i.e. --depth=1 --shallow-submodules)
 
 ARG IDF_CLONE_URL=https://github.com/espressif/esp-idf.git
 ARG IDF_CLONE_BRANCH_OR_TAG=master
 ARG IDF_CHECKOUT_REF=
+ARG IDF_CLONE_SHALLOW=
 
 ENV IDF_PATH=/opt/esp/idf
 ENV IDF_TOOLS_PATH=/opt/esp
 
 RUN echo IDF_CHECKOUT_REF=$IDF_CHECKOUT_REF IDF_CLONE_BRANCH_OR_TAG=$IDF_CLONE_BRANCH_OR_TAG && \
     git clone --recursive \
+      ${IDF_CLONE_SHALLOW:+--depth=1 --shallow-submodules} \
       ${IDF_CLONE_BRANCH_OR_TAG:+-b $IDF_CLONE_BRANCH_OR_TAG} \
       $IDF_CLONE_URL $IDF_PATH && \
     if [ -n "$IDF_CHECKOUT_REF" ]; then \
       cd $IDF_PATH && \
+      if [ -n "$IDF_CLONE_SHALLOW" ]; then \
+        git fetch origin --depth=1 --recurse-submodules ${IDF_CHECKOUT_REF}; \
+      fi && \
       git checkout $IDF_CHECKOUT_REF && \
       git submodule update --init --recursive; \
     fi

tools/docker/hooks/build

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-# This file gets executed to build the image on the Docker Hub.
-# See https://docs.docker.com/docker-hub/builds/advanced/#build-hook-examples for details.
-
-set -euo pipefail
-
-echo "Building for branch ${SOURCE_BRANCH}, commit ${SOURCE_COMMIT}"
-
-docker build \
-    --build-arg IDF_CLONE_BRANCH_OR_TAG=${SOURCE_BRANCH} \
-    --build-arg IDF_CHECKOUT_REF=${SOURCE_COMMIT} \
-    -f $DOCKERFILE_PATH \
-    -t $IMAGE_NAME .