cmake: Build all examples in CI

Includes some fixes for compile errors/warnings in examples.

.gitlab-ci.yml

@@ -177,6 +177,12 @@ build_esp_idf_tests:
     # build some of examples
     - mkdir -p ${LOG_PATH}
     - ${IDF_PATH}/tools/ci/build_examples.sh "${CI_JOB_NAME}"
+    # and again, with cmake!
+    - rm -rf ../build_examples_cmake
+    - mkdir ../build_examples_cmake
+    - cd ../build_examples_cmake
+    - mkdir -p ${LOG_PATH}_cmake
+    - LOG_PATH=${LOG_PATH}_cmake ${IDF_PATH}/tools/ci/build_examples_cmake.sh "${CI_JOB_NAME}"
 
 build_examples_00:
   <<: *build_examples_template

components/bt/CMakeLists.txt

@@ -5,73 +5,78 @@ if(CONFIG_BT_ENABLED)
 
     if(CONFIG_BLUEDROID_ENABLED)
 
-        list(APPEND COMPONENT_ADD_INCLUDEDIRS
+        list(APPEND COMPONENT_PRIV_INCLUDEDIRS
             bluedroid/bta/include
-		    bluedroid/bta/sys/include
-		    bluedroid/btcore/include
-		    bluedroid/device/include
-		    bluedroid/hci/include
-		    bluedroid/osi/include
-		    bluedroid/external/sbc/decoder/include
-		    bluedroid/external/sbc/encoder/include
-		    bluedroid/btc/profile/esp/blufi/include
-		    bluedroid/btc/profile/esp/include
-		    bluedroid/btc/profile/std/a2dp/include
-		    bluedroid/btc/profile/std/include
-		    bluedroid/btc/include
-		    bluedroid/stack/gap/include
-		    bluedroid/stack/gatt/include
-		    bluedroid/stack/l2cap/include
-		    bluedroid/stack/sdp/include
-		    bluedroid/stack/smp/include
-		    bluedroid/stack/avct/include
-		    bluedroid/stack/avrc/include
-		    bluedroid/stack/avdt/include
-		    bluedroid/stack/a2dp/include
-		    bluedroid/stack/rfcomm/include
-		    bluedroid/stack/include
-		    bluedroid/api/include
-		    bluedroid/include
-            )
+            bluedroid/bta/ar/include
+            bluedroid/bta/av/include
+            bluedroid/bta/dm/include
+            bluedroid/bta/gatt/include
+            bluedroid/bta/hh/include
+            bluedroid/bta/jv/include
+            bluedroid/bta/sdp/include
+            bluedroid/bta/sys/include
+            bluedroid/device/include
+            bluedroid/hci/include
+            bluedroid/osi/include
+            bluedroid/external/sbc/decoder/include
+            bluedroid/external/sbc/encoder/include
+            bluedroid/btc/profile/esp/blufi/include
+            bluedroid/btc/profile/esp/include
+            bluedroid/btc/profile/std/a2dp/include
+            bluedroid/btc/profile/std/include
+            bluedroid/btc/include
+            bluedroid/stack/btm/include
+            bluedroid/stack/gap/include
+            bluedroid/stack/gatt/include
+            bluedroid/stack/l2cap/include
+            bluedroid/stack/sdp/include
+            bluedroid/stack/smp/include
+            bluedroid/stack/avct/include
+            bluedroid/stack/avrc/include
+            bluedroid/stack/avdt/include
+            bluedroid/stack/a2dp/include
+            bluedroid/stack/rfcomm/include
+            bluedroid/stack/include
+            bluedroid/common/include)
+
+        list(APPEND COMPONENT_ADD_INCLUDEDIRS bluedroid/api/include/api)
 
         list(APPEND COMPONENT_SRCDIRS
             bluedroid/bta/dm
-		    bluedroid/bta/gatt
-		    bluedroid/bta/hh
-		    bluedroid/bta/sdp
-		    bluedroid/bta/av
-		    bluedroid/bta/ar
-		    bluedroid/bta/sys
-		    bluedroid/bta/jv
-		    bluedroid/btcore
-		    bluedroid/btif
-		    bluedroid/device
-		    bluedroid/hci
-		    bluedroid/main
-		    bluedroid/osi
-		    bluedroid/external/sbc/decoder/srce
-		    bluedroid/external/sbc/encoder/srce
-		    bluedroid/btc/core
-		    bluedroid/btc/profile/esp/blufi
-		    bluedroid/btc/profile/std/gap
-		    bluedroid/btc/profile/std/gatt
-		    bluedroid/btc/profile/std/a2dp
-		    bluedroid/btc/profile/std/avrc
-		    bluedroid/btc/profile/std/spp
-		    bluedroid/stack/btm
-		    bluedroid/stack/btu
-		    bluedroid/stack/gap
-		    bluedroid/stack/gatt
-		    bluedroid/stack/hcic
-		    bluedroid/stack/l2cap
-		    bluedroid/stack/sdp
-		    bluedroid/stack/smp
-		    bluedroid/stack/avct
-		    bluedroid/stack/avrc
-		    bluedroid/stack/avdt
-		    bluedroid/stack/a2dp
-		    bluedroid/stack/rfcomm
-		    bluedroid/api
+            bluedroid/bta/gatt
+            bluedroid/bta/hh
+            bluedroid/bta/sdp
+            bluedroid/bta/av
+            bluedroid/bta/ar
+            bluedroid/bta/sys
+            bluedroid/bta/jv
+            bluedroid/device
+            bluedroid/hci
+            bluedroid/main
+            bluedroid/osi
+            bluedroid/external/sbc/decoder/srce
+            bluedroid/external/sbc/encoder/srce
+            bluedroid/btc/core
+            bluedroid/btc/profile/esp/blufi
+            bluedroid/btc/profile/std/gap
+            bluedroid/btc/profile/std/gatt
+            bluedroid/btc/profile/std/a2dp
+            bluedroid/btc/profile/std/avrc
+            bluedroid/btc/profile/std/spp
+            bluedroid/stack/btm
+            bluedroid/stack/btu
+            bluedroid/stack/gap
+            bluedroid/stack/gatt
+            bluedroid/stack/hcic
+            bluedroid/stack/l2cap
+            bluedroid/stack/sdp
+            bluedroid/stack/smp
+            bluedroid/stack/avct
+            bluedroid/stack/avrc
+            bluedroid/stack/avdt
+            bluedroid/stack/a2dp
+            bluedroid/stack/rfcomm
+            bluedroid/api
             )
     endif()
 endif()

components/lwip/CMakeLists.txt

@@ -36,3 +36,6 @@ set_source_files_properties(apps/dhcpserver.c
     PROPERTIES COMPILE_FLAGS
     "-Wno-unused-variable -Wno-unused-but-set-variable -Wno-type-limits"
     )
+set_source_files_properties(netif/ppp/pppos.c
+    PROPERTIES COMPILE_FLAGS
+    -Wno-type-limits)

components/partition_table/CMakeLists.txt

@@ -6,7 +6,7 @@ register_config_only_component()
 #
 if(CONFIG_PARTITION_TABLE_CUSTOM)
     # Custom filename expands any path relative to the project
-    get_filename_component(partition_csv "${CONFIG_PARTITION_TABLE_CUSTOM}" ABSOLUTE BASE_DIR "${PROJECT_PATH}")
+    get_filename_component(partition_csv "${CONFIG_PARTITION_TABLE_FILENAME}" ABSOLUTE BASE_DIR "${PROJECT_PATH}")
 else()
     # Other .csv files are always in the component directory
     set(partition_csv "${CMAKE_CURRENT_SOURCE_DIR}/${CONFIG_PARTITION_TABLE_FILENAME}")

docs/en/api-guides/build-system-cmake.rst

@@ -586,6 +586,12 @@ The file's contents will be added to the .rodata section in flash, and are avail
 
 The names are generated from the full name of the file, as given in ``COMPONENT_EMBED_FILES``. Characters /, ., etc. are replaced with underscores. The _binary prefix in the symbol name is added by objcopy and is the same for both text and binary files.
 
+To embed a file into a project, rather than a component, you can call the function ``target_add_binary_data`` like this::
+
+  target_add_binary_data(myproject.elf "main/data.bin" TEXT)
+
+Place this line after the ``project()`` line in your project CMakeLists.txt file. Replace ``myproject.elf`` with your project name. The final argument can be ``TEXT`` to embed a null-terminated string, or ``BINARY`` to embed the content as-is.
+
 For an example of using this technique, see :example:`protocols/https_request` - the certificate file contents are loaded from the text .pem file at compile time.
 
 

examples/bluetooth/ble_throughput/throughput_client/CMakeLists.txt

@@ -0,0 +1,10 @@
+# The following lines of boilerplate have to be in your project's
+# CMakeLists in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.5)
+
+set(MAIN_SRCS
+    main/example_ble_client_throughput.c
+    )
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+project(throughput_client_demo)

examples/bluetooth/ble_throughput/throughput_server/CMakeLists.txt

@@ -0,0 +1,10 @@
+# The following lines of boilerplate have to be in your project's
+# CMakeLists in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.5)
+
+set(MAIN_SRCS
+    main/example_ble_server_throughput.c
+    )
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+project(throughput_server_demo)

examples/bluetooth/bt_spp_vfs_acceptor/CMakeLists.txt

@@ -0,0 +1,10 @@
+# The following four lines of boilerplate have to be in your project's CMakeLists
+# in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.5)
+
+set(MAIN_SRCS
+    main/example_spp_vfs_acceptor_demo.c
+    main/spp_task.c)
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+project(bt_spp_vfs_acceptor_demo)

examples/bluetooth/bt_spp_vfs_initiator/CMakeLists.txt

@@ -0,0 +1,10 @@
+# The following four lines of boilerplate have to be in your project's CMakeLists
+# in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.5)
+
+set(MAIN_SRCS
+    main/example_spp_vfs_initiator_demo.c
+    main/spp_task.c)
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+project(bt_spp_vfs_initiator_demo)

examples/peripherals/spi_master/CMakeLists.txt

@@ -2,7 +2,16 @@
 # in this exact order for cmake to work correctly
 cmake_minimum_required(VERSION 3.5)
 
-set(MAIN_SRCS main/spi_master_example_main.c)
+set(MAIN_SRCS
+    main/spi_master_example_main.c
+    main/pretty_effect.c
+    main/decode_image.c)
 
 include($ENV{IDF_PATH}/tools/cmake/project.cmake)
 project(spi_master)
+
+# Embed the image into the final binary
+#
+# (If this was a component, we would set COMPONENT_EMBED_TXTFILES here.)
+target_add_binary_data(spi_master.elf
+    "main/image.jpg" BINARY)

examples/wifi/iperf/components/CMakeLists.txt

@@ -2,4 +2,6 @@ set(COMPONENT_ADD_INCLUDEDIRS .)
 
 set(COMPONENT_SRCDIRS .)
 
+set(COMPONENT_PRIV_REQUIRES lwip)
+
 register_component()

tools/ci/build_examples_cmake.sh

@@ -0,0 +1,173 @@
+#!/bin/bash
+#
+# Build all examples from the examples directory, out of tree to
+# ensure they can run when copied to a new directory.
+#
+# Runs as part of CI process.
+#
+# Assumes PWD is an out-of-tree build directory, and will copy examples
+# to individual subdirectories, one by one.
+#
+#
+# Without arguments it just builds all examples
+#
+# With one argument <JOB_NAME> it builds part of the examples. This is a useful for
+#   parallel execution in CI.
+#   <JOB_NAME> must look like this:
+#               <some_text_label>_<num>
+#   It scans .gitlab-ci.yaml to count number of jobs which have name "<some_text_label>_<num>"
+#   It scans the filesystem to count all examples
+#   Based on this, it decides to run qa set of examples.
+#
+
+# -----------------------------------------------------------------------------
+# Safety settings (see https://gist.github.com/ilg-ul/383869cbb01f61a51c4d).
+
+if [[ ! -z ${DEBUG_SHELL} ]]
+then
+  set -x # Activate the expand mode if DEBUG is anything but empty.
+fi
+
+set -o errexit # Exit if command failed.
+set -o pipefail # Exit if pipe failed.
+
+# Remove the initial space and instead use '\n'.
+IFS=$'\n\t'
+
+export PATH="$IDF_PATH/tools:$PATH"  # for idf.py
+
+# -----------------------------------------------------------------------------
+
+die() {
+    echo "${1:-"Unknown Error"}" 1>&2
+    exit 1
+}
+
+[ -z ${IDF_PATH} ] && die "IDF_PATH is not set"
+[ -z ${LOG_PATH} ] && die "LOG_PATH is not set"
+[ -d ${LOG_PATH} ] || mkdir -p ${LOG_PATH}
+
+set -o nounset # Exit if variable not set.
+
+echo "build_examples running in ${PWD}"
+
+# only 0 or 1 arguments
+[ $# -le 1 ] || die "Have to run as $(basename $0) [<JOB_NAME>]"
+
+export BATCH_BUILD=1
+export V=0 # only build verbose if there's an error
+
+shopt -s lastpipe # Workaround for Bash to use variables in loops (http://mywiki.wooledge.org/BashFAQ/024)
+
+RESULT=0
+FAILED_EXAMPLES=""
+RESULT_ISSUES=22  # magic number result code for issues found
+LOG_SUSPECTED=${LOG_PATH}/common_log.txt
+touch ${LOG_SUSPECTED}
+
+EXAMPLE_PATHS=$( find ${IDF_PATH}/examples/ -type f -name CMakeLists.txt | grep -v "/components/" | sort )
+
+if [ $# -eq 0 ]
+then
+    START_NUM=0
+    END_NUM=999
+else
+    JOB_NAME=$1
+
+    # parse text prefix at the beginning of string 'some_your_text_NUM'
+    # (will be 'some_your_text' without last '_')
+    JOB_PATTERN=$( echo ${JOB_NAME} | sed -n -r 's/^(.*)_[0-9]+$/\1/p' )
+    [ -z ${JOB_PATTERN} ] && die "JOB_PATTERN is bad"
+
+    # parse number 'NUM' at the end of string 'some_your_text_NUM'
+    JOB_NUM=$( echo ${JOB_NAME} | sed -n -r 's/^.*_([0-9]+)$/\1/p' )
+    [ -z ${JOB_NUM} ] && die "JOB_NUM is bad"
+
+    # count number of the jobs
+    NUM_OF_JOBS=$( grep -c -E "^${JOB_PATTERN}_[0-9]+:$" "${IDF_PATH}/.gitlab-ci.yml" )
+    [ -z ${NUM_OF_JOBS} ] && die "NUM_OF_JOBS is bad"
+
+    # count number of examples
+    NUM_OF_EXAMPLES=$(echo ${EXAMPLE_PATHS} | wc -l )
+    [ -z ${NUM_OF_EXAMPLES} ] && die "NUM_OF_EXAMPLES is bad"
+
+    # separate intervals
+    #57 / 5 == 12
+    NUM_OF_EX_PER_JOB=$(( (${NUM_OF_EXAMPLES} + ${NUM_OF_JOBS} - 1) / ${NUM_OF_JOBS} ))
+    [ -z ${NUM_OF_EX_PER_JOB} ] && die "NUM_OF_EX_PER_JOB is bad"
+
+    # ex.: [0; 12); [12; 24); [24; 36); [36; 48); [48; 60)
+    START_NUM=$(( ${JOB_NUM} * ${NUM_OF_EX_PER_JOB} ))
+    [ -z ${START_NUM} ] && die "START_NUM is bad"
+
+    END_NUM=$(( (${JOB_NUM} + 1) * ${NUM_OF_EX_PER_JOB} ))
+    [ -z ${END_NUM} ] && die "END_NUM is bad"
+fi
+
+build_example () {
+    local ID=$1
+    shift
+    local CMAKELISTS=$1
+    shift
+
+    local EXAMPLE_DIR=$(dirname "${CMAKELISTS}")
+    local EXAMPLE_NAME=$(basename "${EXAMPLE_DIR}")
+
+    echo "Building ${EXAMPLE_NAME} as ${ID}..."
+    mkdir -p "example_builds/${ID}"
+    cp -r "${EXAMPLE_DIR}" "example_builds/${ID}"
+    pushd "example_builds/${ID}/${EXAMPLE_NAME}"
+        # be stricter in the CI build than the default IDF settings
+        export EXTRA_CFLAGS="-Werror -Werror=deprecated-declarations"
+        export EXTRA_CXXFLAGS=${EXTRA_CFLAGS}
+
+        # build non-verbose first
+        local BUILDLOG=${LOG_PATH}/ex_${ID}_log.txt
+        touch ${BUILDLOG}
+
+        idf.py fullclean >>${BUILDLOG} 2>&1 &&
+        idf.py build >>${BUILDLOG} 2>&1 &&
+        cp build/flash_project_args build/download.config || # backwards compatible download.config filename
+        {
+            RESULT=$?; FAILED_EXAMPLES+=" ${EXAMPLE_NAME}" ;
+        }
+
+        cat ${BUILDLOG}
+    popd
+
+    grep -i "error\|warning" "${BUILDLOG}" 2>&1 | grep -v "error.c.obj" >> "${LOG_SUSPECTED}" || :
+}
+
+EXAMPLE_NUM=0
+
+for EXAMPLE_PATH in ${EXAMPLE_PATHS}
+do
+    if [[ $EXAMPLE_NUM -lt $START_NUM || $EXAMPLE_NUM -ge $END_NUM ]]
+    then
+        EXAMPLE_NUM=$(( $EXAMPLE_NUM + 1 ))
+        continue
+    fi
+    echo ">>> example [ ${EXAMPLE_NUM} ] - $EXAMPLE_PATH"
+
+    build_example "${EXAMPLE_NUM}" "${EXAMPLE_PATH}"
+
+    EXAMPLE_NUM=$(( $EXAMPLE_NUM + 1 ))
+done
+
+# show warnings
+echo -e "\nFound issues:"
+
+#       Ignore the next messages:
+# "error.o" or "-Werror" in compiler's command line
+# "reassigning to symbol" or "changes choice state" in sdkconfig
+sort -u "${LOG_SUSPECTED}" | \
+grep -v "library/error.o\|\ -Werror\|reassigning to symbol\|changes choice state" \
+    && RESULT=$RESULT_ISSUES \
+    || echo -e "\tNone"
+
+[ -z ${FAILED_EXAMPLES} ] || echo -e "\nThere are errors in the next examples: $FAILED_EXAMPLES"
+[ $RESULT -eq 0 ] || echo -e "\nFix all warnings and errors above to pass the test!"
+
+echo -e "\nReturn code = $RESULT"
+
+exit $RESULT

tools/cmake/scripts/data_file_embed_asm.cmake

@@ -8,7 +8,7 @@
 #
 # Set variables DATA_FILE, SOURCE_FILE, FILE_TYPE when running this.
 #
-# If FILE_TYPE is set to STRING, a null byte is appended to DATA_FILE's contents
+# If FILE_TYPE is set to TEXT, a null byte is appended to DATA_FILE's contents
 # before SOURCE_FILE is created.
 #
 # If FILE_TYPE is unset (or any other value), DATA_FILE is copied