Merge branch 'feature/bootloader_ignore_extra_component' into 'master'

bootloader: add the possibility to ignore extra components

Closes IDF-7825

See merge request espressif/esp-idf!24806

components/bootloader/project_include.cmake

@@ -124,13 +124,20 @@ idf_build_get_property(sdkconfig SDKCONFIG)
 idf_build_get_property(python PYTHON)
 idf_build_get_property(extra_cmake_args EXTRA_CMAKE_ARGS)
 
+# We cannot pass lists are a parameter to the external project without modifying the ';' spearator
+string(REPLACE ";" "|" BOOTLOADER_IGNORE_EXTRA_COMPONENT "${BOOTLOADER_IGNORE_EXTRA_COMPONENT}")
+
 externalproject_add(bootloader
     SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/subproject"
     BINARY_DIR "${BOOTLOADER_BUILD_DIR}"
+    # Modiying the list separator for the arguments, as such, we won't need to manually
+    # replace the new separator by the default ';' in the subproject
+    LIST_SEPARATOR |
     CMAKE_ARGS  -DSDKCONFIG=${sdkconfig} -DIDF_PATH=${idf_path} -DIDF_TARGET=${idf_target}
                 -DPYTHON_DEPS_CHECKED=1 -DPYTHON=${python}
                 -DEXTRA_COMPONENT_DIRS=${CMAKE_CURRENT_LIST_DIR}
                 -DPROJECT_SOURCE_DIR=${PROJECT_SOURCE_DIR}
+                -DIGNORE_EXTRA_COMPONENT=${BOOTLOADER_IGNORE_EXTRA_COMPONENT}
                 ${sign_key_arg} ${ver_key_arg}
                 ${extra_cmake_args}
     INSTALL_COMMAND ""

components/bootloader/subproject/CMakeLists.txt

@@ -41,11 +41,18 @@ if(EXISTS ${PROJECT_EXTRA_COMPONENTS})
     list(APPEND EXTRA_COMPONENT_DIRS "${PROJECT_EXTRA_COMPONENTS}")
 endif()
 
-# Consider each directory in project's bootloader_components as a component to be compiled
+if(IGNORE_EXTRA_COMPONENT)
+    # Prefix all entries of the list with ${PROJECT_EXTRA_COMPONENTS} absolute path
+    list(TRANSFORM IGNORE_EXTRA_COMPONENT
+         PREPEND "${PROJECT_EXTRA_COMPONENTS}/"
+         OUTPUT_VARIABLE EXTRA_COMPONENT_EXCLUDE_DIRS)
+endif()
+
+# Consider each directory in the project's bootloader_components as a component to be compiled
 file(GLOB proj_components RELATIVE ${PROJECT_EXTRA_COMPONENTS} ${PROJECT_EXTRA_COMPONENTS}/*)
 foreach(component ${proj_components})
-  # Only directories are considered as components
+  # Only directories are considered components
-  if(IS_DIRECTORY ${curdir}/${child})
+  if(IS_DIRECTORY "${PROJECT_EXTRA_COMPONENTS}/${component}" AND NOT ${component} IN_LIST IGNORE_EXTRA_COMPONENT)
     list(APPEND COMPONENTS ${component})
   endif()
 endforeach()
@@ -53,6 +60,7 @@ endforeach()
 set(BOOTLOADER_BUILD 1)
 include("${IDF_PATH}/tools/cmake/project.cmake")
 set(common_req log esp_rom esp_common esp_hw_support newlib)
+idf_build_set_property(EXTRA_COMPONENT_EXCLUDE_DIRS "${EXTRA_COMPONENT_EXCLUDE_DIRS}")
 idf_build_set_property(__COMPONENT_REQUIRES_COMMON "${common_req}")
 idf_build_set_property(__OUTPUT_SDKCONFIG 0)
 project(bootloader)

docs/en/api-guides/build-system.rst

@@ -141,6 +141,9 @@ An example project directory tree might look like this::
     - myProject/
                  - CMakeLists.txt
                  - sdkconfig
+                 - bootloader_components/ - boot_component/ - CMakeLists.txt
+                                                            - Kconfig
+                                                            - src1.c
                  - components/ - component1/ - CMakeLists.txt
                                              - Kconfig
                                              - src1.c
@@ -160,6 +163,8 @@ This example "myProject" contains the following elements:
 
 - "sdkconfig" project configuration file. This file is created/updated when ``idf.py menuconfig`` runs, and holds the configuration for all of the components in the project (including ESP-IDF itself). The "sdkconfig" file may or may not be added to the source control system of the project.
 
+- Optional "bootloader_components" directory contains components that need to be compiled and linked inside the bootloader project. A project does not have to contain custom bootloader components of this kind, but it can be useful in case the bootloader needs to be modified to embed new features.
+
 - Optional "components" directory contains components that are part of the project. A project does not have to contain custom components of this kind, but it can be useful for structuring reusable code or including third-party components that aren't part of ESP-IDF. Alternatively, ``EXTRA_COMPONENT_DIRS`` can be set in the top-level CMakeLists.txt to look for components in other places.
 
 - "main" directory is a special component that contains source code for the project itself. "main" is a default name, the CMake variable ``COMPONENT_DIRS`` includes this component but you can modify this variable. See the :ref:`renaming main <rename-main>` section for more info. If you have a lot of source files in your project, we recommend grouping most into components instead of putting them all in "main".
@@ -210,6 +215,8 @@ These variables all have default values that can be overridden for custom behavi
 
 - ``COMPONENTS``: A list of component names to build into the project. Defaults to all components found in the ``COMPONENT_DIRS`` directories. Use this variable to "trim down" the project for faster build times. Note that any component which "requires" another component via the REQUIRES or PRIV_REQUIRES arguments on component registration will automatically have it added to this list, so the ``COMPONENTS`` list can be very short.
 
+- ``BOOTLOADER_IGNORE_EXTRA_COMPONENT``: A list of components, placed in ``bootloader_components/``, that should be ignored by the bootloader compilation. Use this variable if a bootloader component needs to be included conditionally inside the project.
+
 Any paths in these variables can be absolute paths, or set relative to the project directory.
 
 To set these variables, use the `cmake set command <cmake set_>`_ ie ``set(VARIABLE "VALUE")``. The ``set()`` commands should be placed after the ``cmake_minimum(...)`` line but before the ``include(...)`` line.
@@ -672,6 +679,39 @@ The linker will provide a new symbol named ``__real_function_to_redefine`` which
 
 This mechanism is shown in the example :example:`build_system/wrappers`. Check :idf_file:`examples/build_system/wrappers/README.md` for more details.
 
+Override the default Bootloader
+-------------------------------
+
+Thanks to the optional ``bootloader_components`` directory present in your ESP-IDf project, it is possible to override the default ESP-IDF bootloader. To do so, a new ``bootloader_components/main`` component should be defined, which will make the project directory tree look like the following:
+
+    - myProject/
+                 - CMakeLists.txt
+                 - sdkconfig
+                 - bootloader_components/ - main/ - CMakeLists.txt
+                                                  - Kconfig
+                                                  - my_bootloader.c
+                 - main/       - CMakeLists.txt
+                               - app_main.c
+
+                 - build/
+
+
+Here the ``my_bootloader.c`` file becomes source code for the new bootloader, which means that it will need to perform all the required operations to set up and load the ``main`` application from flash.
+
+It is also possible to conditionally replace the bootloader depending on a certain condition, such as the target for example. This can be achieved thanks to the ``BOOTLOADER_IGNORE_EXTRA_COMPONENT`` CMake variable. This list can be used to tell the ESP-IDF bootloader project to ignore and not compile the given components present in ``bootloader_components``. For example, if one wants to use the default bootloader for ESP32 target, then ``myProject/CMakeLists.txt`` should look like the following::
+
+    include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+
+    if(${IDF_TARGET} STREQUAL "esp32")
+        set(BOOTLOADER_IGNORE_EXTRA_COMPONENT "main")
+    endif()
+
+    project(main)
+
+It is important to note that this can also be used for any other bootloader components than `main`. In all cases, the prefix `bootloader_component` must not be specified.
+
+See :example:`custom_bootloader/bootloader_override` for an example of overriding the default bootloader.
+
 .. _config_only_component:
 
 Configuration-Only Components

examples/custom_bootloader/bootloader_override/README.md

@@ -5,11 +5,11 @@
 
 (See the README.md file in the upper level for more information about bootloader examples.)
 
-The purpose of this example is to show how to override the second stage bootloader from a regular project.
+The purpose of this example is to show how to override the second-stage bootloader from a regular project.
 
-**NOTE**: Functions called during the loading stage of the bootloader are expected to be placed in the iram_loader_seg to avoid being overwritten during loading. If you are overriding functions which are called during this stage then special care needs to be taken to avoid issues, e.g. by providing your own linkerscript which places the required functions in the correct sections.
+**NOTE**: Functions called during the loading stage of the bootloader are expected to be placed in the iram_loader_seg to avoid being overwritten during loading. If you are overriding functions that are called during this stage then special care needs to be taken to avoid issues, e.g. by providing your linker script which places the required functions in the correct sections.
 
-## How to use example
+## How to use this example
 
 Simply compile it:
 ```
@@ -21,17 +21,17 @@ And flash it with the following commands:
 idf.py flash
 ```
 
-This custom bootloader does not do more than the older bootloader, it only prints an extra message on start up:
+This custom bootloader does not do more than the older bootloader, it only prints an extra message on startup:
 ```
 [boot] Custom bootloader message defined in the KConfig file.
 ```
 
-## Organisation of this example
+## Organization of this example
 
 This project contains an application, in the `main` directory that represents a user program.
-It also contains a `bootloader_components` directory that, as it name states, contains a component that will override the current bootloader implementation.
+It also contains a `bootloader_components` directory that, as its name states, contains a component that will override the current bootloader implementation.
 
-Below is a short explanation of files in the project folder.
+Below is a short explanation of the files in the project folder.
 
 ```
 ├── CMakeLists.txt
@@ -48,5 +48,24 @@ Below is a short explanation of files in the project folder.
 └── README.md                  This is the file you are currently reading
 ```
 
-As stated in the `README.md` file in the upper level, when the bootloader components is named `main`, it overrides
+As stated in the `README.md` file in the upper level, when the bootloader components are named `main`, it overrides
 the whole second stage bootloader code.
+
+## Conditionally override the bootloader
+
+In case you only want to override the bootloader under a certain condition (target-dependent, KConfig option, etc...), it is possible to let the bootloader project know that the `bootloader_components/main` component shall be ignored.
+
+For example, if the custom bootloader shall not be compiled for ESP32-C3 targets, which should use the default ESP-IDF one, the `CMakeLists.txt` file in this current example must look like this:
+```
+cmake_minimum_required(VERSION 3.16)
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+
+if(${IDF_TARGET} STREQUAL "esp32c3")
+    set(BOOTLOADER_IGNORE_EXTRA_COMPONENT "main")
+endif()
+
+project(main)
+```
+
+It is important to note that this can also be used for any other bootloader components than `main`. In all cases, the prefix `bootloader_component` must not be specified.

tools/cmake/project.cmake

@@ -311,10 +311,11 @@ function(__project_init components_var test_components_var)
         if(EXISTS ${component_dir}/CMakeLists.txt)
             idf_build_component(${component_dir})
         else()
+            idf_build_get_property(exclude_dirs EXTRA_COMPONENT_EXCLUDE_DIRS)
             # otherwise, check whether the subfolders are potential idf components
             file(GLOB component_dirs ${component_dir}/*)
             foreach(component_dir ${component_dirs})
-                if(IS_DIRECTORY ${component_dir})
+                if(IS_DIRECTORY ${component_dir} AND NOT ${component_dir} IN_LIST exclude_dirs)
                     __component_dir_quick_check(is_component ${component_dir})
                     if(is_component)
                         idf_build_component(${component_dir})

tools/test_build_system/test_bootloader.py

@@ -5,7 +5,7 @@ import shutil
 from pathlib import Path
 from typing import Union
 
-from test_build_system_helpers import EnvDict, IdfPyFunc, file_contains
+from test_build_system_helpers import EnvDict, IdfPyFunc, bin_file_contains, file_contains, replace_in_file
 
 
 def get_two_header_bytes(file_path: Union[str, Path]) -> str:
@@ -31,6 +31,29 @@ def test_bootloader_custom_overrides_original(test_app_copy: Path, idf_py: IdfPy
                          str(test_app_copy / 'components' / 'bootloader' / 'subproject' / 'main' / 'bootloader_start.c'))
 
 
+def test_bootloader_custom_ignores_extra_component(test_app_copy: Path, idf_py: IdfPyFunc, default_idf_env: EnvDict) -> None:
+    logging.info('Custom bootloader can ignore extra components')
+    idf_path = Path(default_idf_env.get('IDF_PATH'))
+    # Import the main bootloader component that overrides the default one
+    shutil.copytree(idf_path / 'examples' / 'custom_bootloader' / 'bootloader_override' / 'bootloader_components',
+                    test_app_copy / 'bootloader_components')
+    # Compile it normally and make sure the bootloader is overridden for now
+    idf_py('bootloader')
+    # Search for 'Custom bootloader message defined in the KConfig file.' in the resulting binary
+    # which is the default value for EXAMPLE_BOOTLOADER_WELCOME_MESSAGE Kconfig option.
+    default_message = bytes('Custom bootloader message defined in the KConfig file.', 'ascii')
+    assert bin_file_contains(test_app_copy / 'build' / 'bootloader' / 'bootloader.bin', default_message)
+    # Clean the project
+    idf_py('fullclean')
+    # Ignore bootloader component and rebuild
+    replace_in_file(test_app_copy / 'CMakeLists.txt',
+                    '# placeholder_after_include_project_cmake',
+                    'set(BOOTLOADER_IGNORE_EXTRA_COMPONENT "main")')
+    idf_py('bootloader')
+    # The overridden message must not be present anymore
+    assert not bin_file_contains(test_app_copy / 'build' / 'bootloader' / 'bootloader.bin', default_message)
+
+
 def test_bootloader_correctly_set_image_header(test_app_copy: Path, idf_py: IdfPyFunc) -> None:
     logging.info('Flash size is correctly set in the bootloader image header')
     # Build with the default 2MB setting

tools/test_build_system/test_build_system_helpers/__init__.py

@@ -2,8 +2,8 @@
 # SPDX-License-Identifier: Apache-2.0
 from .build_constants import ALL_ARTIFACTS, APP_BINS, BOOTLOADER_BINS, JSON_METADATA, PARTITION_BIN
 from .editing import append_to_file, replace_in_file
-from .idf_utils import (EXT_IDF_PATH, EnvDict, IdfPyFunc, file_contains, find_python, get_idf_build_env, run_cmake,
+from .idf_utils import (EXT_IDF_PATH, EnvDict, IdfPyFunc, bin_file_contains, file_contains, find_python,
-                        run_cmake_and_build, run_idf_py)
+                        get_idf_build_env, run_cmake, run_cmake_and_build, run_idf_py)
 from .snapshot import Snapshot, get_snapshot
 
 __all__ = [
@@ -11,5 +11,5 @@ __all__ = [
     'get_idf_build_env', 'run_idf_py', 'EXT_IDF_PATH', 'EnvDict', 'IdfPyFunc',
     'Snapshot', 'get_snapshot', 'run_cmake', 'APP_BINS', 'BOOTLOADER_BINS',
     'PARTITION_BIN', 'JSON_METADATA', 'ALL_ARTIFACTS',
-    'run_cmake_and_build', 'find_python', 'file_contains'
+    'run_cmake_and_build', 'find_python', 'file_contains', 'bin_file_contains'
 ]

tools/test_build_system/test_build_system_helpers/idf_utils.py

@@ -149,3 +149,14 @@ def file_contains(filename: Union[str, Path], what: Union[str, Pattern]) -> bool
             return what in data
         else:
             return re.search(what, data) is not None
+
+
+def bin_file_contains(filename: Union[str, Path], what: bytearray) -> bool:
+    """
+    Returns true if the binary file contains the given string
+    :param filename: path to file where lookup is executed
+    :param what: searched bytes
+    """
+    with open(filename, 'rb') as f:
+        data = f.read()
+        return data.find(what) != -1