bootloader: override the 2nd stage bootloader

Add the possibility to have user bootloader components. This is performed
from an application/project, by creating bootloader components. To do so,
it is required to create a `bootloader_component` directory containing
the custom modules to be compiled with the bootloader.

Thanks to this, two solutions are available to override the bootloader now:
- Using hooks within a user bootloader component
- Using a user defined `main` bootloader component to totally override the
  old implementation

Please check the two new examples in `examples/custom_bootloader`

* Closes https://github.com/espressif/esp-idf/issues/7043

components/bootloader/Makefile.projbuild

@@ -27,6 +27,9 @@ BOOTLOADER_OFFSET := 0x1000
 # NB: Some variables are cleared in the environment, not
 # overriden, because they need to be re-defined in the child
 # project.
+#
+# Pass PROJECT_PATH variable, it will let the subproject look
+# for user defined bootloader component(s).
 BOOTLOADER_MAKE= +\
 	PROJECT_PATH= \
 	COMPONENT_DIRS= \
@@ -35,7 +38,8 @@ BOOTLOADER_MAKE= +\
 	BUILD_DIR_BASE=$(BOOTLOADER_BUILD_DIR) \
 	TEST_COMPONENTS= \
 	TESTS_ALL= \
-	EXCLUDE_COMPONENTS=
+	EXCLUDE_COMPONENTS= \
+	PROJECT_SOURCE_DIR=$(PROJECT_PATH)
 
 .PHONY: bootloader-clean bootloader-flash bootloader-list-components bootloader $(BOOTLOADER_BIN)
 

components/bootloader/project_include.cmake

@@ -117,6 +117,7 @@ externalproject_add(bootloader
     CMAKE_ARGS  -DSDKCONFIG=${sdkconfig} -DIDF_PATH=${idf_path} -DIDF_TARGET=${idf_target}
                 -DPYTHON_DEPS_CHECKED=1 -DPYTHON=${python}
                 -DEXTRA_COMPONENT_DIRS=${CMAKE_CURRENT_LIST_DIR}
+                -DPROJECT_SOURCE_DIR=${PROJECT_SOURCE_DIR}
                 ${sign_key_arg} ${ver_key_arg}
                 # LEGACY_INCLUDE_COMMON_HEADERS has to be passed in via cache variable since
                 # the bootloader common component requirements depends on this and

components/bootloader/subproject/CMakeLists.txt

@@ -33,6 +33,21 @@ set(COMPONENTS
     efuse
     esp_system
     newlib)
+
+# Make EXTRA_COMPONENT_DIRS variable to point to the bootloader_components directory
+# of the project being compiled
+set(PROJECT_EXTRA_COMPONENTS "${PROJECT_SOURCE_DIR}/bootloader_components")
+set(EXTRA_COMPONENT_DIRS "${PROJECT_EXTRA_COMPONENTS}" APPEND)
+
+# Consider each directory in project's bootloader_components as a component to be compiled
+file(GLOB proj_components RELATIVE ${PROJECT_EXTRA_COMPONENTS} ${PROJECT_EXTRA_COMPONENTS}/*)
+foreach(component ${proj_components})
+  # Only directories are considered as components
+  if(IS_DIRECTORY ${curdir}/${child})
+    list(APPEND COMPONENTS ${component})
+  endif()
+endforeach()
+
 set(BOOTLOADER_BUILD 1)
 include("${IDF_PATH}/tools/cmake/project.cmake")
 set(common_req log esp_rom esp_common esp_hw_support hal newlib)

components/bootloader/subproject/main/CMakeLists.txt

@@ -4,8 +4,10 @@ idf_component_register(SRCS "bootloader_start.c"
 idf_build_get_property(target IDF_TARGET)
 set(scripts "ld/${target}/bootloader.ld")
 
-if(NOT CONFIG_IDF_TARGET_ESP32C3 AND NOT CONFIG_IDF_TARGET_ESP32H2)
+if(NOT CONFIG_IDF_TARGET_ESP32H2)
     list(APPEND scripts "ld/${target}/bootloader.rom.ld")
 endif()
 
 target_linker_script(${COMPONENT_LIB} INTERFACE "${scripts}")
+
+target_link_libraries(${COMPONENT_LIB} INTERFACE "-u bootloader_hooks_include")

components/bootloader/subproject/main/bootloader_hooks.h

@@ -0,0 +1,38 @@
+// Copyright 2015-2021 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef BOOTLOADER_HOOKS_H
+#define BOOTLOADER_HOOKS_H
+
+/**
+ * @file The 2nd stage bootloader can be overriden or completed by an application.
+ * The functions declared here are weak, and thus, are meant to be defined by a user
+ * project, if required.
+ * Please check `custom_bootloader` ESP-IDF examples for more details about this feature.
+ */
+
+
+/**
+ * @brief Function executed *before* the second stage bootloader initialization,
+ * if provided.
+ */
+void __attribute__((weak)) bootloader_before_init(void);
+
+/**
+ * @brief Function executed *after* the second stage bootloader initialization,
+ * if provided.
+ */
+void __attribute__((weak)) bootloader_after_init(void);
+
+#endif // BOOTLOADER_HOOKS_H

components/bootloader/subproject/main/bootloader_start.c

@@ -16,6 +16,7 @@
 #include "bootloader_init.h"
 #include "bootloader_utility.h"
 #include "bootloader_common.h"
+#include "bootloader_hooks.h"
 
 static const char *TAG = "boot";
 
@@ -29,11 +30,21 @@ static int selected_boot_partition(const bootloader_state_t *bs);
  */
 void __attribute__((noreturn)) call_start_cpu0(void)
 {
+    // (0. Call the before-init hook, if available)
+    if (bootloader_before_init) {
+        bootloader_before_init();
+    }
+
     // 1. Hardware initialization
     if (bootloader_init() != ESP_OK) {
         bootloader_reset();
     }
 
+    // (1.1 Call the after-init hook, if available)
+    if (bootloader_after_init) {
+        bootloader_after_init();
+    }
+
 #ifdef CONFIG_BOOTLOADER_SKIP_VALIDATE_IN_DEEP_SLEEP
     // If this boot is a wake up from the deep sleep then go to the short way,
     // try to load the application which worked before deep sleep.

components/bootloader/subproject/main/ld/esp32c3/bootloader.rom.ld

@@ -0,0 +1 @@
+/* No definition for ESP32-C3 target */

docs/en/api-guides/bootloader.rst

@@ -105,10 +105,14 @@ The bootloader has the :ref:`CONFIG_BOOTLOADER_SKIP_VALIDATE_IN_DEEP_SLEEP` opti
 Custom bootloader
 -----------------
 
-The current bootloader implementation allows a project to override it. To do this, you must copy the directory ``/esp-idf/components/bootloader`` to your project components directory and then edit ``/your_project/components/bootloader/subproject/main/bootloader_start.c``.
+The current bootloader implementation allows a project to extend it or modify it. There are two ways of doing it: by implementing hooks or by overriding it.
+Both ways are presented in :example:`custom_bootloader` folder in ESP-IDF examples:
 
-In the bootloader space, you cannot use the drivers and functions from other components. If necessary, then the required functionality should be placed in the project's ``bootloader`` directory (note that this will increase its size).
+* `bootloader_hooks` which presents how to connect some hooks to the bootloader initialization
+* `bootloader_override` which presents how to override the bootloader implementation
+
+In the bootloader space, you cannot use the drivers and functions from other components. If necessary, then the required functionality should be placed in the project's `bootloader_components` directory (note that this will increase its size).
 
 If the bootloader grows too large then it can collide with the partition table, which is flashed at offset 0x8000 by default. Increase the :ref:`partition table offset <CONFIG_PARTITION_TABLE_OFFSET>` value to place the partition table later in the flash. This increases the space available for the bootloader.
 
-.. note:: The first time you copy the bootloader into an existing project, the project may fail to build as paths have changed unexpectedly. If this happens, run ``idf.py fullclean`` (or delete the project build directory) and then build again.
+.. note:: Customize the bootloader by using either method is only supported with CMake build system (i.e. not supported with legacy Make build system).

examples/custom_bootloader/README.md

@@ -0,0 +1,35 @@
+# Custom bootloader examples
+
+The following directory contains two examples presenting the different ways
+we provide in order to override the second stage bootloader or complete it
+with few hooks.
+
+## Extending the bootloader
+
+In both cases, a project can define custom bootloader components by creating
+them within a directory called `bootloader_components`.
+
+Naming one of them `main` would let the compiler entirely override the
+2nd stage bootloader with the implementation provided.
+
+The bootloader components containing the hooks can have any name, as long
+as it is part of `bootloader_components`, it will be taken into account
+in the build.
+
+## Hooks vs overriding the bootloader
+
+In brief, using hooks will let the application add code to the bootloader.
+They cannot replace the code that is already executed within bootloader.
+
+Two hooks are available at the moment, the first one is called before the
+initialization, and the second one is performed after the bootloader
+initialization, before choosing and loading any partition. The
+signature for these hooks can be found in `bootloader_hooks.h` file in
+`components/bootloader/subproject/main/`.
+
+
+On the other hand, overriding the bootloader offers the possibility to
+totally or partially re-write it, in order to include, remove or modify
+parts of it. Thanks to this, it will be fully customizable.
+This shall only be used if heavy changes are required and they cannot
+be done with hooks or within an application.

examples/custom_bootloader/bootloader_hooks/CMakeLists.txt

@@ -0,0 +1,8 @@
+# For more information about build system see
+# https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/build-system.html
+# The following five lines of boilerplate have to be in your project's
+# CMakeLists in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.5)
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+project(main)

examples/custom_bootloader/bootloader_hooks/Makefile

@@ -0,0 +1,9 @@
+
+#
+# This is a project Makefile. It is assumed the directory this Makefile resides
+# in is a project subdirectory.
+#
+
+PROJECT_NAME := bootloader_hooks
+
+include $(IDF_PATH)/make/project.mk

examples/custom_bootloader/bootloader_hooks/README.md

@@ -0,0 +1,57 @@
+# Bootloader hooks
+
+(See the README.md file in the upper level for more information about bootloader examples.)
+
+The purpose of this example is to show how to add hooks to the 2nd stage bootloader.
+
+## Usage of this example:
+
+Simply compile it:
+```
+idf.py build
+```
+
+Then flash it and open the monitor with the following command:
+```
+idf.py flash monitor
+```
+
+If everything went well, the bootloader output should be as followed:
+```
+I (24) HOOK: This hook is called BEFORE bootloader initialization
+I (37) boot: [...]
+[...]
+I (60) HOOK: This hook is called AFTER bootloader initialization
+```
+
+And finally the application will start and show the message:
+```
+User application is loaded and running.
+```
+
+## Organisation of this example
+
+This project contains an application, in the `main` directory that represents a user program.
+It also contains a `bootloader_components` that, as it name states, contains a component compiled and linked with the bootloader.
+
+Below is a short explanation of files in the project folder.
+
+```
+├── CMakeLists.txt
+├── main
+│   ├── CMakeLists.txt
+│   └── main.c                 User application
+├── bootloader_components
+│   └── my_boot_hooks
+│       ├── CMakeLists.txt   
+│       └── hooks.c            Implementation of the hooks to execute on boot
+└── README.md                  This is the file you are currently reading
+```
+Bootloader hooks are **not** supported in legacy `make` build system. They are only supported with `CMake` build system.
+
+## Note about including weak symbols
+
+The components in ESP-IDF are compiled as static libraries. Moreover, the bootloaders' hooks are declared as `weak`. Thus, when
+defining hooks for the bootloader, we **must** tell the compiler to always include our library (`my_boot_hooks`) in the link process.
+To achieve this, we need to define an extra symbol: `bootloader_hooks_include`. In our case, this symbol is a function defined in
+`bootloader_components/my_boot_hooks/hooks.c`. This will make the linker include all the symbols contained in that file.

examples/custom_bootloader/bootloader_hooks/bootloader_components/my_boot_hooks/CMakeLists.txt

@@ -0,0 +1,8 @@
+idf_component_register(SRCS "hooks.c")
+
+# We need to force GCC to integrate this static library into the
+# bootloader link. Indeed, by default, as the hooks in the bootloader are weak,
+# the linker would just ignore the symbols in the extra. (i.e. not strictly
+# required)
+# To do so, we need to define the symbol (function) `bootloader_hooks_include`
+# within hooks.c source file.

examples/custom_bootloader/bootloader_hooks/bootloader_components/my_boot_hooks/hooks.c

@@ -0,0 +1,19 @@
+#include "esp_log.h"
+
+/* Function used to tell the linker to include this file
+ * with all its symbols.
+ */
+void bootloader_hooks_include(void){
+}
+
+
+void bootloader_before_init(void) {
+    /* Keep in my mind that a lot of functions cannot be called from here
+     * as system initialization has not been performed yet, including
+     * BSS, SPI flash, or memory protection. */
+    ESP_LOGI("HOOK", "This hook is called BEFORE bootloader initialization");
+}
+
+void bootloader_after_init(void) {
+    ESP_LOGI("HOOK", "This hook is called AFTER bootloader initialization");
+}

examples/custom_bootloader/bootloader_hooks/example_test.py

@@ -0,0 +1,18 @@
+from __future__ import print_function
+
+import ttfw_idf
+
+
+@ttfw_idf.idf_example_test(env_tag='Example_GENERIC', target=['esp32', 'esp32s2', 'esp32c3'])
+def test_custom_bootloader_hooks_example(env, _):  # type: ignore
+    # Test with default build configurations
+    dut = env.get_dut('main', 'examples/custom_bootloader/bootloader_hooks')
+    dut.start_app()
+
+    # Expect to read both hooks messages
+    dut.expect('This hook is called BEFORE bootloader initialization')
+    dut.expect('This hook is called AFTER bootloader initialization')
+
+
+if __name__ == '__main__':
+    test_custom_bootloader_hooks_example()

examples/custom_bootloader/bootloader_hooks/main/CMakeLists.txt

@@ -0,0 +1,2 @@
+idf_component_register(SRCS "bootloader_hooks_example_main.c"
+                    INCLUDE_DIRS ".")

examples/custom_bootloader/bootloader_hooks/main/bootloader_hooks_example_main.c

@@ -0,0 +1,11 @@
+#include <stdio.h>
+
+void app_main(void)
+{
+    /**
+     * Nothing special is done here, everything interesting in this example
+     * is done in the custom bootloader code, located in:
+     * `bootloader_components/my_boot_hooks/hooks.c`
+     */
+    printf("User application is loaded and running.\n");
+}

examples/custom_bootloader/bootloader_hooks/main/component.mk

@@ -0,0 +1,4 @@
+#
+# "main" pseudo-component makefile.
+#
+# (Uses default behaviour of compiling all source files in directory, adding 'include' to include path.)

examples/custom_bootloader/bootloader_override/CMakeLists.txt

@@ -0,0 +1,8 @@
+# For more information about build system see
+# https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/build-system.html
+# The following five lines of boilerplate have to be in your project's
+# CMakeLists in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.5)
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+project(main)

examples/custom_bootloader/bootloader_override/Makefile

@@ -0,0 +1,8 @@
+#
+# This is a project Makefile. It is assumed the directory this Makefile resides in is a
+# project subdirectory.
+#
+
+PROJECT_NAME := bootloader_override
+
+include $(IDF_PATH)/make/project.mk

examples/custom_bootloader/bootloader_override/README.md

@@ -0,0 +1,49 @@
+# Bootloader override
+
+(See the README.md file in the upper level for more information about bootloader examples.)
+
+The purpose of this example is to show how to override the second stage bootloader from a regular project.
+
+**NOTE**: Overriding the bootloader is not supported with `Makefile` build system, it is only available with `CMake`.
+
+## How to use example
+
+Simply compile it:
+```
+idf.py build
+```
+
+And flash it with the following commands:
+```
+idf.py flash
+```
+
+This custom bootloader does not do more than the older bootloader, it only prints an extra message on start up:
+```
+[boot] Custom bootloader has been initialized correctly.
+```
+
+## Organisation of this example
+
+This project contains an application, in the `main` directory that represents a user program.
+It also contains a `bootloader_components` directory that, as it name states, contains a component that will override the current bootloader implementation.
+
+Below is a short explanation of files in the project folder.
+
+```
+├── CMakeLists.txt
+├── main
+│   ├── CMakeLists.txt
+│   └── main.c                 User application
+├── bootloader_components
+│   └── main
+│       ├── component.mk          
+│       ├── CMakeLists.txt   
+│       ├── ld/
+│       │   └── ...
+│       └── bootloader_start.c Implementation of the second stage bootloader
+└── README.md                  This is the file you are currently reading
+```
+
+As stated in the `README.md` file in the upper level, when the bootloader components is named `main`, it overrides
+the whole second stage bootloader code.

examples/custom_bootloader/bootloader_override/bootloader_components/main/CMakeLists.txt

@@ -0,0 +1,9 @@
+idf_component_register(SRCS "bootloader_start.c"
+                    REQUIRES bootloader bootloader_support)
+
+idf_build_get_property(target IDF_TARGET)
+# Use the linker script files from the actual bootloader
+set(scripts "${IDF_PATH}/components/bootloader/subproject/main/ld/${target}/bootloader.ld"
+            "${IDF_PATH}/components/bootloader/subproject/main/ld/${target}/bootloader.rom.ld")
+
+target_linker_script(${COMPONENT_LIB} INTERFACE "${scripts}")

examples/custom_bootloader/bootloader_override/bootloader_components/main/bootloader_start.c

@@ -0,0 +1,67 @@
+/*
+ * SPDX-FileCopyrightText: 2015-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#include <stdbool.h>
+#include "esp_log.h"
+#include "bootloader_init.h"
+#include "bootloader_utility.h"
+#include "bootloader_common.h"
+
+static const char* TAG = "boot";
+
+static int select_partition_number(bootloader_state_t *bs);
+
+/*
+ * We arrive here after the ROM bootloader finished loading this second stage bootloader from flash.
+ * The hardware is mostly uninitialized, flash cache is down and the app CPU is in reset.
+ * We do have a stack, so we can do the initialization in C.
+ */
+void __attribute__((noreturn)) call_start_cpu0(void)
+{
+    // 1. Hardware initialization
+    if (bootloader_init() != ESP_OK) {
+        bootloader_reset();
+    }
+
+#ifdef CONFIG_BOOTLOADER_SKIP_VALIDATE_IN_DEEP_SLEEP
+    // If this boot is a wake up from the deep sleep then go to the short way,
+    // try to load the application which worked before deep sleep.
+    // It skips a lot of checks due to it was done before (while first boot).
+    bootloader_utility_load_boot_image_from_deep_sleep();
+    // If it is not successful try to load an application as usual.
+#endif
+
+    // 2. Select the number of boot partition
+    bootloader_state_t bs = {0};
+    int boot_index = select_partition_number(&bs);
+    if (boot_index == INVALID_INDEX) {
+        bootloader_reset();
+    }
+
+    // 2.1 Print a custom message!
+    esp_rom_printf("[%s] Custom bootloader has been initialized correctly.\n", TAG);
+
+    // 3. Load the app image for booting
+    bootloader_utility_load_boot_image(&bs, boot_index);
+}
+
+// Select the number of boot partition
+static int select_partition_number(bootloader_state_t *bs)
+{
+    // 1. Load partition table
+    if (!bootloader_utility_load_partition_table(bs)) {
+        ESP_LOGE(TAG, "load partition table error!");
+        return INVALID_INDEX;
+    }
+
+    // 2. Select the number of boot partition
+    return bootloader_utility_get_selected_boot_partition(bs);
+}
+
+// Return global reent struct if any newlib functions are linked to bootloader
+struct _reent *__getreent(void)
+{
+    return _GLOBAL_REENT;
+}

examples/custom_bootloader/bootloader_override/example_test.py

@@ -0,0 +1,20 @@
+from __future__ import print_function
+
+import ttfw_idf
+
+
+@ttfw_idf.idf_example_test(env_tag='Example_GENERIC', target=['esp32', 'esp32s2', 'esp32c3'])
+def test_custom_bootloader_impl_example(env, _):  # type: ignore
+    # Test with default build configurations
+    dut = env.get_dut('main', 'examples/custom_bootloader/bootloader_override')
+    dut.start_app()
+
+    # Expect to read a message from the custom bootloader
+    dut.expect('Custom bootloader has been initialized correctly.')
+
+    # Expect to read a message from the user application
+    dut.expect('Application started!')
+
+
+if __name__ == '__main__':
+    test_custom_bootloader_impl_example()

examples/custom_bootloader/bootloader_override/main/CMakeLists.txt

@@ -0,0 +1,2 @@
+idf_component_register(SRCS "bootloader_override_example_main.c"
+                    INCLUDE_DIRS ".")

examples/custom_bootloader/bootloader_override/main/bootloader_override_example_main.c

@@ -0,0 +1,16 @@
+/*
+ * SPDX-FileCopyrightText: 2015-2021 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#include <stdio.h>
+
+void app_main(void)
+{
+    /**
+     * Nothing special is done here, everything interesting in this example
+     * is done in the custom bootloader code, located in:
+     * `bootloader_components/main/bootloader_start.c`
+     */
+    printf("Application started!\n");
+}

examples/custom_bootloader/bootloader_override/main/component.mk

@@ -0,0 +1,4 @@
+#
+# "main" pseudo-component makefile.
+#
+# (Uses default behaviour of compiling all source files in directory, adding 'include' to include path.)