esp_system: startup flow modifications

Changes the startup flow to the ff:

hardware -> core libraries init  -> other libraries init -> os
init (optional) -> app_main

- hardware init resides in the port layer, and is the entry point

- core libraries init executes init functions of core components

- other libraries init executes init functions of other components (weak
references)

- after other lib is init, the app_main function is called, however,

 an OS can wrap the real call to app_main to init its own stuff, and
 *then* call the real app_main

components/esp_system/CMakeLists.txt

@@ -1,7 +1,7 @@
 idf_component_register(SRCS "panic.c" "system_api.c" "startup.c"
                     INCLUDE_DIRS include
                     PRIV_INCLUDE_DIRS private_include
-                    PRIV_REQUIRES spi_flash app_update 
+                    PRIV_REQUIRES spi_flash app_update
                                 # requirements due to startup code
                                 nvs_flash pthread app_trace
                     LDFRAGMENTS "linker.lf")
@@ -9,4 +9,5 @@ idf_component_register(SRCS "panic.c" "system_api.c" "startup.c"
 add_subdirectory(port)
 
 # Rely on user code to define app_main
-target_link_libraries(${COMPONENT_LIB} INTERFACE "-u app_main")
+target_link_libraries(${COMPONENT_LIB} INTERFACE "-u app_main")
+target_link_libraries(${COMPONENT_LIB} INTERFACE "-u app_mainX")

components/esp_system/port/cpu_start.c

@@ -67,10 +67,6 @@
 #endif // CONFIG_APP_BUILD_TYPE_ELF_RAM
 #endif
 
-#if !CONFIG_FREERTOS_UNICORE
-static bool app_cpu_started = false;
-#endif //!CONFIG_FREERTOS_UNICORE
-
 extern int _bss_start;
 extern int _bss_end;
 extern int _rtc_bss_start;
@@ -91,18 +87,20 @@ extern int _iram_bss_end;
 #endif
 #endif // CONFIG_IDF_TARGET_ESP32
 
-extern void start_cpu0(void);
+#include "startup_internal.h"
-#if !CONFIG_FREERTOS_UNICORE
-extern void start_cpu1(void);
-#endif //!CONFIG_FREERTOS_UNICORE
 
-extern int _init_start;
+static volatile bool s_cpu_up[SOC_CPU_CORES_NUM] = { false };
+static volatile bool s_cpu_inited[SOC_CPU_CORES_NUM] = { false };
+static volatile bool s_resume_cores;
 
-static const char* TAG = "cpu_start";
+// If CONFIG_SPIRAM_IGNORE_NOTFOUND is set and external RAM is not found or errors out on testing, this is set to false.
-
-//If CONFIG_SPIRAM_IGNORE_NOTFOUND is set and external RAM is not found or errors out on testing, this is set to false.
 bool g_spiram_ok = true;
 
+void startup_resume_other_cores(void)
+{
+    s_resume_cores = true;
+}
+
 static void intr_matrix_clear(void)
 {
 #if CONFIG_IDF_TARGET_ESP32
@@ -112,19 +110,11 @@ static void intr_matrix_clear(void)
     for (int i = ETS_WIFI_MAC_INTR_SOURCE; i < ETS_MAX_INTR_SOURCE; i++) {
 #endif
         intr_matrix_set(0, i, ETS_INVALID_INUM);
-#if !CONFIG_FREERTOS_UNICORE
         intr_matrix_set(1, i, ETS_INVALID_INUM);
-#endif
     }
 }
 
-#if !CONFIG_FREERTOS_UNICORE
+#if SOC_CPU_CORES_NUM > 1
-static void wdt_reset_cpu1_info_enable(void)
-{
-    DPORT_REG_SET_BIT(DPORT_APP_CPU_RECORD_CTRL_REG, DPORT_APP_CPU_PDEBUG_ENABLE | DPORT_APP_CPU_RECORD_ENABLE);
-    DPORT_REG_CLR_BIT(DPORT_APP_CPU_RECORD_CTRL_REG, DPORT_APP_CPU_RECORD_ENABLE);
-}
-
 void IRAM_ATTR call_start_cpu1(void)
 {
     cpu_hal_set_vecbase(&_init_start);
@@ -142,9 +132,11 @@ void IRAM_ATTR call_start_cpu1(void)
     uart_tx_switch(CONFIG_ESP_CONSOLE_UART_NUM);
 #endif
 
-    wdt_reset_cpu1_info_enable();
+    DPORT_REG_SET_BIT(DPORT_APP_CPU_RECORD_CTRL_REG, DPORT_APP_CPU_PDEBUG_ENABLE | DPORT_APP_CPU_RECORD_ENABLE);
+    DPORT_REG_CLR_BIT(DPORT_APP_CPU_RECORD_CTRL_REG, DPORT_APP_CPU_RECORD_ENABLE);
+
+    s_cpu_up[1] = true;
     ESP_EARLY_LOGI(TAG, "App cpu up.");
-    app_cpu_started = 1;
 
     //Take care putting stuff here: if asked, FreeRTOS will happily tell you the scheduler
     //has started, but it isn't active *on this CPU* yet.
@@ -156,7 +148,13 @@ void IRAM_ATTR call_start_cpu1(void)
 #endif
 #endif
 
-    start_cpu1();
+    s_cpu_inited[1] = true;
+
+    while(!s_resume_cores) {
+        cpu_hal_delay_us(100);
+    }
+
+    SYS_STARTUP_FN();
 }
 #endif
 
@@ -164,14 +162,9 @@ void IRAM_ATTR call_start_cpu1(void)
  * We arrive here after the bootloader finished loading the program from flash. The hardware is mostly uninitialized,
  * and the app CPU is in reset. We do have a stack, so we can do the initialization in C.
  */
-
 void IRAM_ATTR call_start_cpu0(void)
 {
-#if CONFIG_FREERTOS_UNICORE
+    RESET_REASON rst_reas[SOC_CPU_CORES_NUM];
-    RESET_REASON rst_reas[1];
-#else
-    RESET_REASON rst_reas[2];
-#endif
 
     bootloader_init_mem();
 
@@ -179,14 +172,14 @@ void IRAM_ATTR call_start_cpu0(void)
     cpu_hal_set_vecbase(&_init_start);
 
     rst_reas[0] = rtc_get_reset_reason(0);
-
+#if SOC_CPU_CORES_NUM > 1
-#if !CONFIG_FREERTOS_UNICORE
     rst_reas[1] = rtc_get_reset_reason(1);
 #endif
 
+#ifndef CONFIG_BOOTLOADER_WDT_ENABLE
     // from panic handler we can be reset by RWDT or TG0WDT
     if (rst_reas[0] == RTCWDT_SYS_RESET || rst_reas[0] == TG0WDT_SYS_RESET
-#if !CONFIG_FREERTOS_UNICORE
+#if SOC_CPU_CORES_NUM > 1
         || rst_reas[1] == RTCWDT_SYS_RESET || rst_reas[1] == TG0WDT_SYS_RESET
 #endif
     ) {
@@ -197,6 +190,7 @@ void IRAM_ATTR call_start_cpu0(void)
         wdt_hal_write_protect_enable(&rtc_wdt_ctx);
 #endif
     }
+#endif
 
     //Clear BSS. Please do not attempt to do any complex stuff (like early logging) before this.
     memset(&_bss_start, 0, (&_bss_end - &_bss_start) * sizeof(_bss_start));
@@ -211,7 +205,6 @@ void IRAM_ATTR call_start_cpu0(void)
         memset(&_rtc_bss_start, 0, (&_rtc_bss_end - &_rtc_bss_start) * sizeof(_rtc_bss_start));
     }
 
-
 #if CONFIG_IDF_TARGET_ESP32S2
     /* Configure the mode of instruction cache : cache size, cache associated ways, cache line size. */
     extern void esp_config_instruction_cache_mode(void);
@@ -247,45 +240,45 @@ void IRAM_ATTR call_start_cpu0(void)
     }
 #endif
 
+    s_cpu_up[0] = true;
     ESP_EARLY_LOGI(TAG, "Pro cpu up.");
 
-#ifdef CONFIG_SECURE_FLASH_ENC_ENABLED
+#if CONFIG_IDF_TARGET_ESP32
-    esp_flash_encryption_init_checks();
+    // If not the single core variant of ESP32 - check this since there is
-#endif
+    // no separate soc_caps.h for the single core variant.
+    if (!REG_GET_BIT(EFUSE_BLK0_RDATA3_REG, EFUSE_RD_CHIP_VER_DIS_APP_CPU)) {
+        ESP_EARLY_LOGI(TAG, "Starting app cpu, entry point is %p", call_start_cpu1);
 
-#if !CONFIG_FREERTOS_UNICORE
+        Cache_Flush(1);
-    if (REG_GET_BIT(EFUSE_BLK0_RDATA3_REG, EFUSE_RD_CHIP_VER_DIS_APP_CPU)) {
+        Cache_Read_Enable(1);
-        ESP_EARLY_LOGE(TAG, "Running on single core chip, but application is built with dual core support.");
+        esp_cpu_unstall(1);
-        ESP_EARLY_LOGE(TAG, "Please enable CONFIG_FREERTOS_UNICORE option in menuconfig.");
+
-        abort();
+        // Enable clock and reset APP CPU. Note that OpenOCD may have already
+        // enabled clock and taken APP CPU out of reset. In this case don't reset
+        // APP CPU again, as that will clear the breakpoints which may have already
+        // been set.
+        if (!DPORT_GET_PERI_REG_MASK(DPORT_APPCPU_CTRL_B_REG, DPORT_APPCPU_CLKGATE_EN)) {
+            DPORT_SET_PERI_REG_MASK(DPORT_APPCPU_CTRL_B_REG, DPORT_APPCPU_CLKGATE_EN);
+            DPORT_CLEAR_PERI_REG_MASK(DPORT_APPCPU_CTRL_C_REG, DPORT_APPCPU_RUNSTALL);
+            DPORT_SET_PERI_REG_MASK(DPORT_APPCPU_CTRL_A_REG, DPORT_APPCPU_RESETTING);
+            DPORT_CLEAR_PERI_REG_MASK(DPORT_APPCPU_CTRL_A_REG, DPORT_APPCPU_RESETTING);
+        }
+        ets_set_appcpu_boot_addr((uint32_t)call_start_cpu1);
+
+        volatile bool cpus_up = false;
+
+        while(!cpus_up){
+            cpus_up = true;
+            for (int i = 0; i < SOC_CPU_CORES_NUM; i++) {
+                cpus_up &= s_cpu_up[i];
+            }
+            cpu_hal_delay_us(100);
+        }
     }
-    // ESP_EARLY_LOGI(TAG, "Starting app cpu, entry point is %p", call_start_cpu1);
+    else {
-
+        s_cpu_inited[1] = true;
-    //Flush and enable icache for APP CPU
+        DPORT_CLEAR_PERI_REG_MASK(DPORT_APPCPU_CTRL_B_REG, DPORT_APPCPU_CLKGATE_EN);
-    Cache_Flush(1);
-    Cache_Read_Enable(1);
-    esp_cpu_unstall(1);
-
-    // Enable clock and reset APP CPU. Note that OpenOCD may have already
-    // enabled clock and taken APP CPU out of reset. In this case don't reset
-    // APP CPU again, as that will clear the breakpoints which may have already
-    // been set.
-    if (!DPORT_GET_PERI_REG_MASK(DPORT_APPCPU_CTRL_B_REG, DPORT_APPCPU_CLKGATE_EN)) {
-        DPORT_SET_PERI_REG_MASK(DPORT_APPCPU_CTRL_B_REG, DPORT_APPCPU_CLKGATE_EN);
-        DPORT_CLEAR_PERI_REG_MASK(DPORT_APPCPU_CTRL_C_REG, DPORT_APPCPU_RUNSTALL);
-        DPORT_SET_PERI_REG_MASK(DPORT_APPCPU_CTRL_A_REG, DPORT_APPCPU_RESETTING);
-        DPORT_CLEAR_PERI_REG_MASK(DPORT_APPCPU_CTRL_A_REG, DPORT_APPCPU_RESETTING);
     }
-    ets_set_appcpu_boot_addr((uint32_t)call_start_cpu1);
-
-    while (!app_cpu_started) {
-        ets_delay_us(100);
-    }
-#else
-#if CONFIG_IDF_TARGET_ESP32 // Single core chips have no 'single core mode'
-    ESP_EARLY_LOGI(TAG, "Single core mode");
-    DPORT_CLEAR_PERI_REG_MASK(DPORT_APPCPU_CTRL_B_REG, DPORT_APPCPU_CLKGATE_EN);
-#endif
 #endif
 
 #if CONFIG_SPIRAM_MEMTEST
@@ -334,8 +327,6 @@ void IRAM_ATTR call_start_cpu0(void)
     memset(&_ext_ram_bss_start, 0, (&_ext_ram_bss_end - &_ext_ram_bss_start) * sizeof(_ext_ram_bss_start));
 #endif
 
-    ///////////////////////////////////////////////////////////////////
-
 //Enable trace memory and immediately start trace.
 #if CONFIG_ESP32_TRAX || CONFIG_ESP32S2_TRAX
 #if CONFIG_IDF_TARGET_ESP32
@@ -358,25 +349,21 @@ void IRAM_ATTR call_start_cpu0(void)
     esp_brownout_init();
 #endif
 
-#if CONFIG_SECURE_DISABLE_ROM_DL_MODE
-    err = esp_efuse_disable_rom_download_mode();
-    assert(err == ESP_OK && "Failed to disable ROM download mode");
-#endif
-#if CONFIG_SECURE_ENABLE_SECURE_ROM_DL_MODE
-    err = esp_efuse_enable_rom_secure_download_mode();
-    assert(err == ESP_OK && "Failed to enable Secure Download mode");
-#endif
-
-#if CONFIG_ESP32_DISABLE_BASIC_ROM_CONSOLE || CONFIG_ESP32S2_DISABLE_BASIC_ROM_CONSOLE
-    esp_efuse_disable_basic_rom_console();
-#endif
-
     rtc_gpio_force_hold_dis_all();
 
     esp_cache_err_int_init();
 
-    bootloader_flash_update_id();
+#if CONFIG_IDF_TARGET_ESP32S2
+#if CONFIG_ESP32S2_MEMPROT_FEATURE
+#if CONFIG_ESP32S2_MEMPROT_FEATURE_LOCK
+    esp_memprot_set_prot(true, true);
+#else
+    esp_memprot_set_prot(true, false);
+#endif
+#endif
+#endif
 
+    bootloader_flash_update_id();
 #if CONFIG_IDF_TARGET_ESP32 
 #if !CONFIG_SPIRAM_BOOT_INIT
     // Read the application binary image header. This will also decrypt the header if the image is encrypted.
@@ -403,5 +390,17 @@ void IRAM_ATTR call_start_cpu0(void)
 #endif //!CONFIG_SPIRAM_BOOT_INIT
 #endif
 
-    start_cpu0();
+    s_cpu_inited[0] = true;
+
+    volatile bool cpus_inited = false;
+
+    while(!cpus_inited) {
+        cpus_inited = true;
+        for (int i = 0; i < SOC_CPU_CORES_NUM; i++) {
+            cpus_inited &= s_cpu_inited[i];
+        }
+        cpu_hal_delay_us(100);
+    }
+
+    SYS_STARTUP_FN();
 }

components/esp_system/private_include/startup_internal.h

@@ -15,6 +15,23 @@
 #pragma once
 
 #include "esp_attr.h"
+
+#include "soc/soc_caps.h"
+#include "hal/cpu_hal.h"
+
+extern bool g_spiram_ok; // [refactor-todo] better way to communicate this from port layer to common startup code
+
+// Port layer defines the entry point. It then transfer control to a `sys_startup_fn_t`, stored in this
+// array, one per core.
+typedef void (*sys_startup_fn_t)(void);
+extern sys_startup_fn_t g_startup_fn[SOC_CPU_CORES_NUM];
+
+// Utility to execute `sys_startup_fn_t` for the current core.
+#define SYS_STARTUP_FN()  ((*g_startup_fn[(cpu_hal_get_core_id())])())
+
+void startup_resume_other_cores(void);
+void startup_core_init();
+
 typedef struct {
   void (*fn)(void);
   uint32_t cores;

components/esp_system/private_include/sys_funcs.h

@@ -1,21 +0,0 @@
-// Copyright 2020 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#pragma once
-
-#include "sdkconfig.h"
-
-extern bool g_spiram_ok;
-
-void sys_startup(void);

components/esp_system/startup.c

@@ -18,75 +18,77 @@
 #include "esp_attr.h"
 #include "esp_err.h"
 
-#include "soc/rtc_wdt.h"
-
-#include "freertos/FreeRTOS.h"
-#include "freertos/task.h"
-#include "freertos/semphr.h"
-#include "freertos/queue.h"
-
-#include "esp_heap_caps_init.h"
 #include "esp_system.h"
-#include "esp_flash_internal.h"
-#include "nvs_flash.h"
-#include "esp_spi_flash.h"
-#include "esp_private/crosscore_int.h"
 #include "esp_log.h"
-#include "esp_vfs_dev.h"
+#include "esp_ota_ops.h"
+
+#include "sdkconfig.h"
+
+#include "soc/rtc_wdt.h"
+#include "soc/soc_caps.h"
+
+#include "esp_system.h"
+#include "esp_log.h"
+#include "esp_heap_caps_init.h"
+#include "esp_spi_flash.h"
+#include "esp_flash_internal.h"
 #include "esp_newlib.h"
-#include "esp_int_wdt.h"
+#include "esp_vfs_dev.h"
-#include "esp_task.h"
+#include "esp_timer.h"
-#include "esp_task_wdt.h"
+#include "esp_efuse.h"
+#include "esp_flash_encrypt.h"
+
+/* Headers for other components init functions */
+#include "nvs_flash.h"
 #include "esp_phy_init.h"
 #include "esp_coexist_internal.h"
 #include "esp_core_dump.h"
 #include "esp_app_trace.h"
 #include "esp_private/dbg_stubs.h"
 #include "esp_flash_encrypt.h"
-#include "esp_clk_internal.h"
-#include "esp_timer.h"
 #include "esp_pm.h"
-#include "esp_private/pm_impl.h"
+#include "esp_pthread.h"
-#include "esp_ota_ops.h"
-
-#include "sdkconfig.h"
 
+// [refactor-todo] make this file completely target-independent
 #if CONFIG_IDF_TARGET_ESP32
 #include "esp32/rom/uart.h"
-#include "esp32/dport_access.h"
+#include "esp32/spiram.h"
 #elif CONFIG_IDF_TARGET_ESP32S2
 #include "esp32s2/rom/uart.h"
-#include "esp32s2/dport_access.h"
+#include "esp32s2/spiram.h"
 #endif
+/***********************************************/
 
-#include "sys_funcs.h"
+#include "startup_internal.h"
-
-#define STRINGIFY(s) STRINGIFY2(s)
-#define STRINGIFY2(s) #s
 
 void start_cpu0(void) __attribute__((weak, alias("start_cpu0_default"))) __attribute__((noreturn));
-void start_cpu0_default(void) IRAM_ATTR __attribute__((noreturn));
+void start_cpuX(void) __attribute__((weak, alias("start_cpuX_default"))) __attribute__((noreturn));
-#if !CONFIG_FREERTOS_UNICORE
+
-void start_cpu1(void) __attribute__((weak, alias("start_cpu1_default"))) __attribute__((noreturn));
+void app_mainX(void) __attribute__((weak, alias("app_mainX_default"))) __attribute__((noreturn));
-void start_cpu1_default(void) IRAM_ATTR __attribute__((noreturn));
-#endif //!CONFIG_FREERTOS_UNICORE
 
 extern void app_main(void);
-extern esp_err_t esp_pthread_init(void);
 
-extern void (*__init_array_start)(void);
+sys_startup_fn_t g_startup_fn[SOC_CPU_CORES_NUM] = { [0] = start_cpu0,
-extern void (*__init_array_end)(void);
+#if SOC_CPU_CORES_NUM > 1
-extern volatile int port_xSchedulerRunning[2];
+    [1 ... SOC_CPU_CORES_NUM - 1] = start_cpuX
+#endif
+};
+
+static volatile bool s_system_inited[SOC_CPU_CORES_NUM] = { false };
+static volatile bool s_system_full_inited = false;
 
 static const char* TAG = "cpu_start";
 
-struct object { long placeholder[ 10 ]; };
+static void IRAM_ATTR do_global_ctors(void)
-void __register_frame_info (const void *begin, struct object *ob);
-extern char __eh_frame[];
-
-static void do_global_ctors(void)
 {
+    extern void (*__init_array_start)(void);
+    extern void (*__init_array_end)(void);
+
 #ifdef CONFIG_COMPILER_CXX_EXCEPTIONS
+    struct object { long placeholder[ 10 ]; };
+    void __register_frame_info (const void *begin, struct object *ob);
+    extern char __eh_frame[];
+
     static struct object ob;
     __register_frame_info( __eh_frame, &ob );
 #endif
@@ -97,7 +99,7 @@ static void do_global_ctors(void)
     }
 }
 
-static void do_system_init_fn(void)
+static void IRAM_ATTR do_system_init_fn(void)
 {
     extern esp_system_init_fn_t _esp_system_init_fn_array_start;
     extern esp_system_init_fn_t _esp_system_init_fn_array_end;
@@ -109,16 +111,54 @@ static void do_system_init_fn(void)
             (*(p->fn))();
         }
     }
+
+    s_system_inited[cpu_hal_get_core_id()] = true;
 }
 
-static void main_task(void* args)
+static void IRAM_ATTR app_mainX_default(void)
 {
-#if !CONFIG_FREERTOS_UNICORE
+    while(1) {
-    // Wait for FreeRTOS initialization to finish on APP CPU, before replacing its startup stack
+        cpu_hal_delay_us(UINT32_MAX);
-    while (port_xSchedulerRunning[1] == 0) {
-        ;
     }
+}
+
+static void IRAM_ATTR start_cpuX_default(void)
+{
+    do_system_init_fn();
+
+    while(!s_system_full_inited) {
+        cpu_hal_delay_us(100);
+    }
+
+    app_mainX();
+}
+
+static void IRAM_ATTR do_core_init(void)
+{
+    /* Initialize heap allocator. WARNING: This *needs* to happen *after* the app cpu has booted.
+       If the heap allocator is initialized first, it will put free memory linked list items into
+       memory also used by the ROM. Starting the app cpu will let its ROM initialize that memory,
+       corrupting those linked lists. Initializing the allocator *after* the app cpu has booted
+       works around this problem.
+       With SPI RAM enabled, there's a second reason: half of the SPI RAM will be managed by the
+       app CPU, and when that is not up yet, the memory will be inaccessible and heap_caps_init may
+       fail initializing it properly. */
+    heap_caps_init();
+
+    esp_setup_syscall_table();
+
+    if (g_spiram_ok) {
+#if CONFIG_SPIRAM_BOOT_INIT && (CONFIG_SPIRAM_USE_CAPS_ALLOC || CONFIG_SPIRAM_USE_MALLOC)
+        esp_err_t r=esp_spiram_add_to_heapalloc();
+        if (r != ESP_OK) {
+            ESP_EARLY_LOGE(TAG, "External RAM could not be added to heap!");
+            abort();
+        }
+#if CONFIG_SPIRAM_USE_MALLOC
+        heap_caps_malloc_extmem_enable(CONFIG_SPIRAM_MALLOC_ALWAYSINTERNAL);
 #endif
+#endif
+    }
 
     // Now we have startup stack RAM available for heap, enable any DMA pool memory
 #if CONFIG_SPIRAM_MALLOC_RESERVE_INTERNAL
@@ -131,75 +171,80 @@ static void main_task(void* args)
     }
 #endif
 
-    //Initialize task wdt if configured to do so
+    esp_reent_init(_GLOBAL_REENT);
-#ifdef CONFIG_ESP_TASK_WDT_PANIC
+
-    ESP_ERROR_CHECK(esp_task_wdt_init(CONFIG_ESP_TASK_WDT_TIMEOUT_S, true));
+#ifndef CONFIG_ESP_CONSOLE_UART_NONE
-#elif CONFIG_ESP_TASK_WDT
+    const int uart_clk_freq = APB_CLK_FREQ;
-    ESP_ERROR_CHECK(esp_task_wdt_init(CONFIG_ESP_TASK_WDT_TIMEOUT_S, false));
+    uart_div_modify(CONFIG_ESP_CONSOLE_UART_NUM, (uart_clk_freq << 4) / CONFIG_ESP_CONSOLE_UART_BAUDRATE);
 #endif
 
-    //Add IDLE 0 to task wdt
+#ifdef CONFIG_VFS_SUPPORT_IO
-#ifdef CONFIG_ESP_TASK_WDT_CHECK_IDLE_TASK_CPU0
+    esp_vfs_dev_uart_register();
-    TaskHandle_t idle_0 = xTaskGetIdleTaskHandleForCPU(0);
+#endif // CONFIG_VFS_SUPPORT_IO
-    if(idle_0 != NULL){
+
-        ESP_ERROR_CHECK(esp_task_wdt_add(idle_0));
+#if defined(CONFIG_VFS_SUPPORT_IO) && !defined(CONFIG_ESP_CONSOLE_UART_NONE)
-    }
+    esp_reent_init(_GLOBAL_REENT);
-#endif
+    const char *default_uart_dev = "/dev/uart/" STRINGIFY(CONFIG_ESP_CONSOLE_UART_NUM);
-    //Add IDLE 1 to task wdt
+    _GLOBAL_REENT->_stdin  = fopen(default_uart_dev, "r");
-#ifdef CONFIG_ESP_TASK_WDT_CHECK_IDLE_TASK_CPU1
+    _GLOBAL_REENT->_stdout = fopen(default_uart_dev, "w");
-    TaskHandle_t idle_1 = xTaskGetIdleTaskHandleForCPU(1);
+    _GLOBAL_REENT->_stderr = fopen(default_uart_dev, "w");
-    if(idle_1 != NULL){
+#else // defined(CONFIG_VFS_SUPPORT_IO) && !defined(CONFIG_ESP_CONSOLE_UART_NONE)
-        ESP_ERROR_CHECK(esp_task_wdt_add(idle_1));
+    _REENT_SMALL_CHECK_INIT(_GLOBAL_REENT);
-    }
+#endif // defined(CONFIG_VFS_SUPPORT_IO) && !defined(CONFIG_ESP_CONSOLE_UART_NONE)
-    esp_spiram_init_cache();
+
+#ifdef CONFIG_SECURE_FLASH_ENC_ENABLED
+    esp_flash_encryption_init_checks();
 #endif
 
-    // Now that the application is about to start, disable boot watchdog
+#if CONFIG_SECURE_DISABLE_ROM_DL_MODE
-#ifndef CONFIG_BOOTLOADER_WDT_DISABLE_IN_USER_CODE
+    err = esp_efuse_disable_rom_download_mode();
-    rtc_wdt_disable();
+    assert(err == ESP_OK && "Failed to disable ROM download mode");
 #endif
-#ifdef CONFIG_BOOTLOADER_EFUSE_SECURE_VERSION_EMULATE
+
-    const esp_partition_t *efuse_partition = esp_partition_find_first(ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_DATA_EFUSE_EM, NULL);
+#if CONFIG_SECURE_ENABLE_SECURE_ROM_DL_MODE
-    if (efuse_partition) {
+    err = esp_efuse_enable_rom_secure_download_mode();
-        esp_efuse_init(efuse_partition->address, efuse_partition->size);
+    assert(err == ESP_OK && "Failed to enable Secure Download mode");
-    }
 #endif
-    app_main();
+
-    vTaskDelete(NULL);
+#if CONFIG_ESP32_DISABLE_BASIC_ROM_CONSOLE
+    esp_efuse_disable_basic_rom_console();
+#endif
+
+    spi_flash_init();
+    /* init default OS-aware flash access critical section */
+    spi_flash_guard_set(&g_flash_guard_default_ops);
+
+    esp_flash_app_init();
+    esp_err_t flash_ret = esp_flash_init_default_chip();
+    assert(flash_ret == ESP_OK);
 }
 
-#if !CONFIG_FREERTOS_UNICORE
+static void IRAM_ATTR do_secondary_init(void)
-void start_cpu1_default(void)
 {
-    // Wait for FreeRTOS initialization to finish on PRO CPU
+    // The port layer transferred control to this function with other cores 'paused',
-    while (port_xSchedulerRunning[0] == 0) {
+    // resume execution so that cores might execute component initialization functions.
-        ;
+    startup_resume_other_cores();
+
+    // Execute initialization functions esp_system_init_fn_t assigned to the main core. While
+    // this is happening, all other cores are executing the initialization functions
+    // assigned to them since they have been resumed already.
+    do_system_init_fn();
+
+    // Wait for all cores to finish secondary init.
+    volatile bool system_inited = false;
+
+    while(!system_inited) {
+        system_inited = true;
+        for (int i = 0; i < SOC_CPU_CORES_NUM; i++) {
+            system_inited &= s_system_inited[i];
+        }
+        cpu_hal_delay_us(100);
     }
-
-#if CONFIG_APPTRACE_ENABLE
-    esp_err_t err = esp_apptrace_init();
-    assert(err == ESP_OK && "Failed to init apptrace module on APP CPU!");
-#endif
-#if CONFIG_ESP_INT_WDT
-    //Initialize the interrupt watch dog for CPU1.
-    esp_int_wdt_cpu_init();
-#endif
-
-    esp_crosscore_int_init();
-    esp_dport_access_int_init();
-
-    ESP_EARLY_LOGI(TAG, "Starting scheduler on APP CPU.");
-    xPortStartScheduler();
-    abort(); /* Only get to here if FreeRTOS somehow very broken */
 }
-#endif //!CONFIG_FREERTOS_UNICORE
 
-/*
- * We arrive here after the bootloader finished loading the program from flash. The hardware is mostly uninitialized,
- * and the app CPU is in reset. We do have a stack, so we can do the initialization in C.
- */
 void IRAM_ATTR start_cpu0_default(void)
 {
+    // Display information about the current running image.
     if (LOG_LOCAL_LEVEL >= ESP_LOG_INFO) {
         const esp_app_desc_t *app_desc = esp_ota_get_app_description();
         ESP_EARLY_LOGI(TAG, "Application information:");
@@ -221,126 +266,46 @@ void IRAM_ATTR start_cpu0_default(void)
         ESP_EARLY_LOGI(TAG, "ESP-IDF:          %s", app_desc->idf_ver);
     }
 
-    /* Initialize heap allocator */
+    // Initialize core components and services.
-    heap_caps_init();
+    do_core_init();
 
+    // Execute constructors.
+    do_global_ctors();
+
+    // Execute init functions of other components; blocks
+    // until all cores finish.
+    do_secondary_init();
+
+    // Now that the application is about to start, disable boot watchdog
+#ifndef CONFIG_BOOTLOADER_WDT_DISABLE_IN_USER_CODE
+    rtc_wdt_disable();
+#endif
+
+    // Finally, we jump to user code.
     ESP_EARLY_LOGI(TAG, "Pro cpu start user code");
 
+    s_system_full_inited = true;
+
+    app_main();
+    while(1);
+}
+
+IRAM_ATTR ESP_SYSTEM_INIT_FN(init_components0, BIT(0))
+{
     esp_err_t err;
-    esp_setup_syscall_table();
 
-    if (g_spiram_ok) {
-#if CONFIG_SPIRAM_BOOT_INIT && (CONFIG_SPIRAM_USE_CAPS_ALLOC || CONFIG_SPIRAM_USE_MALLOC)
-        esp_err_t r=esp_spiram_add_to_heapalloc();
-        if (r != ESP_OK) {
-            ESP_EARLY_LOGE(TAG, "External RAM could not be added to heap!");
-            abort();
-        }
-#if CONFIG_SPIRAM_USE_MALLOC
-        heap_caps_malloc_extmem_enable(CONFIG_SPIRAM_MALLOC_ALWAYSINTERNAL);
-#endif
-#endif
-    }
-
-#ifndef CONFIG_ESP_CONSOLE_UART_NONE
 #ifdef CONFIG_PM_ENABLE
     const int uart_clk_freq = REF_CLK_FREQ;
     /* When DFS is enabled, use REFTICK as UART clock source */
     CLEAR_PERI_REG_MASK(UART_CONF0_REG(CONFIG_ESP_CONSOLE_UART_NUM), UART_TICK_REF_ALWAYS_ON);
-#else
-    const int uart_clk_freq = APB_CLK_FREQ;
-#endif // CONFIG_PM_DFS_ENABLE
     uart_div_modify(CONFIG_ESP_CONSOLE_UART_NUM, (uart_clk_freq << 4) / CONFIG_ESP_CONSOLE_UART_BAUDRATE);
 #endif // CONFIG_ESP_CONSOLE_UART_NONE
 
-
-#ifdef CONFIG_VFS_SUPPORT_IO
-    esp_vfs_dev_uart_register();
-#endif // CONFIG_VFS_SUPPORT_IO
-
-#if defined(CONFIG_VFS_SUPPORT_IO) && !defined(CONFIG_ESP_CONSOLE_UART_NONE)
-    esp_reent_init(_GLOBAL_REENT);
-    const char *default_uart_dev = "/dev/uart/" STRINGIFY(CONFIG_ESP_CONSOLE_UART_NUM);
-    _GLOBAL_REENT->_stdin  = fopen(default_uart_dev, "r");
-    _GLOBAL_REENT->_stdout = fopen(default_uart_dev, "w");
-    _GLOBAL_REENT->_stderr = fopen(default_uart_dev, "w");
-#else // defined(CONFIG_VFS_SUPPORT_IO) && !defined(CONFIG_ESP_CONSOLE_UART_NONE)
-    _REENT_SMALL_CHECK_INIT(_GLOBAL_REENT);
-#endif // defined(CONFIG_VFS_SUPPORT_IO) && !defined(CONFIG_ESP_CONSOLE_UART_NONE)
-    // After setting _GLOBAL_REENT, ESP_LOGIx can be used instead of ESP_EARLY_LOGx.
-
-#ifdef CONFIG_SECURE_FLASH_ENC_ENABLED
-    esp_flash_encryption_init_checks();
-#endif
-
-#if CONFIG_SECURE_DISABLE_ROM_DL_MODE
-    err = esp_efuse_disable_rom_download_mode();
-    assert(err == ESP_OK && "Failed to disable ROM download mode");
-#endif
-
-#if CONFIG_SECURE_ENABLE_SECURE_ROM_DL_MODE
-    err = esp_efuse_enable_rom_secure_download_mode();
-    assert(err == ESP_OK && "Failed to enable Secure Download mode");
-#endif
-
-    esp_timer_init();
-    esp_set_time_from_rtc();
-
-#if CONFIG_APPTRACE_ENABLE
-    err = esp_apptrace_init();
-    assert(err == ESP_OK && "Failed to init apptrace module on PRO CPU!");
-#endif
-
-#if CONFIG_SYSVIEW_ENABLE
-    SEGGER_SYSVIEW_Conf();
-#endif
-#if CONFIG_ESP_DEBUG_STUBS_ENABLE
     esp_dbg_stubs_init();
-#endif
 
     err = esp_pthread_init();
     assert(err == ESP_OK && "Failed to init pthread module!");
 
-#if CONFIG_ESP32S2_MEMPROT_FEATURE
-#if CONFIG_ESP32S2_MEMPROT_FEATURE_LOCK
-    esp_memprot_set_prot(true, true);
-#else
-    esp_memprot_set_prot(true, false);
-#endif
-#endif
-
-    do_global_ctors();
-#if CONFIG_ESP_INT_WDT
-    esp_int_wdt_init();
-    //Initialize the interrupt watch dog for CPU0.
-    esp_int_wdt_cpu_init();
-#else
-#if CONFIG_ESP32_ECO3_CACHE_LOCK_FIX
-    assert(!soc_has_cache_lock_bug() && "ESP32 Rev 3 + Dual Core + PSRAM requires INT WDT enabled in project config!");
-#endif
-#endif
-
-    esp_crosscore_int_init();
-
-#ifndef CONFIG_FREERTOS_UNICORE
-    esp_dport_access_int_init();
-#endif
-
-
-    spi_flash_init();
-    /* init default OS-aware flash access critical section */
-    spi_flash_guard_set(&g_flash_guard_default_ops);
-
-    esp_flash_app_init();
-    esp_err_t flash_ret = esp_flash_init_default_chip();
-    assert(flash_ret == ESP_OK);
-
-#if CONFIG_IDF_TARGET_ESP32
-#if CONFIG_ESP32_ENABLE_COREDUMP
-    esp_core_dump_init();
-#endif
-#endif
-
 #ifdef CONFIG_PM_ENABLE
     esp_pm_impl_init();
 #ifdef CONFIG_PM_DFS_INIT_AUTO
@@ -353,6 +318,12 @@ void IRAM_ATTR start_cpu0_default(void)
 #endif //CONFIG_PM_DFS_INIT_AUTO
 #endif //CONFIG_PM_ENABLE
 
+#if CONFIG_IDF_TARGET_ESP32
+#if CONFIG_ESP32_ENABLE_COREDUMP
+    esp_core_dump_init();
+#endif
+#endif
+
 #if CONFIG_IDF_TARGET_ESP32
 #if CONFIG_ESP32_WIFI_SW_COEXIST_ENABLE
     esp_coex_adapter_register(&g_coex_adapter_funcs);
@@ -360,14 +331,18 @@ void IRAM_ATTR start_cpu0_default(void)
 #endif
 #endif
 
-    do_system_init_fn();
+#ifdef CONFIG_BOOTLOADER_EFUSE_SECURE_VERSION_EMULATE
+    const esp_partition_t *efuse_partition = esp_partition_find_first(ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_DATA_EFUSE_EM, NULL);
+    if (efuse_partition) {
+        esp_efuse_init(efuse_partition->address, efuse_partition->size);
+    }
+#endif
+}
 
-    portBASE_TYPE res = xTaskCreatePinnedToCore(&main_task, "main",
+IRAM_ATTR ESP_SYSTEM_INIT_FN(init_components1, BIT(1))
-                                                ESP_TASK_MAIN_STACK, NULL,
+{
-                                                ESP_TASK_MAIN_PRIO, NULL, 0);
+#if CONFIG_APPTRACE_ENABLE
-    assert(res == pdTRUE);
+    esp_err_t err = esp_apptrace_init();
-
+    assert(err == ESP_OK && "Failed to init apptrace module on APP CPU!");
-    ESP_LOGI(TAG, "Starting scheduler on PRO CPU.");
+#endif
-    vTaskStartScheduler();
-    abort(); /* Only get to here if not enough free heap to start scheduler */
 }

components/freertos/CMakeLists.txt

@@ -62,3 +62,5 @@ set_source_files_properties(
     PROPERTIES COMPILE_DEFINITIONS
     _ESP_FREERTOS_INTERNAL
     )
+
+target_link_libraries(${COMPONENT_LIB} INTERFACE "-Wl,--wrap=app_main")

components/freertos/xtensa/port.c

@@ -110,12 +110,23 @@
 #include "sdkconfig.h"
 #include "esp_compiler.h"
 
+#include "esp_task_wdt.h"
+#include "esp_task.h"
+
+#include "soc/soc_caps.h"
+#include "soc/efuse_reg.h"
+#include "soc/dport_access.h"
+#include "soc/dport_reg.h"
+#include "esp_int_wdt.h"
+
 /* Defined in portasm.h */
 extern void _frxt_tick_timer_init(void);
 
 /* Defined in xtensa_context.S */
 extern void _xt_coproc_init(void);
 
+static const char* TAG = "cpu_start"; // [refactor-todo]: might be appropriate to change in the future, but
+								// for now maintain the same log output
 
 #if CONFIG_FREERTOS_CORETIMER_0
     #define SYSTICK_INTR_ID (ETS_INTERNAL_TIMER0_INTR_SOURCE+ETS_INTERNAL_INTR_SOURCE_OFF)
@@ -124,6 +135,8 @@ extern void _xt_coproc_init(void);
     #define SYSTICK_INTR_ID (ETS_INTERNAL_TIMER1_INTR_SOURCE+ETS_INTERNAL_INTR_SOURCE_OFF)
 #endif
 
+
+
 _Static_assert(tskNO_AFFINITY == CONFIG_FREERTOS_NO_AFFINITY, "incorrect tskNO_AFFINITY value");
 
 /*-----------------------------------------------------------*/
@@ -434,4 +447,122 @@ void  __attribute__((weak)) vApplicationStackOverflowHook( TaskHandle_t xTask, c
 		dest = strcat(dest, str[i]);
 	}
 	esp_system_abort(buf);
+}
+
+// `esp_system` calls the app entry point app_main for core 0 and app_mainX for 
+// the rest of the cores which the app normally provides for non-os builds.
+// If `freertos` is included in the build, wrap the call to app_main and provide 
+// our own definition of app_mainX so that we can do our own initializations for each
+// core and start the scheduler. 
+//
+// We now simply execute the real app_main in the context of the main task that
+// we also start.
+extern void __real_app_main(void);
+
+static void main_task(void* args)
+{
+#if !CONFIG_FREERTOS_UNICORE
+    // Wait for FreeRTOS initialization to finish on APP CPU, before replacing its startup stack
+    while (port_xSchedulerRunning[1] == 0) {
+        ;
+    }
+#endif
+
+    //Initialize task wdt if configured to do so
+#ifdef CONFIG_ESP_TASK_WDT_PANIC
+    ESP_ERROR_CHECK(esp_task_wdt_init(CONFIG_ESP_TASK_WDT_TIMEOUT_S, true));
+#elif CONFIG_ESP_TASK_WDT
+    ESP_ERROR_CHECK(esp_task_wdt_init(CONFIG_ESP_TASK_WDT_TIMEOUT_S, false));
+#endif
+
+    //Add IDLE 0 to task wdt
+#ifdef CONFIG_ESP_TASK_WDT_CHECK_IDLE_TASK_CPU0
+    TaskHandle_t idle_0 = xTaskGetIdleTaskHandleForCPU(0);
+    if(idle_0 != NULL){
+        ESP_ERROR_CHECK(esp_task_wdt_add(idle_0));
+    }
+#endif
+    //Add IDLE 1 to task wdt
+#ifdef CONFIG_ESP_TASK_WDT_CHECK_IDLE_TASK_CPU1
+    TaskHandle_t idle_1 = xTaskGetIdleTaskHandleForCPU(1);
+    if(idle_1 != NULL){
+        ESP_ERROR_CHECK(esp_task_wdt_add(idle_1));
+    }
+#endif
+
+    __real_app_main();
+    vTaskDelete(NULL);
+}
+
+#if !CONFIG_FREERTOS_UNICORE
+void app_mainX(void)
+{
+	if (xPortGetCoreID() >= 2) {
+		// Explicitly support only up to two cores for now.
+		abort();
+	}
+
+    // Wait for FreeRTOS initialization to finish on PRO CPU
+    while (port_xSchedulerRunning[0] == 0) {
+        ;
+    }
+
+#if CONFIG_ESP_INT_WDT
+    //Initialize the interrupt watch dog for CPU1.
+    esp_int_wdt_cpu_init();
+#endif
+
+    esp_crosscore_int_init();
+    esp_dport_access_int_init();
+
+    ESP_EARLY_LOGI(TAG, "Starting scheduler on APP CPU.");
+    xPortStartScheduler();
+    abort(); /* Only get to here if FreeRTOS somehow very broken */
+}
+#endif
+
+void __wrap_app_main(void)
+{
+#if CONFIG_ESP_INT_WDT
+    esp_int_wdt_init();
+    //Initialize the interrupt watch dog for CPU0.
+    esp_int_wdt_cpu_init();
+#else
+#if CONFIG_ESP32_ECO3_CACHE_LOCK_FIX
+    assert(!soc_has_cache_lock_bug() && "ESP32 Rev 3 + Dual Core + PSRAM requires INT WDT enabled in project config!");
+#endif
+#endif
+
+    esp_crosscore_int_init();
+
+#ifndef CONFIG_FREERTOS_UNICORE
+    esp_dport_access_int_init();
+#endif
+
+    portBASE_TYPE res = xTaskCreatePinnedToCore(&main_task, "main",
+                                                ESP_TASK_MAIN_STACK, NULL,
+                                                ESP_TASK_MAIN_PRIO, NULL, 0);
+    assert(res == pdTRUE);
+
+#if !CONFIG_FREERTOS_UNICORE
+	// Check that FreeRTOS is configured properly for the number of cores the target
+	// has at compile and build time.
+#if SOC_CPU_CORES_NUM < 2
+	#error FreeRTOS configured to run on dual core, but target only has a single core.
+#endif
+    if (REG_GET_BIT(EFUSE_BLK0_RDATA3_REG, EFUSE_RD_CHIP_VER_DIS_APP_CPU)) {
+        ESP_EARLY_LOGE(TAG, "Running on single core chip, but application is built with dual core support.");
+        ESP_EARLY_LOGE(TAG, "Please enable CONFIG_FREERTOS_UNICORE option in menuconfig.");
+        abort();
+    }
+
+#else
+	#if SOC_CPU_CORES_NUM > 1 // Single core chips have no 'single core mode'
+		ESP_EARLY_LOGI(TAG, "Single core mode");
+		DPORT_CLEAR_PERI_REG_MASK(DPORT_APPCPU_CTRL_B_REG, DPORT_APPCPU_CLKGATE_EN);
+	#endif
+#endif // !CONFIG_FREERTOS_UNICORE
+
+    ESP_LOGI(TAG, "Starting scheduler on PRO CPU.");
+    vTaskStartScheduler();
 }

components/pthread/include/esp_pthread.h

@@ -82,6 +82,11 @@ esp_err_t esp_pthread_set_cfg(const esp_pthread_cfg_t *cfg);
  */
 esp_err_t esp_pthread_get_cfg(esp_pthread_cfg_t *p);
 
+/**
+ * @brief Initialize pthread library
+ */
+esp_err_t esp_pthread_init(void);
+
 #ifdef __cplusplus
 }
 #endif