feat(esp_system): Add esp_backtrace_print_all_tasks()

This commit adds esp_backtrace_print_all_tasks() which prints the backtraces
of all tasks at runtime.

Closes https://github.com/espressif/esp-idf/issues/9708
CLoses https://github.com/espressif/esp-idf/pull/11575

[Omar Chebib: Prevent task switching while printing backtraces of tasks.]
[Omar Chebib: Ensure all task stacks are flushed from register to RAM.]
[Omar Chebib: Removed esp_task_snapshot_to_backtrace_frame() as task snapshot is private API.]
[Omar Chebib: Added test case for esp_backtrace_print_all_tasks().]

Signed-off-by: Omar Chebib <omar.chebib@espressif.com>

components/esp_system/include/esp_debug_helpers.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2015-2023 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -13,8 +13,9 @@ extern "C" {
 #ifndef __ASSEMBLER__
 
 #include <stdbool.h>
-#include "esp_err.h"
+#include "sdkconfig.h"
 #include "soc/soc.h"  // [refactor-todo] IDF-2297
+#include "esp_err.h"
 #include "esp_cpu.h"
 
 /*
@@ -114,6 +115,20 @@ esp_err_t esp_backtrace_print_from_frame(int depth, const esp_backtrace_frame_t*
  */
 esp_err_t esp_backtrace_print(int depth);
 
+/**
+ * @brief Print the backtrace of all tasks
+ *
+ * @param depth The maximum number of stack frames to print (must be > 0)
+ *
+ * @note Users must ensure that no tasks are created or deleted while this function is running.
+ * @note This function must be called from a task context.
+ *
+ * @return
+ *      - ESP_OK    All backtraces successfully printed to completion or to depth limit
+ *      - ESP_FAIL  One or more backtraces are corrupt
+ */
+esp_err_t esp_backtrace_print_all_tasks(int depth);
+
 /**
  * @brief Set a watchpoint to break/panic when a certain memory range is accessed.
  * Superseded by esp_cpu_set_watchpoint in esp_cpu.h.

components/esp_system/port/arch/xtensa/debug_helpers.c

@@ -1,25 +1,28 @@
 /*
- * SPDX-FileCopyrightText: 2015-2021 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2015-2023 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
 
-#include <string.h>
-
 #include "sdkconfig.h"
+#include <string.h>
+#include <sys/param.h>
+#include "soc/soc_memory_layout.h"
 #include "esp_types.h"
 #include "esp_attr.h"
 #include "esp_err.h"
+#include "esp_check.h"
+#include "esp_ipc.h"
 #include "esp_debug_helpers.h"
-#include "soc/soc_memory_layout.h"
 #include "esp_cpu_utils.h"
 #include "esp_private/panic_internal.h"
-
-#include "xtensa_context.h"
-
-#include "sdkconfig.h"
-
+#include "esp_private/freertos_debug.h"
 #include "esp_rom_sys.h"
+#include "xtensa_context.h"
+#include "freertos/FreeRTOS.h"
+#include "freertos/task.h"
+
+const char *DEBUG_HELPER_TAG = "DBG HLPR";
 
 bool IRAM_ATTR esp_backtrace_get_next_frame(esp_backtrace_frame_t *frame)
 {
@@ -101,3 +104,149 @@ esp_err_t IRAM_ATTR esp_backtrace_print(int depth)
     esp_backtrace_get_start(&(start.pc), &(start.sp), &(start.next_pc));
     return esp_backtrace_print_from_frame(depth, &start, false);
 }
+
+typedef struct {
+#if !CONFIG_FREERTOS_UNICORE
+    volatile bool start_tracing;
+    volatile bool finished_tracing;
+#endif // !CONFIG_FREERTOS_UNICORE
+    struct {
+        TaskHandle_t task_hdl;
+        uint32_t starting_pc;
+        uint32_t starting_sp;
+        uint32_t next_pc;
+    } cur_tasks[configNUMBER_OF_CORES];
+} cur_task_backtrace_ctrl_t;
+
+#if !CONFIG_FREERTOS_UNICORE
+static void backtrace_other_cores_ipc_func(void *arg)
+{
+    cur_task_backtrace_ctrl_t *ctrl = (cur_task_backtrace_ctrl_t *)arg;
+
+    // Suspend the scheduler to prevent task switching
+    vTaskSuspendAll();
+    /*
+    Initialize backtracing for this core:
+
+    - Flush current core's register windows back onto current task's stack using esp_backtrace_get_start()
+    - Get starting frame for backtracing (starting frame is the caller of this function) using esp_backtrace_get_start()
+    - Save the starting frame details into the control block
+    */
+    BaseType_t core_id = xPortGetCoreID();  // Get core ID now that task switching is disabled
+    ctrl->cur_tasks[core_id].task_hdl = xTaskGetCurrentTaskHandle();
+    esp_backtrace_get_start(&ctrl->cur_tasks[core_id].starting_pc,
+                            &ctrl->cur_tasks[core_id].starting_sp,
+                            &ctrl->cur_tasks[core_id].next_pc);
+
+    // Indicate to backtracing core that this core is ready for backtracing
+    ctrl->start_tracing = true;
+    // Wait for backtracing core to indicate completion
+    while (!ctrl->finished_tracing) {
+        ;
+    }
+    // Resume the scheduler to allow task switching again
+    xTaskResumeAll();
+}
+#endif // !CONFIG_FREERTOS_UNICORE
+
+esp_err_t IRAM_ATTR esp_backtrace_print_all_tasks(int depth)
+{
+    esp_err_t ret = ESP_OK;
+    TaskSnapshot_t *task_snapshots;
+    cur_task_backtrace_ctrl_t ctrl = {0};
+
+    /*
+    Allocate array to store task snapshots. Users are responsible for ensuring
+    tasks don't get created/deleted while backtracing.
+    */
+    const UBaseType_t num_tasks = uxTaskGetNumberOfTasks();
+    task_snapshots = calloc(num_tasks, sizeof(TaskSnapshot_t));
+    ESP_GOTO_ON_FALSE(task_snapshots, ESP_ERR_NO_MEM, malloc_err, DEBUG_HELPER_TAG, "Task snapshot alloc failed");
+
+#if !CONFIG_FREERTOS_UNICORE
+    // Use IPC call to prepare other core for backtracing
+    ESP_GOTO_ON_ERROR(esp_ipc_call(!xPortGetCoreID(), backtrace_other_cores_ipc_func, (void *)&ctrl),
+                      ipc_err,
+                      DEBUG_HELPER_TAG,
+                      "IPC call failed");
+    // Wait for other core to confirm its ready for backtracing
+    while (!ctrl.start_tracing) {
+        ;
+    }
+#endif // !CONFIG_FREERTOS_UNICORE
+
+    // Suspend the scheduler to prevent task switching
+    vTaskSuspendAll();
+
+    /*
+    Initialize backtracing for this core:
+
+    - Flush current core's register windows back onto current task's stack using esp_backtrace_get_start()
+    - Get starting frame for backtracing (starting frame is the caller of this function) using esp_backtrace_get_start()
+    - Save the starting frame details into the control block
+    */
+    BaseType_t core_id = xPortGetCoreID();  // Get core ID now that task switching is disabled
+    ctrl.cur_tasks[core_id].task_hdl = xTaskGetCurrentTaskHandle();
+    esp_backtrace_get_start(&ctrl.cur_tasks[core_id].starting_pc,
+                            &ctrl.cur_tasks[core_id].starting_sp,
+                            &ctrl.cur_tasks[core_id].next_pc);
+
+    // Get snapshot of all tasks in the system
+    const UBaseType_t num_snapshots = MIN(num_tasks, uxTaskGetSnapshotAll(task_snapshots, num_tasks, NULL));
+    // Print the backtrace of every task in the system
+    for (UBaseType_t task_idx = 0; task_idx < num_snapshots; task_idx++) {
+        bool cur_running = false;
+        TaskHandle_t task_hdl = (TaskHandle_t) task_snapshots[task_idx].pxTCB;
+        esp_backtrace_frame_t stk_frame;
+
+        // Check if the task is one of the currently running tasks
+        for (BaseType_t core_id = 0; core_id < configNUMBER_OF_CORES; core_id++) {
+            if (task_hdl == ctrl.cur_tasks[core_id].task_hdl) {
+                cur_running = true;
+                break;
+            }
+        }
+        // Initialize the starting backtrace frame of the task
+        if (cur_running) {
+            /*
+            Setting the starting backtrace frame for currently running tasks is different. We cannot
+            use the current frame of each running task as the starting frame (due to the possibility
+            of the SP changing). Thus, each currently running task will have initialized their callers
+            as the starting frame for backtracing, which is saved inside the
+            cur_task_backtrace_ctrl_t block.
+            */
+            stk_frame.pc = ctrl.cur_tasks[core_id].starting_pc;
+            stk_frame.sp = ctrl.cur_tasks[core_id].starting_sp;
+            stk_frame.next_pc = ctrl.cur_tasks[core_id].next_pc;
+        } else {
+            // Set the starting backtrace frame using the task's saved stack pointer
+            XtExcFrame* exc_frame = (XtExcFrame*) task_snapshots[task_idx].pxTopOfStack;
+            stk_frame.pc = exc_frame->pc;
+            stk_frame.sp = exc_frame->a1;
+            stk_frame.next_pc = exc_frame->a0;
+        }
+        // Print backtrace
+        char* name = pcTaskGetName(task_hdl);
+        print_str(name ? name : "No Name" , false);
+        esp_err_t bt_ret = esp_backtrace_print_from_frame(depth, &stk_frame, false);
+        if (bt_ret != ESP_OK) {
+            ret = bt_ret;
+        }
+    }
+
+    // Resume the scheduler to allow task switching again
+    xTaskResumeAll();
+#if !CONFIG_FREERTOS_UNICORE
+    // Indicate to the other core that backtracing is complete
+    ctrl.finished_tracing = true;
+#endif // !CONFIG_FREERTOS_UNICORE
+    free(task_snapshots);
+    return ret;
+
+#if !CONFIG_FREERTOS_UNICORE
+ipc_err:
+    free(task_snapshots);
+#endif // !CONFIG_FREERTOS_UNICORE
+malloc_err:
+    return ret;
+}

components/esp_system/test_apps/esp_system_unity_tests/main/test_backtrace.c

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2021-2023 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -10,12 +10,14 @@
  */
 #include <stdlib.h>
 #include "unity.h"
+#include "test_utils.h"
 
 #if __XTENSA__
 
 #include "freertos/FreeRTOS.h"
 #include "freertos/task.h"
 #include "xtensa_api.h" // Replace with interrupt allocator API (IDF-3891)
+#include "esp_debug_helpers.h"
 #include "esp_intr_alloc.h"
 #include "esp_rom_sys.h"
 #include "esp_rom_uart.h"
@@ -26,6 +28,8 @@
 #define RECUR_DEPTH             3
 #define ACTION_ABORT            -1
 #define ACTION_INT_WDT          -2
+#define TASK_PRIORITY           5
+
 
 // Set to (-1) for abort(), (-2) for interrupt watchdog
 static int backtrace_trigger_source;
@@ -118,4 +122,32 @@ TEST_CASE_MULTIPLE_STAGES("Test backtrace with a ROM function", "[reset_reason][
                            do_rom_crash,
                            check_reset_reason_panic)
 
+
+#define NUM_TEST_FUNCS      2
+
+static void backtrace_suspend_func(void *arg)
+{
+    // Simply suspend and wait to be deleted
+    vTaskSuspend(NULL);
+}
+
+TEST_CASE("Test esp_backtrace_print_all_tasks()", "[esp_system]")
+{
+    TaskHandle_t task_handles[NUM_TEST_FUNCS];
+
+    for (int i = 0; i < NUM_TEST_FUNCS; i++) {
+        // Create multiple unpinned tasks at higher priorities
+        xTaskCreate(backtrace_suspend_func, "trace_func", 2048, NULL, UNITY_FREERTOS_PRIORITY + i + 1, &task_handles[i]);
+    }
+    // Short delay to allow tasks to suspend
+    vTaskDelay(10);
+    // Print backtraces of all tasks
+    esp_backtrace_print_all_tasks(3);
+
+    // Clean up tasks
+    for (int i = 0; i < NUM_TEST_FUNCS; i++) {
+        vTaskDelete(task_handles[i]);
+    }
+}
+
 #endif

components/freertos/esp_additions/freertos_tasks_c_additions.h

@@ -1104,7 +1104,10 @@ UBaseType_t uxTaskGetSnapshotAll( TaskSnapshot_t * const pxTaskSnapshotArray,
         pxCurTaskList = pxGetNextTaskList( pxCurTaskList );
     }
 
-    *pxTCBSize = sizeof( TCB_t );
+    if (pxTCBSize != NULL) {
+        *pxTCBSize = sizeof( TCB_t );
+    }
+
     return uxArrayNumFilled;
 }
 /*----------------------------------------------------------*/

components/freertos/esp_additions/include/esp_private/freertos_debug.h

@@ -73,7 +73,7 @@ BaseType_t vTaskGetSnapshot( TaskHandle_t pxTask,
  *        does not acquire any locks.
  * @param[out] pxTaskSnapshotArray Array of TaskSnapshot_t structures filled by this function
  * @param[in] uxArrayLength Length of the provided array
- * @param[out] pxTCBSize Size of the a task's TCB structure
+ * @param[out] pxTCBSize Size of the a task's TCB structure (can be set to NULL)
  * @return UBaseType_t
  */
 UBaseType_t uxTaskGetSnapshotAll( TaskSnapshot_t * const pxTaskSnapshotArray,