diff --git a/components/esp_hw_support/include/spinlock.h b/components/esp_hw_support/include/spinlock.h index c19d088ee3..7501faabea 100644 --- a/components/esp_hw_support/include/spinlock.h +++ b/components/esp_hw_support/include/spinlock.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: 2015-2023 Espressif Systems (Shanghai) CO LTD + * SPDX-FileCopyrightText: 2015-2024 Espressif Systems (Shanghai) CO LTD * * SPDX-License-Identifier: Apache-2.0 */ @@ -87,7 +87,7 @@ static inline bool __attribute__((always_inline)) spinlock_acquire(spinlock_t *l core_owner_id = xt_utils_get_raw_core_id(); #else //__riscv - irq_status = rv_utils_mask_int_level_lower_than(RVHAL_EXCM_LEVEL); + irq_status = rv_utils_set_intlevel_regval(RVHAL_EXCM_LEVEL_CLIC); core_owner_id = rv_utils_get_core_id() == 0 ? SPINLOCK_OWNER_ID_0 : SPINLOCK_OWNER_ID_1; #endif other_core_owner_id = CORE_ID_REGVAL_XOR_SWAP ^ core_owner_id; @@ -106,7 +106,7 @@ static inline bool __attribute__((always_inline)) spinlock_acquire(spinlock_t *l #if __XTENSA__ XTOS_RESTORE_INTLEVEL(irq_status); #else - rv_utils_restore_intlevel(irq_status); + rv_utils_restore_intlevel_regval(irq_status); #endif return true; } @@ -147,7 +147,7 @@ exit: #if __XTENSA__ XTOS_RESTORE_INTLEVEL(irq_status); #else - rv_utils_restore_intlevel(irq_status); + rv_utils_restore_intlevel_regval(irq_status); #endif return lock_set; @@ -181,7 +181,7 @@ static inline void __attribute__((always_inline)) spinlock_release(spinlock_t *l core_owner_id = xt_utils_get_raw_core_id(); #else - irq_status = rv_utils_mask_int_level_lower_than(RVHAL_EXCM_LEVEL); + irq_status = rv_utils_set_intlevel_regval(RVHAL_EXCM_LEVEL_CLIC); core_owner_id = rv_utils_get_core_id() == 0 ? SPINLOCK_OWNER_ID_0 : SPINLOCK_OWNER_ID_1; #endif assert(core_owner_id == lock->owner); // This is a lock that we didn't acquire, or the lock is corrupt @@ -196,7 +196,7 @@ static inline void __attribute__((always_inline)) spinlock_release(spinlock_t *l #if __XTENSA__ XTOS_RESTORE_INTLEVEL(irq_status); #else - rv_utils_restore_intlevel(irq_status); + rv_utils_restore_intlevel_regval(irq_status); #endif //#if __XTENSA__ #endif //#if !CONFIG_ESP_SYSTEM_SINGLE_CORE_MODE && !BOOTLOADER_BUILD } diff --git a/components/freertos/FreeRTOS-Kernel/portable/riscv/include/freertos/portmacro.h b/components/freertos/FreeRTOS-Kernel/portable/riscv/include/freertos/portmacro.h index c656fd21c3..8ce0e1f8e4 100644 --- a/components/freertos/FreeRTOS-Kernel/portable/riscv/include/freertos/portmacro.h +++ b/components/freertos/FreeRTOS-Kernel/portable/riscv/include/freertos/portmacro.h @@ -466,7 +466,11 @@ void vPortTCBPreDeleteHook( void *pxTCB ); // --------------------- Interrupts ------------------------ #define portDISABLE_INTERRUPTS() portSET_INTERRUPT_MASK_FROM_ISR() +#if !SOC_INT_CLIC_SUPPORTED #define portENABLE_INTERRUPTS() portCLEAR_INTERRUPT_MASK_FROM_ISR(RVHAL_INTR_ENABLE_THRESH) +#else +#define portENABLE_INTERRUPTS() portCLEAR_INTERRUPT_MASK_FROM_ISR(RVHAL_INTR_ENABLE_THRESH_CLIC) +#endif /* !SOC_INT_CLIC_SUPPORTED */ /** * ISR versions to enable/disable interrupts diff --git a/components/freertos/FreeRTOS-Kernel/portable/riscv/port.c b/components/freertos/FreeRTOS-Kernel/portable/riscv/port.c index bcb2855c00..4c4ba5c286 100644 --- a/components/freertos/FreeRTOS-Kernel/portable/riscv/port.c +++ b/components/freertos/FreeRTOS-Kernel/portable/riscv/port.c @@ -476,7 +476,7 @@ UBaseType_t xPortSetInterruptMaskFromISR(void) RV_SET_CSR(mstatus, old_mstatus & MSTATUS_MIE); #else /* When CLIC is supported, all interrupt priority levels less than or equal to the threshold level are masked. */ - prev_int_level = rv_utils_mask_int_level_lower_than(RVHAL_EXCM_LEVEL); + prev_int_level = rv_utils_set_intlevel_regval(RVHAL_EXCM_LEVEL_CLIC); #endif /* !SOC_INIT_CLIC_SUPPORTED */ /** * In theory, this function should not return immediately as there is a @@ -497,7 +497,7 @@ void vPortClearInterruptMaskFromISR(UBaseType_t prev_int_level) #if !SOC_INT_CLIC_SUPPORTED REG_WRITE(INTERRUPT_CURRENT_CORE_INT_THRESH_REG, prev_int_level); #else - rv_utils_restore_intlevel(prev_int_level); + rv_utils_restore_intlevel_regval(prev_int_level); #endif /* SOC_INIT_CLIC_SUPPORTED */ /** * The delay between the moment we unmask the interrupt threshold register diff --git a/components/idf_test/include/esp32p4/idf_performance_target.h b/components/idf_test/include/esp32p4/idf_performance_target.h index 4dc137b1c1..d1efc6b4af 100644 --- a/components/idf_test/include/esp32p4/idf_performance_target.h +++ b/components/idf_test/include/esp32p4/idf_performance_target.h @@ -10,3 +10,7 @@ #define IDF_PERFORMANCE_MAX_SPI_PER_TRANS_POLLING 1000 #define IDF_PERFORMANCE_MAX_SPI_PER_TRANS_NO_POLLING_NO_DMA 1000 #define IDF_PERFORMANCE_MAX_SPI_PER_TRANS_POLLING_NO_DMA 1000 + +/* Spinlock performance on esp32p4 is slower. May need to adjust these values once IDF-7898 is fixed */ +#define IDF_PERFORMANCE_MAX_FREERTOS_SPINLOCK_CYCLES_PER_OP 380 +#define IDF_PERFORMANCE_MAX_FREERTOS_SPINLOCK_CYCLES_PER_OP_UNICORE 135 diff --git a/components/riscv/include/riscv/rv_utils.h b/components/riscv/include/riscv/rv_utils.h index be0f8372c2..1fcc758dc1 100644 --- a/components/riscv/include/riscv/rv_utils.h +++ b/components/riscv/include/riscv/rv_utils.h @@ -51,6 +51,19 @@ extern "C" { #define RVHAL_INTR_ENABLE_THRESH 1 #endif /* SOC_INT_CLIC_SUPPORTED */ +/* On CLIC, the interrupt threshold is stored in the upper (NLBITS) of the mintthresh register, with the other (8 - NLBITS) + * defaulted to 1. We form the interrupt level bits here to avoid doing this at run time */ +#if SOC_INT_CLIC_SUPPORTED +/* Helper macro to translate absolute interrupt level to CLIC interrupt threshold bits in the mintthresh reg */ +#define CLIC_INT_THRESH(intlevel) (((((intlevel) << (8 - NLBITS))) | 0x1f) << CLIC_CPU_INT_THRESH_S) + +/* Helper macro to set interrupt level RVHAL_EXCM_LEVEL. Used during critical sections */ +#define RVHAL_EXCM_LEVEL_CLIC (CLIC_INT_THRESH(RVHAL_EXCM_LEVEL - 1)) + +/* Helper macro to enable interrupts. */ +#define RVHAL_INTR_ENABLE_THRESH_CLIC (CLIC_INT_THRESH(RVHAL_INTR_ENABLE_THRESH)) +#endif /* SOC_INT_CLIC_SUPPORTED */ + /* --------------------------------------------------- CPU Control ----------------------------------------------------- * * ------------------------------------------------------------------------------------------------------------------ */ @@ -192,6 +205,38 @@ FORCE_INLINE_ATTR uint32_t __attribute__((always_inline)) rv_utils_set_intlevel( return old_thresh; } +/* Direct register write version of rv_utils_restore_intlevel(). Used to speed up critical sections. */ +FORCE_INLINE_ATTR void __attribute__((always_inline)) rv_utils_restore_intlevel_regval(uint32_t restoreval) +{ + /* This function expects restoreval to be in the format needed to restore the interrupt level with + * a single write to the mintthresh register without further manipulations needed. + * This is done to quicken up exit for critical sections */ + REG_WRITE(CLIC_INT_THRESH_REG, restoreval); +} + +/* Direct register write version of rv_utils_set_intlevel(). Used to speed up critical sections. */ +FORCE_INLINE_ATTR uint32_t __attribute__((always_inline)) rv_utils_set_intlevel_regval(uint32_t intlevel) +{ + uint32_t old_mstatus = RV_CLEAR_CSR(mstatus, MSTATUS_MIE); + uint32_t old_thresh = REG_READ(CLIC_INT_THRESH_REG); + + /* This function expects the interrupt level to be available in the format needed for mintthresh reg. + * Providing an absolute interrupt level will result in incorrect behavior. + * See CLIC_INT_THRESH() macro for details of how the interrupt level must be provided. */ + REG_WRITE(CLIC_INT_THRESH_REG, intlevel); + /** + * After writing the threshold register, the new threshold is not directly taken into account by the CPU. + * By executing ~8 nop instructions, or by performing a memory load right now, the previous memory write + * operations is forced, making the new threshold active. It is then safe to re-enable MIE bit in mstatus. + */ + REG_READ(CLIC_INT_THRESH_REG); + RV_SET_CSR(mstatus, old_mstatus & MSTATUS_MIE); + + /* We return the mintthresh register value and NOT the absolute interrupt threshold level. + * This is done to avoid extra bit manipulations during critical sections. */ + return old_thresh; +} + FORCE_INLINE_ATTR uint32_t __attribute__((always_inline)) rv_utils_mask_int_level_lower_than(uint32_t intlevel) { /* CLIC's set interrupt level is inclusive, i.e. it does mask the set level */