Merge branch 'fix/aes_operation_using_psram_memory_with_psram_enc' into 'master'

Enable AXI-DMA AES-ECC mean access when external memory encryption is enabled

See merge request espressif/esp-idf!30379
This commit is contained in:
Mahavir Jain 2024-05-14 13:15:33 +08:00
commit ee8a9e8410
16 changed files with 228 additions and 33 deletions

View File

@ -17,6 +17,7 @@
#include "esp_image_format.h"
#include "esp_secure_boot.h"
#include "esp_flash_encrypt.h"
#include "spi_flash_mmap.h"
#include "sdkconfig.h"
#include "esp_ota_ops.h"

View File

@ -13,6 +13,9 @@
#include "hal/efuse_ll.h"
#include "hal/efuse_hal.h"
#ifndef BOOTLOADER_BUILD
#include "spi_flash_mmap.h"
#endif
#include "hal/spi_flash_ll.h"
#include "rom/spi_flash.h"
#if CONFIG_IDF_TARGET_ESP32

View File

@ -9,9 +9,6 @@
#include "esp_attr.h"
#include "esp_err.h"
#include "soc/soc_caps.h"
#ifndef BOOTLOADER_BUILD
#include "spi_flash_mmap.h"
#endif
#include "hal/efuse_ll.h"
#include "sdkconfig.h"

View File

@ -14,7 +14,6 @@
#include "bootloader_random.h"
#include "bootloader_clock.h"
#include "bootloader_common.h"
#include "esp_flash_encrypt.h"
#include "esp_cpu.h"
#include "soc/rtc.h"
#include "hal/wdt_hal.h"

View File

@ -34,6 +34,9 @@
#include "esp_app_desc.h"
#include "esp_secure_boot.h"
#include "esp_flash_encrypt.h"
#ifndef BOOTLOADER_BUILD
#include "spi_flash_mmap.h"
#endif
#include "esp_flash_partitions.h"
#include "bootloader_flash_priv.h"
#include "bootloader_random.h"

View File

@ -27,6 +27,7 @@
#if !CONFIG_IDF_TARGET_LINUX
#include "esp_flash.h"
#include "esp_flash_encrypt.h"
#include "spi_flash_mmap.h"
#endif
#include "esp_log.h"
#include "esp_rom_md5.h"

View File

@ -93,7 +93,6 @@
#include "bootloader_flash_config.h"
#include "bootloader_flash.h"
#include "esp_private/crosscore_int.h"
#include "esp_flash_encrypt.h"
#include "esp_private/sleep_gpio.h"
#include "hal/wdt_hal.h"
@ -114,6 +113,7 @@
#include "esp_rom_spiflash.h"
#include "bootloader_init.h"
#include "esp_private/bootloader_flash_internal.h"
#include "spi_flash_mmap.h"
#endif // CONFIG_APP_BUILD_TYPE_RAM
//This dependency will be removed in the future

View File

@ -12,6 +12,7 @@
#include "esp_flash_encrypt.h"
#include "esp_rom_crc.h"
#include "esp_private/spi_flash_os.h"
#include "spi_flash_mmap.h"
#define BLANK_COREDUMP_SIZE 0xFFFFFFFF

View File

@ -273,6 +273,14 @@ static inline void axi_dma_ll_rx_enable_etm_task(axi_dma_dev_t *dev, uint32_t ch
dev->in[channel].conf.in_conf0.in_etm_en_chn = enable;
}
/**
* @brief Whether to enable the mean access ecc or aes domain
*/
static inline void axi_dma_ll_rx_enable_ext_mem_ecc_aes_access(axi_dma_dev_t *dev, uint32_t channel, bool enable)
{
dev->in[channel].conf.in_conf0.in_ecc_aec_en_chn = enable;
}
///////////////////////////////////// TX /////////////////////////////////////////
/**
* @brief Get DMA TX channel interrupt status word
@ -471,6 +479,14 @@ static inline void axi_dma_ll_tx_enable_etm_task(axi_dma_dev_t *dev, uint32_t ch
dev->out[channel].conf.out_conf0.out_etm_en_chn = enable;
}
/**
* @brief Whether to enable the mean access ecc or aes domain
*/
static inline void axi_dma_ll_tx_enable_ext_mem_ecc_aes_access(axi_dma_dev_t *dev, uint32_t channel, bool enable)
{
dev->out[channel].conf.out_conf0.out_ecc_aec_en_chn = enable;
}
///////////////////////////////////// CRC-TX /////////////////////////////////////////
/**

View File

@ -200,6 +200,9 @@ endif()
if(SHA_PERIPHERAL_TYPE STREQUAL "dma" OR AES_PERIPHERAL_TYPE STREQUAL "dma")
target_link_libraries(mbedcrypto PRIVATE idf::esp_mm)
if(CONFIG_SOC_SHA_GDMA OR CONFIG_SOC_AES_GDMA)
if(CONFIG_SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT)
target_link_libraries(mbedcrypto PRIVATE idf::bootloader_support)
endif()
target_sources(mbedcrypto PRIVATE "${COMPONENT_DIR}/port/crypto_shared_gdma/esp_crypto_shared_gdma.c")
endif()
endif()

View File

@ -16,6 +16,8 @@
#include "esp_memory_utils.h"
#include "esp_private/esp_cache_private.h"
#include "esp_private/periph_ctrl.h"
#include "soc/soc_caps.h"
#include "sdkconfig.h"
#if CONFIG_PM_ENABLE
#include "esp_pm.h"
@ -36,10 +38,19 @@
#include "aes/esp_aes_gcm.h"
#endif
#ifdef SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT
#include "esp_flash_encrypt.h"
#endif /* SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT */
/* Max size of each chunk to process when output buffer is in unaligned external ram
must be a multiple of block size
*/
#if (CONFIG_IDF_TARGET_ESP32P4 && CONFIG_SPIRAM && SOC_PSRAM_DMA_CAPABLE)
/* As P4 has larger memory than other targets, thus we can support a larger chunk write size */
#define AES_MAX_CHUNK_WRITE_SIZE 8*1024
#else
#define AES_MAX_CHUNK_WRITE_SIZE 1600
#endif
/* Input over this length will yield and wait for interrupt instead of
busy-waiting, 30000 bytes is approx 0.5 ms */
@ -163,6 +174,26 @@ static int esp_aes_dma_wait_complete(bool use_intr, crypto_dma_desc_t *output_de
return 0;
}
static inline size_t get_cache_line_size(const void *addr)
{
esp_err_t ret = ESP_FAIL;
size_t cache_line_size = 0;
#if (CONFIG_SPIRAM && SOC_PSRAM_DMA_CAPABLE)
if (esp_ptr_external_ram(addr)) {
ret = esp_cache_get_alignment(MALLOC_CAP_SPIRAM, &cache_line_size);
} else
#endif
{
ret = esp_cache_get_alignment(MALLOC_CAP_DMA, &cache_line_size);
}
if (ret != ESP_OK) {
return 0;
}
return cache_line_size;
}
/* Output buffers in external ram needs to be 16-byte aligned and DMA can't access input in the iCache mem range,
reallocate them into internal memory and encrypt in chunks to avoid
@ -176,13 +207,29 @@ static int esp_aes_process_dma_ext_ram(esp_aes_context *ctx, const unsigned char
size_t chunk_len;
int ret = 0;
int offset = 0;
uint32_t heap_caps = 0;
unsigned char *input_buf = NULL;
unsigned char *output_buf = NULL;
const unsigned char *dma_input;
chunk_len = MIN(AES_MAX_CHUNK_WRITE_SIZE, len);
size_t input_alignment = 1;
size_t output_alignment = 1;
/* When AES-DMA operations are carried out using external memory with external memory encryption enabled,
we need to make sure that the addresses and the sizes of the buffers on which the DMA operates are 16 byte-aligned. */
#ifdef SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT
if (esp_flash_encryption_enabled()) {
if (esp_ptr_external_ram(input) || esp_ptr_external_ram(output) || esp_ptr_in_drom(input) || esp_ptr_in_drom(output)) {
input_alignment = MAX(get_cache_line_size(input), SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT);
output_alignment = MAX(get_cache_line_size(output), SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT);
}
}
#endif /* SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT */
if (realloc_input) {
input_buf = heap_caps_malloc(chunk_len, MALLOC_CAP_DMA);
heap_caps = MALLOC_CAP_8BIT | (esp_ptr_external_ram(input) ? MALLOC_CAP_SPIRAM : MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL);
input_buf = heap_caps_aligned_alloc(input_alignment, chunk_len, heap_caps);
if (input_buf == NULL) {
mbedtls_platform_zeroize(output, len);
@ -192,7 +239,8 @@ static int esp_aes_process_dma_ext_ram(esp_aes_context *ctx, const unsigned char
}
if (realloc_output) {
output_buf = heap_caps_malloc(chunk_len, MALLOC_CAP_DMA);
heap_caps = MALLOC_CAP_8BIT | (esp_ptr_external_ram(output) ? MALLOC_CAP_SPIRAM : MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL);
output_buf = heap_caps_aligned_alloc(output_alignment, chunk_len, heap_caps);
if (output_buf == NULL) {
mbedtls_platform_zeroize(output, len);
@ -284,27 +332,6 @@ static inline void *aes_dma_calloc(size_t num, size_t size, uint32_t caps, size_
return ptr;
}
static inline size_t get_cache_line_size(const void *addr)
{
esp_err_t ret = ESP_FAIL;
size_t cache_line_size = 0;
#if (CONFIG_SPIRAM && SOC_PSRAM_DMA_CAPABLE)
if (esp_ptr_external_ram(addr)) {
ret = esp_cache_get_alignment(MALLOC_CAP_SPIRAM, &cache_line_size);
} else
#endif
{
ret = esp_cache_get_alignment(MALLOC_CAP_DMA, &cache_line_size);
}
if (ret != ESP_OK) {
return 0;
}
return cache_line_size;
}
static inline esp_err_t dma_desc_link(crypto_dma_desc_t *dmadesc, size_t crypto_dma_desc_num, size_t cache_line_size)
{
esp_err_t ret = ESP_OK;
@ -404,7 +431,7 @@ static esp_err_t generate_descriptor_list(const uint8_t *buffer, const size_t le
dma_descs_needed = (unaligned_start_bytes ? 1 : 0) + dma_desc_get_required_num(aligned_block_bytes, max_desc_size) + (unaligned_end_bytes ? 1 : 0);
/* Allocate memory for DMA descriptors of total size aligned up to a multiple of cache line size */
dma_descriptors = (crypto_dma_desc_t *) aes_dma_calloc(dma_descs_needed, sizeof(crypto_dma_desc_t), MALLOC_CAP_DMA, NULL);
dma_descriptors = (crypto_dma_desc_t *) aes_dma_calloc(dma_descs_needed, sizeof(crypto_dma_desc_t), MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL, NULL);
if (dma_descriptors == NULL) {
ESP_LOGE(TAG, "Failed to allocate memory for the array of DMA descriptors");
return ESP_FAIL;
@ -413,7 +440,7 @@ static esp_err_t generate_descriptor_list(const uint8_t *buffer, const size_t le
size_t populated_dma_descs = 0;
if (unaligned_start_bytes) {
start_alignment_stream_buffer = aes_dma_calloc(alignment_buffer_size, sizeof(uint8_t), AES_DMA_ALLOC_CAPS, NULL);
start_alignment_stream_buffer = aes_dma_calloc(alignment_buffer_size, sizeof(uint8_t), AES_DMA_ALLOC_CAPS | (esp_ptr_external_ram(buffer) ? MALLOC_CAP_SPIRAM : MALLOC_CAP_INTERNAL) , NULL);
if (start_alignment_stream_buffer == NULL) {
ESP_LOGE(TAG, "Failed to allocate memory for start alignment buffer");
return ESP_FAIL;
@ -435,7 +462,7 @@ static esp_err_t generate_descriptor_list(const uint8_t *buffer, const size_t le
}
if (unaligned_end_bytes) {
end_alignment_stream_buffer = aes_dma_calloc(alignment_buffer_size, sizeof(uint8_t), AES_DMA_ALLOC_CAPS, NULL);
end_alignment_stream_buffer = aes_dma_calloc(alignment_buffer_size, sizeof(uint8_t), AES_DMA_ALLOC_CAPS | (esp_ptr_external_ram(buffer) ? MALLOC_CAP_SPIRAM : MALLOC_CAP_INTERNAL), NULL);
if (end_alignment_stream_buffer == NULL) {
ESP_LOGE(TAG, "Failed to allocate memory for end alignment buffer");
return ESP_FAIL;
@ -499,6 +526,20 @@ int esp_aes_process_dma(esp_aes_context *ctx, const unsigned char *input, unsign
return MBEDTLS_ERR_AES_INVALID_INPUT_LENGTH;
}
#ifdef SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT
if (esp_flash_encryption_enabled()) {
if (esp_ptr_external_ram(input) || esp_ptr_external_ram(output) || esp_ptr_in_drom(input) || esp_ptr_in_drom(output)) {
if (((intptr_t)(input) & (SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT - 1)) != 0) {
input_needs_realloc = true;
}
if (((intptr_t)(output) & (SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT - 1)) != 0) {
output_needs_realloc = true;
}
}
}
#endif /* SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT */
/* DMA cannot access memory in the iCache range, copy input to internal ram */
if (!s_check_dma_capable(input)) {
input_needs_realloc = true;

View File

@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: 2021-2023 Espressif Systems (Shanghai) CO LTD
* SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
@ -13,7 +13,13 @@
#include "esp_cache.h"
#include "esp_crypto_dma.h"
#include "esp_crypto_lock.h"
#include "esp_memory_utils.h"
#include "soc/soc_caps.h"
#include "sdkconfig.h"
#ifdef SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT
#include "esp_flash_encrypt.h"
#endif /* SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT */
#if SOC_AHB_GDMA_VERSION == 1
#include "hal/gdma_ll.h"
@ -140,6 +146,22 @@ esp_err_t esp_crypto_shared_gdma_start(const lldesc_t *input, const lldesc_t *ou
return ESP_OK;
}
/* The external memory ecc-aes access must be enabled when there exists
at least one buffer in the DMA descriptors that resides in external memory. */
#ifdef SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT
static bool check_dma_descs_need_ext_mem_ecc_aes_access(const crypto_dma_desc_t *dmadesc)
{
crypto_dma_desc_t* desc = (crypto_dma_desc_t*) dmadesc;
while (desc) {
if (esp_ptr_in_drom(desc->buffer) || esp_ptr_external_ram(desc->buffer)) {
return true;
}
desc = desc->next;
}
return false;
}
#endif /* SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT */
esp_err_t esp_crypto_shared_gdma_start_axi_ahb(const crypto_dma_desc_t *input, const crypto_dma_desc_t *output, gdma_trigger_peripheral_t peripheral)
{
int rx_ch_id = 0;
@ -173,6 +195,23 @@ esp_err_t esp_crypto_shared_gdma_start_axi_ahb(const crypto_dma_desc_t *input, c
axi_dma_ll_rx_reset_channel(&AXI_DMA, rx_ch_id);
#endif /* SOC_AHB_GDMA_VERSION */
/* When GDMA operations are carried out using external memory with external memory encryption enabled,
we need to enable AXI-DMA's AES-ECC mean access bit. */
#if (SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT)
if (esp_flash_encryption_enabled()) {
int tx_ch_id = 0;
gdma_get_channel_id(tx_channel, &tx_ch_id);
if (check_dma_descs_need_ext_mem_ecc_aes_access(input) || check_dma_descs_need_ext_mem_ecc_aes_access(output)) {
axi_dma_ll_rx_enable_ext_mem_ecc_aes_access(&AXI_DMA, rx_ch_id, true);
axi_dma_ll_tx_enable_ext_mem_ecc_aes_access(&AXI_DMA, tx_ch_id, true);
} else {
axi_dma_ll_rx_enable_ext_mem_ecc_aes_access(&AXI_DMA, rx_ch_id, false);
axi_dma_ll_tx_enable_ext_mem_ecc_aes_access(&AXI_DMA, tx_ch_id, false);
}
}
#endif /* SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT */
gdma_start(tx_channel, (intptr_t)input);
gdma_start(rx_channel, (intptr_t)output);

View File

@ -53,6 +53,11 @@
#include "hal/sha_ll.h"
#include "soc/soc_caps.h"
#include "esp_sha_dma_priv.h"
#include "sdkconfig.h"
#ifdef SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT
#include "esp_flash_encrypt.h"
#endif /* SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT */
#if SOC_SHA_GDMA
#define SHA_LOCK() esp_crypto_sha_aes_lock_acquire()
@ -168,6 +173,63 @@ static void esp_sha_block_mode(esp_sha_type sha_type, const uint8_t *input, uint
static DRAM_ATTR crypto_dma_desc_t s_dma_descr_input;
static DRAM_ATTR crypto_dma_desc_t s_dma_descr_buf;
static esp_err_t esp_sha_dma_process(esp_sha_type sha_type, const void *input, uint32_t ilen,
const void *buf, uint32_t buf_len, bool is_first_block);
#ifdef SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT
static esp_err_t esp_sha_dma_process_ext(esp_sha_type sha_type, const void *input, uint32_t ilen,
const void *buf, uint32_t buf_len, bool is_first_block,
bool realloc_input, bool realloc_buf)
{
int ret = ESP_FAIL;
void *input_copy = NULL;
void *buf_copy = NULL;
const void *dma_input = NULL;
const void *dma_buf = NULL;
uint32_t heap_caps = 0;
if (realloc_input) {
heap_caps = MALLOC_CAP_8BIT | (esp_ptr_external_ram(input) ? MALLOC_CAP_SPIRAM : MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL);
input_copy = heap_caps_aligned_alloc(SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT, ilen, heap_caps);
if (input_copy == NULL) {
ESP_LOGE(TAG, "Failed to allocate aligned SPIRAM memory");
return ret;
}
memcpy(input_copy, input, ilen);
dma_input = input_copy;
} else {
dma_input = input;
}
if (realloc_buf) {
heap_caps = MALLOC_CAP_8BIT | (esp_ptr_external_ram(buf) ? MALLOC_CAP_SPIRAM : MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL);
buf_copy = heap_caps_aligned_alloc(SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT, buf_len, heap_caps);
if (buf_copy == NULL) {
ESP_LOGE(TAG, "Failed to allocate aligned internal memory");
return ret;
}
memcpy(buf_copy, buf, buf_len);
dma_buf = buf_copy;
} else {
dma_buf = buf;
}
ret = esp_sha_dma_process(sha_type, dma_input, ilen, dma_buf, buf_len, is_first_block);
if (realloc_input) {
free(input_copy);
}
if (realloc_buf) {
free(buf_copy);
}
return ret;
}
#endif /* SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT */
/* Performs SHA on multiple blocks at a time */
static esp_err_t esp_sha_dma_process(esp_sha_type sha_type, const void *input, uint32_t ilen,
const void *buf, uint32_t buf_len, bool is_first_block)
@ -179,6 +241,29 @@ static esp_err_t esp_sha_dma_process(esp_sha_type sha_type, const void *input, u
memset(&s_dma_descr_input, 0, sizeof(crypto_dma_desc_t));
memset(&s_dma_descr_buf, 0, sizeof(crypto_dma_desc_t));
/* When SHA-DMA operations are carried out using external memory with external memory encryption enabled,
we need to make sure that the addresses and the sizes of the buffers on which the DMA operates are 16 byte-aligned. */
#ifdef SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT
if (esp_flash_encryption_enabled()) {
if (esp_ptr_external_ram(input) || esp_ptr_external_ram(buf) || esp_ptr_in_drom(input) || esp_ptr_in_drom(buf)) {
bool input_needs_realloc = false;
bool buf_needs_realloc = false;
if (ilen && ((intptr_t)(input) & (SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT - 1)) != 0) {
input_needs_realloc = true;
}
if (buf_len && ((intptr_t)(buf) & (SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT - 1)) != 0) {
buf_needs_realloc = true;
}
if (input_needs_realloc || buf_needs_realloc) {
return esp_sha_dma_process_ext(sha_type, input, ilen, buf, buf_len, is_first_block, input_needs_realloc, buf_needs_realloc);
}
}
}
#endif /* SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT */
/* DMA descriptor for Memory to DMA-SHA transfer */
if (ilen) {
s_dma_descr_input.dw0.length = ilen;
@ -188,7 +273,7 @@ static esp_err_t esp_sha_dma_process(esp_sha_type sha_type, const void *input, u
s_dma_descr_input.buffer = (void *) input;
dma_descr_head = &s_dma_descr_input;
}
/* Check after input to overide head if there is any buf*/
/* Check after input to override head if there is any buf*/
if (buf_len) {
s_dma_descr_buf.dw0.length = buf_len;
s_dma_descr_buf.dw0.size = buf_len;

View File

@ -17,6 +17,7 @@
#include "esp_log.h"
#include "esp_partition.h"
#include "esp_system.h"
#include "spi_flash_mmap.h"
#include "nvs.h"
#include "nvs_flash.h"

View File

@ -491,6 +491,10 @@ config SOC_GDMA_SUPPORT_ETM
bool
default y
config SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT
int
default 16
config SOC_DMA2D_GROUPS
int
default 1

View File

@ -193,6 +193,7 @@
#define SOC_AXI_GDMA_SUPPORT_PSRAM 1
#define SOC_GDMA_SUPPORT_ETM 1
// #define SOC_GDMA_SUPPORT_SLEEP_RETENTION 1
#define SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT (16)
/*-------------------------- 2D-DMA CAPS -------------------------------------*/
#define SOC_DMA2D_GROUPS (1U) // Number of 2D-DMA groups