mirror of
https://github.com/espressif/esp-idf.git
synced 2024-10-05 20:47:46 -04:00
crypto: allocate all DMA descriptors to DMA capable memory.
These were previously placed on the stack, but the stack could be placed in RTC RAM which is not DMA capable.
This commit is contained in:
parent
c2f14e192e
commit
b957692888
@ -1,16 +1,8 @@
|
||||
// Copyright 2020 Espressif Systems (Shanghai) PTE LTD
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020-2021 Espressif Systems (Shanghai) CO LTD
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
|
@ -1,23 +1,15 @@
|
||||
// Copyright 2020 Espressif Systems (Shanghai) PTE LTD
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020-2021 Espressif Systems (Shanghai) CO LTD
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#define IDF_PERFORMANCE_MIN_AES_CBC_THROUGHPUT_MBSEC 43
|
||||
|
||||
// SHA256 hardware throughput at 240MHz, threshold set lower than worst case
|
||||
#define IDF_PERFORMANCE_MIN_SHA256_THROUGHPUT_MBSEC 19.8
|
||||
#define IDF_PERFORMANCE_MIN_SHA256_THROUGHPUT_MBSEC 90
|
||||
// esp_sha() time to process 32KB of input data from RAM
|
||||
#define IDF_PERFORMANCE_MAX_TIME_SHA1_32KB 1000
|
||||
#define IDF_PERFORMANCE_MAX_TIME_SHA512_32KB 900
|
||||
|
@ -31,6 +31,7 @@
|
||||
#include "esp_intr_alloc.h"
|
||||
#include "driver/periph_ctrl.h"
|
||||
#include "esp_log.h"
|
||||
#include "esp_attr.h"
|
||||
#include "soc/lldesc.h"
|
||||
#include "esp_heap_caps.h"
|
||||
#include "sys/param.h"
|
||||
@ -81,6 +82,14 @@ static esp_pm_lock_handle_t s_pm_sleep_lock;
|
||||
|
||||
static const char *TAG = "esp-aes";
|
||||
|
||||
/* These are static due to:
|
||||
* * Must be in DMA capable memory, so stack is not a safe place to put them
|
||||
* * To avoid having to malloc/free them for every DMA operation
|
||||
*/
|
||||
static DRAM_ATTR lldesc_t s_stream_in_desc;
|
||||
static DRAM_ATTR lldesc_t s_stream_out_desc;
|
||||
static DRAM_ATTR uint8_t s_stream_in[AES_BLOCK_BYTES];
|
||||
static DRAM_ATTR uint8_t s_stream_out[AES_BLOCK_BYTES];
|
||||
|
||||
static inline void esp_aes_wait_dma_done(lldesc_t *output)
|
||||
{
|
||||
@ -289,15 +298,12 @@ cleanup:
|
||||
/* Encrypt/decrypt the input using DMA */
|
||||
static int esp_aes_process_dma(esp_aes_context *ctx, const unsigned char *input, unsigned char *output, size_t len, uint8_t *stream_out)
|
||||
{
|
||||
lldesc_t stream_in_desc, stream_out_desc;
|
||||
lldesc_t *in_desc_head = NULL, *out_desc_head = NULL;
|
||||
lldesc_t *out_desc_tail = NULL; /* pointer to the final output descriptor */
|
||||
lldesc_t *block_desc = NULL, *block_in_desc = NULL, *block_out_desc = NULL;
|
||||
size_t lldesc_num;
|
||||
uint8_t stream_in[16] = {};
|
||||
unsigned stream_bytes = len % AES_BLOCK_BYTES; // bytes which aren't in a full block
|
||||
unsigned block_bytes = len - stream_bytes; // bytes which are in a full block
|
||||
unsigned char *non_icache_input = NULL;
|
||||
unsigned blocks = (block_bytes / AES_BLOCK_BYTES) + ((stream_bytes > 0) ? 1 : 0);
|
||||
bool use_intr = false;
|
||||
bool input_needs_realloc = false;
|
||||
@ -321,7 +327,7 @@ static int esp_aes_process_dma(esp_aes_context *ctx, const unsigned char *input,
|
||||
/* Flush cache if input in external ram */
|
||||
#if (CONFIG_SPIRAM_USE_CAPS_ALLOC || CONFIG_SPIRAM_USE_MALLOC)
|
||||
if (esp_ptr_external_ram(input)) {
|
||||
Cache_WriteBack_All();
|
||||
Cache_WriteBack_Addr((uint32_t)input, len);
|
||||
}
|
||||
if (esp_ptr_external_ram(output)) {
|
||||
if (((intptr_t)(output) & 0xF) != 0) {
|
||||
@ -348,7 +354,7 @@ static int esp_aes_process_dma(esp_aes_context *ctx, const unsigned char *input,
|
||||
lldesc_num = lldesc_get_required_num(block_bytes);
|
||||
|
||||
/* Allocate both in and out descriptors to save a malloc/free per function call */
|
||||
block_desc = heap_caps_malloc(sizeof(lldesc_t) * lldesc_num * 2, MALLOC_CAP_DMA);
|
||||
block_desc = heap_caps_calloc(lldesc_num * 2, sizeof(lldesc_t), MALLOC_CAP_DMA);
|
||||
if (block_desc == NULL) {
|
||||
ESP_LOGE(TAG, "Failed to allocate memory");
|
||||
ret = -1;
|
||||
@ -366,23 +372,30 @@ static int esp_aes_process_dma(esp_aes_context *ctx, const unsigned char *input,
|
||||
|
||||
/* Any leftover bytes which are appended as an additional DMA list */
|
||||
if (stream_bytes > 0) {
|
||||
memcpy(stream_in, input + block_bytes, stream_bytes);
|
||||
|
||||
lldesc_setup_link(&stream_in_desc, stream_in, AES_BLOCK_BYTES, 0);
|
||||
lldesc_setup_link(&stream_out_desc, stream_out, AES_BLOCK_BYTES, 0);
|
||||
memset(&s_stream_in_desc, 0, sizeof(lldesc_t));
|
||||
memset(&s_stream_out_desc, 0, sizeof(lldesc_t));
|
||||
|
||||
memset(s_stream_in, 0, AES_BLOCK_BYTES);
|
||||
memset(s_stream_out, 0, AES_BLOCK_BYTES);
|
||||
|
||||
memcpy(s_stream_in, input + block_bytes, stream_bytes);
|
||||
|
||||
lldesc_setup_link(&s_stream_in_desc, s_stream_in, AES_BLOCK_BYTES, 0);
|
||||
lldesc_setup_link(&s_stream_out_desc, s_stream_out, AES_BLOCK_BYTES, 0);
|
||||
|
||||
if (block_bytes > 0) {
|
||||
/* Link with block descriptors*/
|
||||
block_in_desc[lldesc_num - 1].empty = (uint32_t)&stream_in_desc;
|
||||
block_out_desc[lldesc_num - 1].empty = (uint32_t)&stream_out_desc;
|
||||
block_in_desc[lldesc_num - 1].empty = (uint32_t)&s_stream_in_desc;
|
||||
block_out_desc[lldesc_num - 1].empty = (uint32_t)&s_stream_out_desc;
|
||||
}
|
||||
|
||||
out_desc_tail = &stream_out_desc;
|
||||
out_desc_tail = &s_stream_out_desc;
|
||||
}
|
||||
|
||||
// block buffers are sent to DMA first, unless there aren't any
|
||||
in_desc_head = (block_bytes > 0) ? block_in_desc : &stream_in_desc;
|
||||
out_desc_head = (block_bytes > 0) ? block_out_desc : &stream_out_desc;
|
||||
in_desc_head = (block_bytes > 0) ? block_in_desc : &s_stream_in_desc;
|
||||
out_desc_head = (block_bytes > 0) ? block_out_desc : &s_stream_out_desc;
|
||||
|
||||
|
||||
#if defined (CONFIG_MBEDTLS_AES_USE_INTERRUPT)
|
||||
@ -411,18 +424,18 @@ static int esp_aes_process_dma(esp_aes_context *ctx, const unsigned char *input,
|
||||
#if (CONFIG_SPIRAM_USE_CAPS_ALLOC || CONFIG_SPIRAM_USE_MALLOC)
|
||||
if (block_bytes > 0) {
|
||||
if (esp_ptr_external_ram(output)) {
|
||||
Cache_Invalidate_DCache_All();
|
||||
Cache_Invalidate_Addr((uint32_t)output, block_bytes);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
aes_hal_transform_dma_finish();
|
||||
|
||||
if (stream_bytes > 0) {
|
||||
memcpy(output + block_bytes, stream_out, stream_bytes);
|
||||
memcpy(output + block_bytes, s_stream_out, stream_bytes);
|
||||
memcpy(stream_out, s_stream_out, AES_BLOCK_BYTES);
|
||||
}
|
||||
|
||||
cleanup:
|
||||
free(non_icache_input);
|
||||
free(block_desc);
|
||||
return ret;
|
||||
}
|
||||
|
@ -490,7 +490,7 @@ int esp_aes_gcm_finish( esp_gcm_context *ctx,
|
||||
/* Due to restrictions in the hardware (e.g. need to do the whole conversion in one go),
|
||||
some combinations of inputs are not supported */
|
||||
static bool esp_aes_gcm_input_support_hw_accel(size_t length, const unsigned char *aad, size_t aad_len,
|
||||
const unsigned char *input, unsigned char *output)
|
||||
const unsigned char *input, unsigned char *output, uint8_t *stream_in)
|
||||
{
|
||||
bool support_hw_accel = true;
|
||||
|
||||
@ -505,10 +505,15 @@ static bool esp_aes_gcm_input_support_hw_accel(size_t length, const unsigned cha
|
||||
} else if (!esp_ptr_dma_capable(output) && length > 0) {
|
||||
/* output in non internal DMA memory */
|
||||
support_hw_accel = false;
|
||||
} else if (!esp_ptr_dma_capable(stream_in)) {
|
||||
/* Stream in (and therefor other descriptors and buffers that come from the stack)
|
||||
in non internal DMA memory */
|
||||
support_hw_accel = false;
|
||||
} else if (length == 0) {
|
||||
support_hw_accel = false;
|
||||
}
|
||||
|
||||
|
||||
return support_hw_accel;
|
||||
}
|
||||
|
||||
@ -562,7 +567,7 @@ int esp_aes_gcm_crypt_and_tag( esp_gcm_context *ctx,
|
||||
unsigned block_bytes = aad_len - stream_bytes; // bytes which are in a full block
|
||||
|
||||
/* Due to hardware limition only certain cases are fully supported in HW */
|
||||
if (!esp_aes_gcm_input_support_hw_accel(length, aad, aad_len, input, output)) {
|
||||
if (!esp_aes_gcm_input_support_hw_accel(length, aad, aad_len, input, output, stream_in)) {
|
||||
return esp_aes_gcm_crypt_and_tag_partial_hw(ctx, mode, length, iv, iv_len, aad, aad_len, input, output, tag_len, tag);
|
||||
}
|
||||
|
||||
|
@ -31,6 +31,7 @@
|
||||
|
||||
#include "esp_log.h"
|
||||
#include "esp_crypto_lock.h"
|
||||
#include "esp_attr.h"
|
||||
#include "soc/lldesc.h"
|
||||
#include "soc/cache_memory.h"
|
||||
#include "soc/periph_defs.h"
|
||||
@ -64,6 +65,12 @@
|
||||
|
||||
const static char *TAG = "esp-sha";
|
||||
|
||||
/* These are static due to:
|
||||
* * Must be in DMA capable memory, so stack is not a safe place to put them
|
||||
* * To avoid having to malloc/free them for every DMA operation
|
||||
*/
|
||||
static DRAM_ATTR lldesc_t s_dma_descr_input;
|
||||
static DRAM_ATTR lldesc_t s_dma_descr_buf;
|
||||
|
||||
void esp_sha_write_digest_state(esp_sha_type sha_type, void *digest_state)
|
||||
{
|
||||
@ -273,34 +280,35 @@ static esp_err_t esp_sha_dma_process(esp_sha_type sha_type, const void *input, u
|
||||
const void *buf, uint32_t buf_len, bool is_first_block)
|
||||
{
|
||||
int ret = 0;
|
||||
lldesc_t dma_descr_input = {};
|
||||
lldesc_t dma_descr_buf = {};
|
||||
lldesc_t *dma_descr_head;
|
||||
size_t num_blks = (ilen + buf_len) / block_length(sha_type);
|
||||
|
||||
memset(&s_dma_descr_input, 0, sizeof(lldesc_t));
|
||||
memset(&s_dma_descr_buf, 0, sizeof(lldesc_t));
|
||||
|
||||
/* DMA descriptor for Memory to DMA-SHA transfer */
|
||||
if (ilen) {
|
||||
dma_descr_input.length = ilen;
|
||||
dma_descr_input.size = ilen;
|
||||
dma_descr_input.owner = 1;
|
||||
dma_descr_input.eof = 1;
|
||||
dma_descr_input.buf = (uint8_t *)input;
|
||||
dma_descr_head = &dma_descr_input;
|
||||
s_dma_descr_input.length = ilen;
|
||||
s_dma_descr_input.size = ilen;
|
||||
s_dma_descr_input.owner = 1;
|
||||
s_dma_descr_input.eof = 1;
|
||||
s_dma_descr_input.buf = (uint8_t *)input;
|
||||
dma_descr_head = &s_dma_descr_input;
|
||||
}
|
||||
/* Check after input to overide head if there is any buf*/
|
||||
if (buf_len) {
|
||||
dma_descr_buf.length = buf_len;
|
||||
dma_descr_buf.size = buf_len;
|
||||
dma_descr_buf.owner = 1;
|
||||
dma_descr_buf.eof = 1;
|
||||
dma_descr_buf.buf = (uint8_t *)buf;
|
||||
dma_descr_head = &dma_descr_buf;
|
||||
s_dma_descr_buf.length = buf_len;
|
||||
s_dma_descr_buf.size = buf_len;
|
||||
s_dma_descr_buf.owner = 1;
|
||||
s_dma_descr_buf.eof = 1;
|
||||
s_dma_descr_buf.buf = (uint8_t *)buf;
|
||||
dma_descr_head = &s_dma_descr_buf;
|
||||
}
|
||||
|
||||
/* Link DMA lists */
|
||||
if (buf_len && ilen) {
|
||||
dma_descr_buf.eof = 0;
|
||||
dma_descr_buf.empty = (uint32_t)(&dma_descr_input);
|
||||
s_dma_descr_buf.eof = 0;
|
||||
s_dma_descr_buf.empty = (uint32_t)(&s_dma_descr_input);
|
||||
}
|
||||
|
||||
if (esp_sha_dma_start(dma_descr_head) != ESP_OK) {
|
||||
|
@ -12,6 +12,9 @@
|
||||
#include "esp_timer.h"
|
||||
#include "esp_heap_caps.h"
|
||||
#include "test_utils.h"
|
||||
#include "freertos/FreeRTOS.h"
|
||||
#include "freertos/task.h"
|
||||
#include "freertos/semphr.h"
|
||||
|
||||
static const uint8_t key_256[] = {
|
||||
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
|
||||
@ -296,7 +299,7 @@ TEST_CASE("mbedtls CFB-128 AES-256 test", "[aes]")
|
||||
free(decryptedtext);
|
||||
}
|
||||
|
||||
TEST_CASE("mbedtls CTR stream test", "[aes]")
|
||||
static void aes_ctr_stream_test(void)
|
||||
{
|
||||
const unsigned SZ = 100;
|
||||
mbedtls_aes_context ctx;
|
||||
@ -396,6 +399,11 @@ TEST_CASE("mbedtls CTR stream test", "[aes]")
|
||||
free(decryptedtext);
|
||||
}
|
||||
|
||||
TEST_CASE("mbedtls CTR stream test", "[aes]")
|
||||
{
|
||||
aes_ctr_stream_test();
|
||||
}
|
||||
|
||||
|
||||
TEST_CASE("mbedtls OFB stream test", "[aes]")
|
||||
{
|
||||
@ -1457,3 +1465,32 @@ TEST_CASE("mbedtls AES external flash tests", "[aes]")
|
||||
aes_ext_flash_ctr_test(MALLOC_CAP_DMA | MALLOC_CAP_8BIT | MALLOC_CAP_INTERNAL);
|
||||
}
|
||||
#endif // CONFIG_SPIRAM_USE_MALLOC
|
||||
|
||||
|
||||
#if CONFIG_ESP_SYSTEM_RTC_FAST_MEM_AS_HEAP_DEPCHECK
|
||||
|
||||
RTC_FAST_ATTR uint8_t rtc_stack[4096];
|
||||
static xSemaphoreHandle done_sem;
|
||||
|
||||
static void aes_ctr_stream_test_task(void *pv)
|
||||
{
|
||||
aes_ctr_stream_test();
|
||||
xSemaphoreGive(done_sem);
|
||||
vTaskDelete(NULL);
|
||||
}
|
||||
|
||||
TEST_CASE("mbedtls AES stack in RTC RAM", "[mbedtls]")
|
||||
{
|
||||
done_sem = xSemaphoreCreateBinary();
|
||||
static StaticTask_t rtc_task;
|
||||
memset(rtc_stack, 0, sizeof(rtc_stack));
|
||||
|
||||
TEST_ASSERT(esp_ptr_in_rtc_dram_fast(rtc_stack));
|
||||
|
||||
TEST_ASSERT_NOT_NULL(xTaskCreateStatic(aes_ctr_stream_test_task, "aes_ctr_task", sizeof(rtc_stack), NULL,
|
||||
3, rtc_stack, &rtc_task));
|
||||
TEST_ASSERT_TRUE(xSemaphoreTake(done_sem, 10000 / portTICK_PERIOD_MS));
|
||||
vSemaphoreDelete(done_sem);
|
||||
}
|
||||
|
||||
#endif //CONFIG_ESP_SYSTEM_RTC_FAST_MEM_AS_HEAP_DEPCHECK
|
||||
|
@ -475,11 +475,11 @@ TEST_CASE("mbedtls SHA512/t", "[mbedtls]")
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif //CONFIG_MBEDTLS_HARDWARE_SHA
|
||||
|
||||
#ifdef CONFIG_SPIRAM
|
||||
#ifdef CONFIG_SPIRAM_USE_MALLOC
|
||||
TEST_CASE("mbedtls SHA256 PSRAM DMA", "[mbedtls]")
|
||||
{
|
||||
|
||||
const unsigned CALLS = 256;
|
||||
const unsigned CALL_SZ = 16 * 1024;
|
||||
mbedtls_sha256_context sha256_ctx;
|
||||
@ -510,6 +510,26 @@ TEST_CASE("mbedtls SHA256 PSRAM DMA", "[mbedtls]")
|
||||
TEST_ASSERT_EQUAL_STRING(expected_hash, hash_str);
|
||||
|
||||
}
|
||||
#endif //CONFIG_SPIRAM
|
||||
#endif //CONFIG_SPIRAM_USE_MALLOC
|
||||
|
||||
#endif //CONFIG_MBEDTLS_HARDWARE_SHA
|
||||
#if CONFIG_ESP_SYSTEM_RTC_FAST_MEM_AS_HEAP_DEPCHECK
|
||||
|
||||
extern RTC_FAST_ATTR uint8_t rtc_stack[4096];
|
||||
|
||||
static xSemaphoreHandle done_sem;
|
||||
|
||||
TEST_CASE("mbedtls SHA stack in RTC RAM", "[mbedtls]")
|
||||
{
|
||||
done_sem = xSemaphoreCreateBinary();
|
||||
static StaticTask_t rtc_task;
|
||||
memset(rtc_stack, 0, sizeof(rtc_stack));
|
||||
|
||||
TEST_ASSERT(esp_ptr_in_rtc_dram_fast(rtc_stack));
|
||||
|
||||
TEST_ASSERT_NOT_NULL(xTaskCreateStatic(tskRunSHA256Test, "tskRunSHA256Test_task", sizeof(rtc_stack), NULL,
|
||||
3, rtc_stack, &rtc_task));
|
||||
TEST_ASSERT_TRUE(xSemaphoreTake(done_sem, 10000 / portTICK_PERIOD_MS));
|
||||
vSemaphoreDelete(done_sem);
|
||||
}
|
||||
|
||||
#endif //CONFIG_ESP_SYSTEM_RTC_FAST_MEM_AS_HEAP_DEPCHECK
|
||||
|
@ -1810,8 +1810,6 @@ components/heap/test_multi_heap_host/test_multi_heap.cpp
|
||||
components/idf_test/include/esp32/idf_performance_target.h
|
||||
components/idf_test/include/esp32c3/idf_performance_target.h
|
||||
components/idf_test/include/esp32h2/idf_performance_target.h
|
||||
components/idf_test/include/esp32s2/idf_performance_target.h
|
||||
components/idf_test/include/esp32s3/idf_performance_target.h
|
||||
components/idf_test/include/idf_performance.h
|
||||
components/ieee802154/include/esp_ieee802154.h
|
||||
components/ieee802154/include/esp_ieee802154_types.h
|
||||
|
Loading…
x
Reference in New Issue
Block a user