mirror of
https://github.com/espressif/esp-idf.git
synced 2024-09-20 00:36:01 -04:00
feat(mbedtls/esp_crypto_shared_gdma): support AXI-DMA in the crypto shared gdma layer
- In case of AXI-DMA, the DMA descriptors need to be 8 bytes aligned lldesc_t do not satify this condition thus we need to replace it with dma_descriptor_t (align(4) and align(8)) in esp_crypto_shared_gdma. - Added new shared gdma start API that supports the dma_descriptor_t DMA descriptor. - Added some generic dma descriptor macros and helper functions - replace lldesc_t with dma_descriptor_t
This commit is contained in:
harshal.patil
parent
2c570ed53b
commit
83dd60307f
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020-2023 Espressif Systems (Shanghai) CO LTD
|
||||
* SPDX-FileCopyrightText: 2020-2024 Espressif Systems (Shanghai) CO LTD
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
@ -7,6 +7,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stddef.h>
|
||||
#include "esp_assert.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
@ -56,6 +57,24 @@ ESP_STATIC_ASSERT(sizeof(dma_descriptor_align8_t) == 16, "dma_descriptor_align8_
|
||||
#define DMA_DESCRIPTOR_BUFFER_OWNER_DMA (1) /*!< DMA buffer is allowed to be accessed by DMA engine */
|
||||
#define DMA_DESCRIPTOR_BUFFER_MAX_SIZE (4095) /*!< Maximum size of the buffer that can be attached to descriptor */
|
||||
#define DMA_DESCRIPTOR_BUFFER_MAX_SIZE_4B_ALIGNED (4095-3) /*!< Maximum size of the buffer that can be attached to descriptor, and aligned to 4B */
|
||||
#define DMA_DESCRIPTOR_BUFFER_MAX_SIZE_16B_ALIGNED (4095-15) /*!< Maximum size of the buffer that can be attached to descriptor, and aligned to 16B */
|
||||
|
||||
// the size field has 12 bits, but 0 not for 4096.
|
||||
// to avoid possible problem when the size is not word-aligned, we only use 4096-4 per desc.
|
||||
/** Maximum size of data in the buffer that a DMA descriptor can hold. */
|
||||
#define DMA_DESCRIPTOR_BUFFER_MAX_SIZE_PER_DESC (4096-4)
|
||||
|
||||
/**
|
||||
* Get the number of DMA descriptors required for a given buffer size.
|
||||
*
|
||||
* @param data_size Size to check DMA descriptor number.
|
||||
* @param max_desc_size Maximum length of each descriptor
|
||||
* @return Number of DMA descriptors required.
|
||||
*/
|
||||
static inline size_t dma_desc_get_required_num(size_t data_size, size_t max_desc_size)
|
||||
{
|
||||
return (data_size + max_desc_size - 1) / max_desc_size;
|
||||
}
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
||||
@ -33,7 +33,7 @@
|
||||
#include "esp_private/periph_ctrl.h"
|
||||
#include "esp_log.h"
|
||||
#include "esp_attr.h"
|
||||
#include "soc/lldesc.h"
|
||||
#include "esp_crypto_dma.h"
|
||||
#include "esp_heap_caps.h"
|
||||
#include "esp_memory_utils.h"
|
||||
#include "sys/param.h"
|
||||
@ -42,8 +42,8 @@
|
||||
#endif
|
||||
#include "esp_crypto_lock.h"
|
||||
#include "hal/aes_hal.h"
|
||||
#include "esp_aes_internal.h"
|
||||
#include "esp_aes_dma_priv.h"
|
||||
#include "esp_aes_internal.h"
|
||||
|
||||
#if CONFIG_IDF_TARGET_ESP32S2
|
||||
#include "esp32s2/rom/cache.h"
|
||||
@ -103,25 +103,19 @@ static bool s_check_dma_capable(const void *p);
|
||||
* * Must be in DMA capable memory, so stack is not a safe place to put them
|
||||
* * To avoid having to malloc/free them for every DMA operation
|
||||
*/
|
||||
static DRAM_ATTR lldesc_t s_stream_in_desc;
|
||||
static DRAM_ATTR lldesc_t s_stream_out_desc;
|
||||
static DRAM_ATTR crypto_dma_desc_t s_stream_in_desc;
|
||||
static DRAM_ATTR crypto_dma_desc_t s_stream_out_desc;
|
||||
static DRAM_ATTR uint8_t s_stream_in[AES_BLOCK_BYTES];
|
||||
static DRAM_ATTR uint8_t s_stream_out[AES_BLOCK_BYTES];
|
||||
|
||||
static inline void esp_aes_wait_dma_done(lldesc_t *output)
|
||||
/** Append a descriptor to the chain, set head if chain empty
|
||||
*
|
||||
* @param[out] head Pointer to the first/head node of the DMA descriptor linked list
|
||||
* @param item Pointer to the DMA descriptor node that has to be appended
|
||||
*/
|
||||
static inline void dma_desc_append(crypto_dma_desc_t **head, crypto_dma_desc_t *item)
|
||||
{
|
||||
/* Wait for DMA write operation to complete */
|
||||
while (1) {
|
||||
if ( esp_aes_dma_done(output) ) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Append a descriptor to the chain, set head if chain empty */
|
||||
static inline void lldesc_append(lldesc_t **head, lldesc_t *item)
|
||||
{
|
||||
lldesc_t *it;
|
||||
crypto_dma_desc_t *it;
|
||||
if (*head == NULL) {
|
||||
*head = item;
|
||||
return;
|
||||
@ -129,11 +123,61 @@ static inline void lldesc_append(lldesc_t **head, lldesc_t *item)
|
||||
|
||||
it = *head;
|
||||
|
||||
while (it->empty != 0) {
|
||||
it = (lldesc_t *)it->empty;
|
||||
while (it->next != 0) {
|
||||
it = (crypto_dma_desc_t *)it->next;
|
||||
}
|
||||
it->dw0.suc_eof = 0;
|
||||
it->next = item;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a linked list pointing to a (huge) buffer in an descriptor array.
|
||||
*
|
||||
* The caller should ensure there is enough size to hold the array, by calling
|
||||
* `dma_desc_get_required_num` with the same or less than the max_desc_size argument.
|
||||
*
|
||||
* @param[out] dmadesc Output of a descriptor array, the head should be fed to the DMA.
|
||||
* @param data Buffer for the descriptors to point to.
|
||||
* @param len Size (or length for TX) of the buffer
|
||||
* @param max_desc_size Maximum length of each descriptor
|
||||
* @param isrx The RX DMA may require the buffer to be word-aligned, set to true for a RX link, otherwise false.
|
||||
*/
|
||||
static inline void dma_desc_setup_link(crypto_dma_desc_t* dmadesc, const void *data, int len, int max_desc_size, bool isrx)
|
||||
{
|
||||
int i = 0;
|
||||
while (len) {
|
||||
int dmachunklen = len;
|
||||
if (dmachunklen > max_desc_size) {
|
||||
dmachunklen = max_desc_size;
|
||||
}
|
||||
if (isrx) {
|
||||
//Receive needs DMA length rounded to next 32-bit boundary
|
||||
dmadesc[i].dw0.size = (dmachunklen + 3) & (~3);
|
||||
dmadesc[i].dw0.length = (dmachunklen + 3) & (~3);
|
||||
} else {
|
||||
dmadesc[i].dw0.size = dmachunklen;
|
||||
dmadesc[i].dw0.length = dmachunklen;
|
||||
}
|
||||
dmadesc[i].buffer = (void *)data;
|
||||
dmadesc[i].dw0.suc_eof = 0;
|
||||
dmadesc[i].dw0.owner = DMA_DESCRIPTOR_BUFFER_OWNER_DMA;
|
||||
dmadesc[i].next = &dmadesc[i + 1];
|
||||
len -= dmachunklen;
|
||||
data += dmachunklen;
|
||||
i++;
|
||||
}
|
||||
dmadesc[i - 1].dw0.suc_eof = 1; //Mark last DMA desc as end of stream.
|
||||
dmadesc[i - 1].next = NULL;
|
||||
}
|
||||
|
||||
static inline void esp_aes_wait_dma_done(crypto_dma_desc_t *output)
|
||||
{
|
||||
/* Wait for DMA write operation to complete */
|
||||
while (1) {
|
||||
if ( esp_aes_dma_done(output) ) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
it->eof = 0;
|
||||
it->empty = (uint32_t)item;
|
||||
}
|
||||
|
||||
void esp_aes_acquire_hardware( void )
|
||||
@ -221,7 +265,7 @@ static esp_err_t esp_aes_isr_initialise( void )
|
||||
#endif // CONFIG_MBEDTLS_AES_USE_INTERRUPT
|
||||
|
||||
/* Wait for AES hardware block operation to complete */
|
||||
static int esp_aes_dma_wait_complete(bool use_intr, lldesc_t *output_desc)
|
||||
static int esp_aes_dma_wait_complete(bool use_intr, crypto_dma_desc_t *output_desc)
|
||||
{
|
||||
#if defined (CONFIG_MBEDTLS_AES_USE_INTERRUPT)
|
||||
if (use_intr) {
|
||||
@ -335,10 +379,10 @@ cleanup:
|
||||
*/
|
||||
static int esp_aes_process_dma(esp_aes_context *ctx, const unsigned char *input, unsigned char *output, size_t len, uint8_t *stream_out)
|
||||
{
|
||||
lldesc_t *in_desc_head = NULL, *out_desc_head = NULL;
|
||||
lldesc_t *out_desc_tail = NULL; /* pointer to the final output descriptor */
|
||||
lldesc_t *block_desc = NULL, *block_in_desc = NULL, *block_out_desc = NULL;
|
||||
size_t lldesc_num = 0;
|
||||
crypto_dma_desc_t *in_desc_head = NULL, *out_desc_head = NULL;
|
||||
crypto_dma_desc_t *out_desc_tail = NULL; /* pointer to the final output descriptor */
|
||||
crypto_dma_desc_t *block_desc = NULL, *block_in_desc = NULL, *block_out_desc = NULL;
|
||||
size_t crypto_dma_desc_num = 0;
|
||||
unsigned stream_bytes = len % AES_BLOCK_BYTES; // bytes which aren't in a full block
|
||||
unsigned block_bytes = len - stream_bytes; // bytes which are in a full block
|
||||
unsigned blocks = (block_bytes / AES_BLOCK_BYTES) + ((stream_bytes > 0) ? 1 : 0);
|
||||
@ -388,10 +432,10 @@ static int esp_aes_process_dma(esp_aes_context *ctx, const unsigned char *input,
|
||||
}
|
||||
|
||||
/* Set up dma descriptors for input and output considering the 16 byte alignment requirement for EDMA */
|
||||
lldesc_num = lldesc_get_required_num_constrained(block_bytes, LLDESC_MAX_NUM_PER_DESC_16B_ALIGNED);
|
||||
crypto_dma_desc_num = dma_desc_get_required_num(block_bytes, DMA_DESCRIPTOR_BUFFER_MAX_SIZE_16B_ALIGNED);
|
||||
|
||||
/* Allocate both in and out descriptors to save a malloc/free per function call */
|
||||
block_desc = heap_caps_calloc(lldesc_num * 2, sizeof(lldesc_t), MALLOC_CAP_DMA);
|
||||
block_desc = heap_caps_calloc(crypto_dma_desc_num * 2, sizeof(crypto_dma_desc_t), MALLOC_CAP_DMA);
|
||||
if (block_desc == NULL) {
|
||||
mbedtls_platform_zeroize(output, len);
|
||||
ESP_LOGE(TAG, "Failed to allocate memory");
|
||||
@ -399,36 +443,37 @@ static int esp_aes_process_dma(esp_aes_context *ctx, const unsigned char *input,
|
||||
}
|
||||
|
||||
block_in_desc = block_desc;
|
||||
block_out_desc = block_desc + lldesc_num;
|
||||
block_out_desc = block_desc + crypto_dma_desc_num;
|
||||
|
||||
dma_desc_setup_link(block_in_desc, input, block_bytes, DMA_DESCRIPTOR_BUFFER_MAX_SIZE_PER_DESC, 0);
|
||||
|
||||
lldesc_setup_link(block_in_desc, input, block_bytes, 0);
|
||||
//Limit max inlink descriptor length to be 16 byte aligned, require for EDMA
|
||||
lldesc_setup_link_constrained(block_out_desc, output, block_bytes, LLDESC_MAX_NUM_PER_DESC_16B_ALIGNED, 0);
|
||||
dma_desc_setup_link(block_out_desc, output, block_bytes, DMA_DESCRIPTOR_BUFFER_MAX_SIZE_16B_ALIGNED, 0);
|
||||
|
||||
/* Setup in/out start descriptors */
|
||||
lldesc_append(&in_desc_head, block_in_desc);
|
||||
lldesc_append(&out_desc_head, block_out_desc);
|
||||
dma_desc_append(&in_desc_head, block_in_desc);
|
||||
dma_desc_append(&out_desc_head, block_out_desc);
|
||||
|
||||
out_desc_tail = &block_out_desc[lldesc_num - 1];
|
||||
out_desc_tail = &block_out_desc[crypto_dma_desc_num - 1];
|
||||
}
|
||||
|
||||
/* Any leftover bytes which are appended as an additional DMA list */
|
||||
if (stream_bytes > 0) {
|
||||
|
||||
memset(&s_stream_in_desc, 0, sizeof(lldesc_t));
|
||||
memset(&s_stream_out_desc, 0, sizeof(lldesc_t));
|
||||
memset(&s_stream_in_desc, 0, sizeof(crypto_dma_desc_t));
|
||||
memset(&s_stream_out_desc, 0, sizeof(crypto_dma_desc_t));
|
||||
|
||||
memset(s_stream_in, 0, AES_BLOCK_BYTES);
|
||||
memset(s_stream_out, 0, AES_BLOCK_BYTES);
|
||||
|
||||
memcpy(s_stream_in, input + block_bytes, stream_bytes);
|
||||
|
||||
lldesc_setup_link(&s_stream_in_desc, s_stream_in, AES_BLOCK_BYTES, 0);
|
||||
lldesc_setup_link(&s_stream_out_desc, s_stream_out, AES_BLOCK_BYTES, 0);
|
||||
dma_desc_setup_link(&s_stream_in_desc, s_stream_in, AES_BLOCK_BYTES, DMA_DESCRIPTOR_BUFFER_MAX_SIZE_PER_DESC, 0);
|
||||
dma_desc_setup_link(&s_stream_out_desc, s_stream_out, AES_BLOCK_BYTES, DMA_DESCRIPTOR_BUFFER_MAX_SIZE_PER_DESC, 0);
|
||||
|
||||
/* Link with block descriptors */
|
||||
lldesc_append(&in_desc_head, &s_stream_in_desc);
|
||||
lldesc_append(&out_desc_head, &s_stream_out_desc);
|
||||
dma_desc_append(&in_desc_head, &s_stream_in_desc);
|
||||
dma_desc_append(&out_desc_head, &s_stream_out_desc);
|
||||
|
||||
out_desc_tail = &s_stream_out_desc;
|
||||
}
|
||||
@ -494,13 +539,13 @@ cleanup:
|
||||
* 3. If AES interrupt is enabled and ISR initialisation fails
|
||||
* 4. Failure in any of the AES operations
|
||||
*/
|
||||
int esp_aes_process_dma_gcm(esp_aes_context *ctx, const unsigned char *input, unsigned char *output, size_t len, lldesc_t *aad_desc, size_t aad_len)
|
||||
int esp_aes_process_dma_gcm(esp_aes_context *ctx, const unsigned char *input, unsigned char *output, size_t len, crypto_dma_desc_t *aad_desc, size_t aad_len)
|
||||
{
|
||||
lldesc_t *in_desc_head = NULL, *out_desc_head = NULL, *len_desc = NULL;
|
||||
lldesc_t *out_desc_tail = NULL; /* pointer to the final output descriptor */
|
||||
lldesc_t stream_in_desc, stream_out_desc;
|
||||
lldesc_t *block_desc = NULL, *block_in_desc = NULL, *block_out_desc = NULL;
|
||||
size_t lldesc_num;
|
||||
crypto_dma_desc_t *in_desc_head = NULL, *out_desc_head = NULL, *len_desc = NULL;
|
||||
crypto_dma_desc_t *out_desc_tail = NULL; /* pointer to the final output descriptor */
|
||||
crypto_dma_desc_t stream_in_desc, stream_out_desc;
|
||||
crypto_dma_desc_t *block_desc = NULL, *block_in_desc = NULL, *block_out_desc = NULL;
|
||||
size_t crypto_dma_desc_num = 0;
|
||||
uint32_t len_buf[4] = {};
|
||||
uint8_t stream_in[16] = {};
|
||||
uint8_t stream_out[16] = {};
|
||||
@ -523,10 +568,10 @@ int esp_aes_process_dma_gcm(esp_aes_context *ctx, const unsigned char *input, un
|
||||
}
|
||||
|
||||
/* Set up dma descriptors for input and output */
|
||||
lldesc_num = lldesc_get_required_num(block_bytes);
|
||||
crypto_dma_desc_num = dma_desc_get_required_num(block_bytes, DMA_DESCRIPTOR_BUFFER_MAX_SIZE_PER_DESC);
|
||||
|
||||
/* Allocate both in and out descriptors to save a malloc/free per function call, add 1 for length descriptor */
|
||||
block_desc = heap_caps_calloc( (lldesc_num * 2) + 1, sizeof(lldesc_t), MALLOC_CAP_DMA);
|
||||
block_desc = heap_caps_calloc((crypto_dma_desc_num * 2) + 1, sizeof(crypto_dma_desc_t), MALLOC_CAP_DMA);
|
||||
if (block_desc == NULL) {
|
||||
mbedtls_platform_zeroize(output, len);
|
||||
ESP_LOGE(TAG, "Failed to allocate memory");
|
||||
@ -534,32 +579,32 @@ int esp_aes_process_dma_gcm(esp_aes_context *ctx, const unsigned char *input, un
|
||||
}
|
||||
|
||||
block_in_desc = block_desc;
|
||||
len_desc = block_desc + lldesc_num;
|
||||
block_out_desc = block_desc + lldesc_num + 1;
|
||||
len_desc = block_desc + crypto_dma_desc_num;
|
||||
block_out_desc = block_desc + crypto_dma_desc_num + 1;
|
||||
|
||||
if (aad_desc != NULL) {
|
||||
lldesc_append(&in_desc_head, aad_desc);
|
||||
dma_desc_append(&in_desc_head, aad_desc);
|
||||
}
|
||||
|
||||
if (block_bytes > 0) {
|
||||
lldesc_setup_link(block_in_desc, input, block_bytes, 0);
|
||||
lldesc_setup_link(block_out_desc, output, block_bytes, 0);
|
||||
dma_desc_setup_link(block_in_desc, input, block_bytes, DMA_DESCRIPTOR_BUFFER_MAX_SIZE_PER_DESC, 0);
|
||||
dma_desc_setup_link(block_out_desc, output, block_bytes, DMA_DESCRIPTOR_BUFFER_MAX_SIZE_PER_DESC, 0);
|
||||
|
||||
lldesc_append(&in_desc_head, block_in_desc);
|
||||
lldesc_append(&out_desc_head, block_out_desc);
|
||||
dma_desc_append(&in_desc_head, block_in_desc);
|
||||
dma_desc_append(&out_desc_head, block_out_desc);
|
||||
|
||||
out_desc_tail = &block_out_desc[lldesc_num - 1];
|
||||
out_desc_tail = &block_out_desc[crypto_dma_desc_num - 1];
|
||||
}
|
||||
|
||||
/* Any leftover bytes which are appended as an additional DMA list */
|
||||
if (stream_bytes > 0) {
|
||||
memcpy(stream_in, input + block_bytes, stream_bytes);
|
||||
|
||||
lldesc_setup_link(&stream_in_desc, stream_in, AES_BLOCK_BYTES, 0);
|
||||
lldesc_setup_link(&stream_out_desc, stream_out, AES_BLOCK_BYTES, 0);
|
||||
dma_desc_setup_link(&stream_in_desc, stream_in, AES_BLOCK_BYTES, DMA_DESCRIPTOR_BUFFER_MAX_SIZE_PER_DESC, 0);
|
||||
dma_desc_setup_link(&stream_out_desc, stream_out, AES_BLOCK_BYTES, DMA_DESCRIPTOR_BUFFER_MAX_SIZE_PER_DESC, 0);
|
||||
|
||||
lldesc_append(&in_desc_head, &stream_in_desc);
|
||||
lldesc_append(&out_desc_head, &stream_out_desc);
|
||||
dma_desc_append(&in_desc_head, &stream_in_desc);
|
||||
dma_desc_append(&out_desc_head, &stream_out_desc);
|
||||
|
||||
out_desc_tail = &stream_out_desc;
|
||||
}
|
||||
@ -568,13 +613,13 @@ int esp_aes_process_dma_gcm(esp_aes_context *ctx, const unsigned char *input, un
|
||||
len_buf[1] = __builtin_bswap32(aad_len * 8);
|
||||
len_buf[3] = __builtin_bswap32(len * 8);
|
||||
|
||||
len_desc->length = sizeof(len_buf);
|
||||
len_desc->size = sizeof(len_buf);
|
||||
len_desc->owner = 1;
|
||||
len_desc->eof = 1;
|
||||
len_desc->buf = (uint8_t *)len_buf;
|
||||
len_desc->dw0.length = sizeof(len_buf);
|
||||
len_desc->dw0.size = sizeof(len_buf);
|
||||
len_desc->dw0.owner = 1;
|
||||
len_desc->dw0.suc_eof = 1;
|
||||
len_desc->buffer = (uint8_t *)len_buf;
|
||||
|
||||
lldesc_append(&in_desc_head, len_desc);
|
||||
dma_desc_append(&in_desc_head, len_desc);
|
||||
|
||||
#if defined (CONFIG_MBEDTLS_AES_USE_INTERRUPT)
|
||||
/* Only use interrupt for long AES operations */
|
||||
|
||||
@ -1,18 +1,10 @@
|
||||
// Copyright 2020 Espressif Systems (Shanghai) PTE LTD
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020-2023 Espressif Systems (Shanghai) CO LTD
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#include "esp_crypto_dma.h"
|
||||
#include "esp_aes_dma_priv.h"
|
||||
|
||||
#include "soc/soc_caps.h"
|
||||
@ -20,7 +12,7 @@
|
||||
#include "hal/crypto_dma_ll.h"
|
||||
|
||||
|
||||
esp_err_t esp_aes_dma_start(const lldesc_t *input, const lldesc_t *output)
|
||||
esp_err_t esp_aes_dma_start(const crypto_dma_desc_t *input, const crypto_dma_desc_t *output)
|
||||
{
|
||||
crypto_dma_ll_reset();
|
||||
crypto_dma_ll_set_mode(CRYPTO_DMA_AES);
|
||||
@ -36,7 +28,7 @@ esp_err_t esp_aes_dma_start(const lldesc_t *input, const lldesc_t *output)
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
bool esp_aes_dma_done(const lldesc_t *output)
|
||||
bool esp_aes_dma_done(const crypto_dma_desc_t *output)
|
||||
{
|
||||
return (crypto_dma_ll_inlink_is_eof() && (output->owner == 0));
|
||||
return (crypto_dma_ll_inlink_is_eof() && (output->dw0.owner == 0));
|
||||
}
|
||||
|
||||
@ -5,14 +5,15 @@
|
||||
*/
|
||||
|
||||
#include "esp_aes_dma_priv.h"
|
||||
#include "esp_crypto_dma.h"
|
||||
#include "esp_crypto_shared_gdma.h"
|
||||
|
||||
esp_err_t esp_aes_dma_start(const lldesc_t *input, const lldesc_t *output)
|
||||
esp_err_t esp_aes_dma_start(const crypto_dma_desc_t *input, const crypto_dma_desc_t *output)
|
||||
{
|
||||
return esp_crypto_shared_gdma_start(input, output, GDMA_TRIG_PERIPH_AES);
|
||||
return esp_crypto_shared_gdma_start_axi_ahb(input, output, GDMA_TRIG_PERIPH_AES);
|
||||
}
|
||||
|
||||
bool esp_aes_dma_done(const lldesc_t *output)
|
||||
bool esp_aes_dma_done(const crypto_dma_desc_t *output)
|
||||
{
|
||||
return (output->owner == 0);
|
||||
return (output->dw0.owner == 0);
|
||||
}
|
||||
|
||||
@ -6,7 +6,8 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "soc/lldesc.h"
|
||||
#include <stdbool.h>
|
||||
#include "esp_crypto_dma.h"
|
||||
#include "soc/soc_caps.h"
|
||||
#include "esp_err.h"
|
||||
|
||||
@ -23,7 +24,7 @@ extern "C" {
|
||||
* - ESP_OK: Successfully started the DMA
|
||||
* - ESP_ERR_INVALID_STATE: No DMA channel available
|
||||
*/
|
||||
esp_err_t esp_aes_dma_start(const lldesc_t *input, const lldesc_t *output);
|
||||
esp_err_t esp_aes_dma_start(const crypto_dma_desc_t *input, const crypto_dma_desc_t *output);
|
||||
|
||||
/**
|
||||
* @brief Check if the DMA engine is finished reading the result
|
||||
@ -33,7 +34,7 @@ esp_err_t esp_aes_dma_start(const lldesc_t *input, const lldesc_t *output);
|
||||
* - true: DMA finished
|
||||
* - false: DMA not yet finished
|
||||
*/
|
||||
bool esp_aes_dma_done(const lldesc_t *output);
|
||||
bool esp_aes_dma_done(const crypto_dma_desc_t *output);
|
||||
|
||||
/**
|
||||
* @brief Allocate AES peripheral interrupt handler
|
||||
|
||||
@ -672,8 +672,8 @@ int esp_aes_gcm_crypt_and_tag( esp_gcm_context *ctx,
|
||||
#endif
|
||||
#if CONFIG_MBEDTLS_HARDWARE_GCM
|
||||
int ret;
|
||||
lldesc_t aad_desc[2] = {};
|
||||
lldesc_t *aad_head_desc = NULL;
|
||||
crypto_dma_desc_t aad_desc[2] = {};
|
||||
crypto_dma_desc_t *aad_head_desc = NULL;
|
||||
size_t remainder_bit;
|
||||
uint8_t stream_in[AES_BLOCK_BYTES] = {};
|
||||
unsigned stream_bytes = aad_len % AES_BLOCK_BYTES; // bytes which aren't in a full block
|
||||
@ -687,7 +687,7 @@ int esp_aes_gcm_crypt_and_tag( esp_gcm_context *ctx,
|
||||
/* Limit aad len to a single DMA descriptor to simplify DMA handling
|
||||
In practice, e.g. with mbedtls the length of aad will always be short
|
||||
*/
|
||||
if (aad_len > LLDESC_MAX_NUM_PER_DESC) {
|
||||
if (aad_len > DMA_DESCRIPTOR_BUFFER_MAX_SIZE_PER_DESC) {
|
||||
return MBEDTLS_ERR_GCM_BAD_INPUT;
|
||||
}
|
||||
/* IV and AD are limited to 2^32 bits, so 2^29 bytes */
|
||||
@ -723,20 +723,20 @@ int esp_aes_gcm_crypt_and_tag( esp_gcm_context *ctx,
|
||||
ctx->aes_ctx.key_in_hardware = aes_hal_setkey(ctx->aes_ctx.key, ctx->aes_ctx.key_bytes, mode);
|
||||
|
||||
if (block_bytes > 0) {
|
||||
aad_desc[0].length = block_bytes;
|
||||
aad_desc[0].size = block_bytes;
|
||||
aad_desc[0].owner = 1;
|
||||
aad_desc[0].buf = aad;
|
||||
aad_desc[0].dw0.length = block_bytes;
|
||||
aad_desc[0].dw0.size = block_bytes;
|
||||
aad_desc[0].dw0.owner = 1;
|
||||
aad_desc[0].buffer = (void*)aad;
|
||||
}
|
||||
|
||||
if (stream_bytes > 0) {
|
||||
memcpy(stream_in, aad + block_bytes, stream_bytes);
|
||||
|
||||
aad_desc[0].empty = (uint32_t)&aad_desc[1];
|
||||
aad_desc[1].length = AES_BLOCK_BYTES;
|
||||
aad_desc[1].size = AES_BLOCK_BYTES;
|
||||
aad_desc[1].owner = 1;
|
||||
aad_desc[1].buf = stream_in;
|
||||
aad_desc[0].next = &aad_desc[1];
|
||||
aad_desc[1].dw0.length = AES_BLOCK_BYTES;
|
||||
aad_desc[1].dw0.size = AES_BLOCK_BYTES;
|
||||
aad_desc[1].dw0.owner = 1;
|
||||
aad_desc[1].buffer = (void*)stream_in;
|
||||
}
|
||||
|
||||
if (block_bytes > 0) {
|
||||
|
||||
@ -6,10 +6,10 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "aes/esp_aes.h"
|
||||
#include "aes/esp_aes_gcm.h"
|
||||
#include "soc/soc_caps.h"
|
||||
#include <stdbool.h>
|
||||
#include "aes/esp_aes.h"
|
||||
#include "soc/soc_caps.h"
|
||||
#include "esp_crypto_dma.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
@ -29,7 +29,7 @@ bool valid_key_length(const esp_aes_context *ctx);
|
||||
* @param aad_len GCM additional data length
|
||||
* @return int -1 on error
|
||||
*/
|
||||
int esp_aes_process_dma_gcm(esp_aes_context *ctx, const unsigned char *input, unsigned char *output, size_t len, lldesc_t *aad_desc, size_t aad_len);
|
||||
int esp_aes_process_dma_gcm(esp_aes_context *ctx, const unsigned char *input, unsigned char *output, size_t len, crypto_dma_desc_t *aad_desc, size_t aad_len);
|
||||
#endif
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
|
||||
* SPDX-FileCopyrightText: 2021-2023 Espressif Systems (Shanghai) CO LTD
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
@ -8,12 +8,21 @@
|
||||
|
||||
#include "freertos/FreeRTOS.h"
|
||||
#include "freertos/task.h"
|
||||
|
||||
#include "hal/gdma_ll.h"
|
||||
#include "soc/soc_caps.h"
|
||||
#include "esp_log.h"
|
||||
#include "esp_err.h"
|
||||
#include "esp_crypto_dma.h"
|
||||
#include "esp_crypto_lock.h"
|
||||
#include "soc/soc_caps.h"
|
||||
|
||||
#if CONFIG_IDF_TARGET_ESP32P4
|
||||
#include "esp32p4/rom/cache.h"
|
||||
#endif
|
||||
|
||||
#if SOC_AHB_GDMA_VERSION == 1
|
||||
#include "hal/gdma_ll.h"
|
||||
#elif SOC_AXI_GDMA_SUPPORTED
|
||||
#include "hal/axi_dma_ll.h"
|
||||
#endif /* SOC_AHB_GDMA_VERSION */
|
||||
|
||||
#define NEW_CHANNEL_TIMEOUT_MS 1000
|
||||
#define NEW_CHANNEL_DELAY_MS 100
|
||||
@ -26,12 +35,15 @@ static gdma_channel_handle_t tx_channel;
|
||||
/* Allocate a new GDMA channel, will keep trying until NEW_CHANNEL_TIMEOUT_MS */
|
||||
static inline esp_err_t crypto_shared_gdma_new_channel(gdma_channel_alloc_config_t *channel_config, gdma_channel_handle_t *channel)
|
||||
{
|
||||
esp_err_t ret;
|
||||
esp_err_t ret = ESP_FAIL;
|
||||
int time_waited_ms = 0;
|
||||
|
||||
while (1) {
|
||||
ret = gdma_new_channel(channel_config, channel);
|
||||
|
||||
#if SOC_AXI_GDMA_SUPPORTED
|
||||
ret = gdma_new_axi_channel(channel_config, channel);
|
||||
#else /* !SOC_AXI_GDMA_SUPPORTED */
|
||||
ret = gdma_new_ahb_channel(channel_config, channel);
|
||||
#endif /* SOC_AXI_GDMA_SUPPORTED */
|
||||
if (ret == ESP_OK) {
|
||||
break;
|
||||
} else if (time_waited_ms >= NEW_CHANNEL_TIMEOUT_MS) {
|
||||
@ -92,7 +104,6 @@ err:
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
esp_err_t esp_crypto_shared_gdma_start(const lldesc_t *input, const lldesc_t *output, gdma_trigger_peripheral_t peripheral)
|
||||
{
|
||||
int rx_ch_id = 0;
|
||||
@ -121,7 +132,68 @@ esp_err_t esp_crypto_shared_gdma_start(const lldesc_t *input, const lldesc_t *ou
|
||||
|
||||
/* tx channel is reset by gdma_connect(), also reset rx to ensure a known state */
|
||||
gdma_get_channel_id(rx_channel, &rx_ch_id);
|
||||
|
||||
#if SOC_AHB_GDMA_VERSION == 1
|
||||
gdma_ll_rx_reset_channel(&GDMA, rx_ch_id);
|
||||
#endif /* SOC_AHB_GDMA_VERSION */
|
||||
|
||||
gdma_start(tx_channel, (intptr_t)input);
|
||||
gdma_start(rx_channel, (intptr_t)output);
|
||||
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
esp_err_t esp_crypto_shared_gdma_start_axi_ahb(const crypto_dma_desc_t *input, const crypto_dma_desc_t *output, gdma_trigger_peripheral_t peripheral)
|
||||
{
|
||||
int rx_ch_id = 0;
|
||||
esp_err_t ret = ESP_OK;
|
||||
|
||||
#if SOC_CACHE_INTERNAL_MEM_VIA_L1CACHE
|
||||
// TODO: replace with `esp_cache_msync`
|
||||
const crypto_dma_desc_t *it = input;
|
||||
while(it != NULL) {
|
||||
Cache_WriteBack_Addr(CACHE_MAP_L1_DCACHE | CACHE_MAP_L2_CACHE, (uint32_t)it->buffer, it->dw0.length); // try using esp_cache_msync()
|
||||
Cache_WriteBack_Addr(CACHE_MAP_L1_DCACHE | CACHE_MAP_L2_CACHE, (uint32_t)it, sizeof(crypto_dma_desc_t));
|
||||
it = (const crypto_dma_desc_t*) it->next;
|
||||
}
|
||||
|
||||
it = output;
|
||||
while(it != NULL) {
|
||||
Cache_WriteBack_Addr(CACHE_MAP_L1_DCACHE | CACHE_MAP_L2_CACHE, (uint32_t)it->buffer, it->dw0.length);
|
||||
Cache_WriteBack_Addr(CACHE_MAP_L1_DCACHE | CACHE_MAP_L2_CACHE, (uint32_t)it, sizeof(crypto_dma_desc_t));
|
||||
it = (const crypto_dma_desc_t*) it->next;
|
||||
};
|
||||
#endif /* SOC_CACHE_INTERNAL_MEM_VIA_L1CACHE */
|
||||
|
||||
if (tx_channel == NULL) {
|
||||
/* Allocate a pair of RX and TX for crypto, should only happen the first time we use the GMDA
|
||||
or if user called esp_crypto_shared_gdma_release */
|
||||
ret = crypto_shared_gdma_init();
|
||||
}
|
||||
|
||||
if (ret != ESP_OK) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* Tx channel is shared between AES and SHA, need to connect to peripheral every time */
|
||||
gdma_disconnect(tx_channel);
|
||||
|
||||
if (peripheral == GDMA_TRIG_PERIPH_SHA) {
|
||||
gdma_connect(tx_channel, GDMA_MAKE_TRIGGER(GDMA_TRIG_PERIPH_SHA, 0));
|
||||
} else if (peripheral == GDMA_TRIG_PERIPH_AES) {
|
||||
gdma_connect(tx_channel, GDMA_MAKE_TRIGGER(GDMA_TRIG_PERIPH_AES, 0));
|
||||
} else {
|
||||
return ESP_ERR_INVALID_ARG;
|
||||
}
|
||||
|
||||
/* tx channel is reset by gdma_connect(), also reset rx to ensure a known state */
|
||||
gdma_get_channel_id(rx_channel, &rx_ch_id);
|
||||
|
||||
#if SOC_AHB_GDMA_VERSION == 1
|
||||
gdma_ll_rx_reset_channel(&GDMA, rx_ch_id);
|
||||
#elif SOC_AXI_GDMA_SUPPORTED
|
||||
axi_dma_ll_rx_reset_channel(&AXI_DMA, rx_ch_id);
|
||||
#endif /* SOC_AHB_GDMA_VERSION */
|
||||
|
||||
gdma_start(tx_channel, (intptr_t)input);
|
||||
gdma_start(rx_channel, (intptr_t)output);
|
||||
|
||||
@ -12,7 +12,6 @@
|
||||
|
||||
#include "aes/esp_aes.h"
|
||||
#include "mbedtls/cipher.h"
|
||||
#include "soc/lldesc.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
|
||||
45
components/mbedtls/port/include/esp_crypto_dma.h
Normal file
45
components/mbedtls/port/include/esp_crypto_dma.h
Normal file
@ -0,0 +1,45 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "hal/dma_types.h"
|
||||
#include "soc/gdma_channel.h"
|
||||
#include "soc/soc_caps.h"
|
||||
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C"
|
||||
{
|
||||
#endif
|
||||
|
||||
|
||||
#if (SOC_AES_SUPPORT_DMA) && (SOC_SHA_SUPPORT_DMA)
|
||||
|
||||
#if (SOC_AES_GDMA) && (SOC_SHA_GDMA)
|
||||
|
||||
#if (SOC_GDMA_TRIG_PERIPH_AES0_BUS == SOC_GDMA_BUS_AHB) && (SOC_GDMA_TRIG_PERIPH_SHA0_BUS == SOC_GDMA_BUS_AHB)
|
||||
#define DMA_DESC_MEM_ALIGN_SIZE 4
|
||||
typedef dma_descriptor_align4_t crypto_dma_desc_t;
|
||||
#elif (SOC_GDMA_TRIG_PERIPH_AES0_BUS == SOC_GDMA_BUS_AXI) && (SOC_GDMA_TRIG_PERIPH_SHA0_BUS == SOC_GDMA_BUS_AXI)
|
||||
#define DMA_DESC_MEM_ALIGN_SIZE 8
|
||||
typedef dma_descriptor_align8_t crypto_dma_desc_t;
|
||||
#else
|
||||
#error "As we support a shared crypto GDMA layer for the AES and the SHA peripheral, both the peripherals must use the same GDMA bus"
|
||||
#endif /* (SOC_GDMA_TRIG_PERIPH_AES0_BUS == SOC_GDMA_BUS_AHB) && (SOC_GDMA_TRIG_PERIPH_AES0_BUS == SOC_GDMA_BUS_AHB) */
|
||||
|
||||
#elif (SOC_AES_CRYPTO_DMA) && (SOC_SHA_CRYPTO_DMA)
|
||||
#define DMA_DESC_MEM_ALIGN_SIZE 4
|
||||
typedef dma_descriptor_align4_t crypto_dma_desc_t;
|
||||
|
||||
#endif /* (SOC_AES_GDMA) && (SOC_SHA_GDMA) */
|
||||
|
||||
#endif /* (SOC_AES_SUPPORT_DMA) && (SOC_SHA_SUPPORT_DMA) */
|
||||
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
@ -1,14 +1,15 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
|
||||
* SPDX-FileCopyrightText: 2021-2023 Espressif Systems (Shanghai) CO LTD
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "soc/lldesc.h"
|
||||
#include "esp_crypto_dma.h"
|
||||
#include "esp_private/gdma.h"
|
||||
#include "esp_err.h"
|
||||
#include "soc/lldesc.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
@ -16,17 +17,31 @@ extern "C" {
|
||||
|
||||
/**
|
||||
* @brief Start a GDMA transfer on the shared crypto DMA channel
|
||||
* Only supports AHB-DMA.
|
||||
*
|
||||
* @note Will allocate a GDMA channel for AES & SHA if no such channel is already allocated
|
||||
*
|
||||
* @param input Input linked list descriptor
|
||||
* @param output Output linked list descriptor
|
||||
* @param input Input linked list descriptor (lldesc_t *)
|
||||
* @param output Output linked list descriptor (lldesc_t *)
|
||||
* @param peripheral Crypto peripheral to connect the DMA to, either GDMA_TRIG_PERIPH_AES or
|
||||
* GDMA_TRIG_PERIPH_SHA
|
||||
* @return esp_err_t ESP_FAIL if no GDMA channel available
|
||||
*/
|
||||
esp_err_t esp_crypto_shared_gdma_start(const lldesc_t *input, const lldesc_t *output, gdma_trigger_peripheral_t peripheral);
|
||||
|
||||
/**
|
||||
* @brief Start a GDMA transfer on the shared crypto DMA channel
|
||||
* Supports AXI-DMA and AHB-DMA.
|
||||
*
|
||||
* @note Will allocate a GDMA channel for AES & SHA if no such channel is already allocated
|
||||
*
|
||||
* @param input Input linked list descriptor (crypto_dma_desc_t *)
|
||||
* @param output Output linked list descriptor (crypto_dma_desc_t *)
|
||||
* @param peripheral Crypto peripheral to connect the DMA to, either GDMA_TRIG_PERIPH_AES or
|
||||
* GDMA_TRIG_PERIPH_SHA
|
||||
* @return esp_err_t ESP_FAIL if no GDMA channel available
|
||||
*/
|
||||
esp_err_t esp_crypto_shared_gdma_start_axi_ahb(const crypto_dma_desc_t *input, const crypto_dma_desc_t *output, gdma_trigger_peripheral_t peripheral);
|
||||
|
||||
/**
|
||||
* @brief Frees any shared crypto DMA channel, if esp_crypto_shared_gdma_start is called after
|
||||
|
||||
@ -112,8 +112,6 @@ static void tskRunAES256Test(void *pvParameters)
|
||||
|
||||
}
|
||||
|
||||
#include "esp_crypto_shared_gdma.h"
|
||||
|
||||
#define TASK_STACK_SIZE (20*1024)
|
||||
|
||||
TEST_CASE("mbedtls AES/SHA multithreading", "[mbedtls]")
|
||||
|
||||
@ -504,7 +504,6 @@ components/lwip/include/apps/ping/ping.h
|
||||
components/mbedtls/esp_crt_bundle/test_gen_crt_bundle/test_gen_crt_bundle.py
|
||||
components/mbedtls/port/aes/block/esp_aes.c
|
||||
components/mbedtls/port/aes/dma/esp_aes.c
|
||||
components/mbedtls/port/aes/dma/esp_aes_crypto_dma_impl.c
|
||||
components/mbedtls/port/aes/esp_aes_xts.c
|
||||
components/mbedtls/port/include/aes/esp_aes.h
|
||||
components/mbedtls/port/include/aes_alt.h
|
||||
|
||||
Loading…
Reference in New Issue
Block a user