diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml index eaf1d76124..4dba21edf4 100644 --- a/.gitlab/ci/target-test.yml +++ b/.gitlab/ci/target-test.yml @@ -335,7 +335,7 @@ component_ut_test_001: UT_001: extends: .unit_test_esp32_template - parallel: 48 + parallel: 49 tags: - ESP32_IDF - UT_T1_1 @@ -551,7 +551,7 @@ UT_046: UT_047: extends: .unit_test_esp32s2_template - parallel: 2 + parallel: 3 tags: - ESP32S2_IDF - UT_T1_1 diff --git a/components/esp32c3/esp_crypto_lock.c b/components/esp32c3/esp_crypto_lock.c index ca6c3c49b9..46a90ca08b 100644 --- a/components/esp32c3/esp_crypto_lock.c +++ b/components/esp32c3/esp_crypto_lock.c @@ -17,8 +17,8 @@ #include "esp_crypto_lock.h" /* Lock overview: -SHA: independent -AES: independent +SHA: peripheral independent, but DMA is shared with AES +AES: peripheral independent, but DMA is shared with SHA MPI/RSA: independent HMAC: needs SHA DS: needs HMAC (which needs SHA), AES and MPI @@ -30,24 +30,21 @@ static _lock_t s_crypto_ds_lock; /* Lock for HMAC peripheral */ static _lock_t s_crypto_hmac_lock; -/* Lock for the SHA peripheral, also used by the HMAC and DS peripheral */ -static _lock_t s_crypto_sha_lock; - -/* Lock for the AES peripheral, also used by DS peripheral */ -static _lock_t s_crypto_aes_lock; - /* Lock for the MPI/RSA peripheral, also used by the DS peripheral */ static _lock_t s_crypto_mpi_lock; +/* Single lock for SHA and AES, sharing a reserved GDMA channel */ +static _lock_t s_crypto_sha_aes_lock; + void esp_crypto_hmac_lock_acquire(void) { _lock_acquire(&s_crypto_hmac_lock); - esp_crypto_sha_lock_acquire(); + esp_crypto_sha_aes_lock_acquire(); } void esp_crypto_hmac_lock_release(void) { - esp_crypto_sha_lock_release(); + esp_crypto_sha_aes_lock_release(); _lock_release(&s_crypto_hmac_lock); } @@ -55,36 +52,24 @@ void esp_crypto_ds_lock_acquire(void) { _lock_acquire(&s_crypto_ds_lock); esp_crypto_hmac_lock_acquire(); - esp_crypto_aes_lock_acquire(); esp_crypto_mpi_lock_acquire(); } void esp_crypto_ds_lock_release(void) { esp_crypto_mpi_lock_release(); - esp_crypto_aes_lock_release(); esp_crypto_hmac_lock_release(); _lock_release(&s_crypto_ds_lock); } -void esp_crypto_sha_lock_acquire(void) +void esp_crypto_sha_aes_lock_acquire(void) { - _lock_acquire(&s_crypto_sha_lock); + _lock_acquire(&s_crypto_sha_aes_lock); } -void esp_crypto_sha_lock_release(void) +void esp_crypto_sha_aes_lock_release(void) { - _lock_release(&s_crypto_sha_lock); -} - -void esp_crypto_aes_lock_acquire(void) -{ - _lock_acquire(&s_crypto_aes_lock); -} - -void esp_crypto_aes_lock_release(void) -{ - _lock_release(&s_crypto_aes_lock); + _lock_release(&s_crypto_sha_aes_lock); } void esp_crypto_mpi_lock_acquire(void) diff --git a/components/esp32c3/include/esp_crypto_lock.h b/components/esp32c3/include/esp_crypto_lock.h index cd0e7e3203..3149d34692 100644 --- a/components/esp32c3/include/esp_crypto_lock.h +++ b/components/esp32c3/include/esp_crypto_lock.h @@ -47,28 +47,17 @@ void esp_crypto_ds_lock_acquire(void); void esp_crypto_ds_lock_release(void); /** - * @brief Acquire lock for the SHA cryptography peripheral. + * @brief Acquire lock for the SHA and AES cryptography peripheral. * */ -void esp_crypto_sha_lock_acquire(void); +void esp_crypto_sha_aes_lock_acquire(void); /** - * @brief Release lock for the SHA cryptography peripheral. + * @brief Release lock for the SHA and AES cryptography peripheral. * */ -void esp_crypto_sha_lock_release(void); +void esp_crypto_sha_aes_lock_release(void); -/** - * @brief Acquire lock for the aes cryptography peripheral. - * - */ -void esp_crypto_aes_lock_acquire(void); - -/** - * @brief Release lock for the aes cryptography peripheral. - * - */ -void esp_crypto_aes_lock_release(void); /** * @brief Acquire lock for the mpi cryptography peripheral. diff --git a/components/esp32s3/esp_crypto_lock.c b/components/esp32s3/esp_crypto_lock.c index 86fcbab449..98fa6296ce 100644 --- a/components/esp32s3/esp_crypto_lock.c +++ b/components/esp32s3/esp_crypto_lock.c @@ -16,33 +16,28 @@ #include "esp_crypto_lock.h" -/* Lock for the SHA peripheral, also used by the HMAC and DS peripheral */ -static _lock_t s_crypto_sha_lock; +/* Lock overview: +SHA: peripheral independent, but DMA is shared with AES +AES: peripheral independent, but DMA is shared with SHA +MPI/RSA: independent +HMAC: needs SHA +DS: needs HMAC (which needs SHA), AES and MPI +*/ -/* Lock for the AES peripheral, also used by DS peripheral */ -static _lock_t s_crypto_aes_lock; +/* Single lock for SHA and AES, sharing a reserved GDMA channel */ +static _lock_t s_crypto_sha_aes_lock; /* Lock for the MPI/RSA peripheral, also used by the DS peripheral */ static _lock_t s_crypto_mpi_lock; -void esp_crypto_sha_lock_acquire(void) +void esp_crypto_sha_aes_lock_acquire(void) { - _lock_acquire(&s_crypto_sha_lock); + _lock_acquire(&s_crypto_sha_aes_lock); } -void esp_crypto_sha_lock_release(void) +void esp_crypto_sha_aes_lock_release(void) { - _lock_release(&s_crypto_sha_lock); -} - -void esp_crypto_aes_lock_acquire(void) -{ - _lock_acquire(&s_crypto_aes_lock); -} - -void esp_crypto_aes_lock_release(void) -{ - _lock_release(&s_crypto_aes_lock); + _lock_release(&s_crypto_sha_aes_lock); } void esp_crypto_mpi_lock_acquire(void) diff --git a/components/esp32s3/include/esp_crypto_lock.h b/components/esp32s3/include/esp_crypto_lock.h index bf30ba318c..781ac07780 100644 --- a/components/esp32s3/include/esp_crypto_lock.h +++ b/components/esp32s3/include/esp_crypto_lock.h @@ -27,24 +27,16 @@ extern "C" { */ /** - * Acquire lock for the SHA cryptography peripheral + * @brief Acquire lock for the SHA and AES cryptography peripheral. + * */ -void esp_crypto_sha_lock_acquire(void); +void esp_crypto_sha_aes_lock_acquire(void); /** - * Release lock for the SHA cryptography peripheral + * @brief Release lock for the SHA and AES cryptography peripheral. + * */ -void esp_crypto_sha_lock_release(void); - -/** - * Acquire lock for the AES cryptography peripheral - */ -void esp_crypto_aes_lock_acquire(void); - -/** - * Release lock for the AES cryptography peripheral - */ -void esp_crypto_aes_lock_release(void); +void esp_crypto_sha_aes_lock_release(void); /** * Acquire lock for the MPI/RSA cryptography peripheral diff --git a/components/mbedtls/CMakeLists.txt b/components/mbedtls/CMakeLists.txt index 8817b9fa36..ce4298fc75 100644 --- a/components/mbedtls/CMakeLists.txt +++ b/components/mbedtls/CMakeLists.txt @@ -96,7 +96,8 @@ if(SHA_PERIPHERAL_TYPE STREQUAL "dma") if(CONFIG_IDF_TARGET_ESP32S2) set(SHA_DMA_SRCS "${COMPONENT_DIR}/port/sha/dma/esp_sha_crypto_dma_impl.c") else() - set(SHA_DMA_SRCS "${COMPONENT_DIR}/port/sha/dma/esp_sha_gdma_impl.c") + set(SHA_DMA_SRCS "${COMPONENT_DIR}/port/sha/dma/esp_sha_gdma_impl.c" + "${COMPONENT_DIR}/port/crypto_shared_gdma/esp_crypto_shared_gdma.c") endif() endif() diff --git a/components/mbedtls/port/aes/dma/esp_aes.c b/components/mbedtls/port/aes/dma/esp_aes.c index 52c46d1b00..a0b9af4ca5 100644 --- a/components/mbedtls/port/aes/dma/esp_aes.c +++ b/components/mbedtls/port/aes/dma/esp_aes.c @@ -54,8 +54,8 @@ #endif #if SOC_AES_GDMA -#define AES_LOCK() esp_crypto_aes_lock_acquire() -#define AES_RELEASE() esp_crypto_aes_lock_release() +#define AES_LOCK() esp_crypto_sha_aes_lock_acquire() +#define AES_RELEASE() esp_crypto_sha_aes_lock_release() #elif SOC_AES_CRYPTO_DMA #define AES_LOCK() esp_crypto_dma_lock_acquire() #define AES_RELEASE() esp_crypto_dma_lock_release() diff --git a/components/mbedtls/port/aes/dma/esp_aes_gdma_impl.c b/components/mbedtls/port/aes/dma/esp_aes_gdma_impl.c index 0c25bc4a32..94e970709a 100644 --- a/components/mbedtls/port/aes/dma/esp_aes_gdma_impl.c +++ b/components/mbedtls/port/aes/dma/esp_aes_gdma_impl.c @@ -11,157 +11,16 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + #include "esp_aes_dma_priv.h" - -#include - -#include "freertos/FreeRTOS.h" -#include "freertos/task.h" - -#include "driver/periph_ctrl.h" #include "hal/gdma_ll.h" -#include "soc/soc_caps.h" -#include "esp_private/gdma.h" -#include "esp_log.h" - -#define NEW_CHANNEL_TIMEOUT_MS 1000 -#define NEW_CHANNEL_DELAY_MS 100 - -static const char *TAG = "esp_aes_gdma"; - -static _lock_t gdma_ch_lock; -/* For GDMA we allocate and reserve a single DMA pair for AES at esp_aes_init - and release it esp_aes_free - This is done to avoid the GDMA associated overhead when doing multiple AES transforms in a row. - - The channel is shared between any AES operations that are running in parallel, - access will be limited by the peripheral lock - */ -static uint8_t ref_counts; - -/* The GDMA channel is protected from concurrent access by the general AES peripheral lock */ -static gdma_channel_handle_t tx_channel; -static gdma_channel_handle_t rx_channel; - -/* Allocate a new GDMA channel, will keep trying until NEW_CHANNEL_TIMEOUT_MS */ -static inline esp_err_t esp_aes_gdma_new_channel(gdma_channel_alloc_config_t *channel_config, gdma_channel_handle_t *channel) -{ - esp_err_t ret; - int time_waited_ms = 0; - - while(1) { - ret = gdma_new_channel(channel_config, channel); - - if (ret == ESP_OK) { - break; - } else if (time_waited_ms >= NEW_CHANNEL_TIMEOUT_MS) { - *channel = NULL; - break; - } - - time_waited_ms += NEW_CHANNEL_DELAY_MS; - vTaskDelay(NEW_CHANNEL_DELAY_MS / portTICK_PERIOD_MS); - } - return ret; -} - -/* Initialize GDMA module and channels */ -static inline void esp_aes_gdma_init(void) -{ - esp_err_t ret; - - gdma_channel_alloc_config_t channel_config = { - .direction = GDMA_CHANNEL_DIRECTION_TX, - }; - - ret = esp_aes_gdma_new_channel(&channel_config, &tx_channel); - if (ret != ESP_OK) { - goto err; - } - - channel_config.direction = GDMA_CHANNEL_DIRECTION_RX; - ret = esp_aes_gdma_new_channel(&channel_config, &rx_channel); - if (ret != ESP_OK) { - gdma_del_channel(tx_channel); // Clean up already allocated TX channel - goto err; - } - - gdma_connect(tx_channel, GDMA_MAKE_TRIGGER(GDMA_TRIG_PERIPH_AES, 0)); - gdma_connect(rx_channel, GDMA_MAKE_TRIGGER(GDMA_TRIG_PERIPH_AES, 0)); - - return; - -err: - /* mbedtls_aes_init do not have a way of signaling errors to the caller - so we set the channel to NULL and detect it in esp_aes_dma_start */ - ESP_LOGE(TAG, "Failed to acquire DMA channel, Err=0x%X", ret); - tx_channel = NULL; - rx_channel = NULL; -} - -void esp_aes_dma_init() -{ - - _lock_acquire(&gdma_ch_lock); - if (ref_counts == 0) { - esp_aes_gdma_init(); - } - ref_counts++; - _lock_release(&gdma_ch_lock); -} - -static inline void esp_aes_gdma_free(void) -{ - gdma_disconnect(tx_channel); - gdma_disconnect(rx_channel); - gdma_del_channel(tx_channel); - gdma_del_channel(rx_channel); - - tx_channel = NULL; - rx_channel = NULL; -} - -void esp_aes_dma_free() -{ - _lock_acquire(&gdma_ch_lock); - ref_counts--; - if (ref_counts == 0) { - esp_aes_gdma_free(); - } - _lock_release(&gdma_ch_lock); -} - +#include "esp_crypto_shared_gdma.h" esp_err_t esp_aes_dma_start(const lldesc_t *input, const lldesc_t *output) { -#if SOC_GDMA_SUPPORT_EXTMEM - int tx_ch_id = 0; - int rx_ch_id = 0; -#endif //SOC_GDMA_SUPPORT_EXTMEM - - if (!tx_channel || !rx_channel) { - /* Will happen if no channel was acquired before timeout */ - return ESP_FAIL; - } - -#if SOC_GDMA_SUPPORT_EXTMEM - gdma_get_channel_id(tx_channel, &tx_ch_id); - gdma_get_channel_id(rx_channel, &rx_ch_id); - /* An L2 FIFO bigger than 40 bytes is need when accessing external ram */ - gdma_ll_tx_extend_fifo_size_to(&GDMA, tx_ch_id, 40); - gdma_ll_rx_extend_l2_fifo_size_to(&GDMA, rx_ch_id, 40); - gdma_ll_tx_set_block_size_psram(&GDMA, tx_ch_id, GDMA_OUT_EXT_MEM_BK_SIZE_16B); - gdma_ll_rx_set_block_size_psram(&GDMA, rx_ch_id, GDMA_OUT_EXT_MEM_BK_SIZE_16B); -#endif //SOC_GDMA_SUPPORT_EXTMEM - - gdma_start(tx_channel, (intptr_t)input); - gdma_start(rx_channel, (intptr_t)output); - - return ESP_OK; + return esp_crypto_shared_gdma_start(input, output, GDMA_TRIG_PERIPH_AES); } - - bool esp_aes_dma_done(const lldesc_t *output) { return (output->owner == 0); diff --git a/components/mbedtls/port/aes/dma/include/esp_aes_dma_priv.h b/components/mbedtls/port/aes/dma/include/esp_aes_dma_priv.h index 7dc1e6a6b0..7880f18806 100644 --- a/components/mbedtls/port/aes/dma/include/esp_aes_dma_priv.h +++ b/components/mbedtls/port/aes/dma/include/esp_aes_dma_priv.h @@ -22,27 +22,6 @@ extern "C" { #endif -#if SOC_AES_GDMA - -/** - * @brief Initialize the GDMA channel - * - * @note Allocate and initialize a DMA channel (rx and tx) for the AES peripheral - * Only one channel will be initialized at any given time. If two or more AES operations are - * run in parallel the channel will be shared sequentially. - * - */ -void esp_aes_dma_init(void); - -/** - * @brief Free the GDMA channel - * - * @note The channel will only be freed if there are no other AES operations currently using it - * - */ -void esp_aes_dma_free(void); -#endif //SOC_AES_GDMA - /** * @brief Start the DMA engine * diff --git a/components/mbedtls/port/aes/esp_aes_common.c b/components/mbedtls/port/aes/esp_aes_common.c index 55b873bac1..c0dd78a968 100644 --- a/components/mbedtls/port/aes/esp_aes_common.c +++ b/components/mbedtls/port/aes/esp_aes_common.c @@ -54,10 +54,6 @@ bool valid_key_length(const esp_aes_context *ctx) void esp_aes_init( esp_aes_context *ctx ) { bzero( ctx, sizeof( esp_aes_context ) ); - -#if SOC_AES_GDMA - esp_aes_dma_init(); -#endif } void esp_aes_free( esp_aes_context *ctx ) @@ -66,10 +62,6 @@ void esp_aes_free( esp_aes_context *ctx ) return; } -#if SOC_AES_GDMA - esp_aes_dma_free(); -#endif - bzero( ctx, sizeof( esp_aes_context ) ); } diff --git a/components/mbedtls/port/crypto_shared_gdma/esp_crypto_shared_gdma.c b/components/mbedtls/port/crypto_shared_gdma/esp_crypto_shared_gdma.c new file mode 100644 index 0000000000..118fee0a4f --- /dev/null +++ b/components/mbedtls/port/crypto_shared_gdma/esp_crypto_shared_gdma.c @@ -0,0 +1,170 @@ +// Copyright 2021 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "esp_crypto_shared_gdma.h" + +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" + +#include "hal/gdma_ll.h" +#include "soc/soc_caps.h" +#include "esp_log.h" +#include "esp_err.h" +#include "esp_crypto_lock.h" + +#define NEW_CHANNEL_TIMEOUT_MS 1000 +#define NEW_CHANNEL_DELAY_MS 100 + +static const char *TAG = "crypto_shared_gdma"; + +static gdma_channel_handle_t rx_channel; +static gdma_channel_handle_t tx_channel; + +/* Allocate a new GDMA channel, will keep trying until NEW_CHANNEL_TIMEOUT_MS */ +static inline esp_err_t crypto_shared_gdma_new_channel(gdma_channel_alloc_config_t *channel_config, gdma_channel_handle_t *channel) +{ + esp_err_t ret; + int time_waited_ms = 0; + + while(1) { + ret = gdma_new_channel(channel_config, channel); + + if (ret == ESP_OK) { + break; + } else if (time_waited_ms >= NEW_CHANNEL_TIMEOUT_MS) { + *channel = NULL; + break; + } + + time_waited_ms += NEW_CHANNEL_DELAY_MS; + vTaskDelay(NEW_CHANNEL_DELAY_MS / portTICK_PERIOD_MS); + } + return ret; +} + + +#if SOC_GDMA_SUPPORT_EXTMEM +/* Initialize external memory specific DMA configs */ +static void esp_crypto_shared_dma_init_extmem(void) +{ + int tx_ch_id = 0; + int rx_ch_id = 0; + + gdma_get_channel_id(tx_channel, &tx_ch_id); + gdma_get_channel_id(rx_channel, &rx_ch_id); + + /* An L2 FIFO bigger than 40 bytes is need when accessing external ram */ + gdma_ll_tx_extend_fifo_size_to(&GDMA, tx_ch_id, 40); + gdma_ll_rx_extend_l2_fifo_size_to(&GDMA, rx_ch_id, 40); + gdma_ll_tx_set_block_size_psram(&GDMA, tx_ch_id, GDMA_OUT_EXT_MEM_BK_SIZE_16B); + gdma_ll_rx_set_block_size_psram(&GDMA, rx_ch_id, GDMA_OUT_EXT_MEM_BK_SIZE_16B); +} +#endif //SOC_GDMA_SUPPORT_EXTMEM + +/* Initialize GDMA module and channels */ +static esp_err_t crypto_shared_gdma_init(void) +{ + esp_err_t ret; + + gdma_channel_alloc_config_t channel_config_tx = { + .direction = GDMA_CHANNEL_DIRECTION_TX, + }; + + gdma_channel_alloc_config_t channel_config_rx = { + .direction = GDMA_CHANNEL_DIRECTION_RX, + }; + + ret = crypto_shared_gdma_new_channel(&channel_config_tx, &tx_channel); + if (ret != ESP_OK) { + goto err; + } + + ret = crypto_shared_gdma_new_channel(&channel_config_rx, &rx_channel); + if (ret != ESP_OK) { + gdma_del_channel(tx_channel); // Clean up already allocated TX channel + goto err; + } + +#if SOC_GDMA_SUPPORT_EXTMEM + esp_crypto_shared_dma_init_extmem(); +#endif //SOC_GDMA_SUPPORT_EXTMEM + + gdma_connect(rx_channel, GDMA_MAKE_TRIGGER(GDMA_TRIG_PERIPH_AES, 0)); + gdma_connect(tx_channel, GDMA_MAKE_TRIGGER(GDMA_TRIG_PERIPH_AES, 0)); + + return ESP_OK; + +err: + ESP_LOGE(TAG, "Failed to acquire DMA channel, Err=0x%X", ret); + tx_channel = NULL; + rx_channel = NULL; + + return ret; +} + + +esp_err_t esp_crypto_shared_gdma_start(const lldesc_t *input, const lldesc_t *output, gdma_trigger_peripheral_t peripheral) +{ + int rx_ch_id = 0; + esp_err_t ret = ESP_OK; + + if (tx_channel == NULL) { + /* Allocate a pair of RX and TX for crypto, should only happen the first time we use the GMDA + or if user called esp_crypto_shared_gdma_release */ + ret = crypto_shared_gdma_init(); + } + + if (ret != ESP_OK) { + return ret; + } + + /* Tx channel is shared between AES and SHA, need to connect to peripheral every time */ + gdma_disconnect(tx_channel); + + if (peripheral == GDMA_TRIG_PERIPH_SHA) { + gdma_connect(tx_channel, GDMA_MAKE_TRIGGER(GDMA_TRIG_PERIPH_SHA, 0)); + } else if (peripheral == GDMA_TRIG_PERIPH_AES) { + gdma_connect(tx_channel, GDMA_MAKE_TRIGGER(GDMA_TRIG_PERIPH_AES, 0)); + } else { + return ESP_ERR_INVALID_ARG; + } + + /* tx channel is reset by gdma_connect(), also reset rx to ensure a known state */ + gdma_get_channel_id(tx_channel, &rx_ch_id); + gdma_ll_rx_reset_channel(&GDMA, rx_ch_id); + + gdma_start(tx_channel, (intptr_t)input); + gdma_start(rx_channel, (intptr_t)output); + + return ESP_OK; +} + +void esp_crypto_shared_gdma_free() +{ + esp_crypto_sha_aes_lock_acquire(); + + if (rx_channel != NULL) { + gdma_disconnect(rx_channel); + gdma_del_channel(rx_channel); + rx_channel = NULL; + } + + if (tx_channel != NULL) { + gdma_disconnect(tx_channel); + gdma_del_channel(tx_channel); + tx_channel = NULL; + } + + esp_crypto_sha_aes_lock_release(); +} diff --git a/components/mbedtls/port/include/esp_crypto_shared_gdma.h b/components/mbedtls/port/include/esp_crypto_shared_gdma.h new file mode 100644 index 0000000000..46aa0383b1 --- /dev/null +++ b/components/mbedtls/port/include/esp_crypto_shared_gdma.h @@ -0,0 +1,55 @@ +// Copyright 2021 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "soc/lldesc.h" +#include "esp_private/gdma.h" +#include "esp_err.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief Start a GDMA transfer on the shared crypto DMA channel + * + * @note Will allocate a GDMA channel for AES & SHA if no such channel is already allocated + * + * @param input Input linked list descriptor + * @param output Output linked list descriptor + * @param peripheral Crypto peripheral to connect the DMA to, either GDMA_TRIG_PERIPH_AES or + * GDMA_TRIG_PERIPH_SHA + * @return esp_err_t ESP_FAIL if no GDMA channel available + */ +esp_err_t esp_crypto_shared_gdma_start(const lldesc_t *input, const lldesc_t *output, gdma_trigger_peripheral_t peripheral); + + +/** + * @brief Frees any shared crypto DMA channel, if esp_crypto_shared_gdma_start is called after + * this, new GDMA channels will be allocated. + * + * @note Function is meant to be called from user code, and thus takes AES/SHA lock. + * This means this function should not be called from code which already takes these locks, + * i.e. inside our AES/SHA code. + * + * If you are continously using AES/SHA (e.g. because of a wifi connection) then it's not recommended + * to use this API. Freeing the channel is mainly for use cases where you are finished with the crypto peripherals + * and need the DMA channel for other peripherals. An example would be doing some processing after disconnecting WiFi + */ +void esp_crypto_shared_gdma_free(void); + +#ifdef __cplusplus +} +#endif diff --git a/components/mbedtls/port/sha/dma/esp_sha1.c b/components/mbedtls/port/sha/dma/esp_sha1.c index 5d22de58f6..91b011f386 100644 --- a/components/mbedtls/port/sha/dma/esp_sha1.c +++ b/components/mbedtls/port/sha/dma/esp_sha1.c @@ -47,10 +47,6 @@ #include "sha/sha_dma.h" -#if SOC_SHA_GDMA -#include "esp_sha_dma_priv.h" -#endif - /* Implementation that should never be optimized out by the compiler */ static void mbedtls_zeroize( void *v, size_t n ) { @@ -77,10 +73,6 @@ static void mbedtls_zeroize( void *v, size_t n ) void mbedtls_sha1_init( mbedtls_sha1_context *ctx ) { memset( ctx, 0, sizeof( mbedtls_sha1_context ) ); - -#if SOC_SHA_GDMA - esp_sha_dma_init(); -#endif } void mbedtls_sha1_free( mbedtls_sha1_context *ctx ) @@ -88,10 +80,6 @@ void mbedtls_sha1_free( mbedtls_sha1_context *ctx ) if ( ctx == NULL ) { return; } -#if SOC_SHA_GDMA - esp_sha_dma_free(); -#endif - mbedtls_zeroize( ctx, sizeof( mbedtls_sha1_context ) ); } diff --git a/components/mbedtls/port/sha/dma/esp_sha256.c b/components/mbedtls/port/sha/dma/esp_sha256.c index 3797cd28f3..307b338498 100644 --- a/components/mbedtls/port/sha/dma/esp_sha256.c +++ b/components/mbedtls/port/sha/dma/esp_sha256.c @@ -48,10 +48,6 @@ #include "sha/sha_dma.h" -#if SOC_SHA_GDMA -#include "esp_sha_dma_priv.h" -#endif - /* Implementation that should never be optimized out by the compiler */ static void mbedtls_zeroize( void *v, size_t n ) { @@ -87,10 +83,6 @@ do { \ void mbedtls_sha256_init( mbedtls_sha256_context *ctx ) { memset( ctx, 0, sizeof( mbedtls_sha256_context ) ); - -#if SOC_SHA_GDMA - esp_sha_dma_init(); -#endif } void mbedtls_sha256_free( mbedtls_sha256_context *ctx ) @@ -99,10 +91,6 @@ void mbedtls_sha256_free( mbedtls_sha256_context *ctx ) return; } -#if SOC_SHA_GDMA - esp_sha_dma_free(); -#endif - mbedtls_zeroize( ctx, sizeof( mbedtls_sha256_context ) ); } diff --git a/components/mbedtls/port/sha/dma/esp_sha512.c b/components/mbedtls/port/sha/dma/esp_sha512.c index 950c9298bc..fa4465d0f8 100644 --- a/components/mbedtls/port/sha/dma/esp_sha512.c +++ b/components/mbedtls/port/sha/dma/esp_sha512.c @@ -54,10 +54,6 @@ #include "sha/sha_dma.h" -#if SOC_SHA_GDMA -#include "esp_sha_dma_priv.h" -#endif - /* Implementation that should never be optimized out by the compiler */ static void mbedtls_zeroize( void *v, size_t n ) { @@ -109,10 +105,6 @@ void esp_sha512_set_t( mbedtls_sha512_context *ctx, uint16_t t_val) void mbedtls_sha512_init( mbedtls_sha512_context *ctx ) { memset( ctx, 0, sizeof( mbedtls_sha512_context ) ); - -#if SOC_SHA_GDMA - esp_sha_dma_init(); -#endif } void mbedtls_sha512_free( mbedtls_sha512_context *ctx ) @@ -121,10 +113,6 @@ void mbedtls_sha512_free( mbedtls_sha512_context *ctx ) return; } -#if SOC_SHA_GDMA - esp_sha_dma_free(); -#endif - mbedtls_zeroize( ctx, sizeof( mbedtls_sha512_context ) ); } diff --git a/components/mbedtls/port/sha/dma/esp_sha_gdma_impl.c b/components/mbedtls/port/sha/dma/esp_sha_gdma_impl.c index 31fe219a70..0dc827fec2 100644 --- a/components/mbedtls/port/sha/dma/esp_sha_gdma_impl.c +++ b/components/mbedtls/port/sha/dma/esp_sha_gdma_impl.c @@ -11,130 +11,11 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + #include "esp_sha_dma_priv.h" - -#include - -#include "freertos/FreeRTOS.h" -#include "freertos/task.h" - -#include "driver/periph_ctrl.h" -#include "hal/gdma_ll.h" -#include "soc/soc_caps.h" -#include "esp_private/gdma.h" -#include "esp_log.h" - -#define NEW_CHANNEL_TIMEOUT_MS 1000 -#define NEW_CHANNEL_DELAY_MS 100 - -static const char *TAG = "esp_sha_gdma"; - -static _lock_t gdma_ch_lock; -/* For GDMA we allocate and reserve a single DMA pair for sha at esp_sha_init - and release it esp_sha_free - This is done to avoid the GDMA associated overhead when doing multiple sha transforms in a row. - - The channel is shared between any sha operations that are running in parallel, - access will be limited by the peripheral lock - */ -static uint8_t ref_counts; - -/* The GDMA channel is protected from concurrent access by the general sha peripheral lock */ -static gdma_channel_handle_t tx_channel; - -/* Allocate a new GDMA channel, will keep trying until NEW_CHANNEL_TIMEOUT_MS */ -static inline esp_err_t esp_sha_gdma_new_channel(gdma_channel_alloc_config_t *channel_config, gdma_channel_handle_t *channel) -{ - esp_err_t ret; - int time_waited_ms = 0; - - while(1) { - ret = gdma_new_channel(channel_config, channel); - - if (ret == ESP_OK) { - break; - } else if (time_waited_ms >= 1 - ) { - *channel = NULL; - break; - } - - time_waited_ms += NEW_CHANNEL_DELAY_MS; - vTaskDelay(NEW_CHANNEL_DELAY_MS / portTICK_PERIOD_MS); - } - return ret; -} - -/* Initialize GDMA module and channels */ -static inline void esp_sha_gdma_init(void) -{ - esp_err_t ret; - - gdma_channel_alloc_config_t channel_config = { - .direction = GDMA_CHANNEL_DIRECTION_TX, - }; - - ret = esp_sha_gdma_new_channel(&channel_config, &tx_channel); - if (ret != ESP_OK) { - - /* mbedtls_sha_init do not have a way of signaling errors to the caller - so we set the channel to NULL and detect it in esp_sha_dma_start */ - ESP_LOGE(TAG, "Failed to acquire DMA channel, Err=0x%X", ret); - tx_channel = NULL; - return; - } - - gdma_connect(tx_channel, GDMA_MAKE_TRIGGER(GDMA_TRIG_PERIPH_SHA, 0)); -} - -void esp_sha_dma_init() -{ - - _lock_acquire(&gdma_ch_lock); - if (ref_counts == 0) { - esp_sha_gdma_init(); - } - ref_counts++; - _lock_release(&gdma_ch_lock); -} - -static inline void esp_sha_gdma_free(void) -{ - gdma_disconnect(tx_channel); - gdma_del_channel(tx_channel); - tx_channel = NULL; -} - -void esp_sha_dma_free() -{ - _lock_acquire(&gdma_ch_lock); - ref_counts--; - if (ref_counts == 0) { - esp_sha_gdma_free(); - } - _lock_release(&gdma_ch_lock); -} - +#include "esp_crypto_shared_gdma.h" esp_err_t esp_sha_dma_start(const lldesc_t *input) { -#if SOC_GDMA_SUPPORT_EXTMEM - int tx_ch_id = 0; -#endif //SOC_GDMA_SUPPORT_EXTMEM - - if (!tx_channel) { - /* Will happen if no channel was acquired before timeout */ - return ESP_FAIL; - } - -#if SOC_GDMA_SUPPORT_EXTMEM - gdma_get_channel_id(tx_channel, &tx_ch_id); - /* An L2 FIFO bigger than 40 bytes is need when accessing external ram */ - gdma_ll_tx_extend_fifo_size_to(&GDMA, tx_ch_id, 40); - gdma_ll_tx_set_block_size_psram(&GDMA, tx_ch_id, GDMA_OUT_EXT_MEM_BK_SIZE_16B); -#endif //SOC_GDMA_SUPPORT_EXTMEM - - gdma_start(tx_channel, (intptr_t)input); - - return ESP_OK; + return esp_crypto_shared_gdma_start(input, NULL, GDMA_TRIG_PERIPH_SHA); } diff --git a/components/mbedtls/port/sha/dma/include/esp_sha_dma_priv.h b/components/mbedtls/port/sha/dma/include/esp_sha_dma_priv.h index 6909a6f91e..fdb758fc3c 100644 --- a/components/mbedtls/port/sha/dma/include/esp_sha_dma_priv.h +++ b/components/mbedtls/port/sha/dma/include/esp_sha_dma_priv.h @@ -22,27 +22,6 @@ extern "C" { #endif -#if SOC_SHA_GDMA - -/** - * @brief Initialize the GDMA channel - * - * @note Allocate and initialize a DMA channel (tx) for the SHA peripheral - * Only one channel will be initialized at any given time. If two or more SHA operations are - * run in parallel the channel will be shared sequentially. - * - */ -void esp_sha_dma_init(void); - -/** - * @brief Free the GDMA channel - * - * @note The channel will only be freed if there are no other SHA operations currently using it - * - */ -void esp_sha_dma_free(void); -#endif //SOC_SHA_GDMA - /** * @brief Start the DMA engine * diff --git a/components/mbedtls/port/sha/dma/sha.c b/components/mbedtls/port/sha/dma/sha.c index e17e6ce232..91f098411c 100644 --- a/components/mbedtls/port/sha/dma/sha.c +++ b/components/mbedtls/port/sha/dma/sha.c @@ -55,13 +55,11 @@ #endif #if SOC_SHA_GDMA -#define SHA_LOCK() esp_crypto_sha_lock_acquire() -#define SHA_RELEASE() esp_crypto_sha_lock_release() +#define SHA_LOCK() esp_crypto_sha_aes_lock_acquire() +#define SHA_RELEASE() esp_crypto_sha_aes_lock_release() #elif SOC_SHA_CRYPTO_DMA #define SHA_LOCK() esp_crypto_dma_lock_acquire() #define SHA_RELEASE() esp_crypto_dma_lock_release() -#else -#define SHA_LOCK() () #endif const static char *TAG = "esp-sha"; diff --git a/components/mbedtls/test/test_aes_sha_parallel.c b/components/mbedtls/test/test_aes_sha_parallel.c new file mode 100644 index 0000000000..6b2744d0a7 --- /dev/null +++ b/components/mbedtls/test/test_aes_sha_parallel.c @@ -0,0 +1,127 @@ +#include +#include +#include +#include "mbedtls/aes.h" +#include "mbedtls/sha256.h" +#include "unity.h" +#include "sdkconfig.h" +#include "esp_heap_caps.h" +#include "test_utils.h" +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" +#include "freertos/semphr.h" + +static xSemaphoreHandle done_sem; + +static const unsigned char *one_hundred_bs = (unsigned char *) + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + +static const uint8_t sha256_thousand_bs[32] = { + 0xf6, 0xf1, 0x18, 0xe1, 0x20, 0xe5, 0x2b, 0xe0, 0xbd, 0x0c, 0xfd, 0xf2, 0x79, 0x4c, 0xd1, 0x2c, 0x07, 0x68, 0x6c, 0xc8, 0x71, 0x23, 0x5a, 0xc2, 0xf1, 0x14, 0x59, 0x37, 0x8e, 0x6d, 0x23, 0x5b +}; + +static void tskRunSHA256Test(void *pvParameters) +{ + mbedtls_sha256_context sha256_ctx; + unsigned char sha256[32]; + + for (int i = 0; i < 1000; i++) { + + mbedtls_sha256_init(&sha256_ctx); + TEST_ASSERT_EQUAL(0, mbedtls_sha256_starts_ret(&sha256_ctx, false)); + for (int j = 0; j < 10; j++) { + TEST_ASSERT_EQUAL(0, mbedtls_sha256_update_ret(&sha256_ctx, (unsigned char *)one_hundred_bs, 100)); + } + TEST_ASSERT_EQUAL(0, mbedtls_sha256_finish_ret(&sha256_ctx, sha256)); + mbedtls_sha256_free(&sha256_ctx); + TEST_ASSERT_EQUAL_MEMORY_MESSAGE(sha256_thousand_bs, sha256, 32, "SHA256 calculation"); + } + xSemaphoreGive(done_sem); + vTaskDelete(NULL); +} + + +static void tskRunAES256Test(void *pvParameters) +{ + static const uint8_t iv[] = { + 0x10, 0x0f, 0x0e, 0x0d, 0x0c, 0x0b, 0x0a, 0x09, + 0x08, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, + }; + + static const uint8_t key_256[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + }; + + for (int i = 0; i <1000; i++) + { + const unsigned SZ = 1600; + mbedtls_aes_context ctx; + uint8_t nonce[16]; + + const uint8_t expected_cipher_end[] = { + 0x3e, 0x68, 0x8a, 0x02, 0xe6, 0xf2, 0x6a, 0x9e, + 0x9b, 0xb2, 0xc0, 0xc4, 0x63, 0x63, 0xd9, 0x25, + 0x51, 0xdc, 0xc2, 0x71, 0x96, 0xb3, 0xe5, 0xcd, + 0xbd, 0x0e, 0xf2, 0xef, 0xa9, 0xab, 0xab, 0x2d, + }; + + memcpy(nonce, iv, 16); + + // allocate internal memory + uint8_t *chipertext = heap_caps_malloc(SZ, MALLOC_CAP_8BIT|MALLOC_CAP_INTERNAL); + uint8_t *plaintext = heap_caps_malloc(SZ, MALLOC_CAP_8BIT|MALLOC_CAP_INTERNAL); + uint8_t *decryptedtext = heap_caps_malloc(SZ, MALLOC_CAP_8BIT|MALLOC_CAP_INTERNAL); + + TEST_ASSERT_NOT_NULL(chipertext); + TEST_ASSERT_NOT_NULL(plaintext); + TEST_ASSERT_NOT_NULL(decryptedtext); + + mbedtls_aes_init(&ctx); + mbedtls_aes_setkey_enc(&ctx, key_256, 256); + + memset(plaintext, 0x3A, SZ); + memset(decryptedtext, 0x0, SZ); + + // Encrypt + mbedtls_aes_crypt_cbc(&ctx, MBEDTLS_AES_ENCRYPT, SZ, nonce, plaintext, chipertext); + TEST_ASSERT_EQUAL_HEX8_ARRAY(expected_cipher_end, chipertext + SZ - 32, 32); + + // Decrypt + memcpy(nonce, iv, 16); + mbedtls_aes_setkey_dec(&ctx, key_256, 256); + mbedtls_aes_crypt_cbc(&ctx, MBEDTLS_AES_DECRYPT, SZ, nonce, chipertext, decryptedtext); + + TEST_ASSERT_EQUAL_HEX8_ARRAY(plaintext, decryptedtext, SZ); + + mbedtls_aes_free(&ctx); + free(plaintext); + free(chipertext); + free(decryptedtext); + } + xSemaphoreGive(done_sem); + vTaskDelete(NULL); + +} + +#include "esp_crypto_shared_gdma.h" + +#define TASK_STACK_SIZE (20*1024) + +TEST_CASE("mbedtls AES/SHA multithreading", "[mbedtls]") +{ + done_sem = xSemaphoreCreateCounting(2, 0); + + xTaskCreate(tskRunSHA256Test, "SHA256Task", TASK_STACK_SIZE, NULL, 3, NULL); + xTaskCreate(tskRunAES256Test, "AES256Task", TASK_STACK_SIZE, NULL, 3, NULL); + + for (int i = 0; i < 2; i++) { + if (!xSemaphoreTake(done_sem, 10000 / portTICK_PERIOD_MS)) { + TEST_FAIL_MESSAGE("done_sem not released by test task"); + } + } + + vSemaphoreDelete(done_sem); +} diff --git a/components/soc/esp32s3/include/soc/soc.h b/components/soc/esp32s3/include/soc/soc.h index 7aaa424cd3..dcc0572303 100644 --- a/components/soc/esp32s3/include/soc/soc.h +++ b/components/soc/esp32s3/include/soc/soc.h @@ -261,8 +261,9 @@ #define SOC_DMA_HIGH 0x3FD00000 // Region of memory accessible via DMA in external memory. See esp_ptr_dma_ext_capable(). -#define SOC_DMA_EXT_LOW 0x3C000000 -#define SOC_DMA_EXT_HIGH 0x3DFFFFFF +#define SOC_DMA_EXT_LOW SOC_EXTRAM_DATA_LOW +#define SOC_DMA_EXT_HIGH SOC_EXTRAM_DATA_HIGH + // Region of memory that is byte-accessible. See esp_ptr_byte_accessible(). #define SOC_BYTE_ACCESSIBLE_LOW 0x3FC88000