/* * SPDX-FileCopyrightText: 2020-2024 Espressif Systems (Shanghai) CO LTD * * SPDX-License-Identifier: Apache-2.0 */ #include #include #include #include #include "sdkconfig.h" #include "freertos/FreeRTOS.h" #include "soc/soc_caps.h" #include "esp_log.h" #include "esp_check.h" #include "esp_attr.h" #include "esp_err.h" #include "esp_private/gdma.h" #include "esp_memory_utils.h" #include "esp_async_memcpy.h" #include "esp_async_memcpy_priv.h" #include "esp_cache.h" #include "hal/dma_types.h" #include "hal/cache_hal.h" #include "hal/cache_ll.h" static const char *TAG = "async_mcp.gdma"; #ifdef CACHE_LL_L2MEM_NON_CACHE_ADDR #define MCP_GET_NON_CACHE_ADDR(addr) ((addr) ? CACHE_LL_L2MEM_NON_CACHE_ADDR(addr) : 0) #else #define MCP_GET_NON_CACHE_ADDR(addr) (addr) #endif #if SOC_AXI_GDMA_SUPPORTED #define MCP_DMA_DESC_ALIGN 8 typedef dma_descriptor_align8_t mcp_dma_descriptor_t; #elif SOC_AHB_GDMA_SUPPORTED #define MCP_DMA_DESC_ALIGN 4 typedef dma_descriptor_align4_t mcp_dma_descriptor_t; #else #error "Unsupported GDMA type" #endif /// @brief Transaction object for async memcpy /// @note - GDMA requires the DMA descriptors to be 4 or 8 bytes aligned /// @note - The DMA descriptor link list is allocated dynamically from DMA-able memory /// @note - Because of the eof_node, the transaction object should also be allocated from DMA-able memory typedef struct async_memcpy_transaction_t { mcp_dma_descriptor_t eof_node; // this is the DMA node which act as the EOF descriptor (RX path only) mcp_dma_descriptor_t *tx_desc_link; // descriptor link list, the length of the link is determined by the copy buffer size mcp_dma_descriptor_t *tx_desc_nc; // non-cacheable version of tx_desc_link mcp_dma_descriptor_t *rx_desc_link; // descriptor link list, the length of the link is determined by the copy buffer size mcp_dma_descriptor_t *rx_desc_nc; // non-cacheable version of rx_desc_link intptr_t tx_start_desc_addr; // TX start descriptor address intptr_t rx_start_desc_addr; // RX start descriptor address void *memcpy_dst_addr; // memcpy destination address size_t memcpy_size; // memcpy size async_memcpy_isr_cb_t cb; // user callback void *cb_args; // user callback args STAILQ_ENTRY(async_memcpy_transaction_t) idle_queue_entry; // Entry for the idle queue STAILQ_ENTRY(async_memcpy_transaction_t) ready_queue_entry; // Entry for the ready queue } async_memcpy_transaction_t; /// @brief Context of async memcpy driver /// @note - It saves two queues, one for idle transaction objects, one for ready transaction objects /// @note - Transaction objects are allocated from DMA-able memory /// @note - Number of transaction objects are determined by the backlog parameter typedef struct { async_memcpy_context_t parent; // Parent IO interface size_t descriptor_align; // DMA descriptor alignment size_t rx_int_mem_alignment; // DMA buffer alignment (both in size and address) for internal RX memory size_t rx_ext_mem_alignment; // DMA buffer alignment (both in size and address) for external RX memory size_t tx_int_mem_alignment; // DMA buffer alignment (both in size and address) for internal TX memory size_t tx_ext_mem_alignment; // DMA buffer alignment (both in size and address) for external TX memory size_t max_single_dma_buffer; // max DMA buffer size by a single descriptor int gdma_bus_id; // GDMA bus id (AHB, AXI, etc.) gdma_channel_handle_t tx_channel; // GDMA TX channel handle gdma_channel_handle_t rx_channel; // GDMA RX channel handle portMUX_TYPE spin_lock; // spin lock to avoid threads and isr from accessing the same resource simultaneously _Atomic async_memcpy_fsm_t fsm; // driver state machine, changing state should be atomic async_memcpy_transaction_t *transaction_pool; // transaction object pool STAILQ_HEAD(, async_memcpy_transaction_t) idle_queue_head; // Head of the idle queue STAILQ_HEAD(, async_memcpy_transaction_t) ready_queue_head; // Head of the ready queue } async_memcpy_gdma_context_t; static bool mcp_gdma_rx_eof_callback(gdma_channel_handle_t dma_chan, gdma_event_data_t *event_data, void *user_data); static esp_err_t mcp_gdma_del(async_memcpy_context_t *ctx); static esp_err_t mcp_gdma_memcpy(async_memcpy_context_t *ctx, void *dst, void *src, size_t n, async_memcpy_isr_cb_t cb_isr, void *cb_args); #if SOC_GDMA_SUPPORT_ETM static esp_err_t mcp_new_etm_event(async_memcpy_context_t *ctx, async_memcpy_etm_event_t event_type, esp_etm_event_handle_t *out_event); #endif // SOC_GDMA_SUPPORT_ETM static esp_err_t mcp_gdma_destroy(async_memcpy_gdma_context_t *mcp_gdma) { if (mcp_gdma->transaction_pool) { free(mcp_gdma->transaction_pool); } if (mcp_gdma->tx_channel) { gdma_disconnect(mcp_gdma->tx_channel); gdma_del_channel(mcp_gdma->tx_channel); } if (mcp_gdma->rx_channel) { gdma_disconnect(mcp_gdma->rx_channel); gdma_del_channel(mcp_gdma->rx_channel); } free(mcp_gdma); return ESP_OK; } static esp_err_t esp_async_memcpy_install_gdma_template(const async_memcpy_config_t *config, async_memcpy_handle_t *mcp, esp_err_t (*new_channel)(const gdma_channel_alloc_config_t *, gdma_channel_handle_t *), int gdma_bus_id) { esp_err_t ret = ESP_OK; async_memcpy_gdma_context_t *mcp_gdma = NULL; ESP_RETURN_ON_FALSE(config && mcp, ESP_ERR_INVALID_ARG, TAG, "invalid argument"); // allocate memory of driver context from internal memory mcp_gdma = heap_caps_calloc(1, sizeof(async_memcpy_gdma_context_t), MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); ESP_GOTO_ON_FALSE(mcp_gdma, ESP_ERR_NO_MEM, err, TAG, "no mem for driver context"); uint32_t trans_queue_len = config->backlog ? config->backlog : DEFAULT_TRANSACTION_QUEUE_LENGTH; // allocate memory for transaction pool uint32_t data_cache_line_size = cache_hal_get_cache_line_size(CACHE_LL_LEVEL_INT_MEM, CACHE_TYPE_DATA); uint32_t alignment = MAX(data_cache_line_size, MCP_DMA_DESC_ALIGN); mcp_gdma->transaction_pool = heap_caps_aligned_calloc(alignment, trans_queue_len, sizeof(async_memcpy_transaction_t), MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT | MALLOC_CAP_DMA); ESP_GOTO_ON_FALSE(mcp_gdma->transaction_pool, ESP_ERR_NO_MEM, err, TAG, "no mem for transaction pool"); mcp_gdma->descriptor_align = alignment; // create TX channel and RX channel, they should reside in the same DMA pair gdma_channel_alloc_config_t tx_alloc_config = { .flags.reserve_sibling = 1, .direction = GDMA_CHANNEL_DIRECTION_TX, }; ESP_GOTO_ON_ERROR(new_channel(&tx_alloc_config, &mcp_gdma->tx_channel), err, TAG, "failed to create GDMA TX channel"); gdma_channel_alloc_config_t rx_alloc_config = { .direction = GDMA_CHANNEL_DIRECTION_RX, .sibling_chan = mcp_gdma->tx_channel, }; ESP_GOTO_ON_ERROR(new_channel(&rx_alloc_config, &mcp_gdma->rx_channel), err, TAG, "failed to create GDMA RX channel"); // initialize GDMA channels gdma_trigger_t m2m_trigger = GDMA_MAKE_TRIGGER(GDMA_TRIG_PERIPH_M2M, 0); // get a free DMA trigger ID for memory copy uint32_t free_m2m_id_mask = 0; gdma_get_free_m2m_trig_id_mask(mcp_gdma->tx_channel, &free_m2m_id_mask); m2m_trigger.instance_id = __builtin_ctz(free_m2m_id_mask); ESP_GOTO_ON_ERROR(gdma_connect(mcp_gdma->rx_channel, m2m_trigger), err, TAG, "GDMA rx connect failed"); ESP_GOTO_ON_ERROR(gdma_connect(mcp_gdma->tx_channel, m2m_trigger), err, TAG, "GDMA tx connect failed"); gdma_transfer_config_t transfer_cfg = { .max_data_burst_size = config->dma_burst_size ? config->dma_burst_size : 16, .access_ext_mem = true, // allow to do memory copy from/to external memory }; ESP_GOTO_ON_ERROR(gdma_config_transfer(mcp_gdma->tx_channel, &transfer_cfg), err, TAG, "config transfer for tx channel failed"); ESP_GOTO_ON_ERROR(gdma_config_transfer(mcp_gdma->rx_channel, &transfer_cfg), err, TAG, "config transfer for rx channel failed"); // register rx eof callback gdma_rx_event_callbacks_t cbs = { .on_recv_eof = mcp_gdma_rx_eof_callback, }; ESP_GOTO_ON_ERROR(gdma_register_rx_event_callbacks(mcp_gdma->rx_channel, &cbs, mcp_gdma), err, TAG, "failed to register RX EOF callback"); // initialize transaction queue STAILQ_INIT(&mcp_gdma->idle_queue_head); STAILQ_INIT(&mcp_gdma->ready_queue_head); // pick transactions from the pool and insert to the idle queue for (int i = 0; i < trans_queue_len; i++) { STAILQ_INSERT_TAIL(&mcp_gdma->idle_queue_head, &mcp_gdma->transaction_pool[i], idle_queue_entry); } // initialize other members portMUX_INITIALIZE(&mcp_gdma->spin_lock); atomic_init(&mcp_gdma->fsm, MCP_FSM_IDLE); mcp_gdma->gdma_bus_id = gdma_bus_id; // get the buffer alignment required by the GDMA channel gdma_get_alignment_constraints(mcp_gdma->rx_channel, &mcp_gdma->rx_int_mem_alignment, &mcp_gdma->rx_ext_mem_alignment); gdma_get_alignment_constraints(mcp_gdma->tx_channel, &mcp_gdma->tx_int_mem_alignment, &mcp_gdma->tx_ext_mem_alignment); size_t buf_align = MAX(MAX(mcp_gdma->rx_int_mem_alignment, mcp_gdma->rx_ext_mem_alignment), MAX(mcp_gdma->tx_int_mem_alignment, mcp_gdma->tx_ext_mem_alignment)); mcp_gdma->max_single_dma_buffer = ALIGN_DOWN(DMA_DESCRIPTOR_BUFFER_MAX_SIZE, buf_align); mcp_gdma->parent.del = mcp_gdma_del; mcp_gdma->parent.memcpy = mcp_gdma_memcpy; #if SOC_GDMA_SUPPORT_ETM mcp_gdma->parent.new_etm_event = mcp_new_etm_event; #endif // return driver object *mcp = &mcp_gdma->parent; return ESP_OK; err: if (mcp_gdma) { mcp_gdma_destroy(mcp_gdma); } return ret; } #if SOC_AHB_GDMA_SUPPORTED esp_err_t esp_async_memcpy_install_gdma_ahb(const async_memcpy_config_t *config, async_memcpy_handle_t *mcp) { return esp_async_memcpy_install_gdma_template(config, mcp, gdma_new_ahb_channel, SOC_GDMA_BUS_AHB); } #endif // SOC_AHB_GDMA_SUPPORTED #if SOC_AXI_GDMA_SUPPORTED esp_err_t esp_async_memcpy_install_gdma_axi(const async_memcpy_config_t *config, async_memcpy_handle_t *mcp) { return esp_async_memcpy_install_gdma_template(config, mcp, gdma_new_axi_channel, SOC_GDMA_BUS_AXI); } #endif // SOC_AXI_GDMA_SUPPORTED #if SOC_AHB_GDMA_SUPPORTED /// default installation falls back to use the AHB GDMA esp_err_t esp_async_memcpy_install(const async_memcpy_config_t *config, async_memcpy_handle_t *asmcp) __attribute__((alias("esp_async_memcpy_install_gdma_ahb"))); #elif SOC_AXI_GDMA_SUPPORTED /// default installation falls back to use the AXI GDMA esp_err_t esp_async_memcpy_install(const async_memcpy_config_t *config, async_memcpy_handle_t *asmcp) __attribute__((alias("esp_async_memcpy_install_gdma_axi"))); #endif static esp_err_t mcp_gdma_del(async_memcpy_context_t *ctx) { async_memcpy_gdma_context_t *mcp_gdma = __containerof(ctx, async_memcpy_gdma_context_t, parent); // check if there are pending transactions ESP_RETURN_ON_FALSE(STAILQ_EMPTY(&mcp_gdma->ready_queue_head), ESP_ERR_INVALID_STATE, TAG, "there are pending transactions"); // check if the driver is in IDLE state ESP_RETURN_ON_FALSE(atomic_load(&mcp_gdma->fsm) == MCP_FSM_IDLE, ESP_ERR_INVALID_STATE, TAG, "driver is not in IDLE state"); return mcp_gdma_destroy(mcp_gdma); } static void mount_tx_buffer_to_dma(async_memcpy_transaction_t *trans, int num_desc, uint8_t *buf, size_t buf_sz, size_t max_single_dma_buffer) { mcp_dma_descriptor_t *desc_array = trans->tx_desc_link; mcp_dma_descriptor_t *desc_nc = trans->tx_desc_nc; uint32_t prepared_length = 0; size_t len = buf_sz; for (int i = 0; i < num_desc - 1; i++) { desc_nc[i].buffer = &buf[prepared_length]; desc_nc[i].dw0.owner = DMA_DESCRIPTOR_BUFFER_OWNER_DMA; desc_nc[i].dw0.suc_eof = 0; desc_nc[i].dw0.size = max_single_dma_buffer; desc_nc[i].dw0.length = max_single_dma_buffer; desc_nc[i].next = &desc_array[i + 1]; prepared_length += max_single_dma_buffer; len -= max_single_dma_buffer; } // take special care to the EOF descriptor desc_nc[num_desc - 1].buffer = &buf[prepared_length]; desc_nc[num_desc - 1].next = NULL; desc_nc[num_desc - 1].dw0.owner = DMA_DESCRIPTOR_BUFFER_OWNER_DMA; desc_nc[num_desc - 1].dw0.suc_eof = 1; desc_nc[num_desc - 1].dw0.size = len; desc_nc[num_desc - 1].dw0.length = len; } static void mount_rx_buffer_to_dma(async_memcpy_transaction_t *trans, int num_desc, uint8_t *buf, size_t buf_sz, size_t max_single_dma_buffer) { mcp_dma_descriptor_t *desc_array = trans->rx_desc_link; mcp_dma_descriptor_t *desc_nc = trans->rx_desc_nc; mcp_dma_descriptor_t *eof_desc = &trans->eof_node; mcp_dma_descriptor_t *eof_nc = (mcp_dma_descriptor_t *)MCP_GET_NON_CACHE_ADDR(eof_desc); uint32_t prepared_length = 0; size_t len = buf_sz; if (desc_array) { assert(num_desc > 0); for (int i = 0; i < num_desc; i++) { desc_nc[i].buffer = &buf[prepared_length]; desc_nc[i].dw0.owner = DMA_DESCRIPTOR_BUFFER_OWNER_DMA; desc_nc[i].dw0.size = max_single_dma_buffer; desc_nc[i].dw0.length = max_single_dma_buffer; desc_nc[i].next = &desc_array[i + 1]; prepared_length += max_single_dma_buffer; len -= max_single_dma_buffer; } desc_nc[num_desc - 1].next = eof_desc; } eof_nc->buffer = &buf[prepared_length]; eof_nc->next = NULL; eof_nc->dw0.owner = DMA_DESCRIPTOR_BUFFER_OWNER_DMA; eof_nc->dw0.size = len; eof_nc->dw0.length = len; } /// @brief help function to get one transaction from the ready queue /// @note this function is allowed to be called in ISR static async_memcpy_transaction_t *try_pop_trans_from_ready_queue(async_memcpy_gdma_context_t *mcp_gdma) { async_memcpy_transaction_t *trans = NULL; portENTER_CRITICAL_SAFE(&mcp_gdma->spin_lock); trans = STAILQ_FIRST(&mcp_gdma->ready_queue_head); if (trans) { STAILQ_REMOVE_HEAD(&mcp_gdma->ready_queue_head, ready_queue_entry); } portEXIT_CRITICAL_SAFE(&mcp_gdma->spin_lock); return trans; } /// @brief help function to start a pending transaction /// @note this function is allowed to be called in ISR static void try_start_pending_transaction(async_memcpy_gdma_context_t *mcp_gdma) { async_memcpy_fsm_t expected_fsm = MCP_FSM_IDLE; async_memcpy_transaction_t *trans = NULL; if (atomic_compare_exchange_strong(&mcp_gdma->fsm, &expected_fsm, MCP_FSM_RUN_WAIT)) { trans = try_pop_trans_from_ready_queue(mcp_gdma); if (trans) { atomic_store(&mcp_gdma->fsm, MCP_FSM_RUN); gdma_start(mcp_gdma->rx_channel, trans->rx_start_desc_addr); gdma_start(mcp_gdma->tx_channel, trans->tx_start_desc_addr); } else { atomic_store(&mcp_gdma->fsm, MCP_FSM_IDLE); } } } /// @brief help function to get one transaction from the idle queue /// @note this function is allowed to be called in ISR static async_memcpy_transaction_t *try_pop_trans_from_idle_queue(async_memcpy_gdma_context_t *mcp_gdma) { async_memcpy_transaction_t *trans = NULL; portENTER_CRITICAL_SAFE(&mcp_gdma->spin_lock); trans = STAILQ_FIRST(&mcp_gdma->idle_queue_head); if (trans) { STAILQ_REMOVE_HEAD(&mcp_gdma->idle_queue_head, idle_queue_entry); } portEXIT_CRITICAL_SAFE(&mcp_gdma->spin_lock); return trans; } static bool check_buffer_alignment(async_memcpy_gdma_context_t *mcp_gdma, void *src, void *dst, size_t n) { bool valid = true; if (esp_ptr_external_ram(dst)) { valid = valid && (((uint32_t)dst & (mcp_gdma->rx_ext_mem_alignment - 1)) == 0); valid = valid && ((n & (mcp_gdma->rx_ext_mem_alignment - 1)) == 0); } else { valid = valid && (((uint32_t)dst & (mcp_gdma->rx_int_mem_alignment - 1)) == 0); valid = valid && ((n & (mcp_gdma->rx_int_mem_alignment - 1)) == 0); } if (esp_ptr_external_ram(src)) { valid = valid && (((uint32_t)src & (mcp_gdma->tx_ext_mem_alignment - 1)) == 0); valid = valid && ((n & (mcp_gdma->tx_ext_mem_alignment - 1)) == 0); } else { valid = valid && (((uint32_t)src & (mcp_gdma->tx_int_mem_alignment - 1)) == 0); valid = valid && ((n & (mcp_gdma->tx_int_mem_alignment - 1)) == 0); } return valid; } static esp_err_t mcp_gdma_memcpy(async_memcpy_context_t *ctx, void *dst, void *src, size_t n, async_memcpy_isr_cb_t cb_isr, void *cb_args) { esp_err_t ret = ESP_OK; async_memcpy_gdma_context_t *mcp_gdma = __containerof(ctx, async_memcpy_gdma_context_t, parent); // buffer location check #if SOC_AHB_GDMA_SUPPORTED && !SOC_AHB_GDMA_SUPPORT_PSRAM if (mcp_gdma->gdma_bus_id == SOC_GDMA_BUS_AHB) { ESP_RETURN_ON_FALSE(esp_ptr_internal(src) && esp_ptr_internal(dst), ESP_ERR_INVALID_ARG, TAG, "AHB GDMA can only access SRAM"); } #endif // SOC_AHB_GDMA_SUPPORTED && !SOC_AHB_GDMA_SUPPORT_PSRAM #if SOC_AXI_GDMA_SUPPORTED && !SOC_AXI_GDMA_SUPPORT_PSRAM if (mcp_gdma->gdma_bus_id == SOC_GDMA_BUS_AXI) { ESP_RETURN_ON_FALSE(esp_ptr_internal(src) && esp_ptr_internal(dst), ESP_ERR_INVALID_ARG, TAG, "AXI DMA can only access SRAM"); } #endif // SOC_AXI_GDMA_SUPPORTED && !SOC_AXI_GDMA_SUPPORT_PSRAM // alignment check ESP_RETURN_ON_FALSE(check_buffer_alignment(mcp_gdma, src, dst, n), ESP_ERR_INVALID_ARG, TAG, "buffer not aligned: %p -> %p, sz=%zu", src, dst, n); async_memcpy_transaction_t *trans = NULL; // pick one transaction node from idle queue trans = try_pop_trans_from_idle_queue(mcp_gdma); // check if we get the transaction object successfully ESP_RETURN_ON_FALSE(trans, ESP_ERR_INVALID_STATE, TAG, "no free node in the idle queue"); // calculate how many descriptors we want size_t max_single_dma_buffer = mcp_gdma->max_single_dma_buffer; uint32_t num_desc_per_path = (n + max_single_dma_buffer - 1) / max_single_dma_buffer; // allocate DMA descriptors, descriptors need a strict alignment trans->tx_desc_link = heap_caps_aligned_calloc(mcp_gdma->descriptor_align, num_desc_per_path, sizeof(mcp_dma_descriptor_t), MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT | MALLOC_CAP_DMA); ESP_GOTO_ON_FALSE(trans->tx_desc_link, ESP_ERR_NO_MEM, err, TAG, "no mem for DMA descriptors"); trans->tx_desc_nc = (mcp_dma_descriptor_t *)MCP_GET_NON_CACHE_ADDR(trans->tx_desc_link); // don't have to allocate the EOF descriptor, we will use trans->eof_node as the RX EOF descriptor if (num_desc_per_path > 1) { trans->rx_desc_link = heap_caps_aligned_calloc(mcp_gdma->descriptor_align, num_desc_per_path - 1, sizeof(mcp_dma_descriptor_t), MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT | MALLOC_CAP_DMA); ESP_GOTO_ON_FALSE(trans->rx_desc_link, ESP_ERR_NO_MEM, err, TAG, "no mem for DMA descriptors"); trans->rx_desc_nc = (mcp_dma_descriptor_t *)MCP_GET_NON_CACHE_ADDR(trans->rx_desc_link); } else { // small copy buffer, use the trans->eof_node is sufficient trans->rx_desc_link = NULL; trans->rx_desc_nc = NULL; } // (preload) mount src data to the TX descriptor mount_tx_buffer_to_dma(trans, num_desc_per_path, src, n, max_single_dma_buffer); // (preload) mount dst data to the RX descriptor mount_rx_buffer_to_dma(trans, num_desc_per_path - 1, dst, n, max_single_dma_buffer); // if the data is in the cache, write back, then DMA can see the latest data bool need_write_back = false; if (esp_ptr_external_ram(src)) { need_write_back = true; } else if (esp_ptr_internal(src)) { #if SOC_CACHE_INTERNAL_MEM_VIA_L1CACHE need_write_back = true; #endif } if (need_write_back) { esp_cache_msync(src, n, ESP_CACHE_MSYNC_FLAG_DIR_C2M); } // save other transaction context trans->cb = cb_isr; trans->cb_args = cb_args; trans->memcpy_size = n; trans->memcpy_dst_addr = dst; // save the destination buffer address, because we may need to do data cache invalidate later trans->tx_start_desc_addr = (intptr_t)trans->tx_desc_link; trans->rx_start_desc_addr = trans->rx_desc_link ? (intptr_t)trans->rx_desc_link : (intptr_t)&trans->eof_node; portENTER_CRITICAL(&mcp_gdma->spin_lock); // insert the trans to ready queue STAILQ_INSERT_TAIL(&mcp_gdma->ready_queue_head, trans, ready_queue_entry); portEXIT_CRITICAL(&mcp_gdma->spin_lock); // check driver state, if there's no running transaction, start a new one try_start_pending_transaction(mcp_gdma); return ESP_OK; err: if (trans) { if (trans->tx_desc_link) { free(trans->tx_desc_link); trans->tx_desc_link = NULL; } if (trans->rx_desc_link) { free(trans->rx_desc_link); trans->rx_desc_link = NULL; } // return back the trans to idle queue portENTER_CRITICAL(&mcp_gdma->spin_lock); STAILQ_INSERT_TAIL(&mcp_gdma->idle_queue_head, trans, idle_queue_entry); portEXIT_CRITICAL(&mcp_gdma->spin_lock); } return ret; } static bool mcp_gdma_rx_eof_callback(gdma_channel_handle_t dma_chan, gdma_event_data_t *event_data, void *user_data) { bool need_yield = false; async_memcpy_gdma_context_t *mcp_gdma = (async_memcpy_gdma_context_t *)user_data; mcp_dma_descriptor_t *eof_desc = (mcp_dma_descriptor_t *)event_data->rx_eof_desc_addr; // get the transaction object address by the EOF descriptor address async_memcpy_transaction_t *trans = __containerof(eof_desc, async_memcpy_transaction_t, eof_node); // switch driver state from RUN to IDLE async_memcpy_fsm_t expected_fsm = MCP_FSM_RUN; if (atomic_compare_exchange_strong(&mcp_gdma->fsm, &expected_fsm, MCP_FSM_IDLE_WAIT)) { void *dst = trans->memcpy_dst_addr; // if the data is in the cache, invalidate, then CPU can see the latest data bool need_invalidate = false; if (esp_ptr_external_ram(dst)) { need_invalidate = true; } else if (esp_ptr_internal(dst)) { #if SOC_CACHE_INTERNAL_MEM_VIA_L1CACHE need_invalidate = true; #endif } if (need_invalidate) { esp_cache_msync(dst, trans->memcpy_size, ESP_CACHE_MSYNC_FLAG_DIR_M2C); } // invoked callback registered by user async_memcpy_isr_cb_t cb = trans->cb; if (cb) { async_memcpy_event_t e = { // No event data for now }; need_yield = cb(&mcp_gdma->parent, &e, trans->cb_args); } // recycle descriptor memory if (trans->tx_desc_link) { free(trans->tx_desc_link); trans->tx_desc_link = NULL; } if (trans->rx_desc_link) { free(trans->rx_desc_link); trans->rx_desc_link = NULL; } trans->cb = NULL; portENTER_CRITICAL_ISR(&mcp_gdma->spin_lock); // insert the trans object to the idle queue STAILQ_INSERT_TAIL(&mcp_gdma->idle_queue_head, trans, idle_queue_entry); portEXIT_CRITICAL_ISR(&mcp_gdma->spin_lock); atomic_store(&mcp_gdma->fsm, MCP_FSM_IDLE); } // try start the next pending transaction try_start_pending_transaction(mcp_gdma); return need_yield; } #if SOC_GDMA_SUPPORT_ETM static esp_err_t mcp_new_etm_event(async_memcpy_context_t *ctx, async_memcpy_etm_event_t event_type, esp_etm_event_handle_t *out_event) { async_memcpy_gdma_context_t *mcp_gdma = __containerof(ctx, async_memcpy_gdma_context_t, parent); if (event_type == ASYNC_MEMCPY_ETM_EVENT_COPY_DONE) { // use the RX EOF to indicate the async memcpy done event gdma_etm_event_config_t etm_event_conf = { .event_type = GDMA_ETM_EVENT_EOF, }; return gdma_new_etm_event(mcp_gdma->rx_channel, &etm_event_conf, out_event); } else { return ESP_ERR_NOT_SUPPORTED; } } #endif // SOC_GDMA_SUPPORT_ETM