Merge branch 'feat/async_memcpy_any_alignment_v5.3' into 'release/v5.3'

async memcpy destination address doesn't have to be cache aligned (v5.3)

See merge request espressif/esp-idf!36634

Author: suda-morris

components/esp_hw_support/dma/async_memcpy_gdma.c

@@ -13,8 +13,6 @@
 #include "esp_async_memcpy.h"
 #include "soc/soc_caps.h"
 
-#define ALIGN_DOWN(val, align)  ((val) & ~((align) - 1))
-
 #define DEFAULT_TRANSACTION_QUEUE_LENGTH 4
 
 #ifdef __cplusplus

components/esp_hw_support/dma/esp_async_memcpy_priv.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -13,14 +13,118 @@
 #include "esp_heap_caps.h"
 #include "esp_memory_utils.h"
 #include "esp_dma_utils.h"
+#include "esp_private/esp_dma_utils.h"
 #include "esp_private/esp_cache_private.h"
 #include "soc/soc_caps.h"
 #include "hal/hal_utils.h"
+#include "hal/cache_hal.h"
+#include "hal/cache_ll.h"
+#include "esp_cache.h"
 
 static const char *TAG = "dma_utils";
 
 #define ALIGN_UP_BY(num, align) (((num) + ((align) - 1)) & ~((align) - 1))
-#define ALIGN_DOWN_BY(num, align) ((num) & (~((align) - 1)))
+
+esp_err_t esp_dma_split_rx_buffer_to_cache_aligned(void *rx_buffer, size_t buffer_len, dma_buffer_split_array_t *align_buf_array, uint8_t** ret_stash_buffer)
+{
+    ESP_RETURN_ON_FALSE(rx_buffer && buffer_len && align_buf_array, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
+
+    // read the cache line size of internal and external memory, we also use this information to check if a given memory is behind the cache
+    size_t int_mem_cache_line_size = cache_hal_get_cache_line_size(CACHE_LL_LEVEL_INT_MEM, CACHE_TYPE_DATA);
+    size_t ext_mem_cache_line_size = cache_hal_get_cache_line_size(CACHE_LL_LEVEL_EXT_MEM, CACHE_TYPE_DATA);
+
+    size_t split_line_size = 0;
+    if (esp_ptr_external_ram(rx_buffer)) {
+        split_line_size = ext_mem_cache_line_size;
+    } else if (esp_ptr_internal(rx_buffer)) {
+        split_line_size = int_mem_cache_line_size;
+    }
+    ESP_LOGV(TAG, "split_line_size:%zu", split_line_size);
+
+    // allocate the stash buffer from internal RAM
+    // Note, the split_line_size can be 0, in this case, the stash_buffer is also NULL, which is fine
+    uint8_t* stash_buffer = heap_caps_calloc(2, split_line_size, MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT);
+    ESP_RETURN_ON_FALSE(!(split_line_size && !stash_buffer), ESP_ERR_NO_MEM, TAG, "no mem for stash buffer");
+
+    // clear align_array to avoid garbage data
+    memset(align_buf_array, 0, sizeof(dma_buffer_split_array_t));
+    bool need_cache_sync[3] = {false};
+
+    // if split_line_size is non-zero, split the buffer into head, body and tail
+    if (split_line_size > 0) {
+        // calculate head_overflow_len
+        size_t head_overflow_len = (uintptr_t)rx_buffer % split_line_size;
+        head_overflow_len = head_overflow_len ? split_line_size - head_overflow_len : 0;
+        ESP_LOGV(TAG, "head_addr:%p head_overflow_len:%zu", rx_buffer, head_overflow_len);
+        // calculate tail_overflow_len
+        size_t tail_overflow_len = ((uintptr_t)rx_buffer + buffer_len) % split_line_size;
+        ESP_LOGV(TAG, "tail_addr:%p tail_overflow_len:%zu", rx_buffer + buffer_len - tail_overflow_len, tail_overflow_len);
+
+        uint8_t extra_buf_count = 0;
+        uint8_t* input_buffer = (uint8_t*)rx_buffer;
+        align_buf_array->buf.head.recovery_address = input_buffer;
+        align_buf_array->buf.head.aligned_buffer = stash_buffer + split_line_size * extra_buf_count++;
+        align_buf_array->buf.head.length = head_overflow_len;
+        need_cache_sync[0] = int_mem_cache_line_size > 0;
+        align_buf_array->buf.body.recovery_address = input_buffer + head_overflow_len;
+        align_buf_array->buf.body.aligned_buffer = input_buffer + head_overflow_len;
+        align_buf_array->buf.body.length = buffer_len - head_overflow_len - tail_overflow_len;
+        need_cache_sync[1] = true;
+        align_buf_array->buf.tail.recovery_address = input_buffer + buffer_len - tail_overflow_len;
+        align_buf_array->buf.tail.aligned_buffer = stash_buffer + split_line_size * extra_buf_count++;
+        align_buf_array->buf.tail.length = tail_overflow_len;
+        need_cache_sync[2] = int_mem_cache_line_size > 0;
+
+        // special handling when input_buffer length is no more than buffer alignment
+        if (head_overflow_len >= buffer_len || tail_overflow_len >= buffer_len) {
+            align_buf_array->buf.head.length  = buffer_len ;
+            align_buf_array->buf.body.length  = 0 ;
+            align_buf_array->buf.tail.length  = 0 ;
+        }
+    } else {
+        align_buf_array->buf.body.aligned_buffer = rx_buffer;
+        align_buf_array->buf.body.recovery_address = rx_buffer;
+        align_buf_array->buf.body.length = buffer_len;
+        need_cache_sync[1] = false;
+    }
+
+    for (int i = 0; i < 3; i++) {
+        if (align_buf_array->aligned_buffer[i].length == 0) {
+            align_buf_array->aligned_buffer[i].aligned_buffer = NULL;
+            align_buf_array->aligned_buffer[i].recovery_address = NULL;
+            need_cache_sync[i] = false;
+        }
+    }
+
+    // invalidate the aligned buffer if necessary
+    for (int i = 0; i < 3; i++) {
+        if (need_cache_sync[i]) {
+            size_t sync_size = align_buf_array->aligned_buffer[i].length;
+            if (sync_size < split_line_size) {
+                // If the size is smaller than the cache line, we need to sync the split buffer (must be cache line sized)
+                sync_size = split_line_size;
+            }
+            esp_cache_msync(align_buf_array->aligned_buffer[i].aligned_buffer, sync_size, ESP_CACHE_MSYNC_FLAG_DIR_M2C);
+        }
+    }
+
+    *ret_stash_buffer = stash_buffer;
+    return ESP_OK;
+}
+
+esp_err_t esp_dma_merge_aligned_rx_buffers(dma_buffer_split_array_t *align_array)
+{
+    ESP_RETURN_ON_FALSE_ISR(align_array, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
+
+    // only need to copy the head and tail buffer
+    if (align_array->buf.head.length) {
+        memcpy(align_array->buf.head.recovery_address, align_array->buf.head.aligned_buffer, align_array->buf.head.length);
+    }
+    if (align_array->buf.tail.length) {
+        memcpy(align_array->buf.tail.recovery_address, align_array->buf.tail.aligned_buffer, align_array->buf.tail.length);
+    }
+    return ESP_OK;
+}
 
 esp_err_t esp_dma_capable_malloc(size_t size, const esp_dma_mem_info_t *dma_mem_info, void **out_ptr, size_t *actual_size)
 {

components/esp_hw_support/dma/esp_dma_utils.c

@@ -6,14 +6,8 @@
 
 #include <stdlib.h>
 #include <string.h>
-#include <stdatomic.h>
 #include <sys/cdefs.h>
-#include <sys/lock.h>
-#include "sdkconfig.h"
-#include "freertos/FreeRTOS.h"
-#include "freertos/task.h"
 #include "soc/soc_caps.h"
-#include "soc/ext_mem_defs.h"
 #include "esp_log.h"
 #include "esp_check.h"
 #include "esp_memory_utils.h"

components/esp_hw_support/dma/gdma_link.c

@@ -0,0 +1,88 @@
+/*
+ * SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#pragma once
+
+#include <stdbool.h>
+#include "esp_err.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * @brief DMA buffer information
+ */
+typedef struct {
+    void *aligned_buffer;   //!< Buffer address
+    void *recovery_address; //!< Origin buffer address that aligned buffer should be recovered
+    size_t length;          //!< Buffer length
+} dma_buffer_split_info_t;
+
+/**
+ * @brief DMA buffer aligned array
+ * The array contains three parts: head, body and tail.
+ * Length of each part will be >=0, especially, length=0 means that there is no such part.
+ */
+typedef struct {
+    union {
+        struct {
+            dma_buffer_split_info_t head;               //!< Aligned head part. Corresponds to the part of the original buffer where the head is not aligned
+            dma_buffer_split_info_t body;               //!< Aligned body part. Corresponds to the part of the original aligned buffer
+            dma_buffer_split_info_t tail;               //!< Aligned tail part. Corresponds to the part of the original buffer where the tail is not aligned
+        } buf;
+        dma_buffer_split_info_t aligned_buffer[3];      //!< DMA aligned buffer array, consist of `head`, `body` and `tail`
+    };
+} dma_buffer_split_array_t;
+
+/**
+ * @brief Split DMA RX buffer to cache aligned buffers
+ *
+ * @note After the original RX buffer is split into an array, caller should mount the buffer array to the DMA controller in scatter-gather mode.
+ *       Don't read/write the aligned buffers before the DMA finished using them.
+ *
+ * @param[in]   rx_buffer        The origin DMA buffer used for receiving data
+ * @param[in]   buffer_len       rx_buffer length
+ * @param[out]  align_buf_array  Aligned DMA buffer array
+ * @param[out]  ret_stash_buffer Allocated stash buffer (caller should free it after use)
+ * @return
+ *      - ESP_OK: Split to aligned buffer successfully
+ *      - ESP_ERR_INVALID_ARG: Split to aligned buffer failed because of invalid argument
+ *
+ *  brief sketch:
+ *                  cache alignment delimiter    cache alignment delimiter
+ *                              │                             │
+ *     Origin Buffer            │        Origin Buffer        │
+ *           │                  │              │              │
+ *           │                  ▼              ▼              ▼
+ *           │       ...---xxxxx|xxxxxxxxxxxxxxxxxxxxxxxxxxxxx|xxxxx----...
+ *           │               │                 │                 │
+ *           │               │                 ▼                 │
+ *           │               │  |xxxxxxxxxxxxxxxxxxxxxxxxxxxxx|  │
+ *           │               │                 ▲                 │
+ *           ▼               │                 │                 │
+ *     Aligned buffers       └──► Head        Body   Tail ◄──────┘
+ *                                 │                  │
+ *                                 ▼                  ▼
+ *                              |xxxxx......|     |xxxxx......|
+ */
+esp_err_t esp_dma_split_rx_buffer_to_cache_aligned(void *rx_buffer, size_t buffer_len, dma_buffer_split_array_t *align_buf_array, uint8_t** ret_stash_buffer);
+
+/**
+ * @brief Merge aligned RX buffer array to origin buffer
+ *
+ * @note This function can be used in the ISR context.
+ *
+ * @param[in] align_buf_array Aligned DMA buffer array
+ * @return
+ *      - ESP_OK: Merge aligned buffer to origin buffer successfully
+ *      - ESP_ERR_INVALID_ARG: Merge aligned buffer to origin buffer failed because of invalid argument
+ */
+esp_err_t esp_dma_merge_aligned_rx_buffers(dma_buffer_split_array_t *align_buf_array);
+
+#ifdef __cplusplus
+}
+#endif