fix(mbedtls/crypto_shared_gdma): Enable AXI-DMA enable external memory AES-ECC access

- When external memory encryption is enabled, set the aes_ecc bit of AXI-DMA to enable memory access

Author: Harshal5

components/hal/esp32p4/include/hal/axi_dma_ll.h

@@ -273,6 +273,14 @@ static inline void axi_dma_ll_rx_enable_etm_task(axi_dma_dev_t *dev, uint32_t ch
     dev->in[channel].conf.in_conf0.in_etm_en_chn = enable;
 }
 
+/**
+ * @brief Whether to enable the mean access ecc or aes domain
+ */
+static inline void axi_dma_ll_rx_enable_ext_mem_ecc_aes_access(axi_dma_dev_t *dev, uint32_t channel, bool enable)
+{
+    dev->in[channel].conf.in_conf0.in_ecc_aec_en_chn = enable;
+}
+
 ///////////////////////////////////// TX /////////////////////////////////////////
 /**
  * @brief Get DMA TX channel interrupt status word
@@ -471,6 +479,14 @@ static inline void axi_dma_ll_tx_enable_etm_task(axi_dma_dev_t *dev, uint32_t ch
     dev->out[channel].conf.out_conf0.out_etm_en_chn = enable;
 }
 
+/**
+ * @brief Whether to enable the mean access ecc or aes domain
+ */
+static inline void axi_dma_ll_tx_enable_ext_mem_ecc_aes_access(axi_dma_dev_t *dev, uint32_t channel, bool enable)
+{
+    dev->out[channel].conf.out_conf0.out_ecc_aec_en_chn = enable;
+}
+
 ///////////////////////////////////// CRC-TX /////////////////////////////////////////
 
 /**

components/hal/test_apps/crypto/main/CMakeLists.txt

@@ -68,7 +68,7 @@ if(CONFIG_SOC_SHA_SUPPORTED)
 endif()
 
 idf_component_register(SRCS ${srcs}
-                       PRIV_REQUIRES efuse mbedtls esp_mm
+                       PRIV_REQUIRES efuse mbedtls esp_mm bootloader_support
                        REQUIRES test_utils unity
                        WHOLE_ARCHIVE
                        PRIV_INCLUDE_DIRS "${priv_include_dirs}"

components/mbedtls/CMakeLists.txt

@@ -201,6 +201,9 @@ if(SHA_PERIPHERAL_TYPE STREQUAL "dma" OR AES_PERIPHERAL_TYPE STREQUAL "dma")
     target_link_libraries(mbedcrypto PRIVATE idf::esp_mm)
     if(CONFIG_SOC_SHA_GDMA OR CONFIG_SOC_AES_GDMA)
         target_sources(mbedcrypto PRIVATE "${COMPONENT_DIR}/port/crypto_shared_gdma/esp_crypto_shared_gdma.c")
+        if(CONFIG_SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT)
+            target_link_libraries(mbedcrypto PRIVATE idf::spi_flash idf::bootloader_support)
+        endif()
     endif()
 endif()
 

components/mbedtls/port/aes/dma/esp_aes_dma_core.c

@@ -16,6 +16,8 @@
 #include "esp_memory_utils.h"
 #include "esp_private/esp_cache_private.h"
 #include "esp_private/periph_ctrl.h"
+#include "soc/soc_caps.h"
+#include "sdkconfig.h"
 
 #if CONFIG_PM_ENABLE
 #include "esp_pm.h"
@@ -36,10 +38,19 @@
 #include "aes/esp_aes_gcm.h"
 #endif
 
+#ifdef SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT
+#include "esp_flash_encrypt.h"
+#endif /* SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT */
+
 /* Max size of each chunk to process when output buffer is in unaligned external ram
    must be a multiple of block size
 */
+#if (CONFIG_IDF_TARGET_ESP32P4 && CONFIG_SPIRAM && SOC_PSRAM_DMA_CAPABLE)
+/* As P4 has larger memory than other targets, thus we can support a larger chunk write size */
+#define AES_MAX_CHUNK_WRITE_SIZE 8*1024
+#else
 #define AES_MAX_CHUNK_WRITE_SIZE 1600
+#endif
 
 /* Input over this length will yield and wait for interrupt instead of
    busy-waiting, 30000 bytes is approx 0.5 ms */
@@ -163,6 +174,26 @@ static int esp_aes_dma_wait_complete(bool use_intr, crypto_dma_desc_t *output_de
     return 0;
 }
 
+static inline size_t get_cache_line_size(const void *addr)
+{
+    esp_err_t ret = ESP_FAIL;
+    size_t cache_line_size = 0;
+
+#if (CONFIG_SPIRAM && SOC_PSRAM_DMA_CAPABLE)
+    if (esp_ptr_external_ram(addr)) {
+        ret = esp_cache_get_alignment(MALLOC_CAP_SPIRAM, &cache_line_size);
+    } else
+#endif
+    {
+        ret = esp_cache_get_alignment(MALLOC_CAP_DMA, &cache_line_size);
+    }
+
+    if (ret != ESP_OK) {
+        return 0;
+    }
+
+    return cache_line_size;
+}
 
 /* Output buffers in external ram needs to be 16-byte aligned and DMA can't access input in the iCache mem range,
    reallocate them into internal memory and encrypt in chunks to avoid
@@ -176,14 +207,32 @@ static int esp_aes_process_dma_ext_ram(esp_aes_context *ctx, const unsigned char
     size_t chunk_len;
     int ret = 0;
     int offset = 0;
+    uint32_t input_heap_caps = MALLOC_CAP_DMA;
+    uint32_t output_heap_caps = MALLOC_CAP_DMA;
     unsigned char *input_buf = NULL;
     unsigned char *output_buf = NULL;
     const unsigned char *dma_input;
     chunk_len = MIN(AES_MAX_CHUNK_WRITE_SIZE, len);
 
-    if (realloc_input) {
-        input_buf = heap_caps_malloc(chunk_len, MALLOC_CAP_DMA);
+    size_t input_alignment = 1;
+    size_t output_alignment = 1;
 
+/* When AES-DMA operations are carried out using external memory with external memory encryption enabled,
+   we need to make sure that the addresses and the sizes of the buffers on which the DMA operates are 16 byte-aligned. */
+#ifdef SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT
+    if (esp_flash_encryption_enabled()) {
+        if (esp_ptr_external_ram(input) || esp_ptr_external_ram(output) || esp_ptr_in_drom(input) || esp_ptr_in_drom(output)) {
+            input_alignment = MAX(get_cache_line_size(input), SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT);
+            output_alignment = MAX(get_cache_line_size(output), SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT);
+
+            input_heap_caps = MALLOC_CAP_8BIT | (esp_ptr_external_ram(input) ? MALLOC_CAP_SPIRAM : MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL);
+            output_heap_caps = MALLOC_CAP_8BIT | (esp_ptr_external_ram(output) ? MALLOC_CAP_SPIRAM : MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL);
+        }
+    }
+#endif /* SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT */
+
+    if (realloc_input) {
+        input_buf = heap_caps_aligned_alloc(input_alignment, chunk_len, input_heap_caps);
         if (input_buf == NULL) {
             mbedtls_platform_zeroize(output, len);
             ESP_LOGE(TAG, "Failed to allocate memory");
@@ -192,8 +241,7 @@ static int esp_aes_process_dma_ext_ram(esp_aes_context *ctx, const unsigned char
     }
 
     if (realloc_output) {
-        output_buf = heap_caps_malloc(chunk_len, MALLOC_CAP_DMA);
-
+        output_buf = heap_caps_aligned_alloc(output_alignment, chunk_len, output_heap_caps);
         if (output_buf == NULL) {
             mbedtls_platform_zeroize(output, len);
             ESP_LOGE(TAG, "Failed to allocate memory");
@@ -284,27 +332,6 @@ static inline void *aes_dma_calloc(size_t num, size_t size, uint32_t caps, size_
     return ptr;
 }
 
-static inline size_t get_cache_line_size(const void *addr)
-{
-    esp_err_t ret = ESP_FAIL;
-    size_t cache_line_size = 0;
-
-#if (CONFIG_SPIRAM && SOC_PSRAM_DMA_CAPABLE)
-    if (esp_ptr_external_ram(addr)) {
-        ret = esp_cache_get_alignment(MALLOC_CAP_SPIRAM, &cache_line_size);
-    } else
-#endif
-    {
-        ret = esp_cache_get_alignment(MALLOC_CAP_DMA, &cache_line_size);
-    }
-
-    if (ret != ESP_OK) {
-        return 0;
-    }
-
-    return cache_line_size;
-}
-
 static inline esp_err_t dma_desc_link(crypto_dma_desc_t *dmadesc, size_t crypto_dma_desc_num, size_t cache_line_size)
 {
     esp_err_t ret = ESP_OK;
@@ -404,7 +431,7 @@ static esp_err_t generate_descriptor_list(const uint8_t *buffer, const size_t le
     dma_descs_needed = (unaligned_start_bytes ? 1 : 0) + dma_desc_get_required_num(aligned_block_bytes, max_desc_size) + (unaligned_end_bytes ? 1 : 0);
 
     /* Allocate memory for DMA descriptors of total size aligned up to a multiple of cache line size */
-    dma_descriptors = (crypto_dma_desc_t *) aes_dma_calloc(dma_descs_needed, sizeof(crypto_dma_desc_t), MALLOC_CAP_DMA, NULL);
+    dma_descriptors = (crypto_dma_desc_t *) aes_dma_calloc(dma_descs_needed, sizeof(crypto_dma_desc_t), MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL, NULL);
     if (dma_descriptors == NULL) {
         ESP_LOGE(TAG, "Failed to allocate memory for the array of DMA descriptors");
         return ESP_FAIL;
@@ -413,7 +440,7 @@ static esp_err_t generate_descriptor_list(const uint8_t *buffer, const size_t le
     size_t populated_dma_descs = 0;
 
     if (unaligned_start_bytes) {
-        start_alignment_stream_buffer = aes_dma_calloc(alignment_buffer_size, sizeof(uint8_t), AES_DMA_ALLOC_CAPS, NULL);
+        start_alignment_stream_buffer = aes_dma_calloc(alignment_buffer_size, sizeof(uint8_t), AES_DMA_ALLOC_CAPS | (esp_ptr_external_ram(buffer) ? MALLOC_CAP_SPIRAM : MALLOC_CAP_INTERNAL) , NULL);
         if (start_alignment_stream_buffer == NULL) {
             ESP_LOGE(TAG, "Failed to allocate memory for start alignment buffer");
             return ESP_FAIL;
@@ -435,7 +462,7 @@ static esp_err_t generate_descriptor_list(const uint8_t *buffer, const size_t le
     }
 
     if (unaligned_end_bytes) {
-        end_alignment_stream_buffer = aes_dma_calloc(alignment_buffer_size, sizeof(uint8_t), AES_DMA_ALLOC_CAPS, NULL);
+        end_alignment_stream_buffer = aes_dma_calloc(alignment_buffer_size, sizeof(uint8_t), AES_DMA_ALLOC_CAPS | (esp_ptr_external_ram(buffer) ? MALLOC_CAP_SPIRAM : MALLOC_CAP_INTERNAL), NULL);
         if (end_alignment_stream_buffer == NULL) {
             ESP_LOGE(TAG, "Failed to allocate memory for end alignment buffer");
             return ESP_FAIL;
@@ -499,6 +526,20 @@ int esp_aes_process_dma(esp_aes_context *ctx, const unsigned char *input, unsign
         return MBEDTLS_ERR_AES_INVALID_INPUT_LENGTH;
     }
 
+#ifdef SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT
+    if (esp_flash_encryption_enabled()) {
+        if (esp_ptr_external_ram(input) || esp_ptr_external_ram(output) || esp_ptr_in_drom(input) || esp_ptr_in_drom(output)) {
+            if (((intptr_t)(input) & (SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT - 1)) != 0) {
+                input_needs_realloc = true;
+            }
+
+            if (((intptr_t)(output) & (SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT - 1)) != 0) {
+                output_needs_realloc = true;
+            }
+        }
+    }
+#endif /* SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT */
+
     /* DMA cannot access memory in the iCache range, copy input to internal ram */
     if (!s_check_dma_capable(input)) {
         input_needs_realloc = true;

components/mbedtls/port/crypto_shared_gdma/esp_crypto_shared_gdma.c

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2021-2023 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -13,7 +13,13 @@
 #include "esp_cache.h"
 #include "esp_crypto_dma.h"
 #include "esp_crypto_lock.h"
+#include "esp_memory_utils.h"
 #include "soc/soc_caps.h"
+#include "sdkconfig.h"
+
+#ifdef SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT
+#include "esp_flash_encrypt.h"
+#endif /* SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT */
 
 #if SOC_AHB_GDMA_VERSION == 1
 #include "hal/gdma_ll.h"
@@ -140,6 +146,22 @@ esp_err_t esp_crypto_shared_gdma_start(const lldesc_t *input, const lldesc_t *ou
     return ESP_OK;
 }
 
+/* The external memory ecc-aes access must be enabled when there exists
+   at least one buffer in the DMA descriptors that resides in external memory. */
+#ifdef SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT
+static bool check_dma_descs_need_ext_mem_ecc_aes_access(const crypto_dma_desc_t *dmadesc)
+{
+    crypto_dma_desc_t* desc = (crypto_dma_desc_t*) dmadesc;
+    while (desc) {
+        if (esp_ptr_in_drom(desc->buffer) || esp_ptr_external_ram(desc->buffer)) {
+            return true;
+        }
+        desc = desc->next;
+    }
+    return false;
+}
+#endif /* SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT */
+
 esp_err_t esp_crypto_shared_gdma_start_axi_ahb(const crypto_dma_desc_t *input, const crypto_dma_desc_t *output, gdma_trigger_peripheral_t peripheral)
 {
     int rx_ch_id = 0;
@@ -173,6 +195,23 @@ esp_err_t esp_crypto_shared_gdma_start_axi_ahb(const crypto_dma_desc_t *input, c
     axi_dma_ll_rx_reset_channel(&AXI_DMA, rx_ch_id);
 #endif /* SOC_AHB_GDMA_VERSION */
 
+/* When GDMA operations are carried out using external memory with external memory encryption enabled,
+   we need to enable AXI-DMA's AES-ECC mean access bit. */
+#if (SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT)
+    if (esp_flash_encryption_enabled()) {
+        int tx_ch_id = 0;
+        gdma_get_channel_id(tx_channel, &tx_ch_id);
+
+        if (check_dma_descs_need_ext_mem_ecc_aes_access(input) || check_dma_descs_need_ext_mem_ecc_aes_access(output)) {
+            axi_dma_ll_rx_enable_ext_mem_ecc_aes_access(&AXI_DMA, rx_ch_id, true);
+            axi_dma_ll_tx_enable_ext_mem_ecc_aes_access(&AXI_DMA, tx_ch_id, true);
+        } else {
+            axi_dma_ll_rx_enable_ext_mem_ecc_aes_access(&AXI_DMA, rx_ch_id, false);
+            axi_dma_ll_tx_enable_ext_mem_ecc_aes_access(&AXI_DMA, tx_ch_id, false);
+        }
+    }
+#endif /* SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT */
+
     gdma_start(tx_channel, (intptr_t)input);
     gdma_start(rx_channel, (intptr_t)output);