esp-idf/components/esp_psram/esp_psram.c
Armando 2d44dc1eed mmu: driver framework, for vaddr maintenance
This commit gives basic mmu driver framework. Now it is able to maintain
mmu virtual address usage on esp32, esp32s2 and esp32s3. Usage to
external virtual address should rely on mmu functions to know which
address range is available, instead of hardcoded.

This commit also improves psram memory that is added to the heap
allocator. Now it's added to the heap, according to the memory
alignment.

Closes https://github.com/espressif/esp-idf/issues/8295
2022-08-26 17:59:06 +08:00

468 lines
19 KiB
C

/*
* SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
/*----------------------------------------------------------------------------------------------------
* Abstraction layer for PSRAM. PSRAM device related registers and MMU/Cache related code shouls be
* abstracted to lower layers.
*
* When we add more types of external RAM memory, this can be made into a more intelligent dispatcher.
*----------------------------------------------------------------------------------------------------*/
#include <sys/param.h>
#include "sdkconfig.h"
#include "esp_attr.h"
#include "esp_err.h"
#include "esp_log.h"
#include "freertos/FreeRTOS.h"
#include "freertos/xtensa_api.h"
#include "esp_heap_caps_init.h"
#include "hal/mmu_hal.h"
#include "hal/cache_ll.h"
#include "esp_private/esp_psram_io.h"
#include "esp_private/esp_psram_extram.h"
#include "esp_private/mmu_psram_flash.h"
#include "esp_psram_impl.h"
#include "esp_psram.h"
#include "mmu.h"
#if CONFIG_IDF_TARGET_ESP32
#include "esp32/himem.h"
#include "esp32/rom/cache.h"
#endif
#if CONFIG_IDF_TARGET_ESP32
#if CONFIG_FREERTOS_UNICORE
#define PSRAM_MODE PSRAM_VADDR_MODE_NORMAL
#else
#define PSRAM_MODE PSRAM_VADDR_MODE_LOWHIGH
#endif
#else
#define PSRAM_MODE PSRAM_VADDR_MODE_NORMAL
#endif
/**
* Two types of PSRAM memory regions for now:
* - 8bit aligned
* - 32bit aligned
*/
#define PSRAM_MEM_TYPE_NUM 2
#define PSRAM_MEM_8BIT_ALIGNED 0
#define PSRAM_MEM_32BIT_ALIGNED 1
#if CONFIG_SPIRAM_ALLOW_BSS_SEG_EXTERNAL_MEMORY
extern uint8_t _ext_ram_bss_start;
extern uint8_t _ext_ram_bss_end;
#endif //#if CONFIG_SPIRAM_ALLOW_BSS_SEG_EXTERNAL_MEMORY
#if CONFIG_SPIRAM_ALLOW_NOINIT_SEG_EXTERNAL_MEMORY
extern uint8_t _ext_ram_noinit_start;
extern uint8_t _ext_ram_noinit_end;
#endif //#if CONFIG_SPIRAM_ALLOW_NOINIT_SEG_EXTERNAL_MEMORY
typedef struct {
intptr_t vaddr_start;
intptr_t vaddr_end;
size_t size; //in bytes
} psram_mem_t;
typedef struct {
bool is_initialised;
/**
* @note 1
* As we can't use heap allocator during this stage, we need to statically declare these regions.
* Luckily only S2 has two different types of memory regions:
* - byte-aligned memory
* - word-aligned memory
* On the other hand, the type number usually won't be very big
*
* On other chips, only one region is needed.
* So for chips other than S2, size of `regions_to_heap[1]` and `mapped_regions[1]`will always be zero.
*
* If in the future, this condition is worse (dbus memory isn't consecutive), we need to delegate this context
* to chip-specific files, and only keep a (void *) pointer here pointing to those chip-specific contexts
*/
psram_mem_t regions_to_heap[PSRAM_MEM_TYPE_NUM]; //memory regions that are available to be added to the heap allocator
psram_mem_t mapped_regions[PSRAM_MEM_TYPE_NUM]; //mapped memory regions
} psram_ctx_t;
static psram_ctx_t s_psram_ctx;
static const char* TAG = "esp_psram";
#if CONFIG_IDF_TARGET_ESP32
//If no function in esp_himem.c is used, this function will be linked into the
//binary instead of the one in esp_himem.c, automatically making sure no memory
//is reserved if no himem function is used.
size_t __attribute__((weak)) esp_himem_reserved_area_size(void) {
return 0;
}
static void IRAM_ATTR s_mapping(int v_start, int size)
{
//Enable external RAM in MMU
cache_sram_mmu_set(0, 0, v_start, 0, 32, (size / 1024 / 32));
//Flush and enable icache for APP CPU
#if !CONFIG_FREERTOS_UNICORE
DPORT_CLEAR_PERI_REG_MASK(DPORT_APP_CACHE_CTRL1_REG, DPORT_APP_CACHE_MASK_DRAM1);
cache_sram_mmu_set(1, 0, v_start, 0, 32, (size / 1024 / 32));
#endif
}
#endif //CONFIG_IDF_TARGET_ESP32
esp_err_t esp_psram_init(void)
{
if (s_psram_ctx.is_initialised) {
return ESP_ERR_INVALID_STATE;
}
esp_err_t ret = ESP_FAIL;
ret = esp_psram_impl_enable(PSRAM_MODE);
if (ret != ESP_OK) {
#if CONFIG_SPIRAM_IGNORE_NOTFOUND
ESP_EARLY_LOGE(TAG, "PSRAM enabled but initialization failed. Bailing out.");
#endif
return ret;
}
s_psram_ctx.is_initialised = true;
uint32_t psram_physical_size = 0;
ret = esp_psram_impl_get_physical_size(&psram_physical_size);
assert(ret == ESP_OK);
ESP_EARLY_LOGI(TAG, "Found %dMB PSRAM device", psram_physical_size / (1024 * 1024));
ESP_EARLY_LOGI(TAG, "Speed: %dMHz", CONFIG_SPIRAM_SPEED);
#if CONFIG_IDF_TARGET_ESP32
ESP_EARLY_LOGI(TAG, "PSRAM initialized, cache is in %s mode.", \
(PSRAM_MODE==PSRAM_VADDR_MODE_EVENODD)?"even/odd (2-core)": \
(PSRAM_MODE==PSRAM_VADDR_MODE_LOWHIGH)?"low/high (2-core)": \
(PSRAM_MODE==PSRAM_VADDR_MODE_NORMAL)?"normal (1-core)":"ERROR");
#endif
uint32_t psram_available_size = 0;
ret = esp_psram_impl_get_available_size(&psram_available_size);
assert(ret == ESP_OK);
__attribute__((unused)) uint32_t total_available_size = psram_available_size;
/**
* `start_page` is the psram physical address in MMU page size.
* MMU page size on ESP32S2 is 64KB
* e.g.: psram physical address 16 is in page 0
*
* Here we plan to copy FLASH instructions to psram physical address 0, which is the No.0 page.
*/
__attribute__((unused)) uint32_t start_page = 0;
#if CONFIG_SPIRAM_FETCH_INSTRUCTIONS || CONFIG_SPIRAM_RODATA
uint32_t used_page = 0;
#endif
//------------------------------------Copy Flash .text to PSRAM-------------------------------------//
#if CONFIG_SPIRAM_FETCH_INSTRUCTIONS
ret = mmu_config_psram_text_segment(start_page, total_available_size, &used_page);
if (ret != ESP_OK) {
ESP_EARLY_LOGE(TAG, "No enough psram memory for instructon!");
abort();
}
start_page += used_page;
psram_available_size -= MMU_PAGE_TO_BYTES(used_page);
ESP_EARLY_LOGV(TAG, "after copy .text, used page is %d, start_page is %d, psram_available_size is %d B", used_page, start_page, psram_available_size);
#endif //#if CONFIG_SPIRAM_FETCH_INSTRUCTIONS
//------------------------------------Copy Flash .rodata to PSRAM-------------------------------------//
#if CONFIG_SPIRAM_RODATA
ret = mmu_config_psram_rodata_segment(start_page, total_available_size, &used_page);
if (ret != ESP_OK) {
ESP_EARLY_LOGE(TAG, "No enough psram memory for rodata!");
abort();
}
start_page += used_page;
psram_available_size -= MMU_PAGE_TO_BYTES(used_page);
ESP_EARLY_LOGV(TAG, "after copy .rodata, used page is %d, start_page is %d, psram_available_size is %d B", used_page, start_page, psram_available_size);
#endif //#if CONFIG_SPIRAM_RODATA
/**
* For now,
* - we only need to use MMU driver when PSRAM is enabled
* - MMU driver isn't public
*
* So we call `esp_mmu_init()` here, instead of calling it in startup code.
*/
esp_mmu_init();
//----------------------------------Map the PSRAM physical range to MMU-----------------------------//
/**
* @note 2
* Similarly to @note 1, we expect HW DBUS memory to be consecutive.
*
* If situation is worse in the future (memory region isn't consecutive), we need to put these logics into chip-specific files
*/
size_t total_mapped_size = 0;
size_t size_to_map = 0;
size_t byte_aligned_size = 0;
ret = esp_mmu_get_largest_free_block(MMU_MEM_CAP_READ | MMU_MEM_CAP_WRITE | MMU_MEM_CAP_8BIT | MMU_MEM_CAP_32BIT, &byte_aligned_size);
assert(ret == ESP_OK);
size_to_map = MIN(byte_aligned_size, psram_available_size);
const void *v_start_8bit_aligned = NULL;
ret = esp_mmu_find_vaddr_range(size_to_map, MMU_MEM_CAP_READ | MMU_MEM_CAP_WRITE | MMU_MEM_CAP_8BIT | MMU_MEM_CAP_32BIT, &v_start_8bit_aligned);
assert(ret == ESP_OK);
#if CONFIG_IDF_TARGET_ESP32
s_mapping((int)v_start_8bit_aligned, size_to_map);
#else
uint32_t actual_mapped_len = 0;
mmu_hal_map_region(0, MMU_TARGET_PSRAM0, (intptr_t)v_start_8bit_aligned, MMU_PAGE_TO_BYTES(start_page), size_to_map, &actual_mapped_len);
start_page += BYTES_TO_MMU_PAGE(actual_mapped_len);
ESP_EARLY_LOGV(TAG, "8bit-aligned-region: actual_mapped_len is 0x%x bytes", actual_mapped_len);
cache_bus_mask_t bus_mask = cache_ll_l1_get_bus(0, (uint32_t)v_start_8bit_aligned, actual_mapped_len);
cache_ll_l1_enable_bus(0, bus_mask);
#if !CONFIG_FREERTOS_UNICORE
bus_mask = cache_ll_l1_get_bus(1, (uint32_t)v_start_8bit_aligned, actual_mapped_len);
cache_ll_l1_enable_bus(1, bus_mask);
#endif
#endif //#if CONFIG_IDF_TARGET_ESP32
s_psram_ctx.mapped_regions[PSRAM_MEM_8BIT_ALIGNED].size = size_to_map;
s_psram_ctx.mapped_regions[PSRAM_MEM_8BIT_ALIGNED].vaddr_start = (intptr_t)v_start_8bit_aligned;
s_psram_ctx.mapped_regions[PSRAM_MEM_8BIT_ALIGNED].vaddr_end = (intptr_t)v_start_8bit_aligned + size_to_map;
s_psram_ctx.regions_to_heap[PSRAM_MEM_8BIT_ALIGNED].size = size_to_map;
s_psram_ctx.regions_to_heap[PSRAM_MEM_8BIT_ALIGNED].vaddr_start = (intptr_t)v_start_8bit_aligned;
s_psram_ctx.regions_to_heap[PSRAM_MEM_8BIT_ALIGNED].vaddr_end = (intptr_t)v_start_8bit_aligned + size_to_map;
ESP_EARLY_LOGV(TAG, "8bit-aligned-range: 0x%x B, starting from: 0x%x", s_psram_ctx.mapped_regions[PSRAM_MEM_8BIT_ALIGNED].size, v_start_8bit_aligned);
total_mapped_size += size_to_map;
#if CONFIG_IDF_TARGET_ESP32S2
/**
* On ESP32S2, there are 2 types of DBUS memory:
* - byte-aligned-memory
* - word-aligned-memory
*
* If byte-aligned-memory isn't enough, we search for word-aligned-memory to do mapping
*/
if (total_mapped_size < psram_available_size) {
size_to_map = psram_available_size - total_mapped_size;
size_t word_aligned_size = 0;
ret = esp_mmu_get_largest_free_block(MMU_MEM_CAP_READ | MMU_MEM_CAP_WRITE | MMU_MEM_CAP_32BIT, &word_aligned_size);
assert(ret == ESP_OK);
size_to_map = MIN(word_aligned_size, size_to_map);
const void *v_start_32bit_aligned = NULL;
ret = esp_mmu_find_vaddr_range(size_to_map, MMU_MEM_CAP_READ | MMU_MEM_CAP_WRITE | MMU_MEM_CAP_32BIT, &v_start_32bit_aligned);
assert(ret == ESP_OK);
mmu_hal_map_region(0, MMU_TARGET_PSRAM0, (intptr_t)v_start_32bit_aligned, MMU_PAGE_TO_BYTES(start_page), size_to_map, &actual_mapped_len);
ESP_EARLY_LOGV(TAG, "32bit-aligned-region: actual_mapped_len is 0x%x bytes", actual_mapped_len);
cache_bus_mask_t bus_mask = cache_ll_l1_get_bus(0, (uint32_t)v_start_32bit_aligned, actual_mapped_len);
cache_ll_l1_enable_bus(0, bus_mask);
s_psram_ctx.mapped_regions[PSRAM_MEM_32BIT_ALIGNED].size = size_to_map;
s_psram_ctx.mapped_regions[PSRAM_MEM_32BIT_ALIGNED].vaddr_start = (intptr_t)v_start_32bit_aligned;
s_psram_ctx.mapped_regions[PSRAM_MEM_32BIT_ALIGNED].vaddr_end = (intptr_t)v_start_32bit_aligned + size_to_map;
s_psram_ctx.regions_to_heap[PSRAM_MEM_32BIT_ALIGNED].size = size_to_map;
s_psram_ctx.regions_to_heap[PSRAM_MEM_32BIT_ALIGNED].vaddr_start = (intptr_t)v_start_32bit_aligned;
s_psram_ctx.regions_to_heap[PSRAM_MEM_32BIT_ALIGNED].vaddr_end = (intptr_t)v_start_32bit_aligned + size_to_map;
ESP_EARLY_LOGV(TAG, "32bit-aligned-range: 0x%x B, starting from: 0x%x", s_psram_ctx.mapped_regions[PSRAM_MEM_32BIT_ALIGNED].size, v_start_32bit_aligned);
total_mapped_size += size_to_map;
}
#endif // #if CONFIG_IDF_TARGET_ESP32S2
if (total_mapped_size < psram_available_size) {
ESP_EARLY_LOGW(TAG, "Virtual address not enough for PSRAM, map as much as we can. %dMB is mapped", total_mapped_size / 1024 / 1024);
}
/*------------------------------------------------------------------------------
* After mapping, we DON'T care about the PSRAM PHYSICAL ADDRESSS ANYMORE!
*----------------------------------------------------------------------------*/
//------------------------------------Configure .bss in PSRAM-------------------------------------//
#if CONFIG_SPIRAM_ALLOW_BSS_SEG_EXTERNAL_MEMORY
//should never be negative number
uint32_t ext_bss_size = ((intptr_t)&_ext_ram_bss_end - (intptr_t)&_ext_ram_bss_start);
ESP_EARLY_LOGV(TAG, "ext_bss_size is %d", ext_bss_size);
s_psram_ctx.regions_to_heap[PSRAM_MEM_8BIT_ALIGNED].vaddr_start += ext_bss_size;
s_psram_ctx.regions_to_heap[PSRAM_MEM_8BIT_ALIGNED].size -= ext_bss_size;
#endif //#if CONFIG_SPIRAM_ALLOW_BSS_SEG_EXTERNAL_MEMORY
#if CONFIG_SPIRAM_ALLOW_NOINIT_SEG_EXTERNAL_MEMORY
uint32_t ext_noinit_size = ((intptr_t)&_ext_ram_noinit_end - (intptr_t)&_ext_ram_noinit_start);
ESP_EARLY_LOGV(TAG, "ext_noinit_size is %d", ext_noinit_size);
s_psram_ctx.regions_to_heap[PSRAM_MEM_8BIT_ALIGNED].vaddr_start += ext_noinit_size;
s_psram_ctx.regions_to_heap[PSRAM_MEM_8BIT_ALIGNED].size -= ext_noinit_size;
#endif
#if CONFIG_IDF_TARGET_ESP32
s_psram_ctx.regions_to_heap[PSRAM_MEM_8BIT_ALIGNED].size -= esp_himem_reserved_area_size() - 1;
#endif
return ESP_OK;
}
esp_err_t esp_psram_extram_add_to_heap_allocator(void)
{
esp_err_t ret = ESP_FAIL;
uint32_t byte_aligned_caps[] = {MALLOC_CAP_SPIRAM|MALLOC_CAP_DEFAULT, 0, MALLOC_CAP_8BIT|MALLOC_CAP_32BIT};
ret = heap_caps_add_region_with_caps(byte_aligned_caps,
s_psram_ctx.regions_to_heap[PSRAM_MEM_8BIT_ALIGNED].vaddr_start,
s_psram_ctx.regions_to_heap[PSRAM_MEM_8BIT_ALIGNED].vaddr_end);
if (ret != ESP_OK) {
return ret;
}
if (s_psram_ctx.regions_to_heap[PSRAM_MEM_32BIT_ALIGNED].size) {
assert(s_psram_ctx.regions_to_heap[PSRAM_MEM_32BIT_ALIGNED].vaddr_start);
uint32_t word_aligned_caps[] = {MALLOC_CAP_SPIRAM|MALLOC_CAP_DEFAULT, 0, MALLOC_CAP_32BIT};
ret = heap_caps_add_region_with_caps(word_aligned_caps,
s_psram_ctx.regions_to_heap[PSRAM_MEM_32BIT_ALIGNED].vaddr_start,
s_psram_ctx.regions_to_heap[PSRAM_MEM_32BIT_ALIGNED].vaddr_end);
if (ret != ESP_OK) {
return ret;
}
}
ESP_EARLY_LOGI(TAG, "Adding pool of %dK of PSRAM memory to heap allocator",
(s_psram_ctx.regions_to_heap[PSRAM_MEM_8BIT_ALIGNED].size + s_psram_ctx.regions_to_heap[PSRAM_MEM_32BIT_ALIGNED].size) / 1024);
return ESP_OK;
}
bool IRAM_ATTR esp_psram_check_ptr_addr(const void *p)
{
if (!s_psram_ctx.is_initialised) {
return false;
}
return ((intptr_t)p >= s_psram_ctx.mapped_regions[PSRAM_MEM_8BIT_ALIGNED].vaddr_start && (intptr_t)p < s_psram_ctx.mapped_regions[PSRAM_MEM_8BIT_ALIGNED].vaddr_end) ||
((intptr_t)p >= s_psram_ctx.mapped_regions[PSRAM_MEM_32BIT_ALIGNED].vaddr_start && (intptr_t)p < s_psram_ctx.mapped_regions[PSRAM_MEM_32BIT_ALIGNED].vaddr_end);
}
esp_err_t esp_psram_extram_reserve_dma_pool(size_t size)
{
if (size == 0) {
return ESP_OK; //no-op
}
ESP_EARLY_LOGI(TAG, "Reserving pool of %dK of internal memory for DMA/internal allocations", size / 1024);
/* Pool may be allocated in multiple non-contiguous chunks, depending on available RAM */
while (size > 0) {
size_t next_size = heap_caps_get_largest_free_block(MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL);
next_size = MIN(next_size, size);
ESP_EARLY_LOGD(TAG, "Allocating block of size %d bytes", next_size);
uint8_t *dma_heap = heap_caps_malloc(next_size, MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL);
if (!dma_heap || next_size == 0) {
return ESP_ERR_NO_MEM;
}
uint32_t caps[] = {0, MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL, MALLOC_CAP_8BIT | MALLOC_CAP_32BIT};
esp_err_t e = heap_caps_add_region_with_caps(caps, (intptr_t)dma_heap, (intptr_t)dma_heap + next_size - 1);
if (e != ESP_OK) {
return e;
}
size -= next_size;
}
return ESP_OK;
}
bool IRAM_ATTR __attribute__((pure)) esp_psram_is_initialized(void)
{
return s_psram_ctx.is_initialised;
}
size_t esp_psram_get_size(void)
{
uint32_t available_size = 0;
esp_err_t ret = esp_psram_impl_get_available_size(&available_size);
if (ret != ESP_OK) {
//This means PSRAM isn't initialised, to keep back-compatibility, set size to 0.
available_size = 0;
}
return (size_t)available_size;
}
uint8_t esp_psram_io_get_cs_io(void)
{
return esp_psram_impl_get_cs_io();
}
/*
Simple RAM test. Writes a word every 32 bytes. Takes about a second to complete for 4MiB. Returns
true when RAM seems OK, false when test fails. WARNING: Do not run this before the 2nd cpu has been
initialized (in a two-core system) or after the heap allocator has taken ownership of the memory.
*/
static bool s_test_psram(intptr_t v_start, size_t size, intptr_t reserved_start, intptr_t reserved_end)
{
volatile int *spiram = (volatile int *)v_start;
size_t p;
int errct = 0;
int initial_err = -1;
for (p = 0; p < (size / sizeof(int)); p += 8) {
intptr_t addr = (intptr_t)&spiram[p];
if ((reserved_start <= addr) && (addr < reserved_end)) {
continue;
}
spiram[p] = p ^ 0xAAAAAAAA;
}
for (p = 0; p < (size / sizeof(int)); p += 8) {
intptr_t addr = (intptr_t)&spiram[p];
if ((reserved_start <= addr) && (addr < reserved_end)) {
continue;
}
if (spiram[p] != (p ^ 0xAAAAAAAA)) {
errct++;
if (errct == 1) {
initial_err = p * 4;
}
}
}
if (errct) {
ESP_EARLY_LOGE(TAG, "SPI SRAM memory test fail. %d/%d writes failed, first @ %X\n", errct, size/32, initial_err + v_start);
return false;
} else {
ESP_EARLY_LOGI(TAG, "SPI SRAM memory test OK");
return true;
}
}
bool esp_psram_extram_test(void)
{
bool test_success = false;
#if CONFIG_SPIRAM_ALLOW_NOINIT_SEG_EXTERNAL_MEMORY
intptr_t noinit_vstart = (intptr_t)&_ext_ram_noinit_start;
intptr_t noinit_vend = (intptr_t)&_ext_ram_noinit_end;
#else
intptr_t noinit_vstart = 0;
intptr_t noinit_vend = 0;
#endif
test_success = s_test_psram(s_psram_ctx.mapped_regions[PSRAM_MEM_8BIT_ALIGNED].vaddr_start,
s_psram_ctx.mapped_regions[PSRAM_MEM_8BIT_ALIGNED].size,
noinit_vstart,
noinit_vend);
if (!test_success) {
return false;
}
if (s_psram_ctx.mapped_regions[PSRAM_MEM_32BIT_ALIGNED].size) {
test_success = s_test_psram(s_psram_ctx.mapped_regions[PSRAM_MEM_32BIT_ALIGNED].vaddr_start,
s_psram_ctx.mapped_regions[PSRAM_MEM_32BIT_ALIGNED].size,
0,
0);
}
if (!test_success) {
return false;
}
return true;
}