Merge branch 'feature/fatfs_use_dynamic_buffer' into 'master'

feat(storage/fatfs): Use dynamic buffers for fatfs

Closes IDFGH-9561

See merge request espressif/esp-idf!27501
This commit is contained in:
Tomas Rohlinek 2024-04-19 13:15:38 +08:00
commit 22d3473f18
9 changed files with 79 additions and 41 deletions

View File

@ -253,4 +253,15 @@ menu "FAT Filesystem support"
If enabled, the whole link operation (including file copying) is performed under lock.
This ensures that the link operation is atomic, but may cause performance for large files.
It may create less fragmented file copy.
config FATFS_USE_DYN_BUFFERS
bool "Use dynamic buffers"
depends on CONFIG_WL_SECTOR_SIZE_4096
default y
help
If enabled, the buffers used by FATFS will be allocated separately from the rest of the structure.
This option is useful when using multiple FATFS instances with different
sector sizes, as the buffers will be allocated according to the sector size.
If disabled, the greatest sector size will be used for all FATFS instances.
(In most cases, this would be the sector size of Wear Levelling library)
This might cause more memory to be used than necessary.
endmenu

View File

@ -1118,7 +1118,7 @@ static FRESULT sync_fs ( /* Returns FR_OK or FR_DISK_ERR */
if (res == FR_OK) {
if (fs->fs_type == FS_FAT32 && fs->fsi_flag == 1) { /* FAT32: Update FSInfo sector if needed */
/* Create FSInfo structure */
memset(fs->win, 0, sizeof fs->win);
memset(fs->win, 0, SS(fs));
st_word(fs->win + BS_55AA, 0xAA55); /* Boot signature */
st_dword(fs->win + FSI_LeadSig, 0x41615252); /* Leading signature */
st_dword(fs->win + FSI_StrucSig, 0x61417272); /* Structure signature */
@ -1670,7 +1670,7 @@ static FRESULT dir_clear ( /* Returns FR_OK or FR_DISK_ERR */
if (sync_window(fs) != FR_OK) return FR_DISK_ERR; /* Flush disk access window */
sect = clst2sect(fs, clst); /* Top of the cluster */
fs->winsect = sect; /* Set window to top of the cluster */
memset(fs->win, 0, sizeof fs->win); /* Clear window buffer */
memset(fs->win, 0, SS(fs)); /* Clear window buffer */
#if FF_USE_LFN == 3 /* Quick table clear by using multi-secter write */
/* Allocate a temporary buffer */
for (szb = ((DWORD)fs->csize * SS(fs) >= MAX_MALLOC) ? MAX_MALLOC : fs->csize * SS(fs), ibuf = 0; szb > SS(fs) && (ibuf = ff_memalloc(szb)) == 0; szb /= 2) ;
@ -3438,6 +3438,10 @@ static FRESULT mount_volume ( /* FR_OK(0): successful, !=0: an error occurred */
if (disk_ioctl(fs->pdrv, GET_SECTOR_SIZE, &SS(fs)) != RES_OK) return FR_DISK_ERR;
if (SS(fs) > FF_MAX_SS || SS(fs) < FF_MIN_SS || (SS(fs) & (SS(fs) - 1))) return FR_DISK_ERR;
#endif
#if FF_USE_DYN_BUFFER
fs->win = ff_memalloc(SS(fs)); /* Allocate memory for sector buffer */
if (!fs->win) return FR_NOT_ENOUGH_CORE;
#endif
/* Find an FAT volume on the hosting drive */
fmt = find_volume(fs, LD2PT(vol));
@ -3680,6 +3684,10 @@ FRESULT f_mount (
#endif
#if FF_FS_REENTRANT /* Discard mutex of the current volume */
ff_mutex_delete(vol);
#endif
#if FF_USE_DYN_BUFFER
if (cfs->fs_type) /* Check if the buffer was ever allocated */
ff_memfree(cfs->win); /* Deallocate buffer allocated for the filesystem object */
#endif
cfs->fs_type = 0; /* Invalidate the filesystem object to be unregistered */
}
@ -3868,7 +3876,19 @@ FRESULT f_open (
fp->fptr = 0; /* Set file pointer top of the file */
#if !FF_FS_READONLY
#if !FF_FS_TINY
memset(fp->buf, 0, sizeof fp->buf); /* Clear sector buffer */
#if FF_USE_DYN_BUFFER
fp->buf = NULL;
if (res == FR_OK) {
fp->buf = ff_memalloc(SS(fs));
if (!fp->buf) {
res = FR_NOT_ENOUGH_CORE; /* Not enough memory */
goto fail;
}
memset(fp->buf, 0, SS(fs)); /* Clear sector buffer */
}
#else
memset(fp->buf, 0, SS(fs)); /* Clear sector buffer */
#endif
#endif
if ((mode & FA_SEEKEND) && fp->obj.objsize > 0) { /* Seek to end of file if FA_OPEN_APPEND is specified */
fp->fptr = fp->obj.objsize; /* Offset to seek */
@ -3901,7 +3921,19 @@ FRESULT f_open (
FREE_NAMBUF();
}
if (res != FR_OK) fp->obj.fs = 0; /* Invalidate file object on error */
if (res != FR_OK) {
fp->obj.fs = 0; /* Invalidate file object on error */
#if !FF_FS_TINY && FF_USE_DYN_BUFFER
if (fp->buf) {
ff_memfree(fp->buf);
fp->buf = NULL;
}
#endif
}
#if FF_USE_DYN_BUFFER
fail:
#endif
LEAVE_FF(fs, res);
}
@ -4235,6 +4267,10 @@ FRESULT f_close (
#else
fp->obj.fs = 0; /* Invalidate file object */
#endif
#if !FF_FS_TINY && FF_USE_DYN_BUFFER
ff_memfree(fp->buf);
fp->buf = NULL;
#endif
#if FF_FS_REENTRANT
unlock_volume(fs, FR_OK); /* Unlock volume */
#endif

View File

@ -170,7 +170,11 @@ typedef struct {
LBA_t bitbase; /* Allocation bitmap base sector */
#endif
LBA_t winsect; /* Current sector appearing in the win[] */
#if FF_USE_DYN_BUFFER
BYTE* win; /* Disk access window for Directory, FAT (and file data at tiny cfg) */
#else
BYTE win[FF_MAX_SS]; /* Disk access window for Directory, FAT (and file data at tiny cfg) */
#endif
} FATFS;
@ -215,8 +219,12 @@ typedef struct {
DWORD* cltbl; /* Pointer to the cluster link map table (nulled on open, set by application) */
#endif
#if !FF_FS_TINY
#if FF_USE_DYN_BUFFER
BYTE* buf; /* File private data read/write window */
#else
BYTE buf[FF_MAX_SS]; /* File private data read/write window */
#endif
#endif
} FIL;
@ -289,7 +297,7 @@ typedef enum {
FR_MKFS_ABORTED, /* (14) The f_mkfs() aborted due to any problem */
FR_TIMEOUT, /* (15) Could not get a grant to access the volume within defined period */
FR_LOCKED, /* (16) The operation is rejected according to the file sharing policy */
FR_NOT_ENOUGH_CORE, /* (17) LFN working buffer could not be allocated */
FR_NOT_ENOUGH_CORE, /* (17) Buffer could not be allocated */
FR_TOO_MANY_OPEN_FILES, /* (18) Number of open files > FF_FS_LOCK */
FR_INVALID_PARAMETER /* (19) Given parameter is invalid */
} FRESULT;

View File

@ -311,6 +311,13 @@
/ The FF_FS_TIMEOUT defines timeout period in unit of O/S time tick.
*/
#define FF_USE_DYN_BUFFER CONFIG_FATFS_USE_DYN_BUFFERS
/* The option FF_USE_DYN_BUFFER controls source of size used for buffers in the FS and FIL objects.
/
/ 0: Disable dynamic buffer size and use static size buffers defined by FF_MAX_SS.
/ 1: Enable dynamic buffer size and use ff_memmalloc() to allocate buffers.
*/
#include <sys/param.h>
#include "freertos/FreeRTOS.h"
#include "freertos/semphr.h"

View File

@ -1,4 +1,4 @@
# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: CC0-1.0
import pytest
from pytest_embedded import Dut
@ -8,8 +8,4 @@ from pytest_embedded import Dut
@pytest.mark.esp32c3
@pytest.mark.generic
def test_fatfs_flash_ro(dut: Dut) -> None:
dut.expect_exact('Press ENTER to see the list of tests')
dut.write('')
dut.expect_exact('Enter test for running.')
dut.write('*')
dut.expect_unity_test_output()
dut.run_all_single_board_cases()

View File

@ -13,14 +13,12 @@ from pytest_embedded import Dut
'default',
'release',
'fastseek',
'auto_fsync',
'no_dyn_buffers',
]
)
def test_fatfs_flash_wl_generic(dut: Dut) -> None:
dut.expect_exact('Press ENTER to see the list of tests')
dut.write('')
dut.expect_exact('Enter test for running.')
dut.write('*')
dut.expect_unity_test_output(timeout=180)
dut.run_all_single_board_cases(timeout=240)
@pytest.mark.esp32
@ -33,8 +31,4 @@ def test_fatfs_flash_wl_generic(dut: Dut) -> None:
]
)
def test_fatfs_flash_wl_psram(dut: Dut) -> None:
dut.expect_exact('Press ENTER to see the list of tests')
dut.write('')
dut.expect_exact('Enter test for running.')
dut.write('*')
dut.expect_unity_test_output(timeout=180)
dut.run_all_single_board_cases(timeout=180)

View File

@ -1 +1,2 @@
CONFIG_FATFS_IMMEDIATE_FSYNC=y
CONFIG_ESP_MAIN_TASK_STACK_SIZE=4096

View File

@ -0,0 +1 @@
CONFIG_FATFS_USE_DYN_BUFFERS=n

View File

@ -15,11 +15,7 @@ from pytest_embedded import Dut
]
)
def test_fatfs_sdcard_generic_sdmmc(dut: Dut) -> None:
dut.expect_exact('Press ENTER to see the list of tests')
dut.write('')
dut.expect_exact('Enter test for running.')
dut.write('[sdmmc]')
dut.expect_unity_test_output(timeout=180)
dut.run_all_single_board_cases(group='sdmmc', timeout=180)
@pytest.mark.esp32
@ -34,11 +30,7 @@ def test_fatfs_sdcard_generic_sdmmc(dut: Dut) -> None:
]
)
def test_fatfs_sdcard_generic_sdspi(dut: Dut) -> None:
dut.expect_exact('Press ENTER to see the list of tests')
dut.write('')
dut.expect_exact('Enter test for running.')
dut.write('[sdspi]')
dut.expect_unity_test_output(timeout=180)
dut.run_all_single_board_cases(group='sdspi', timeout=180)
@pytest.mark.esp32
@ -51,11 +43,7 @@ def test_fatfs_sdcard_generic_sdspi(dut: Dut) -> None:
]
)
def test_fatfs_sdcard_psram_sdmmc(dut: Dut) -> None:
dut.expect_exact('Press ENTER to see the list of tests')
dut.write('')
dut.expect_exact('Enter test for running.')
dut.write('[sdmmc]')
dut.expect_unity_test_output(timeout=180)
dut.run_all_single_board_cases(group='sdmmc', timeout=180)
@pytest.mark.esp32
@ -69,8 +57,4 @@ def test_fatfs_sdcard_psram_sdmmc(dut: Dut) -> None:
]
)
def test_fatfs_sdcard_psram_sdspi(dut: Dut) -> None:
dut.expect_exact('Press ENTER to see the list of tests')
dut.write('')
dut.expect_exact('Enter test for running.')
dut.write('[sdspi]')
dut.expect_unity_test_output(timeout=180)
dut.run_all_single_board_cases(group='sdspi', timeout=180)