Merge branch 'feature/fatfs_use_dynamic_buffer' into 'master'

feat(storage/fatfs): Use dynamic buffers for fatfs

Closes IDFGH-9561

See merge request espressif/esp-idf!27501

components/fatfs/Kconfig

@@ -253,4 +253,15 @@ menu "FAT Filesystem support"
             If enabled, the whole link operation (including file copying) is performed under lock.
             This ensures that the link operation is atomic, but may cause performance for large files.
             It may create less fragmented file copy.
+    config FATFS_USE_DYN_BUFFERS
+        bool "Use dynamic buffers"
+        depends on CONFIG_WL_SECTOR_SIZE_4096
+        default y
+        help
+            If enabled, the buffers used by FATFS will be allocated separately from the rest of the structure.
+            This option is useful when using multiple FATFS instances with different
+            sector sizes, as the buffers will be allocated according to the sector size.
+            If disabled, the greatest sector size will be used for all FATFS instances.
+            (In most cases, this would be the sector size of Wear Levelling library)
+            This might cause more memory to be used than necessary.
 endmenu

components/fatfs/src/ff.c

@@ -1118,7 +1118,7 @@ static FRESULT sync_fs (	/* Returns FR_OK or FR_DISK_ERR */
 	if (res == FR_OK) {
 		if (fs->fs_type == FS_FAT32 && fs->fsi_flag == 1) {	/* FAT32: Update FSInfo sector if needed */
 			/* Create FSInfo structure */
-			memset(fs->win, 0, sizeof fs->win);
+			memset(fs->win, 0, SS(fs));
 			st_word(fs->win + BS_55AA, 0xAA55);					/* Boot signature */
 			st_dword(fs->win + FSI_LeadSig, 0x41615252);		/* Leading signature */
 			st_dword(fs->win + FSI_StrucSig, 0x61417272);		/* Structure signature */
@@ -1670,7 +1670,7 @@ static FRESULT dir_clear (	/* Returns FR_OK or FR_DISK_ERR */
 	if (sync_window(fs) != FR_OK) return FR_DISK_ERR;	/* Flush disk access window */
 	sect = clst2sect(fs, clst);		/* Top of the cluster */
 	fs->winsect = sect;				/* Set window to top of the cluster */
-	memset(fs->win, 0, sizeof fs->win);	/* Clear window buffer */
+	memset(fs->win, 0, SS(fs));	/* Clear window buffer */
 #if FF_USE_LFN == 3		/* Quick table clear by using multi-secter write */
 	/* Allocate a temporary buffer */
 	for (szb = ((DWORD)fs->csize * SS(fs) >= MAX_MALLOC) ? MAX_MALLOC : fs->csize * SS(fs), ibuf = 0; szb > SS(fs) && (ibuf = ff_memalloc(szb)) == 0; szb /= 2) ;
@@ -3438,6 +3438,10 @@ static FRESULT mount_volume (	/* FR_OK(0): successful, !=0: an error occurred */
 	if (disk_ioctl(fs->pdrv, GET_SECTOR_SIZE, &SS(fs)) != RES_OK) return FR_DISK_ERR;
 	if (SS(fs) > FF_MAX_SS || SS(fs) < FF_MIN_SS || (SS(fs) & (SS(fs) - 1))) return FR_DISK_ERR;
 #endif
+#if FF_USE_DYN_BUFFER
+	fs->win = ff_memalloc(SS(fs));		/* Allocate memory for sector buffer */
+	if (!fs->win) return FR_NOT_ENOUGH_CORE;
+#endif
 
 	/* Find an FAT volume on the hosting drive */
 	fmt = find_volume(fs, LD2PT(vol));
@@ -3680,6 +3684,10 @@ FRESULT f_mount (
 #endif
 #if FF_FS_REENTRANT				/* Discard mutex of the current volume */
 		ff_mutex_delete(vol);
+#endif
+#if FF_USE_DYN_BUFFER
+		if (cfs->fs_type)           /* Check if the buffer was ever allocated */
+			ff_memfree(cfs->win);   /* Deallocate buffer allocated for the filesystem object */
 #endif
 		cfs->fs_type = 0;		/* Invalidate the filesystem object to be unregistered */
 	}
@@ -3868,7 +3876,19 @@ FRESULT f_open (
 			fp->fptr = 0;		/* Set file pointer top of the file */
 #if !FF_FS_READONLY
 #if !FF_FS_TINY
-			memset(fp->buf, 0, sizeof fp->buf);	/* Clear sector buffer */
+#if FF_USE_DYN_BUFFER
+			fp->buf = NULL;
+			if (res == FR_OK) {
+				fp->buf = ff_memalloc(SS(fs));
+				if (!fp->buf) {
+					res = FR_NOT_ENOUGH_CORE;	/* Not enough memory */
+					goto fail;
+				}
+				memset(fp->buf, 0, SS(fs));	/* Clear sector buffer */
+			}
+#else
+			memset(fp->buf, 0, SS(fs));    /* Clear sector buffer */
+#endif
 #endif
 			if ((mode & FA_SEEKEND) && fp->obj.objsize > 0) {	/* Seek to end of file if FA_OPEN_APPEND is specified */
 				fp->fptr = fp->obj.objsize;			/* Offset to seek */
@@ -3901,7 +3921,19 @@ FRESULT f_open (
 		FREE_NAMBUF();
 	}
 
-	if (res != FR_OK) fp->obj.fs = 0;	/* Invalidate file object on error */
+	if (res != FR_OK) {
+		fp->obj.fs = 0;	/* Invalidate file object on error */
+#if !FF_FS_TINY && FF_USE_DYN_BUFFER
+		if (fp->buf) {
+			ff_memfree(fp->buf);
+			fp->buf = NULL;
+		}
+#endif
+	}
+
+#if FF_USE_DYN_BUFFER
+fail:
+#endif
 
 	LEAVE_FF(fs, res);
 }
@@ -4235,6 +4267,10 @@ FRESULT f_close (
 #else
 			fp->obj.fs = 0;	/* Invalidate file object */
 #endif
+#if !FF_FS_TINY && FF_USE_DYN_BUFFER
+			ff_memfree(fp->buf);
+			fp->buf = NULL;
+#endif
 #if FF_FS_REENTRANT
 			unlock_volume(fs, FR_OK);		/* Unlock volume */
 #endif

components/fatfs/src/ff.h

@@ -170,7 +170,11 @@ typedef struct {
 	LBA_t	bitbase;		/* Allocation bitmap base sector */
 #endif
 	LBA_t	winsect;		/* Current sector appearing in the win[] */
+#if FF_USE_DYN_BUFFER
+	BYTE*	win;	        /* Disk access window for Directory, FAT (and file data at tiny cfg) */
+#else
 	BYTE	win[FF_MAX_SS];	/* Disk access window for Directory, FAT (and file data at tiny cfg) */
+#endif
 } FATFS;
 
 
@@ -215,8 +219,12 @@ typedef struct {
 	DWORD*	cltbl;			/* Pointer to the cluster link map table (nulled on open, set by application) */
 #endif
 #if !FF_FS_TINY
+#if FF_USE_DYN_BUFFER
+	BYTE*	buf;	        /* File private data read/write window */
+#else
 	BYTE	buf[FF_MAX_SS];	/* File private data read/write window */
 #endif
+#endif
 } FIL;
 
 
@@ -289,7 +297,7 @@ typedef enum {
 	FR_MKFS_ABORTED,		/* (14) The f_mkfs() aborted due to any problem */
 	FR_TIMEOUT,				/* (15) Could not get a grant to access the volume within defined period */
 	FR_LOCKED,				/* (16) The operation is rejected according to the file sharing policy */
-	FR_NOT_ENOUGH_CORE,		/* (17) LFN working buffer could not be allocated */
+	FR_NOT_ENOUGH_CORE,		/* (17) Buffer could not be allocated */
 	FR_TOO_MANY_OPEN_FILES,	/* (18) Number of open files > FF_FS_LOCK */
 	FR_INVALID_PARAMETER	/* (19) Given parameter is invalid */
 } FRESULT;

components/fatfs/src/ffconf.h

@@ -311,6 +311,13 @@
 /  The FF_FS_TIMEOUT defines timeout period in unit of O/S time tick.
 */
 
+#define FF_USE_DYN_BUFFER CONFIG_FATFS_USE_DYN_BUFFERS
+/* The option FF_USE_DYN_BUFFER controls source of size used for buffers in the FS and FIL objects.
+/
+/   0: Disable dynamic buffer size and use static size buffers defined by FF_MAX_SS.
+/   1: Enable dynamic buffer size and use ff_memmalloc() to allocate buffers.
+*/
+
 #include <sys/param.h>
 #include "freertos/FreeRTOS.h"
 #include "freertos/semphr.h"

components/fatfs/test_apps/flash_ro/pytest_fatfs_flash_ro.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: CC0-1.0
 import pytest
 from pytest_embedded import Dut
@@ -8,8 +8,4 @@ from pytest_embedded import Dut
 @pytest.mark.esp32c3
 @pytest.mark.generic
 def test_fatfs_flash_ro(dut: Dut) -> None:
-    dut.expect_exact('Press ENTER to see the list of tests')
-    dut.write('')
-    dut.expect_exact('Enter test for running.')
-    dut.write('*')
-    dut.expect_unity_test_output()
+    dut.run_all_single_board_cases()

components/fatfs/test_apps/flash_wl/pytest_fatfs_flash_wl.py

@@ -13,14 +13,12 @@ from pytest_embedded import Dut
         'default',
         'release',
         'fastseek',
+        'auto_fsync',
+        'no_dyn_buffers',
     ]
 )
 def test_fatfs_flash_wl_generic(dut: Dut) -> None:
-    dut.expect_exact('Press ENTER to see the list of tests')
-    dut.write('')
-    dut.expect_exact('Enter test for running.')
-    dut.write('*')
-    dut.expect_unity_test_output(timeout=180)
+    dut.run_all_single_board_cases(timeout=240)
 
 
 @pytest.mark.esp32
@@ -33,8 +31,4 @@ def test_fatfs_flash_wl_generic(dut: Dut) -> None:
     ]
 )
 def test_fatfs_flash_wl_psram(dut: Dut) -> None:
-    dut.expect_exact('Press ENTER to see the list of tests')
-    dut.write('')
-    dut.expect_exact('Enter test for running.')
-    dut.write('*')
-    dut.expect_unity_test_output(timeout=180)
+    dut.run_all_single_board_cases(timeout=180)

components/fatfs/test_apps/flash_wl/sdkconfig.ci.auto_fsync

@@ -1 +1,2 @@
 CONFIG_FATFS_IMMEDIATE_FSYNC=y
+CONFIG_ESP_MAIN_TASK_STACK_SIZE=4096

components/fatfs/test_apps/flash_wl/sdkconfig.ci.no_dyn_buffers

@@ -0,0 +1 @@
+CONFIG_FATFS_USE_DYN_BUFFERS=n

components/fatfs/test_apps/sdcard/pytest_fatfs_sdcard.py

@@ -15,11 +15,7 @@ from pytest_embedded import Dut
     ]
 )
 def test_fatfs_sdcard_generic_sdmmc(dut: Dut) -> None:
-    dut.expect_exact('Press ENTER to see the list of tests')
-    dut.write('')
-    dut.expect_exact('Enter test for running.')
-    dut.write('[sdmmc]')
-    dut.expect_unity_test_output(timeout=180)
+    dut.run_all_single_board_cases(group='sdmmc', timeout=180)
 
 
 @pytest.mark.esp32
@@ -34,11 +30,7 @@ def test_fatfs_sdcard_generic_sdmmc(dut: Dut) -> None:
     ]
 )
 def test_fatfs_sdcard_generic_sdspi(dut: Dut) -> None:
-    dut.expect_exact('Press ENTER to see the list of tests')
-    dut.write('')
-    dut.expect_exact('Enter test for running.')
-    dut.write('[sdspi]')
-    dut.expect_unity_test_output(timeout=180)
+    dut.run_all_single_board_cases(group='sdspi', timeout=180)
 
 
 @pytest.mark.esp32
@@ -51,11 +43,7 @@ def test_fatfs_sdcard_generic_sdspi(dut: Dut) -> None:
     ]
 )
 def test_fatfs_sdcard_psram_sdmmc(dut: Dut) -> None:
-    dut.expect_exact('Press ENTER to see the list of tests')
-    dut.write('')
-    dut.expect_exact('Enter test for running.')
-    dut.write('[sdmmc]')
-    dut.expect_unity_test_output(timeout=180)
+    dut.run_all_single_board_cases(group='sdmmc', timeout=180)
 
 
 @pytest.mark.esp32
@@ -69,8 +57,4 @@ def test_fatfs_sdcard_psram_sdmmc(dut: Dut) -> None:
     ]
 )
 def test_fatfs_sdcard_psram_sdspi(dut: Dut) -> None:
-    dut.expect_exact('Press ENTER to see the list of tests')
-    dut.write('')
-    dut.expect_exact('Enter test for running.')
-    dut.write('[sdspi]')
-    dut.expect_unity_test_output(timeout=180)
+    dut.run_all_single_board_cases(group='sdspi', timeout=180)