From 7c65370d84f8134c410bdca5762541af28fa9906 Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Tue, 22 Mar 2022 17:50:01 +0100 Subject: [PATCH] spiffs: add esp_spiffs_gc function to force garbage collection Closes https://github.com/espressif/esp-idf/issues/8626 --- components/spiffs/Kconfig | 2 +- components/spiffs/esp_spiffs.c | 18 ++++++ components/spiffs/include/esp_spiffs.h | 29 +++++++++ components/spiffs/test/test_spiffs.c | 83 ++++++++++++++++++++++++++ tools/unit-test-app/sdkconfig.defaults | 4 ++ 5 files changed, 135 insertions(+), 1 deletion(-) diff --git a/components/spiffs/Kconfig b/components/spiffs/Kconfig index 0f20920d97..785f3acb3e 100644 --- a/components/spiffs/Kconfig +++ b/components/spiffs/Kconfig @@ -42,7 +42,7 @@ menu "SPIFFS Configuration" config SPIFFS_GC_MAX_RUNS int "Set Maximum GC Runs" default 10 - range 1 255 + range 1 10000 help Define maximum number of GC runs to perform to reach desired free pages. diff --git a/components/spiffs/esp_spiffs.c b/components/spiffs/esp_spiffs.c index bf7bc136f2..0709efc356 100644 --- a/components/spiffs/esp_spiffs.c +++ b/components/spiffs/esp_spiffs.c @@ -378,6 +378,24 @@ esp_err_t esp_spiffs_format(const char* partition_label) return ESP_OK; } +esp_err_t esp_spiffs_gc(const char* partition_label, size_t size_to_gc) +{ + int index; + if (esp_spiffs_by_label(partition_label, &index) != ESP_OK) { + return ESP_ERR_INVALID_STATE; + } + int res = SPIFFS_gc(_efs[index]->fs, size_to_gc); + if (res != SPIFFS_OK) { + ESP_LOGE(TAG, "SPIFFS_gc failed, %d", res); + SPIFFS_clearerr(_efs[index]->fs); + if (res == SPIFFS_ERR_FULL) { + return ESP_ERR_NOT_FINISHED; + } + return ESP_FAIL; + } + return ESP_OK; +} + esp_err_t esp_vfs_spiffs_register(const esp_vfs_spiffs_conf_t * conf) { assert(conf->base_path); diff --git a/components/spiffs/include/esp_spiffs.h b/components/spiffs/include/esp_spiffs.h index 5dbb4a2e51..1b59b9cfde 100644 --- a/components/spiffs/include/esp_spiffs.h +++ b/components/spiffs/include/esp_spiffs.h @@ -95,6 +95,35 @@ esp_err_t esp_spiffs_info(const char* partition_label, size_t *total_bytes, size */ esp_err_t esp_spiffs_check(const char* partition_label); + +/** + * @brief Perform garbage collection in SPIFFS partition + * + * Call this function to run GC and ensure that at least the given amount of + * space is available in the partition. This function will fail with ESP_ERR_NOT_FINISHED + * if it is not possible to reclaim the requested space (that is, not enough free + * or deleted pages in the filesystem). This function will also fail if it fails to + * reclaim the requested space after CONFIG_SPIFFS_GC_MAX_RUNS number of GC iterations. + * On one GC iteration, SPIFFS will erase one logical block (4kB). Therefore the value + * of CONFIG_SPIFFS_GC_MAX_RUNS should be set at least to the maximum expected size_to_gc, + * divided by 4096. For example, if the application expects to make room for a 1MB file and + * calls esp_spiffs_gc(label, 1024 * 1024), CONFIG_SPIFFS_GC_MAX_RUNS should be set to + * at least 256. + * On the other hand, increasing CONFIG_SPIFFS_GC_MAX_RUNS value increases the maximum + * amount of time for which any SPIFFS GC or write operation may potentially block. + * + * @param partition_label Label of the partition to be garbage-collected. + * The partition must be already mounted. + * @param size_to_gc The number of bytes that the GC process should attempt + * to make available. + * @return + * - ESP_OK on success + * - ESP_ERR_NOT_FINISHED if GC fails to reclaim the size given by size_to_gc + * - ESP_ERR_INVALID_STATE if the partition is not mounted + * - ESP_FAIL on all other errors + */ +esp_err_t esp_spiffs_gc(const char* partition_label, size_t size_to_gc); + #ifdef __cplusplus } #endif diff --git a/components/spiffs/test/test_spiffs.c b/components/spiffs/test/test_spiffs.c index cf800f18f8..7340e4f8cd 100644 --- a/components/spiffs/test/test_spiffs.c +++ b/components/spiffs/test/test_spiffs.c @@ -22,6 +22,7 @@ #include "freertos/queue.h" #include "freertos/semphr.h" #include "esp_partition.h" +#include "esp_random.h" #include "esp_rom_sys.h" const char* spiffs_test_hello_str = "Hello, World!\n"; @@ -828,3 +829,85 @@ TEST_CASE("utime() works well", "[spiffs]") test_teardown(); } #endif // CONFIG_SPIFFS_USE_MTIME + +static void test_spiffs_rw_speed(const char* filename, void* buf, size_t buf_size, size_t file_size, bool is_write) +{ + const size_t buf_count = file_size / buf_size; + + FILE* f = fopen(filename, (is_write) ? "wb" : "rb"); + TEST_ASSERT_NOT_NULL(f); + + struct timeval tv_start; + gettimeofday(&tv_start, NULL); + for (size_t n = 0; n < buf_count; ++n) { + if (is_write) { + TEST_ASSERT_EQUAL(buf_size, write(fileno(f), buf, buf_size)); + } else { + if (read(fileno(f), buf, buf_size) != buf_size) { + printf("reading at n=%d, eof=%d", n, feof(f)); + TEST_FAIL(); + } + } + } + + struct timeval tv_end; + gettimeofday(&tv_end, NULL); + + TEST_ASSERT_EQUAL(0, fclose(f)); + + float t_s = tv_end.tv_sec - tv_start.tv_sec + 1e-6f * (tv_end.tv_usec - tv_start.tv_usec); + printf("%s %d bytes (block size %d) in %.3fms (%.3f MB/s)\n", + (is_write)?"Wrote":"Read", file_size, buf_size, t_s * 1e3, + file_size / (1024.0f * 1024.0f * t_s)); +} + +TEST_CASE("write/read speed test", "[spiffs][timeout=60]") +{ + /* Erase partition before running the test to get consistent results */ + const esp_partition_t* part = get_test_data_partition(); + esp_partition_erase_range(part, 0, part->size); + + test_setup(); + + const size_t buf_size = 16 * 1024; + uint32_t* buf = (uint32_t*) calloc(1, buf_size); + esp_fill_random(buf, buf_size); + const size_t file_size = 256 * 1024; + const char* file = "/spiffs/256k.bin"; + + test_spiffs_rw_speed(file, buf, 4 * 1024, file_size, true); + TEST_ASSERT_EQUAL(0, unlink(file)); + TEST_ESP_OK(esp_spiffs_gc(spiffs_test_partition_label, file_size)); + + test_spiffs_rw_speed(file, buf, 8 * 1024, file_size, true); + TEST_ASSERT_EQUAL(0, unlink(file)); + TEST_ESP_OK(esp_spiffs_gc(spiffs_test_partition_label, file_size)); + + test_spiffs_rw_speed(file, buf, 16 * 1024, file_size, true); + + test_spiffs_rw_speed(file, buf, 4 * 1024, file_size, false); + test_spiffs_rw_speed(file, buf, 8 * 1024, file_size, false); + test_spiffs_rw_speed(file, buf, 16 * 1024, file_size, false); + TEST_ASSERT_EQUAL(0, unlink(file)); + TEST_ESP_OK(esp_spiffs_gc(spiffs_test_partition_label, file_size)); + + free(buf); + test_teardown(); +} + +TEST_CASE("SPIFFS garbage-collect", "[spiffs][timeout=60]") +{ + // should fail until the partition is initialized + TEST_ESP_ERR(ESP_ERR_INVALID_STATE, esp_spiffs_gc(spiffs_test_partition_label, 4096)); + + test_setup(); + + // reclaiming one block should be possible + TEST_ESP_OK(esp_spiffs_gc(spiffs_test_partition_label, 4096)); + + // shouldn't be possible to reclaim more than the partition size + const esp_partition_t* part = get_test_data_partition(); + TEST_ESP_ERR(ESP_ERR_NOT_FINISHED, esp_spiffs_gc(spiffs_test_partition_label, part->size * 2)); + + test_teardown(); +} diff --git a/tools/unit-test-app/sdkconfig.defaults b/tools/unit-test-app/sdkconfig.defaults index c47a5da82c..f9a791794d 100644 --- a/tools/unit-test-app/sdkconfig.defaults +++ b/tools/unit-test-app/sdkconfig.defaults @@ -26,3 +26,7 @@ CONFIG_FATFS_ALLOC_PREFER_EXTRAM=y CONFIG_UNITY_ENABLE_BACKTRACE_ON_FAIL=y CONFIG_FREERTOS_TIMER_TASK_STACK_DEPTH=3000 CONFIG_MQTT_TEST_BROKER_URI="mqtt://${EXAMPLE_MQTT_BROKER_TCP}" +# Set sufficient number of GC runs for SPIFFS: +# size of the storage partition divided by flash sector size. +# See esp_spiffs_gc description for more info. +CONFIG_SPIFFS_GC_MAX_RUNS=132