Merge branch 'feature/storage_host_test_nvs_flash2' into 'master'

nvs_flash: finished migration of host tests to the linux emulator of esp_partition

See merge request espressif/esp-idf!23926
This commit is contained in:
Martin Vychodil 2023-08-28 20:20:16 +08:00
commit e44d4260ad
10 changed files with 1600 additions and 1398 deletions

View File

@ -9,10 +9,11 @@ if(${target} STREQUAL "linux")
list(APPEND srcs "partition_linux.c")
set(priv_reqs partition_table)
# Steal some include directories from bootloader_support and hal components:
# Steal some include directories from bootloader_support hal and spi_flash components:
idf_component_get_property(hal_dir hal COMPONENT_DIR)
idf_component_get_property(bootloader_support_dir bootloader_support COMPONENT_DIR)
list(APPEND include_dirs include ${hal_dir}/include ${bootloader_support_dir}/include)
idf_component_get_property(spi_flash_dir spi_flash COMPONENT_DIR)
list(APPEND include_dirs include ${hal_dir}/include ${bootloader_support_dir}/include ${spi_flash_dir}/include)
else()
list(APPEND srcs "partition_target.c")
endif()

View File

@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
* SPDX-FileCopyrightText: 2021-2023 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*
@ -18,6 +18,7 @@
#include "unity.h"
#include "unity_fixture.h"
#include "esp_log.h"
#include "spi_flash_mmap.h"
const char *TAG = "partition_api_test";
@ -694,16 +695,16 @@ TEST(partition_api, test_partition_power_off_emulation)
// --- power-off on, write ---
// ensure power-off emulation is on, below the limit for size
// esp_partition_write consumes one power off failure cycle per 4 bytes written
esp_partition_fail_after(size / 4 - 1, ESP_PARTITION_FAIL_AFTER_MODE_BOTH);
esp_partition_fail_after(size / 4, ESP_PARTITION_FAIL_AFTER_MODE_BOTH);
// write data - should fail
err = esp_partition_write(partition_data, offset, test_data_ptr, size);
TEST_ASSERT_EQUAL(ESP_FAIL, err);
TEST_ASSERT_EQUAL(ESP_ERR_FLASH_OP_FAIL, err);
// --- power-off on, erase has just enough power off failure cycles available---
// ensure power-off emulation is on, at the limit for size
// esp_partition_erase_range consumes one power-off emulation cycle per one virtual sector erased
esp_partition_fail_after(size / ESP_PARTITION_EMULATED_SECTOR_SIZE, ESP_PARTITION_FAIL_AFTER_MODE_BOTH);
esp_partition_fail_after(size / ESP_PARTITION_EMULATED_SECTOR_SIZE + 1, ESP_PARTITION_FAIL_AFTER_MODE_BOTH);
// write data - should be ok
err = esp_partition_erase_range(partition_data, offset, size);
@ -712,11 +713,11 @@ TEST(partition_api, test_partition_power_off_emulation)
// --- power-off on, erase has one cycle less than required---
// ensure power-off emulation is on, below the limit for size
// esp_partition_erase_range consumes one power-off emulation cycle per one virtual sector erased
esp_partition_fail_after(size / ESP_PARTITION_EMULATED_SECTOR_SIZE - 1, ESP_PARTITION_FAIL_AFTER_MODE_BOTH);
esp_partition_fail_after(size / ESP_PARTITION_EMULATED_SECTOR_SIZE, ESP_PARTITION_FAIL_AFTER_MODE_BOTH);
// write data - should fail
err = esp_partition_erase_range(partition_data, offset, size);
TEST_ASSERT_EQUAL(ESP_FAIL, err);
TEST_ASSERT_EQUAL(ESP_ERR_FLASH_OP_FAIL, err);
// ---cleanup ---
// disable power-off emulation

View File

@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
* SPDX-FileCopyrightText: 2021-2023 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
@ -22,6 +22,7 @@
#include "esp_flash_partitions.h"
#include "esp_private/partition_linux.h"
#include "esp_log.h"
#include "spi_flash_mmap.h"
static const char *TAG = "linux_spiflash";
@ -50,8 +51,8 @@ static size_t *s_esp_partition_stat_sector_erase_count = NULL;
// forward declaration of hooks
static void esp_partition_hook_read(const void *srcAddr, const size_t size);
static bool esp_partition_hook_write(const void *dstAddr, const size_t size);
static bool esp_partition_hook_erase(const void *dstAddr, const size_t size);
static bool esp_partition_hook_write(const void *dstAddr, size_t *size);
static bool esp_partition_hook_erase(const void *dstAddr, size_t *size);
// redirect hooks to functions
#define ESP_PARTITION_HOOK_READ(srcAddr, size) esp_partition_hook_read(srcAddr, size)
@ -378,29 +379,35 @@ esp_err_t esp_partition_write(const esp_partition_t *partition, size_t dst_offse
return ESP_ERR_INVALID_SIZE;
}
uint8_t *write_buf = malloc(size);
if (write_buf == NULL) {
return ESP_ERR_NO_MEM;
}
void *dst_addr = s_spiflash_mem_file_buf + partition->address + dst_offset;
ESP_LOGV(TAG, "esp_partition_write(): partition=%s dst_offset=%zu src=%p size=%zu (real dst address: %p)", partition->label, dst_offset, src, size, dst_addr);
// local size, can be modified by the write hook in case of simulated power-off
size_t new_size = size;
esp_err_t ret = ESP_OK;
// hook gathers statistics and can emulate power-off
if (!ESP_PARTITION_HOOK_WRITE(dst_addr, size)) {
free(write_buf);
return ESP_FAIL;
// in case of power - off it decreases new_size to the number of bytes written
// before power event occured
if (!ESP_PARTITION_HOOK_WRITE(dst_addr, &new_size)) {
ret = ESP_ERR_FLASH_OP_FAIL;
}
//read the contents first, AND with the write buffer (to emulate real NOR FLASH behavior)
memcpy(write_buf, dst_addr, size);
for (size_t x = 0; x < size; x++) {
write_buf[x] &= ((uint8_t *)src)[x];
}
memcpy(dst_addr, write_buf, size);
free(write_buf);
for (size_t x = 0; x < new_size; x++) {
return ESP_OK;
// Check if address to be written was erased first
if((~((uint8_t *)dst_addr)[x] & ((uint8_t *)src)[x]) != 0) {
ESP_LOGW(TAG, "invalid flash operation detected");
ret = ESP_ERR_FLASH_OP_FAIL;
break;
}
// AND with destination byte (to emulate real NOR FLASH behavior)
((uint8_t *)dst_addr)[x] &= ((uint8_t *)src)[x];
}
return ret;
}
esp_err_t esp_partition_read(const esp_partition_t *partition, size_t src_offset, void *dst, size_t size)
@ -453,15 +460,20 @@ esp_err_t esp_partition_erase_range(const esp_partition_t *partition, size_t off
void *target_addr = s_spiflash_mem_file_buf + partition->address + offset;
ESP_LOGV(TAG, "esp_partition_erase_range(): partition=%s offset=%zu size=%zu (real target address: %p)", partition->label, offset, size, target_addr);
// local size to be potentially updated by the hook in case of power-off event
size_t new_size = size;
// hook gathers statistics and can emulate power-off
if (!ESP_PARTITION_HOOK_ERASE(target_addr, size)) {
return ESP_FAIL;
esp_err_t ret = ESP_OK;
if(!ESP_PARTITION_HOOK_ERASE(target_addr, &new_size)) {
ret = ESP_ERR_FLASH_OP_FAIL;
}
//set all bits to 1 (NOR FLASH default)
memset(target_addr, 0xFF, size);
memset(target_addr, 0xFF, new_size);
return ESP_OK;
return ret;
}
/*
@ -562,89 +574,89 @@ static void esp_partition_hook_read(const void *srcAddr, const size_t size)
}
// Registers write access statistics of emulated SPI FLASH device (Linux host)
// If enabled by the esp_partition_fail_after, function emulates power-off event during write/erase operations by
// If enabled by the esp_partition_fail_after, function emulates power-off event during write operations by
// decrementing the s_esp_partition_emulated_power_off_counter for each 4 bytes written
// If zero threshold is reached, false is returned.
// If zero threshold is reached, false is returned. In this case the size parameter contains number of successfully written bytes
// Else the function increases nmuber of write operations, accumulates number
// of bytes written and accumulates emulated write operation time (size dependent) and returns true.
static bool esp_partition_hook_write(const void *dstAddr, const size_t size)
static bool esp_partition_hook_write(const void *dstAddr, size_t *size)
{
ESP_LOGV(TAG, "%s", __FUNCTION__);
// power-off emulation
for (size_t i = 0; i < size / 4; ++i) {
if (s_esp_partition_emulated_power_off_counter != SIZE_MAX && s_esp_partition_emulated_power_off_counter-- == 0) {
return false;
}
}
bool ret_val = true;
// one power down cycle per 4 bytes written
size_t write_cycles = size / 4;
size_t write_cycles = *size / 4;
// check whether power off simulation is active for write
if (s_esp_partition_emulated_power_off_counter != SIZE_MAX &&
s_esp_partition_emulated_power_off_counter & ESP_PARTITION_FAIL_AFTER_MODE_WRITE) {
ESP_PARTITION_FAIL_AFTER_MODE_WRITE) {
// check if power down happens during this call
if (s_esp_partition_emulated_power_off_counter >= write_cycles) {
if (s_esp_partition_emulated_power_off_counter > write_cycles) {
// OK
s_esp_partition_emulated_power_off_counter -= write_cycles;
} else {
// failure in this call - reduce cycle count to the number of remainint power on cycles
write_cycles = s_esp_partition_emulated_power_off_counter;
// clear remaining cycles
s_esp_partition_emulated_power_off_counter = 0;
// failure in this call
// update number of bytes written to the in/out parameter
*size = s_esp_partition_emulated_power_off_counter * 4;
// disable power on cycles for further calls
s_esp_partition_emulated_power_off_counter = SIZE_MAX;
// final result value will be false
ret_val = false;
}
}
// stats
++s_esp_partition_stat_write_ops;
s_esp_partition_stat_write_bytes += write_cycles * 4;
s_esp_partition_stat_total_time += esp_partition_stat_time_interpolate((uint32_t) (write_cycles * 4), s_esp_partition_stat_write_times);
if(ret_val) {
// stats
++s_esp_partition_stat_write_ops;
s_esp_partition_stat_write_bytes += write_cycles * 4;
s_esp_partition_stat_total_time += esp_partition_stat_time_interpolate((uint32_t) (*size), s_esp_partition_stat_write_times);
}
return ret_val;
}
// Registers erase access statistics of emulated SPI FLASH device (Linux host)
// If enabled by 'esp_partition_fail_after' parameter, the function emulates a power-off event during write/erase
// operations by decrementing the s_esp_partition_emulated_power_off_counterpower for each erased virtual sector.
// If zero threshold is reached, false is returned.
// If enabled by 'esp_partition_fail_after' parameter, the function emulates a power-off event during erase
// operation by decrementing the s_esp_partition_emulated_power_off_counterpower for each erased virtual sector.
// If zero threshold is reached, false is returned. In out parameter size is updated with number of bytes erased until power-off
// Else, for statistics purpose, the impacted virtual sectors are identified based on
// ESP_PARTITION_EMULATED_SECTOR_SIZE and their respective counts of erase operations are incremented
// Total number of erase operations is increased by the number of impacted virtual sectors
static bool esp_partition_hook_erase(const void *dstAddr, const size_t size)
static bool esp_partition_hook_erase(const void *dstAddr, size_t *size)
{
ESP_LOGV(TAG, "%s", __FUNCTION__);
if (size == 0) {
if (*size == 0) {
return true;
}
// cycle over virtual sectors
ptrdiff_t offset = dstAddr - s_spiflash_mem_file_buf;
size_t first_sector_idx = offset / ESP_PARTITION_EMULATED_SECTOR_SIZE;
size_t last_sector_idx = (offset + size - 1) / ESP_PARTITION_EMULATED_SECTOR_SIZE;
size_t last_sector_idx = (offset + *size - 1) / ESP_PARTITION_EMULATED_SECTOR_SIZE;
size_t sector_count = 1 + last_sector_idx - first_sector_idx;
bool ret_val = true;
// check whether power off simulation is active for erase
if (s_esp_partition_emulated_power_off_counter != SIZE_MAX &&
s_esp_partition_emulated_power_off_counter & ESP_PARTITION_FAIL_AFTER_MODE_ERASE) {
ESP_PARTITION_FAIL_AFTER_MODE_ERASE) {
// check if power down happens during this call
if (s_esp_partition_emulated_power_off_counter >= sector_count) {
if (s_esp_partition_emulated_power_off_counter > sector_count) {
// OK
s_esp_partition_emulated_power_off_counter -= sector_count;
} else {
// failure in this call - reduce sector_count to the number of remainint power on cycles
// failure in this call - reduce sector_count to the number of remaining power on cycles
sector_count = s_esp_partition_emulated_power_off_counter;
// clear remaining cycles
s_esp_partition_emulated_power_off_counter = 0;
// disable power on cycles for further calls
s_esp_partition_emulated_power_off_counter = SIZE_MAX;
// update number of bytes to be really erased before power-off event
*size = sector_count * ESP_PARTITION_EMULATED_SECTOR_SIZE;
// final result value will be false
ret_val = false;
}

View File

@ -1,11 +1,14 @@
/*
* SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD
* SPDX-FileCopyrightText: 2015-2023 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
#include "nvs_partition.hpp"
#include "esp_private/partition_linux.h"
#include "nvs.h"
#include <random>
#include <fcntl.h>
#include <unistd.h>
class PartitionEmulationFixture {
public:
@ -16,20 +19,132 @@ public:
{
if (esp_partition_file_mmap((const uint8_t **) &p_part_desc_addr_start) != ESP_OK) {
throw ("Failed to initialize esp_partition_file_mmap");
FAIL("Failed to initialize esp_partition_file_mmap");
}
esp_partition.address = start_sector * SPI_FLASH_SEC_SIZE;
esp_partition.size = sector_size * SPI_FLASH_SEC_SIZE;
esp_partition.size = (start_sector + sector_size) * SPI_FLASH_SEC_SIZE;
esp_partition.erase_size = ESP_PARTITION_EMULATED_SECTOR_SIZE;
esp_partition.type = ESP_PARTITION_TYPE_DATA;
esp_partition.subtype = ESP_PARTITION_SUBTYPE_DATA_NVS;
strncpy(esp_partition.label, partition_name, PART_NAME_MAX_SIZE);
p_part = new (std::nothrow) nvs::NVSPartition(&esp_partition);
CHECK(p_part != nullptr);
REQUIRE(p_part != nullptr);
}
// initializes the partition and loads partition binary file into it
PartitionEmulationFixture( uint32_t start_sector,
uint32_t sector_size,
const char *partition_name,
const char *partition_binary) : PartitionEmulationFixture(start_sector, sector_size, partition_name)
{
int file_fd = -1;
off_t size = -1;
void *p_buff = nullptr;
char const *fail_msg = nullptr;
do {
// get file size
file_fd = open(partition_binary, O_RDONLY);
if (file_fd == -1) {
fail_msg = "Failed to open file with partition content";
break;
}
size = lseek(file_fd, 0L, SEEK_END);
if (size < 0) {
fail_msg = "falied to seek in file with partition content";
break;
}
// check if file fits into the partitiion
if (size > sector_size * SPI_FLASH_SEC_SIZE) {
fail_msg = "file with partition content doesn't fit into the partition";
break;
}
// allocate local buffer
p_buff = malloc((size_t) size);
if (p_buff == nullptr) {
fail_msg = "unable to allocate buffer for reading file with partition content";
break;
}
// laoad file into local buffer
int res = lseek(file_fd, 0L, SEEK_SET);
if (res < 0) {
fail_msg = "falied to seek in file with partition content";
break;
}
size = read(file_fd, p_buff, size);
if (size < 0) {
fail_msg = "cannot read file with partition content";
break;
}
// erase whole partition
if (ESP_OK != esp_partition_erase_range(&esp_partition, 0, sector_size * SPI_FLASH_SEC_SIZE)) {
fail_msg = "cannot erase partition prior to write partition binary from file";
break;
}
// write local buffer to the partition
if (ESP_OK != esp_partition_write_raw(&esp_partition, 0, p_buff, size)) {
fail_msg = "cannot write to the partition";
break;
}
} while (false);
// close file
if (file_fd >= 0) {
close(file_fd);
}
// deallocate buffer
if (p_buff != nullptr) {
free(p_buff);
}
if(fail_msg != nullptr) {
FAIL(fail_msg);
}
}
void randomize(uint32_t seed)
{
std::random_device rd;
std::mt19937 gen(rd());
gen.seed(seed);
esp_partition_file_mmap_ctrl_t *p_ctrl = esp_partition_get_file_mmap_ctrl_act();
REQUIRE(p_ctrl != nullptr);
std::generate_n(p_part_desc_addr_start, p_ctrl->flash_file_size, gen);
}
// absolute sectorNumber is used here
bool erase(size_t sectorNumber)
{
size_t offset = sectorNumber * SPI_FLASH_SEC_SIZE;
// check the upper bound
esp_partition_file_mmap_ctrl_t *p_ctrl = esp_partition_get_file_mmap_ctrl_act();
REQUIRE(p_ctrl != nullptr);
if (offset > p_ctrl->flash_file_size) {
return false;
}
// esp_partition_erase_range uses offset relative to the begining of partition
return (esp_partition_erase_range(&esp_partition,
offset - esp_partition.address,
SPI_FLASH_SEC_SIZE) == ESP_OK);
}
~PartitionEmulationFixture()
{
delete p_part;
// ensure underlying mmaped file gets deleted after unmap.
esp_partition_file_mmap_ctrl_t *p_ctrl = esp_partition_get_file_mmap_ctrl_input();
p_ctrl->remove_dump = true;
esp_partition_file_munmap();
}
@ -38,7 +153,49 @@ public:
return p_part;
}
const esp_partition_t *get_esp_partition() const
{
return &esp_partition;
}
nvs::NVSPartition *p_part;
esp_partition_t esp_partition;
uint8_t *p_part_desc_addr_start;
};
// fixture with 2 partitions
class PartitionEmulationFixture2 : public PartitionEmulationFixture {
public:
PartitionEmulationFixture2( uint32_t start_sector1 = 0,
uint32_t sector_size1 = 1,
const char *partition_name1 = "nvs1",
uint32_t start_sector2 = 1,
uint32_t sector_size2 = 1,
const char *partition_name2 = "nvs2"
) :
PartitionEmulationFixture(start_sector1, sector_size1, partition_name1), esp_partition2()
{
// for 2nd partition
esp_partition2.address = start_sector2 * SPI_FLASH_SEC_SIZE;
esp_partition2.size = (start_sector2 + sector_size2) * SPI_FLASH_SEC_SIZE;
esp_partition2.erase_size = ESP_PARTITION_EMULATED_SECTOR_SIZE;
esp_partition2.type = ESP_PARTITION_TYPE_DATA;
esp_partition2.subtype = ESP_PARTITION_SUBTYPE_DATA_NVS;
strncpy(esp_partition2.label, partition_name2, PART_NAME_MAX_SIZE);
p_part2 = new (std::nothrow) nvs::NVSPartition(&esp_partition2);
REQUIRE(p_part2 != nullptr);
}
~PartitionEmulationFixture2()
{
delete p_part2;
}
nvs::NVSPartition *part2()
{
return p_part2;
}
nvs::NVSPartition *p_part2;
esp_partition_t esp_partition2;
};

File diff suppressed because it is too large Load Diff

View File

@ -68,3 +68,40 @@ TEST_CASE("Partition manager invalidates handle on partition de-init", "[partiti
delete handle;
}
TEST_CASE("Partition manager initializes multiple partitions", "[partition_mgr]")
{
const uint32_t NVS_FLASH_SECTOR_BEGIN1 = 0;
const uint32_t NVS_FLASH_SECTOR_SIZE1 = 3;
const char* NVS_FLASH_PARTITION1 = "test1";
const uint32_t NVS_FLASH_SECTOR_BEGIN2 = 3;
const uint32_t NVS_FLASH_SECTOR_SIZE2 = 3;
const char* NVS_FLASH_PARTITION2 = "test2";
PartitionEmulationFixture2 f(NVS_FLASH_SECTOR_BEGIN1,
NVS_FLASH_SECTOR_SIZE1,
NVS_FLASH_PARTITION1,
NVS_FLASH_SECTOR_BEGIN2,
NVS_FLASH_SECTOR_SIZE2,
NVS_FLASH_PARTITION2
);
REQUIRE(nvs::NVSPartitionManager::get_instance()->init_custom(f.part(),
NVS_FLASH_SECTOR_BEGIN1,
NVS_FLASH_SECTOR_SIZE1)
== ESP_OK);
REQUIRE(nvs::NVSPartitionManager::get_instance()->init_custom(f.part2(),
NVS_FLASH_SECTOR_BEGIN2,
NVS_FLASH_SECTOR_SIZE2)
== ESP_OK);
nvs::Storage *storage1 = nvs::NVSPartitionManager::get_instance()->lookup_storage_from_name(NVS_FLASH_PARTITION1);
REQUIRE(storage1 != nullptr);
nvs::Storage *storage2 = nvs::NVSPartitionManager::get_instance()->lookup_storage_from_name(NVS_FLASH_PARTITION2);
REQUIRE(storage2 != nullptr);
CHECK(storage1 != storage2);
REQUIRE(nvs::NVSPartitionManager::get_instance()->deinit_partition(NVS_FLASH_PARTITION1) == ESP_OK);
REQUIRE(nvs::NVSPartitionManager::get_instance()->deinit_partition(NVS_FLASH_PARTITION2) == ESP_OK);
}

View File

@ -1,3 +1,8 @@
CONFIG_IDF_TARGET="linux"
CONFIG_COMPILER_CXX_EXCEPTIONS=y
CONFIG_UNITY_ENABLE_IDF_TEST_RUNNER=n
CONFIG_ESP_PARTITION_ENABLE_STATS=y
CONFIG_PARTITION_TABLE_SINGLE_APP=y
CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv"
CONFIG_PARTITION_TABLE_FILENAME="partitions_singleapp.csv"
CONFIG_PARTITION_TABLE_OFFSET=0x8000

File diff suppressed because it is too large Load Diff

View File

@ -13,6 +13,7 @@
#include "nvs_test_api.h"
#include "test_fixtures.hpp"
/*
TEST_CASE("Partition manager initializes multiple partitions", "[partition_mgr]")
{
const uint32_t NVS_FLASH_SECTOR = 6;
@ -35,3 +36,4 @@ TEST_CASE("Partition manager initializes multiple partitions", "[partition_mgr]"
REQUIRE(nvs::NVSPartitionManager::get_instance()->deinit_partition(part_0.get_partition_name()) == ESP_OK);
REQUIRE(nvs::NVSPartitionManager::get_instance()->deinit_partition(part_1.get_partition_name()) == ESP_OK);
}
*/

View File

@ -195,7 +195,8 @@ TEST(spiffs, format_disk_open_file_write_and_read_file)
// Generate data
spiffs_file file = spiffs_res;
uint32_t data_size = 100000;
uint32_t data_count = 5000;
uint32_t data_size = data_count * sizeof(uint32_t);
char *data = (char *) malloc(data_size);
char *read = (char *) malloc(data_size);