diff --git a/components/nvs_flash/README.rst b/components/nvs_flash/README.rst index 7225ba52b4..2352236f8f 100644 --- a/components/nvs_flash/README.rst +++ b/components/nvs_flash/README.rst @@ -147,6 +147,8 @@ Erased (2'b00) A key-value pair in this entry has been discarded. Contents of this entry will not be parsed anymore. +.. _structure_of_entry: + Structure of entry ^^^^^^^^^^^^^^^^^^ diff --git a/components/nvs_flash/nvs_partition_generator/README.rst b/components/nvs_flash/nvs_partition_generator/README.rst index 2dacb79a4a..34314da4a8 100644 --- a/components/nvs_flash/nvs_partition_generator/README.rst +++ b/components/nvs_flash/nvs_partition_generator/README.rst @@ -46,21 +46,58 @@ When a new namespace entry is encountered in the CSV file, each follow-up entrie .. note:: First entry in a CSV file should always be ``namespace`` entry. +Multipage Blob Support +---------------------- + +By default, binary blobs are allowed to span over multiple pages and written in the format mentioned in section :ref:`structure_of_entry`. +If older format is intended to be used, the utility provides an option to disable this feature. + Running the utility ------------------- -You can run the utility using below command:: +You can run the utility in two modes using below command: + - Multipage Blob Support Enabled (v2) + - Multipage Blob Support Disabled (v1) - python nvs_partition_gen.py [-h] input output size + +*Usage*:: + + python nvs_partition_gen.py [--version {v1,v2}] input output Positional arguments: -| Arguments | Description -| --- | --- -| input | Path to CSV file to parse. Will use stdin if omitted (a sample.csv is provided) -| output | Path to output converted binary file. Will use stdout if omitted -| size | Size of NVS Partition in KB. E.g. 12KB ++------------------------+----------------------------------------------------------------------------------------------+ +| Arguments | Description | ++========================+==============================================================================================+ +| input | Path to CSV file to parse. Will use stdin if omitted (sample files are provided) | ++------------------------+----------------------------------------------------------------------------------------------+ +| output | Path to output converted binary file. Will use stdout if omitted | ++------------------------+----------------------------------------------------------------------------------------------+ + +Optional arguments: + ++-------------------------------+---------------------------------------------------------------------------------------+ +| Arguments | Description | ++===============================+=======================================================================================+ +| --version {v1,v2} | Set version. Default: v2 | ++-------------------------------+---------------------------------------------------------------------------------------+ + + +*Multipage Blob Support Enabled Mode:* + +You can run the utility in this mode by setting the version parameter to v2, as shown below. +A sample CSV file is provided with the utility:: + + python nvs_partition_gen.py sample_multipage_blob.csv partition_multipage_blob.bin --version v2 + + +*Multipage Blob Support Disabled Mode:* + +You can run the utility in this mode by setting the version parameter to v1, as shown below. +A sample CSV file is provided with the utility:: + + python nvs_partition_gen.py sample_singlepage_blob.csv partition_single_page.bin --version v1 Caveats diff --git a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py index cd07d7f41c..4446d084fc 100755 --- a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py +++ b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py @@ -34,7 +34,8 @@ from os import path class Page(object): PAGE_PARAMS = { "max_size": 4096, - "max_blob_size": 4000, + "max_old_blob_size": 1984, + "max_new_blob_size": 4000, "max_entries": 126 } @@ -59,10 +60,13 @@ class Page(object): CHUNK_ANY = 0xFF ACTIVE = 0xFFFFFFFE FULL = 0xFFFFFFFC + VERSION1=0xFF + VERSION2=0xFE def __init__(self, page_num, is_rsrv_page=False): self.entry_num = 0 self.bitmap_array = array.array('B') + self.version = Page.VERSION2 self.page_buf = bytearray(b'\xff')*Page.PAGE_PARAMS["max_size"] if not is_rsrv_page: self.bitmap_array = self.create_bitmap_array() @@ -77,6 +81,11 @@ class Page(object): page_header[0:4] = struct.pack(' Page.PAGE_PARAMS["max_blob_size"]: + if version == Page.VERSION1: + if datalen > Page.PAGE_PARAMS["max_old_blob_size"]: raise InputError("%s: Size exceeds max allowed length." % key) + if version == Page.VERSION2: + if encoding == "string": + if datalen > Page.PAGE_PARAMS["max_new_blob_size"]: + raise InputError("%s: Size exceeds max allowed length." % key) + # Calculate no. of entries data will require rounded_size = (datalen + 31) & ~31 data_entry_count = rounded_size // 32 @@ -239,11 +265,14 @@ class Page(object): # Set Namespace Index entry_struct[0] = ns_index # Set Span - if encoding == "string": + if version == Page.VERSION2: + if encoding == "string": + entry_struct[2] = data_entry_count + 1 + # Set Chunk Index + chunk_index = Page.CHUNK_ANY + entry_struct[3] = chunk_index + else: entry_struct[2] = data_entry_count + 1 - # Set Chunk Index - chunk_index = Page.CHUNK_ANY - entry_struct[3] = chunk_index # set key key_array = bytearray('\x00')*16 @@ -256,22 +285,12 @@ class Page(object): elif encoding in ["hex2bin", "binary", "base64"]: entry_struct[1] = Page.BLOB - if encoding == "binary" or encoding == "hex2bin" or encoding == "base64": - entry_struct = self.write_varlen_binary_data(entry_struct,ns_index,key,data,\ - datalen,total_entry_count,nvs_obj) + if version == Page.VERSION2 and encoding == "binary" or encoding == "hex2bin" or encoding == "base64": + entry_struct = self.write_varlen_binary_data(entry_struct,ns_index,key,data,\ + datalen,total_entry_count, nvs_obj) else: - # compute CRC of data - entry_struct[24:26] = struct.pack(' 0); int status; @@ -2008,9 +2010,9 @@ TEST_CASE("check partition generation utility", "[nvs_part_gen]") } } -TEST_CASE("read data from partition generated via partition generation utility", "[nvs_part_gen]") +TEST_CASE("read data from partition generated via partition generation utility with multipage blob support disabled", "[nvs_part_gen]") { - SpiFlashEmulator emu("../nvs_partition_generator/partition.bin"); + SpiFlashEmulator emu("../nvs_partition_generator/partition_single_page.bin"); nvs_handle handle; TEST_ESP_OK( nvs_flash_init_custom("test", 0, 3) ); TEST_ESP_OK( nvs_open_from_partition("test", "dummyNamespace", NVS_READONLY, &handle)); @@ -2059,7 +2061,7 @@ TEST_CASE("read data from partition generated via partition generation utility", size_t bin_len = sizeof(bin_data); char binfiledata[5200]; ifstream file; - file.open("../nvs_partition_generator/testdata/sample.bin"); + file.open("../nvs_partition_generator/testdata/sample_singlepage_blob.bin"); file.read(binfiledata,5200); TEST_ESP_OK( nvs_get_blob(handle, "binFileKey", bin_data, &bin_len)); CHECK(memcmp(bin_data, binfiledata, bin_len) == 0); @@ -2068,6 +2070,86 @@ TEST_CASE("read data from partition generated via partition generation utility", } +TEST_CASE("check partition generation utility with multipage blob support enabled", "[nvs_part_gen]") +{ + int childpid = fork(); + if (childpid == 0) { + exit(execlp("python", "python", + "../nvs_partition_generator/nvs_partition_gen.py", + "../nvs_partition_generator/sample_multipage_blob.csv", + "../nvs_partition_generator/partition_multipage_blob.bin", + "12KB", + "--version", + "v2",NULL)); + } else { + CHECK(childpid > 0); + int status; + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) != -1); + } +} + +TEST_CASE("read data from partition generated via partition generation utility with multipage blob support enabled", "[nvs_part_gen]") +{ + SpiFlashEmulator emu("../nvs_partition_generator/partition_multipage_blob.bin"); + nvs_handle handle; + TEST_ESP_OK( nvs_flash_init_custom("test", 0, 3) ); + TEST_ESP_OK( nvs_open_from_partition("test", "dummyNamespace", NVS_READONLY, &handle)); + uint8_t u8v; + TEST_ESP_OK( nvs_get_u8(handle, "dummyU8Key", &u8v)); + CHECK(u8v == 127); + int8_t i8v; + TEST_ESP_OK( nvs_get_i8(handle, "dummyI8Key", &i8v)); + CHECK(i8v == -128); + uint16_t u16v; + TEST_ESP_OK( nvs_get_u16(handle, "dummyU16Key", &u16v)); + CHECK(u16v == 32768); + uint32_t u32v; + TEST_ESP_OK( nvs_get_u32(handle, "dummyU32Key", &u32v)); + CHECK(u32v == 4294967295); + int32_t i32v; + TEST_ESP_OK( nvs_get_i32(handle, "dummyI32Key", &i32v)); + CHECK(i32v == -2147483648); + + char buf[64] = {0}; + size_t buflen = 64; + TEST_ESP_OK( nvs_get_str(handle, "dummyStringKey", buf, &buflen)); + CHECK(strncmp(buf, "0A:0B:0C:0D:0E:0F", buflen) == 0); + + uint8_t hexdata[] = {0x01, 0x02, 0x03, 0xab, 0xcd, 0xef}; + buflen = 64; + int j; + TEST_ESP_OK( nvs_get_blob(handle, "dummyHex2BinKey", buf, &buflen)); + CHECK(memcmp(buf, hexdata, buflen) == 0); + + uint8_t base64data[] = {'1', '2', '3', 'a', 'b', 'c'}; + TEST_ESP_OK( nvs_get_blob(handle, "dummyBase64Key", buf, &buflen)); + CHECK(memcmp(buf, base64data, buflen) == 0); + + buflen = 64; + uint8_t hexfiledata[] = {0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}; + TEST_ESP_OK( nvs_get_blob(handle, "hexFileKey", buf, &buflen)); + CHECK(memcmp(buf, hexfiledata, buflen) == 0); + + buflen = 64; + uint8_t strfiledata[64] = "abcdefghijklmnopqrstuvwxyz\0"; + TEST_ESP_OK( nvs_get_str(handle, "stringFileKey", buf, &buflen)); + CHECK(memcmp(buf, strfiledata, buflen) == 0); + + char bin_data[5200]; + size_t bin_len = sizeof(bin_data); + char binfiledata[5200]; + ifstream file; + file.open("../nvs_partition_generator/testdata/sample_multipage_blob.bin"); + file.read(binfiledata,5200); + TEST_ESP_OK( nvs_get_blob(handle, "binFileKey", bin_data, &bin_len)); + CHECK(memcmp(bin_data, binfiledata, bin_len) == 0); + + file.close(); + +} + + TEST_CASE("dump all performance data", "[nvs]") { std::cout << "====================" << std::endl << "Dumping benchmarks" << std::endl;