Merge branch 'bugfix/nvs_util_write_single_page_big_blob_data' into 'master'

nvs_util: Fix to support write operation of multiple single page big blob data

See merge request idf/esp-idf!4268
This commit is contained in:
Ivan Grokhotkov 2019-03-21 19:10:29 +08:00
commit ffa8bed7c8
8 changed files with 100 additions and 104 deletions

View File

@ -160,6 +160,5 @@ exclude =
components/wifi_provisioning/python/wifi_constants_pb2.py,
examples/provisioning/custom_config/components/custom_provisioning/python/custom_config_pb2.py,
# temporary list (should be empty)
components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py,
tools/esp_app_trace/pylibelf,
tools/mass_mfg/mfg_gen.py,

View File

@ -158,7 +158,7 @@ You can also provide the format version number (in any of the two modes):
You can run the utility in this format by setting the version parameter to v2, as shown below.
A sample CSV file is provided with the utility::
python nvs_partition_gen.py --input sample_multipage_blob.csv --output partition_multipage_blob.bin --size 0x3000 --version v2
python nvs_partition_gen.py --input sample_multipage_blob.csv --output partition_multipage_blob.bin --size 0x4000 --version v2
**Multipage Blob Support Disabled (v1):**

View File

@ -39,6 +39,8 @@ VERSION2_PRINT = "v2 - Multipage Blob Support Enabled"
""" Class for standard NVS page structure """
class Page(object):
PAGE_PARAMS = {
"max_size": 4096,
@ -102,7 +104,6 @@ class Page(object):
struct.pack_into('<I', page_header, 28, crc & 0xFFFFFFFF)
self.page_buf[0:len(page_header)] = page_header
def create_bitmap_array(self):
bitarray = array.array('B')
charsize = 32 # bitmaparray has 256 bits, hence 32 bytes
@ -110,7 +111,6 @@ class Page(object):
bitarray.extend((fill,) * charsize)
return bitarray
def write_bitmaparray(self):
bitnum = self.entry_num * 2
byte_idx = bitnum // 8 # Find byte index in the array
@ -121,7 +121,6 @@ class Page(object):
end_idx = Page.BITMAPARRAY_OFFSET + Page.BITMAPARRAY_SIZE_IN_BYTES
self.page_buf[start_idx:end_idx] = self.bitmap_array
def encrypt_entry(self, data_arr, tweak_arr, encr_key):
# Encrypt 32 bytes of data using AES-XTS encryption
backend = default_backend()
@ -134,7 +133,6 @@ class Page(object):
return encrypted_data
def reverse_hexbytes(self, addr_tmp):
addr = []
reversed_bytes = ""
@ -144,7 +142,6 @@ class Page(object):
return reversed_bytes
def encrypt_data(self, data_input, no_of_entries, nvs_obj):
# Set values needed for encryption and encrypt data byte wise
encr_data_to_write = bytearray()
@ -155,7 +152,6 @@ class Page(object):
tweak_tmp = ''
encr_key_input = None
# Extract encryption key and tweak key from given key input
if len(self.encr_key) == key_len_needed:
encr_key_input = self.encr_key
@ -207,7 +203,6 @@ class Page(object):
return encr_data_to_write
def write_entry_to_buf(self, data, entrycount,nvs_obj):
encr_data = bytearray()
@ -226,7 +221,6 @@ class Page(object):
self.write_bitmaparray()
self.entry_num += 1
def set_crc_header(self, entry_struct):
crc_data = bytearray(b'28')
crc_data[0:4] = entry_struct[0:4]
@ -236,7 +230,6 @@ class Page(object):
struct.pack_into('<I', entry_struct, 4, crc & 0xFFFFFFFF)
return entry_struct
def write_varlen_binary_data(self, entry_struct, ns_index, key, data, data_size, total_entry_count, encoding, nvs_obj):
chunk_start = 0
chunk_count = 0
@ -306,7 +299,6 @@ class Page(object):
offset = offset + chunk_size
# All chunks are stored, now store the index
if not remaining_size:
# Initialise data field to 0xff
@ -336,7 +328,6 @@ class Page(object):
return entry_struct
def write_single_page_entry(self, entry_struct, data, datalen, data_entry_count, nvs_obj):
# compute CRC of data
struct.pack_into('<H', entry_struct, 24, datalen)
@ -355,7 +346,6 @@ class Page(object):
# write actual data
self.write_entry_to_buf(data, data_entry_count, nvs_obj)
"""
Low-level function to write variable length data into page buffer. Data should be formatted
according to encoding specified.
@ -364,13 +354,11 @@ class Page(object):
# Set size of data
datalen = len(data)
if version == Page.VERSION1:
if datalen > Page.PAGE_PARAMS["max_old_blob_size"]:
if version == Page.VERSION1:
raise InputError("Version %s\n%s: Size exceeds max allowed length." % (VERSION1_PRINT,key))
if version == Page.VERSION2:
else:
if encoding == "string":
if datalen > Page.PAGE_PARAMS["max_new_blob_size"]:
raise InputError("Version %s\n%s: Size exceeds max allowed length." % (VERSION2_PRINT,key))
# Calculate no. of entries data will require
@ -379,8 +367,10 @@ class Page(object):
total_entry_count = data_entry_count + 1 # +1 for the entry header
# Check if page is already full and new page is needed to be created right away
if encoding == "string":
if (self.entry_num + total_entry_count) >= Page.PAGE_PARAMS["max_entries"]:
if self.entry_num >= Page.PAGE_PARAMS["max_entries"]:
raise PageFullError()
elif (self.entry_num + total_entry_count) >= Page.PAGE_PARAMS["max_entries"]:
if not (version == Page.VERSION2 and encoding in ["hex2bin", "binary", "base64"]):
raise PageFullError()
# Entry header
@ -409,13 +399,11 @@ class Page(object):
entry_struct[1] = Page.BLOB
if version == Page.VERSION2 and (encoding in ["hex2bin", "binary", "base64"]):
entry_struct = self.write_varlen_binary_data(entry_struct,ns_index,key,data,\
entry_struct = self.write_varlen_binary_data(entry_struct,ns_index,key,data,
datalen,total_entry_count, encoding, nvs_obj)
else:
self.write_single_page_entry(entry_struct, data, datalen, data_entry_count, nvs_obj)
""" Low-level function to write data of primitive type into page buffer. """
def write_primitive_data(self, key, data, encoding, ns_index,nvs_obj):
# Check if entry exceeds max number of entries allowed per page
@ -464,9 +452,13 @@ class Page(object):
def get_data(self):
return self.page_buf
"""
NVS class encapsulates all NVS specific operations to create a binary with given key-value pairs. Binary can later be flashed onto device via a flashing utility.
NVS class encapsulates all NVS specific operations to create a binary with given key-value pairs.
Binary can later be flashed onto device via a flashing utility.
"""
class NVS(object):
def __init__(self, fout, input_size):
self.size = input_size
@ -480,11 +472,11 @@ class NVS(object):
return self
def __exit__(self, exc_type, exc_value, traceback):
if exc_type == None and exc_value == None:
if exc_type is None and exc_value is None:
# Create pages for remaining available size
while True:
try:
new_page = self.create_new_page()
self.create_new_page()
except InsufficientSizeError:
self.size = None
# Creating the last reserved page
@ -572,6 +564,7 @@ class NVS(object):
data += page.get_data()
return data
class PageFullError(RuntimeError):
"""
Represents error when current page doesn't have sufficient entries left
@ -580,6 +573,7 @@ class PageFullError(RuntimeError):
def __init__(self):
super(PageFullError, self).__init__()
class InputError(RuntimeError):
"""
Represents error on the input
@ -587,6 +581,7 @@ class InputError(RuntimeError):
def __init__(self, e):
super(InputError, self).__init__(e)
class InsufficientSizeError(RuntimeError):
"""
Represents error when NVS Partition size given is insufficient
@ -595,6 +590,7 @@ class InsufficientSizeError(RuntimeError):
def __init__(self, e):
super(InsufficientSizeError, self).__init__(e)
def nvs_open(result_obj, input_size):
""" Wrapper to create and NVS class object. This object can later be used to set key-value pairs
@ -604,6 +600,7 @@ def nvs_open(result_obj, input_size):
"""
return NVS(result_obj, input_size)
def write_entry(nvs_instance, key, datatype, encoding, value):
""" Wrapper to set key-value pair in NVS format
@ -617,7 +614,7 @@ def write_entry(nvs_instance, key, datatype, encoding, value):
if datatype == "file":
abs_file_path = value
if os.path.isabs(value) == False:
if os.path.isabs(value) is False:
script_dir = os.path.dirname(__file__)
abs_file_path = os.path.join(script_dir, value)
@ -629,6 +626,7 @@ def write_entry(nvs_instance, key, datatype, encoding, value):
else:
nvs_instance.write_entry(key, value, encoding)
def nvs_close(nvs_instance):
""" Wrapper to finish writing to NVS and write data to file/stream object provided to nvs_open method
@ -638,7 +636,7 @@ def nvs_close(nvs_instance):
nvs_instance.__exit__(None, None, None)
def check_input_args(input_filename=None, output_filename=None, input_part_size=None, is_key_gen=None,\
def check_input_args(input_filename=None, output_filename=None, input_part_size=None, is_key_gen=None,
encrypt_mode=None, key_file=None, version_no=None, print_arg_str=None, print_encrypt_arg_str=None):
global version, is_encrypt_data, input_size, key_gen
@ -653,7 +651,6 @@ encrypt_mode=None, key_file=None, version_no=None, print_arg_str=None, print_enc
elif is_encrypt_data.lower() == 'false':
is_encrypt_data = False
if version == 'v1':
version = Page.VERSION1
elif version == 'v2':
@ -664,7 +661,6 @@ encrypt_mode=None, key_file=None, version_no=None, print_arg_str=None, print_enc
elif key_gen.lower() == 'false':
key_gen = False
if key_gen:
if all(arg is not None for arg in [input_filename, output_filename, input_size]):
if not is_encrypt_data:
@ -676,7 +672,6 @@ encrypt_mode=None, key_file=None, version_no=None, print_arg_str=None, print_enc
if not all(arg is not None for arg in [input_filename, output_filename]):
sys.exit(print_arg_str)
if is_encrypt_data and not key_gen and not key_file:
sys.exit(print_encrypt_arg_str)
@ -700,8 +695,6 @@ encrypt_mode=None, key_file=None, version_no=None, print_arg_str=None, print_enc
sys.exit("Minimum NVS partition size needed is 0x3000 bytes.")
def nvs_part_gen(input_filename=None, output_filename=None, input_part_size=None, is_key_gen=None, encrypt_mode=None, key_file=None, version_no=None):
""" Wrapper to generate nvs partition binary
@ -744,7 +737,6 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_part_size=None
input_file.close()
output_file.close()
if key_gen:
keys_page_buf = bytearray(b'\xff') * Page.PAGE_PARAMS["max_size"]
key_bytes = bytearray()
@ -819,13 +811,13 @@ def main():
is_encrypt_data = args.encrypt
key_file = args.keyfile
print_arg_str = "Invalid.\nTo generate nvs partition binary --input, --output and --size arguments are mandatory.\nTo generate encryption keys --keygen argument is mandatory."
print_arg_str = "Invalid.\nTo generate nvs partition binary --input, --output and --size arguments are mandatory.\n \
To generate encryption keys --keygen argument is mandatory."
print_encrypt_arg_str = "Missing parameter. Enter --keyfile or --keygen."
check_input_args(input_filename,output_filename, part_size, is_key_gen, is_encrypt_data, key_file, version_no, print_arg_str, print_encrypt_arg_str)
nvs_part_gen(input_filename, output_filename, part_size, is_key_gen, is_encrypt_data, key_file, version_no)
if __name__ == "__main__":
main()

View File

@ -11,4 +11,6 @@ dummyBase64Key,data,base64,MTIzYWJj
hexFileKey,file,hex2bin,testdata/sample.hex
base64FileKey,file,base64,testdata/sample.base64
stringFileKey,file,string,testdata/sample.txt
blobFileAKey,file,binary,testdata/sample_blob.bin
blobFileBKey,file,binary,testdata/sample_blob.bin
binFileKey,file,binary,testdata/sample_multipage_blob.bin

1 key type encoding value
11 hexFileKey file hex2bin testdata/sample.hex
12 base64FileKey file base64 testdata/sample.base64
13 stringFileKey file string testdata/sample.txt
14 blobFileAKey file binary testdata/sample_blob.bin
15 blobFileBKey file binary testdata/sample_blob.bin
16 binFileKey file binary testdata/sample_multipage_blob.bin

View File

@ -11,4 +11,6 @@ dummyBase64Key,data,base64,MTIzYWJj
hexFileKey,file,hex2bin,testdata/sample.hex
base64FileKey,file,base64,testdata/sample.base64
stringFileKey,file,string,testdata/sample.txt
blobFileAKey,file,binary,testdata/sample_blob.bin
blobFileBKey,file,binary,testdata/sample_blob.bin
binFileKey,file,binary,testdata/sample_singlepage_blob.bin

1 key type encoding value
11 hexFileKey file hex2bin testdata/sample.hex
12 base64FileKey file base64 testdata/sample.base64
13 stringFileKey file string testdata/sample.txt
14 blobFileAKey file binary testdata/sample_blob.bin
15 blobFileBKey file binary testdata/sample_blob.bin
16 binFileKey file binary testdata/sample_singlepage_blob.bin

View File

@ -0,0 +1 @@
start0000000000000000000000start0123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef0000000000000000end00000000000000000000000000end

View File

@ -2057,14 +2057,14 @@ TEST_CASE("Recovery from power-off during modification of blob present in old-fo
TEST_ESP_ERR(p3.findItem(1, ItemType::BLOB, "singlepage"), ESP_ERR_NVS_NOT_FOUND);
}
static void check_nvs_part_gen_args(char const *part_name, char const *filename, bool is_encr, nvs_sec_cfg_t* xts_cfg)
static void check_nvs_part_gen_args(char const *part_name, int size, char const *filename, bool is_encr, nvs_sec_cfg_t* xts_cfg)
{
nvs_handle handle;
if (is_encr)
TEST_ESP_OK(nvs_flash_secure_init_custom(part_name, 0, 3, xts_cfg));
TEST_ESP_OK(nvs_flash_secure_init_custom(part_name, 0, size, xts_cfg));
else
TEST_ESP_OK( nvs_flash_init_custom(part_name, 0, 3) );
TEST_ESP_OK( nvs_flash_init_custom(part_name, 0, size) );
TEST_ESP_OK( nvs_open_from_partition(part_name, "dummyNamespace", NVS_READONLY, &handle));
uint8_t u8v;
@ -2148,7 +2148,7 @@ TEST_CASE("check and read data from partition generated via partition generation
TEST_ESP_OK(nvs_flash_deinit());
check_nvs_part_gen_args("test", "../nvs_partition_generator/testdata/sample_singlepage_blob.bin", false, NULL);
check_nvs_part_gen_args("test", 3, "../nvs_partition_generator/testdata/sample_singlepage_blob.bin", false, NULL);
}
@ -2163,7 +2163,7 @@ TEST_CASE("check and read data from partition generated via partition generation
"--output",
"../nvs_partition_generator/partition_multipage_blob.bin",
"--size",
"0x3000",
"0x4000",
"--version",
"v2",NULL));
} else {
@ -2175,7 +2175,7 @@ TEST_CASE("check and read data from partition generated via partition generation
SpiFlashEmulator emu("../nvs_partition_generator/partition_multipage_blob.bin");
check_nvs_part_gen_args("test", "../nvs_partition_generator/testdata/sample_multipage_blob.bin",false,NULL);
check_nvs_part_gen_args("test", 4, "../nvs_partition_generator/testdata/sample_multipage_blob.bin",false,NULL);
}
@ -2329,7 +2329,7 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena
"--output",
"../nvs_partition_generator/partition_encrypted.bin",
"--size",
"0x3000",
"0x4000",
"--encrypt",
"True",
"--keyfile",
@ -2349,7 +2349,7 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena
cfg.tky[count] = 0x22;
}
check_nvs_part_gen_args(NVS_DEFAULT_PART_NAME, "../nvs_partition_generator/testdata/sample_multipage_blob.bin", true, &cfg);
check_nvs_part_gen_args(NVS_DEFAULT_PART_NAME, 4, "../nvs_partition_generator/testdata/sample_multipage_blob.bin", true, &cfg);
}
@ -2366,7 +2366,7 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena
"--output",
"../nvs_partition_generator/partition_encrypted_using_keygen.bin",
"--size",
"0x3000",
"0x4000",
"--encrypt",
"True",
"--keygen",
@ -2396,7 +2396,7 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena
cfg.tky[count] = buffer[count+32] & 255;
}
check_nvs_part_gen_args(NVS_DEFAULT_PART_NAME, "../nvs_partition_generator/testdata/sample_multipage_blob.bin", true, &cfg);
check_nvs_part_gen_args(NVS_DEFAULT_PART_NAME, 4, "../nvs_partition_generator/testdata/sample_multipage_blob.bin", true, &cfg);
}
@ -2413,7 +2413,7 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena
"--output",
"../nvs_partition_generator/partition_encrypted_using_keyfile.bin",
"--size",
"0x3000",
"0x4000",
"--encrypt",
"True",
"--keyfile",
@ -2443,7 +2443,7 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena
cfg.tky[count] = buffer[count+32] & 255;
}
check_nvs_part_gen_args(NVS_DEFAULT_PART_NAME, "../nvs_partition_generator/testdata/sample_multipage_blob.bin", true, &cfg);
check_nvs_part_gen_args(NVS_DEFAULT_PART_NAME, 4, "../nvs_partition_generator/testdata/sample_multipage_blob.bin", true, &cfg);
childpid = fork();
if (childpid == 0) {

View File

@ -5,7 +5,7 @@
# - UT_009_ - multi-device tests are not compatible
# - UT_014_ - multi-device tests are not compatible
# - UT_017_ - multi-device tests are not compatible
py3_incomp='assign_test|nvs_compatible_test|IT|UT_009_|UT_014_|UT_017_'
py3_incomp='assign_test|nvs_compatible_test|IT|UT_009_|UT_013_|UT_014_|UT_017_'
if [ -z ${PYTHON_VER+x} ] || [[ $CI_JOB_NAME =~ $py3_incomp ]]; then
# Use this version of the Python interpreter if it was not defined before or