mirror of
https://github.com/espressif/esp-idf.git
synced 2024-10-05 20:47:46 -04:00
Merge branch 'update/mfg_gen' into 'master'
Update mfg utility and nvs partition utility to make Python2 and Python3 compatible See merge request idf/esp-idf!3243
This commit is contained in:
Angus Gratton
commit
dad3531f96
@ -116,6 +116,7 @@ A sample CSV file is provided with the utility::
|
||||
|
||||
python nvs_partition_gen.py sample_singlepage_blob.csv partition_single_page.bin --version v1
|
||||
|
||||
|
||||
+------------------------+----------------------------------------------------------------------------------------------+
|
||||
| Arguments | Description |
|
||||
+========================+==============================================================================================+
|
||||
@ -123,7 +124,7 @@ A sample CSV file is provided with the utility::
|
||||
+------------------------+----------------------------------------------------------------------------------------------+
|
||||
| output | Path to output converted binary file. Will use stdout if omitted |
|
||||
+------------------------+----------------------------------------------------------------------------------------------+
|
||||
| size | Size of NVS Partition in KB. Eg. 12KB |
|
||||
| size | Size of NVS Partition in bytes (must be multiple of 4096) |
|
||||
+------------------------+----------------------------------------------------------------------------------------------+
|
||||
| --version {v1,v2} | Set version. Default: v2 |
|
||||
+-------------------------------+---------------------------------------------------------------------------------------+
|
||||
|
||||
@ -17,8 +17,10 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
from __future__ import division, print_function
|
||||
from builtins import int, range
|
||||
from io import open
|
||||
import sys
|
||||
import argparse
|
||||
import binascii
|
||||
@ -28,6 +30,7 @@ import os
|
||||
import array
|
||||
import csv
|
||||
import zlib
|
||||
import codecs
|
||||
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
|
||||
from cryptography.hazmat.backends import default_backend
|
||||
|
||||
@ -79,20 +82,20 @@ class Page(object):
|
||||
global page_header
|
||||
|
||||
# set page state to active
|
||||
page_header= bytearray(b'\xff')*32
|
||||
page_header= bytearray(b'\xff') *32
|
||||
page_state_active_seq = Page.ACTIVE
|
||||
page_header[0:4] = struct.pack('<I', page_state_active_seq)
|
||||
struct.pack_into('<I', page_header, 0, page_state_active_seq)
|
||||
# set page sequence number
|
||||
page_header[4:8] = struct.pack('<I', page_num)
|
||||
struct.pack_into('<I', page_header, 4, page_num)
|
||||
# set version
|
||||
if version == Page.VERSION2:
|
||||
page_header[8] = Page.VERSION2
|
||||
elif version == Page.VERSION1:
|
||||
page_header[8] = Page.VERSION1
|
||||
# set header's CRC
|
||||
crc_data = page_header[4:28]
|
||||
crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
|
||||
page_header[28:32] = struct.pack('<I', crc & 0xFFFFFFFF)
|
||||
crc_data = bytes(page_header[4:28])
|
||||
crc = zlib.crc32(crc_data, 0xFFFFFFFF)
|
||||
struct.pack_into('<I', page_header, 28, crc & 0xFFFFFFFF)
|
||||
self.page_buf[0:len(page_header)] = page_header
|
||||
|
||||
|
||||
@ -118,8 +121,9 @@ class Page(object):
|
||||
def encrypt_entry(self, data_arr, tweak_arr, encr_key):
|
||||
# Encrypt 32 bytes of data using AES-XTS encryption
|
||||
backend = default_backend()
|
||||
plain_text = data_arr.decode('hex')
|
||||
tweak = tweak_arr.decode('hex')
|
||||
plain_text = codecs.decode(data_arr, 'hex')
|
||||
tweak = codecs.decode(tweak_arr, 'hex')
|
||||
|
||||
cipher = Cipher(algorithms.AES(encr_key), modes.XTS(tweak), backend=backend)
|
||||
encryptor = cipher.encryptor()
|
||||
encrypted_data = encryptor.update(plain_text)
|
||||
@ -139,7 +143,7 @@ class Page(object):
|
||||
|
||||
def encrypt_data(self, data_input, no_of_entries, nvs_obj):
|
||||
# Set values needed for encryption and encrypt data byte wise
|
||||
encr_data_to_write = ''
|
||||
encr_data_to_write = bytearray()
|
||||
data_len_needed = 64 #in hex
|
||||
tweak_len_needed = 32 #in hex
|
||||
init_tweak_val = '0'
|
||||
@ -147,17 +151,18 @@ class Page(object):
|
||||
tweak_tmp = ''
|
||||
encr_key_input = None
|
||||
|
||||
|
||||
# Extract encryption key and tweak key from given key input
|
||||
encr_key_input = self.encr_key.decode('hex')
|
||||
encr_key_input = codecs.decode(self.encr_key, 'hex')
|
||||
|
||||
rel_addr = nvs_obj.page_num * Page.PAGE_PARAMS["max_size"] + Page.FIRST_ENTRY_OFFSET
|
||||
|
||||
if type(data_input) != bytearray:
|
||||
byte_arr = bytearray('\xff') * 32
|
||||
if not isinstance(data_input, bytearray):
|
||||
byte_arr = bytearray(b'\xff') * 32
|
||||
byte_arr[0:len(data_input)] = data_input
|
||||
data_input = byte_arr
|
||||
|
||||
data_input = binascii.hexlify(bytearray(data_input))
|
||||
data_input = binascii.hexlify(data_input)
|
||||
|
||||
entry_no = self.entry_num
|
||||
start_idx = 0
|
||||
@ -182,6 +187,9 @@ class Page(object):
|
||||
|
||||
# Encrypt data
|
||||
data_bytes = data_input[start_idx:end_idx]
|
||||
if type(data_bytes) == bytes:
|
||||
data_bytes = data_bytes.decode()
|
||||
|
||||
data_val = data_bytes + (init_data_val * (data_len_needed - len(data_bytes)))
|
||||
encr_data_ret = self.encrypt_entry(data_val, tweak_val, encr_key_input)
|
||||
encr_data_to_write = encr_data_to_write + encr_data_ret
|
||||
@ -190,12 +198,12 @@ class Page(object):
|
||||
end_idx = start_idx + 64
|
||||
entry_no += 1
|
||||
|
||||
|
||||
return encr_data_to_write
|
||||
|
||||
|
||||
def write_entry_to_buf(self, data, entrycount,nvs_obj):
|
||||
encr_data = bytearray()
|
||||
|
||||
if self.is_encrypt:
|
||||
encr_data_ret = self.encrypt_data(data, entrycount,nvs_obj)
|
||||
encr_data[0:len(encr_data_ret)] = encr_data_ret
|
||||
@ -213,13 +221,15 @@ class Page(object):
|
||||
|
||||
|
||||
def set_crc_header(self, entry_struct):
|
||||
crc_data = bytearray(28)
|
||||
crc_data = bytearray(b'28')
|
||||
crc_data[0:4] = entry_struct[0:4]
|
||||
crc_data[4:28] = entry_struct[8:32]
|
||||
crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
|
||||
entry_struct[4:8] = struct.pack('<I', crc & 0xFFFFFFFF)
|
||||
crc_data = bytes(crc_data)
|
||||
crc = zlib.crc32(crc_data, 0xFFFFFFFF)
|
||||
struct.pack_into('<I', entry_struct, 4, crc & 0xFFFFFFFF)
|
||||
return entry_struct
|
||||
|
||||
|
||||
def write_varlen_binary_data(self, entry_struct, ns_index, key, data, data_size, total_entry_count,nvs_obj):
|
||||
chunk_start = 0
|
||||
chunk_count = 0
|
||||
@ -262,9 +272,10 @@ class Page(object):
|
||||
data_chunk = data[offset:offset + chunk_size]
|
||||
|
||||
# Compute CRC of data chunk
|
||||
entry_struct[24:26] = struct.pack('<H', chunk_size)
|
||||
struct.pack_into('<H', entry_struct, 24, chunk_size)
|
||||
data_chunk = bytes(data_chunk)
|
||||
crc = zlib.crc32(data_chunk, 0xFFFFFFFF)
|
||||
entry_struct[28:32] = struct.pack('<I', crc & 0xFFFFFFFF)
|
||||
struct.pack_into('<I', entry_struct, 28, crc & 0xFFFFFFFF)
|
||||
|
||||
# compute crc of entry header
|
||||
entry_struct = self.set_crc_header(entry_struct)
|
||||
@ -279,7 +290,7 @@ class Page(object):
|
||||
if remaining_size or (tailroom - chunk_size) < Page.SINGLE_ENTRY_SIZE:
|
||||
if page_header[0:4] != Page.FULL:
|
||||
page_state_full_seq = Page.FULL
|
||||
page_header[0:4] = struct.pack('<I', page_state_full_seq)
|
||||
struct.pack_into('<I', page_header, 0, page_state_full_seq)
|
||||
nvs_obj.create_new_page()
|
||||
self = nvs_obj.cur_page
|
||||
|
||||
@ -289,7 +300,7 @@ class Page(object):
|
||||
# All chunks are stored, now store the index
|
||||
if not remaining_size:
|
||||
# Initialise data field to 0xff
|
||||
data_array = bytearray('\xff')*8
|
||||
data_array = bytearray(b'\xff')*8
|
||||
entry_struct[24:32] = data_array
|
||||
|
||||
# change type of data to BLOB_IDX
|
||||
@ -302,7 +313,7 @@ class Page(object):
|
||||
chunk_index = Page.CHUNK_ANY
|
||||
entry_struct[3] = chunk_index
|
||||
|
||||
entry_struct[24:28] = struct.pack('<I', data_size)
|
||||
struct.pack_into('<I', entry_struct, 24, data_size)
|
||||
entry_struct[28] = chunk_count
|
||||
entry_struct[29] = chunk_start
|
||||
|
||||
@ -318,9 +329,11 @@ class Page(object):
|
||||
|
||||
def write_single_page_entry(self, entry_struct, data, datalen, data_entry_count, nvs_obj):
|
||||
# compute CRC of data
|
||||
entry_struct[24:26] = struct.pack('<H', datalen)
|
||||
struct.pack_into('<H', entry_struct, 24, datalen)
|
||||
if not type(data) == bytes:
|
||||
data = data.encode()
|
||||
crc = zlib.crc32(data, 0xFFFFFFFF)
|
||||
entry_struct[28:32] = struct.pack('<I', crc & 0xFFFFFFFF)
|
||||
struct.pack_into('<I', entry_struct, 28, crc & 0xFFFFFFFF)
|
||||
|
||||
# compute crc of entry header
|
||||
entry_struct = self.set_crc_header(entry_struct)
|
||||
@ -359,7 +372,7 @@ class Page(object):
|
||||
raise PageFullError()
|
||||
|
||||
# Entry header
|
||||
entry_struct = bytearray('\xff')*32
|
||||
entry_struct = bytearray(b'\xff')*32
|
||||
# Set Namespace Index
|
||||
entry_struct[0] = ns_index
|
||||
# Set Span
|
||||
@ -373,9 +386,9 @@ class Page(object):
|
||||
entry_struct[2] = data_entry_count + 1
|
||||
|
||||
# set key
|
||||
key_array = bytearray('\x00')*16
|
||||
key_array = b'\x00' * 16
|
||||
entry_struct[8:24] = key_array
|
||||
entry_struct[8:8 + len(key)] = key
|
||||
entry_struct[8:8 + len(key)] = key.encode()
|
||||
|
||||
# set Type
|
||||
if encoding == "string":
|
||||
@ -397,39 +410,40 @@ class Page(object):
|
||||
if self.entry_num >= Page.PAGE_PARAMS["max_entries"]:
|
||||
raise PageFullError()
|
||||
|
||||
entry_struct = bytearray('\xff')*32
|
||||
entry_struct = bytearray(b'\xff')*32
|
||||
entry_struct[0] = ns_index # namespace index
|
||||
entry_struct[2] = 0x01 # Span
|
||||
chunk_index = Page.CHUNK_ANY
|
||||
entry_struct[3] = chunk_index
|
||||
|
||||
# write key
|
||||
key_array = bytearray('\x00')*16
|
||||
key_array = b'\x00' *16
|
||||
entry_struct[8:24] = key_array
|
||||
entry_struct[8:8 + len(key)] = key
|
||||
entry_struct[8:8 + len(key)] = key.encode()
|
||||
|
||||
if encoding == "u8":
|
||||
entry_struct[1] = Page.U8
|
||||
entry_struct[24] = struct.pack('<B', data)
|
||||
struct.pack_into('<B', entry_struct, 24, data)
|
||||
elif encoding == "i8":
|
||||
entry_struct[1] = Page.I8
|
||||
entry_struct[24] = struct.pack('<b', data)
|
||||
struct.pack_into('<b', entry_struct, 24, data)
|
||||
elif encoding == "u16":
|
||||
entry_struct[1] = Page.U16
|
||||
entry_struct[24:26] = struct.pack('<H', data)
|
||||
struct.pack_into('<H', entry_struct, 24, data)
|
||||
elif encoding == "u32":
|
||||
entry_struct[1] = Page.U32
|
||||
entry_struct[24:28] = struct.pack('<I', data)
|
||||
struct.pack_into('<I', entry_struct, 24, data)
|
||||
elif encoding == "i32":
|
||||
entry_struct[1] = Page.I32
|
||||
entry_struct[24:28] = struct.pack('<i', data)
|
||||
struct.pack_into('<i', entry_struct, 24, data)
|
||||
|
||||
# Compute CRC
|
||||
crc_data = bytearray(28)
|
||||
crc_data = bytearray(b'28')
|
||||
crc_data[0:4] = entry_struct[0:4]
|
||||
crc_data[4:28] = entry_struct[8:32]
|
||||
crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
|
||||
entry_struct[4:8] = struct.pack('<I', crc & 0xFFFFFFFF)
|
||||
crc_data = bytes(crc_data)
|
||||
crc = zlib.crc32(crc_data, 0xFFFFFFFF)
|
||||
struct.pack_into('<I', entry_struct, 4, crc & 0xFFFFFFFF)
|
||||
|
||||
# write to file
|
||||
self.write_entry_to_buf(entry_struct, 1,nvs_obj)
|
||||
@ -478,7 +492,8 @@ class NVS(object):
|
||||
new_page = Page(self.page_num, is_rsrv_page)
|
||||
new_page.version = version
|
||||
new_page.is_encrypt = is_encrypt_data
|
||||
new_page.encr_key = key_input
|
||||
if new_page.is_encrypt:
|
||||
new_page.encr_key = key_input
|
||||
self.pages.append(new_page)
|
||||
self.cur_page = new_page
|
||||
return new_page
|
||||
@ -494,7 +509,6 @@ class NVS(object):
|
||||
except PageFullError:
|
||||
new_page = self.create_new_page()
|
||||
new_page.write_primitive_data(key, self.namespace_idx, "u8", 0,self)
|
||||
pass
|
||||
|
||||
"""
|
||||
Write key-value pair. Function accepts value in the form of ascii character and converts
|
||||
@ -512,6 +526,8 @@ class NVS(object):
|
||||
value = binascii.a2b_base64(value)
|
||||
|
||||
if encoding == "string":
|
||||
if type(value) == bytes:
|
||||
value = value.decode()
|
||||
value += '\0'
|
||||
|
||||
encoding = encoding.lower()
|
||||
@ -523,15 +539,12 @@ class NVS(object):
|
||||
except PageFullError:
|
||||
new_page = self.create_new_page()
|
||||
new_page.write_varlen_data(key, value, encoding, self.namespace_idx,self)
|
||||
pass
|
||||
elif encoding in primitive_encodings:
|
||||
try:
|
||||
self.cur_page.write_primitive_data(key, int(value), encoding, self.namespace_idx,self)
|
||||
except PageFullError:
|
||||
new_page = self.create_new_page()
|
||||
new_page.write_primitive_data(key, int(value), encoding, self.namespace_idx,self)
|
||||
sys.exc_clear()
|
||||
pass
|
||||
else:
|
||||
raise InputError("%s: Unsupported encoding" % encoding)
|
||||
|
||||
@ -590,6 +603,7 @@ def write_entry(nvs_instance, key, datatype, encoding, value):
|
||||
if os.path.isabs(value) == False:
|
||||
script_dir = os.path.dirname(__file__)
|
||||
abs_file_path = os.path.join(script_dir, value)
|
||||
|
||||
with open(abs_file_path, 'rb') as f:
|
||||
value = f.read()
|
||||
|
||||
@ -611,9 +625,9 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_size=None, key
|
||||
|
||||
:param input_filename: Name of input file containing data
|
||||
:param output_filename: Name of output file to store generated binary
|
||||
:param input_size: Size of partition
|
||||
:param input_size: Size of partition in bytes (must be multiple of 4096)
|
||||
:param key_gen: Enable encryption key generation in encryption mode
|
||||
:param encryption_mode: Enable/Disable encryption mode
|
||||
:param encrypt_mode: Enable/Disable encryption mode
|
||||
:param key_file: Input file having encryption keys in encryption mode
|
||||
:return: None
|
||||
"""
|
||||
@ -623,16 +637,16 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_size=None, key
|
||||
is_encrypt_data = encrypt_mode
|
||||
|
||||
# Set size
|
||||
input_size = int(input_size.split('KB')[0]) * 1024
|
||||
input_size = int(input_size, 0)
|
||||
|
||||
if input_size % 4096 !=0:
|
||||
sys.exit("Size parameter should be a multiple of 4KB.")
|
||||
sys.exit("Size of partition (must be multiple of 4096)")
|
||||
|
||||
if version == 'v1':
|
||||
version = Page.VERSION1
|
||||
elif version == 'v2':
|
||||
version = Page.VERSION2
|
||||
|
||||
|
||||
# Update size as a page needs to be reserved of size 4KB
|
||||
input_size = input_size - Page.PAGE_PARAMS["max_size"]
|
||||
|
||||
@ -662,13 +676,13 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_size=None, key
|
||||
sys.exit("Invalid. Cannot give --key_file as --encrypt is set to False.")
|
||||
|
||||
if key_gen:
|
||||
key_input = ''.join(random.choice('0123456789abcdef') for _ in xrange(128))
|
||||
key_input = ''.join(random.choice('0123456789abcdef') for _ in range(128)).strip()
|
||||
elif key_file:
|
||||
with open(key_file) as key_f:
|
||||
with open(key_file, 'rt', encoding='utf8') as key_f:
|
||||
key_input = key_f.readline()
|
||||
key_input = key_input.strip()
|
||||
|
||||
input_file = open(input_filename, 'rb')
|
||||
input_file = open(input_filename, 'rt', encoding='utf8')
|
||||
output_file = open(output_filename, 'wb')
|
||||
|
||||
with nvs_open(output_file, input_size) as nvs_obj:
|
||||
@ -686,15 +700,21 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_size=None, key
|
||||
output_file.close()
|
||||
|
||||
if is_encrypt_data:
|
||||
output_keys_file = open("encryption_keys.bin",'wb')
|
||||
keys_page_buf = bytearray(b'\xff')*Page.PAGE_PARAMS["max_size"]
|
||||
key_bytes = key_input.decode('hex')
|
||||
key_bytes = bytearray()
|
||||
|
||||
key_bytes = codecs.decode(key_input, 'hex')
|
||||
key_len = len(key_bytes)
|
||||
keys_page_buf[0:key_len] = key_bytes
|
||||
|
||||
crc_data = keys_page_buf[0:key_len]
|
||||
crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
|
||||
keys_page_buf[64:68] = struct.pack('<I', crc & 0xFFFFFFFF)
|
||||
output_keys_file.write(keys_page_buf)
|
||||
crc_data = bytes(crc_data)
|
||||
crc = zlib.crc32(crc_data, 0xFFFFFFFF)
|
||||
|
||||
struct.pack_into('<I', keys_page_buf, key_len, crc & 0xFFFFFFFF)
|
||||
|
||||
with open("encryption_keys.bin",'wb') as output_keys_file:
|
||||
output_keys_file.write(keys_page_buf)
|
||||
|
||||
|
||||
|
||||
@ -712,7 +732,7 @@ def main():
|
||||
|
||||
parser.add_argument(
|
||||
"size",
|
||||
help='Size of NVS Partition in KB. Eg. 12KB')
|
||||
help='Size of NVS Partition in bytes (must be multiple of 4096)')
|
||||
|
||||
parser.add_argument(
|
||||
"--version",
|
||||
|
||||
@ -2003,7 +2003,7 @@ TEST_CASE("check partition generation utility with multipage blob support disabl
|
||||
"../nvs_partition_generator/nvs_partition_gen.py",
|
||||
"../nvs_partition_generator/sample_singlepage_blob.csv",
|
||||
"../nvs_partition_generator/partition_single_page.bin",
|
||||
"12KB",
|
||||
"0x3000",
|
||||
"--version",
|
||||
"v1",NULL));
|
||||
} else {
|
||||
@ -2071,6 +2071,7 @@ TEST_CASE("read data from partition generated via partition generation utility w
|
||||
CHECK(memcmp(bin_data, binfiledata, bin_len) == 0);
|
||||
|
||||
file.close();
|
||||
|
||||
nvs_close(handle);
|
||||
}
|
||||
|
||||
@ -2082,7 +2083,7 @@ TEST_CASE("check partition generation utility with multipage blob support enable
|
||||
"../nvs_partition_generator/nvs_partition_gen.py",
|
||||
"../nvs_partition_generator/sample_multipage_blob.csv",
|
||||
"../nvs_partition_generator/partition_multipage_blob.bin",
|
||||
"12KB",
|
||||
"0x3000",
|
||||
"--version",
|
||||
"v2",NULL));
|
||||
} else {
|
||||
@ -2148,8 +2149,9 @@ TEST_CASE("read data from partition generated via partition generation utility w
|
||||
file.read(binfiledata,5200);
|
||||
TEST_ESP_OK( nvs_get_blob(handle, "binFileKey", bin_data, &bin_len));
|
||||
CHECK(memcmp(bin_data, binfiledata, bin_len) == 0);
|
||||
|
||||
file.close();
|
||||
|
||||
nvs_close(handle);
|
||||
|
||||
}
|
||||
|
||||
@ -2300,7 +2302,7 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena
|
||||
"../nvs_partition_generator/nvs_partition_gen.py",
|
||||
"../nvs_partition_generator/sample_multipage_blob.csv",
|
||||
"../nvs_partition_generator/partition_encrypted.bin",
|
||||
"12KB",
|
||||
"0x3000",
|
||||
"--encrypt",
|
||||
"True",
|
||||
"--keyfile",
|
||||
@ -2329,6 +2331,7 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena
|
||||
uint8_t u8v;
|
||||
TEST_ESP_OK( nvs_get_u8(handle, "dummyU8Key", &u8v));
|
||||
CHECK(u8v == 127);
|
||||
|
||||
int8_t i8v;
|
||||
TEST_ESP_OK( nvs_get_i8(handle, "dummyI8Key", &i8v));
|
||||
CHECK(i8v == -128);
|
||||
@ -2380,7 +2383,7 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena
|
||||
file.read(binfiledata,5120);
|
||||
TEST_ESP_OK( nvs_get_blob(handle, "binFileKey", bin_data, &bin_len));
|
||||
CHECK(memcmp(bin_data, binfiledata, bin_len) == 0);
|
||||
|
||||
|
||||
nvs_close(handle);
|
||||
TEST_ESP_OK(nvs_flash_deinit());
|
||||
|
||||
|
||||
@ -115,11 +115,13 @@ The mfg\_gen.py utility is using the generated CSV Configuration file and Master
|
||||
|
||||
**Usage**::
|
||||
|
||||
$ ./mfg_gen.py [-h] --conf CONFIG_FILE --values VALUES_FILE --prefix PREFIX [--fileid FILEID] [--outdir OUTDIR]
|
||||
$ ./mfg_gen.py [-h] --size PART_SIZE --conf CONFIG_FILE --values VALUES_FILE --prefix PREFIX [--fileid FILEID] [--outdir OUTDIR]
|
||||
|
||||
+------------------------+----------------------------------------------------------------------------------------------+
|
||||
| Arguments | Description |
|
||||
+========================+==============================================================================================+
|
||||
| --size PART_SIZE | Size of NVS Partition in bytes (must be multiple of 4096) |
|
||||
+------------------------+----------------------------------------------------------------------------------------------+
|
||||
| --conf CONFIG_FILE | the input configuration csv file |
|
||||
+------------------------+----------------------------------------------------------------------------------------------+
|
||||
| --values VALUES_FILE | the input values csv file |
|
||||
@ -134,7 +136,7 @@ The mfg\_gen.py utility is using the generated CSV Configuration file and Master
|
||||
|
||||
**You can use the below command to run this utility with the sample files provided**::
|
||||
|
||||
$ ./mfg_gen.py --conf sample_config.csv --values sample_values.csv --prefix Fan
|
||||
$ ./mfg_gen.py --size 0x3000 --conf samples/sample_config.csv --values samples/sample_values.csv --prefix Fan
|
||||
|
||||
|
||||
.. note:: The default numeric value: 1,2,3... of ``fileid`` argument, corresponds to each row having device instance values in master csv values file.
|
||||
|
||||
107
tools/mass_mfg/mfg_gen.py
Normal file → Executable file
107
tools/mass_mfg/mfg_gen.py
Normal file → Executable file
@ -15,6 +15,9 @@
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
from __future__ import print_function
|
||||
from builtins import range
|
||||
from future.moves.itertools import zip_longest
|
||||
import sys
|
||||
import os
|
||||
import csv
|
||||
@ -31,15 +34,15 @@ def verify_values_exist(input_values_file, keys_in_values_file):
|
||||
line_no = 1
|
||||
key_count_in_values_file = len(keys_in_values_file)
|
||||
|
||||
values_file = open(input_values_file,'rb')
|
||||
values_file = open(input_values_file, 'r')
|
||||
values_file_reader = csv.reader(values_file, delimiter=',')
|
||||
keys = values_file_reader.next()
|
||||
keys = next(values_file_reader)
|
||||
|
||||
for values_data in values_file_reader:
|
||||
line_no +=1
|
||||
if len(values_data) != key_count_in_values_file:
|
||||
raise SystemExit("\nOops...Number of values is not equal to number of keys in file: '" + \
|
||||
str(input_values_file) + "' at line No:" + str(line_no) )
|
||||
raise SystemExit("\nOops...Number of values is not equal to number of keys in file: %s at line No:%s\n"\
|
||||
% (str(input_values_file), str(line_no)))
|
||||
|
||||
|
||||
def verify_keys_exist(values_file_keys, input_config_file):
|
||||
@ -59,12 +62,11 @@ def verify_keys_exist(values_file_keys, input_config_file):
|
||||
else:
|
||||
keys_missing.append([config_data[0], line_no])
|
||||
|
||||
|
||||
|
||||
if keys_missing:
|
||||
print "Oops..."
|
||||
for key, line_no in keys_missing:
|
||||
print "Key:`" + str(key) + "` at line no:" + str(line_no) + \
|
||||
" in config file is not found in values file..."
|
||||
print("Key:`", str(key), "` at line no:", str(line_no),\
|
||||
" in config file is not found in values file.")
|
||||
config_file.close()
|
||||
raise SystemExit(1)
|
||||
|
||||
@ -84,12 +86,12 @@ def verify_datatype_encoding(input_config_file):
|
||||
for config_data in config_file_reader:
|
||||
line_no+=1
|
||||
if config_data[1] not in valid_datatypes:
|
||||
raise SystemExit("Oops...config file: `" + str(input_config_file) + \
|
||||
"` has invalid datatype at line no:" + str(line_no))
|
||||
raise SystemExit("Oops...config file: %s has invalid datatype at line no:%s\n`" \
|
||||
% (str(input_config_file), str(line_no)))
|
||||
if 'namespace' not in config_data:
|
||||
if config_data[2] not in valid_encodings:
|
||||
raise SystemExit("Oops...config file: `" + str(input_config_file) + \
|
||||
"` has invalid encoding at line no:" + str(line_no))
|
||||
raise SystemExit("Oops...config file: %s has invalid encoding at line no:%s\n`" \
|
||||
% (str(input_config_file), str(line_no)))
|
||||
|
||||
|
||||
|
||||
@ -103,9 +105,8 @@ def verify_file_data_count(input_config_file, keys_repeat):
|
||||
for line in config_file_reader:
|
||||
line_no += 1
|
||||
if len(line) != 3 and line[0] not in keys_repeat:
|
||||
raise SystemExit("Oops...data missing in config file at line no: " + str(line_no) + \
|
||||
" <format needed:key,type,encoding>")
|
||||
|
||||
raise SystemExit("Oops...data missing in config file at line no:%s <format needed:key,type,encoding>\n" \
|
||||
% str(line_no) )
|
||||
config_file.close()
|
||||
|
||||
|
||||
@ -131,7 +132,7 @@ def verify_data_in_file(input_config_file, input_values_file, config_file_keys,
|
||||
verify_values_exist(input_values_file, keys_in_values_file)
|
||||
|
||||
except StandardError as std_err:
|
||||
print std_err
|
||||
print(std_err)
|
||||
except:
|
||||
raise
|
||||
|
||||
@ -203,6 +204,7 @@ def add_data_to_file(config_data_to_write, key_value_pair, output_csv_file):
|
||||
data_to_write = []
|
||||
|
||||
target_csv_file = open(output_csv_file, 'w')
|
||||
|
||||
output_file_writer = csv.writer(target_csv_file, delimiter=',')
|
||||
output_file_writer.writerow(header)
|
||||
|
||||
@ -246,18 +248,18 @@ def set_repeat_value(total_keys_repeat, keys, csv_file):
|
||||
target_filename = filename + "_created" + file_ext
|
||||
with open(csv_file, 'r') as read_from, open(target_filename,'w') as write_to:
|
||||
csv_file_reader = csv.reader(read_from, delimiter=',')
|
||||
headers = csv_file_reader.next()
|
||||
values = csv_file_reader.next()
|
||||
total_keys_values = map(None, keys, values)
|
||||
|
||||
headers = next(csv_file_reader)
|
||||
values = next(csv_file_reader)
|
||||
csv_file_writer = csv.writer(write_to, delimiter=',')
|
||||
csv_file_writer.writerow(headers)
|
||||
csv_file_writer.writerow(values)
|
||||
total_keys_values = list(zip_longest(keys, values))
|
||||
|
||||
# read new data, add value if key has repeat tag, write to new file
|
||||
for row in csv_file_reader:
|
||||
index = -1
|
||||
key_val_new = map(None, keys, row)
|
||||
key_val_new = list(zip_longest(keys, row))
|
||||
|
||||
key_val_pair = total_keys_values[:]
|
||||
key_repeated = total_keys_repeat[:]
|
||||
while key_val_new and key_repeated:
|
||||
@ -270,7 +272,7 @@ def set_repeat_value(total_keys_repeat, keys, csv_file):
|
||||
del key_repeated[0]
|
||||
del key_val_new[0]
|
||||
del key_val_pair[0]
|
||||
|
||||
|
||||
|
||||
return target_filename
|
||||
|
||||
@ -284,6 +286,11 @@ file_identifier=None,output_dir_path=None):
|
||||
description="Create binary files from input config and values file",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
|
||||
parser.add_argument("--size",
|
||||
dest='part_size',
|
||||
required=True,
|
||||
help='Size of NVS Partition in bytes (must be multiple of 4096)')
|
||||
|
||||
parser.add_argument('--conf',
|
||||
dest='config_file',
|
||||
required=True,
|
||||
@ -320,6 +327,7 @@ file_identifier=None,output_dir_path=None):
|
||||
if not args.outdir.endswith('/'):
|
||||
args.outdir = args.outdir + '/'
|
||||
|
||||
input_part_size = args.part_size
|
||||
input_config_file = args.config_file
|
||||
input_values_file = args.values_file
|
||||
target_file_name_prefix = args.prefix
|
||||
@ -345,11 +353,11 @@ file_identifier=None,output_dir_path=None):
|
||||
|
||||
# Verify config file is not empty
|
||||
if os.stat(input_config_file).st_size == 0:
|
||||
raise SystemExit("Oops...config file: " + input_config_file + " is empty...")
|
||||
raise SystemExit("Oops...config file: %s is empty." % input_config_file)
|
||||
|
||||
# Verify values file is not empty
|
||||
if os.stat(input_values_file).st_size == 0:
|
||||
raise SystemExit("Oops...values file: " + input_values_file + " is empty...")
|
||||
raise SystemExit("Oops...values file: %s is empty." % input_values_file )
|
||||
|
||||
# Verify config file does not have empty lines
|
||||
csv_config_file = open(input_config_file,'r')
|
||||
@ -364,9 +372,9 @@ file_identifier=None,output_dir_path=None):
|
||||
is_empty_line = False
|
||||
break
|
||||
if is_empty_line:
|
||||
raise SystemExit("Oops...config file: " + input_config_file + " cannot have empty lines...")
|
||||
raise SystemExit("Oops...config file: %s cannot have empty lines. " % input_config_file )
|
||||
if not config_data:
|
||||
raise SystemExit("Oops...config file: " + input_config_file + " cannot have empty lines...")
|
||||
raise SystemExit("Oops...config file: %s cannot have empty lines." % input_config_file )
|
||||
|
||||
csv_config_file.seek(0)
|
||||
|
||||
@ -381,7 +389,7 @@ file_identifier=None,output_dir_path=None):
|
||||
|
||||
csv_config_file.close()
|
||||
except Exception as e:
|
||||
print e
|
||||
print(e)
|
||||
finally:
|
||||
csv_config_file.close()
|
||||
|
||||
@ -389,7 +397,7 @@ file_identifier=None,output_dir_path=None):
|
||||
|
||||
|
||||
# Verify values file does not have empty lines
|
||||
csv_values_file = open(input_values_file,'rb')
|
||||
csv_values_file = open(input_values_file, 'r')
|
||||
try:
|
||||
values_file_reader = csv.reader(csv_values_file, delimiter=',')
|
||||
for values_data in values_file_reader:
|
||||
@ -401,18 +409,18 @@ file_identifier=None,output_dir_path=None):
|
||||
is_empty_line = False
|
||||
break
|
||||
if is_empty_line:
|
||||
raise SystemExit("Oops...values file: " + input_values_file + " cannot have empty lines...")
|
||||
raise SystemExit("Oops...values file: %s cannot have empty lines." % input_values_file )
|
||||
if not values_data:
|
||||
raise SystemExit("Oops...values file: " + input_values_file + " cannot have empty lines...")
|
||||
raise SystemExit("Oops...values file: %s cannot have empty lines." % input_values_file )
|
||||
|
||||
csv_values_file.seek(0)
|
||||
|
||||
# Extract keys from values file
|
||||
keys_in_values_file = values_file_reader.next()
|
||||
keys_in_values_file = next(values_file_reader)
|
||||
|
||||
csv_values_file.close()
|
||||
except Exception as e:
|
||||
print e
|
||||
print(e)
|
||||
exit(1)
|
||||
finally:
|
||||
csv_values_file.close()
|
||||
@ -420,8 +428,8 @@ file_identifier=None,output_dir_path=None):
|
||||
# Verify file identifier exists in values file
|
||||
if file_identifier:
|
||||
if file_identifier not in keys_in_values_file:
|
||||
raise SystemExit('Oops...target_file_identifier: ' + file_identifier + \
|
||||
' does not exist in values file...\n')
|
||||
raise SystemExit('Oops...target_file_identifier: %s does not exist in values file.\n' % file_identifier )
|
||||
|
||||
|
||||
# Verify data in the input_config_file and input_values_file
|
||||
verify_data_in_file(input_config_file, input_values_file, keys_in_config_file,\
|
||||
@ -431,16 +439,20 @@ file_identifier=None,output_dir_path=None):
|
||||
config_data_to_write = add_config_data_per_namespace(input_config_file)
|
||||
|
||||
try:
|
||||
with open(input_values_file,'rb') as csv_values_file:
|
||||
with open(input_values_file, 'r') as csv_values_file:
|
||||
values_file_reader = csv.reader(csv_values_file, delimiter=',')
|
||||
keys = values_file_reader.next()
|
||||
keys = next(values_file_reader)
|
||||
|
||||
target_values_file = set_repeat_value(keys_repeat, keys, input_values_file)
|
||||
csv_values_file = open(target_values_file, 'rb')
|
||||
|
||||
csv_values_file = open(target_values_file, 'r')
|
||||
|
||||
values_file_reader = csv.reader(csv_values_file, delimiter=',')
|
||||
values_file_reader.next()
|
||||
next(values_file_reader)
|
||||
|
||||
|
||||
for values_data_line in values_file_reader:
|
||||
key_value_data = map(None,keys_in_values_file,values_data_line)
|
||||
key_value_data = list(zip_longest(keys_in_values_file,values_data_line))
|
||||
|
||||
# Get file identifier value from values file
|
||||
file_identifier_value = get_fileid_val(file_identifier, keys_in_config_file, \
|
||||
@ -456,7 +468,7 @@ file_identifier=None,output_dir_path=None):
|
||||
csv_file_list.append(csv_filename)
|
||||
output_csv_file = output_target_dir + csv_filename
|
||||
if os.path.isfile(output_csv_file):
|
||||
raise SystemExit("Target csv file: `" + output_csv_file + "` already exists...")
|
||||
raise SystemExit("Target csv file: %s already exists.`" % output_csv_file )
|
||||
|
||||
# Add values corresponding to each key to csv target file
|
||||
add_data_to_file(config_data_to_write, key_value_pair, output_csv_file)
|
||||
@ -468,18 +480,19 @@ file_identifier=None,output_dir_path=None):
|
||||
output_bin_file = output_target_dir + target_file_name_prefix + "-" +\
|
||||
file_identifier_value + ".bin"
|
||||
if os.path.isfile(output_bin_file):
|
||||
raise SystemExit("Target csv file: `" + output_bin_file + "` already exists...")
|
||||
raise SystemExit("Target csv file: %s already exists.`" % output_bin_file )
|
||||
|
||||
# Create output csv and bin file
|
||||
print "CSV Generated: " + str(output_csv_file)
|
||||
nvs_partition_gen.nvs_part_gen(input_filename = output_csv_file, output_filename = output_bin_file)
|
||||
print "NVS Flash Binary Generated: " + str(output_bin_file)
|
||||
print("CSV Generated: ", str(output_csv_file))
|
||||
nvs_partition_gen.nvs_part_gen(input_filename = output_csv_file, output_filename = output_bin_file,\
|
||||
input_size=input_part_size)
|
||||
print("NVS Flash Binary Generated: ", str(output_bin_file))
|
||||
|
||||
files_created = True
|
||||
|
||||
csv_values_file.close()
|
||||
except Exception as e:
|
||||
print e
|
||||
print(e)
|
||||
exit(1)
|
||||
finally:
|
||||
csv_values_file.close()
|
||||
@ -487,8 +500,8 @@ file_identifier=None,output_dir_path=None):
|
||||
|
||||
return csv_file_list, files_created
|
||||
|
||||
except StandardError as std_err:
|
||||
print std_err
|
||||
except ValueError as err:
|
||||
print(err)
|
||||
except:
|
||||
raise
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user