alex/esp-idf
alex/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf.git synced 2024-10-05 20:47:46 -04:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Merge branch 'feature/mfg_util_optimise' into 'master'

Browse Source 
mfg_util: Optimise by adding subparser changes w.r.t similar changes in nvs util changes

See merge request espressif/esp-idf!5336
This commit is contained in:
Angus Gratton 2019-07-05 10:08:10 +08:00
commit 8be982f60c
3 changed files with 306 additions and 327 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
components/nvs_flash/test_nvs_host/test_nvs.cpp
View File

@ -2462,7 +2462,6 @@ TEST_CASE("check and read data from partition generated via partition generation
} }
} }
#if false
TEST_CASE("check and read data from partition generated via manufacturing utility with multipage blob support disabled", "[mfg_gen]") TEST_CASE("check and read data from partition generated via manufacturing utility with multipage blob support disabled", "[mfg_gen]")
{ {
int childpid = fork(); int childpid = fork();
@ -2484,18 +2483,15 @@ TEST_CASE("check and read data from partition generated via manufacturing utilit
if (childpid == 0) { if (childpid == 0) {
exit(execlp("python", "python", exit(execlp("python", "python",
"../../../tools/mass_mfg/mfg_gen.py", "../../../tools/mass_mfg/mfg_gen.py",
"--conf", "generate",
"../../../tools/mass_mfg/samples/sample_config.csv", "../../../tools/mass_mfg/samples/sample_config.csv",
"--values",
"../../../tools/mass_mfg/samples/sample_values_singlepage_blob.csv", "../../../tools/mass_mfg/samples/sample_values_singlepage_blob.csv",
"--prefix",
"Test", "Test",
"--size",
"0x3000", "0x3000",
"--outdir", "--outdir",
"../../../tools/mass_mfg/host_test", "../../../tools/mass_mfg/host_test",
"--version", "--version",
"v1",NULL)); "1",NULL));
} else { } else {
CHECK(childpid > 0); CHECK(childpid > 0);
@ -2506,14 +2502,12 @@ TEST_CASE("check and read data from partition generated via manufacturing utilit
if (childpid == 0) { if (childpid == 0) {
exit(execlp("python", "python", exit(execlp("python", "python",
"../nvs_partition_generator/nvs_partition_gen.py", "../nvs_partition_generator/nvs_partition_gen.py",
"--input", "generate",
"../../../tools/mass_mfg/host_test/csv/Test-1.csv", "../../../tools/mass_mfg/host_test/csv/Test-1.csv",
"--output",
"../nvs_partition_generator/Test-1-partition.bin", "../nvs_partition_generator/Test-1-partition.bin",
"--size",
"0x3000", "0x3000",
"--version", "--version",
"v1",NULL)); "1",NULL));
} else { } else {
CHECK(childpid > 0); CHECK(childpid > 0);
@ -2570,18 +2564,15 @@ TEST_CASE("check and read data from partition generated via manufacturing utilit
if (childpid == 0) { if (childpid == 0) {
exit(execlp("python", "python", exit(execlp("python", "python",
"../../../tools/mass_mfg/mfg_gen.py", "../../../tools/mass_mfg/mfg_gen.py",
"--conf", "generate",
"../../../tools/mass_mfg/samples/sample_config.csv", "../../../tools/mass_mfg/samples/sample_config.csv",
"--values",
"../../../tools/mass_mfg/samples/sample_values_multipage_blob.csv", "../../../tools/mass_mfg/samples/sample_values_multipage_blob.csv",
"--prefix",
"Test", "Test",
"--size",
"0x4000", "0x4000",
"--outdir", "--outdir",
"../../../tools/mass_mfg/host_test", "../../../tools/mass_mfg/host_test",
"--version", "--version",
"v2",NULL)); "2",NULL));
} else { } else {
CHECK(childpid > 0); CHECK(childpid > 0);
@ -2592,14 +2583,12 @@ TEST_CASE("check and read data from partition generated via manufacturing utilit
if (childpid == 0) { if (childpid == 0) {
exit(execlp("python", "python", exit(execlp("python", "python",
"../nvs_partition_generator/nvs_partition_gen.py", "../nvs_partition_generator/nvs_partition_gen.py",
"--input", "generate",
"../../../tools/mass_mfg/host_test/csv/Test-1.csv", "../../../tools/mass_mfg/host_test/csv/Test-1.csv",
"--output",
"../nvs_partition_generator/Test-1-partition.bin", "../nvs_partition_generator/Test-1-partition.bin",
"--size",
"0x4000", "0x4000",
"--version", "--version",
"v2",NULL)); "2",NULL));
} else { } else {
CHECK(childpid > 0); CHECK(childpid > 0);
@ -2633,7 +2622,6 @@ TEST_CASE("check and read data from partition generated via manufacturing utilit
} }
} }
#endif
#if CONFIG_NVS_ENCRYPTION #if CONFIG_NVS_ENCRYPTION
TEST_CASE("check underlying xts code for 32-byte size sector encryption", "[nvs]") TEST_CASE("check underlying xts code for 32-byte size sector encryption", "[nvs]")
@ -3015,8 +3003,7 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena
} }
#if false TEST_CASE("check and read data from partition generated via manufacturing utility with encryption enabled using sample inputkey", "[mfg_gen]")
TEST_CASE("check and read data from partition generated via manufacturing utility with encryption enabled using sample keyfile", "[mfg_gen]")
{ {
int childpid = fork(); int childpid = fork();
int status; int status;
@ -3037,21 +3024,16 @@ TEST_CASE("check and read data from partition generated via manufacturing utilit
if (childpid == 0) { if (childpid == 0) {
exit(execlp("python", "python", exit(execlp("python", "python",
"../../../tools/mass_mfg/mfg_gen.py", "../../../tools/mass_mfg/mfg_gen.py",
"--conf", "generate",
"../../../tools/mass_mfg/samples/sample_config.csv", "../../../tools/mass_mfg/samples/sample_config.csv",
"--values",
"../../../tools/mass_mfg/samples/sample_values_multipage_blob.csv", "../../../tools/mass_mfg/samples/sample_values_multipage_blob.csv",
"--prefix",
"Test", "Test",
"--size",
"0x4000", "0x4000",
"--outdir", "--outdir",
"../../../tools/mass_mfg/host_test", "../../../tools/mass_mfg/host_test",
"--version", "--version",
"v2", "2",
"--encrypt", "--inputkey",
"true",
"--keyfile",
"mfg_testdata/sample_encryption_keys.bin",NULL)); "mfg_testdata/sample_encryption_keys.bin",NULL));
} else { } else {
@ -3063,17 +3045,13 @@ TEST_CASE("check and read data from partition generated via manufacturing utilit
if (childpid == 0) { if (childpid == 0) {
exit(execlp("python", "python", exit(execlp("python", "python",
"../nvs_partition_generator/nvs_partition_gen.py", "../nvs_partition_generator/nvs_partition_gen.py",
"--input", "encrypt",
"../../../tools/mass_mfg/host_test/csv/Test-1.csv", "../../../tools/mass_mfg/host_test/csv/Test-1.csv",
"--output",
"../nvs_partition_generator/Test-1-partition-encrypted.bin", "../nvs_partition_generator/Test-1-partition-encrypted.bin",
"--size",
"0x4000", "0x4000",
"--version", "--version",
"v2", "2",
"--encrypt", "--inputkey",
"true",
"--keyfile",
"testdata/sample_encryption_keys.bin",NULL)); "testdata/sample_encryption_keys.bin",NULL));
} else { } else {
@ -3143,8 +3121,7 @@ TEST_CASE("check and read data from partition generated via manufacturing utilit
if (childpid == 0) { if (childpid == 0) {
exit(execlp("python", "python", exit(execlp("python", "python",
"../../../tools/mass_mfg/mfg_gen.py", "../../../tools/mass_mfg/mfg_gen.py",
"--keygen", "generate-key",
"true",
"--outdir", "--outdir",
"../../../tools/mass_mfg/host_test", "../../../tools/mass_mfg/host_test",
"--keyfile", "--keyfile",
@ -3159,21 +3136,16 @@ TEST_CASE("check and read data from partition generated via manufacturing utilit
if (childpid == 0) { if (childpid == 0) {
exit(execlp("python", "python", exit(execlp("python", "python",
"../../../tools/mass_mfg/mfg_gen.py", "../../../tools/mass_mfg/mfg_gen.py",
"--conf", "generate",
"../../../tools/mass_mfg/samples/sample_config.csv", "../../../tools/mass_mfg/samples/sample_config.csv",
"--values",
"../../../tools/mass_mfg/samples/sample_values_multipage_blob.csv", "../../../tools/mass_mfg/samples/sample_values_multipage_blob.csv",
"--prefix",
"Test", "Test",
"--size",
"0x4000", "0x4000",
"--outdir", "--outdir",
"../../../tools/mass_mfg/host_test", "../../../tools/mass_mfg/host_test",
"--version", "--version",
"v2", "2",
"--encrypt", "--inputkey",
"true",
"--keyfile",
"../../../tools/mass_mfg/host_test/keys/encr_keys_host_test.bin",NULL)); "../../../tools/mass_mfg/host_test/keys/encr_keys_host_test.bin",NULL));
} else { } else {
@ -3185,17 +3157,13 @@ TEST_CASE("check and read data from partition generated via manufacturing utilit
if (childpid == 0) { if (childpid == 0) {
exit(execlp("python", "python", exit(execlp("python", "python",
"../nvs_partition_generator/nvs_partition_gen.py", "../nvs_partition_generator/nvs_partition_gen.py",
"--input", "encrypt",
"../../../tools/mass_mfg/host_test/csv/Test-1.csv", "../../../tools/mass_mfg/host_test/csv/Test-1.csv",
"--output",
"../nvs_partition_generator/Test-1-partition-encrypted.bin", "../nvs_partition_generator/Test-1-partition-encrypted.bin",
"--size",
"0x4000", "0x4000",
"--version", "--version",
"v2", "2",
"--encrypt", "--inputkey",
"true",
"--keyfile",
"../../../tools/mass_mfg/host_test/keys/encr_keys_host_test.bin",NULL)); "../../../tools/mass_mfg/host_test/keys/encr_keys_host_test.bin",NULL));
} else { } else {
@ -3256,7 +3224,6 @@ TEST_CASE("check and read data from partition generated via manufacturing utilit
} }
#endif #endif
#endif
/* Add new tests above */ /* Add new tests above */
/* This test has to be the final one */ /* This test has to be the final one */

143
tools/mass_mfg/docs/README.rst
View File

@ -127,75 +127,117 @@ An instance of an intermediate CSV file will be created for each device on an in
Running the utility Running the utility
------------------- -------------------
The mfg\_gen.py utility uses the generated CSV Configuration file and the master value CSV file to generate factory images for each device.
*A sample CSV Configuration file and a master value CSV file are both provided with this utility.*
**Usage**:: **Usage**::
./mfg_gen.py [-h] [--conf CONFIG_FILE] [--values VALUES_FILE]
[--prefix PREFIX] [--fileid FILEID] [--outdir OUTDIR]
[--size PART_SIZE] [--version {v1,v2}]
[--keygen {true,false}] [--encrypt {true,false}]
[--keyfile KEYFILE]
The description of the arguments is given in the table below. python mfg_gen.py [-h] {generate,generate-key} ...
+------------------------+------------------------------------------------------------+-------------------+ Optional Arguments:
| Arguments | Description | Default Value | +-----+------------+----------------------------------------------------------------------+
+========================+============================================================+===================+ | No. | Parameter | Description |
| --conf CONFIG_FILE | Path to existing CSV configuration file | | +=====+============+======================================================================+
+------------------------+------------------------------------------------------------+-------------------+ | 1 | -h, --help | show this help message and exit |
| --values VALUES_FILE | Path to existing master value CSV file | | +-----+------------+----------------------------------------------------------------------+
+------------------------+------------------------------------------------------------+-------------------+
| --prefix PREFIX | Unique filename prefix | |
+------------------------+------------------------------------------------------------+-------------------+
| --fileid FILEID | Unique file identifier (any key in the file with values) | numeric value |
| | as a filename suffix | (1,2,3...) |
+------------------------+------------------------------------------------------------+-------------------+
| --outdir OUTDIR | Output directory to store created files | current directory |
+------------------------+------------------------------------------------------------+-------------------+
| --size PART_SIZE | Size of NVS Partition in bytes (must be multiple of 4096) | |
+------------------------+------------------------------------------------------------+-------------------+
| --version {v1,v2} | Set version | v2 |
+------------------------+------------------------------------------------------------+-------------------+
| --keygen {true,false} | Generate keys for encryption | false |
+------------------------+------------------------------------------------------------+-------------------+
| --encrypt {true,false} | Set encryption mode | false |
+------------------------+------------------------------------------------------------+-------------------+
| --keyfile KEYFILE | File storing key for encryption (Applicable only if | |
| | Encryption mode is true). | |
+------------------------+------------------------------------------------------------+-------------------+
*To run this utility with the provided sample files, use the commands below*:: Commands:
Run mfg_gen.py {command} -h for additional help
+-----+--------------+--------------------------------------------------------------------+
| No. | Parameter | Description |
+=====+==============+====================================================================+
| 1 | generate | Generate NVS partition |
+-----+--------------+--------------------------------------------------------------------+
| 2 | generate-key | Generate keys for encryption |
+-----+--------------+--------------------------------------------------------------------+
./mfg_gen.py --conf samples/sample_config.csv --values samples/sample_values_singlepage_blob.csv --prefix Fan --size 0x3000 **To generate factory images for each device (Default):**
**Usage**::
./mfg_gen.py --conf samples/sample_config.csv --values samples/sample_values_multipage_blob.csv --prefix Fan --size 0x4000 python mfg_gen.py generate [-h] [--fileid FILEID] [--version {1,2}] [--keygen]
[--keyfile KEYFILE] [--inputkey INPUTKEY]
[--outdir OUTDIR]
conf values prefix size
Positional Arguments:
+--------------+----------------------------------------------------------------------+
| Parameter | Description |
+==============+======================================================================+
| conf | Path to configuration csv file to parse |
+--------------+----------------------------------------------------------------------+
| values | Path to values csv file to parse |
+--------------+----------------------------------------------------------------------+
| prefix | Unique name for each output filename prefix |
+-----+--------------+----------------------------------------------------------------+
| size | Size of NVS partition in bytes |
| | (must be multiple of 4096) |
+--------------+----------------------------------------------------------------------+
When you use this utility to generate factory images on a per device basis, keep in mind that the arguments --conf, --values, --prefix, and --size are mandatory. Optional Arguments:
+---------------------+--------------------------------------------------------------------+
| Parameter | Description |
+=====================+====================================================================+
| -h, --help | show this help message and exit |
+---------------------+--------------------------------------------------------------------+
| --fileid FILEID | Unique file identifier(any key in values file) |
| | for each filename suffix (Default: numeric value(1,2,3...) |
+---------------------+--------------------------------------------------------------------+
| --version {1,2} | Set multipage blob version. |
| | Version 1 - Multipage blob support disabled. |
| | Version 2 - Multipage blob support enabled. |
| | Default: Version 2 |
+---------------------+--------------------------------------------------------------------+
| --keygen | Generates key for encrypting NVS partition |
+---------------------+--------------------------------------------------------------------+
| --inputkey INPUTKEY | File having key for encrypting NVS partition |
+---------------------+--------------------------------------------------------------------+
| --outdir OUTDIR | Output directory to store files created |
| | (Default: current directory) |
+---------------------+--------------------------------------------------------------------+
./mfg_gen.py --conf samples/sample_config.csv --values samples/sample_values_singlepage_blob.csv --prefix Fan --size 0x3000 --outdir tmp You can run the utility to generate factory images for each device using the command below. A sample CSV file is provided with the utility::
.. note:: If the --outdir directory does not exist, it will be created. python mfg_gen.py generate samples/sample_config.csmples/sample_values_singlepage_blob.csv Sample 0x3000
The master value CSV file should have the path in the ``file`` type relative to the directory from which you are running the utility. The master value CSV file should have the path in the ``file`` type relative to the directory from which you are running the utility.
./mfg_gen.py --conf samples/sample_config.csv --values samples/sample_values_singlepage_blob.csv --prefix Fan --size 0x3000 --encrypt true --keygen true **To generate encrypted factory images for each device:**
.. note:: The generated ``keys/`` directory is named as the file with encryption keys of the form ``prefix-fileid-keys.bin``. You can run the utility to encrypt factory images for each device using the command below. A sample CSV file is provided with the utility:
*If you* **only** *want to generate a binary file with encryption keys, you can run the command below.*:: - Encrypt by allowing the utility to generate encryption keys::
./mfg_gen.py --keygen true python mfg_gen.py generate samples/sample_config.csv samples/sample_values_singlepage_blob.csv Sample 0x3000 --keygen
.. note:: When you use this utility to generate encryption keys only, the --keygen argument is mandatory. .. note:: Encryption key of the following format ``<outdir>/keys/keys-<prefix>-<fileid>.bin`` is created.
.. note:: This newly created file having encryption keys in ``keys/`` directory is compatible with NVS key-partition structure. Refer to :ref:`nvs_key_partition` for more details.
In the following example, the 'keys/' directory will be created at the current path. This binary file can further be used to encrypt factory images created on the per device basis*.:: - Encrypt by providing the encryption keys as input binary file::
./mfg_gen.py --keygen true --keyfile encr_keys.bin python mfg_gen.py generate samples/sample_config.csv samples/sample_values_singlepage_blob.csv Sample 0x3000 --inputkey keys/sample_keys.bin
.. note:: When running the utility to generate encryption keys only, if --keyfile is given, it will generate encryption keys with the filename given in the --keyfile argument. **To generate only encryption keys:**
**Usage**::
python mfg_gen.py generate-key [-h] [--keyfile KEYFILE] [--outdir OUTDIR]
Optional Arguments:
+--------------------+----------------------------------------------------------------------+
| Parameter | Description |
+====================+======================================================================+
| -h, --help | show this help message and exit |
+--------------------+----------------------------------------------------------------------+
| --keyfile KEYFILE | Path to output encryption keys file |
+--------------------+----------------------------------------------------------------------+
| --outdir OUTDIR | Output directory to store files created. |
| | (Default: current directory) |
+--------------------+----------------------------------------------------------------------+
You can run the utility to generate only encryption keys using the command below::
python mfg_gen.py generate-key
.. note:: Encryption key of the following format ``<outdir>/keys/keys-<timestamp>.bin`` is created. Timestamp format is: ``%m-%d_%H-%M``.
.. note:: To provide custom target filename use the --keyfile argument.
Generated encryption key binary file can further be used to encrypt factory images created on the per device basis.
The default numeric value: 1,2,3... of the ``fileid`` argument corresponds to each line bearing device instance values in the master value CSV file. The default numeric value: 1,2,3... of the ``fileid`` argument corresponds to each line bearing device instance values in the master value CSV file.
@ -203,3 +245,4 @@ While running the manufacturing utility, the following folders will be created i
- ``bin/`` for storing the generated binary files - ``bin/`` for storing the generated binary files
- ``csv/`` for storing the generated intermediate CSV files - ``csv/`` for storing the generated intermediate CSV files
- ``keys/`` for storing encryption keys (when generating encrypted factory images)

413
tools/mass_mfg/mfg_gen.py
View File

@ -45,7 +45,7 @@ def verify_values_exist(input_values_file, keys_in_values_file):
for values_data in values_file_reader: for values_data in values_file_reader:
line_no += 1 line_no += 1
if len(values_data) != key_count_in_values_file: if len(values_data) != key_count_in_values_file:
raise SystemExit("\nOops...Number of values is not equal to number of keys in file: %s at line No:%s\n" raise SystemExit("\nError: Number of values is not equal to number of keys in file: %s at line No:%s\n"
% (str(input_values_file), str(line_no))) % (str(input_values_file), str(line_no)))
@ -88,11 +88,11 @@ def verify_datatype_encoding(input_config_file):
for config_data in config_file_reader: for config_data in config_file_reader:
line_no += 1 line_no += 1
if config_data[1] not in valid_datatypes: if config_data[1] not in valid_datatypes:
raise SystemExit("Oops...config file: %s has invalid datatype at line no:%s\n`" raise SystemExit("Error: config file: %s has invalid datatype at line no:%s\n`"
% (str(input_config_file), str(line_no))) % (str(input_config_file), str(line_no)))
if 'namespace' not in config_data: if 'namespace' not in config_data:
if config_data[2] not in valid_encodings: if config_data[2] not in valid_encodings:
raise SystemExit("Oops...config file: %s has invalid encoding at line no:%s\n`" raise SystemExit("Error: config file: %s has invalid encoding at line no:%s\n`"
% (str(input_config_file), str(line_no))) % (str(input_config_file), str(line_no)))
@ -106,7 +106,7 @@ def verify_file_data_count(input_config_file, keys_repeat):
for line in config_file_reader: for line in config_file_reader:
line_no += 1 line_no += 1
if len(line) != 3 and line[0] not in keys_repeat: if len(line) != 3 and line[0] not in keys_repeat:
raise SystemExit("Oops...data missing in config file at line no:%s <format needed:key,type,encoding>\n" raise SystemExit("Error: data missing in config file at line no:%s <format needed:key,type,encoding>\n"
% str(line_no)) % str(line_no))
config_file.close() config_file.close()
@ -134,7 +134,7 @@ def verify_data_in_file(input_config_file, input_values_file, config_file_keys,
except Exception as err: except Exception as err:
print(err) print(err)
raise exit(1)
def get_keys(keys_in_values_file, config_file_keys): def get_keys(keys_in_values_file, config_file_keys):
@ -226,14 +226,13 @@ def add_data_to_file(config_data_to_write, key_value_pair, output_csv_file):
# Set index to start of file # Set index to start of file
target_csv_file.seek(0) target_csv_file.seek(0)
target_csv_file.close() target_csv_file.close()
def create_dir(filetype, output_dir_path): def create_dir(filetype, output_dir_path):
""" Create new directory(if doesn't exist) to store file generated """ Create new directory(if doesn't exist) to store file generated
""" """
output_target_dir = output_dir_path + filetype output_target_dir = os.path.join(output_dir_path,filetype,'')
if not os.path.isdir(output_target_dir): if not os.path.isdir(output_target_dir):
distutils.dir_util.mkpath(output_target_dir) distutils.dir_util.mkpath(output_target_dir)
@ -272,152 +271,116 @@ def set_repeat_value(total_keys_repeat, keys, csv_file, target_filename):
return target_filename return target_filename
def main(input_config_file=None,input_values_file=None,target_file_name_prefix=None, def create_intermediate_csv(args, keys_in_config_file, keys_in_values_file, keys_repeat, is_encr=False):
file_identifier=None,output_dir_path=None,part_size=None,input_version=None, file_identifier_value = '0'
input_is_keygen=None,input_is_encrypt=None,input_is_keyfile=None): csv_str = 'csv'
bin_str = 'bin'
set_output_keyfile = False
# Add config data per namespace to `config_data_to_write` list
config_data_to_write = add_config_data_per_namespace(args.conf)
try: try:
if all(arg is None for arg in [input_config_file,input_values_file,target_file_name_prefix, with open(args.values, 'r') as csv_values_file:
file_identifier,output_dir_path]): values_file_reader = csv.reader(csv_values_file, delimiter=',')
parser = argparse.ArgumentParser(prog='./mfg_gen.py', keys = next(values_file_reader)
description="Create binary files from input config and values file",
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('--conf', filename, file_ext = os.path.splitext(args.values)
dest='config_file', target_filename = filename + "_created" + file_ext
help='the input configuration csv file', if keys_repeat:
default=None) target_values_file = set_repeat_value(keys_repeat, keys, args.values, target_filename)
else:
target_values_file = args.values
parser.add_argument('--values', csv_values_file = open(target_values_file, 'r')
dest='values_file',
help='the input values csv file',
default=None)
parser.add_argument('--prefix', values_file_reader = csv.reader(csv_values_file, delimiter=',')
dest='prefix', next(values_file_reader)
help='the unique name as each filename prefix')
parser.add_argument('--fileid', # Create new directory(if doesn't exist) to store csv file generated
dest='fileid', output_csv_target_dir = create_dir(csv_str, args.outdir)
help='the unique file identifier(any key in values file) \ # Create new directory(if doesn't exist) to store bin file generated
as each filename suffix (Default: numeric value(1,2,3...)') output_bin_target_dir = create_dir(bin_str, args.outdir)
if args.keygen:
set_output_keyfile = True
parser.add_argument('--outdir', for values_data_line in values_file_reader:
dest='outdir', key_value_data = list(zip_longest(keys_in_values_file, values_data_line))
default=os.getcwd(),
help='the output directory to store the files created\
(Default: current directory)')
parser.add_argument("--size", # Get file identifier value from values file
dest='part_size', file_identifier_value = get_fileid_val(args.fileid, keys_in_config_file,
help='Size of NVS Partition in bytes (must be multiple of 4096)') keys_in_values_file, values_data_line, key_value_data,
file_identifier_value)
parser.add_argument("--version", key_value_pair = key_value_data[:]
dest="version",
help='Set version. Default: v2',
choices=['v1','v2'],
default='v2',
type=str.lower)
parser.add_argument("--keygen", # Verify if output csv file does not exist
dest="keygen", csv_filename = args.prefix + "-" + file_identifier_value + "." + csv_str
help='Generate keys for encryption. Default: false', output_csv_file = output_csv_target_dir + csv_filename
choices=['true','false'], if os.path.isfile(output_csv_file):
default='false', raise SystemExit("Target csv file: %s already exists.`" % output_csv_file)
type=str.lower)
parser.add_argument("--encrypt", # Add values corresponding to each key to csv intermediate file
dest="encrypt", add_data_to_file(config_data_to_write, key_value_pair, output_csv_file)
help='Set encryption mode. Default: false', print("\nCreated CSV file: ===>", output_csv_file)
choices=['true','false'],
default='false',
type=str.lower)
parser.add_argument("--keyfile", # Verify if output bin file does not exist
dest="keyfile", bin_filename = args.prefix + "-" + file_identifier_value + "." + bin_str
help='File having key for encryption (Applicable only if encryption mode is true)', output_bin_file = output_bin_target_dir + bin_filename
default=None) if os.path.isfile(output_bin_file):
raise SystemExit("Target binary file: %s already exists.`" % output_bin_file)
args = parser.parse_args() args.input = output_csv_file
args.output = os.path.join(bin_str, bin_filename)
if set_output_keyfile:
args.keyfile = "keys-" + args.prefix + "-" + file_identifier_value
args.outdir = os.path.join(args.outdir, '') if is_encr:
nvs_partition_gen.encrypt(args)
else:
nvs_partition_gen.generate(args)
input_config_file = args.config_file print("\nFiles generated in %s ..." % args.outdir)
input_values_file = args.values_file
target_file_name_prefix = args.prefix
output_dir_path = args.outdir
part_size = args.part_size
input_version = args.version
input_is_keygen = args.keygen
input_is_encrypt = args.encrypt
input_is_keyfile = args.keyfile
file_identifier = ''
print_arg_str = "Invalid.\nTo generate binary --conf, --values, --prefix and --size arguments are mandatory.\
\nTo generate encryption keys --keygen argument is mandatory."
print_encrypt_arg_str = "Missing parameter. Enter --keygen or --keyfile."
if args.fileid: except Exception as e:
file_identifier = args.fileid print(e)
exit(1)
finally:
csv_values_file.close()
if input_config_file and input_is_encrypt.lower() == 'true' and input_is_keygen.lower() == 'true' and input_is_keyfile:
sys.exit('Invalid. Cannot provide both --keygen and --keyfile argument together.')
nvs_partition_gen.check_input_args(input_config_file, input_values_file, part_size, input_is_keygen, def verify_empty_lines_exist(args, input_file):
input_is_encrypt, input_is_keyfile, input_version, print_arg_str, input_file_reader = csv.reader(input_file, delimiter=',')
print_encrypt_arg_str, output_dir_path) for file_data in input_file_reader:
for data in file_data:
if len(data.strip()) == 0:
raise SystemExit("Error: config file: %s cannot have empty lines. " % args.conf)
else:
break
if not file_data:
raise SystemExit("Error: config file: %s cannot have empty lines." % args.conf)
if not input_config_file and input_is_keygen: input_file.seek(0)
if input_is_encrypt == 'true': return input_file_reader
sys.exit("Invalid.\nOnly --keyfile and --outdir arguments allowed.\n")
# Generate Key Only
nvs_partition_gen.nvs_part_gen(input_filename=input_config_file, output_filename=input_values_file,
input_part_size=part_size, is_key_gen=input_is_keygen,
encrypt_mode=input_is_encrypt, key_file=input_is_keyfile,
version_no=input_version, output_dir=output_dir_path)
exit(0)
if not (input_config_file and input_values_file and target_file_name_prefix and part_size):
sys.exit(print_arg_str)
keys_in_values_file = [] def verify_file_format(args):
keys_in_config_file = [] keys_in_config_file = []
config_data_to_write = [] keys_in_values_file = []
key_value_data = [] keys_repeat = []
csv_file_list = []
keys_repeat = []
is_empty_line = False
files_created = False
file_identifier_value = '0'
output_target_dir = ''
target_values_file = None
output_file_prefix = None
# Verify config file is not empty # Verify config file is not empty
if os.stat(input_config_file).st_size == 0: if os.stat(args.conf).st_size == 0:
raise SystemExit("Oops...config file: %s is empty." % input_config_file) raise SystemExit("Error: config file: %s is empty." % args.conf)
# Verify values file is not empty # Verify values file is not empty
if os.stat(input_values_file).st_size == 0: if os.stat(args.values).st_size == 0:
raise SystemExit("Oops...values file: %s is empty." % input_values_file) raise SystemExit("Error: values file: %s is empty." % args.values)
# Verify config file does not have empty lines # Verify config file does not have empty lines
csv_config_file = open(input_config_file,'r') with open(args.conf, 'r') as csv_config_file:
try: try:
config_file_reader = csv.reader(csv_config_file, delimiter=',') config_file_reader = verify_empty_lines_exist(args, csv_config_file)
for config_data in config_file_reader:
for data in config_data:
empty_line = data.strip()
if empty_line is '':
is_empty_line = True
else:
is_empty_line = False
break
if is_empty_line:
raise SystemExit("Oops...config file: %s cannot have empty lines. " % input_config_file)
if not config_data:
raise SystemExit("Oops...config file: %s cannot have empty lines." % input_config_file)
csv_config_file.seek(0)
# Extract keys from config file # Extract keys from config file
for config_data in config_file_reader: for config_data in config_file_reader:
if 'namespace' not in config_data: if 'namespace' not in config_data:
@ -425,124 +388,130 @@ def main(input_config_file=None,input_values_file=None,target_file_name_prefix=N
if 'REPEAT' in config_data: if 'REPEAT' in config_data:
keys_repeat.append(config_data[0]) keys_repeat.append(config_data[0])
csv_config_file.close()
except Exception as e: except Exception as e:
print(e) print(e)
finally:
csv_config_file.close()
is_empty_line = False # Verify values file does not have empty lines
# Verify values file does not have empty lines with open(args.values, 'r') as csv_values_file:
csv_values_file = open(input_values_file, 'r')
try: try:
values_file_reader = csv.reader(csv_values_file, delimiter=',') values_file_reader = verify_empty_lines_exist(args, csv_values_file)
for values_data in values_file_reader:
for data in values_data:
empty_line = data.strip()
if empty_line is '':
is_empty_line = True
else:
is_empty_line = False
break
if is_empty_line:
raise SystemExit("Oops...values file: %s cannot have empty lines." % input_values_file)
if not values_data:
raise SystemExit("Oops...values file: %s cannot have empty lines." % input_values_file)
csv_values_file.seek(0)
# Extract keys from values file # Extract keys from values file
keys_in_values_file = next(values_file_reader) keys_in_values_file = next(values_file_reader)
csv_values_file.close()
except Exception as e: except Exception as e:
print(e) print(e)
exit(1)
finally:
csv_values_file.close()
# Verify file identifier exists in values file # Verify file identifier exists in values file
if file_identifier: if args.fileid:
if file_identifier not in keys_in_values_file: if args.fileid not in keys_in_values_file:
raise SystemExit('Oops...target_file_identifier: %s does not exist in values file.\n' % file_identifier) raise SystemExit('Error: target_file_identifier: %s does not exist in values file.\n' % args.fileid)
else:
args.fileid = 1
# Verify data in the input_config_file and input_values_file return keys_in_config_file, keys_in_values_file, keys_repeat
verify_data_in_file(input_config_file, input_values_file, keys_in_config_file,
keys_in_values_file, keys_repeat)
# Add config data per namespace to `config_data_to_write` list
config_data_to_write = add_config_data_per_namespace(input_config_file)
try: def generate(args):
with open(input_values_file, 'r') as csv_values_file: keys_in_config_file = []
values_file_reader = csv.reader(csv_values_file, delimiter=',') keys_in_values_file = []
keys = next(values_file_reader) keys_repeat = []
encryption_enabled = False
filename, file_ext = os.path.splitext(input_values_file) args.outdir = os.path.join(args.outdir, '')
target_filename = filename + "_created" + file_ext # Verify input config and values file format
if keys_repeat: keys_in_config_file, keys_in_values_file, keys_repeat = verify_file_format(args)
target_values_file = set_repeat_value(keys_repeat, keys, input_values_file, target_filename)
else:
target_values_file = input_values_file
csv_values_file = open(target_values_file, 'r') # Verify data in the input_config_file and input_values_file
verify_data_in_file(args.conf, args.values, keys_in_config_file,
keys_in_values_file, keys_repeat)
values_file_reader = csv.reader(csv_values_file, delimiter=',') if (args.keygen or args.inputkey):
next(values_file_reader) encryption_enabled = True
for values_data_line in values_file_reader: print("\nGenerating encrypted NVS binary images...")
key_value_data = list(zip_longest(keys_in_values_file,values_data_line)) # Create intermediate csv file
create_intermediate_csv(args, keys_in_config_file, keys_in_values_file,
keys_repeat, is_encr=encryption_enabled)
# Get file identifier value from values file
file_identifier_value = get_fileid_val(file_identifier, keys_in_config_file,
keys_in_values_file, values_data_line, key_value_data, file_identifier_value)
key_value_pair = key_value_data[:] def generate_key(args):
nvs_partition_gen.generate_key(args)
# Create new directory(if doesn't exist) to store csv file generated
output_target_dir = create_dir("csv/", output_dir_path)
# Verify if output csv file does not exist def main():
csv_filename = target_file_name_prefix + "-" + file_identifier_value + ".csv" try:
csv_file_list.append(csv_filename) parser = argparse.ArgumentParser(description="\nESP Manufacturing Utility", formatter_class=argparse.RawTextHelpFormatter)
output_csv_file = output_target_dir + csv_filename subparser = parser.add_subparsers(title='Commands',
if os.path.isfile(output_csv_file): dest='command',
raise SystemExit("Target csv file: %s already exists.`" % output_csv_file) help='\nRun mfg_gen.py {command} -h for additional help\n\n')
# Add values corresponding to each key to csv target file parser_gen = subparser.add_parser('generate',
add_data_to_file(config_data_to_write, key_value_pair, output_csv_file) help='Generate NVS partition',
formatter_class=argparse.RawTextHelpFormatter)
parser_gen.set_defaults(func=generate)
parser_gen.add_argument('conf',
default=None,
help='Path to configuration csv file to parse')
parser_gen.add_argument('values',
default=None,
help='Path to values csv file to parse')
parser_gen.add_argument('prefix',
default=None,
help='Unique name for each output filename prefix')
parser_gen.add_argument('size',
default=None,
help='Size of NVS partition in bytes\
\n(must be multiple of 4096)')
parser_gen.add_argument('--fileid',
default=None,
help='''Unique file identifier(any key in values file) \
\nfor each filename suffix (Default: numeric value(1,2,3...)''')
parser_gen.add_argument('--version',
choices=[1, 2],
default=2,
type=int,
help='''Set multipage blob version.\
\nVersion 1 - Multipage blob support disabled.\
\nVersion 2 - Multipage blob support enabled.\
\nDefault: Version 2 ''')
parser_gen.add_argument('--keygen',
action="store_true",
default=False,
help='Generates key for encrypting NVS partition')
parser_gen.add_argument('--keyfile',
default=None,
help=argparse.SUPPRESS)
parser_gen.add_argument('--inputkey',
default=None,
help='File having key for encrypting NVS partition')
parser_gen.add_argument('--outdir',
default=os.getcwd(),
help='Output directory to store files created\
\n(Default: current directory)')
parser_gen.add_argument('--input',
default=None,
help=argparse.SUPPRESS)
parser_gen.add_argument('--output',
default=None,
help=argparse.SUPPRESS)
parser_gen_key = subparser.add_parser('generate-key',
help='Generate keys for encryption',
formatter_class=argparse.RawTextHelpFormatter)
parser_gen_key.set_defaults(func=generate_key)
parser_gen_key.add_argument('--keyfile',
default=None,
help='Path to output encryption keys file')
parser_gen_key.add_argument('--outdir',
default=os.getcwd(),
help='Output directory to store files created.\
\n(Default: current directory)')
# Create new directory(if doesn't exist) to store bin file generated args = parser.parse_args()
output_target_dir = create_dir("bin/", output_dir_path) args.func(args)
# Verify if output bin file does not exist
output_file_prefix = target_file_name_prefix + "-" + file_identifier_value
output_bin_file = output_target_dir + output_file_prefix + ".bin"
if os.path.isfile(output_bin_file):
raise SystemExit("Target csv file: %s already exists.`" % output_bin_file)
# Create output csv and bin file
if input_is_keygen.lower() == 'true' and input_is_keyfile:
input_is_keyfile = os.path.basename(input_is_keyfile)
nvs_partition_gen.nvs_part_gen(input_filename=output_csv_file, output_filename=output_bin_file,
input_part_size=part_size, is_key_gen=input_is_keygen,
encrypt_mode=input_is_encrypt, key_file=input_is_keyfile,
version_no=input_version, encr_key_prefix=output_file_prefix, output_dir=output_dir_path)
print("CSV Generated: ", str(output_csv_file))
files_created = True
csv_values_file.close()
except Exception as e:
print(e)
exit(1)
finally:
csv_values_file.close()
return csv_file_list, files_created, target_values_file
except ValueError as err: except ValueError as err:
print(err) print(err)
except Exception: except Exception as e:
raise print(e)
if __name__ == "__main__": if __name__ == "__main__":