From 0331d0aa63fb86798077935ac294fc2515d7cfeb Mon Sep 17 00:00:00 2001 From: Djordje Nedic Date: Thu, 30 Jun 2022 17:53:12 +0200 Subject: [PATCH] tools: Introduce support for blank lines in config and value files for mfg utility Some users have requested this feature. In order to avoid complete refactoring or introducing more code complexity, a design choice to create temporary files without blank lines is made. Additionally, an extension check is added and there are multiple smaller code style and structure improvements. Closes https://github.com/espressif/esp-idf/issues/8421 --- .../nvs_flash/test_nvs_host/test_nvs.cpp | 81 ++++++++++++++ tools/mass_mfg/mfg_gen.py | 100 ++++++++---------- .../samples/sample_config_blank_lines.csv | 29 +++++ ...ple_values_singlepage_blob_blank_lines.csv | 7 ++ 4 files changed, 164 insertions(+), 53 deletions(-) create mode 100644 tools/mass_mfg/samples/sample_config_blank_lines.csv create mode 100644 tools/mass_mfg/samples/sample_values_singlepage_blob_blank_lines.csv diff --git a/components/nvs_flash/test_nvs_host/test_nvs.cpp b/components/nvs_flash/test_nvs_host/test_nvs.cpp index 6d111990ff..d71c6ec961 100644 --- a/components/nvs_flash/test_nvs_host/test_nvs.cpp +++ b/components/nvs_flash/test_nvs_host/test_nvs.cpp @@ -2997,6 +2997,87 @@ TEST_CASE("check and read data from partition generated via manufacturing utilit } +TEST_CASE("check and read data from partition generated via manufacturing utility with blank lines in csv files and multipage blob support disabled", "[mfg_gen]") +{ + int childpid = fork(); + int status; + + if (childpid == 0) { + exit(execlp("bash", "bash", + "-c", + "rm -rf ../../../tools/mass_mfg/host_test && \ + cp -rf ../../../tools/mass_mfg/testdata mfg_testdata && \ + cp -rf ../nvs_partition_generator/testdata . && \ + mkdir -p ../../../tools/mass_mfg/host_test", NULL)); + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + + childpid = fork(); + if (childpid == 0) { + exit(execlp("python", "python", + "../../../tools/mass_mfg/mfg_gen.py", + "generate", + "../../../tools/mass_mfg/samples/sample_config_blank_lines.csv", + "../../../tools/mass_mfg/samples/sample_values_singlepage_blob_blank_lines.csv", + "Test", + "0x3000", + "--outdir", + "../../../tools/mass_mfg/host_test", + "--version", + "1",NULL)); + + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + + childpid = fork(); + if (childpid == 0) { + exit(execlp("python", "python", + "../nvs_partition_generator/nvs_partition_gen.py", + "generate", + "../../../tools/mass_mfg/host_test/csv/Test-1.csv", + "../nvs_partition_generator/Test-1-partition.bin", + "0x3000", + "--version", + "1",NULL)); + + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + + } + + } + + } + + SpiFlashEmulator emu1("../../../tools/mass_mfg/host_test/bin/Test-1.bin"); + check_nvs_part_gen_args_mfg(&emu1, "test", 3, "mfg_testdata/sample_singlepage_blob.bin", false, NULL); + + SpiFlashEmulator emu2("../nvs_partition_generator/Test-1-partition.bin"); + check_nvs_part_gen_args_mfg(&emu2, "test", 3, "testdata/sample_singlepage_blob.bin", false, NULL); + + + childpid = fork(); + if (childpid == 0) { + exit(execlp("bash", " bash", + "-c", + "rm -rf ../../../tools/mass_mfg/host_test | \ + rm -rf mfg_testdata | \ + rm -rf testdata",NULL)); + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) == 0); + + } + +} + TEST_CASE("check and read data from partition generated via manufacturing utility with multipage blob support enabled", "[mfg_gen]") { int childpid = fork(); diff --git a/tools/mass_mfg/mfg_gen.py b/tools/mass_mfg/mfg_gen.py index fa81c4bc83..0d4dce1605 100644 --- a/tools/mass_mfg/mfg_gen.py +++ b/tools/mass_mfg/mfg_gen.py @@ -23,6 +23,23 @@ except Exception as e: sys.exit('Please check IDF_PATH') +def create_temp_files(args): + new_filenames = [] + for filename in [args.conf, args.values]: + name, ext = os.path.splitext(filename) + new_filename = name + '_tmp' + ext + strip_blank_lines(filename, new_filename) + new_filenames.append(new_filename) + return new_filenames + + +def strip_blank_lines(input_filename, output_filename): + with open(input_filename, 'r') as read_from, open(output_filename,'w', newline='') as write_to: + for line in read_from: + if not line.isspace(): + write_to.write(line) + + def verify_values_exist(input_values_file, keys_in_values_file): """ Verify all keys have corresponding values in values file """ @@ -237,10 +254,6 @@ def set_repeat_value(total_keys_repeat, keys, csv_file, target_filename): def create_intermediate_csv(args, keys_in_values_file, keys_repeat, is_encr=False): file_identifier_value = '0' - csv_str = 'csv' - bin_str = 'bin' - set_output_keyfile = False - # Add config data per namespace to `config_data_to_write` list config_data_to_write = add_config_data_per_namespace(args.conf) @@ -261,11 +274,9 @@ def create_intermediate_csv(args, keys_in_values_file, keys_repeat, is_encr=Fals next(values_file_reader) # Create new directory(if doesn't exist) to store csv file generated - output_csv_target_dir = create_dir(csv_str, args.outdir) + output_csv_target_dir = create_dir('csv', args.outdir) # Create new directory(if doesn't exist) to store bin file generated - output_bin_target_dir = create_dir(bin_str, args.outdir) - if args.keygen: - set_output_keyfile = True + output_bin_target_dir = create_dir('bin', args.outdir) for values_data_line in values_file_reader: key_value_data = list(zip_longest(keys_in_values_file, values_data_line)) @@ -276,7 +287,7 @@ def create_intermediate_csv(args, keys_in_values_file, keys_repeat, is_encr=Fals key_value_pair = key_value_data[:] # Verify if output csv file does not exist - csv_filename = args.prefix + '-' + file_identifier_value + '.' + csv_str + csv_filename = args.prefix + '-' + file_identifier_value + '.' + 'csv' output_csv_file = output_csv_target_dir + csv_filename if os.path.isfile(output_csv_file): raise SystemExit('Target csv file: %s already exists.`' % output_csv_file) @@ -286,14 +297,14 @@ def create_intermediate_csv(args, keys_in_values_file, keys_repeat, is_encr=Fals print('\nCreated CSV file: ===>', output_csv_file) # Verify if output bin file does not exist - bin_filename = args.prefix + '-' + file_identifier_value + '.' + bin_str + bin_filename = args.prefix + '-' + file_identifier_value + '.' + 'bin' output_bin_file = output_bin_target_dir + bin_filename if os.path.isfile(output_bin_file): raise SystemExit('Target binary file: %s already exists.`' % output_bin_file) args.input = output_csv_file - args.output = os.path.join(bin_str, bin_filename) - if set_output_keyfile: + args.output = os.path.join('bin', bin_filename) + if args.keygen: args.keyfile = 'keys-' + args.prefix + '-' + file_identifier_value if is_encr: @@ -308,57 +319,38 @@ def create_intermediate_csv(args, keys_in_values_file, keys_repeat, is_encr=Fals exit(1) -def verify_empty_lines_exist(args, input_file): - input_file_reader = csv.reader(input_file, delimiter=',') - for file_data in input_file_reader: - for data in file_data: - if len(data.strip()) == 0: - raise SystemExit('Error: config file: %s cannot have empty lines. ' % args.conf) - else: - break - if not file_data: - raise SystemExit('Error: config file: %s cannot have empty lines.' % args.conf) - - input_file.seek(0) - return input_file_reader - - def verify_file_format(args): keys_in_config_file = [] keys_in_values_file = [] keys_repeat = [] - # Verify config file is not empty + # Verify files have csv extension + conf_name, conf_extension = os.path.splitext(args.conf) + if conf_extension != '.csv': + raise SystemExit('Error: config file: %s does not have the .csv extension.' % args.conf) + values_name, values_extension = os.path.splitext(args.values) + if values_extension != '.csv': + raise SystemExit('Error: values file: %s does not have the .csv extension.' % args.values) + + # Verify files are not empty if os.stat(args.conf).st_size == 0: raise SystemExit('Error: config file: %s is empty.' % args.conf) - - # Verify values file is not empty if os.stat(args.values).st_size == 0: raise SystemExit('Error: values file: %s is empty.' % args.values) - # Verify config file does not have empty lines - with open(args.conf, 'r') as csv_config_file: - try: - config_file_reader = verify_empty_lines_exist(args, csv_config_file) - # Extract keys from config file - for config_data in config_file_reader: - if 'namespace' not in config_data: - keys_in_config_file.append(config_data[0]) - if 'REPEAT' in config_data: - keys_repeat.append(config_data[0]) + # Extract keys from config file + with open(args.conf, 'r') as config_file: + config_file_reader = csv.reader(config_file, delimiter=',') + for config_data in config_file_reader: + if 'namespace' not in config_data: + keys_in_config_file.append(config_data[0]) + if 'REPEAT' in config_data: + keys_repeat.append(config_data[0]) - except Exception as e: - print(e) - - # Verify values file does not have empty lines - with open(args.values, 'r') as csv_values_file: - try: - values_file_reader = verify_empty_lines_exist(args, csv_values_file) - # Extract keys from values file - keys_in_values_file = next(values_file_reader) - - except Exception as e: - print(e) + # Extract keys from values file + with open(args.values, 'r') as values_file: + values_file_reader = csv.reader(values_file, delimiter=',') + keys_in_values_file = next(values_file_reader) # Verify file identifier exists in values file if args.fileid: @@ -371,7 +363,9 @@ def verify_file_format(args): def generate(args): - args.outdir = os.path.join(args.outdir, '') + # Create work files with no blank lines + args.conf, args.values = create_temp_files(args) + # Verify input config and values file format keys_in_config_file, keys_in_values_file, keys_repeat = verify_file_format(args) diff --git a/tools/mass_mfg/samples/sample_config_blank_lines.csv b/tools/mass_mfg/samples/sample_config_blank_lines.csv new file mode 100644 index 0000000000..66832ba885 --- /dev/null +++ b/tools/mass_mfg/samples/sample_config_blank_lines.csv @@ -0,0 +1,29 @@ +dummyNamespace,namespace, + + +dummyU8Key,data,u8 + +dummyI8Key,data,i8 + +dummyU16Key,data,u16 + + +dummyU32Key,data,u32 + +dummyI32Key,data,i32,REPEAT + +dummyStringKey,data,string + +dummyHex2BinKey,data,hex2bin + +dummyBase64Key,data,base64 +hexFileKey,file,hex2bin +base64FileKey,file,base64 + +stringFileKey,file,string + +blobFileAKey,file,binary + + +blobFileBKey,file,binary +binFileKey,file,binary diff --git a/tools/mass_mfg/samples/sample_values_singlepage_blob_blank_lines.csv b/tools/mass_mfg/samples/sample_values_singlepage_blob_blank_lines.csv new file mode 100644 index 0000000000..e3fb10685b --- /dev/null +++ b/tools/mass_mfg/samples/sample_values_singlepage_blob_blank_lines.csv @@ -0,0 +1,7 @@ +id,dummyU8Key,dummyI8Key,dummyU16Key,dummyU32Key,dummyI32Key,dummyStringKey,dummyHex2BinKey,dummyBase64Key,hexFileKey,base64FileKey,stringFileKey,blobFileAKey,blobFileBKey,binFileKey + +1,127,-128,32768,4294967295,-2147483648,0A:0B:0C:0D:0E:0F,010203abcdef,MTIzYWJj,testdata/sample.hex,testdata/sample.base64,testdata/sample.txt,testdata/sample_blob.bin,testdata/sample_blob.bin,testdata/sample_singlepage_blob.bin + + +2,126,-127,32767,4294967294,,A0:B0:C0:D0:E0:F0,102030abcdef,MTIzYWFh,testdata/sample.hex,testdata/sample.base64,testdata/sample.txt,testdata/sample_blob.bin,testdata/sample_blob.bin,testdata/sample_singlepage_blob.bin +3,125,-126,32766,4294967293,,00:B3:C4:BD:E2:0F,010203efcdab,MTIzYmJi,testdata/sample.hex,testdata/sample.base64,testdata/sample.txt,testdata/sample_blob.bin,testdata/sample_blob.bin,testdata/sample_singlepage_blob.bin