Merge branch 'tools/mfg_fix_blank_lines_csv_v4.4' into 'release/v4.4'

tools: Introduce support for blank lines in config and value files for mfg utility (v4.4)

See merge request espressif/esp-idf!19253
This commit is contained in:
Roland Dobai 2022-08-03 22:49:14 +08:00
commit b2770bd18e
4 changed files with 164 additions and 53 deletions

View File

@ -2911,6 +2911,87 @@ TEST_CASE("check and read data from partition generated via manufacturing utilit
}
TEST_CASE("check and read data from partition generated via manufacturing utility with blank lines in csv files and multipage blob support disabled", "[mfg_gen]")
{
int childpid = fork();
int status;
if (childpid == 0) {
exit(execlp("bash", "bash",
"-c",
"rm -rf ../../../tools/mass_mfg/host_test && \
cp -rf ../../../tools/mass_mfg/testdata mfg_testdata && \
cp -rf ../nvs_partition_generator/testdata . && \
mkdir -p ../../../tools/mass_mfg/host_test", NULL));
} else {
CHECK(childpid > 0);
waitpid(childpid, &status, 0);
CHECK(WEXITSTATUS(status) == 0);
childpid = fork();
if (childpid == 0) {
exit(execlp("python", "python",
"../../../tools/mass_mfg/mfg_gen.py",
"generate",
"../../../tools/mass_mfg/samples/sample_config_blank_lines.csv",
"../../../tools/mass_mfg/samples/sample_values_singlepage_blob_blank_lines.csv",
"Test",
"0x3000",
"--outdir",
"../../../tools/mass_mfg/host_test",
"--version",
"1",NULL));
} else {
CHECK(childpid > 0);
waitpid(childpid, &status, 0);
CHECK(WEXITSTATUS(status) == 0);
childpid = fork();
if (childpid == 0) {
exit(execlp("python", "python",
"../nvs_partition_generator/nvs_partition_gen.py",
"generate",
"../../../tools/mass_mfg/host_test/csv/Test-1.csv",
"../nvs_partition_generator/Test-1-partition.bin",
"0x3000",
"--version",
"1",NULL));
} else {
CHECK(childpid > 0);
waitpid(childpid, &status, 0);
CHECK(WEXITSTATUS(status) == 0);
}
}
}
SpiFlashEmulator emu1("../../../tools/mass_mfg/host_test/bin/Test-1.bin");
check_nvs_part_gen_args_mfg(&emu1, "test", 3, "mfg_testdata/sample_singlepage_blob.bin", false, NULL);
SpiFlashEmulator emu2("../nvs_partition_generator/Test-1-partition.bin");
check_nvs_part_gen_args_mfg(&emu2, "test", 3, "testdata/sample_singlepage_blob.bin", false, NULL);
childpid = fork();
if (childpid == 0) {
exit(execlp("bash", " bash",
"-c",
"rm -rf ../../../tools/mass_mfg/host_test | \
rm -rf mfg_testdata | \
rm -rf testdata",NULL));
} else {
CHECK(childpid > 0);
waitpid(childpid, &status, 0);
CHECK(WEXITSTATUS(status) == 0);
}
}
TEST_CASE("check and read data from partition generated via manufacturing utility with multipage blob support enabled", "[mfg_gen]")
{
int childpid = fork();

View File

@ -23,6 +23,23 @@ except Exception as e:
sys.exit('Please check IDF_PATH')
def create_temp_files(args):
new_filenames = []
for filename in [args.conf, args.values]:
name, ext = os.path.splitext(filename)
new_filename = name + '_tmp' + ext
strip_blank_lines(filename, new_filename)
new_filenames.append(new_filename)
return new_filenames
def strip_blank_lines(input_filename, output_filename):
with open(input_filename, 'r') as read_from, open(output_filename,'w', newline='') as write_to:
for line in read_from:
if not line.isspace():
write_to.write(line)
def verify_values_exist(input_values_file, keys_in_values_file):
""" Verify all keys have corresponding values in values file
"""
@ -237,10 +254,6 @@ def set_repeat_value(total_keys_repeat, keys, csv_file, target_filename):
def create_intermediate_csv(args, keys_in_values_file, keys_repeat, is_encr=False):
file_identifier_value = '0'
csv_str = 'csv'
bin_str = 'bin'
set_output_keyfile = False
# Add config data per namespace to `config_data_to_write` list
config_data_to_write = add_config_data_per_namespace(args.conf)
@ -261,11 +274,9 @@ def create_intermediate_csv(args, keys_in_values_file, keys_repeat, is_encr=Fals
next(values_file_reader)
# Create new directory(if doesn't exist) to store csv file generated
output_csv_target_dir = create_dir(csv_str, args.outdir)
output_csv_target_dir = create_dir('csv', args.outdir)
# Create new directory(if doesn't exist) to store bin file generated
output_bin_target_dir = create_dir(bin_str, args.outdir)
if args.keygen:
set_output_keyfile = True
output_bin_target_dir = create_dir('bin', args.outdir)
for values_data_line in values_file_reader:
key_value_data = list(zip_longest(keys_in_values_file, values_data_line))
@ -276,7 +287,7 @@ def create_intermediate_csv(args, keys_in_values_file, keys_repeat, is_encr=Fals
key_value_pair = key_value_data[:]
# Verify if output csv file does not exist
csv_filename = args.prefix + '-' + file_identifier_value + '.' + csv_str
csv_filename = args.prefix + '-' + file_identifier_value + '.' + 'csv'
output_csv_file = output_csv_target_dir + csv_filename
if os.path.isfile(output_csv_file):
raise SystemExit('Target csv file: %s already exists.`' % output_csv_file)
@ -286,14 +297,14 @@ def create_intermediate_csv(args, keys_in_values_file, keys_repeat, is_encr=Fals
print('\nCreated CSV file: ===>', output_csv_file)
# Verify if output bin file does not exist
bin_filename = args.prefix + '-' + file_identifier_value + '.' + bin_str
bin_filename = args.prefix + '-' + file_identifier_value + '.' + 'bin'
output_bin_file = output_bin_target_dir + bin_filename
if os.path.isfile(output_bin_file):
raise SystemExit('Target binary file: %s already exists.`' % output_bin_file)
args.input = output_csv_file
args.output = os.path.join(bin_str, bin_filename)
if set_output_keyfile:
args.output = os.path.join('bin', bin_filename)
if args.keygen:
args.keyfile = 'keys-' + args.prefix + '-' + file_identifier_value
if is_encr:
@ -308,57 +319,38 @@ def create_intermediate_csv(args, keys_in_values_file, keys_repeat, is_encr=Fals
exit(1)
def verify_empty_lines_exist(args, input_file):
input_file_reader = csv.reader(input_file, delimiter=',')
for file_data in input_file_reader:
for data in file_data:
if len(data.strip()) == 0:
raise SystemExit('Error: config file: %s cannot have empty lines. ' % args.conf)
else:
break
if not file_data:
raise SystemExit('Error: config file: %s cannot have empty lines.' % args.conf)
input_file.seek(0)
return input_file_reader
def verify_file_format(args):
keys_in_config_file = []
keys_in_values_file = []
keys_repeat = []
# Verify config file is not empty
# Verify files have csv extension
conf_name, conf_extension = os.path.splitext(args.conf)
if conf_extension != '.csv':
raise SystemExit('Error: config file: %s does not have the .csv extension.' % args.conf)
values_name, values_extension = os.path.splitext(args.values)
if values_extension != '.csv':
raise SystemExit('Error: values file: %s does not have the .csv extension.' % args.values)
# Verify files are not empty
if os.stat(args.conf).st_size == 0:
raise SystemExit('Error: config file: %s is empty.' % args.conf)
# Verify values file is not empty
if os.stat(args.values).st_size == 0:
raise SystemExit('Error: values file: %s is empty.' % args.values)
# Verify config file does not have empty lines
with open(args.conf, 'r') as csv_config_file:
try:
config_file_reader = verify_empty_lines_exist(args, csv_config_file)
# Extract keys from config file
for config_data in config_file_reader:
if 'namespace' not in config_data:
keys_in_config_file.append(config_data[0])
if 'REPEAT' in config_data:
keys_repeat.append(config_data[0])
# Extract keys from config file
with open(args.conf, 'r') as config_file:
config_file_reader = csv.reader(config_file, delimiter=',')
for config_data in config_file_reader:
if 'namespace' not in config_data:
keys_in_config_file.append(config_data[0])
if 'REPEAT' in config_data:
keys_repeat.append(config_data[0])
except Exception as e:
print(e)
# Verify values file does not have empty lines
with open(args.values, 'r') as csv_values_file:
try:
values_file_reader = verify_empty_lines_exist(args, csv_values_file)
# Extract keys from values file
keys_in_values_file = next(values_file_reader)
except Exception as e:
print(e)
# Extract keys from values file
with open(args.values, 'r') as values_file:
values_file_reader = csv.reader(values_file, delimiter=',')
keys_in_values_file = next(values_file_reader)
# Verify file identifier exists in values file
if args.fileid:
@ -371,7 +363,9 @@ def verify_file_format(args):
def generate(args):
args.outdir = os.path.join(args.outdir, '')
# Create work files with no blank lines
args.conf, args.values = create_temp_files(args)
# Verify input config and values file format
keys_in_config_file, keys_in_values_file, keys_repeat = verify_file_format(args)

View File

@ -0,0 +1,29 @@
dummyNamespace,namespace,
dummyU8Key,data,u8
dummyI8Key,data,i8
dummyU16Key,data,u16
dummyU32Key,data,u32
dummyI32Key,data,i32,REPEAT
dummyStringKey,data,string
dummyHex2BinKey,data,hex2bin
dummyBase64Key,data,base64
hexFileKey,file,hex2bin
base64FileKey,file,base64
stringFileKey,file,string
blobFileAKey,file,binary
blobFileBKey,file,binary
binFileKey,file,binary
Can't render this file because it has a wrong number of fields in line 13.

View File

@ -0,0 +1,7 @@
id,dummyU8Key,dummyI8Key,dummyU16Key,dummyU32Key,dummyI32Key,dummyStringKey,dummyHex2BinKey,dummyBase64Key,hexFileKey,base64FileKey,stringFileKey,blobFileAKey,blobFileBKey,binFileKey
1,127,-128,32768,4294967295,-2147483648,0A:0B:0C:0D:0E:0F,010203abcdef,MTIzYWJj,testdata/sample.hex,testdata/sample.base64,testdata/sample.txt,testdata/sample_blob.bin,testdata/sample_blob.bin,testdata/sample_singlepage_blob.bin
2,126,-127,32767,4294967294,,A0:B0:C0:D0:E0:F0,102030abcdef,MTIzYWFh,testdata/sample.hex,testdata/sample.base64,testdata/sample.txt,testdata/sample_blob.bin,testdata/sample_blob.bin,testdata/sample_singlepage_blob.bin
3,125,-126,32766,4294967293,,00:B3:C4:BD:E2:0F,010203efcdab,MTIzYmJi,testdata/sample.hex,testdata/sample.base64,testdata/sample.txt,testdata/sample_blob.bin,testdata/sample_blob.bin,testdata/sample_singlepage_blob.bin
1 id dummyU8Key dummyI8Key dummyU16Key dummyU32Key dummyI32Key dummyStringKey dummyHex2BinKey dummyBase64Key hexFileKey base64FileKey stringFileKey blobFileAKey blobFileBKey binFileKey
2 1 127 -128 32768 4294967295 -2147483648 0A:0B:0C:0D:0E:0F 010203abcdef MTIzYWJj testdata/sample.hex testdata/sample.base64 testdata/sample.txt testdata/sample_blob.bin testdata/sample_blob.bin testdata/sample_singlepage_blob.bin
3 2 126 -127 32767 4294967294 A0:B0:C0:D0:E0:F0 102030abcdef MTIzYWFh testdata/sample.hex testdata/sample.base64 testdata/sample.txt testdata/sample_blob.bin testdata/sample_blob.bin testdata/sample_singlepage_blob.bin
4 3 125 -126 32766 4294967293 00:B3:C4:BD:E2:0F 010203efcdab MTIzYmJi testdata/sample.hex testdata/sample.base64 testdata/sample.txt testdata/sample_blob.bin testdata/sample_blob.bin testdata/sample_singlepage_blob.bin