alex/esp-idf
alex/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf.git synced 2024-09-20 00:36:01 -04:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Merge branch 'feature/add-wl-support-fatfsparse' into 'master'

Browse Source 
fatfsparse.py: Add support for WL

Closes IDF-4994 and IDF-5522

See merge request espressif/esp-idf!18760
This commit is contained in:
Ivan Grokhotkov 2022-07-27 16:01:02 +08:00
commit 0330e952d1
20 changed files with 260 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
components/fatfs/fatfs_utils/entry.py
View File

@ -47,7 +47,7 @@ class Entry:
'DIR_Name' / PaddedString(MAX_NAME_SIZE, SHORT_NAMES_ENCODING),
'DIR_Name_ext' / PaddedString(MAX_EXT_SIZE, SHORT_NAMES_ENCODING),
'DIR_Attr' / Int8ul,
'DIR_NTRes' / Const(EMPTY_BYTE),
'DIR_NTRes' / Int8ul, # this tagged for lfn (0x00 for lfn prefix, 0x18 for short name in lfn)
'DIR_CrtTimeTenth' / Const(EMPTY_BYTE), # ignored by esp-idf fatfs library
'DIR_CrtTime' / Int16ul, # ignored by esp-idf fatfs library
'DIR_CrtDate' / Int16ul, # ignored by esp-idf fatfs library
@ -159,6 +159,7 @@ class Entry:
lfn_order: int = SHORT_ENTRY,
lfn_names: Optional[List[bytes]] = None,
lfn_checksum_: int = 0,
fits_short: bool = False,
lfn_is_last: bool = False) -> None:
"""
:param first_cluster_id: id of the first data cluster for given entry
@ -172,6 +173,7 @@ class Entry:
:param lfn_names: if the entry is dedicated for long names the lfn_names contains
LDIR_Name1, LDIR_Name2 and LDIR_Name3 in this order
:param lfn_checksum_: use only for long file names, checksum calculated lfn_checksum function
:param fits_short: determines if the name fits in 8.3 filename
:param lfn_is_last: determines if the long file name entry is holds last part of the name,
thus its address is first in the physical order
:returns: None
@ -213,6 +215,7 @@ class Entry:
DIR_Name=pad_string(object_name, size=MAX_NAME_SIZE),
DIR_Name_ext=pad_string(object_extension, size=MAX_EXT_SIZE),
DIR_Attr=entity_type,
DIR_NTRes=0x00 if (not self.fatfs_state.long_names_enabled) or (not fits_short) else 0x18,
DIR_FstClusLO=first_cluster_id,
DIR_FileSize=size,
DIR_CrtDate=date_entry_, # ignored by esp-idf fatfs library

1
components/fatfs/fatfs_utils/fs_object.py
View File

@ -260,6 +260,7 @@ class Directory:
entity_extension=extension,
date=fatfs_date_,
time=fatfs_time_,
fits_short=True,
entity_type=entity_type)
return free_cluster, free_entry, target_dir
return self.allocate_long_name_object(free_entry=free_entry,

6
components/fatfs/fatfs_utils/utils.py
View File

@ -42,7 +42,7 @@ FATFS_SECONDS_GRANULARITY: int = 2
LONG_NAMES_ENCODING: str = 'utf-16'
SHORT_NAMES_ENCODING: str = 'utf-8'
ALLOWED_SECTOR_SIZES: List[int] = [512, 1024, 2048, 4096]
ALLOWED_SECTOR_SIZES: List[int] = [4096]
ALLOWED_SECTORS_PER_CLUSTER: List[int] = [1, 2, 4, 8, 16, 32, 64, 128]
@ -181,7 +181,7 @@ def get_args_for_partition_generator(desc: str) -> argparse.Namespace:
parser.add_argument('--fat_type',
default=0,
type=int,
choices=[12, 16, 0],
choices=[FAT12, FAT16, 0],
help="""
Type of fat. Select 12 for fat12, 16 for fat16. Don't set, or set to 0 for automatic
calculation using cluster size and partition size.
@ -269,3 +269,5 @@ class FATDefaults:
VERSION: int = 2
TEMP_BUFFER_SIZE: int = 32
UPDATE_RATE: int = 16
WR_SIZE: int = 16
WL_SECTOR_SIZE: int = 4096

21
components/fatfs/fatfsparse.py Normal file → Executable file
View File

@ -1,3 +1,4 @@
#!/usr/bin/env python
# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import argparse
@ -9,6 +10,7 @@ from fatfs_utils.entry import Entry
from fatfs_utils.fat import FAT
from fatfs_utils.fatfs_state import BootSectorState
from fatfs_utils.utils import FULL_BYTE, LONG_NAMES_ENCODING, PAD_CHAR, FATDefaults, lfn_checksum, read_filesystem
from wl_fatfsgen import remove_wl
def build_file_name(name1: bytes, name2: bytes, name3: bytes) -> str:
@ -42,7 +44,7 @@ def traverse_folder_tree(directory_bytes_: bytes,
name: str,
state_: BootSectorState,
fat_: FAT,
binary_array_: bytearray) -> None:
binary_array_: bytes) -> None:
os.makedirs(name)
assert len(directory_bytes_) % FATDefaults.ENTRY_SIZE == 0
@ -89,9 +91,26 @@ if __name__ == '__main__':
argument_parser.add_argument('--long-name-support',
action='store_true',
help='Set flag to enable long names support.')
argument_parser.add_argument('--wear-leveling',
action='store_true',
help='Set flag to parse an image encoded using wear levelling.')
args = argument_parser.parse_args()
fs = read_filesystem(args.input_image)
# An algorithm for removing wear levelling:
# 1. find an remove dummy sector:
# a) dummy sector is at the position defined by the number of records in the state sector
# b) dummy may not be placed in state nor cfg sectors
# c) first (boot) sector position (boot_s_pos) is calculated using value of move count
# boot_s_pos = - mc
# 2. remove state sectors (trivial)
# 3. remove cfg sector (trivial)
# 4. valid fs is then old_fs[-mc:] + old_fs[:-mc]
if args.wear_leveling:
fs = remove_wl(fs)
boot_sector_ = BootSector()
boot_sector_.parse_boot_sector(fs)
fat = FAT(boot_sector_.boot_sector_state, init_=False)

8
components/fatfs/test_fatfsgen/test_fatfsgen.py
View File

@ -294,7 +294,7 @@ class FatFSGen(unittest.TestCase):
fatfs.create_file('HELLO', extension='TXT')
fatfs.write_filesystem(CFG['output_file'])
file_system = read_filesystem(CFG['output_file'])
self.assertEqual(file_system[0x2000: 0x2019], b'HELLO TXT \x00\x00\x00\x00!\x00!\x00\x00\x00\x00\x00!')
self.assertEqual(file_system[0x2000: 0x2019], b'HELLO TXT \x18\x00\x00\x00!\x00!\x00\x00\x00\x00\x00!')
def test_lfn_short_name(self) -> None:
fatfs = fatfsgen.FATFS(long_names_enabled=True)
@ -302,7 +302,7 @@ class FatFSGen(unittest.TestCase):
fatfs.write_content(path_from_root=['HELLO.TXT'], content=b'this is a test')
fatfs.write_filesystem(CFG['output_file'])
file_system = read_filesystem(CFG['output_file'])
self.assertEqual(file_system[0x2000: 0x2010], b'HELLO TXT \x00\x00\x00\x00')
self.assertEqual(file_system[0x2000: 0x2010], b'HELLO TXT \x18\x00\x00\x00')
self.assertEqual(file_system[0x2010: 0x2020], b'!\x00!\x00\x00\x00\x00\x00!\x00\x02\x00\x0e\x00\x00\x00')
self.assertEqual(file_system[0x6000: 0x6010], b'this is a test\x00\x00')
@ -367,7 +367,7 @@ class FatFSGen(unittest.TestCase):
self.assertEqual(file_system[0x6012: 0x6020], b'!\x00\x00\x00\x00\x00!\x00\x02\x00\x00\x00\x00\x00')
self.assertEqual(file_system[0x6020: 0x6030], b'.. \x10\x00\x00\x00\x00')
self.assertEqual(file_system[0x6030: 0x6040], b'!\x00!\x00\x00\x00\x00\x00!\x00\x01\x00\x00\x00\x00\x00')
self.assertEqual(file_system[0x6040: 0x6050], b'HELLO TXT \x00\x00\x00\x00')
self.assertEqual(file_system[0x6040: 0x6050], b'HELLO TXT \x18\x00\x00\x00')
self.assertEqual(file_system[0x6050: 0x6060], b'!\x00!\x00\x00\x00\x00\x00!\x00\x03\x00\x00\x00\x00\x00')
def test_lfn_nested_long_empty(self) -> None:
@ -410,7 +410,7 @@ class FatFSGen(unittest.TestCase):
self.assertEqual(file_system[0x6012: 0x6020], b'!\x00\x00\x00\x00\x00!\x00\x02\x00\x00\x00\x00\x00')
self.assertEqual(file_system[0x6020: 0x6030], b'.. \x10\x00\x00\x00\x00')
self.assertEqual(file_system[0x6030: 0x6040], b'!\x00!\x00\x00\x00\x00\x00!\x00\x01\x00\x00\x00\x00\x00')
self.assertEqual(file_system[0x6040: 0x6050], b'HELLO TXT \x00\x00\x00\x00')
self.assertEqual(file_system[0x6040: 0x6050], b'HELLO TXT \x18\x00\x00\x00')
self.assertEqual(file_system[0x6050: 0x6060], b'!\x00!\x00\x00\x00\x00\x00!\x00\x03\x00\x0e\x00\x00\x00')
self.assertEqual(file_system[0x7000: 0x7010], b'this is a test\x00\x00')

29
components/fatfs/test_fatfsgen/test_fatfsparse.py
View File

@ -23,6 +23,7 @@ class FatFSGen(unittest.TestCase):
shutil.rmtree('output_data', ignore_errors=True)
shutil.rmtree('Espressif', ignore_errors=True)
shutil.rmtree('testf', ignore_errors=True)
shutil.rmtree('testf_wl', ignore_errors=True)
if os.path.exists('fatfs_image.img'):
os.remove('fatfs_image.img')
@ -138,6 +139,15 @@ class FatFSGen(unittest.TestCase):
], stderr=STDOUT)
run(['python', '../fatfsparse.py', 'fatfs_image.img'], stderr=STDOUT)
assert compare_folders('testf', 'Espressif')
shutil.rmtree('Espressif', ignore_errors=True)
run([
'python',
f'{os.path.join(os.path.dirname(__file__), "..", "wl_fatfsgen.py")}',
'testf'
], stderr=STDOUT)
run(['python', '../fatfsparse.py', '--wear-leveling', 'fatfs_image.img'], stderr=STDOUT)
assert compare_folders('testf', 'Espressif')
def test_e2e_deeper(self) -> None:
folder_ = {
@ -159,6 +169,7 @@ class FatFSGen(unittest.TestCase):
folder_
]
}
generate_local_folder_structure(struct_, path_='.')
run([
'python',
@ -167,6 +178,15 @@ class FatFSGen(unittest.TestCase):
], stderr=STDOUT)
run(['python', '../fatfsparse.py', 'fatfs_image.img'], stderr=STDOUT)
assert compare_folders('testf', 'Espressif')
shutil.rmtree('Espressif', ignore_errors=True)
run([
'python',
f'{os.path.join(os.path.dirname(__file__), "..", "wl_fatfsgen.py")}',
'testf'
], stderr=STDOUT)
run(['python', '../fatfsparse.py', '--wear-leveling', 'fatfs_image.img'], stderr=STDOUT)
assert compare_folders('testf', 'Espressif')
def test_e2e_deeper_large(self) -> None:
folder_ = {
@ -214,6 +234,15 @@ class FatFSGen(unittest.TestCase):
], stderr=STDOUT)
run(['python', '../fatfsparse.py', 'fatfs_image.img'], stderr=STDOUT)
assert compare_folders('testf', 'Espressif')
shutil.rmtree('Espressif', ignore_errors=True)
run([
'python',
f'{os.path.join(os.path.dirname(__file__), "..", "wl_fatfsgen.py")}',
'testf'
], stderr=STDOUT)
run(['python', '../fatfsparse.py', '--wear-leveling', 'fatfs_image.img'], stderr=STDOUT)
assert compare_folders('testf', 'Espressif')
def test_e2e_very_deep(self) -> None:
folder_ = {

3
components/fatfs/test_fatfsgen/test_wl_fatfsgen.py
View File

@ -105,9 +105,6 @@ class WLFatFSGen(unittest.TestCase):
fatfs.wl_create_directory('TESTFOLD')
self.assertRaises(WLNotInitialized, fatfs.wl_write_filesystem, CFG['output_file'])
def test_wrong_sector_size(self) -> None:
self.assertRaises(NotImplementedError, wl_fatfsgen.WLFATFS, sector_size=1024)
def test_e2e_deep_folder_into_image_ext(self) -> None:
fatfs = wl_fatfsgen.WLFATFS()
fatfs.wl_generate(CFG['test_dir2'])

82
components/fatfs/wl_fatfsgen.py
View File

@ -11,6 +11,41 @@ from fatfs_utils.utils import (FULL_BYTE, UINT32_MAX, FATDefaults, crc32, genera
from fatfsgen import FATFS
def remove_wl(binary_image: bytes) -> bytes:
partition_size: int = len(binary_image)
total_sectors: int = partition_size // FATDefaults.WL_SECTOR_SIZE
wl_state_size: int = WLFATFS.WL_STATE_HEADER_SIZE + WLFATFS.WL_STATE_RECORD_SIZE * total_sectors
wl_state_sectors_cnt: int = (wl_state_size + FATDefaults.WL_SECTOR_SIZE - 1) // FATDefaults.WL_SECTOR_SIZE
wl_state_total_size: int = wl_state_sectors_cnt * FATDefaults.WL_SECTOR_SIZE
wl_sectors_size: int = (wl_state_sectors_cnt
* FATDefaults.WL_SECTOR_SIZE
* WLFATFS.WL_STATE_COPY_COUNT
+ FATDefaults.WL_SECTOR_SIZE)
correct_wl_configuration = binary_image[-wl_sectors_size:]
data_ = WLFATFS.WL_STATE_T_DATA.parse(correct_wl_configuration[:WLFATFS.WL_STATE_HEADER_SIZE])
total_records = 0
# iterating over records field of the first copy of the state sector
for i in range(WLFATFS.WL_STATE_HEADER_SIZE, wl_state_total_size, WLFATFS.WL_STATE_RECORD_SIZE):
if correct_wl_configuration[i:i + WLFATFS.WL_STATE_RECORD_SIZE] != WLFATFS.WL_STATE_RECORD_SIZE * b'\xff':
total_records += 1
else:
break
before_dummy = binary_image[:total_records * FATDefaults.WL_SECTOR_SIZE]
after_dummy = binary_image[total_records * FATDefaults.WL_SECTOR_SIZE + FATDefaults.WL_SECTOR_SIZE:]
new_image: bytes = before_dummy + after_dummy
# remove wl sectors
new_image = new_image[:len(new_image) - (FATDefaults.WL_SECTOR_SIZE + 2 * wl_state_total_size)]
# reorder to preserve original order
new_image = (new_image[-data_['move_count'] * FATDefaults.WL_SECTOR_SIZE:]
+ new_image[:-data_['move_count'] * FATDefaults.WL_SECTOR_SIZE])
return new_image
class WLFATFS:
# pylint: disable=too-many-instance-attributes
WL_CFG_SECTORS_COUNT = 1
@ -18,7 +53,7 @@ class WLFATFS:
WL_CONFIG_HEADER_SIZE = 48
WL_STATE_RECORD_SIZE = 16
WL_STATE_HEADER_SIZE = 64
WL_STATE_COPY_COUNT = 2
WL_STATE_COPY_COUNT = 2 # always 2 copies for power failure safety
WL_SECTOR_SIZE = 0x1000
WL_STATE_T_DATA = Struct(
@ -37,7 +72,7 @@ class WLFATFS:
'start_addr' / Int32ul,
'full_mem_size' / Int32ul,
'page_size' / Int32ul,
'sector_size' / Int32ul,
'sector_size' / Int32ul, # always 4096 for the types of NOR flash supported by ESP-IDF!
'updaterate' / Int32ul,
'wr_size' / Int32ul,
'version' / Int32ul,
@ -50,7 +85,6 @@ class WLFATFS:
reserved_sectors_cnt: int = FATDefaults.RESERVED_SECTORS_COUNT,
fat_tables_cnt: int = FATDefaults.FAT_TABLES_COUNT,
sectors_per_cluster: int = FATDefaults.SECTORS_PER_CLUSTER,
sector_size: int = FATDefaults.SECTOR_SIZE,
sectors_per_fat: int = FATDefaults.SECTORS_PER_FAT,
explicit_fat_type: int = None,
hidden_sectors: int = FATDefaults.HIDDEN_SECTORS,
@ -63,28 +97,22 @@ class WLFATFS:
use_default_datetime: bool = True,
version: int = FATDefaults.VERSION,
temp_buff_size: int = FATDefaults.TEMP_BUFFER_SIZE,
update_rate: int = FATDefaults.UPDATE_RATE,
device_id: int = None,
root_entry_count: int = FATDefaults.ROOT_ENTRIES_COUNT,
media_type: int = FATDefaults.MEDIA_TYPE) -> None:
if sector_size != WLFATFS.WL_SECTOR_SIZE:
raise NotImplementedError(f'The only supported sector size is currently {WLFATFS.WL_SECTOR_SIZE}')
self._initialized = False
self.sector_size = sector_size
self._version = version
self._temp_buff_size = temp_buff_size
self._device_id = device_id
self._update_rate = update_rate
self.partition_size = size
self.total_sectors = self.partition_size // self.sector_size
self.total_sectors = self.partition_size // FATDefaults.WL_SECTOR_SIZE
self.wl_state_size = WLFATFS.WL_STATE_HEADER_SIZE + WLFATFS.WL_STATE_RECORD_SIZE * self.total_sectors
# determine the number of required sectors (roundup to sector size)
self.wl_state_sectors = (self.wl_state_size + self.sector_size - 1) // self.sector_size
self.wl_state_sectors = (self.wl_state_size + FATDefaults.WL_SECTOR_SIZE - 1) // FATDefaults.WL_SECTOR_SIZE
self.boot_sector_start = self.sector_size # shift by one "dummy" sector
self.fat_table_start = self.boot_sector_start + reserved_sectors_cnt * self.sector_size
self.boot_sector_start = FATDefaults.WL_SECTOR_SIZE # shift by one "dummy" sector
self.fat_table_start = self.boot_sector_start + reserved_sectors_cnt * FATDefaults.WL_SECTOR_SIZE
wl_sectors = (WLFATFS.WL_DUMMY_SECTORS_COUNT + WLFATFS.WL_CFG_SECTORS_COUNT +
self.wl_state_sectors * WLFATFS.WL_STATE_COPY_COUNT)
@ -92,11 +120,11 @@ class WLFATFS:
self.plain_fatfs = FATFS(
explicit_fat_type=explicit_fat_type,
size=self.plain_fat_sectors * self.sector_size,
size=self.plain_fat_sectors * FATDefaults.WL_SECTOR_SIZE,
reserved_sectors_cnt=reserved_sectors_cnt,
fat_tables_cnt=fat_tables_cnt,
sectors_per_cluster=sectors_per_cluster,
sector_size=sector_size,
sector_size=FATDefaults.WL_SECTOR_SIZE,
sectors_per_fat=sectors_per_fat,
root_entry_count=root_entry_count,
hidden_sectors=hidden_sectors,
@ -121,17 +149,17 @@ class WLFATFS:
self._initialized = True
def _add_dummy_sector(self) -> None:
self.fatfs_binary_image = self.sector_size * FULL_BYTE + self.fatfs_binary_image
self.fatfs_binary_image = FATDefaults.WL_SECTOR_SIZE * FULL_BYTE + self.fatfs_binary_image
def _add_config_sector(self) -> None:
wl_config_data = WLFATFS.WL_CONFIG_T_DATA.build(
dict(
start_addr=0,
full_mem_size=self.partition_size,
page_size=self.sector_size,
sector_size=self.sector_size,
updaterate=self._update_rate,
wr_size=16,
page_size=FATDefaults.WL_SECTOR_SIZE, # equal to sector size (always 4096)
sector_size=FATDefaults.WL_SECTOR_SIZE,
updaterate=FATDefaults.UPDATE_RATE,
wr_size=FATDefaults.WR_SIZE,
version=self._version,
temp_buff_size=self._temp_buff_size
)
@ -143,7 +171,8 @@ class WLFATFS:
# adding three 4 byte zeros to align the structure
wl_config = wl_config_data + wl_config_crc + Int32ul.build(0) + Int32ul.build(0) + Int32ul.build(0)
self.fatfs_binary_image += (wl_config + (self.sector_size - WLFATFS.WL_CONFIG_HEADER_SIZE) * FULL_BYTE)
self.fatfs_binary_image += (
wl_config + (FATDefaults.WL_SECTOR_SIZE - WLFATFS.WL_CONFIG_HEADER_SIZE) * FULL_BYTE)
def _add_state_sectors(self) -> None:
wl_state_data = WLFATFS.WL_STATE_T_DATA.build(
@ -152,8 +181,8 @@ class WLFATFS:
max_pos=self.plain_fat_sectors + WLFATFS.WL_DUMMY_SECTORS_COUNT,
move_count=0,
access_count=0,
max_count=self._update_rate,
block_size=self.sector_size,
max_count=FATDefaults.UPDATE_RATE,
block_size=FATDefaults.WL_SECTOR_SIZE, # equal to page size, thus equal to wl sector size (4096)
version=self._version,
device_id=self._device_id or generate_4bytes_random(),
)
@ -161,9 +190,9 @@ class WLFATFS:
crc = crc32(list(wl_state_data), UINT32_MAX)
wl_state_crc = Int32ul.build(crc)
wl_state = wl_state_data + wl_state_crc
wl_state_sector_padding: bytes = (self.sector_size - WLFATFS.WL_STATE_HEADER_SIZE) * FULL_BYTE
wl_state_sector_padding: bytes = (FATDefaults.WL_SECTOR_SIZE - WLFATFS.WL_STATE_HEADER_SIZE) * FULL_BYTE
wl_state_sector: bytes = (
wl_state + wl_state_sector_padding + (self.wl_state_sectors - 1) * self.sector_size * FULL_BYTE
wl_state + wl_state_sector_padding + (self.wl_state_sectors - 1) * FATDefaults.WL_SECTOR_SIZE * FULL_BYTE
)
self.fatfs_binary_image += (WLFATFS.WL_STATE_COPY_COUNT * wl_state_sector)
@ -193,8 +222,7 @@ if __name__ == '__main__':
desc = 'Create a FAT filesystem with support for wear levelling and populate it with directory content'
args = get_args_for_partition_generator(desc)
wl_fatfs = WLFATFS(sector_size=args.sector_size,
sectors_per_cluster=args.sectors_per_cluster,
wl_fatfs = WLFATFS(sectors_per_cluster=args.sectors_per_cluster,
size=args.partition_size,
root_entry_count=args.root_entry_count,
explicit_fat_type=args.fat_type,

7
examples/storage/fatfsgen/main/Kconfig.projbuild
View File

@ -7,6 +7,13 @@ menu "Example Configuration"
If read-only mode is set, the generated fatfs image will be raw (without wear levelling support).
Otherwise it will support wear levelling that enables read-write mounting.
config EXAMPLE_FATFS_WRITE_COUNT
int "Number of volumes"
default 1
range 1 600
help
Number of writes to the file (for testing purposes).
config EXAMPLE_FATFS_DEFAULT_DATETIME
bool "Default modification date and time for generated FATFS image"
default n

16
examples/storage/fatfsgen/main/fatfsgen_example_main.c
View File

@ -66,13 +66,17 @@ void app_main(void)
if (!EXAMPLE_FATFS_MODE_READ_ONLY){
// Open file for reading
ESP_LOGI(TAG, "Opening file");
FILE *f = fopen(device_filename, "wb");
if (f == NULL) {
ESP_LOGE(TAG, "Failed to open file for writing");
return;
FILE *f;
for(int i = 0; i < CONFIG_EXAMPLE_FATFS_WRITE_COUNT; i++){
f = fopen(device_filename, "wb");
if (f == NULL) {
ESP_LOGE(TAG, "Failed to open file for writing");
return;
}
fprintf(f, "This is written by the device");
fclose(f);
}
fprintf(f, "This is written by the device");
fclose(f);
ESP_LOGI(TAG, "File written");
// Open file for reading

117
examples/storage/fatfsgen/pytest_fatfsgen_example.py
View File

@ -1,13 +1,55 @@
# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Unlicense OR CC0-1.0
import os
import re
import shutil
import sys
from datetime import datetime
from subprocess import STDOUT, run
from typing import List
import pytest
from pytest_embedded import Dut
idf_path = os.environ['IDF_PATH'] # get value of IDF_PATH from environment
parttool_dir = os.path.join(idf_path, 'components', 'partition_table')
sys.path.append(parttool_dir)
from parttool import PartitionName, ParttoolTarget # noqa E402 # pylint: disable=C0413
def file_(x: str, content_: str = 'hey this is a test') -> dict:
return {
'type': 'file',
'name': x,
'content': content_
}
def generate_local_folder_structure(structure_: dict, path_: str) -> None:
if structure_['type'] == 'folder':
new_path_ = os.path.join(path_, structure_['name'])
os.makedirs(new_path_)
for item_ in structure_['content']:
generate_local_folder_structure(item_, new_path_)
else:
new_path_ = os.path.join(path_, structure_['name'])
with open(new_path_, 'w') as f_:
f_.write(structure_['content'])
def compare_folders(fp1: str, fp2: str) -> bool:
if os.path.isdir(fp1) != os.path.isdir(fp2):
return False
if os.path.isdir(fp1):
if set(os.listdir(fp1)) != set(os.listdir(fp2)):
return False
return all([compare_folders(os.path.join(fp1, path_), os.path.join(fp2, path_)) for path_ in os.listdir(fp1)])
with open(fp1, 'rb') as f1_, open(fp2, 'rb') as f2_:
return f1_.read() == f2_.read()
# Example_GENERIC
@pytest.mark.esp32
@ -51,6 +93,7 @@ def test_examples_fatfsgen(config: str, dut: Dut) -> None:
filename_sn = 'sub/test.txt'
date_modified = datetime.today()
date_default = datetime(1980, 1, 1)
fatfs_parser_path = os.path.join(idf_path, 'components', 'fatfs', 'fatfsparse.py')
if config in ['test_read_write_partition_gen', 'test_read_write_partition_gen_default_dt']:
filename = filename_sn
@ -68,6 +111,32 @@ def test_examples_fatfsgen(config: str, dut: Dut) -> None:
'example: Unmounting FAT filesystem',
'example: Done'], timeout)
target = ParttoolTarget(dut.port)
target.read_partition(PartitionName('storage'), 'temp.img')
run(['python', fatfs_parser_path, '--wear-leveling', 'temp.img'], stderr=STDOUT)
folder_ = {
'type': 'folder',
'name': 'SUB',
'content': [
file_('TEST.TXT', content_='this is test\n'),
]
}
struct_: dict = {
'type': 'folder',
'name': 'testf',
'content': [
file_('HELLO.TXT', content_='This is generated on the host\n'),
file_('INNER.TXT', content_='This is written by the device'),
folder_
]
}
generate_local_folder_structure(struct_, path_='.')
try:
assert compare_folders('testf', 'Espressif')
finally:
shutil.rmtree('Espressif', ignore_errors=True)
shutil.rmtree('testf', ignore_errors=True)
elif config in ['test_read_only_partition_gen', 'test_read_only_partition_gen_default_dt']:
filename = filename_sn
filename_expected = f'/spiflash/{filename}'
@ -79,6 +148,30 @@ def test_examples_fatfsgen(config: str, dut: Dut) -> None:
expect_all(['example: Read from file: \'this is test\'',
'example: Unmounting FAT filesystem',
'example: Done'], timeout)
target = ParttoolTarget(dut.port)
target.read_partition(PartitionName('storage'), 'temp.img')
run(['python', fatfs_parser_path, '--long-name-support', 'temp.img'], stderr=STDOUT)
folder_ = {
'type': 'folder',
'name': 'sublongnames',
'content': [
file_('testlongfilenames.txt', content_='this is test; long name it has\n'),
]
}
struct_ = {
'type': 'folder',
'name': 'testf',
'content': [
file_('hellolongname.txt', content_='This is generated on the host; long name it has\n'),
folder_
]
}
generate_local_folder_structure(struct_, path_='.')
try:
assert compare_folders('testf', 'Espressif')
finally:
shutil.rmtree('Espressif', ignore_errors=True)
shutil.rmtree('testf', ignore_errors=True)
elif config in ['test_read_write_partition_gen_ln', 'test_read_write_partition_gen_ln_default_dt']:
filename = filename_ln
@ -107,3 +200,27 @@ def test_examples_fatfsgen(config: str, dut: Dut) -> None:
expect_all(['example: Read from file: \'this is test; long name it has\'',
'example: Unmounting FAT filesystem',
'example: Done'], timeout)
target = ParttoolTarget(dut.port)
target.read_partition(PartitionName('storage'), 'temp.img')
run(['python', fatfs_parser_path, 'temp.img'], stderr=STDOUT)
folder_ = {
'type': 'folder',
'name': 'SUB',
'content': [
file_('TEST.TXT', content_='this is test\n'),
]
}
struct_ = {
'type': 'folder',
'name': 'testf',
'content': [
file_('HELLO.TXT', content_='This is generated on the host\n'),
folder_
]
}
generate_local_folder_structure(struct_, path_='.')
try:
assert compare_folders('testf', 'Espressif')
finally:
shutil.rmtree('Espressif', ignore_errors=True)
shutil.rmtree('testf', ignore_errors=True)

1
examples/storage/fatfsgen/sdkconfig.ci.test_read_only_partition_gen
View File

@ -2,3 +2,4 @@ CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY=y
CONFIG_FATFS_LFN_HEAP=n
CONFIG_FATFS_LFN_STACK=n
CONFIG_FATFS_LFN_NONE=y
CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300

1
examples/storage/fatfsgen/sdkconfig.ci.test_read_only_partition_gen_default_dt
View File

@ -3,3 +3,4 @@ CONFIG_FATFS_LFN_HEAP=n
CONFIG_FATFS_LFN_STACK=n
CONFIG_FATFS_LFN_NONE=y
CONFIG_EXAMPLE_FATFS_DEFAULT_DATETIME=y
CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300

1
examples/storage/fatfsgen/sdkconfig.ci.test_read_only_partition_gen_ln
View File

@ -2,3 +2,4 @@ CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY=y
CONFIG_FATFS_LFN_HEAP=y
CONFIG_FATFS_LFN_NONE=n
CONFIG_FATFS_LFN_STACK=n
CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300

1
examples/storage/fatfsgen/sdkconfig.ci.test_read_only_partition_gen_ln_default_dt
View File

@ -3,3 +3,4 @@ CONFIG_FATFS_LFN_HEAP=y
CONFIG_FATFS_LFN_NONE=n
CONFIG_FATFS_LFN_STACK=n
CONFIG_EXAMPLE_FATFS_DEFAULT_DATETIME=y
CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300

1
examples/storage/fatfsgen/sdkconfig.ci.test_read_write_partition_gen
View File

@ -2,3 +2,4 @@ CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY=n
CONFIG_FATFS_LFN_HEAP=n
CONFIG_FATFS_LFN_STACK=n
CONFIG_FATFS_LFN_NONE=y
CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300

1
examples/storage/fatfsgen/sdkconfig.ci.test_read_write_partition_gen_default_dt
View File

@ -3,3 +3,4 @@ CONFIG_FATFS_LFN_HEAP=n
CONFIG_FATFS_LFN_STACK=n
CONFIG_FATFS_LFN_NONE=y
CONFIG_EXAMPLE_FATFS_DEFAULT_DATETIME=y
CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300

1
examples/storage/fatfsgen/sdkconfig.ci.test_read_write_partition_gen_ln
View File

@ -2,3 +2,4 @@ CONFIG_EXAMPLE_FATFS_MODE_READ_ONLY=n
CONFIG_FATFS_LFN_HEAP=y
CONFIG_FATFS_LFN_NONE=n
CONFIG_FATFS_LFN_STACK=n
CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300

1
examples/storage/fatfsgen/sdkconfig.ci.test_read_write_partition_gen_ln_default_dt
View File

@ -3,3 +3,4 @@ CONFIG_FATFS_LFN_HEAP=y
CONFIG_FATFS_LFN_NONE=n
CONFIG_FATFS_LFN_STACK=n
CONFIG_EXAMPLE_FATFS_DEFAULT_DATETIME=y
CONFIG_EXAMPLE_FATFS_WRITE_COUNT=300

1
tools/ci/executable-list.txt
View File

@ -8,6 +8,7 @@ components/espcoredump/test/test_espcoredump.py
components/espcoredump/test/test_espcoredump.sh
components/espcoredump/test_apps/build_espcoredump.sh
components/fatfs/fatfsgen.py
components/fatfs/fatfsparse.py
components/fatfs/test_fatfsgen/test_fatfsgen.py
components/fatfs/test_fatfsgen/test_fatfsparse.py
components/fatfs/test_fatfsgen/test_wl_fatfsgen.py