alex/esp-idf
alex/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf.git synced 2024-10-05 20:47:46 -04:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

fatfsgen.py: enabled automatic detection of the FATFS type for FAT12 and FAT16

Browse Source
This commit is contained in:
Martin Gaňo 2021-12-06 18:17:20 +01:00
parent 6ddf2ea05e
commit f3425dea96
13 changed files with 390 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
components/fatfs/Kconfig
View File

@ -7,6 +7,46 @@ menu "FAT Filesystem support"
help
Number of volumes (logical drives) to use.
choice FATFS_SECTOR_SIZE
prompt "Sector size"
default FATFS_SECTOR_4096
help
Specify the size of the sector in bytes for FATFS partition generator.
config FATFS_SECTOR_512
bool "512"
config FATFS_SECTOR_1024
bool "1024"
config FATFS_SECTOR_2048
bool "2048"
config FATFS_SECTOR_4096
bool "4096"
endchoice
choice FATFS_SECTORS_PER_CLUSTER
prompt "Sectors per cluster"
default FATFS_SECTORS_PER_CLUSTER_1
help
This value specifies how many sectors there are in one cluster.
config FATFS_SECTORS_PER_CLUSTER_1
bool "1"
config FATFS_SECTORS_PER_CLUSTER_2
bool "2"
config FATFS_SECTORS_PER_CLUSTER_4
bool "4"
config FATFS_SECTORS_PER_CLUSTER_8
bool "8"
config FATFS_SECTORS_PER_CLUSTER_16
bool "16"
config FATFS_SECTORS_PER_CLUSTER_32
bool "32"
config FATFS_SECTORS_PER_CLUSTER_64
bool "64"
config FATFS_SECTORS_PER_CLUSTER_128
bool "128"
endchoice
choice FATFS_CHOOSE_CODEPAGE
prompt "OEM Code Page"
default FATFS_CODEPAGE_437
@ -64,6 +104,21 @@ menu "FAT Filesystem support"
endchoice
choice FATFS_CHOOSE_TYPE
prompt "FAT type"
default FATFS_AUTO_TYPE
help
If user specifies automatic detection of the FAT type,
the FATFS generator will determine the type by the size.
config FATFS_AUTO_TYPE
bool "Select a suitable FATFS type automatically."
config FATFS_FAT12
bool "FAT12"
config FATFS_FAT16
bool "FAT16"
endchoice
config FATFS_CODEPAGE
int
default 0 if FATFS_CODEPAGE_DYNAMIC
@ -96,7 +151,7 @@ menu "FAT Filesystem support"
help
Support long filenames in FAT. Long filename data increases
memory usage. FATFS can be configured to store the buffer for
long filename data in stack or heap.
long filename data in stack or heap (Currently not supported by FATFS partition generator).
config FATFS_LFN_NONE
bool "No long filenames"

87
components/fatfs/fatfsgen.py
View File

@ -5,11 +5,12 @@
import os
from typing import Any, List, Optional
from construct import Const, Int8ul, Int16ul, Int32ul, PaddedString, Struct
from fatfsgen_utils.fat import FAT
from fatfsgen_utils.fatfs_parser import FATFSParser
from fatfsgen_utils.fatfs_state import FATFSState
from fatfsgen_utils.fs_object import Directory
from fatfsgen_utils.utils import generate_4bytes_random, get_args_for_partition_generator, pad_string
from fatfsgen_utils.utils import (BYTES_PER_DIRECTORY_ENTRY, FAT32, generate_4bytes_random,
get_args_for_partition_generator, pad_string, read_filesystem)
class FATFS:
@ -17,35 +18,6 @@ class FATFS:
The class FATFS provides API for generating FAT file system.
It contains reference to the FAT table and to the root directory.
"""
MAX_VOL_LAB_SIZE = 11
MAX_OEM_NAME_SIZE = 8
MAX_FS_TYPE_SIZE = 8
BOOT_HEADER_SIZE = 512
BOOT_SECTOR_HEADER = Struct(
'BS_jmpBoot' / Const(b'\xeb\xfe\x90'),
'BS_OEMName' / PaddedString(MAX_OEM_NAME_SIZE, 'utf-8'),
'BPB_BytsPerSec' / Int16ul,
'BPB_SecPerClus' / Int8ul,
'BPB_RsvdSecCnt' / Int16ul,
'BPB_NumFATs' / Int8ul,
'BPB_RootEntCnt' / Int16ul,
'BPB_TotSec16' / Int16ul,
'BPB_Media' / Int8ul,
'BPB_FATSz16' / Int16ul,
'BPB_SecPerTrk' / Int16ul,
'BPB_NumHeads' / Int16ul,
'BPB_HiddSec' / Int32ul,
'BPB_TotSec32' / Int32ul,
'BS_DrvNum' / Const(b'\x80'),
'BS_Reserved1' / Const(b'\x00'),
'BS_BootSig' / Const(b'\x29'),
'BS_VolID' / Int32ul,
'BS_VolLab' / PaddedString(MAX_VOL_LAB_SIZE, 'utf-8'),
'BS_FilSysType' / PaddedString(MAX_FS_TYPE_SIZE, 'utf-8'),
'BS_EMPTY' / Const(448 * b'\x00'),
'Signature_word' / Const(b'\x55\xAA')
)
def __init__(self,
binary_image_path: Optional[str] = None,
@ -55,7 +27,6 @@ class FATFS:
sectors_per_cluster: int = 1,
sector_size: int = 0x1000,
sectors_per_fat: int = 1,
root_dir_sectors_cnt: int = 4,
hidden_sectors: int = 0,
long_names_enabled: bool = False,
entry_size: int = 32,
@ -64,10 +35,18 @@ class FATFS:
sec_per_track: int = 0x3f,
volume_label: str = 'Espressif',
file_sys_type: str = 'FAT',
root_entry_count: int = 512,
explicit_fat_type: int = None,
media_type: int = 0xf8) -> None:
assert (root_entry_count * BYTES_PER_DIRECTORY_ENTRY) % sector_size == 0
assert ((root_entry_count * BYTES_PER_DIRECTORY_ENTRY) // sector_size) % 2 == 0
root_dir_sectors_cnt = (root_entry_count * BYTES_PER_DIRECTORY_ENTRY) // sector_size
self.state = FATFSState(entry_size=entry_size,
sector_size=sector_size,
explicit_fat_type=explicit_fat_type,
reserved_sectors_cnt=reserved_sectors_cnt,
root_dir_sectors_cnt=root_dir_sectors_cnt,
size=size,
@ -83,13 +62,13 @@ class FATFS:
volume_label=volume_label,
oem_name=oem_name)
binary_image = bytearray(
self.read_filesystem(binary_image_path) if binary_image_path else self.create_empty_fatfs())
read_filesystem(binary_image_path) if binary_image_path else self.create_empty_fatfs())
self.state.binary_image = binary_image
self.fat = FAT(fatfs_state=self.state,
reserved_sectors_cnt=self.state.reserved_sectors_cnt)
self.root_directory = Directory(name='A', # the name is not important
self.root_directory = Directory(name='A', # the name is not important, must be string
size=self.state.root_dir_sectors_cnt * self.state.sector_size,
fat=self.fat,
cluster=self.fat.clusters[1],
@ -105,6 +84,7 @@ class FATFS:
parent_dir = self.root_directory
if path_from_root:
parent_dir = self.root_directory.recursive_search(path_from_root, self.root_directory)
self.root_directory.new_directory(name=name, parent=parent_dir, path_from_root=path_from_root)
def write_content(self, path_from_root: List[str], content: bytes) -> None:
@ -117,36 +97,31 @@ class FATFS:
sectors_count = self.state.size // self.state.sector_size
volume_uuid = generate_4bytes_random()
return (
FATFS.BOOT_SECTOR_HEADER.build(
dict(BS_OEMName=pad_string(self.state.oem_name, size=FATFS.MAX_OEM_NAME_SIZE),
BPB_BytsPerSec=self.state.sectors_per_cluster * self.state.sector_size,
FATFSParser.BOOT_SECTOR_HEADER.build(
dict(BS_OEMName=pad_string(self.state.oem_name, size=FATFSParser.MAX_OEM_NAME_SIZE),
BPB_BytsPerSec=self.state.sector_size,
BPB_SecPerClus=self.state.sectors_per_cluster,
BPB_RsvdSecCnt=self.state.reserved_sectors_cnt,
BPB_NumFATs=self.state.fat_tables_cnt,
BPB_RootEntCnt=self.state.entries_root_count,
BPB_TotSec16=0x00 if self.state.fatfs_type == FATFSState.FAT32 else sectors_count,
BPB_TotSec16=0x00 if self.state.fatfs_type == FAT32 else sectors_count,
BPB_Media=self.state.media_type,
BPB_FATSz16=self.state.sectors_per_fat_cnt,
BPB_SecPerTrk=self.state.sec_per_track,
BPB_NumHeads=self.state.num_heads,
BPB_HiddSec=self.state.hidden_sectors,
BPB_TotSec32=sectors_count if self.state.fatfs_type == FATFSState.FAT32 else 0x00,
BPB_TotSec32=sectors_count if self.state.fatfs_type == FAT32 else 0x00,
BS_VolID=volume_uuid,
BS_VolLab=pad_string(self.state.volume_label, size=FATFS.MAX_VOL_LAB_SIZE),
BS_FilSysType=pad_string(self.state.file_sys_type, size=FATFS.MAX_FS_TYPE_SIZE)
BS_VolLab=pad_string(self.state.volume_label, size=FATFSParser.MAX_VOL_LAB_SIZE),
BS_FilSysType=pad_string(self.state.file_sys_type, size=FATFSParser.MAX_FS_TYPE_SIZE)
)
)
+ (self.state.sector_size - FATFS.BOOT_HEADER_SIZE) * b'\x00'
+ (self.state.sector_size - FATFSParser.BOOT_HEADER_SIZE) * b'\x00'
+ self.state.sectors_per_fat_cnt * self.state.fat_tables_cnt * self.state.sector_size * b'\x00'
+ self.state.root_dir_sectors_cnt * self.state.sector_size * b'\x00'
+ self.state.data_sectors * self.state.sector_size * b'\xff'
)
@staticmethod
def read_filesystem(path: str) -> bytearray:
with open(path, 'rb') as fs_file:
return bytearray(fs_file.read())
def write_filesystem(self, output_path: str) -> None:
with open(output_path, 'wb') as output:
output.write(bytearray(self.state.binary_image))
@ -190,13 +165,17 @@ class FATFS:
self._generate_partition_from_folder(folder_name, folder_path=path_to_folder, is_dir=True)
if __name__ == '__main__':
def main() -> None:
args = get_args_for_partition_generator('Create a FAT filesystem and populate it with directory content')
input_dir = args.input_directory
fatfs = FATFS(sector_size=args.sector_size,
sectors_per_cluster=args.sectors_per_cluster,
size=args.partition_size,
root_entry_count=args.root_entry_count,
explicit_fat_type=args.fat_type)
partition_size = int(str(args.partition_size), 0)
sector_size_bytes = int(str(args.sector_size), 0)
fatfs = FATFS(size=partition_size, sector_size=sector_size_bytes)
fatfs.generate(input_dir)
fatfs.generate(args.input_directory)
fatfs.write_filesystem(args.output_file)
if __name__ == '__main__':
main()

34
components/fatfs/fatfsgen_utils/cluster.py
View File

@ -1,10 +1,13 @@
# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
from typing import Optional
from construct import Int16ul
from .fatfs_state import FATFSState
from .utils import build_byte, clean_first_half_byte, clean_second_half_byte, split_by_half_byte_12_bit_little_endian
from .utils import (FAT12, FAT16, build_byte, clean_first_half_byte, clean_second_half_byte,
split_by_half_byte_12_bit_little_endian)
class Cluster:
@ -13,7 +16,10 @@ class Cluster:
"""
RESERVED_BLOCK_ID = 0
ROOT_BLOCK_ID = 1
ALLOCATED_BLOCK_VALUE = 0xFFF # for fat 12
ALLOCATED_BLOCK_FAT12 = 0xFFF
ALLOCATED_BLOCK_FAT16 = 0xFFFF
ALLOCATED_BLOCK_SWITCH = {FAT12: ALLOCATED_BLOCK_FAT12, FAT16: ALLOCATED_BLOCK_FAT16}
INITIAL_BLOCK_SWITCH = {FAT12: 0xFF8, FAT16: 0xFFF8}
def __init__(self,
cluster_id: int,
@ -26,7 +32,7 @@ class Cluster:
self._next_cluster = None # type: Optional[Cluster]
if self.id == Cluster.RESERVED_BLOCK_ID:
self.is_empty = False
self.set_in_fat(0xff8)
self.set_in_fat(self.INITIAL_BLOCK_SWITCH[self.fatfs_state.fatfs_type])
return
self.cluster_data_address = self._compute_cluster_data_address()
@ -86,14 +92,16 @@ class Cluster:
assert value <= (1 << self.fatfs_state.fatfs_type) - 1
half_bytes = split_by_half_byte_12_bit_little_endian(value)
# hardcoded for fat 12
# IDF-4046 will extend it for fat 16
if self.fat_cluster_address % 8 == 0:
self.fatfs_state.binary_image[self.real_cluster_address] = build_byte(half_bytes[1], half_bytes[0])
self._set_second_half_byte(self.real_cluster_address + 1, half_bytes[2])
elif self.fat_cluster_address % 8 != 0:
self._set_first_half_byte(self.real_cluster_address, half_bytes[0])
self.fatfs_state.binary_image[self.real_cluster_address + 1] = build_byte(half_bytes[2], half_bytes[1])
if self.fatfs_state.fatfs_type == FAT12:
if self.fat_cluster_address % 8 == 0:
self.fatfs_state.binary_image[self.real_cluster_address] = build_byte(half_bytes[1], half_bytes[0])
self._set_second_half_byte(self.real_cluster_address + 1, half_bytes[2])
elif self.fat_cluster_address % 8 != 0:
self._set_first_half_byte(self.real_cluster_address, half_bytes[0])
self.fatfs_state.binary_image[self.real_cluster_address + 1] = build_byte(half_bytes[2], half_bytes[1])
elif self.fatfs_state.fatfs_type == FAT16:
self.fatfs_state.binary_image[self.real_cluster_address:self.real_cluster_address + 2] = Int16ul.build(
value)
@property
def is_root(self) -> bool:
@ -104,7 +112,7 @@ class Cluster:
This method sets bits in FAT table to `allocated` and clean the corresponding sector(s)
"""
self.is_empty = False
self.set_in_fat(Cluster.ALLOCATED_BLOCK_VALUE)
self.set_in_fat(self.ALLOCATED_BLOCK_SWITCH[self.fatfs_state.fatfs_type])
cluster_start = self.cluster_data_address
dir_size = self.fatfs_state.get_dir_size(self.is_root)

6
components/fatfs/fatfsgen_utils/exceptions.py
View File

@ -1,4 +1,4 @@
# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
class WriteDirectoryException(Exception):
@ -38,3 +38,7 @@ class WLNotInitialized(Exception):
class FatalError(Exception):
pass
class InconsistentFATAttributes(Exception):
pass

4
components/fatfs/fatfsgen_utils/fat.py
View File

@ -1,4 +1,4 @@
# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
from .cluster import Cluster
@ -18,7 +18,7 @@ class FAT:
self.reserved_sectors_cnt = reserved_sectors_cnt
self.clusters = [Cluster(cluster_id=i, fatfs_state=self.fatfs_state) for i in
range(1, self.fatfs_state.max_clusters)]
range(1, self.fatfs_state.clusters)]
# update root directory record
self.clusters[0].allocate_cluster()

67
components/fatfs/fatfsgen_utils/fatfs_parser.py Normal file
View File

@ -0,0 +1,67 @@
# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
from construct import Const, Int8ul, Int16ul, Int32ul, PaddedString, Struct
from .utils import (BYTES_PER_DIRECTORY_ENTRY, get_fatfs_type, get_non_data_sectors_cnt, number_of_clusters,
read_filesystem)
class FATFSParser:
MAX_VOL_LAB_SIZE = 11
MAX_OEM_NAME_SIZE = 8
MAX_FS_TYPE_SIZE = 8
# the FAT specification defines 512 bytes for the boot sector header
BOOT_HEADER_SIZE = 512
BOOT_SECTOR_HEADER = Struct(
'BS_jmpBoot' / Const(b'\xeb\xfe\x90'),
'BS_OEMName' / PaddedString(MAX_OEM_NAME_SIZE, 'utf-8'),
'BPB_BytsPerSec' / Int16ul,
'BPB_SecPerClus' / Int8ul,
'BPB_RsvdSecCnt' / Int16ul,
'BPB_NumFATs' / Int8ul,
'BPB_RootEntCnt' / Int16ul,
'BPB_TotSec16' / Int16ul,
'BPB_Media' / Int8ul,
'BPB_FATSz16' / Int16ul,
'BPB_SecPerTrk' / Int16ul,
'BPB_NumHeads' / Int16ul,
'BPB_HiddSec' / Int32ul,
'BPB_TotSec32' / Int32ul,
'BS_DrvNum' / Const(b'\x80'),
'BS_Reserved1' / Const(b'\x00'),
'BS_BootSig' / Const(b'\x29'),
'BS_VolID' / Int32ul,
'BS_VolLab' / PaddedString(MAX_VOL_LAB_SIZE, 'utf-8'),
'BS_FilSysType' / PaddedString(MAX_FS_TYPE_SIZE, 'utf-8'),
'BS_EMPTY' / Const(448 * b'\x00'),
'Signature_word' / Const(b'\x55\xAA')
)
assert BOOT_SECTOR_HEADER.sizeof() == BOOT_HEADER_SIZE
def __init__(self, image_file_path: str, wl_support: bool = False) -> None:
if wl_support:
raise NotImplementedError('Parser is not implemented for WL yet.')
self.fatfs = read_filesystem(image_file_path)
# when wl is not supported we expect boot sector to be the first
self.parsed_header = FATFSParser.BOOT_SECTOR_HEADER.parse(self.fatfs[:FATFSParser.BOOT_HEADER_SIZE])
def print(self) -> None:
print('Properties of the FATFS:')
for key in self.parsed_header.keys():
if key in ('_io', 'BS_EMPTY', 'Signature_word'):
continue
print(' {}: {}'.format(key.replace('BPB_', '').replace('BS_', ''), self.parsed_header[key]))
root_dir_cnt = (self.parsed_header['BPB_RootEntCnt'] * BYTES_PER_DIRECTORY_ENTRY) // self.parsed_header[
'BPB_BytsPerSec']
non_data_sectors = get_non_data_sectors_cnt(self.parsed_header['BPB_RsvdSecCnt'],
# this has to be changed when FAT32 is supported
self.parsed_header['BPB_FATSz16'], root_dir_cnt)
data_clusters = self.parsed_header['BPB_TotSec16'] - non_data_sectors
clusters_num = number_of_clusters(data_clusters, self.parsed_header['BPB_SecPerClus'])
assert self.parsed_header['BPB_BytsPerSec'] in (512, 1024, 2048, 4096)
assert self.parsed_header['BPB_SecPerClus'] in (1, 2, 4, 8, 16, 32, 64, 128)
print(f' Clusters number: {clusters_num}')
print(f' FATFS type: FAT{get_fatfs_type(clusters_num)}')

47
components/fatfs/fatfsgen_utils/fatfs_state.py
View File

@ -1,16 +1,17 @@
# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
from textwrap import dedent
from .exceptions import InconsistentFATAttributes
from .utils import (FAT12, FAT12_MAX_CLUSTERS, FAT16, FAT16_MAX_CLUSTERS, get_fatfs_type, get_non_data_sectors_cnt,
number_of_clusters)
class FATFSState:
"""
The class represents the state and the configuration of the FATFS.
"""
FAT12_MAX_CLUSTERS = 4085
FAT16_MAX_CLUSTERS = 65525
FAT12 = 12
FAT16 = 16
FAT32 = 32
def __init__(self,
entry_size: int,
@ -28,7 +29,10 @@ class FATFSState:
num_heads: int,
hidden_sectors: int,
file_sys_type: str,
explicit_fat_type: int = None,
long_names_enabled: bool = False):
self._explicit_fat_type = explicit_fat_type
self._binary_image: bytearray = bytearray(b'')
self.fat_tables_cnt: int = fat_tables_cnt
self.oem_name: str = oem_name
@ -47,6 +51,11 @@ class FATFSState:
self.sectors_per_fat_cnt: int = sectors_per_fat
self.sectors_per_cluster: int = sectors_per_cluster
if self.clusters in (FAT12_MAX_CLUSTERS, FAT16_MAX_CLUSTERS):
print('WARNING: It is not recommended to create FATFS with bounding '
f'count of clusters: {FAT12_MAX_CLUSTERS} or {FAT16_MAX_CLUSTERS}')
self.check_fat_type()
@property
def binary_image(self) -> bytearray:
return self._binary_image
@ -68,28 +77,36 @@ class FATFSState:
@property
def non_data_sectors(self) -> int:
return self.reserved_sectors_cnt + self.sectors_per_fat_cnt + self.root_dir_sectors_cnt
return get_non_data_sectors_cnt(self.reserved_sectors_cnt, self.sectors_per_fat_cnt, # type: ignore
self.root_dir_sectors_cnt)
@property
def data_region_start(self) -> int:
return self.non_data_sectors * self.sector_size
@property
def max_clusters(self) -> int:
return self.data_sectors // self.sectors_per_cluster
def clusters(self) -> int:
return number_of_clusters(self.data_sectors, self.sectors_per_cluster) # type: ignore
@property
def root_directory_start(self) -> int:
return (self.reserved_sectors_cnt + self.sectors_per_fat_cnt) * self.sector_size
def check_fat_type(self) -> None:
_type = self.fatfs_type
if self._explicit_fat_type is not None and self._explicit_fat_type != _type:
raise InconsistentFATAttributes(dedent(
f"""FAT type you specified is inconsistent with other attributes of the system.
The specified FATFS type: FAT{self._explicit_fat_type}
The actual FATFS type: FAT{_type}"""))
if _type not in (FAT12, FAT16):
raise NotImplementedError('FAT32 is currently not supported.')
@property
def fatfs_type(self) -> int:
if self.max_clusters < FATFSState.FAT12_MAX_CLUSTERS:
return FATFSState.FAT12
elif self.max_clusters < FATFSState.FAT16_MAX_CLUSTERS:
return FATFSState.FAT16
# fat is FAT.FAT32, not supported now
raise NotImplementedError('FAT32 is currently not supported.')
# variable typed_fatfs_type must be explicitly typed to avoid mypy error
typed_fatfs_type: int = get_fatfs_type(self.clusters)
return typed_fatfs_type
@property
def entries_root_count(self) -> int:

50
components/fatfs/fatfsgen_utils/utils.py
View File

@ -9,6 +9,13 @@ from typing import List, Optional, Tuple
from construct import Int16ul
FAT12_MAX_CLUSTERS: int = 4085
FAT16_MAX_CLUSTERS: int = 65525
FAT12: int = 12
FAT16: int = 16
FAT32: int = 32
BYTES_PER_DIRECTORY_ENTRY = 32
def crc32(input_values: List[int], crc: int) -> int:
"""
@ -18,6 +25,22 @@ def crc32(input_values: List[int], crc: int) -> int:
return binascii.crc32(bytearray(input_values), crc)
def number_of_clusters(number_of_sectors: int, sectors_per_cluster: int) -> int:
return number_of_sectors // sectors_per_cluster
def get_non_data_sectors_cnt(reserved_sectors_cnt: int, sectors_per_fat_cnt: int, root_dir_sectors_cnt: int) -> int:
return reserved_sectors_cnt + sectors_per_fat_cnt + root_dir_sectors_cnt
def get_fatfs_type(clusters_count: int) -> int:
if clusters_count < FAT12_MAX_CLUSTERS:
return FAT12
if clusters_count < FAT16_MAX_CLUSTERS:
return FAT16
return FAT32
def required_clusters_count(cluster_size: int, content: bytes) -> int:
# compute number of required clusters for file text
return (len(content) + cluster_size - 1) // cluster_size
@ -88,8 +111,35 @@ def get_args_for_partition_generator(desc: str) -> argparse.Namespace:
help='Size of the partition in bytes')
parser.add_argument('--sector_size',
default=4096,
type=int,
choices=[512, 1024, 2048, 4096],
help='Size of the partition in bytes')
parser.add_argument('--sectors_per_cluster',
default=1,
type=int,
choices=[1, 2, 4, 8, 16, 32, 64, 128],
help='Number of sectors per cluster')
parser.add_argument('--root_entry_count',
default=512,
help='Number of entries in the root directory')
parser.add_argument('--fat_type',
default=0,
type=int,
choices=[12, 16, 0],
help="""
Type of fat. Select 12 for fat12, 16 for fat16. Don't set, or set to 0 for automatic
calculation using cluster size and partition size.
""")
args = parser.parse_args()
if args.fat_type == 0:
args.fat_type = None
args.partition_size = int(str(args.partition_size), 0)
if not os.path.isdir(args.input_directory):
raise NotADirectoryError(f'The target directory `{args.input_directory}` does not exist!')
return args
def read_filesystem(path: str) -> bytearray:
with open(path, 'rb') as fs_file:
return bytearray(fs_file.read())

40
components/fatfs/project_include.cmake
View File

@ -16,6 +16,43 @@ function(fatfs_create_partition_image partition base_dir)
set(fatfsgen_py ${python} ${idf_path}/components/fatfs/fatfsgen.py)
endif()
if("${CONFIG_FATFS_SECTORS_PER_CLUSTER_1}")
set(sectors_per_cluster 1)
elseif("${CONFIG_FATFS_SECTORS_PER_CLUSTER_2}")
set(sectors_per_cluster 2)
elseif("${CONFIG_FATFS_SECTORS_PER_CLUSTER_4}")
set(sectors_per_cluster 4)
elseif("${CONFIG_FATFS_SECTORS_PER_CLUSTER_8}")
set(sectors_per_cluster 8)
elseif("${CONFIG_FATFS_SECTORS_PER_CLUSTER_16}")
set(sectors_per_cluster 16)
elseif("${CONFIG_FATFS_SECTORS_PER_CLUSTER_32}")
set(sectors_per_cluster 32)
elseif("${CONFIG_FATFS_SECTORS_PER_CLUSTER_64}")
set(sectors_per_cluster 64)
elseif("${CONFIG_FATFS_SECTORS_PER_CLUSTER_128}")
set(sectors_per_cluster 128)
endif()
if("${CONFIG_FATFS_SECTOR_512}")
set(fatfs_sector_size 512)
elseif("${CONFIG_FATFS_SECTOR_1024}")
set(fatfs_sector_size 1024)
elseif("${CONFIG_FATFS_SECTOR_2048}")
set(fatfs_sector_size 2048)
else()
set(fatfs_sector_size 4096)
endif()
if("${CONFIG_FATFS_AUTO_TYPE}")
set(fatfs_explicit_type 0)
elseif("${CONFIG_FATFS_FAT12}")
set(fatfs_explicit_type 12)
elseif("${CONFIG_FATFS_FAT16}")
set(fatfs_explicit_type 16)
endif()
get_filename_component(base_dir_full_path ${base_dir} ABSOLUTE)
partition_table_get_partition_info(size "--partition-name ${partition}" "size")
partition_table_get_partition_info(offset "--partition-name ${partition}" "offset")
@ -28,6 +65,9 @@ function(fatfs_create_partition_image partition base_dir)
COMMAND ${fatfsgen_py} ${base_dir_full_path}
--partition_size ${size}
--output_file ${image_file}
--sector_size "${fatfs_sector_size}"
--sectors_per_cluster "${sectors_per_cluster}"
--fat_type "${fatfs_explicit_type}"
)
set_property(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" APPEND PROPERTY

79
components/fatfs/test_fatfsgen/test_fatfsgen.py
View File

@ -7,12 +7,14 @@ import shutil
import sys
import unittest
from test_utils import CFG, generate_test_dir_1, generate_test_dir_2
from test_utils import CFG, fill_sector, generate_test_dir_1, generate_test_dir_2
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
import fatfsgen # noqa E402
from fatfsgen_utils.exceptions import WriteDirectoryException # noqa E402
from fatfsgen_utils.exceptions import LowerCaseException, NoFreeClusterException, TooLongNameException # noqa E402
import fatfsgen # noqa E402 # pylint: disable=C0413
from fatfsgen_utils.exceptions import TooLongNameException # noqa E402 # pylint: disable=C0413
from fatfsgen_utils.exceptions import WriteDirectoryException # noqa E402 # pylint: disable=C0413
from fatfsgen_utils.exceptions import LowerCaseException, NoFreeClusterException # noqa E402 # pylint: disable=C0413
from fatfsgen_utils.utils import read_filesystem # noqa E402 # pylint: disable=C0413
class FatFSGen(unittest.TestCase):
@ -27,8 +29,9 @@ class FatFSGen(unittest.TestCase):
def test_empty_file_sn_fat12(self) -> None:
fatfs = fatfsgen.FATFS()
fatfs.create_file('TESTFILE')
fatfs.write_filesystem(CFG['output_file'])
file_system = fatfs.read_filesystem(CFG['output_file'])
file_system = read_filesystem(CFG['output_file'])
self.assertEqual(file_system[0x2000:0x200c], b'TESTFILE \x20') # check entry name and type
self.assertEqual(file_system[0x1000:0x1006], b'\xf8\xff\xff\xff\x0f\x00') # check fat
@ -37,7 +40,7 @@ class FatFSGen(unittest.TestCase):
fatfs = fatfsgen.FATFS()
fatfs.create_directory('TESTFOLD')
fatfs.write_filesystem(CFG['output_file'])
file_system = fatfs.read_filesystem(CFG['output_file'])
file_system = read_filesystem(CFG['output_file'])
self.assertEqual(file_system[0x2000:0x200c], b'TESTFOLD \x10') # check entry name and type
self.assertEqual(file_system[0x1000:0x1006], b'\xf8\xff\xff\xff\x0f\x00') # check fat
@ -48,7 +51,7 @@ class FatFSGen(unittest.TestCase):
fatfs = fatfsgen.FATFS()
fatfs.create_file('TESTF', extension='TXT')
fatfs.write_filesystem(CFG['output_file'])
file_system = fatfs.read_filesystem(CFG['output_file'])
file_system = read_filesystem(CFG['output_file'])
self.assertEqual(file_system[0x2000:0x200c], b'TESTF TXT\x20') # check entry name and type
self.assertEqual(file_system[0x1000:0x1006], b'\xf8\xff\xff\xff\x0f\x00') # check fat
@ -57,7 +60,7 @@ class FatFSGen(unittest.TestCase):
fatfs.create_file('WRITEF', extension='TXT')
fatfs.write_content(path_from_root=['WRITEF.TXT'], content=b'testcontent')
fatfs.write_filesystem(CFG['output_file'])
file_system = fatfs.read_filesystem(CFG['output_file'])
file_system = read_filesystem(CFG['output_file'])
self.assertEqual(file_system[0x2000:0x200c], b'WRITEF TXT\x20') # check entry name and type
self.assertEqual(file_system[0x201a:0x2020], b'\x02\x00\x0b\x00\x00\x00') # check size and cluster ref
@ -70,7 +73,7 @@ class FatFSGen(unittest.TestCase):
fatfs.create_file('WRITEF', extension='TXT', path_from_root=['TESTFOLD'])
fatfs.write_content(path_from_root=['TESTFOLD', 'WRITEF.TXT'], content=b'testcontent')
fatfs.write_filesystem(CFG['output_file'])
file_system = fatfs.read_filesystem(CFG['output_file'])
file_system = read_filesystem(CFG['output_file'])
self.assertEqual(file_system[0x2000:0x200c], b'TESTFOLD \x10')
self.assertEqual(
@ -91,7 +94,7 @@ class FatFSGen(unittest.TestCase):
fatfs.fat.clusters[3].set_in_fat(4)
fatfs.fat.clusters[4].set_in_fat(5)
fatfs.write_filesystem(CFG['output_file'])
file_system = fatfs.read_filesystem(CFG['output_file'])
file_system = read_filesystem(CFG['output_file'])
self.assertEqual(
file_system[0x1000:0x1010],
b'\xf8\xff\xff\xe8\x43\x00\x05\xf0\xff\xff\x0f\x00\x00\x00\x00\x00')
@ -101,7 +104,7 @@ class FatFSGen(unittest.TestCase):
fatfs.create_file('WRITEF', extension='TXT')
fatfs.write_content(path_from_root=['WRITEF.TXT'], content=CFG['sector_size'] * b'a')
fatfs.write_filesystem(CFG['output_file'])
file_system = fatfs.read_filesystem(CFG['output_file'])
file_system = read_filesystem(CFG['output_file'])
self.assertEqual(file_system[0x1000: 0x100e], b'\xf8\xff\xff\xff\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00')
self.assertEqual(file_system[0x6000: 0x7000], CFG['sector_size'] * b'a')
@ -110,7 +113,7 @@ class FatFSGen(unittest.TestCase):
fatfs.create_file('WRITEF', extension='TXT')
fatfs.write_content(path_from_root=['WRITEF.TXT'], content=CFG['sector_size'] * b'a' + b'a')
fatfs.write_filesystem(CFG['output_file'])
file_system = fatfs.read_filesystem(CFG['output_file'])
file_system = read_filesystem(CFG['output_file'])
self.assertEqual(file_system[0x1000: 0x100e], b'\xf8\xff\xff\x03\xf0\xff\x00\x00\x00\x00\x00\x00\x00\x00')
self.assertEqual(file_system[0x7000: 0x8000], b'a' + (CFG['sector_size'] - 1) * b'\x00')
@ -118,12 +121,11 @@ class FatFSGen(unittest.TestCase):
fatfs = fatfsgen.FATFS()
fatfs.create_directory('TESTFOLD')
for i in range(CFG['sector_size'] // CFG['entry_size']):
fatfs.create_file(f'A{str(i).upper()}', path_from_root=['TESTFOLD'])
fill_sector(fatfs)
fatfs.write_content(path_from_root=['TESTFOLD', 'A0'], content=b'first')
fatfs.write_content(path_from_root=['TESTFOLD', 'A126'], content=b'later')
fatfs.write_filesystem(CFG['output_file'])
file_system = fatfs.read_filesystem(CFG['output_file'])
file_system = read_filesystem(CFG['output_file'])
self.assertEqual(file_system[0x1000: 0x10d0],
b'\xf8\xff\xff\x82\xf0\xff' + 192 * b'\xff' + 10 * b'\x00')
self.assertEqual(file_system[0x85000:0x85005], b'later')
@ -162,7 +164,7 @@ class FatFSGen(unittest.TestCase):
fatfs.write_content(path_from_root=['TESTFOLD', 'A253'], content=b'later')
fatfs.write_content(path_from_root=['TESTFOLD', 'A255'], content=b'last')
fatfs.write_filesystem(CFG['output_file'])
file_system = fatfs.read_filesystem(CFG['output_file'])
file_system = read_filesystem(CFG['output_file'])
self.assertEqual(file_system[0x105000:0x105010], b'later\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00')
self.assertEqual(file_system[0x108000:0x108010], b'last\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00')
@ -193,7 +195,7 @@ class FatFSGen(unittest.TestCase):
fatfs.create_file('WRITEF', extension='TXT', path_from_root=['TESTFOLD', 'TESTFOLL', 'TESTFOLO'])
fatfs.write_content(path_from_root=['TESTFOLD', 'TESTFOLL', 'TESTFOLO', 'WRITEF.TXT'], content=b'later')
fatfs.write_filesystem(CFG['output_file'])
file_system = fatfs.read_filesystem(CFG['output_file'])
file_system = read_filesystem(CFG['output_file'])
self.assertEqual(file_system[0x9000:0x9010], b'later\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00')
@ -205,7 +207,7 @@ class FatFSGen(unittest.TestCase):
fatfs.create_file('WRITEF', extension='TXT', path_from_root=['TESTFOLD', 'TESTFOLD', 'TESTFOLD'])
fatfs.write_content(path_from_root=['TESTFOLD', 'TESTFOLD', 'TESTFOLD', 'WRITEF.TXT'], content=b'later')
fatfs.write_filesystem(CFG['output_file'])
file_system = fatfs.read_filesystem(CFG['output_file'])
file_system = read_filesystem(CFG['output_file'])
self.assertEqual(file_system[0x2000:0x2010], b'TESTFOLD \x10\x00\x00\x01\x00')
self.assertEqual(file_system[0x2010:0x2020], b'!\x00\x00\x00\x00\x00\x01\x00\x01\x00\x02\x00\x00\x00\x00\x00')
@ -220,7 +222,7 @@ class FatFSGen(unittest.TestCase):
fatfs = fatfsgen.FATFS()
fatfs.generate(CFG['test_dir'])
fatfs.write_filesystem(CFG['output_file'])
file_system = fatfs.read_filesystem(CFG['output_file'])
file_system = read_filesystem(CFG['output_file'])
self.assertEqual(file_system[0x6060:0x6070], b'TESTFIL2 \x00\x00\x01\x00')
self.assertEqual(file_system[0x6070:0x6080], b'!\x00\x00\x00\x00\x00\x01\x00\x01\x00\x05\x00\x0b\x00\x00\x00')
self.assertEqual(file_system[0x7040:0x7050], b'LASTFILE \x00\x00\x01\x00')
@ -232,7 +234,7 @@ class FatFSGen(unittest.TestCase):
fatfs = fatfsgen.FATFS()
fatfs.generate(CFG['test_dir2'])
fatfs.write_filesystem(CFG['output_file'])
file_system = fatfs.read_filesystem(CFG['output_file'])
file_system = read_filesystem(CFG['output_file'])
self.assertEqual(file_system[0x2020:0x2030], b'TESTFILE \x00\x00\x01\x00')
self.assertEqual(file_system[0x6060:0x6070], b'TESTFIL2 \x00\x00\x01\x00')
@ -243,6 +245,43 @@ class FatFSGen(unittest.TestCase):
self.assertEqual(file_system[0xa000:0xa010], b'ahoj\n\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00')
self.assertEqual(file_system[0xb000:0xb009], b'\xff\xff\xff\xff\xff\xff\xff\xff\xff')
def test_empty_fat16(self) -> None:
fatfs = fatfsgen.FATFS(size=17 * 1024 * 1024)
fatfs.write_filesystem(CFG['output_file'])
file_system = read_filesystem(CFG['output_file'])
self.assertEqual(file_system[0x1000:0x1007], b'\xf8\xff\xff\xff\x00\x00\x00')
def test_simple_fat16(self) -> None:
fatfs = fatfsgen.FATFS(size=17 * 1024 * 1024)
fatfs.create_directory('TESTFOLD')
fatfs.write_filesystem(CFG['output_file'])
file_system = read_filesystem(CFG['output_file'])
self.assertEqual(file_system[0x1000:0x1007], b'\xf8\xff\xff\xff\xff\xff\x00')
def test_chaining_fat16(self) -> None:
fatfs = fatfsgen.FATFS(size=17 * 1024 * 1024)
fatfs.create_file('WRITEF', extension='TXT')
fatfs.write_content(path_from_root=['WRITEF.TXT'], content=CFG['sector_size'] * b'a' + b'a')
fatfs.write_filesystem(CFG['output_file'])
file_system = read_filesystem(CFG['output_file'])
self.assertEqual(file_system[0x1000: 0x100e], b'\xf8\xff\xff\xff\x03\x00\xff\xff\x00\x00\x00\x00\x00\x00')
self.assertEqual(file_system[0x7000: 0x8000], b'a' + (CFG['sector_size'] - 1) * b'\x00')
def test_full_sector_folder_fat16(self) -> None:
fatfs = fatfsgen.FATFS(size=17 * 1024 * 1024)
fatfs.create_directory('TESTFOLD')
fill_sector(fatfs)
fatfs.write_content(path_from_root=['TESTFOLD', 'A0'], content=b'first')
fatfs.write_content(path_from_root=['TESTFOLD', 'A126'], content=b'later')
fatfs.write_filesystem(CFG['output_file'])
file_system = read_filesystem(CFG['output_file'])
self.assertEqual(file_system[0x1000: 0x1110],
b'\xf8\xff\xff\xff\x82\x00' + 258 * b'\xff' + 8 * b'\x00')
self.assertEqual(file_system[0x85000:0x85005], b'later')
self.assertEqual(file_system[0x86000:0x86010], b'A126 \x00\x00\x01\x00')
self.assertEqual(file_system[0x86020:0x86030], b'A127 \x00\x00\x01\x00')
if __name__ == '__main__':
unittest.main()

11
components/fatfs/test_fatfsgen/test_utils.py
View File

@ -1,8 +1,12 @@
# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import os
import sys
from typing import Any, Dict, Union
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
import fatfsgen # noqa E402 # pylint: disable=C0413
CFG = dict(
sector_size=4096,
entry_size=32,
@ -33,3 +37,8 @@ def generate_test_dir_2() -> None:
file.write('thisistest\n')
with open(os.path.join(CFG['test_dir2'], 'testfile'), 'w') as file:
file.write('ahoj\n')
def fill_sector(fatfs: fatfsgen.FATFS) -> None:
for i in range(CFG['sector_size'] // CFG['entry_size']):
fatfs.create_file(f'A{str(i).upper()}', path_from_root=['TESTFOLD'])

25
components/fatfs/wl_fatfsgen.py
View File

@ -52,7 +52,7 @@ class WLFATFS:
sectors_per_cluster: int = 1,
sector_size: int = 0x1000,
sectors_per_fat: int = 1,
root_dir_sectors_cnt: int = 4,
explicit_fat_type: int = None,
hidden_sectors: int = 0,
long_names_enabled: bool = False,
entry_size: int = 32,
@ -63,8 +63,9 @@ class WLFATFS:
file_sys_type: str = 'FAT',
version: int = 2,
temp_buff_size: int = 32,
updaterate: int = 16,
update_rate: int = 16,
device_id: int = None,
root_entry_count: int = 512,
media_type: int = 0xf8) -> None:
if sector_size != WLFATFS.WL_SECTOR_SIZE:
raise NotImplementedError(f'The only supported sector size is currently {WLFATFS.WL_SECTOR_SIZE}')
@ -74,7 +75,7 @@ class WLFATFS:
self._version = version
self._temp_buff_size = temp_buff_size
self._device_id = device_id
self._updaterate = updaterate
self._update_rate = update_rate
self.partition_size = size
self.total_sectors = self.partition_size // self.sector_size
self.wl_state_size = WLFATFS.WL_STATE_HEADER_SIZE + WLFATFS.WL_STATE_RECORD_SIZE * self.total_sectors
@ -89,13 +90,14 @@ class WLFATFS:
self.plain_fat_sectors = self.total_sectors - wl_sectors
self.plain_fatfs = FATFS(
explicit_fat_type=explicit_fat_type,
size=self.plain_fat_sectors * self.sector_size,
reserved_sectors_cnt=reserved_sectors_cnt,
fat_tables_cnt=fat_tables_cnt,
sectors_per_cluster=sectors_per_cluster,
sector_size=sector_size,
sectors_per_fat=sectors_per_fat,
root_dir_sectors_cnt=root_dir_sectors_cnt,
root_entry_count=root_entry_count,
hidden_sectors=hidden_sectors,
long_names_enabled=long_names_enabled,
entry_size=entry_size,
@ -127,7 +129,7 @@ class WLFATFS:
full_mem_size=self.partition_size,
page_size=self.sector_size,
sector_size=self.sector_size,
updaterate=self._updaterate,
updaterate=self._update_rate,
wr_size=16,
version=self._version,
temp_buff_size=self._temp_buff_size
@ -149,7 +151,7 @@ class WLFATFS:
max_pos=self.plain_fat_sectors + WLFATFS.DUMMY_SECTORS_COUNT,
move_count=0,
access_count=0,
max_count=self._updaterate,
max_count=self._update_rate,
block_size=self.sector_size,
version=self._version,
device_id=self._device_id or generate_4bytes_random(),
@ -187,12 +189,13 @@ class WLFATFS:
if __name__ == '__main__':
desc = 'Create a FAT filesystem with support for wear levelling and populate it with directory content'
args = get_args_for_partition_generator(desc)
input_dir = args.input_directory
partition_size = int(str(args.partition_size), 0)
sector_size_bytes = int(str(args.sector_size), 0)
wl_fatfs = WLFATFS(sector_size=args.sector_size,
sectors_per_cluster=args.sectors_per_cluster,
size=args.partition_size,
root_entry_count=args.root_entry_count,
explicit_fat_type=args.fat_type)
wl_fatfs = WLFATFS(size=partition_size, sector_size=sector_size_bytes)
wl_fatfs.wl_generate(input_dir)
wl_fatfs.wl_generate(args.input_directory)
wl_fatfs.init_wl()
wl_fatfs.wl_write_filesystem(args.output_file)

2
docs/en/api-reference/storage/fatfs.rst
View File

@ -95,7 +95,7 @@ The tool is used to create filesystem images on a host and populate it with cont
The script is based on the partition generator (:component_file:`fatfsgen.py<fatfs/fatfsgen.py>`) and except for generating partition also initializes wear levelling.
Current implementation supports short file names and FAT12. Long file names, and FAT16 are subjects of the future work.
Current implementation supports short file names, FAT12 and FAT16. Long file names support is the subject of the future work.
Build system integration with FATFS partition generator