esp-idf/components/fatfs/fatfs_utils/utils.py
2024-03-19 09:29:10 +01:00

311 lines
11 KiB
Python

# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import argparse
import binascii
import os
import re
import uuid
from datetime import datetime
from typing import List
from typing import Optional
from typing import Tuple
from construct import BitsInteger
from construct import BitStruct
from construct import Int16ul
# the regex pattern defines symbols that are allowed by long file names but not by short file names
INVALID_SFN_CHARS_PATTERN = re.compile(r'[.+,;=\[\]]')
FATFS_MIN_ALLOC_UNIT: int = 128
FAT12_MAX_CLUSTERS: int = 4085
FAT16_MAX_CLUSTERS: int = 65525
RESERVED_CLUSTERS_COUNT: int = 2
PAD_CHAR: int = 0x20
FAT12: int = 12
FAT16: int = 16
FAT32: int = 32
FULL_BYTE: bytes = b'\xff'
EMPTY_BYTE: bytes = b'\x00'
# redundant
BYTES_PER_DIRECTORY_ENTRY: int = 32
UINT32_MAX: int = (1 << 32) - 1
MAX_NAME_SIZE: int = 8
MAX_EXT_SIZE: int = 3
DATETIME = Tuple[int, int, int]
FATFS_INCEPTION_YEAR: int = 1980
FATFS_INCEPTION: datetime = datetime(FATFS_INCEPTION_YEAR, 1, 1, 0, 0, 0, 0)
FATFS_MAX_HOURS = 24
FATFS_MAX_MINUTES = 60
FATFS_MAX_SECONDS = 60
FATFS_MAX_DAYS = 31
FATFS_MAX_MONTHS = 12
FATFS_MAX_YEARS = 127
FATFS_SECONDS_GRANULARITY: int = 2
# long names are encoded to two bytes in utf-16
LONG_NAMES_ENCODING: str = 'utf-16'
SHORT_NAMES_ENCODING: str = 'utf-8'
# compatible with WL_SECTOR_SIZE
# choices for WL are WL_SECTOR_SIZE_512 and WL_SECTOR_SIZE_4096
ALLOWED_WL_SECTOR_SIZES: List[int] = [512, 4096]
ALLOWED_SECTOR_SIZES: List[int] = [512, 1024, 2048, 4096]
ALLOWED_SECTORS_PER_CLUSTER: List[int] = [1, 2, 4, 8, 16, 32, 64, 128]
def crc32(input_values: List[int], crc: int) -> int:
"""
Name Polynomial Reversed? Init-value XOR-out
crc32 0x104C11DB7 True 4294967295 (UINT32_MAX) 0xFFFFFFFF
"""
return binascii.crc32(bytearray(input_values), crc)
def number_of_clusters(number_of_sectors: int, sectors_per_cluster: int) -> int:
return number_of_sectors // sectors_per_cluster
def get_non_data_sectors_cnt(reserved_sectors_cnt: int, sectors_per_fat_cnt: int, root_dir_sectors_cnt: int) -> int:
return reserved_sectors_cnt + sectors_per_fat_cnt + root_dir_sectors_cnt
def get_fatfs_type(clusters_count: int) -> int:
if clusters_count < FAT12_MAX_CLUSTERS:
return FAT12
if clusters_count <= FAT16_MAX_CLUSTERS:
return FAT16
return FAT32
def get_fat_sectors_count(clusters_count: int, sector_size: int) -> int:
fatfs_type_ = get_fatfs_type(clusters_count)
if fatfs_type_ == FAT32:
raise NotImplementedError('FAT32 is not supported!')
# number of byte halves
cluster_s: int = fatfs_type_ // 4
fat_size_bytes: int = (
clusters_count * 2 + cluster_s) if fatfs_type_ == FAT16 else (clusters_count * 3 + 1) // 2 + cluster_s
return (fat_size_bytes + sector_size - 1) // sector_size
def required_clusters_count(cluster_size: int, content: bytes) -> int:
# compute number of required clusters for file text
return (len(content) + cluster_size - 1) // cluster_size
def generate_4bytes_random() -> int:
return uuid.uuid4().int & 0xFFFFFFFF
def pad_string(content: str, size: Optional[int] = None, pad: int = PAD_CHAR) -> str:
# cut string if longer and fill with pad character if shorter than size
return content.ljust(size or len(content), chr(pad))[:size]
def right_strip_string(content: str, pad: int = PAD_CHAR) -> str:
return content.rstrip(chr(pad))
def build_lfn_short_entry_name(name: str, extension: str, order: int) -> str:
return '{}{}'.format(pad_string(content=name[:MAX_NAME_SIZE - 2] + '~' + chr(order), size=MAX_NAME_SIZE),
pad_string(extension[:MAX_EXT_SIZE], size=MAX_EXT_SIZE))
def lfn_checksum(short_entry_name: str) -> int:
"""
Function defined by FAT specification. Computes checksum out of name in the short file name entry.
"""
checksum_result = 0
for i in range(MAX_NAME_SIZE + MAX_EXT_SIZE):
# operation is a right rotation on 8 bits (Python equivalent for unsigned char in C)
checksum_result = (0x80 if checksum_result & 1 else 0x00) + (checksum_result >> 1) + ord(short_entry_name[i])
checksum_result &= 0xff
return checksum_result
def convert_to_utf16_and_pad(content: str,
expected_size: int,
pad: bytes = FULL_BYTE) -> bytes:
# we need to get rid of the Byte order mark 0xfeff or 0xfffe, fatfs does not use it
bom_utf16: bytes = b'\xfe\xff'
encoded_content_utf16: bytes = content.encode(LONG_NAMES_ENCODING)[len(bom_utf16):]
return encoded_content_utf16.ljust(2 * expected_size, pad)
def split_to_name_and_extension(full_name: str) -> Tuple[str, str]:
name, extension = os.path.splitext(full_name)
return name, extension.replace('.', '')
def is_valid_fatfs_name(string: str) -> bool:
return string == string.upper()
def split_by_half_byte_12_bit_little_endian(value: int) -> Tuple[int, int, int]:
value_as_bytes: bytes = Int16ul.build(value)
return value_as_bytes[0] & 0x0f, value_as_bytes[0] >> 4, value_as_bytes[1] & 0x0f
def merge_by_half_byte_12_bit_little_endian(v1: int, v2: int, v3: int) -> int:
return v1 | v2 << 4 | v3 << 8
def build_byte(first_half: int, second_half: int) -> int:
return (first_half << 4) | second_half
def split_content_into_sectors(content: bytes, sector_size: int) -> List[bytes]:
result = []
clusters_cnt: int = required_clusters_count(cluster_size=sector_size, content=content)
for i in range(clusters_cnt):
result.append(content[sector_size * i:(i + 1) * sector_size])
return result
def get_args_for_partition_generator(desc: str, wl: bool) -> argparse.Namespace:
parser: argparse.ArgumentParser = argparse.ArgumentParser(description=desc)
parser.add_argument('input_directory',
help='Path to the directory that will be encoded into fatfs image')
parser.add_argument('--output_file',
default='fatfs_image.img',
help='Filename of the generated fatfs image')
parser.add_argument('--partition_size',
default=FATDefaults.SIZE,
help='Size of the partition in bytes.' +
('' if wl else ' Use `--partition_size detect` for detecting the minimal partition size.')
)
parser.add_argument('--sector_size',
default=FATDefaults.SECTOR_SIZE,
type=int,
choices=ALLOWED_WL_SECTOR_SIZES if wl else ALLOWED_SECTOR_SIZES,
help='Size of the partition in bytes')
parser.add_argument('--sectors_per_cluster',
default=1,
type=int,
choices=ALLOWED_SECTORS_PER_CLUSTER,
help='Number of sectors per cluster')
parser.add_argument('--root_entry_count',
default=FATDefaults.ROOT_ENTRIES_COUNT,
help='Number of entries in the root directory')
parser.add_argument('--long_name_support',
action='store_true',
help='Set flag to enable long names support.')
parser.add_argument('--use_default_datetime',
action='store_true',
help='For test purposes. If the flag is set the files are created with '
'the default timestamp that is the 1st of January 1980')
parser.add_argument('--fat_type',
default=0,
type=int,
choices=[FAT12, FAT16, 0],
help="""
Type of fat. Select 12 for fat12, 16 for fat16. Don't set, or set to 0 for automatic
calculation using cluster size and partition size.
""")
parser.add_argument('--wl_mode',
default=None,
type=str,
choices=['safe', 'perf'],
help='Wear levelling mode to use. Safe or performance. Only for sector size of 512')
args = parser.parse_args()
if args.fat_type == 0:
args.fat_type = None
if args.partition_size == 'detect' and not wl:
args.partition_size = -1
args.partition_size = int(str(args.partition_size), 0)
if not os.path.isdir(args.input_directory):
raise NotADirectoryError(f'The target directory `{args.input_directory}` does not exist!')
if args.wl_mode is not None:
if args.sector_size != 512:
raise ValueError('Wear levelling mode can be set only for sector size 512')
return args
def read_filesystem(path: str) -> bytearray:
with open(path, 'rb') as fs_file:
return bytearray(fs_file.read())
DATE_ENTRY = BitStruct(
'year' / BitsInteger(7),
'month' / BitsInteger(4),
'day' / BitsInteger(5))
TIME_ENTRY = BitStruct(
'hour' / BitsInteger(5),
'minute' / BitsInteger(6),
'second' / BitsInteger(5),
)
def build_name(name: str, extension: str) -> str:
return f'{name}.{extension}' if len(extension) > 0 else name
def build_date_entry(year: int, mon: int, mday: int) -> int:
"""
:param year: denotes year starting from 1980 (0 ~ 1980, 1 ~ 1981, etc), valid values are 1980 + 0..127 inclusive
thus theoretically 1980 - 2107
:param mon: denotes number of month of year in common order (1 ~ January, 2 ~ February, etc.),
valid values: 1..12 inclusive
:param mday: denotes number of day in month, valid values are 1..31 inclusive
:returns: 16 bit integer number (7 bits for year, 4 bits for month and 5 bits for day of the month)
"""
assert year in range(FATFS_INCEPTION_YEAR, FATFS_INCEPTION_YEAR + FATFS_MAX_YEARS)
assert mon in range(1, FATFS_MAX_MONTHS + 1)
assert mday in range(1, FATFS_MAX_DAYS + 1)
return int.from_bytes(DATE_ENTRY.build(dict(year=year - FATFS_INCEPTION_YEAR, month=mon, day=mday)), 'big')
def build_time_entry(hour: int, minute: int, sec: int) -> int:
"""
:param hour: denotes number of hour, valid values are 0..23 inclusive
:param minute: denotes minutes, valid range 0..59 inclusive
:param sec: denotes seconds with granularity 2 seconds (e.g. 1 ~ 2, 29 ~ 58), valid range 0..29 inclusive
:returns: 16 bit integer number (5 bits for hour, 6 bits for minute and 5 bits for second)
"""
assert hour in range(FATFS_MAX_HOURS)
assert minute in range(FATFS_MAX_MINUTES)
assert sec in range(FATFS_MAX_SECONDS)
return int.from_bytes(TIME_ENTRY.build(
dict(hour=hour, minute=minute, second=sec // FATFS_SECONDS_GRANULARITY)),
byteorder='big'
)
class FATDefaults:
# FATFS defaults
SIZE: int = 1024 * 1024
RESERVED_SECTORS_COUNT: int = 1
FAT_TABLES_COUNT: int = 1
SECTORS_PER_CLUSTER: int = 1
SECTOR_SIZE: int = 0x1000
HIDDEN_SECTORS: int = 0
ENTRY_SIZE: int = 32
NUM_HEADS: int = 0xff
OEM_NAME: str = 'MSDOS5.0'
SEC_PER_TRACK: int = 0x3f
VOLUME_LABEL: str = 'Espressif'
FILE_SYS_TYPE: str = 'FAT'
ROOT_ENTRIES_COUNT: int = 512 # number of entries in the root directory, recommended 512
MEDIA_TYPE: int = 0xf8
SIGNATURE_WORD: bytes = b'\x55\xAA'
# wear levelling defaults
VERSION: int = 2
TEMP_BUFFER_SIZE: int = 32
UPDATE_RATE: int = 16
WR_SIZE: int = 16
# wear leveling metadata (config sector) contains always sector size 4096
WL_SECTOR_SIZE: int = 4096