# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import argparse import binascii import os import re import uuid from datetime import datetime from typing import List, Optional, Tuple from construct import BitsInteger, BitStruct, Int16ul # the regex pattern defines symbols that are allowed by long file names but not by short file names INVALID_SFN_CHARS_PATTERN = re.compile(r'[.+,;=\[\]]') FAT12_MAX_CLUSTERS: int = 4085 FAT16_MAX_CLUSTERS: int = 65525 RESERVED_CLUSTERS_COUNT: int = 2 PAD_CHAR: int = 0x20 FAT12: int = 12 FAT16: int = 16 FAT32: int = 32 FULL_BYTE: bytes = b'\xff' EMPTY_BYTE: bytes = b'\x00' # redundant BYTES_PER_DIRECTORY_ENTRY: int = 32 UINT32_MAX: int = (1 << 32) - 1 MAX_NAME_SIZE: int = 8 MAX_EXT_SIZE: int = 3 DATETIME = Tuple[int, int, int] FATFS_INCEPTION_YEAR: int = 1980 FATFS_INCEPTION: datetime = datetime(FATFS_INCEPTION_YEAR, 1, 1, 0, 0, 0, 0) FATFS_MAX_HOURS = 24 FATFS_MAX_MINUTES = 60 FATFS_MAX_SECONDS = 60 FATFS_MAX_DAYS = 31 FATFS_MAX_MONTHS = 12 FATFS_MAX_YEARS = 127 FATFS_SECONDS_GRANULARITY: int = 2 # long names are encoded to two bytes in utf-16 LONG_NAMES_ENCODING: str = 'utf-16' SHORT_NAMES_ENCODING: str = 'utf-8' # compatible with WL_SECTOR_SIZE # choices for WL are WL_SECTOR_SIZE_512 and WL_SECTOR_SIZE_4096 ALLOWED_WL_SECTOR_SIZES: List[int] = [512, 4096] ALLOWED_SECTOR_SIZES: List[int] = [512, 1024, 2048, 4096] ALLOWED_SECTORS_PER_CLUSTER: List[int] = [1, 2, 4, 8, 16, 32, 64, 128] def crc32(input_values: List[int], crc: int) -> int: """ Name Polynomial Reversed? Init-value XOR-out crc32 0x104C11DB7 True 4294967295 (UINT32_MAX) 0xFFFFFFFF """ return binascii.crc32(bytearray(input_values), crc) def number_of_clusters(number_of_sectors: int, sectors_per_cluster: int) -> int: return number_of_sectors // sectors_per_cluster def get_non_data_sectors_cnt(reserved_sectors_cnt: int, sectors_per_fat_cnt: int, root_dir_sectors_cnt: int) -> int: return reserved_sectors_cnt + sectors_per_fat_cnt + root_dir_sectors_cnt def get_fatfs_type(clusters_count: int) -> int: if clusters_count < FAT12_MAX_CLUSTERS: return FAT12 if clusters_count <= FAT16_MAX_CLUSTERS: return FAT16 return FAT32 def get_fat_sectors_count(clusters_count: int, sector_size: int) -> int: fatfs_type_ = get_fatfs_type(clusters_count) if fatfs_type_ == FAT32: raise NotImplementedError('FAT32 is not supported!') # number of byte halves cluster_s: int = fatfs_type_ // 4 fat_size_bytes: int = ( clusters_count * 2 + cluster_s) if fatfs_type_ == FAT16 else (clusters_count * 3 + 1) // 2 + cluster_s return (fat_size_bytes + sector_size - 1) // sector_size def required_clusters_count(cluster_size: int, content: bytes) -> int: # compute number of required clusters for file text return (len(content) + cluster_size - 1) // cluster_size def generate_4bytes_random() -> int: return uuid.uuid4().int & 0xFFFFFFFF def pad_string(content: str, size: Optional[int] = None, pad: int = PAD_CHAR) -> str: # cut string if longer and fill with pad character if shorter than size return content.ljust(size or len(content), chr(pad))[:size] def right_strip_string(content: str, pad: int = PAD_CHAR) -> str: return content.rstrip(chr(pad)) def build_lfn_short_entry_name(name: str, extension: str, order: int) -> str: return '{}{}'.format(pad_string(content=name[:MAX_NAME_SIZE - 2] + '~' + chr(order), size=MAX_NAME_SIZE), pad_string(extension[:MAX_EXT_SIZE], size=MAX_EXT_SIZE)) def lfn_checksum(short_entry_name: str) -> int: """ Function defined by FAT specification. Computes checksum out of name in the short file name entry. """ checksum_result = 0 for i in range(MAX_NAME_SIZE + MAX_EXT_SIZE): # operation is a right rotation on 8 bits (Python equivalent for unsigned char in C) checksum_result = (0x80 if checksum_result & 1 else 0x00) + (checksum_result >> 1) + ord(short_entry_name[i]) checksum_result &= 0xff return checksum_result def convert_to_utf16_and_pad(content: str, expected_size: int, pad: bytes = FULL_BYTE) -> bytes: # we need to get rid of the Byte order mark 0xfeff or 0xfffe, fatfs does not use it bom_utf16: bytes = b'\xfe\xff' encoded_content_utf16: bytes = content.encode(LONG_NAMES_ENCODING)[len(bom_utf16):] return encoded_content_utf16.ljust(2 * expected_size, pad) def split_to_name_and_extension(full_name: str) -> Tuple[str, str]: name, extension = os.path.splitext(full_name) return name, extension.replace('.', '') def is_valid_fatfs_name(string: str) -> bool: return string == string.upper() def split_by_half_byte_12_bit_little_endian(value: int) -> Tuple[int, int, int]: value_as_bytes: bytes = Int16ul.build(value) return value_as_bytes[0] & 0x0f, value_as_bytes[0] >> 4, value_as_bytes[1] & 0x0f def merge_by_half_byte_12_bit_little_endian(v1: int, v2: int, v3: int) -> int: return v1 | v2 << 4 | v3 << 8 def build_byte(first_half: int, second_half: int) -> int: return (first_half << 4) | second_half def split_content_into_sectors(content: bytes, sector_size: int) -> List[bytes]: result = [] clusters_cnt: int = required_clusters_count(cluster_size=sector_size, content=content) for i in range(clusters_cnt): result.append(content[sector_size * i:(i + 1) * sector_size]) return result def get_args_for_partition_generator(desc: str, wl: bool) -> argparse.Namespace: parser: argparse.ArgumentParser = argparse.ArgumentParser(description=desc) parser.add_argument('input_directory', help='Path to the directory that will be encoded into fatfs image') parser.add_argument('--output_file', default='fatfs_image.img', help='Filename of the generated fatfs image') parser.add_argument('--partition_size', default=FATDefaults.SIZE, help='Size of the partition in bytes') parser.add_argument('--sector_size', default=FATDefaults.SECTOR_SIZE, type=int, choices=ALLOWED_WL_SECTOR_SIZES if wl else ALLOWED_SECTOR_SIZES, help='Size of the partition in bytes') parser.add_argument('--sectors_per_cluster', default=1, type=int, choices=ALLOWED_SECTORS_PER_CLUSTER, help='Number of sectors per cluster') parser.add_argument('--root_entry_count', default=FATDefaults.ROOT_ENTRIES_COUNT, help='Number of entries in the root directory') parser.add_argument('--long_name_support', action='store_true', help='Set flag to enable long names support.') parser.add_argument('--use_default_datetime', action='store_true', help='For test purposes. If the flag is set the files are created with ' 'the default timestamp that is the 1st of January 1980') parser.add_argument('--fat_type', default=0, type=int, choices=[FAT12, FAT16, 0], help=""" Type of fat. Select 12 for fat12, 16 for fat16. Don't set, or set to 0 for automatic calculation using cluster size and partition size. """) args = parser.parse_args() if args.fat_type == 0: args.fat_type = None args.partition_size = int(str(args.partition_size), 0) if not os.path.isdir(args.input_directory): raise NotADirectoryError(f'The target directory `{args.input_directory}` does not exist!') return args def read_filesystem(path: str) -> bytearray: with open(path, 'rb') as fs_file: return bytearray(fs_file.read()) DATE_ENTRY = BitStruct( 'year' / BitsInteger(7), 'month' / BitsInteger(4), 'day' / BitsInteger(5)) TIME_ENTRY = BitStruct( 'hour' / BitsInteger(5), 'minute' / BitsInteger(6), 'second' / BitsInteger(5), ) def build_name(name: str, extension: str) -> str: return f'{name}.{extension}' if len(extension) > 0 else name def build_date_entry(year: int, mon: int, mday: int) -> int: """ :param year: denotes year starting from 1980 (0 ~ 1980, 1 ~ 1981, etc), valid values are 1980 + 0..127 inclusive thus theoretically 1980 - 2107 :param mon: denotes number of month of year in common order (1 ~ January, 2 ~ February, etc.), valid values: 1..12 inclusive :param mday: denotes number of day in month, valid values are 1..31 inclusive :returns: 16 bit integer number (7 bits for year, 4 bits for month and 5 bits for day of the month) """ assert year in range(FATFS_INCEPTION_YEAR, FATFS_INCEPTION_YEAR + FATFS_MAX_YEARS) assert mon in range(1, FATFS_MAX_MONTHS + 1) assert mday in range(1, FATFS_MAX_DAYS + 1) return int.from_bytes(DATE_ENTRY.build(dict(year=year - FATFS_INCEPTION_YEAR, month=mon, day=mday)), 'big') def build_time_entry(hour: int, minute: int, sec: int) -> int: """ :param hour: denotes number of hour, valid values are 0..23 inclusive :param minute: denotes minutes, valid range 0..59 inclusive :param sec: denotes seconds with granularity 2 seconds (e.g. 1 ~ 2, 29 ~ 58), valid range 0..29 inclusive :returns: 16 bit integer number (5 bits for hour, 6 bits for minute and 5 bits for second) """ assert hour in range(FATFS_MAX_HOURS) assert minute in range(FATFS_MAX_MINUTES) assert sec in range(FATFS_MAX_SECONDS) return int.from_bytes(TIME_ENTRY.build( dict(hour=hour, minute=minute, second=sec // FATFS_SECONDS_GRANULARITY)), byteorder='big' ) class FATDefaults: # FATFS defaults SIZE: int = 1024 * 1024 RESERVED_SECTORS_COUNT: int = 1 FAT_TABLES_COUNT: int = 1 SECTORS_PER_CLUSTER: int = 1 SECTOR_SIZE: int = 0x1000 HIDDEN_SECTORS: int = 0 ENTRY_SIZE: int = 32 NUM_HEADS: int = 0xff OEM_NAME: str = 'MSDOS5.0' SEC_PER_TRACK: int = 0x3f VOLUME_LABEL: str = 'Espressif' FILE_SYS_TYPE: str = 'FAT' ROOT_ENTRIES_COUNT: int = 512 # number of entries in the root directory, recommended 512 MEDIA_TYPE: int = 0xf8 SIGNATURE_WORD: bytes = b'\x55\xAA' # wear levelling defaults VERSION: int = 2 TEMP_BUFFER_SIZE: int = 32 UPDATE_RATE: int = 16 WR_SIZE: int = 16 # wear leveling metadata (config sector) contains always sector size 4096 WL_SECTOR_SIZE: int = 4096