mirror of
https://github.com/espressif/esp-idf.git
synced 2024-10-05 20:47:46 -04:00
781 lines
31 KiB
Python
781 lines
31 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
|
||
|
|
# SPDX-License-Identifier: Apache-2.0
|
||
|
|
import argparse
|
||
|
|
import sys
|
||
|
|
import traceback
|
||
|
|
from typing import Any, Dict, List
|
||
|
|
from zlib import crc32
|
||
|
|
|
||
|
|
|
||
|
|
class Logger:
|
||
|
|
ansi = {
|
||
|
|
'red': '\033[31m',
|
||
|
|
'green': '\033[32m',
|
||
|
|
'yellow': '\033[33m',
|
||
|
|
'blue': '\033[34m',
|
||
|
|
'cyan': '\033[36m',
|
||
|
|
'bold': '\033[1m',
|
||
|
|
'clear': '\033[0m'
|
||
|
|
}
|
||
|
|
|
||
|
|
def __init__(self, *, color:str='auto'):
|
||
|
|
self.color = color == 'always' or (color == 'auto' and sys.stdout.isatty())
|
||
|
|
|
||
|
|
def set_color(self, color:str) -> None:
|
||
|
|
self.color = color == 'always' or (color == 'auto' and sys.stdout.isatty())
|
||
|
|
|
||
|
|
def info(self, *args, **kwargs) -> None: # type: ignore
|
||
|
|
kwargs['file'] = kwargs.get('file', sys.stdout) # Set default output to be stdout, but can be overwritten
|
||
|
|
print(*args, **kwargs)
|
||
|
|
|
||
|
|
def error(self, *args, **kwargs) -> None: # type: ignore
|
||
|
|
kwargs['file'] = kwargs.get('file', sys.stderr) # Set default output to be stderr, but can be overwritten
|
||
|
|
print(*args, **kwargs)
|
||
|
|
|
||
|
|
def red(self, text:str) -> str:
|
||
|
|
if self.color:
|
||
|
|
return Logger.ansi['red'] + text + Logger.ansi['clear']
|
||
|
|
return text
|
||
|
|
|
||
|
|
def green(self, text:str) -> str:
|
||
|
|
if self.color:
|
||
|
|
return Logger.ansi['green'] + text + Logger.ansi['clear']
|
||
|
|
return text
|
||
|
|
|
||
|
|
def yellow(self, text:str) -> str:
|
||
|
|
if self.color:
|
||
|
|
return Logger.ansi['yellow'] + text + Logger.ansi['clear']
|
||
|
|
return text
|
||
|
|
|
||
|
|
def blue(self, text:str) -> str:
|
||
|
|
if self.color:
|
||
|
|
return Logger.ansi['blue'] + text + Logger.ansi['clear']
|
||
|
|
return text
|
||
|
|
|
||
|
|
def cyan(self, text:str) -> str:
|
||
|
|
if self.color:
|
||
|
|
return Logger.ansi['cyan'] + text + Logger.ansi['clear']
|
||
|
|
return text
|
||
|
|
|
||
|
|
def bold(self, text:str) -> str:
|
||
|
|
if self.color:
|
||
|
|
return Logger.ansi['bold'] + text + Logger.ansi['clear']
|
||
|
|
return text
|
||
|
|
|
||
|
|
|
||
|
|
# Constants
|
||
|
|
class Constants:
|
||
|
|
class ConstantError(AttributeError):
|
||
|
|
pass
|
||
|
|
|
||
|
|
def __init__(self) -> None:
|
||
|
|
self.page_size = 4096
|
||
|
|
self.entry_size = 32
|
||
|
|
self.item_type = {
|
||
|
|
0x01: 'uint8_t',
|
||
|
|
0x11: 'int8_t',
|
||
|
|
0x02: 'uint16_t',
|
||
|
|
0x12: 'int16_t',
|
||
|
|
0x04: 'uint32_t',
|
||
|
|
0x14: 'int32_t',
|
||
|
|
0x08: 'uint64_t',
|
||
|
|
0x18: 'int64_t',
|
||
|
|
0x21: 'string',
|
||
|
|
0x41: 'blob',
|
||
|
|
0x42: 'blob_data',
|
||
|
|
0x48: 'blob_index',
|
||
|
|
}
|
||
|
|
self.page_status = {
|
||
|
|
0xffffffff: 'Empty',
|
||
|
|
0xfffffffe: 'Active',
|
||
|
|
0xfffffffc: 'Full',
|
||
|
|
0xfffffff8: 'Freeing',
|
||
|
|
}
|
||
|
|
self.entry_status = {
|
||
|
|
0b11: 'Empty',
|
||
|
|
0b10: 'Written',
|
||
|
|
0b00: 'Erased',
|
||
|
|
}
|
||
|
|
|
||
|
|
def __setattr__(self, key:str, val:Any) -> None:
|
||
|
|
if self.__dict__.get(key, None) is None:
|
||
|
|
self.__dict__[key] = val
|
||
|
|
else:
|
||
|
|
raise Constants.ConstantError('Cannot change a constant!')
|
||
|
|
|
||
|
|
|
||
|
|
const = Constants()
|
||
|
|
log = Logger()
|
||
|
|
|
||
|
|
|
||
|
|
class NotAlignedError(ValueError):
|
||
|
|
pass
|
||
|
|
|
||
|
|
|
||
|
|
class NVS_Partition:
|
||
|
|
def __init__(self, name:str, raw_data:bytearray):
|
||
|
|
if len(raw_data) % const.page_size != 0:
|
||
|
|
raise NotAlignedError(f'Given partition data is not aligned to page size ({len(raw_data)} % {const.page_size} = {len(raw_data)%const.page_size})')
|
||
|
|
|
||
|
|
# Divide partition into pages
|
||
|
|
self.name = name
|
||
|
|
self.pages = []
|
||
|
|
for i in range(0, len(raw_data), const.page_size):
|
||
|
|
self.pages.append(NVS_Page(raw_data[i:i + const.page_size], i))
|
||
|
|
|
||
|
|
|
||
|
|
class NVS_Page:
|
||
|
|
def __init__(self, page_data:bytearray, address:int):
|
||
|
|
if len(page_data) != const.page_size:
|
||
|
|
raise NotAlignedError(f'Size of given page does not match page size ({len(page_data)} != {const.page_size})')
|
||
|
|
|
||
|
|
# Initialize class
|
||
|
|
self.is_empty = page_data[0:const.entry_size] == bytearray({0xff}) * const.entry_size
|
||
|
|
self.start_address = address
|
||
|
|
self.raw_header = page_data[0:const.entry_size]
|
||
|
|
self.raw_entry_state_bitmap = page_data[const.entry_size:2 * const.entry_size]
|
||
|
|
self.entries = []
|
||
|
|
|
||
|
|
# Load header
|
||
|
|
self.header:Dict[str, Any] = {
|
||
|
|
'status': const.page_status.get(int.from_bytes(page_data[0:4], byteorder='little'), 'Invalid'),
|
||
|
|
'page_index': int.from_bytes(page_data[4:8], byteorder='little'),
|
||
|
|
'version': 256 - page_data[8],
|
||
|
|
'crc':{
|
||
|
|
'original': int.from_bytes(page_data[28:32], byteorder='little'),
|
||
|
|
'computed': crc32(page_data[4:28], 0xFFFFFFFF)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
# Load entry state bitmap
|
||
|
|
entry_states = []
|
||
|
|
for c in self.raw_entry_state_bitmap:
|
||
|
|
for index in range(0, 8, 2):
|
||
|
|
entry_states.append(const.entry_status.get((c >> index) & 3, 'Invalid'))
|
||
|
|
entry_states = entry_states[:-2]
|
||
|
|
|
||
|
|
# Load entries
|
||
|
|
i = 2
|
||
|
|
while i < int(const.page_size / const.entry_size): # Loop through every entry
|
||
|
|
span = page_data[(i * const.entry_size) + 2]
|
||
|
|
if span in [0xff, 0]: # 'Default' span length to prevent span overflow
|
||
|
|
span = 1
|
||
|
|
|
||
|
|
# Load an entry
|
||
|
|
entry = NVS_Entry(i - 2, page_data[i * const.entry_size:(i + 1) * const.entry_size], entry_states[i - 2])
|
||
|
|
self.entries.append(entry)
|
||
|
|
|
||
|
|
# Load all children entries
|
||
|
|
if span != 1:
|
||
|
|
for span_idx in range(1, span):
|
||
|
|
page_addr = i + span_idx
|
||
|
|
entry_idx = page_addr - 2
|
||
|
|
if page_addr * const.entry_size >= const.page_size:
|
||
|
|
break
|
||
|
|
child_entry = NVS_Entry(entry_idx, page_data[page_addr * const.entry_size:(page_addr + 1) * const.entry_size], entry_states[entry_idx])
|
||
|
|
entry.child_assign(child_entry)
|
||
|
|
entry.compute_crc()
|
||
|
|
i += span
|
||
|
|
|
||
|
|
|
||
|
|
class NVS_Entry:
|
||
|
|
def __init__(self, index:int, entry_data:bytearray, entry_state:str):
|
||
|
|
if len(entry_data) != const.entry_size:
|
||
|
|
raise NotAlignedError(f'Given entry is not aligned to entry size ({len(entry_data)} % {const.entry_size} = {len(entry_data)%const.entry_size})')
|
||
|
|
|
||
|
|
def item_convert(i_type:int, data:bytearray) -> Dict:
|
||
|
|
byte_size_mask = 0x0f
|
||
|
|
number_sign_mask = 0xf0
|
||
|
|
fixed_entry_length_threshold = 0x20 # Fixed length entry type number is always smaller than this
|
||
|
|
if i_type in const.item_type:
|
||
|
|
# Deal with non variable length entries
|
||
|
|
if i_type < fixed_entry_length_threshold:
|
||
|
|
size = i_type & byte_size_mask
|
||
|
|
num = int.from_bytes(data[:size], byteorder='little', signed=bool(i_type & number_sign_mask))
|
||
|
|
return {'value':num}
|
||
|
|
|
||
|
|
# Deal with variable length entries
|
||
|
|
if const.item_type[i_type] in ['string', 'blob_data', 'blob']:
|
||
|
|
size = int.from_bytes(data[:2], byteorder='little')
|
||
|
|
crc = int.from_bytes(data[4:8], byteorder='little')
|
||
|
|
return {'value': [size, crc],
|
||
|
|
'size': size,
|
||
|
|
'crc': crc}
|
||
|
|
if const.item_type[i_type] == 'blob_index':
|
||
|
|
size = int.from_bytes(data[:4], byteorder='little')
|
||
|
|
chunk_count = data[4]
|
||
|
|
chunk_start = data[5]
|
||
|
|
return {'value': [size, chunk_count, chunk_start],
|
||
|
|
'size': size,
|
||
|
|
'chunk_count': chunk_count,
|
||
|
|
'chunk_start': chunk_start}
|
||
|
|
return {'value': log.red('Cannot parse')}
|
||
|
|
|
||
|
|
def key_decode(data:bytearray) -> str:
|
||
|
|
decoded = ''
|
||
|
|
for n in data:
|
||
|
|
char = chr(n)
|
||
|
|
if char.isprintable():
|
||
|
|
decoded += char
|
||
|
|
return decoded
|
||
|
|
|
||
|
|
self.raw = entry_data
|
||
|
|
self.state = entry_state
|
||
|
|
self.is_empty = self.raw == bytearray({0xff}) * const.entry_size
|
||
|
|
self.index = index
|
||
|
|
|
||
|
|
namespace = self.raw[0]
|
||
|
|
entry_type = self.raw[1]
|
||
|
|
span = self.raw[2]
|
||
|
|
chunk_index = self.raw[3]
|
||
|
|
crc = self.raw[4:8]
|
||
|
|
key = self.raw[8:24]
|
||
|
|
data = self.raw[24:32]
|
||
|
|
raw_without_crc = self.raw[:4] + self.raw[8:32]
|
||
|
|
self.metadata: Dict[str, Any] = {
|
||
|
|
'namespace': namespace,
|
||
|
|
'type': const.item_type.get(entry_type, f'0x{entry_type:02x}'),
|
||
|
|
'span': span,
|
||
|
|
'chunk_index': chunk_index,
|
||
|
|
'crc':{
|
||
|
|
'original': int.from_bytes(crc, byteorder='little'),
|
||
|
|
'computed': crc32(raw_without_crc, 0xFFFFFFFF),
|
||
|
|
'data_original': int.from_bytes(data[-4:], byteorder='little'),
|
||
|
|
'data_computed': 0
|
||
|
|
}
|
||
|
|
}
|
||
|
|
self.children: List['NVS_Entry'] = []
|
||
|
|
self.key = key_decode(key)
|
||
|
|
self.data = item_convert(entry_type, data)
|
||
|
|
|
||
|
|
def dump_raw(self) -> str:
|
||
|
|
hex_bytes = ''
|
||
|
|
decoded = ''
|
||
|
|
for i, c in enumerate(self.raw):
|
||
|
|
middle_index = int(len(self.raw) / 2)
|
||
|
|
if i == middle_index: # Add a space in the middle
|
||
|
|
hex_bytes += ' '
|
||
|
|
decoded += ' '
|
||
|
|
hex_bytes += f'{c:02x} '
|
||
|
|
decoded += chr(c) if chr(c).isprintable() else '.'
|
||
|
|
return hex_bytes + ' ' + decoded
|
||
|
|
|
||
|
|
def child_assign(self, entry:'NVS_Entry') -> None:
|
||
|
|
if not isinstance(entry, type(self)):
|
||
|
|
raise ValueError('You can assign only NVS_Entry')
|
||
|
|
self.children.append(entry)
|
||
|
|
|
||
|
|
def compute_crc(self) -> None:
|
||
|
|
if self.metadata['span'] == 1:
|
||
|
|
return
|
||
|
|
|
||
|
|
# Merge entries into one buffer
|
||
|
|
children_data = bytearray()
|
||
|
|
for entry in self.children:
|
||
|
|
children_data += entry.raw
|
||
|
|
children_data = children_data[:self.data['size']] # Discard padding
|
||
|
|
|
||
|
|
self.metadata['crc']['data_computed'] = crc32(children_data, 0xFFFFFFFF)
|
||
|
|
|
||
|
|
|
||
|
|
def storage_stats(nvs_partition:NVS_Partition) -> None:
|
||
|
|
global_stats = {
|
||
|
|
'written_entries': 0,
|
||
|
|
'free_entries': 0,
|
||
|
|
'erased_entries': 0,
|
||
|
|
'invalid_entries': 0
|
||
|
|
}
|
||
|
|
|
||
|
|
for page in nvs_partition.pages:
|
||
|
|
written_e = 0
|
||
|
|
free_e = 0
|
||
|
|
erased_e = 0
|
||
|
|
invalid_e = 0
|
||
|
|
for entry in page.entries:
|
||
|
|
if entry.state == 'Written':
|
||
|
|
written_e += 1
|
||
|
|
elif entry.state == 'Empty':
|
||
|
|
free_e += 1
|
||
|
|
elif entry.state == 'Erased':
|
||
|
|
erased_e += 1
|
||
|
|
else:
|
||
|
|
invalid_e += 1
|
||
|
|
|
||
|
|
log.info(log.bold(f'Page {page.header["status"]}'))
|
||
|
|
log.info(' Found entries:')
|
||
|
|
log.info(f' Written: {written_e: 5d}')
|
||
|
|
log.info(f' Erased: {erased_e: 5d}')
|
||
|
|
log.info(f' Empty: {free_e: 5d}')
|
||
|
|
log.info(f' Invalid: {invalid_e: 5d}')
|
||
|
|
log.info(f' Total: {written_e + free_e + erased_e + invalid_e: 5d}')
|
||
|
|
log.info()
|
||
|
|
global_stats['written_entries'] += written_e
|
||
|
|
global_stats['erased_entries'] += erased_e
|
||
|
|
global_stats['free_entries'] += free_e
|
||
|
|
global_stats['invalid_entries'] += invalid_e
|
||
|
|
|
||
|
|
log.info(log.bold('Global'))
|
||
|
|
log.info(' Config:')
|
||
|
|
log.info(f' Page size: {const.page_size: 5d}')
|
||
|
|
log.info(f' Entry size: {const.entry_size: 5d}')
|
||
|
|
log.info(f' Total pages: {len(nvs_partition.pages): 5d}')
|
||
|
|
log.info(' Entries:')
|
||
|
|
log.info(f' Written: {global_stats["written_entries"]: 5d}')
|
||
|
|
log.info(f' Erased: {global_stats["erased_entries"]: 5d}')
|
||
|
|
log.info(f' Empty: {global_stats["free_entries"]: 5d}')
|
||
|
|
log.info(f' Invalid: {global_stats["invalid_entries"]: 5d}')
|
||
|
|
log.info(f' Total: {sum([global_stats[key] for key in global_stats]): 5d}')
|
||
|
|
log.info()
|
||
|
|
|
||
|
|
|
||
|
|
def dump_everything(nvs_partition:NVS_Partition, written_only:bool=False) -> None:
|
||
|
|
for page in nvs_partition.pages:
|
||
|
|
# Print page header
|
||
|
|
if page.is_empty:
|
||
|
|
log.info(log.bold(f'Page Empty, Page address: 0x{page.start_address:x}'))
|
||
|
|
else:
|
||
|
|
if page.header['crc']['original'] == page.header['crc']['computed']: # Color CRC32
|
||
|
|
crc = log.green(f'{page.header["crc"]["original"]: >8x}')
|
||
|
|
else:
|
||
|
|
crc = log.red(f'{page.header["crc"]["original"]: >8x}')
|
||
|
|
log.info(log.bold(f'Page no. {page.header["page_index"]}'
|
||
|
|
+ f', Status: {page.header["status"]}'
|
||
|
|
+ f', Version: {page.header["version"]}'
|
||
|
|
+ f', CRC32: {crc}')
|
||
|
|
+ log.bold(f', Page address: 0x{page.start_address:x}'))
|
||
|
|
|
||
|
|
log.info(log.bold(' Entry state bitmap: '), end='')
|
||
|
|
for x in page.raw_entry_state_bitmap:
|
||
|
|
log.info(f'{x:02x} ', end='')
|
||
|
|
log.info()
|
||
|
|
|
||
|
|
# Dump entries
|
||
|
|
empty_entries = []
|
||
|
|
for entry in page.entries:
|
||
|
|
# Skip non-written entries if needed
|
||
|
|
if written_only and not entry.state == 'Written':
|
||
|
|
continue
|
||
|
|
|
||
|
|
# Compress all empty entries
|
||
|
|
if entry.state == 'Empty' and entry.is_empty: # Gather all subsequent empty entries
|
||
|
|
empty_entries.append(entry)
|
||
|
|
continue
|
||
|
|
else:
|
||
|
|
# Print the empty entries
|
||
|
|
if len(empty_entries) >= 3: # There is enough entries to compress
|
||
|
|
log.info(log.bold(f' {empty_entries[0].index:03d}.'), 'Empty')
|
||
|
|
log.info(log.bold(' ...'))
|
||
|
|
log.info(log.bold(f' {empty_entries[-1].index:03d}.'), 'Empty')
|
||
|
|
else: # No need for compression
|
||
|
|
for e in empty_entries:
|
||
|
|
log.info(log.bold(f' {e.index:03d}.'), 'Empty')
|
||
|
|
empty_entries.clear()
|
||
|
|
|
||
|
|
# Dump a single entry
|
||
|
|
status = entry.state
|
||
|
|
if status == 'Written':
|
||
|
|
status = log.green(f'{status: <7}')
|
||
|
|
elif status == 'Erased':
|
||
|
|
status = log.red(f'{status: <7}')
|
||
|
|
|
||
|
|
crc = ''
|
||
|
|
if entry.metadata['crc']['original'] == entry.metadata['crc']['computed']: # Color CRC32
|
||
|
|
crc = log.green(f'{entry.metadata["crc"]["original"]: >8x}')
|
||
|
|
else:
|
||
|
|
crc = log.red(f'{entry.metadata["crc"]["original"]: >8x}')
|
||
|
|
|
||
|
|
log.info(log.bold(f' {entry.index:03d}.')
|
||
|
|
+ ' ' + status
|
||
|
|
+ f', Namespace Index: {entry.metadata["namespace"]:03d}'
|
||
|
|
+ f', Type: {entry.metadata["type"]:<10}'
|
||
|
|
+ f', Span: {entry.metadata["span"]:03d}'
|
||
|
|
+ f', Chunk Index: {entry.metadata["chunk_index"]:03d}'
|
||
|
|
+ f', CRC32: {crc}'
|
||
|
|
+ f' | {entry.key} : ', end='')
|
||
|
|
|
||
|
|
if entry.metadata['type'] not in ['string', 'blob_data', 'blob_index', 'blob']: # Entry is non-variable length
|
||
|
|
log.info(entry.data['value'])
|
||
|
|
else:
|
||
|
|
if entry.metadata['type'] == 'blob_index':
|
||
|
|
log.info(f'Size={entry.data["size"]}'
|
||
|
|
+ f', ChunkCount={entry.data["chunk_count"]}'
|
||
|
|
+ f', ChunkStart={entry.data["chunk_start"]}')
|
||
|
|
else:
|
||
|
|
if entry.metadata['crc']['data_original'] == entry.metadata['crc']['data_computed']: # Color CRC32
|
||
|
|
crc = log.green(f'{entry.metadata["crc"]["data_original"]:x}')
|
||
|
|
else:
|
||
|
|
crc = log.red(f'{entry.metadata["crc"]["data_original"]:x}')
|
||
|
|
log.info(f'Size={entry.data["size"]}, CRC32={crc}')
|
||
|
|
|
||
|
|
# Dump all children entries
|
||
|
|
if entry.metadata['span'] != 1:
|
||
|
|
for i, data in enumerate(entry.children):
|
||
|
|
log.info(f'{"": >6}0x{(i*const.entry_size):03x} {data.dump_raw()}')
|
||
|
|
|
||
|
|
# Dump trailing empty entries
|
||
|
|
if len(empty_entries) >= 3:
|
||
|
|
log.info(log.bold(f' {empty_entries[0].index:03d}.'), 'Empty')
|
||
|
|
log.info(log.bold(' ...'))
|
||
|
|
log.info(log.bold(f' {empty_entries[-1].index:03d}.'), 'Empty')
|
||
|
|
else:
|
||
|
|
for e in empty_entries:
|
||
|
|
log.info(log.bold(f' {e.index:03d}.'), 'Empty')
|
||
|
|
empty_entries.clear()
|
||
|
|
log.info()
|
||
|
|
|
||
|
|
|
||
|
|
def dump_written_entries(nvs_partition:NVS_Partition) -> None:
|
||
|
|
dump_everything(nvs_partition, True)
|
||
|
|
|
||
|
|
|
||
|
|
def list_namespaces(nvs_partition:NVS_Partition) -> None:
|
||
|
|
# Gather namespaces
|
||
|
|
ns = {}
|
||
|
|
for page in nvs_partition.pages:
|
||
|
|
for entry in page.entries:
|
||
|
|
if entry.state == 'Written' and entry.metadata['namespace'] == 0:
|
||
|
|
ns[entry.data['value']] = entry.key
|
||
|
|
|
||
|
|
# Print found namespaces
|
||
|
|
log.info(log.bold(f'Index : Namespace'))
|
||
|
|
for ns_index in sorted(ns):
|
||
|
|
log.info(f' {ns_index:03d} :', log.cyan(ns[ns_index]))
|
||
|
|
|
||
|
|
|
||
|
|
def dump_key_value_pairs(nvs_partition:NVS_Partition) -> None:
|
||
|
|
# Get namespace list
|
||
|
|
ns = {}
|
||
|
|
for page in nvs_partition.pages:
|
||
|
|
for entry in page.entries:
|
||
|
|
if entry.state == 'Written' and entry.metadata['namespace'] == 0:
|
||
|
|
ns[entry.data['value']] = entry.key
|
||
|
|
|
||
|
|
# Print key-value pairs
|
||
|
|
for page in nvs_partition.pages:
|
||
|
|
# Print page header
|
||
|
|
if page.is_empty:
|
||
|
|
log.info(log.bold('Page Empty'))
|
||
|
|
else:
|
||
|
|
log.info(log.bold(f'Page no. {page.header["page_index"]}'
|
||
|
|
+ f', Status: {page.header["status"]}'))
|
||
|
|
|
||
|
|
# Print entries
|
||
|
|
for entry in page.entries:
|
||
|
|
if entry.state == 'Written' and entry.metadata['namespace'] != 0: # Ignore non-written entries
|
||
|
|
chunk_index = ''
|
||
|
|
data = ''
|
||
|
|
if entry.metadata['type'] not in ['string', 'blob_data', 'blob_index', 'blob']: # Non-variable length entry
|
||
|
|
data = entry.data['value']
|
||
|
|
elif entry.metadata['type'] == 'blob_index':
|
||
|
|
continue
|
||
|
|
else: # Variable length entries
|
||
|
|
tmp = b''
|
||
|
|
for e in entry.children: # Merge all children entries
|
||
|
|
tmp += bytes(e.raw)
|
||
|
|
tmp = tmp[:entry.data['size']] # Discard padding
|
||
|
|
if entry.metadata['type'] == 'blob_data':
|
||
|
|
if entry.metadata['chunk_index'] >= 128: # Get real chunk index
|
||
|
|
chunk_index = f'[{entry.metadata["chunk_index"] - 128}]'
|
||
|
|
else:
|
||
|
|
chunk_index = f'[{entry.metadata["chunk_index"]}]'
|
||
|
|
data = str(tmp)
|
||
|
|
|
||
|
|
if entry.metadata['namespace'] not in ns:
|
||
|
|
continue
|
||
|
|
else:
|
||
|
|
log.info(' '
|
||
|
|
+ log.cyan(ns[entry.metadata['namespace']])
|
||
|
|
+ ':'
|
||
|
|
+ log.yellow(entry.key)
|
||
|
|
+ f'{chunk_index} = {data}')
|
||
|
|
log.info()
|
||
|
|
|
||
|
|
|
||
|
|
def dump_written_blobs(nvs_partition:NVS_Partition) -> None:
|
||
|
|
blobs: Dict = {}
|
||
|
|
strings: List[NVS_Entry] = []
|
||
|
|
legacy_blobs: List[NVS_Entry] = []
|
||
|
|
ns = {}
|
||
|
|
empty_entry = NVS_Entry(-1, bytearray(32), 'Erased')
|
||
|
|
|
||
|
|
# Gather namespaces, blob indexes and legacy blobs
|
||
|
|
for page in nvs_partition.pages:
|
||
|
|
for entry in page.entries:
|
||
|
|
if entry.state == 'Written':
|
||
|
|
if entry.metadata['type'] == 'blob_index':
|
||
|
|
blobs[f'{entry.metadata["namespace"]:03d}{entry.key}'] = [entry] + [empty_entry] * entry.data['chunk_count']
|
||
|
|
elif entry.metadata['type'] == 'blob':
|
||
|
|
legacy_blobs.append(entry)
|
||
|
|
elif entry.metadata['type'] == 'string':
|
||
|
|
strings.append(entry)
|
||
|
|
elif entry.metadata['namespace'] == 0:
|
||
|
|
ns[entry.data['value']] = entry.key
|
||
|
|
|
||
|
|
# Dump blobs
|
||
|
|
for key in blobs:
|
||
|
|
for page in nvs_partition.pages:
|
||
|
|
for entry in page.entries:
|
||
|
|
# Gather all blob chunks
|
||
|
|
if entry.state == 'Written' \
|
||
|
|
and entry.metadata['type'] != 'blob_index' \
|
||
|
|
and entry.metadata['namespace'] == blobs[key][0].metadata['namespace'] \
|
||
|
|
and entry.key == blobs[key][0].key:
|
||
|
|
blobs[key][1 + entry.metadata['chunk_index'] - blobs[key][0].data['chunk_start']] = entry
|
||
|
|
|
||
|
|
blob_index = blobs[key][0]
|
||
|
|
blob_chunks = blobs[key][1:]
|
||
|
|
|
||
|
|
# Print blob info
|
||
|
|
log.info(log.cyan(ns.get(blob_index.metadata['namespace'], blob_index.metadata['namespace']))
|
||
|
|
+ ':'
|
||
|
|
+ log.yellow(blob_index.key)
|
||
|
|
+ ' - '
|
||
|
|
+ f'Type: Blob (Version 2), '
|
||
|
|
+ f'Size: {blob_index.data["size"]}')
|
||
|
|
|
||
|
|
# Print blob data
|
||
|
|
raw_entries = []
|
||
|
|
for kid in blob_chunks: # Gather all chunk entries
|
||
|
|
if kid is empty_entry:
|
||
|
|
raw_entries += [empty_entry]
|
||
|
|
else:
|
||
|
|
raw_entries += kid.children
|
||
|
|
|
||
|
|
for i, entry in enumerate(raw_entries):
|
||
|
|
if entry is empty_entry:
|
||
|
|
log.info(log.yellow(f' {"":->63} Missing data {"":-<64}'))
|
||
|
|
else:
|
||
|
|
log.info(f' 0x{(i * const.entry_size):05x} {entry.dump_raw()}')
|
||
|
|
log.info()
|
||
|
|
|
||
|
|
# Dump strings
|
||
|
|
for string in strings:
|
||
|
|
log.info(log.cyan(ns.get(string.metadata['namespace'], string.metadata['namespace']))
|
||
|
|
+ ':'
|
||
|
|
+ log.yellow(string.key)
|
||
|
|
+ ' - '
|
||
|
|
+ 'Type: String, '
|
||
|
|
+ f'Size: {string.data["size"]}')
|
||
|
|
for i, entry in enumerate(string.children):
|
||
|
|
log.info(f' 0x{(i * const.entry_size):05x} {entry.dump_raw()}')
|
||
|
|
log.info()
|
||
|
|
|
||
|
|
# Dump legacy blobs
|
||
|
|
for blob in legacy_blobs:
|
||
|
|
log.info(log.cyan(ns.get(blob.metadata['namespace'], blob.metadata['namespace']))
|
||
|
|
+ ':'
|
||
|
|
+ log.yellow(blob.key)
|
||
|
|
+ ' - '
|
||
|
|
+ 'Type: Blob (Version 1), '
|
||
|
|
+ f'Size: {blob.data["size"]}')
|
||
|
|
for i, entry in enumerate(blob.children):
|
||
|
|
log.info(f' 0x{(i * const.entry_size):05x} {entry.dump_raw()}')
|
||
|
|
log.info()
|
||
|
|
|
||
|
|
|
||
|
|
def integrity_check(nvs_partition:NVS_Partition) -> None:
|
||
|
|
used_namespaces: Dict[int, None] = {}
|
||
|
|
found_namespaces: Dict[int, str] = {}
|
||
|
|
blobs:Dict = {}
|
||
|
|
blob_chunks: List[NVS_Entry] = []
|
||
|
|
empty_entry = NVS_Entry(-1, bytearray(32), 'Erased')
|
||
|
|
|
||
|
|
# Partition size
|
||
|
|
if len(nvs_partition.pages) < 3:
|
||
|
|
log.info(log.yellow('Partition has to have at least 3 pages to function properly!'))
|
||
|
|
|
||
|
|
for page in nvs_partition.pages:
|
||
|
|
# Print page header
|
||
|
|
if page.header['status'] == 'Empty':
|
||
|
|
log.info(log.cyan('Page Empty'))
|
||
|
|
|
||
|
|
# Check if page is truly empty
|
||
|
|
if page.raw_entry_state_bitmap != bytearray({0xff}) * const.entry_size:
|
||
|
|
log.info(log.red('Page is reported as empty but entry state bitmap is not empty!'))
|
||
|
|
if any([not e.is_empty for e in page.entries]):
|
||
|
|
log.info(log.red('Page is reported as emtpy but there is written data!'))
|
||
|
|
else:
|
||
|
|
# Check page header CRC32
|
||
|
|
if page.header['crc']['original'] == page.header['crc']['computed']:
|
||
|
|
log.info(log.cyan(f'Page no. {page.header["page_index"]}'))
|
||
|
|
else:
|
||
|
|
log.info(log.cyan(f'Page no. {page.header["page_index"]}'),
|
||
|
|
f'Written CRC32:',
|
||
|
|
log.red(f'{page.header["crc"]["original"]:x}'),
|
||
|
|
f'Generated CRC32:',
|
||
|
|
log.green(f'{page.header["crc"]["computed"]:x}'))
|
||
|
|
|
||
|
|
# Check all entries
|
||
|
|
for entry in page.entries:
|
||
|
|
# Entry state check
|
||
|
|
if entry.is_empty:
|
||
|
|
if entry.state == 'Written':
|
||
|
|
log.info(log.red(f' Entry #{entry.index:03d} is reported as written but is empty!'))
|
||
|
|
continue
|
||
|
|
elif entry.state == 'Erased':
|
||
|
|
log.info(log.yellow(f' Entry #{entry.index:03d} is reported as erased but is empty!'))
|
||
|
|
|
||
|
|
if entry.state == 'Written':
|
||
|
|
# Entry CRC32 check
|
||
|
|
if entry.metadata['crc']['original'] != entry.metadata['crc']['computed']:
|
||
|
|
log.info(log.red(f' Entry #{entry.index:03d} {entry.key} has wrong CRC32!{"": <5}'),
|
||
|
|
f'Written:',
|
||
|
|
log.red(f'{entry.metadata["crc"]["original"]:x}'),
|
||
|
|
f'Generated:',
|
||
|
|
log.green(f'{entry.metadata["crc"]["computed"]:x}'))
|
||
|
|
|
||
|
|
# Entry children CRC32 check
|
||
|
|
if entry.metadata['span'] > 1 and entry.metadata['crc']['data_original'] != entry.metadata['crc']['data_computed']:
|
||
|
|
log.info(log.red(f' Entry #{entry.index:03d} {entry.key} data has wrong CRC32!'),
|
||
|
|
f'Written:',
|
||
|
|
log.red(f'{entry.metadata["crc"]["original"]:x}'),
|
||
|
|
f'Generated:',
|
||
|
|
log.green(f'{entry.metadata["crc"]["computed"]:x}'))
|
||
|
|
|
||
|
|
# Entry type check
|
||
|
|
if entry.metadata['type'] not in [const.item_type[key] for key in const.item_type]:
|
||
|
|
log.info(log.yellow(f' Type of entry #{entry.index:03d} {entry.key} is unrecognized!'),
|
||
|
|
f'Type: {entry.metadata["type"]}')
|
||
|
|
|
||
|
|
# Span check
|
||
|
|
if entry.index + entry.metadata['span'] - 1 >= int(const.page_size / const.entry_size) - 2:
|
||
|
|
log.info(log.red(f' Variable length entry #{entry.index:03d} {entry.key} is out of bounds!'))
|
||
|
|
# Spanned entry state checks
|
||
|
|
elif entry.metadata['span'] > 1:
|
||
|
|
parent_state = entry.state
|
||
|
|
for kid in entry.children:
|
||
|
|
if parent_state != kid.state:
|
||
|
|
log.info(log.yellow(' Inconsistent data state!'),
|
||
|
|
f'Entry #{entry.index:03d} {entry.key} state: {parent_state},',
|
||
|
|
f'Data entry #{kid.index:03d} {entry.key} state: {kid.state}')
|
||
|
|
|
||
|
|
# Gather blobs & namespaces
|
||
|
|
if entry.metadata['type'] == 'blob_index':
|
||
|
|
blobs[f'{entry.metadata["namespace"]:03d}{entry.key}'] = [entry] + [empty_entry] * entry.data['chunk_count']
|
||
|
|
elif entry.metadata['type'] == 'blob_data':
|
||
|
|
blob_chunks.append(entry)
|
||
|
|
|
||
|
|
if entry.metadata['namespace'] == 0:
|
||
|
|
found_namespaces[entry.data['value']] = entry.key
|
||
|
|
else:
|
||
|
|
used_namespaces[entry.metadata['namespace']] = None
|
||
|
|
log.info()
|
||
|
|
|
||
|
|
# Assemble blobs
|
||
|
|
for chunk in blob_chunks:
|
||
|
|
parent = blobs.get(f'{chunk.metadata["namespace"]:03d}{chunk.key}', [empty_entry])[0]
|
||
|
|
if parent is empty_entry:
|
||
|
|
log.info(log.red(f'Blob {chunk.key} chunk has no blob index!'),
|
||
|
|
f'Namespace index: {chunk.metadata["namespace"]:03d}',
|
||
|
|
f'[{found_namespaces.get(chunk.metadata["namespace"], "undefined")}],',
|
||
|
|
f'Chunk Index: {chunk.metadata["chunk_index"]:03d}')
|
||
|
|
else:
|
||
|
|
blob_key = f'{chunk.metadata["namespace"]:03d}{chunk.key}'
|
||
|
|
chunk_index = chunk.metadata['chunk_index'] - parent.data['chunk_start']
|
||
|
|
blobs[blob_key][chunk_index + 1] = chunk
|
||
|
|
|
||
|
|
# Check all blobs
|
||
|
|
for blob_key in blobs:
|
||
|
|
blob_index = blobs[blob_key][0]
|
||
|
|
blob_chunks = blobs[blob_key][1:]
|
||
|
|
blob_size = blob_index.data['size']
|
||
|
|
|
||
|
|
# Blob data check
|
||
|
|
for i, chunk in enumerate(blob_chunks):
|
||
|
|
if chunk is empty_entry:
|
||
|
|
log.info(log.red(f'Blob {blob_index.key} is missing a chunk!'),
|
||
|
|
f'Namespace index: {blob_index.metadata["namespace"]:03d}',
|
||
|
|
f'[{found_namespaces.get(blob_index.metadata["namespace"], "undefined")}],',
|
||
|
|
f'Chunk Index: {i:03d}')
|
||
|
|
else:
|
||
|
|
blob_size -= len(chunk.children) * const.entry_size
|
||
|
|
|
||
|
|
if blob_size > 0:
|
||
|
|
log.info(log.red(f'Blob {blob_index.key} is missing {blob_size} B of data!'),
|
||
|
|
f'Namespace index: {blob_index.metadata["namespace"]:03d}')
|
||
|
|
|
||
|
|
# Namespaces
|
||
|
|
for used_ns in used_namespaces:
|
||
|
|
key = found_namespaces.pop(used_ns, '')
|
||
|
|
if key == '':
|
||
|
|
log.info(log.red('Undefined namespace index!'),
|
||
|
|
f'Namespace index: {used_ns:03d}',
|
||
|
|
f'[undefined]')
|
||
|
|
|
||
|
|
for unused_ns in found_namespaces:
|
||
|
|
log.info(log.yellow('Found unused namespace.'),
|
||
|
|
f'Namespace index: {unused_ns:03d}',
|
||
|
|
f'[{found_namespaces[unused_ns]}]')
|
||
|
|
|
||
|
|
|
||
|
|
def program_args() -> argparse.Namespace:
|
||
|
|
parser = argparse.ArgumentParser(description='Parse NVS partition')
|
||
|
|
parser.add_argument('file', help='Path to dumped NVS partition')
|
||
|
|
parser.add_argument('-i', '--integrity-check', action='store_true',
|
||
|
|
help='Check partition for potential errors')
|
||
|
|
tmp = {'all': 'Print everything',
|
||
|
|
'written': 'Print only currently written entries',
|
||
|
|
'minimal': 'Print only namespace:key=value pairs',
|
||
|
|
'namespaces': 'List all written namespaces',
|
||
|
|
'blobs': 'Print all blobs and strings',
|
||
|
|
'storage_info': 'Print storage related information (free/used entries, etc)',
|
||
|
|
'none': 'Do not print anything (if you only want to do integrity check)'}
|
||
|
|
parser.add_argument('-d', '--dump', choices=tmp, default='all', metavar='type',
|
||
|
|
help=f"type={{{str(list(tmp.keys()))[1:-1]}}} : {' ; '.join([f'{opt} - {tmp[opt]}' for opt in tmp])}")
|
||
|
|
parser.add_argument('--color', choices=['never', 'auto', 'always'], default='auto', help='Enable color (ANSI)')
|
||
|
|
return parser.parse_args()
|
||
|
|
|
||
|
|
|
||
|
|
def main() -> None:
|
||
|
|
args = program_args()
|
||
|
|
|
||
|
|
if const.entry_size != 32:
|
||
|
|
raise ValueError(f'Entry size is not 32 B! This is currently non negotiable.')
|
||
|
|
|
||
|
|
log.set_color(args.color)
|
||
|
|
|
||
|
|
try:
|
||
|
|
with open(args.file, 'rb') as f:
|
||
|
|
partition = f.read()
|
||
|
|
except IndexError:
|
||
|
|
log.error('No file given')
|
||
|
|
raise
|
||
|
|
except FileNotFoundError:
|
||
|
|
log.error('Bad filename')
|
||
|
|
raise
|
||
|
|
|
||
|
|
nvs = NVS_Partition(args.file.split('/')[-1], bytearray(partition))
|
||
|
|
|
||
|
|
def noop(_:NVS_Partition) -> None:
|
||
|
|
pass
|
||
|
|
|
||
|
|
def not_implemented(_:NVS_Partition) -> None:
|
||
|
|
raise RuntimeError(f'{args.dump} is not implemented')
|
||
|
|
|
||
|
|
cmds = {
|
||
|
|
'all': dump_everything,
|
||
|
|
'written': dump_written_entries,
|
||
|
|
'minimal': dump_key_value_pairs,
|
||
|
|
'namespaces': list_namespaces,
|
||
|
|
'blobs': dump_written_blobs,
|
||
|
|
'storage_info': storage_stats,
|
||
|
|
'none': noop,
|
||
|
|
}
|
||
|
|
cmds.get(args.dump, not_implemented)(nvs) # type: ignore
|
||
|
|
|
||
|
|
if args.integrity_check:
|
||
|
|
log.info()
|
||
|
|
integrity_check(nvs)
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
try:
|
||
|
|
main()
|
||
|
|
except ValueError:
|
||
|
|
traceback.print_exc(file=sys.stderr)
|
||
|
|
sys.exit(1)
|
||
|
|
except Constants.ConstantError:
|
||
|
|
traceback.print_exc(file=sys.stderr)
|
||
|
|
sys.exit(1)
|