From b66b98a25f8d8ee103c21478ccb163bb0638c117 Mon Sep 17 00:00:00 2001 From: Roland Dobai Date: Fri, 14 May 2021 14:22:54 +0200 Subject: [PATCH] tools: Split up large binaries into smaller chunks in the DFU binary ROM will erase the region a partition is in as soon as it receives the first bit of the data that is in the partition. For large partitions it takes more than 5 seconds to erase which is a hard-coded limit in dfu-utils. This splits large binaries and adds them by chunks which should avoid timing-out during flashing. Closes https://github.com/espressif/esp-idf/issues/6999 --- tools/idf_py_actions/dfu_ext.py | 18 ++++- tools/mkdfu.py | 39 ++++++++-- tools/test_mkdfu/1/flasher_args.json | 7 -- tools/test_mkdfu/2/dfu.bin | Bin 0 -> 10256 bytes tools/test_mkdfu/test_mkdfu.py | 105 +++++++++++++++++---------- 5 files changed, 115 insertions(+), 54 deletions(-) delete mode 100644 tools/test_mkdfu/1/flasher_args.json create mode 100644 tools/test_mkdfu/2/dfu.bin diff --git a/tools/idf_py_actions/dfu_ext.py b/tools/idf_py_actions/dfu_ext.py index 0062978b9a..f637a36534 100644 --- a/tools/idf_py_actions/dfu_ext.py +++ b/tools/idf_py_actions/dfu_ext.py @@ -6,7 +6,11 @@ def action_extensions(base_actions, project_path): SUPPORTED_TARGETS = ['esp32s2'] - def dfu_target(target_name, ctx, args): + def dfu_target(target_name, ctx, args, part_size): + ensure_build_directory(args, ctx.info_name) + run_target(target_name, args, {'ESP_DFU_PART_SIZE': part_size} if part_size else {}) + + def dfu_list_target(target_name, ctx, args): ensure_build_directory(args, ctx.info_name) run_target(target_name, args) @@ -27,9 +31,17 @@ def action_extensions(base_actions, project_path): 'callback': dfu_target, 'short_help': 'Build the DFU binary', 'dependencies': ['all'], + 'options': [ + { + 'names': ['--part-size'], + 'help': 'Large files are split up into smaller partitions in order to avoid timeout during ' + 'erasing flash. This option allows to overwrite the default partition size of ' + 'mkdfu.py.' + } + ], }, 'dfu-list': { - 'callback': dfu_target, + 'callback': dfu_list_target, 'short_help': 'List DFU capable devices', 'dependencies': [], }, @@ -42,7 +54,7 @@ def action_extensions(base_actions, project_path): 'names': ['--path'], 'default': '', 'help': 'Specify path to DFU device. The default empty path works if there is just one ' - 'ESP device with the same product identificator. See the device list for paths ' + 'ESP device with the same product identifier. See the device list for paths ' 'of available devices.' } ], diff --git a/tools/mkdfu.py b/tools/mkdfu.py index e521354fe1..980cf278ba 100755 --- a/tools/mkdfu.py +++ b/tools/mkdfu.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Copyright 2020 Espressif Systems (Shanghai) PTE LTD +# Copyright 2020-2021 Espressif Systems (Shanghai) CO LTD # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,6 +21,8 @@ # This file must be the first one in the archive. It contains binary structures describing each # subsequent file (for example, where the file needs to be flashed/loaded). +from __future__ import print_function, unicode_literals + import argparse import hashlib import json @@ -28,6 +30,7 @@ import os import struct import zlib from collections import namedtuple +from functools import partial from future.utils import iteritems @@ -125,16 +128,31 @@ def pad_bytes(b, multiple, padding=b'\x00'): # type: (bytes, int, bytes) -> byt class EspDfuWriter(object): - def __init__(self, dest_file, pid): # type: (typing.BinaryIO) -> None + def __init__(self, dest_file, pid, part_size): # type: (typing.BinaryIO, int, int) -> None self.dest = dest_file self.pid = pid + self.part_size = part_size self.entries = [] # type: typing.List[bytes] self.index = [] # type: typing.List[DFUInfo] def add_file(self, flash_addr, path): # type: (int, str) -> None - """ Add file to be written into flash at given address """ + """ + Add file to be written into flash at given address + + Files are split up into chunks in order avoid timing-out during erasing large regions. Instead of adding + "app.bin" at flash_addr it will add: + 1. app.bin at flash_addr # sizeof(app.bin) == self.part_size + 2. app.bin.1 at flash_addr + self.part_size + 3. app.bin.2 at flash_addr + 2 * self.part_size + ... + + """ + f_name = os.path.basename(path) with open(path, 'rb') as f: - self._add_cpio_flash_entry(os.path.basename(path), flash_addr, f.read()) + for i, chunk in enumerate(iter(partial(f.read, self.part_size), b'')): + n = f_name if i == 0 else '.'.join([f_name, str(i)]) + self._add_cpio_flash_entry(n, flash_addr, chunk) + flash_addr += len(chunk) def finish(self): # type: () -> None """ Write DFU file """ @@ -187,13 +205,15 @@ class EspDfuWriter(object): self.entries.insert(0, entry) -def action_write(args): - writer = EspDfuWriter(args['output_file'], args['pid']) +def action_write(args): # type: (typing.Mapping[str, typing.Any]) -> None + writer = EspDfuWriter(args['output_file'], args['pid'], args['part_size']) for addr, f in args['files']: print('Adding {} at {:#x}'.format(f, addr)) writer.add_file(addr, f) writer.finish() print('"{}" has been written. You may proceed with DFU flashing.'.format(args['output_file'].name)) + if args['part_size'] % (4 * 1024) != 0: + print('WARNING: Partition size of DFU is not multiple of 4k (4096). You might get unexpected behavior.') def main(): @@ -212,6 +232,10 @@ def main(): help='Hexa-decimal product indentificator') write_parser.add_argument('--json', help='Optional file for loading "flash_files" dictionary with
items') + write_parser.add_argument('--part-size', + default=os.environ.get('ESP_DFU_PART_SIZE', 512 * 1024), + type=lambda x: int(x, 0), + help='Larger files are split-up into smaller partitions of this size') write_parser.add_argument('files', metavar='
', help='Add at
', nargs='*') @@ -241,12 +265,13 @@ def main(): files += [(int(addr, 0), process_json_file(f_name)) for addr, f_name in iteritems(json.load(f)['flash_files'])] - files = sorted([(addr, f_name) for addr, f_name in iteritems(dict(files))], + files = sorted([(addr, f_name.decode('utf-8') if isinstance(f_name, type(b'')) else f_name) for addr, f_name in iteritems(dict(files))], key=lambda x: x[0]) # remove possible duplicates and sort based on the address cmd_args = {'output_file': args.output_file, 'files': files, 'pid': args.pid, + 'part_size': args.part_size, } {'write': action_write diff --git a/tools/test_mkdfu/1/flasher_args.json b/tools/test_mkdfu/1/flasher_args.json deleted file mode 100644 index 06a54d0cc7..0000000000 --- a/tools/test_mkdfu/1/flasher_args.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "flash_files" : { - "0x8000" : "2.bin", - "0x1000" : "1.bin", - "0x10000" : "3.bin" - } -} diff --git a/tools/test_mkdfu/2/dfu.bin b/tools/test_mkdfu/2/dfu.bin new file mode 100644 index 0000000000000000000000000000000000000000..31774a80cfcbcb5206ff02925e98d32484bb9d5d GIT binary patch literal 10256 zcmeIwF$%&k6oBC>IO!lbxVUt6C@H8FH$erVQ>h0~N}(Wi(aFsd2zmjp;sHE@XYdBK z(x$Yc6)HH$|CYSuH80_7gdq%}i&XUWc_%f^Nf$Y_Ov#Zm$K80oyAql+Un@y0JU_{c zB>Ua1)>vM*LDf994)(DUHwpV{dXa|lp69-yRADR8l?ETi#6D8FJR-c$PtNYiyG3&E z&z9f47&C9_Mg84U@|bLEI5o$Xs;Xc7I<_K!00IagfB*srAb