From 88c81c67b7dae1144b949d184672c6b1349afeab Mon Sep 17 00:00:00 2001 From: Renz Christian Bagaporo Date: Fri, 16 Nov 2018 04:59:37 +0800 Subject: [PATCH 1/5] partition_table: implement new parttool functionality --- components/partition_table/gen_esp32part.py | 7 +- components/partition_table/parttool.py | 306 +++++++++++++----- .../partition_table/project_include.cmake | 18 +- .../gen_esp32part_tests.py | 24 +- 4 files changed, 253 insertions(+), 102 deletions(-) diff --git a/components/partition_table/gen_esp32part.py b/components/partition_table/gen_esp32part.py index d029127abf..53a65efc08 100755 --- a/components/partition_table/gen_esp32part.py +++ b/components/partition_table/gen_esp32part.py @@ -35,6 +35,9 @@ MAX_PARTITION_LENGTH = 0xC00 # 3K for partition data (96 entries) leaves 1K in MD5_PARTITION_BEGIN = b"\xEB\xEB" + b"\xFF" * 14 # The first 2 bytes are like magic numbers for MD5 sum PARTITION_TABLE_SIZE = 0x1000 # Size of partition table +MIN_PARTITION_SUBTYPE_APP_OTA = 0x10 +NUM_PARTITION_SUBTYPE_APP_OTA = 16 + __version__ = '1.2' APP_TYPE = 0x00 @@ -254,8 +257,8 @@ class PartitionDefinition(object): } # add subtypes for the 16 OTA slot values ("ota_XX, etc.") - for ota_slot in range(16): - SUBTYPES[TYPES["app"]]["ota_%d" % ota_slot] = 0x10 + ota_slot + for ota_slot in range(NUM_PARTITION_SUBTYPE_APP_OTA): + SUBTYPES[TYPES["app"]]["ota_%d" % ota_slot] = MIN_PARTITION_SUBTYPE_APP_OTA + ota_slot def __init__(self): self.name = "" diff --git a/components/partition_table/parttool.py b/components/partition_table/parttool.py index 83c5ce9d9d..7153018edb 100755 --- a/components/partition_table/parttool.py +++ b/components/partition_table/parttool.py @@ -1,9 +1,7 @@ #!/usr/bin/env python # -# parttool returns info about the required partition. -# -# This utility is used by the make system to get information -# about the start addresses: partition table, factory area, phy area. +# parttool is used to perform partition level operations - reading, +# writing, erasing and getting info about the partition. # # Copyright 2018 Espressif Systems (Shanghai) PTE LTD # @@ -21,113 +19,259 @@ from __future__ import print_function, division import argparse import os -import re -import struct import sys -import hashlib -import binascii +import subprocess +import tempfile import gen_esp32part as gen __version__ = '1.0' +IDF_COMPONENTS_PATH = os.path.expandvars(os.path.join("$IDF_PATH", "components")) + +ESPTOOL_PY = os.path.join(IDF_COMPONENTS_PATH, "esptool_py", "esptool", "esptool.py") + quiet = False def status(msg): """ Print status message to stderr """ if not quiet: - critical(msg) + print(msg) -def critical(msg): - """ Print critical message to stderr """ - if not quiet: - sys.stderr.write(msg) - sys.stderr.write('\n') - -def main(): - global quiet - parser = argparse.ArgumentParser(description='Returns info about the required partition.') - parser.add_argument('--quiet', '-q', help="Don't print status messages to stderr", action='store_true') +def _invoke_esptool(esptool_args, args): + m_esptool_args = [sys.executable, ESPTOOL_PY] - parser.add_argument('--partition-table-offset', help='The offset of the partition table in flash. Only consulted if partition table is in CSV format.', type=str, default='0x8000') + if args.port != "": + m_esptool_args.extend(["--port", args.port]) - search_type = parser.add_mutually_exclusive_group() - search_type.add_argument('--partition-name', '-p', help='The name of the required partition', type=str, default=None) - search_type.add_argument('--type', '-t', help='The type of the required partition', type=str, default=None) - search_type.add_argument('--default-boot-partition', help='Select the default boot partition, '+ - 'using the same fallback logic as the IDF bootloader', action="store_true") + m_esptool_args.extend(esptool_args) - parser.add_argument('--subtype', '-s', help='The subtype of the required partition', type=str, default=None) + if quiet: + with open(os.devnull, "w") as fnull: + subprocess.check_call(m_esptool_args, stdout=fnull, stderr=fnull) + else: + subprocess.check_call(m_esptool_args) - parser.add_argument('--offset', '-o', help='Return offset of required partition', action="store_true") - parser.add_argument('--size', help='Return size of required partition', action="store_true") - parser.add_argument('input', help='Path to CSV or binary file to parse. Will use stdin if omitted.', - type=argparse.FileType('rb'), default=sys.stdin) - - args = parser.parse_args() - - if args.type is not None and args.subtype is None: - status("If --type is specified, --subtype is required") - return 2 - if args.type is None and args.subtype is not None: - status("--subtype is only used with --type") - return 2 - - quiet = args.quiet +def _get_partition_table(args): + partition_table = None gen.offset_part_table = int(args.partition_table_offset, 0) - input = args.input.read() - input_is_binary = input[0:2] == gen.PartitionDefinition.MAGIC_BYTES - if input_is_binary: - status("Parsing binary partition input...") - table = gen.PartitionTable.from_binary(input) + if args.partition_table_file: + status("Reading partition table from partition table file...") + + try: + with open(args.partition_table_file, "rb") as partition_table_file: + partition_table = gen.PartitionTable.from_binary(partition_table_file.read()) + status("Partition table read from binary file {}".format(partition_table_file.name)) + except (gen.InputError, TypeError): + with open(args.partition_table_file, "r") as partition_table_file: + partition_table_file.seek(0) + partition_table = gen.PartitionTable.from_csv(partition_table_file.read()) + status("Partition table read from CSV file {}".format(partition_table_file.name)) else: - input = input.decode() - status("Parsing CSV input...") - table = gen.PartitionTable.from_csv(input) + port_info = (" on port " + args.port if args.port else "") + status("Reading partition table from device{}...".format(port_info)) + with tempfile.NamedTemporaryFile() as partition_table_file: + invoke_args = ["read_flash", str(gen.offset_part_table), str(gen.MAX_PARTITION_LENGTH), partition_table_file.name] + _invoke_esptool(invoke_args, args) + partition_table = gen.PartitionTable.from_binary(partition_table_file.read()) + status("Partition table read from device" + port_info) - found_partition = None + return partition_table - if args.default_boot_partition: - search = [ "factory" ] + [ "ota_%d" % d for d in range(16) ] + +def _get_partition(args): + partition_table = _get_partition_table(args) + + partition = None + + if args.partition_name: + partition = partition_table.find_by_name(args.partition_name) + elif args.partition_type and args.partition_subtype: + partition = partition_table.find_by_type(args.partition_type, args.partition_subtype) + elif args.partition_boot_default: + search = [ "factory" ] + [ "ota_{}".format(d) for d in range(16) ] for subtype in search: - found_partition = table.find_by_type("app", subtype) - if found_partition is not None: + partition = partition_table.find_by_type("app", subtype) + if partition is not None: break - elif args.partition_name is not None: - found_partition = table.find_by_name(args.partition_name) - elif args.type is not None: - found_partition = table.find_by_type(args.type, args.subtype) else: - raise RuntimeError("invalid partition selection choice") + raise RuntimeError("Invalid partition selection arguments. Specify --partition-name OR \ + --partition-type and --partition-subtype OR --partition--boot-default.") - if found_partition is None: - return 1 # nothing found + if partition: + status("Found partition {}".format(str(partition))) - if args.offset: - print('0x%x' % (found_partition.offset)) - if args.size: - print('0x%x' % (found_partition.size)) - - return 0 - -class InputError(RuntimeError): - def __init__(self, e): - super(InputError, self).__init__(e) + return partition -class ValidationError(InputError): - def __init__(self, partition, message): - super(ValidationError, self).__init__( - "Partition %s invalid: %s" % (partition.name, message)) +def _get_and_check_partition(args): + partition = None + + partition = _get_partition(args) + + if not partition: + raise RuntimeError("Unable to find specified partition.") + + return partition + + +def write_partition(args): + erase_partition(args) + + partition = _get_and_check_partition(args) + + status("Checking input file size...") + + with open(args.input, "rb") as input_file: + content_len = len(input_file.read()) + + if content_len != partition.size: + status("File size (0x{:x}) does not match partition size (0x{:x})".format(content_len, partition.size)) + else: + status("File size matches partition size (0x{:x})".format(partition.size)) + + _invoke_esptool(["write_flash", str(partition.offset), args.input], args) + + status("Written contents of file '{}' to device at offset 0x{:x}".format(args.input, partition.offset)) + + +def read_partition(args): + partition = _get_and_check_partition(args) + _invoke_esptool(["read_flash", str(partition.offset), str(partition.size), args.output], args) + status("Read partition contents from device at offset 0x{:x} to file '{}'".format(partition.offset, args.output)) + + +def erase_partition(args): + partition = _get_and_check_partition(args) + _invoke_esptool(["erase_region", str(partition.offset), str(partition.size)], args) + status("Erased partition at offset 0x{:x} on device".format(partition.offset)) + + +def get_partition_info(args): + partition = None + + if args.table: + partition_table = _get_partition_table(args) + + if args.table.endswith(".csv"): + partition_table = partition_table.to_csv() + else: + partition_table = partition_table.to_binary() + + with open(args.table, "wb") as table_file: + table_file.write(partition_table) + status("Partition table written to " + table_file.name) + else: + partition = _get_partition(args) + + if partition: + info_dict = { + "offset" : '0x{:x}'.format(partition.offset), + "size": '0x{:x}'.format(partition.size) + } + + infos = [] + + try: + for info in args.info: + infos += [info_dict[info]] + except KeyError: + raise RuntimeError("Request for unknown partition info {}".format(info)) + + status("Requested partition information [{}]:".format( ", ".join(args.info))) + print(" ".join(infos)) + else: + status("Partition not found") + + +def generate_blank_partition_file(args): + output = None + stdout_binary = None + + partition = _get_and_check_partition(args) + output = b"\xFF" * partition.size + + try: + stdout_binary = sys.stdout.buffer # Python 3 + except AttributeError: + stdout_binary = sys.stdout + + with stdout_binary if args.output == "" else open(args.output, 'wb') as f: + f.write(output) + status("Blank partition file '{}' generated".format(args.output)) + + +def main(): + global quiet + + parser = argparse.ArgumentParser("ESP-IDF Partitions Tool") + + parser.add_argument("--quiet", "-q", help="suppress stderr messages", action="store_true") + + # There are two possible sources for the partition table: a device attached to the host + # or a partition table CSV/binary file. These sources are mutually exclusive. + partition_table_info_source_args = parser.add_mutually_exclusive_group() + + partition_table_info_source_args.add_argument("--port", "-p", help="port where the device to read the partition table from is attached", default="") + partition_table_info_source_args.add_argument("--partition-table-file", "-f", help="file (CSV/binary) to read the partition table from") + + parser.add_argument("--partition-table-offset", "-o", help="offset to read the partition table from", default="0x8000") + + # Specify what partition to perform the operation on. This can either be specified using the + # partition name or the first partition that matches the specified type/subtype + partition_selection_args = parser.add_mutually_exclusive_group() + + partition_selection_args.add_argument("--partition-name", "-n", help="name of the partition") + partition_selection_args.add_argument("--partition-type", "-t", help="type of the partition") + partition_selection_args.add_argument('--partition-boot-default', "-d", help='select the default boot partition, '+ + 'using the same fallback logic as the IDF bootloader', action="store_true") + + parser.add_argument("--partition-subtype", "-s", help="subtype of the partition") + + subparsers = parser.add_subparsers(dest="operation", help="run parttool -h for additional help") + + # Specify the supported operations + read_part_subparser = subparsers.add_parser("read_partition", help="read partition from device and dump contents into a file") + read_part_subparser.add_argument("--output", help="file to dump the read partition contents to") + + write_part_subparser = subparsers.add_parser("write_partition", help="write contents of a binary file to partition on device") + write_part_subparser.add_argument("--input", help="file whose contents are to be written to the partition offset") + + subparsers.add_parser("erase_partition", help="erase the contents of a partition on the device") + + print_partition_info_subparser = subparsers.add_parser("get_partition_info", help="get partition information") + print_partition_info_subparser_info_type = print_partition_info_subparser.add_mutually_exclusive_group() + print_partition_info_subparser_info_type.add_argument("--info", help="type of partition information to get", nargs="+") + print_partition_info_subparser_info_type.add_argument("--table", help="dump the partition table to a file") + + generate_blank_subparser = subparsers.add_parser("generate_blank_partition_file", help="generate a blank (all 0xFF) partition file of the specified partition that can be flashed to the device") + generate_blank_subparser.add_argument("--output", help="blank partition file filename") + + args = parser.parse_args() + + quiet = args.quiet + + # No operation specified, display help and exit + if args.operation is None: + if not quiet: + parser.print_help() + sys.exit(1) + + # Else execute the operation + operation_func = globals()[args.operation] + + if quiet: + # If exceptions occur, suppress and exit quietly + try: + operation_func(args) + except Exception: + sys.exit(2) + else: + operation_func(args) if __name__ == '__main__': - try: - r = main() - sys.exit(r) - except InputError as e: - print(e, file=sys.stderr) - sys.exit(2) + main() \ No newline at end of file diff --git a/components/partition_table/project_include.cmake b/components/partition_table/project_include.cmake index 80841ed742..9d564ca51f 100644 --- a/components/partition_table/project_include.cmake +++ b/components/partition_table/project_include.cmake @@ -28,13 +28,13 @@ endif() set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${PARTITION_CSV_PATH}) # Parse the partition table to get variable partition offsets & sizes which must be known at CMake runtime -function(get_partition_info variable get_part_info_args) +function(get_partition_info variable get_part_info_args part_info) separate_arguments(get_part_info_args) execute_process(COMMAND ${PYTHON} ${COMPONENT_PATH}/parttool.py -q --partition-table-offset ${PARTITION_TABLE_OFFSET} - ${get_part_info_args} - ${PARTITION_CSV_PATH} + --partition-table-file ${PARTITION_CSV_PATH} + ${get_part_info_args} get_partition_info --info ${part_info} OUTPUT_VARIABLE result RESULT_VARIABLE exit_code OUTPUT_STRIP_TRAILING_WHITESPACE) @@ -46,15 +46,19 @@ function(get_partition_info variable get_part_info_args) endfunction() if(CONFIG_ESP32_PHY_INIT_DATA_IN_PARTITION) - get_partition_info(PHY_PARTITION_OFFSET "--type data --subtype phy --offset") + get_partition_info(PHY_PARTITION_OFFSET + "--partition-type data --partition-subtype phy" "offset") set(PHY_PARTITION_BIN_FILE "esp32/phy_init_data.bin") endif() -get_partition_info(APP_PARTITION_OFFSET "--default-boot-partition --offset") +get_partition_info(APP_PARTITION_OFFSET + "--partition-boot-default" "offset") -get_partition_info(OTADATA_PARTITION_OFFSET "--type data --subtype ota --offset") +get_partition_info(OTADATA_PARTITION_OFFSET + "--partition-type data --partition-subtype ota" "offset") -get_partition_info(OTADATA_PARTITION_SIZE "--type data --subtype ota --size") +get_partition_info(OTADATA_PARTITION_SIZE + "--partition-type data --partition-subtype ota" "size") endif() diff --git a/components/partition_table/test_gen_esp32part_host/gen_esp32part_tests.py b/components/partition_table/test_gen_esp32part_host/gen_esp32part_tests.py index e6188dd160..bd61266283 100755 --- a/components/partition_table/test_gen_esp32part_host/gen_esp32part_tests.py +++ b/components/partition_table/test_gen_esp32part_host/gen_esp32part_tests.py @@ -398,13 +398,13 @@ app,app, factory, 32K, 1M class PartToolTests(Py23TestCase): - def _run_parttool(self, csvcontents, args): + def _run_parttool(self, csvcontents, args, info): csvpath = tempfile.mktemp() with open(csvpath, "w") as f: f.write(csvcontents) try: - output = subprocess.check_output([sys.executable, "../parttool.py"] + args.split(" ") + [ csvpath ], - stderr=subprocess.STDOUT) + output = subprocess.check_output([sys.executable, "../parttool.py"] + args.split(" ") + + ["--partition-table-file", csvpath , "get_partition_info", "--info", info], stderr=subprocess.STDOUT) self.assertNotIn(b"WARNING", output) m = re.search(b"0x[0-9a-fA-F]+", output) return m.group(0) if m else "" @@ -418,16 +418,16 @@ otadata, data, ota, 0xd000, 0x2000 phy_init, data, phy, 0xf000, 0x1000 factory, app, factory, 0x10000, 1M """ - rpt = lambda args: self._run_parttool(csv, args) + rpt = lambda args, info: self._run_parttool(csv, args, info) self.assertEqual( - rpt("--type data --subtype nvs --offset"), b"0x9000") + rpt("--partition-type=data --partition-subtype=nvs -q", "offset"), b"0x9000") self.assertEqual( - rpt("--type data --subtype nvs --size"), b"0x4000") + rpt("--partition-type=data --partition-subtype=nvs -q", "size"), b"0x4000") self.assertEqual( - rpt("--partition-name otadata --offset"), b"0xd000") + rpt("--partition-name=otadata -q", "offset"), b"0xd000") self.assertEqual( - rpt("--default-boot-partition --offset"), b"0x10000") + rpt("--partition-boot-default -q", "offset"), b"0x10000") def test_fallback(self): csv = """ @@ -437,15 +437,15 @@ phy_init, data, phy, 0xf000, 0x1000 ota_0, app, ota_0, 0x30000, 1M ota_1, app, ota_1, , 1M """ - rpt = lambda args: self._run_parttool(csv, args) + rpt = lambda args, info: self._run_parttool(csv, args, info) self.assertEqual( - rpt("--type app --subtype ota_1 --offset"), b"0x130000") + rpt("--partition-type=app --partition-subtype=ota_1 -q", "offset"), b"0x130000") self.assertEqual( - rpt("--default-boot-partition --offset"), b"0x30000") # ota_0 + rpt("--partition-boot-default -q", "offset"), b"0x30000") # ota_0 csv_mod = csv.replace("ota_0", "ota_2") self.assertEqual( - self._run_parttool(csv_mod, "--default-boot-partition --offset"), + self._run_parttool(csv_mod, "--partition-boot-default -q", "offset"), b"0x130000") # now default is ota_1 if __name__ =="__main__": From 8ca6904d5542fcf9773a7868c14edf7498a58e3a Mon Sep 17 00:00:00 2001 From: Renz Christian Bagaporo Date: Fri, 16 Nov 2018 05:00:27 +0800 Subject: [PATCH 2/5] ota: implement otatool functionality --- components/app_update/dump_otadata.py | 88 ----- components/app_update/gen_empty_partition.py | 82 ----- components/app_update/otatool.py | 327 +++++++++++++++++++ examples/system/ota/README.md | 4 +- tools/ci/executable-list.txt | 2 +- 5 files changed, 330 insertions(+), 173 deletions(-) delete mode 100755 components/app_update/dump_otadata.py delete mode 100755 components/app_update/gen_empty_partition.py create mode 100755 components/app_update/otatool.py diff --git a/components/app_update/dump_otadata.py b/components/app_update/dump_otadata.py deleted file mode 100755 index 040bd2b032..0000000000 --- a/components/app_update/dump_otadata.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python -# -# gen_otadata prints info about the otadata partition. -# -# Copyright 2018 Espressif Systems (Shanghai) PTE LTD -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http:#www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from __future__ import print_function, division -import argparse -import os -import re -import struct -import sys -import hashlib -import binascii - -__version__ = '1.0' - -quiet = False - -def status(msg): - """ Print status message to stderr """ - if not quiet: - critical(msg) - -def critical(msg): - """ Print critical message to stderr """ - if not quiet: - sys.stderr.write(msg) - sys.stderr.write('\n') - -def little_endian(buff, offset): - data = buff[offset:offset+4] - data.reverse() - data = ''.join(data) - return data - -def main(): - global quiet - parser = argparse.ArgumentParser(description='Prints otadata partition in human readable form.') - - parser.add_argument('--quiet', '-q', help="Don't print status messages to stderr", action='store_true') - - search_type = parser.add_mutually_exclusive_group() - - parser.add_argument('input', help='Path to binary file containing otadata partition to parse.', - type=argparse.FileType('rb')) - - args = parser.parse_args() - - quiet = args.quiet - - input = args.input.read() - - hex_input_0 = binascii.hexlify(input) - hex_input_0 = map(''.join, zip(*[iter(hex_input_0)]*2)) - hex_input_1 = binascii.hexlify(input[4096:]) - hex_input_1 = map(''.join, zip(*[iter(hex_input_1)]*2)) - - print("\t%11s\t%8s |\t%8s\t%8s" %("OTA_SEQ", "CRC", "OTA_SEQ", "CRC")) - print("Firmware: 0x%s \t 0x%s |\t0x%s \t 0x%s" % (little_endian(hex_input_0, 0), little_endian(hex_input_0, 28), \ - little_endian(hex_input_1, 0), little_endian(hex_input_1, 28))) -class InputError(RuntimeError): - def __init__(self, e): - super(InputError, self).__init__(e) - -class ValidationError(InputError): - def __init__(self, partition, message): - super(ValidationError, self).__init__( - "Partition %s invalid: %s" % (partition.name, message)) - -if __name__ == '__main__': - try: - r = main() - sys.exit(r) - except InputError as e: - print(e, file=sys.stderr) - sys.exit(2) diff --git a/components/app_update/gen_empty_partition.py b/components/app_update/gen_empty_partition.py deleted file mode 100755 index 365b2ada3c..0000000000 --- a/components/app_update/gen_empty_partition.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python -# -# generates an empty binary file -# -# This tool generates an empty binary file of the required size. -# -# Copyright 2018 Espressif Systems (Shanghai) PTE LTD -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http:#www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from __future__ import print_function, division -from __future__ import unicode_literals -import argparse -import os -import re -import struct -import sys -import hashlib -import binascii - -__version__ = '1.0' - -quiet = False - -def status(msg): - """ Print status message to stderr """ - if not quiet: - critical(msg) - -def critical(msg): - """ Print critical message to stderr """ - if not quiet: - sys.stderr.write(msg) - sys.stderr.write('\n') - -def generate_blanked_file(size, output_path): - output = b"\xFF" * size - try: - stdout_binary = sys.stdout.buffer # Python 3 - except AttributeError: - stdout_binary = sys.stdout - with stdout_binary if output_path == '-' else open(output_path, 'wb') as f: - f.write(output) - -def main(): - global quiet - parser = argparse.ArgumentParser(description='Generates an empty binary file of the required size.') - - parser.add_argument('--quiet', '-q', help="Don't print status messages to stderr", action='store_true') - - parser.add_argument('--size', help='Size of generated the file', type=str, required=True) - - parser.add_argument('output', help='Path for binary file.', nargs='?', default='-') - args = parser.parse_args() - - quiet = args.quiet - - size = int(args.size, 0) - if size > 0 : - generate_blanked_file(size, args.output) - return 0 - -class InputError(RuntimeError): - def __init__(self, e): - super(InputError, self).__init__(e) - -if __name__ == '__main__': - try: - r = main() - sys.exit(r) - except InputError as e: - print(e, file=sys.stderr) - sys.exit(2) diff --git a/components/app_update/otatool.py b/components/app_update/otatool.py new file mode 100755 index 0000000000..f7216b0dc1 --- /dev/null +++ b/components/app_update/otatool.py @@ -0,0 +1,327 @@ +#!/usr/bin/env python +# +# otatool is used to perform ota-level operations - flashing ota partition +# erasing ota partition and switching ota partition +# +# Copyright 2018 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import print_function, division +import argparse +import os +import sys +import binascii +import subprocess +import tempfile +import collections +import struct + +__version__ = '1.0' + +IDF_COMPONENTS_PATH = os.path.expandvars(os.path.join("$IDF_PATH", "components")) + +PARTTOOL_PY = os.path.join(IDF_COMPONENTS_PATH, "partition_table", "parttool.py") + +SPI_FLASH_SEC_SIZE = 0x2000 + +quiet = False + + +def status(msg): + if not quiet: + print(msg) + + +def _invoke_parttool(parttool_args, args, output=False, partition=None): + invoke_args = [] + + if partition: + invoke_args += [sys.executable, PARTTOOL_PY] + partition + else: + invoke_args += [sys.executable, PARTTOOL_PY, "--partition-type", "data", "--partition-subtype", "ota"] + + if quiet: + invoke_args += ["-q"] + + if args.port != "": + invoke_args += ["--port", args.port] + + if args.partition_table_file: + invoke_args += ["--partition-table-file", args.partition_table_file] + + if args.partition_table_offset: + invoke_args += ["--partition-table-offset", args.partition_table_offset] + + invoke_args += parttool_args + + if output: + return subprocess.check_output(invoke_args) + else: + return subprocess.check_call(invoke_args) + + +def _get_otadata_contents(args, check=True): + global quiet + + if check: + check_args = ["get_partition_info", "--info", "offset", "size"] + + quiet = True + output = _invoke_parttool(check_args, args, True).split(b" ") + quiet = args.quiet + + if not output: + raise RuntimeError("No ota_data partition found") + + with tempfile.NamedTemporaryFile() as otadata_file: + invoke_args = ["read_partition", "--output", otadata_file.name] + _invoke_parttool(invoke_args, args) + return otadata_file.read() + + +def _get_otadata_status(otadata_contents): + status = [] + + otadata_status = collections.namedtuple("otadata_status", "seq crc") + + for i in range(2): + start = i * (SPI_FLASH_SEC_SIZE >> 1) + + seq = bytearray(otadata_contents[start:start + 4]) + crc = bytearray(otadata_contents[start + 28:start + 32]) + + seq = struct.unpack('>I', seq) + crc = struct.unpack('>I', crc) + + status.append(otadata_status(seq[0], crc[0])) + + return status + + +def read_otadata(args): + status("Reading ota_data partition contents...") + otadata_info = _get_otadata_contents(args) + otadata_info = _get_otadata_status(otadata_info) + + print(otadata_info) + + print("\t\t{:11}\t{:8s}|\t{:8s}\t{:8s}".format("OTA_SEQ", "CRC", "OTA_SEQ", "CRC")) + print("Firmware: 0x{:8x} \t 0x{:8x} |\t0x{:8x} \t 0x{:8x}".format(otadata_info[0].seq, otadata_info[0].crc, + otadata_info[1].seq, otadata_info[1].crc)) + + +def erase_otadata(args): + status("Erasing ota_data partition contents...") + _invoke_parttool(["erase_partition"], args) + status("Erased ota_data partition contents") + + +def switch_otadata(args): + sys.path.append(os.path.join(IDF_COMPONENTS_PATH, "partition_table")) + import gen_esp32part as gen + + def is_otadata_status_valid(status): + seq = status.seq % (1 << 32) + crc = hex(binascii.crc32(struct.pack("I", seq), 0xFFFFFFFF) % (1 << 32)) + return seq < (int('0xFFFFFFFF', 16) % (1 << 32)) and status.crc == crc + + status("Looking for ota app partitions...") + + # In order to get the number of ota app partitions, we need the partition table + partition_table = None + + with tempfile.NamedTemporaryFile() as partition_table_file: + invoke_args = ["get_partition_info", "--table", partition_table_file.name] + + _invoke_parttool(invoke_args, args) + + partition_table = partition_table_file.read() + partition_table = gen.PartitionTable.from_binary(partition_table) + + ota_partitions = list() + + for i in range(gen.NUM_PARTITION_SUBTYPE_APP_OTA): + ota_partition = filter(lambda p: p.subtype == (gen.MIN_PARTITION_SUBTYPE_APP_OTA + i), partition_table) + + try: + ota_partitions.append(list(ota_partition)[0]) + except IndexError: + break + + ota_partitions = sorted(ota_partitions, key=lambda p: p.subtype) + + if not ota_partitions: + raise RuntimeError("No ota app partitions found") + + status("Verifying partition to switch to exists...") + + # Look for the app partition to switch to + ota_partition_next = None + + try: + if args.name: + ota_partition_next = filter(lambda p: p.name == args.name, ota_partitions) + else: + ota_partition_next = filter(lambda p: p.subtype - gen.MIN_PARTITION_SUBTYPE_APP_OTA == args.slot, ota_partitions) + + ota_partition_next = list(ota_partition_next)[0] + except IndexError: + raise RuntimeError("Partition to switch to not found") + + otadata_contents = _get_otadata_contents(args) + otadata_status = _get_otadata_status(otadata_contents) + + # Find the copy to base the computation for ota sequence number on + otadata_compute_base = -1 + + # Both are valid, take the max as computation base + if is_otadata_status_valid(otadata_status[0]) and is_otadata_status_valid(otadata_status[1]): + if otadata_status[0].seq >= otadata_status[1].seq: + otadata_compute_base = 0 + else: + otadata_compute_base = 1 + # Only one copy is valid, use that + elif is_otadata_status_valid(otadata_status[0]): + otadata_compute_base = 0 + elif is_otadata_status_valid(otadata_status[1]): + otadata_compute_base = 1 + # Both are invalid (could be initial state - all 0xFF's) + else: + pass + + ota_seq_next = 0 + ota_partitions_num = len(ota_partitions) + + target_seq = (ota_partition_next.subtype & 0x0F) + 1 + + # Find the next ota sequence number + if otadata_compute_base == 0 or otadata_compute_base == 1: + base_seq = otadata_status[otadata_compute_base].seq % (1 << 32) + + i = 0 + while base_seq > target_seq % ota_partitions_num + i * ota_partitions_num: + i += 1 + + ota_seq_next = target_seq % ota_partitions_num + i * ota_partitions_num + else: + ota_seq_next = target_seq + + # Create binary data from computed values + ota_seq_next = struct.pack("I", ota_seq_next) + ota_seq_crc_next = binascii.crc32(ota_seq_next, 0xFFFFFFFF) % (1 << 32) + ota_seq_crc_next = struct.pack("I", ota_seq_crc_next) + + with tempfile.NamedTemporaryFile() as otadata_next_file: + start = (1 if otadata_compute_base == 0 else 0) * (SPI_FLASH_SEC_SIZE >> 1) + + otadata_next_file.write(otadata_contents) + + otadata_next_file.seek(start) + otadata_next_file.write(ota_seq_next) + + otadata_next_file.seek(start + 28) + otadata_next_file.write(ota_seq_crc_next) + + otadata_next_file.flush() + + _invoke_parttool(["write_partition", "--input", otadata_next_file.name], args) + status("Updated ota_data partition") + + +def _get_partition_specifier(args): + if args.name: + return ["--partition-name", args.name] + else: + return ["--partition-type", "app", "--partition-subtype", "ota_" + str(args.slot)] + + +def read_ota_partition(args): + invoke_args = ["read_partition", "--output", args.output] + _invoke_parttool(invoke_args, args, partition=_get_partition_specifier(args)) + status("Read ota partition contents to file {}".format(args.output)) + + +def write_ota_partition(args): + invoke_args = ["write_partition", "--input", args.input] + _invoke_parttool(invoke_args, args, partition=_get_partition_specifier(args)) + status("Written contents of file {} to ota partition".format(args.input)) + + +def erase_ota_partition(args): + invoke_args = ["erase_partition"] + _invoke_parttool(invoke_args, args, partition=_get_partition_specifier(args)) + status("Erased contents of ota partition") + + +def main(): + global quiet + + parser = argparse.ArgumentParser("ESP-IDF OTA Partitions Tool") + + parser.add_argument("--quiet", "-q", help="suppress stderr messages", action="store_true") + + # There are two possible sources for the partition table: a device attached to the host + # or a partition table CSV/binary file. These sources are mutually exclusive. + partition_table_info_source_args = parser.add_mutually_exclusive_group() + + partition_table_info_source_args.add_argument("--port", "-p", help="port where the device to read the partition table from is attached", default="") + partition_table_info_source_args.add_argument("--partition-table-file", "-f", help="file (CSV/binary) to read the partition table from", default="") + + parser.add_argument("--partition-table-offset", "-o", help="offset to read the partition table from", default="0x8000") + + subparsers = parser.add_subparsers(dest="operation", help="run otatool -h for additional help") + + # Specify the supported operations + subparsers.add_parser("read_otadata", help="read otadata partition") + subparsers.add_parser("erase_otadata", help="erase otadata partition") + + slot_or_name_parser = argparse.ArgumentParser(add_help=False) + slot_or_name_parser_args = slot_or_name_parser.add_mutually_exclusive_group() + slot_or_name_parser_args.add_argument("--slot", help="slot number of the ota partition", type=int) + slot_or_name_parser_args.add_argument("--name", help="name of the ota partition") + + subparsers.add_parser("switch_otadata", help="switch otadata partition", parents=[slot_or_name_parser]) + + read_ota_partition_subparser = subparsers.add_parser("read_ota_partition", help="read contents of an ota partition", parents=[slot_or_name_parser]) + read_ota_partition_subparser.add_argument("--output", help="file to write the contents of the ota partition to") + + write_ota_partition_subparser = subparsers.add_parser("write_ota_partition", help="write contents to an ota partition", parents=[slot_or_name_parser]) + write_ota_partition_subparser.add_argument("--input", help="file whose contents to write to the ota partition") + + subparsers.add_parser("erase_ota_partition", help="erase contents of an ota partition", parents=[slot_or_name_parser]) + + args = parser.parse_args() + + quiet = args.quiet + + # No operation specified, display help and exit + if args.operation is None: + if not quiet: + parser.print_help() + sys.exit(1) + + # Else execute the operation + operation_func = globals()[args.operation] + + if quiet: + # If exceptions occur, suppress and exit quietly + try: + operation_func(args) + except Exception: + sys.exit(2) + else: + operation_func(args) + + +if __name__ == '__main__': + main() diff --git a/examples/system/ota/README.md b/examples/system/ota/README.md index 2641419380..a5b77ea03d 100644 --- a/examples/system/ota/README.md +++ b/examples/system/ota/README.md @@ -97,8 +97,8 @@ It allows to run the newly loaded app from a factory partition. make flash ``` -After first update, if you want to return back to factory app (or the first OTA partition, if factory partition is not present) then use the command `make erase_ota`. -It erases ota_data partition to initial. +After first update, if you want to return back to factory app (or the first OTA partition, if factory partition is not present) then use the command `make erase_otadata`. +It erases the ota_data partition to initial state. **Take note that this assumes that the partition table of this project is the one that is on the device**. ### Step 5: Run the OTA Example diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index 6c3756fe6f..deb554c90a 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -8,7 +8,7 @@ components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py components/partition_table/gen_esp32part.py components/partition_table/parttool.py components/app_update/gen_empty_partition.py -components/app_update/dump_otadata.py +components/app_update/otatool.py components/partition_table/test_gen_esp32part_host/gen_esp32part_tests.py components/ulp/esp32ulp_mapgen.py docs/check_doc_warnings.sh From bceec35d0ee543c8c3f43c0013ed80583d965cf4 Mon Sep 17 00:00:00 2001 From: Renz Christian Bagaporo Date: Fri, 16 Nov 2018 05:01:15 +0800 Subject: [PATCH 3/5] cmake: use otatool and parttool for build --- components/app_update/CMakeLists.txt | 17 +++++++-- components/app_update/project_include.cmake | 3 +- tools/idf.py | 42 +++++++++++---------- 3 files changed, 37 insertions(+), 25 deletions(-) diff --git a/components/app_update/CMakeLists.txt b/components/app_update/CMakeLists.txt index 8eac369358..e559ef1020 100644 --- a/components/app_update/CMakeLists.txt +++ b/components/app_update/CMakeLists.txt @@ -7,12 +7,21 @@ set(COMPONENT_PRIV_REQUIRES bootloader_support) register_component() # Add custom target for generating empty otadata partition for flashing -if(OTADATA_PARTITION_OFFSET AND IDF_BUILD_ARTIFACTS) +if(OTADATA_PARTITION_OFFSET AND OTADATA_PARTITION_SIZE) add_custom_command(OUTPUT "${IDF_BUILD_ARTIFACTS_DIR}/${BLANK_OTADATA_FILE}" - COMMAND ${PYTHON} ${CMAKE_CURRENT_SOURCE_DIR}/gen_empty_partition.py - --size ${OTADATA_PARTITION_SIZE} "${IDF_BUILD_ARTIFACTS_DIR}/${BLANK_OTADATA_FILE}") + COMMAND ${PYTHON} ${IDF_PATH}/components/partition_table/parttool.py + --partition-type data --partition-subtype ota -q + --partition-table-file ${PARTITION_CSV_PATH} generate_blank_partition_file + --output "${IDF_BUILD_ARTIFACTS_DIR}/${BLANK_OTADATA_FILE}") add_custom_target(blank_ota_data ALL DEPENDS "${IDF_BUILD_ARTIFACTS_DIR}/${BLANK_OTADATA_FILE}") - add_dependencies(flash blank_ota_data) endif() + +set(otatool_py ${PYTHON} ${COMPONENT_PATH}/otatool.py) + +add_custom_target(read_otadata DEPENDS "${PARTITION_CSV_PATH}" + COMMAND ${otatool_py} --partition-table-file ${PARTITION_CSV_PATH} read_otadata) + +add_custom_target(erase_otadata DEPENDS "${PARTITION_CSV_PATH}" + COMMAND ${otatool_py} --partition-table-file ${PARTITION_CSV_PATH} erase_otadata) diff --git a/components/app_update/project_include.cmake b/components/app_update/project_include.cmake index 8d959bd7cb..0f7001056a 100644 --- a/components/app_update/project_include.cmake +++ b/components/app_update/project_include.cmake @@ -3,6 +3,7 @@ # partition table # (NB: because of component dependency, we know partition_table # project_include.cmake has already been included.) -if(OTADATA_PARTITION_OFFSET AND IDF_BUILD_ARTIFACTS) + +if(OTADATA_PARTITION_OFFSET AND OTADATA_PARTITION_SIZE AND IDF_BUILD_ARTIFACTS) set(BLANK_OTADATA_FILE "ota_data_initial.bin") endif() diff --git a/tools/idf.py b/tools/idf.py index 830ff97133..5d61d3ec52 100755 --- a/tools/idf.py +++ b/tools/idf.py @@ -386,27 +386,29 @@ def print_closing_message(args): ACTIONS = { # action name : ( function (or alias), dependencies, order-only dependencies ) - "all" : ( build_target, [], [ "reconfigure", "menuconfig", "clean", "fullclean" ] ), - "build": ( "all", [], [] ), # build is same as 'all' target - "clean": ( clean, [], [ "fullclean" ] ), - "fullclean": ( fullclean, [], [] ), - "reconfigure": ( reconfigure, [], [ "menuconfig" ] ), - "menuconfig": ( build_target, [], [] ), + "all" : ( build_target, [], [ "reconfigure", "menuconfig", "clean", "fullclean" ] ), + "build": ( "all", [], [] ), # build is same as 'all' target + "clean": ( clean, [], [ "fullclean" ] ), + "fullclean": ( fullclean, [], [] ), + "reconfigure": ( reconfigure, [], [ "menuconfig" ] ), + "menuconfig": ( build_target, [], [] ), "defconfig": ( build_target, [], [] ), - "confserver": ( build_target, [], [] ), - "size": ( build_target, [ "app" ], [] ), - "size-components": ( build_target, [ "app" ], [] ), - "size-files": ( build_target, [ "app" ], [] ), - "bootloader": ( build_target, [], [] ), - "bootloader-clean": ( build_target, [], [] ), - "bootloader-flash": ( flash, [ "bootloader" ], [ "erase_flash"] ), - "app": ( build_target, [], [ "clean", "fullclean", "reconfigure" ] ), - "app-flash": ( flash, [ "app" ], [ "erase_flash"]), - "partition_table": ( build_target, [], [ "reconfigure" ] ), - "partition_table-flash": ( flash, [ "partition_table" ], [ "erase_flash" ]), - "flash": ( flash, [ "all" ], [ "erase_flash" ] ), - "erase_flash": ( erase_flash, [], []), - "monitor": ( monitor, [], [ "flash", "partition_table-flash", "bootloader-flash", "app-flash" ]), + "confserver": ( build_target, [], [] ), + "size": ( build_target, [ "app" ], [] ), + "size-components": ( build_target, [ "app" ], [] ), + "size-files": ( build_target, [ "app" ], [] ), + "bootloader": ( build_target, [], [] ), + "bootloader-clean": ( build_target, [], [] ), + "bootloader-flash": ( flash, [ "bootloader" ], [ "erase_flash"] ), + "app": ( build_target, [], [ "clean", "fullclean", "reconfigure" ] ), + "app-flash": ( flash, [ "app" ], [ "erase_flash"]), + "partition_table": ( build_target, [], [ "reconfigure" ] ), + "partition_table-flash": ( flash, [ "partition_table" ], [ "erase_flash" ]), + "flash": ( flash, [ "all" ], [ "erase_flash" ] ), + "erase_flash": ( erase_flash, [], []), + "monitor": ( monitor, [], [ "flash", "partition_table-flash", "bootloader-flash", "app-flash" ]), + "erase_otadata": ( build_target, [], []), + "read_otadata": ( build_target, [], []), } def get_commandline_options(): From 5e086980396998cf205f04ec898f851d256e1cac Mon Sep 17 00:00:00 2001 From: Renz Christian Bagaporo Date: Fri, 16 Nov 2018 05:01:41 +0800 Subject: [PATCH 4/5] make: use otatool and parttool for build --- components/app_update/Makefile.projbuild | 62 ++++++------------- components/partition_table/Makefile.projbuild | 12 ++-- make/project.mk | 3 +- 3 files changed, 30 insertions(+), 47 deletions(-) diff --git a/components/app_update/Makefile.projbuild b/components/app_update/Makefile.projbuild index f4cf1d0555..c9ee591373 100644 --- a/components/app_update/Makefile.projbuild +++ b/components/app_update/Makefile.projbuild @@ -1,60 +1,38 @@ # Generate partition binary # -.PHONY: dump_otadata erase_ota blank_ota_data +.PHONY: blank_ota_data erase_otadata read_otadata -GEN_EMPTY_PART := $(PYTHON) $(COMPONENT_PATH)/gen_empty_partition.py +OTATOOL_PY := $(PYTHON) $(COMPONENT_PATH)/otatool.py +PARTTOOL_PY := $(PYTHON) $(IDF_PATH)/components/partition_table/parttool.py + +# Generate blank partition file BLANK_OTA_DATA_FILE = $(BUILD_DIR_BASE)/ota_data_initial.bin -PARTITION_TABLE_LEN := 0xC00 -OTADATA_LEN := 0x2000 +$(BLANK_OTA_DATA_FILE): partition_table_get_info $(PARTITION_TABLE_BIN) | check_python_dependencies + $(shell if [ $(OTA_DATA_OFFSET) != "" ] && [ $(OTA_DATA_SIZE) != "" ]; then \ + $(PARTTOOL_PY) --partition-type data --partition-subtype ota --partition-table-file $(PARTITION_TABLE_BIN) \ + -q generate_blank_partition_file --output $(BLANK_OTA_DATA_FILE); \ + fi; ) + $(eval BLANK_OTA_DATA_FILE = $(shell if [ $(OTA_DATA_OFFSET) != "" ] && [ $(OTA_DATA_SIZE) != "" ]; then \ + echo $(BLANK_OTA_DATA_FILE); else echo " "; fi) ) -PARTITION_TABLE_ONCHIP_BIN_PATH := $(call dequote,$(abspath $(BUILD_DIR_BASE))) -PARTITION_TABLE_ONCHIP_BIN_NAME := "onchip_partition.bin" -OTADATA_ONCHIP_BIN_NAME := "onchip_otadata.bin" - -PARTITION_TABLE_ONCHIP_BIN := $(PARTITION_TABLE_ONCHIP_BIN_PATH)/$(call dequote,$(PARTITION_TABLE_ONCHIP_BIN_NAME)) -OTADATA_ONCHIP_BIN := $(PARTITION_TABLE_ONCHIP_BIN_PATH)/$(call dequote,$(OTADATA_ONCHIP_BIN_NAME)) - -PARTITION_TABLE_GET_BIN_CMD = $(ESPTOOLPY_SERIAL) read_flash $(PARTITION_TABLE_OFFSET) $(PARTITION_TABLE_LEN) $(PARTITION_TABLE_ONCHIP_BIN) -OTADATA_GET_BIN_CMD = $(ESPTOOLPY_SERIAL) read_flash $(OTADATA_OFFSET) $(OTADATA_LEN) $(OTADATA_ONCHIP_BIN) - -GEN_OTADATA = $(IDF_PATH)/components/app_update/dump_otadata.py -ERASE_OTADATA_CMD = $(ESPTOOLPY_SERIAL) erase_region $(OTADATA_OFFSET) $(OTADATA_LEN) +blank_ota_data: $(BLANK_OTA_DATA_FILE) # If there is no otadata partition, both OTA_DATA_OFFSET and BLANK_OTA_DATA_FILE # expand to empty values. ESPTOOL_ALL_FLASH_ARGS += $(OTA_DATA_OFFSET) $(BLANK_OTA_DATA_FILE) -$(PARTITION_TABLE_ONCHIP_BIN): - $(PARTITION_TABLE_GET_BIN_CMD) +erase_otadata: $(PARTITION_TABLE_BIN) partition_table_get_info | check_python_dependencies + $(OTATOOL_PY) --partition-table-file $(PARTITION_TABLE_BIN) erase_otadata -onchip_otadata_get_info: $(PARTITION_TABLE_ONCHIP_BIN) - $(eval OTADATA_OFFSET:=$(shell $(GET_PART_INFO) --type data --subtype ota --offset $(PARTITION_TABLE_ONCHIP_BIN))) - @echo $(if $(OTADATA_OFFSET), $(shell export OTADATA_OFFSET), $(shell rm -f $(PARTITION_TABLE_ONCHIP_BIN));$(error "ERROR: ESP32 does not have otadata partition.")) +read_otadata: $(PARTITION_TABLE_BIN) partition_table_get_info | check_python_dependencies + $(OTATOOL_PY) --partition-table-file $(PARTITION_TABLE_BIN) read_otadata -$(OTADATA_ONCHIP_BIN): - $(OTADATA_GET_BIN_CMD) - -dump_otadata: onchip_otadata_get_info $(OTADATA_ONCHIP_BIN) $(PARTITION_TABLE_ONCHIP_BIN) - @echo "otadata retrieved. Contents:" - @echo $(SEPARATOR) - $(GEN_OTADATA) $(OTADATA_ONCHIP_BIN) - @echo $(SEPARATOR) - rm -f $(PARTITION_TABLE_ONCHIP_BIN) - rm -f $(OTADATA_ONCHIP_BIN) - -$(BLANK_OTA_DATA_FILE): partition_table_get_info - $(GEN_EMPTY_PART) --size $(OTA_DATA_SIZE) $(BLANK_OTA_DATA_FILE) - $(eval BLANK_OTA_DATA_FILE = $(shell if [ $(OTA_DATA_SIZE) != 0 ]; then echo $(BLANK_OTA_DATA_FILE); else echo " "; fi) ) - -blank_ota_data: $(BLANK_OTA_DATA_FILE) - -erase_ota: partition_table_get_info | check_python_dependencies - @echo $(if $(OTA_DATA_OFFSET), "Erase ota_data [addr=$(OTA_DATA_OFFSET) size=$(OTA_DATA_SIZE)] ...", $(error "ERROR: Partition table does not have ota_data partition.")) - $(ESPTOOLPY_SERIAL) erase_region $(OTA_DATA_OFFSET) $(OTA_DATA_SIZE) +erase_ota: erase_otadata + @echo "WARNING: erase_ota is deprecated. Use erase_otadata instead." all: blank_ota_data flash: blank_ota_data clean: - rm -f $(BLANK_OTA_DATA_FILE) + rm -f $(BLANK_OTA_DATA_FILE) \ No newline at end of file diff --git a/components/partition_table/Makefile.projbuild b/components/partition_table/Makefile.projbuild index 71e26868b8..7128e6ab6f 100644 --- a/components/partition_table/Makefile.projbuild +++ b/components/partition_table/Makefile.projbuild @@ -63,10 +63,14 @@ $(PARTITION_TABLE_BIN_UNSIGNED): $(PARTITION_TABLE_CSV_PATH) $(SDKCONFIG_MAKEFIL all_binaries: $(PARTITION_TABLE_BIN) partition_table_get_info partition_table_get_info: $(PARTITION_TABLE_BIN) - $(eval PHY_DATA_OFFSET:=$(shell $(GET_PART_INFO) --type data --subtype phy --offset $(PARTITION_TABLE_BIN))) - $(eval APP_OFFSET:=$(shell $(GET_PART_INFO) --default-boot-partition --offset $(PARTITION_TABLE_BIN))) - $(eval OTA_DATA_SIZE := $(shell $(GET_PART_INFO) --type data --subtype ota --size $(PARTITION_TABLE_BIN) || echo 0)) - $(eval OTA_DATA_OFFSET := $(shell $(GET_PART_INFO) --type data --subtype ota --offset $(PARTITION_TABLE_BIN))) + $(eval PHY_DATA_OFFSET:=$(shell $(GET_PART_INFO) --partition-type data --partition-subtype phy \ + --partition-table-file $(PARTITION_TABLE_BIN) get_partition_info --info offset)) + $(eval APP_OFFSET:=$(shell $(GET_PART_INFO) --partition-boot-default \ + --partition-table-file $(PARTITION_TABLE_BIN) get_partition_info --info offset)) + $(eval OTA_DATA_OFFSET:=$(shell $(GET_PART_INFO) --partition-type data --partition-subtype ota \ + --partition-table-file $(PARTITION_TABLE_BIN) get_partition_info --info offset)) + $(eval OTA_DATA_SIZE:=$(shell $(GET_PART_INFO) --partition-type data --partition-subtype ota \ + --partition-table-file $(PARTITION_TABLE_BIN) get_partition_info --info size)) export APP_OFFSET export PHY_DATA_OFFSET diff --git a/make/project.mk b/make/project.mk index cf5548c0ce..feab67c5e2 100644 --- a/make/project.mk +++ b/make/project.mk @@ -34,7 +34,8 @@ help: @echo "make size-components, size-files - Finer-grained memory footprints" @echo "make size-symbols - Per symbol memory footprint. Requires COMPONENT=" @echo "make erase_flash - Erase entire flash contents" - @echo "make erase_ota - Erase ota_data partition. After that will boot first bootable partition (factory or OTAx)." + @echo "make erase_otadata - Erase ota_data partition; First bootable partition (factory or OTAx) will be used on next boot." + @echo " This assumes this project's partition table is the one flashed on the device." @echo "make monitor - Run idf_monitor tool to monitor serial output from app" @echo "make simple_monitor - Monitor serial output on terminal console" @echo "make list-components - List all components in the project" From b926764385de1fdf7687ee694b88f2b3dba93bd7 Mon Sep 17 00:00:00 2001 From: Renz Christian Bagaporo Date: Fri, 16 Nov 2018 05:02:34 +0800 Subject: [PATCH 5/5] examples: add otatool and parttool examples --- .flake8 | 2 + examples/storage/parttool/CMakeLists.txt | 6 + examples/storage/parttool/Makefile | 9 + examples/storage/parttool/README.md | 68 ++++++ examples/storage/parttool/example_test.py | 40 ++++ examples/storage/parttool/main/CMakeLists.txt | 4 + examples/storage/parttool/main/component.mk | 4 + .../storage/parttool/main/parttool_main.c | 22 ++ .../storage/parttool/partitions_example.csv | 6 + examples/storage/parttool/parttool_example.py | 207 ++++++++++++++++++ examples/storage/parttool/sdkconfig.defaults | 5 + examples/system/ota/otatool/CMakeLists.txt | 6 + examples/system/ota/otatool/Makefile | 9 + examples/system/ota/otatool/README.md | 70 ++++++ examples/system/ota/otatool/example_test.py | 43 ++++ .../system/ota/otatool/main/CMakeLists.txt | 4 + examples/system/ota/otatool/main/component.mk | 4 + .../system/ota/otatool/main/otatool_main.c | 26 +++ .../system/ota/otatool/otatool_example.py | 194 ++++++++++++++++ .../system/ota/otatool/sdkconfig.defaults | 4 + tools/ci/executable-list.txt | 2 + 21 files changed, 735 insertions(+) create mode 100644 examples/storage/parttool/CMakeLists.txt create mode 100644 examples/storage/parttool/Makefile create mode 100644 examples/storage/parttool/README.md create mode 100644 examples/storage/parttool/example_test.py create mode 100644 examples/storage/parttool/main/CMakeLists.txt create mode 100644 examples/storage/parttool/main/component.mk create mode 100644 examples/storage/parttool/main/parttool_main.c create mode 100644 examples/storage/parttool/partitions_example.csv create mode 100755 examples/storage/parttool/parttool_example.py create mode 100644 examples/storage/parttool/sdkconfig.defaults create mode 100644 examples/system/ota/otatool/CMakeLists.txt create mode 100644 examples/system/ota/otatool/Makefile create mode 100644 examples/system/ota/otatool/README.md create mode 100644 examples/system/ota/otatool/example_test.py create mode 100644 examples/system/ota/otatool/main/CMakeLists.txt create mode 100644 examples/system/ota/otatool/main/component.mk create mode 100644 examples/system/ota/otatool/main/otatool_main.c create mode 100755 examples/system/ota/otatool/otatool_example.py create mode 100644 examples/system/ota/otatool/sdkconfig.defaults diff --git a/.flake8 b/.flake8 index be4d0fd62b..c35d620a64 100644 --- a/.flake8 +++ b/.flake8 @@ -88,11 +88,13 @@ exclude = examples/provisioning/custom_config/components/custom_provisioning/python/custom_config_pb2.py, examples/provisioning/softap_prov/softap_prov_test.py, examples/provisioning/softap_prov/utils/wifi_tools.py, + examples/storage/parttool/example_test.py, examples/system/cpp_exceptions/example_test.py, examples/system/esp_event/default_event_loop/example_test.py, examples/system/esp_event/user_event_loops/example_test.py, examples/system/esp_timer/example_test.py, examples/system/light_sleep/example_test.py, + examples/system/ota/otatool/example_test.py, examples/wifi/iperf/iperf_test.py, examples/wifi/iperf/test_report.py, tools/check_python_dependencies.py, diff --git a/examples/storage/parttool/CMakeLists.txt b/examples/storage/parttool/CMakeLists.txt new file mode 100644 index 0000000000..2be2660074 --- /dev/null +++ b/examples/storage/parttool/CMakeLists.txt @@ -0,0 +1,6 @@ +# The following lines of boilerplate have to be in your project's CMakeLists +# in this exact order for cmake to work correctly +cmake_minimum_required(VERSION 3.5) + +include($ENV{IDF_PATH}/tools/cmake/project.cmake) +project(parttool) diff --git a/examples/storage/parttool/Makefile b/examples/storage/parttool/Makefile new file mode 100644 index 0000000000..24352c1e6a --- /dev/null +++ b/examples/storage/parttool/Makefile @@ -0,0 +1,9 @@ +# +# This is a project Makefile. It is assumed the directory this Makefile resides in is a +# project subdirectory. +# + +PROJECT_NAME := parttool + +include $(IDF_PATH)/make/project.mk + diff --git a/examples/storage/parttool/README.md b/examples/storage/parttool/README.md new file mode 100644 index 0000000000..b84203e03f --- /dev/null +++ b/examples/storage/parttool/README.md @@ -0,0 +1,68 @@ +# Partitions Tool Example + +This example demonstrates common operations the partitions tool [parttool.py](../../../components/partition_table/parttool.py) allows the user to perform: + +- reading, writing and erasing partitions, +- retrieving info on a certain partition, +- dumping the entire partition table, and +- generating a blank partition file. + +Users taking a look at this example should focus on the contents of the python script [parttool_example.py](parttool_example.py). The script contains programmatic invocations of [parttool.py](../../../components/partition_table/parttool.py) in Python for the operations mentioned above; and can serve as a guide for users wanting to do the same in their applications. + +The example performs the operations mentioned above in a straightforward manner: it performs writes to partitions and then verifies correct content +by reading it back. For partitions, contents are compared to the originally written file. For the partition table, contents are verified against the partition table CSV +file. An erased partition's contents is compared to a generated blank file. + +## How to use example + +### Build and Flash + +Before running the example script [parttool_example.py](parttool_example.py), it is necessary to build and flash the firmware using the usual means: + +```bash +# If using Make +make build flash + +# If using CMake +idf.py build flash +``` + +### Running [parttool_example.py](parttool_example.py) + +The example can be executed by running the script [parttool_example.py](parttool_example.py). Either run it directly using + +```bash +./parttool_example.py +``` + +or run it using + +```bash +python parttool_example.py +``` + +The script searches for valid target devices connected to the host and performs the operations on the first one it finds. To perform the operations on a specific device, specify the port it is attached to during script invocation: + +```bash +# The target device is attached to /dev/ttyUSB2, for example +python parttool_example.py --port /dev/ttyUSB2 +``` + +## Example output + +Running the script produces the following output: + +``` +Checking if device app binary matches built binary +Checking if device partition table matches partition table csv +Retrieving data partition offset and size +Found data partition at offset 0x110000 with size 0x10000 +Writing to data partition +Reading data partition +Erasing data partition +Generating blank data partition file +Reading data partition + +Partition tool operations performed successfully! +``` + diff --git a/examples/storage/parttool/example_test.py b/examples/storage/parttool/example_test.py new file mode 100644 index 0000000000..7808e5d136 --- /dev/null +++ b/examples/storage/parttool/example_test.py @@ -0,0 +1,40 @@ +from __future__ import print_function +import os +import sys +import subprocess + +test_fw_path = os.getenv('TEST_FW_PATH') +if test_fw_path and test_fw_path not in sys.path: + sys.path.insert(0, test_fw_path) + +import TinyFW +import IDF + + +@IDF.idf_example_test(env_tag='Example_WIFI') +def test_examples_parttool(env, extra_data): + dut = env.get_dut('parttool', 'examples/storage/parttool') + dut.start_app(False) + + # Verify factory firmware + dut.expect("Partitions Tool Example") + dut.expect("Example end") + + # Close connection to DUT + dut.receive_thread.exit() + dut.port_inst.close() + + # Run the example python script + script_path = os.path.join(os.getenv("IDF_PATH"), "examples", "storage", "parttool", "parttool_example.py") + + binary_path = "" + for config in dut.download_config: + if "parttool.bin" in config: + binary_path = config + break + + subprocess.check_call([sys.executable, script_path, "--binary", binary_path]) + + +if __name__ == '__main__': + test_examples_parttool() diff --git a/examples/storage/parttool/main/CMakeLists.txt b/examples/storage/parttool/main/CMakeLists.txt new file mode 100644 index 0000000000..a574d5ffe6 --- /dev/null +++ b/examples/storage/parttool/main/CMakeLists.txt @@ -0,0 +1,4 @@ +set(COMPONENT_SRCS "parttool_main.c") +set(COMPONENT_ADD_INCLUDEDIRS ".") + +register_component() \ No newline at end of file diff --git a/examples/storage/parttool/main/component.mk b/examples/storage/parttool/main/component.mk new file mode 100644 index 0000000000..a98f634eae --- /dev/null +++ b/examples/storage/parttool/main/component.mk @@ -0,0 +1,4 @@ +# +# "main" pseudo-component makefile. +# +# (Uses default behaviour of compiling all source files in directory, adding 'include' to include path.) diff --git a/examples/storage/parttool/main/parttool_main.c b/examples/storage/parttool/main/parttool_main.c new file mode 100644 index 0000000000..0006d86244 --- /dev/null +++ b/examples/storage/parttool/main/parttool_main.c @@ -0,0 +1,22 @@ +/* Partitions Tool Example + This example code is in the Public Domain (or CC0 licensed, at your option.) + + Unless required by applicable law or agreed to in writing, this + software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + CONDITIONS OF ANY KIND, either express or implied. +*/ + +#include +#include +#include +#include +#include "esp_err.h" +#include "esp_log.h" + +static const char *TAG = "example"; + +void app_main(void) +{ + ESP_LOGI(TAG, "Partitions Tool Example"); + ESP_LOGI(TAG, "Example end"); +} diff --git a/examples/storage/parttool/partitions_example.csv b/examples/storage/parttool/partitions_example.csv new file mode 100644 index 0000000000..f76d1ca37a --- /dev/null +++ b/examples/storage/parttool/partitions_example.csv @@ -0,0 +1,6 @@ +# Name, Type, SubType, Offset, Size, Flags +# Note: if you change the phy_init or app partition offset, make sure to change the offset in Kconfig.projbuild +nvs, data, nvs, 0x9000, 0x6000, +phy_init, data, phy, 0xf000, 0x1000, +factory, app, factory, 0x10000, 1M, +storage, data, spiffs, , 0x10000, diff --git a/examples/storage/parttool/parttool_example.py b/examples/storage/parttool/parttool_example.py new file mode 100755 index 0000000000..24b8fcd18f --- /dev/null +++ b/examples/storage/parttool/parttool_example.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python +# +# Demonstrates the use of parttool.py, a tool for performing partition level +# operations. +# +# Copyright 2018 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import sys +import subprocess +import argparse + +IDF_PATH = os.path.expandvars("$IDF_PATH") + +PARTTOOL_PY = os.path.join(IDF_PATH, "components", "partition_table", "parttool.py") + +PARTITION_TABLE_OFFSET = 0x8000 + +INVOKE_ARGS = [sys.executable, PARTTOOL_PY, "-q", "--partition-table-offset", str(PARTITION_TABLE_OFFSET)] + + +def sized_file_compare(file1, file2): + with open(file1, "rb") as f1: + with open(file2, "rb") as f2: + f1 = f1.read() + f2 = f2.read() + + if len(f1) < len(f2): + f2 = f2[:len(f1)] + else: + f1 = f1[:len(f2)] + + return f1 == f2 + + +def check(condition, message): + if not condition: + print("Error: " + message) + sys.exit(1) + + +def write_data_partition(size): + print("Writing to data partition") + with open("write.bin", "wb") as f: + # Create a file to write to the data partition with randomly generated content + f.write(os.urandom(int(size, 16))) + + # Invokes the command + # + # parttool.py --partition-table-offset 0x8000 -q --partition-name storage write_partition --input write.bin + # + # to write the contents of a file to a partition in the device. + invoke_args = INVOKE_ARGS + ["--partition-name", "storage", "write_partition", "--input", f.name] + subprocess.check_call(invoke_args) + return f.name + + +def read_data_partition(): + print("Reading data partition") + # Invokes the command + # + # parttool.py --partition-table-offset 0x8000 -q --partition-name storage read_partition --output read.bin + # + # to read the contents of a partition in the device, which is then written to a file. + f = "read.bin" + invoke_args = INVOKE_ARGS + ["--partition-name", "storage", "read_partition", "--output", f] + subprocess.check_call(invoke_args) + return f + + +def get_data_partition_info(): + print("Retrieving data partition offset and size") + # Invokes the command + # + # parttool.py --partition-table-offset 0x8000 -q --partition-name storage get_partition_info --info offset size + # + # to get the offset and size of a partition named 'storage'. + invoke_args = INVOKE_ARGS + ["--partition-name", "storage", "get_partition_info", "--info", "offset", "size"] + + (offset, size) = subprocess.check_output(invoke_args).strip().split(b" ") + return (offset, size) + + +def check_app(args): + print("Checking if device app binary matches built binary") + # Invokes the command + # + # parttool.py --partition-table-offset 0x8000 --partition-type app --partition-subtype factory read_partition --output app.bin" + # + # to read the app binary and write it to a file. The read app binary is compared to the built binary in the build folder. + invoke_args = INVOKE_ARGS + ["--partition-type", "app", "--partition-subtype", "factory", "read_partition", "--output", "app.bin"] + subprocess.check_call(invoke_args) + + app_same = sized_file_compare("app.bin", args.binary) + check(app_same, "Device app binary does not match built binary") + + +def check_partition_table(): + sys.path.append(os.path.join(IDF_PATH, "components", "partition_table")) + import gen_esp32part as gen + + print("Checking if device partition table matches partition table csv") + # Invokes the command + # + # parttool.py --partition-table-offset 0x8000 get_partition_info --table table.bin + # + # to read the device partition table and write it to a file. The read partition table is compared to + # the partition table csv. + invoke_args = INVOKE_ARGS + ["get_partition_info", "--table", "table.bin"] + subprocess.check_call(invoke_args) + + with open("table.bin", "rb") as read: + partition_table_csv = os.path.join(IDF_PATH, "examples", "storage", "parttool", "partitions_example.csv") + with open(partition_table_csv, "r") as csv: + read = gen.PartitionTable.from_binary(read.read()) + csv = gen.PartitionTable.from_csv(csv.read()) + check(read == csv, "Device partition table does not match csv partition table") + + +def erase_data_partition(): + print("Erasing data partition") + # Invokes the command + # + # parttool.py --partition-table-offset 0x8000 --partition-name storage erase_partition + # + # to erase the 'storage' partition. + invoke_args = INVOKE_ARGS + ["--partition-name", "storage", "erase_partition"] + subprocess.check_call(invoke_args) + + +def generate_blank_data_file(): + print("Generating blank data partition file") + + # Invokes the command + # + # parttool.py --partition-table-offset 0x8000 --partition-name storage generate_blank_partition_file --output blank.bin + # + # to generate a blank partition file and write it to a file. The blank partition file has the same size as the + # 'storage' partition. + f = "blank.bin" + invoke_args = INVOKE_ARGS + ["--partition-name", "storage", "generate_blank_partition_file", "--output", f] + subprocess.check_call(invoke_args) + return f + + +def main(): + global INVOKE_ARGS + + parser = argparse.ArgumentParser("ESP-IDF Partitions Tool Example") + + parser.add_argument("--port", "-p", help="port where the device to perform operations on is connected") + parser.add_argument("--binary", "-b", help="path to built example binary", default=os.path.join("build", "parttool.bin")) + + args = parser.parse_args() + + if args.port: + INVOKE_ARGS += ["--port", args.port] + + # Before proceeding, do checks to verify whether the app and partition table in the device matches the built binary and + # the generated partition table during build + check_app(args) + check_partition_table() + + # Get the offset and size of the data partition + (offset, size) = get_data_partition_info() + + print("Found data partition at offset %s with size %s" % (offset, size)) + + # Write a generated file of random bytes to the found data partition + written = write_data_partition(size) + + # Read back the contents of the data partition + read = read_data_partition() + + # Compare the written and read back data + data_same = sized_file_compare(read, written) + check(data_same, "Read contents of the data partition does not match written data") + + # Erase the data partition + erase_data_partition() + + # Read back the erase data partition, which should be all 0xFF's after erasure + read = read_data_partition() + + # Generate blank partition file (all 0xFF's) + blank = generate_blank_data_file() + + # Verify that the partition has been erased by comparing the contents to the generated blank file + data_same = sized_file_compare(read, blank) + check(data_same, "Erased data partition contents does not match blank partition file") + + print("\nPartition tool operations performed successfully!") + + +if __name__ == '__main__': + main() diff --git a/examples/storage/parttool/sdkconfig.defaults b/examples/storage/parttool/sdkconfig.defaults new file mode 100644 index 0000000000..61aebc7e90 --- /dev/null +++ b/examples/storage/parttool/sdkconfig.defaults @@ -0,0 +1,5 @@ +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions_example.csv" +CONFIG_PARTITION_TABLE_CUSTOM_APP_BIN_OFFSET=0x10000 +CONFIG_PARTITION_TABLE_FILENAME="partitions_example.csv" +CONFIG_APP_OFFSET=0x10000 \ No newline at end of file diff --git a/examples/system/ota/otatool/CMakeLists.txt b/examples/system/ota/otatool/CMakeLists.txt new file mode 100644 index 0000000000..e3cd4d944c --- /dev/null +++ b/examples/system/ota/otatool/CMakeLists.txt @@ -0,0 +1,6 @@ +# The following lines of boilerplate have to be in your project's CMakeLists +# in this exact order for cmake to work correctly +cmake_minimum_required(VERSION 3.5) + +include($ENV{IDF_PATH}/tools/cmake/project.cmake) +project(otatool) diff --git a/examples/system/ota/otatool/Makefile b/examples/system/ota/otatool/Makefile new file mode 100644 index 0000000000..2f02dfe789 --- /dev/null +++ b/examples/system/ota/otatool/Makefile @@ -0,0 +1,9 @@ +# +# This is a project Makefile. It is assumed the directory this Makefile resides in is a +# project subdirectory. +# + +PROJECT_NAME := otatool + +include $(IDF_PATH)/make/project.mk + diff --git a/examples/system/ota/otatool/README.md b/examples/system/ota/otatool/README.md new file mode 100644 index 0000000000..c14e1cfb0d --- /dev/null +++ b/examples/system/ota/otatool/README.md @@ -0,0 +1,70 @@ +# OTA Tool Example + +This example demonstrates common operations the OTA tool [otatool.py](../../../components/app_update/otatool.py) allows the user to perform: + +- reading, writing and erasing OTA partitions, +- switching boot partitions, and +- switching to factory partition. + +Users taking a look at this example should focus on the contents of the python script [otatool_example.py](otatool_example.py). The script contains programmatic invocations of the tool [otatool.py](../../../components/app_update/otatool.py) in Python for the operations mentioned above; and can serve as a guide for users wanting to do the same in their applications. + +The built application in this example outputs the currently running partition, whose output is used to verify if the tool switched OTA +partitions succesfully. The built application binary is written to all OTA partitions at the start of the example to be able to determine the running +partition for all switches performed. + +## How to use example + +### Build and Flash + +Before running the example script [otatool_example.py](otatool_example.py), it is necessary to build and flash the firmware using the usual means: + +```bash +# If using Make +make build flash + +# If using CMake +idf.py build flash +``` + +### Running [otatool_example.py](otatool_example.py) + +The example can be executed by running the script [otatool_example.py](otatool_example.py). Either run it directly using + +```bash +./otatool_example.py +``` + +or run it using + +```bash +python otatool_example.py +``` + +The script searches for valid target devices connected to the host and performs the operations on the first one it finds. This could present problems if there +are multiple viable target devices attached to the host. To perform the operations on a specific device, specify the port it is attached to during script invocation: + +```bash +# The target device is attached to /dev/ttyUSB2, for example +python otatool_example.py --port /dev/ttyUSB2 +``` + +## Example output + +Running the script produces the following output: + +``` +Writing factory firmware to ota_0 +Writing factory firmware to ota_1 +Checking written firmware to ota_0 and ota_1 match factory firmware +Switching to ota partition name factory +Switching to ota partition name factory +Switching to ota partition slot 0 +Switching to ota partition name ota_1 +Switching to ota partition slot 1 +Switching to ota partition name ota_0 +Switching to ota partition slot 0 +Switching to ota partition name factory +Switching to ota partition slot 1 + +OTA tool operations executed successfully! +``` diff --git a/examples/system/ota/otatool/example_test.py b/examples/system/ota/otatool/example_test.py new file mode 100644 index 0000000000..3bcab86ce9 --- /dev/null +++ b/examples/system/ota/otatool/example_test.py @@ -0,0 +1,43 @@ +from __future__ import print_function +import os +import sys +import subprocess + +# this is a test case write with tiny-test-fw. +# to run test cases outside tiny-test-fw, +# we need to set environment variable `TEST_FW_PATH`, +# then get and insert `TEST_FW_PATH` to sys path before import FW module +test_fw_path = os.getenv('TEST_FW_PATH') +if test_fw_path and test_fw_path not in sys.path: + sys.path.insert(0, test_fw_path) + +import TinyFW +import IDF + + +@IDF.idf_example_test(env_tag='Example_WIFI') +def test_otatool_example(env, extra_data): + dut = env.get_dut('otatool', 'examples/system/ota/otatool') + + # Verify factory firmware + dut.start_app() + dut.expect("OTA Tool Example") + dut.expect("Example end") + + # Close connection to DUT + dut.receive_thread.exit() + dut.port_inst.close() + + script_path = os.path.join(os.getenv("IDF_PATH"), "examples", "system", "ota", "otatool", "otatool_example.py") + binary_path = "" + + for config in dut.download_config: + if "otatool.bin" in config: + binary_path = config + break + + subprocess.check_call([sys.executable, script_path, "--binary", binary_path]) + + +if __name__ == '__main__': + test_otatool_example() diff --git a/examples/system/ota/otatool/main/CMakeLists.txt b/examples/system/ota/otatool/main/CMakeLists.txt new file mode 100644 index 0000000000..2dc1cb53a0 --- /dev/null +++ b/examples/system/ota/otatool/main/CMakeLists.txt @@ -0,0 +1,4 @@ +set(COMPONENT_SRCS "otatool_main.c") +set(COMPONENT_ADD_INCLUDEDIRS ".") + +register_component() diff --git a/examples/system/ota/otatool/main/component.mk b/examples/system/ota/otatool/main/component.mk new file mode 100644 index 0000000000..a98f634eae --- /dev/null +++ b/examples/system/ota/otatool/main/component.mk @@ -0,0 +1,4 @@ +# +# "main" pseudo-component makefile. +# +# (Uses default behaviour of compiling all source files in directory, adding 'include' to include path.) diff --git a/examples/system/ota/otatool/main/otatool_main.c b/examples/system/ota/otatool/main/otatool_main.c new file mode 100644 index 0000000000..50ea3f2ec4 --- /dev/null +++ b/examples/system/ota/otatool/main/otatool_main.c @@ -0,0 +1,26 @@ +/* OTA Tool example + + This example code is in the Public Domain (or CC0 licensed, at your option.) + + Unless required by applicable law or agreed to in writing, this + software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + CONDITIONS OF ANY KIND, either express or implied. +*/ +#include "esp_system.h" +#include "esp_log.h" +#include "esp_ota_ops.h" +#include "esp_partition.h" + +static const char *TAG = "example"; + +void app_main() +{ + ESP_LOGI(TAG, "OTA Tool Example"); + + const esp_partition_t *running = esp_ota_get_running_partition(); + + // Display the running partition + ESP_LOGI(TAG, "Running partition: %s", running->label); + + ESP_LOGI(TAG, "Example end"); +} diff --git a/examples/system/ota/otatool/otatool_example.py b/examples/system/ota/otatool/otatool_example.py new file mode 100755 index 0000000000..17ed0cdb9e --- /dev/null +++ b/examples/system/ota/otatool/otatool_example.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python +# +# Demonstrates the use of otatool.py, a tool for performing ota partition level +# operations. +# +# Copyright 2018 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import sys +import subprocess +import argparse +import serial +import re + +IDF_PATH = os.path.expandvars("$IDF_PATH") + +OTATOOL_PY = os.path.join(IDF_PATH, "components", "app_update", "otatool.py") +ESPTOOL_PY = os.path.join(IDF_PATH, "components", "esptool_py", "esptool", "esptool.py") + +INVOKE_ARGS = [sys.executable, OTATOOL_PY, "-q"] + + +def sized_file_compare(file1, file2): + with open(file1, "rb") as f1: + with open(file2, "rb") as f2: + f1 = f1.read() + f2 = f2.read() + + if len(f1) < len(f2): + f2 = f2[:len(f1)] + else: + f1 = f1[:len(f2)] + + return f1 == f2 + + +def check(condition, message): + if not condition: + print("Error: " + message) + sys.exit(1) + + +def flash_example_firmware_to_ota_partitions(args): + # Invokes the command + # + # otatool.py -q write_ota_partition --slot or + # otatool.py -q write_ota_partition --name + # + # to write the contents of a file to the specified ota partition (either using name or the slot number) + print("Writing factory firmware to ota_0") + invoke_args = INVOKE_ARGS + ["write_ota_partition", "--slot", "0", "--input", args.binary] + subprocess.check_call(invoke_args) + + print("Writing factory firmware to ota_1") + invoke_args = INVOKE_ARGS + ["write_ota_partition", "--name", "ota_1", "--input", args.binary] + subprocess.check_call(invoke_args) + + # Verify that the contents of the two ota slots are the same as that of the factory partition + print("Checking written firmware to ota_0 and ota_1 match factory firmware") + + # Invokes the command + # + # otatool.py -q read_ota_partition --slot or + # otatool.py -q read_ota_partition --name + # + # to read the contents of a specified ota partition (either using name or the slot number) and write to a file + invoke_args = INVOKE_ARGS + ["read_ota_partition", "--slot", "0", "--output", "app_0.bin"] + subprocess.check_call(invoke_args) + + invoke_args = INVOKE_ARGS + ["read_ota_partition", "--name", "ota_1", "--output", "app_1.bin"] + subprocess.check_call(invoke_args) + + ota_same = sized_file_compare("app_0.bin", args.binary) + check(ota_same, "Slot 0 app does not match factory app") + + ota_same = sized_file_compare("app_1.bin", args.binary) + check(ota_same, "Slot 1 app does not match factory app") + + +def check_running_ota_partition(expected, port=None): + # Monitor the serial output of target device. The firmware outputs the currently + # running partition. It should match the partition the otatool switched to. + + if expected == 0 or expected == "ota_0": + expected = b"ota_0" + elif expected == 1 or expected == "ota_1": + expected = b"ota_1" + else: + expected = b"factory" + + sys.path.append(os.path.join(IDF_PATH, 'components', 'esptool_py', 'esptool')) + import esptool + + baud = os.environ.get("ESPTOOL_BAUD", esptool.ESPLoader.ESP_ROM_BAUD) + + if not port: + # Check what esptool.py finds on what port the device is connected to + output = subprocess.check_output([sys.executable, ESPTOOL_PY, "chip_id"]) + pattern = r"Serial port ([\S]+)" + pattern = re.compile(pattern.encode()) + port = re.search(pattern, output).group(1) + + serial_instance = serial.serial_for_url(port.decode("utf-8"), baud, do_not_open=True) + + serial_instance.dtr = False + serial_instance.rts = False + + serial_instance.rts = True + serial_instance.open() + serial_instance.rts = False + + # Read until example end and find the currently running partition string + content = serial_instance.read_until(b"Example end") + pattern = re.compile(b"Running partition: ([a-z0-9_]+)") + running = re.search(pattern, content).group(1) + + check(expected == running, "Running partition %s does not match expected %s" % (running, expected)) + + +def switch_partition(part, port): + if isinstance(part, int): + spec = "slot" + else: + spec = "name" + + print("Switching to ota partition %s %s" % (spec, str(part))) + + if str(part) == "factory": + # Invokes the command + # + # otatool.py -q erase_otadata + # + # to erase the otadata partition, effectively setting boot firmware to + # factory + subprocess.check_call(INVOKE_ARGS + ["erase_otadata"]) + else: + # Invokes the command + # + # otatool.py -q switch_otadata --slot or + # otatool.py -q switch_otadata --name + # + # to switch to the indicated ota partition (either using name or the slot number) + subprocess.check_call(INVOKE_ARGS + ["switch_otadata", "--" + spec, str(part)]) + + check_running_ota_partition(part, port) + + +def main(): + global INVOKE_ARGS + + parser = argparse.ArgumentParser("ESP-IDF OTA Tool Example") + + parser.add_argument("--port", "-p", help="port where the device to perform operations on is connected") + parser.add_argument("--binary", "-b", help="path to built example binary", default=os.path.join("build", "otatool.bin")) + args = parser.parse_args() + + if args.port: + INVOKE_ARGS += ["--port", args.port] + + # Flash the factory firmware to all ota partitions + flash_example_firmware_to_ota_partitions(args) + + # Perform switching ota partitions + switch_partition("factory", args.port) + switch_partition("factory", args.port) # check switching to factory partition twice in a row + + switch_partition(0, args.port) + + switch_partition("ota_1", args.port) + switch_partition(1, args.port) # check switching to ota_1 partition twice in a row + + switch_partition("ota_0", args.port) + switch_partition(0, args.port) # check switching to ota_0 partition twice in a row + + switch_partition("factory", args.port) + + switch_partition(1, args.port) # check switching to ota_1 partition from factory + + print("\nOTA tool operations executed successfully!") + + +if __name__ == '__main__': + main() diff --git a/examples/system/ota/otatool/sdkconfig.defaults b/examples/system/ota/otatool/sdkconfig.defaults new file mode 100644 index 0000000000..2289a82300 --- /dev/null +++ b/examples/system/ota/otatool/sdkconfig.defaults @@ -0,0 +1,4 @@ +# Default sdkconfig parameters to use the OTA +# partition table layout, with a 4MB flash size +CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y +CONFIG_PARTITION_TABLE_TWO_OTA=y diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index deb554c90a..d021f01078 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -68,3 +68,5 @@ tools/ldgen/ldgen.py tools/ldgen/test/test_fragments.py tools/ldgen/test/test_generation.py examples/build_system/cmake/idf_as_lib/build.sh +examples/storage/parttool/parttool_example.py +examples/system/ota/otatool/otatool_example.py