From e30329ffe21f536dc82c550216dbfcc2410d4b78 Mon Sep 17 00:00:00 2001 From: Aleksei Apaseev Date: Wed, 5 Jan 2022 11:20:17 +0800 Subject: [PATCH] idf_monitor: Changed the elf file to be optional when running "idf.py monitor" command This feature allows to start IDF Monitor without (re-)building the project. --- tools/ci/check_copyright_ignore.txt | 1 - tools/idf_monitor.py | 37 +++++++---- tools/idf_monitor_base/argument_parser.py | 19 ++---- tools/idf_monitor_base/serial_handler.py | 76 ++++++++++++++++++++++- tools/idf_py_actions/serial_ext.py | 10 ++- 5 files changed, 106 insertions(+), 37 deletions(-) diff --git a/tools/ci/check_copyright_ignore.txt b/tools/ci/check_copyright_ignore.txt index 0733f96dc9..c81e116de1 100644 --- a/tools/ci/check_copyright_ignore.txt +++ b/tools/ci/check_copyright_ignore.txt @@ -3000,7 +3000,6 @@ tools/find_build_apps/cmake.py tools/find_build_apps/common.py tools/gdb_panic_server.py tools/gen_esp_err_to_name.py -tools/idf_monitor_base/argument_parser.py tools/idf_monitor_base/chip_specific_config.py tools/idf_monitor_base/console_reader.py tools/idf_monitor_base/constants.py diff --git a/tools/idf_monitor.py b/tools/idf_monitor.py index 93c672e44e..e0620ab843 100755 --- a/tools/idf_monitor.py +++ b/tools/idf_monitor.py @@ -9,7 +9,7 @@ # - If core dump output is detected, it is converted to a human-readable report # by espcoredump.py. # -# SPDX-FileCopyrightText: 2015-2021 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # # Contains elements taken from miniterm "Very simple serial terminal" which @@ -20,6 +20,7 @@ # import codecs +import io import os import queue import re @@ -49,7 +50,7 @@ from idf_monitor_base.gdbhelper import GDBHelper from idf_monitor_base.line_matcher import LineMatcher from idf_monitor_base.logger import Logger from idf_monitor_base.output_helpers import normal_print, yellow_print -from idf_monitor_base.serial_handler import SerialHandler, run_make +from idf_monitor_base.serial_handler import SerialHandler, SerialHandlerNoElf, run_make from idf_monitor_base.serial_reader import LinuxReader, SerialReader from idf_monitor_base.web_socket_client import WebSocketClient from serial.tools import miniterm @@ -92,10 +93,13 @@ class Monitor: self.console.output = get_converter(self.console.output) self.console.byte_output = get_converter(self.console.byte_output) - self.elf_file = elf_file + self.elf_file = elf_file or '' + self.elf_exists = os.path.exists(self.elf_file) self.logger = Logger(self.elf_file, self.console, timestamps, timestamp_format, b'', enable_address_decoding, toolchain_prefix) - self.coredump = CoreDump(decode_coredumps, self.event_queue, self.logger, websocket_client, self.elf_file) + + self.coredump = CoreDump(decode_coredumps, self.event_queue, self.logger, websocket_client, + self.elf_file) if self.elf_exists else None # allow for possibility the "make" arg is a list of arguments (for idf.py) self.make = make if os.path.exists(make) else shlex.split(make) # type: Any[Union[str, List[str]], str] @@ -108,17 +112,19 @@ class Monitor: self.serial_reader = SerialReader(self.serial, self.event_queue) self.gdb_helper = GDBHelper(toolchain_prefix, websocket_client, self.elf_file, self.serial.port, - self.serial.baudrate) + self.serial.baudrate) if self.elf_exists else None + else: socket_mode = False - self.serial = subprocess.Popen([elf_file], stdin=subprocess.PIPE, stdout=subprocess.PIPE, + self.serial = subprocess.Popen([self.elf_file], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) self.serial_reader = LinuxReader(self.serial, self.event_queue) self.gdb_helper = None - self.serial_handler = SerialHandler(b'', socket_mode, self.logger, decode_panic, PANIC_IDLE, b'', target, - False, False, self.serial, encrypted) + cls = SerialHandler if self.elf_exists else SerialHandlerNoElf + self.serial_handler = cls(b'', socket_mode, self.logger, decode_panic, PANIC_IDLE, b'', target, + False, False, self.serial, encrypted, self.elf_file) self.console_parser = ConsoleParser(eol) self.console_reader = ConsoleReader(self.console, self.event_queue, self.cmd_queue, self.console_parser, @@ -227,7 +233,8 @@ class SerialMonitor(Monitor): def _pre_start(self) -> None: super()._pre_start() - self.gdb_helper.gdb_exit = False + if self.elf_exists: + self.gdb_helper.gdb_exit = False self.serial_handler.start_cmd_sent = False def serial_write(self, *args, **kwargs): # type: ignore @@ -240,12 +247,12 @@ class SerialMonitor(Monitor): pass # this can happen if a non-ascii character was passed, ignoring def check_gdb_stub_and_run(self, line: bytes) -> None: # type: ignore # The base class one is a None value - if self.gdb_helper.check_gdb_stub_trigger(line): + if self.gdb_helper and self.gdb_helper.check_gdb_stub_trigger(line): with self: # disable console control self.gdb_helper.run_gdb() def _main_loop(self) -> None: - if self.gdb_helper.gdb_exit: + if self.elf_exists and self.gdb_helper.gdb_exit: self.gdb_helper.gdb_exit = False time.sleep(GDB_EXIT_TIMEOUT) # Continue the program after exit from the GDB @@ -283,7 +290,11 @@ def main() -> None: yellow_print('--- WARNING: Serial ports accessed as /dev/tty.* will hang gdb if launched.') yellow_print('--- Using %s instead...' % args.port) - args.elf_file.close() # don't need this as a file + if isinstance(args.elf_file, io.BufferedReader): + elf_file = args.elf_file.name + args.elf_file.close() # don't need this as a file + else: + elf_file = args.elf_file # remove the parallel jobserver arguments from MAKEFLAGS, as any # parent make is only running 1 job (monitor), so we can re-spawn @@ -318,7 +329,7 @@ def main() -> None: yellow_print('--- idf_monitor on {p.name} {p.baudrate} ---'.format(p=serial_instance)) monitor = cls(serial_instance, - args.elf_file.name, + elf_file, args.print_filter, args.make, args.encrypted, diff --git a/tools/idf_monitor_base/argument_parser.py b/tools/idf_monitor_base/argument_parser.py index 149ffba6fc..2cff301d5f 100644 --- a/tools/idf_monitor_base/argument_parser.py +++ b/tools/idf_monitor_base/argument_parser.py @@ -1,16 +1,5 @@ -# Copyright 2015-2021 Espressif Systems (Shanghai) CO LTD -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 import argparse import os @@ -65,7 +54,9 @@ def get_parser(): # type: () -> argparse.ArgumentParser parser.add_argument( 'elf_file', help='ELF file of application', - type=argparse.FileType('rb')) + type=lambda f: open(f, 'rb') if os.path.exists(f) else f'{f}', + nargs='?' + ) parser.add_argument( '--print_filter', diff --git a/tools/idf_monitor_base/serial_handler.py b/tools/idf_monitor_base/serial_handler.py index af267088f2..fc63d9237f 100644 --- a/tools/idf_monitor_base/serial_handler.py +++ b/tools/idf_monitor_base/serial_handler.py @@ -1,6 +1,7 @@ -# SPDX-FileCopyrightText: 2015-2021 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 +import hashlib import os import queue # noqa: F401 import re @@ -26,6 +27,14 @@ from .output_helpers import yellow_print from .serial_reader import Reader +def get_sha256(filename, block_size=65536): # type: (str, int) -> str + sha256 = hashlib.sha256() + with open(filename, 'rb') as f: + for block in iter(lambda: f.read(block_size), b''): + sha256.update(block) + return sha256.hexdigest() + + def run_make(target, make, console, console_parser, event_queue, cmd_queue, logger): # type: (str, str, miniterm.Console, ConsoleParser, queue.Queue, queue.Queue, Logger) -> None if isinstance(make, list): @@ -49,8 +58,8 @@ class SerialHandler: The class is responsible for buffering serial input and performing corresponding commands. """ def __init__(self, last_line_part, serial_check_exit, logger, decode_panic, reading_panic, panic_buffer, target, - force_line_print, start_cmd_sent, serial_instance, encrypted): - # type: (bytes, bool, Logger, str, int, bytes,str, bool, bool, serial.Serial, bool) -> None + force_line_print, start_cmd_sent, serial_instance, encrypted, elf_file): + # type: (bytes, bool, Logger, str, int, bytes,str, bool, bool, serial.Serial, bool, str) -> None self._last_line_part = last_line_part self._serial_check_exit = serial_check_exit self.logger = logger @@ -62,6 +71,7 @@ class SerialHandler: self.start_cmd_sent = start_cmd_sent self.serial_instance = serial_instance self.encrypted = encrypted + self.elf_file = elf_file def handle_serial_input(self, data, console_parser, coredump, gdb_helper, line_matcher, check_gdb_stub_and_run, finalize_line=False): @@ -91,6 +101,7 @@ class SerialHandler: with coredump.check(line): if self._force_line_print or line_matcher.match(line.decode(errors='ignore')): self.logger.print(line + b'\n') + self.compare_elf_sha256(line.decode(errors='ignore')) self.logger.handle_possible_pc_address_in_line(line) check_gdb_stub_and_run(line) self._force_line_print = False @@ -141,6 +152,24 @@ class SerialHandler: gdb_helper.process_panic_output(self._panic_buffer, self.logger, self.target) self._panic_buffer = b'' + def compare_elf_sha256(self, line): # type: (str) -> None + elf_sha256_matcher = re.compile( + r'ELF file SHA256:\s+(?P[a-z0-9]+)' + ) + file_sha256_flashed_match = re.search(elf_sha256_matcher, line) + if not file_sha256_flashed_match: + return + file_sha256_flashed = file_sha256_flashed_match.group('sha256_flashed') + if not os.path.exists(self.elf_file): + yellow_print(f'ELF file not found. ' + f"You need to build & flash the project before running 'monitor', " + f'and the binary on the device must match the one in the build directory exactly. ') + else: + file_sha256_build = get_sha256(self.elf_file) + if file_sha256_flashed not in f'{file_sha256_build}': + yellow_print(f'Warning: checksum mismatch between flashed and built applications. ' + f'Checksum of built application is {file_sha256_build}') + def handle_commands(self, cmd, chip, run_make_func, console_reader, serial_reader): # type: (int, str, Callable, ConsoleReader, Reader) -> None config = get_chip_config(chip) @@ -192,3 +221,44 @@ class SerialHandler: self.serial_instance.setDTR(high) # IO0=HIGH, done else: raise RuntimeError('Bad command data %d' % cmd) # type: ignore + + +class SerialHandlerNoElf(SerialHandler): + # This method avoids using 'gdb_helper,' 'coredump' and 'handle_possible_pc_address_in_line' + # where the elf file is required to be passed + def handle_serial_input(self, data, console_parser, coredump, gdb_helper, line_matcher, + check_gdb_stub_and_run, finalize_line=False): + # type: (bytes, ConsoleParser, CoreDump, Optional[GDBHelper], LineMatcher, Callable, bool) -> None + + if self.start_cmd_sent: + self.start_cmd_sent = False + pos = data.find(b'+') + if pos != -1: + data = data[(pos + 1):] + + sp = data.split(b'\n') + if self._last_line_part != b'': + # add unprocessed part from previous "data" to the first line + sp[0] = self._last_line_part + sp[0] + self._last_line_part = b'' + if sp[-1] != b'': + # last part is not a full line + self._last_line_part = sp.pop() + for line in sp: + if line == b'': + continue + if self._serial_check_exit and line == console_parser.exit_key.encode('latin-1'): + raise SerialStopException() + + self.logger.print(line + b'\n') + self.compare_elf_sha256(line.decode(errors='ignore')) + self._force_line_print = False + + force_print_or_matched = any(( + self._force_line_print, + (finalize_line and line_matcher.match(self._last_line_part.decode(errors='ignore'))) + )) + + if self._last_line_part != b'' and force_print_or_matched: + self._force_line_print = True + self.logger.print(self._last_line_part) diff --git a/tools/idf_py_actions/serial_ext.py b/tools/idf_py_actions/serial_ext.py index b78a71e007..9d14761e65 100644 --- a/tools/idf_py_actions/serial_ext.py +++ b/tools/idf_py_actions/serial_ext.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import json @@ -89,10 +89,6 @@ def action_extensions(base_actions, project_path): """ project_desc = _get_project_desc(ctx, args) elf_file = os.path.join(args.build_dir, project_desc['app_elf']) - if not os.path.exists(elf_file): - raise FatalError("ELF file '%s' not found. You need to build & flash the project before running 'monitor', " - 'and the binary on the device must match the one in the build directory exactly. ' - "Try '%s flash monitor'." % (elf_file, ctx.info_name), ctx) idf_monitor = os.path.join(os.environ['IDF_PATH'], 'tools/idf_monitor.py') monitor_args = [PYTHON, idf_monitor] @@ -123,7 +119,9 @@ def action_extensions(base_actions, project_path): if print_filter is not None: monitor_args += ['--print_filter', print_filter] - monitor_args += [elf_file] + + if elf_file: + monitor_args += [elf_file] if encrypted: monitor_args += ['--encrypted']