idf_monitor: Changed the elf file to be optional when running "idf.py monitor" command

This feature allows to start IDF Monitor without (re-)building the project.

tools/ci/check_copyright_ignore.txt

@@ -3000,7 +3000,6 @@ tools/find_build_apps/cmake.py
 tools/find_build_apps/common.py
 tools/gdb_panic_server.py
 tools/gen_esp_err_to_name.py
-tools/idf_monitor_base/argument_parser.py
 tools/idf_monitor_base/chip_specific_config.py
 tools/idf_monitor_base/console_reader.py
 tools/idf_monitor_base/constants.py

tools/idf_monitor.py

@@ -9,7 +9,7 @@
 # - If core dump output is detected, it is converted to a human-readable report
 #   by espcoredump.py.
 #
-# SPDX-FileCopyrightText: 2015-2021 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 #
 # Contains elements taken from miniterm "Very simple serial terminal" which
@@ -20,6 +20,7 @@
 #
 
 import codecs
+import io
 import os
 import queue
 import re
@@ -49,7 +50,7 @@ from idf_monitor_base.gdbhelper import GDBHelper
 from idf_monitor_base.line_matcher import LineMatcher
 from idf_monitor_base.logger import Logger
 from idf_monitor_base.output_helpers import normal_print, yellow_print
-from idf_monitor_base.serial_handler import SerialHandler, run_make
+from idf_monitor_base.serial_handler import SerialHandler, SerialHandlerNoElf, run_make
 from idf_monitor_base.serial_reader import LinuxReader, SerialReader
 from idf_monitor_base.web_socket_client import WebSocketClient
 from serial.tools import miniterm
@@ -92,10 +93,13 @@ class Monitor:
         self.console.output = get_converter(self.console.output)
         self.console.byte_output = get_converter(self.console.byte_output)
 
-        self.elf_file = elf_file
+        self.elf_file = elf_file or ''
+        self.elf_exists = os.path.exists(self.elf_file)
         self.logger = Logger(self.elf_file, self.console, timestamps, timestamp_format, b'', enable_address_decoding,
                              toolchain_prefix)
-        self.coredump = CoreDump(decode_coredumps, self.event_queue, self.logger, websocket_client, self.elf_file)
+
+        self.coredump = CoreDump(decode_coredumps, self.event_queue, self.logger, websocket_client,
+                                 self.elf_file) if self.elf_exists else None
 
         # allow for possibility the "make" arg is a list of arguments (for idf.py)
         self.make = make if os.path.exists(make) else shlex.split(make)  # type: Any[Union[str, List[str]], str]
@@ -108,17 +112,19 @@ class Monitor:
             self.serial_reader = SerialReader(self.serial, self.event_queue)
 
             self.gdb_helper = GDBHelper(toolchain_prefix, websocket_client, self.elf_file, self.serial.port,
-                                        self.serial.baudrate)
+                                        self.serial.baudrate) if self.elf_exists else None
+
         else:
             socket_mode = False
-            self.serial = subprocess.Popen([elf_file], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+            self.serial = subprocess.Popen([self.elf_file], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                                            stderr=subprocess.STDOUT)
             self.serial_reader = LinuxReader(self.serial, self.event_queue)
 
             self.gdb_helper = None
 
-        self.serial_handler = SerialHandler(b'', socket_mode, self.logger, decode_panic, PANIC_IDLE, b'', target,
-                                            False, False, self.serial, encrypted)
+        cls = SerialHandler if self.elf_exists else SerialHandlerNoElf
+        self.serial_handler = cls(b'', socket_mode, self.logger, decode_panic, PANIC_IDLE, b'', target,
+                                  False, False, self.serial, encrypted, self.elf_file)
 
         self.console_parser = ConsoleParser(eol)
         self.console_reader = ConsoleReader(self.console, self.event_queue, self.cmd_queue, self.console_parser,
@@ -227,7 +233,8 @@ class SerialMonitor(Monitor):
 
     def _pre_start(self) -> None:
         super()._pre_start()
-        self.gdb_helper.gdb_exit = False
+        if self.elf_exists:
+            self.gdb_helper.gdb_exit = False
         self.serial_handler.start_cmd_sent = False
 
     def serial_write(self, *args, **kwargs):  # type: ignore
@@ -240,12 +247,12 @@ class SerialMonitor(Monitor):
             pass  # this can happen if a non-ascii character was passed, ignoring
 
     def check_gdb_stub_and_run(self, line: bytes) -> None:  # type: ignore # The base class one is a None value
-        if self.gdb_helper.check_gdb_stub_trigger(line):
+        if self.gdb_helper and self.gdb_helper.check_gdb_stub_trigger(line):
             with self:  # disable console control
                 self.gdb_helper.run_gdb()
 
     def _main_loop(self) -> None:
-        if self.gdb_helper.gdb_exit:
+        if self.elf_exists and self.gdb_helper.gdb_exit:
             self.gdb_helper.gdb_exit = False
             time.sleep(GDB_EXIT_TIMEOUT)
             # Continue the program after exit from the GDB
@@ -283,7 +290,11 @@ def main() -> None:
         yellow_print('--- WARNING: Serial ports accessed as /dev/tty.* will hang gdb if launched.')
         yellow_print('--- Using %s instead...' % args.port)
 
-    args.elf_file.close()  # don't need this as a file
+    if isinstance(args.elf_file, io.BufferedReader):
+        elf_file = args.elf_file.name
+        args.elf_file.close()  # don't need this as a file
+    else:
+        elf_file = args.elf_file
 
     # remove the parallel jobserver arguments from MAKEFLAGS, as any
     # parent make is only running 1 job (monitor), so we can re-spawn
@@ -318,7 +329,7 @@ def main() -> None:
             yellow_print('--- idf_monitor on {p.name} {p.baudrate} ---'.format(p=serial_instance))
 
         monitor = cls(serial_instance,
-                      args.elf_file.name,
+                      elf_file,
                       args.print_filter,
                       args.make,
                       args.encrypted,

tools/idf_monitor_base/argument_parser.py

@@ -1,16 +1,5 @@
-# Copyright 2015-2021 Espressif Systems (Shanghai) CO LTD
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
 
 import argparse
 import os
@@ -65,7 +54,9 @@ def get_parser():  # type: () -> argparse.ArgumentParser
 
     parser.add_argument(
         'elf_file', help='ELF file of application',
-        type=argparse.FileType('rb'))
+        type=lambda f: open(f, 'rb') if os.path.exists(f) else f'{f}',
+        nargs='?'
+    )
 
     parser.add_argument(
         '--print_filter',

tools/idf_monitor_base/serial_handler.py

@@ -1,6 +1,7 @@
-# SPDX-FileCopyrightText: 2015-2021 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 
+import hashlib
 import os
 import queue  # noqa: F401
 import re
@@ -26,6 +27,14 @@ from .output_helpers import yellow_print
 from .serial_reader import Reader
 
 
+def get_sha256(filename, block_size=65536):  # type: (str, int) -> str
+    sha256 = hashlib.sha256()
+    with open(filename, 'rb') as f:
+        for block in iter(lambda: f.read(block_size), b''):
+            sha256.update(block)
+    return sha256.hexdigest()
+
+
 def run_make(target, make, console, console_parser, event_queue, cmd_queue, logger):
     # type: (str, str, miniterm.Console, ConsoleParser, queue.Queue, queue.Queue, Logger) -> None
     if isinstance(make, list):
@@ -49,8 +58,8 @@ class SerialHandler:
     The class is responsible for buffering serial input and performing corresponding commands.
     """
     def __init__(self, last_line_part, serial_check_exit, logger, decode_panic, reading_panic, panic_buffer, target,
-                 force_line_print, start_cmd_sent, serial_instance, encrypted):
-        # type: (bytes, bool, Logger, str, int, bytes,str, bool, bool, serial.Serial, bool) -> None
+                 force_line_print, start_cmd_sent, serial_instance, encrypted, elf_file):
+        # type: (bytes, bool, Logger, str, int, bytes,str, bool, bool, serial.Serial, bool, str) -> None
         self._last_line_part = last_line_part
         self._serial_check_exit = serial_check_exit
         self.logger = logger
@@ -62,6 +71,7 @@ class SerialHandler:
         self.start_cmd_sent = start_cmd_sent
         self.serial_instance = serial_instance
         self.encrypted = encrypted
+        self.elf_file = elf_file
 
     def handle_serial_input(self, data, console_parser, coredump, gdb_helper, line_matcher,
                             check_gdb_stub_and_run, finalize_line=False):
@@ -91,6 +101,7 @@ class SerialHandler:
             with coredump.check(line):
                 if self._force_line_print or line_matcher.match(line.decode(errors='ignore')):
                     self.logger.print(line + b'\n')
+                    self.compare_elf_sha256(line.decode(errors='ignore'))
                     self.logger.handle_possible_pc_address_in_line(line)
             check_gdb_stub_and_run(line)
             self._force_line_print = False
@@ -141,6 +152,24 @@ class SerialHandler:
             gdb_helper.process_panic_output(self._panic_buffer, self.logger, self.target)
             self._panic_buffer = b''
 
+    def compare_elf_sha256(self, line):  # type: (str) -> None
+        elf_sha256_matcher = re.compile(
+            r'ELF file SHA256:\s+(?P<sha256_flashed>[a-z0-9]+)'
+        )
+        file_sha256_flashed_match = re.search(elf_sha256_matcher, line)
+        if not file_sha256_flashed_match:
+            return
+        file_sha256_flashed = file_sha256_flashed_match.group('sha256_flashed')
+        if not os.path.exists(self.elf_file):
+            yellow_print(f'ELF file not found. '
+                         f"You need to build & flash the project before running 'monitor', "
+                         f'and the binary on the device must match the one in the build directory exactly. ')
+        else:
+            file_sha256_build = get_sha256(self.elf_file)
+            if file_sha256_flashed not in f'{file_sha256_build}':
+                yellow_print(f'Warning: checksum mismatch between flashed and built applications. '
+                             f'Checksum of built application is {file_sha256_build}')
+
     def handle_commands(self, cmd, chip, run_make_func, console_reader, serial_reader):
         # type: (int, str, Callable, ConsoleReader, Reader) -> None
         config = get_chip_config(chip)
@@ -192,3 +221,44 @@ class SerialHandler:
             self.serial_instance.setDTR(high)  # IO0=HIGH, done
         else:
             raise RuntimeError('Bad command data %d' % cmd)  # type: ignore
+
+
+class SerialHandlerNoElf(SerialHandler):
+    # This method avoids using 'gdb_helper,' 'coredump' and 'handle_possible_pc_address_in_line'
+    # where the elf file is required to be passed
+    def handle_serial_input(self, data, console_parser, coredump, gdb_helper, line_matcher,
+                            check_gdb_stub_and_run, finalize_line=False):
+        #  type: (bytes, ConsoleParser, CoreDump, Optional[GDBHelper], LineMatcher, Callable, bool) -> None
+
+        if self.start_cmd_sent:
+            self.start_cmd_sent = False
+            pos = data.find(b'+')
+            if pos != -1:
+                data = data[(pos + 1):]
+
+        sp = data.split(b'\n')
+        if self._last_line_part != b'':
+            # add unprocessed part from previous "data" to the first line
+            sp[0] = self._last_line_part + sp[0]
+            self._last_line_part = b''
+        if sp[-1] != b'':
+            # last part is not a full line
+            self._last_line_part = sp.pop()
+        for line in sp:
+            if line == b'':
+                continue
+            if self._serial_check_exit and line == console_parser.exit_key.encode('latin-1'):
+                raise SerialStopException()
+
+            self.logger.print(line + b'\n')
+            self.compare_elf_sha256(line.decode(errors='ignore'))
+            self._force_line_print = False
+
+        force_print_or_matched = any((
+            self._force_line_print,
+            (finalize_line and line_matcher.match(self._last_line_part.decode(errors='ignore')))
+        ))
+
+        if self._last_line_part != b'' and force_print_or_matched:
+            self._force_line_print = True
+            self.logger.print(self._last_line_part)

tools/idf_py_actions/serial_ext.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 
 import json
@@ -89,10 +89,6 @@ def action_extensions(base_actions, project_path):
         """
         project_desc = _get_project_desc(ctx, args)
         elf_file = os.path.join(args.build_dir, project_desc['app_elf'])
-        if not os.path.exists(elf_file):
-            raise FatalError("ELF file '%s' not found. You need to build & flash the project before running 'monitor', "
-                             'and the binary on the device must match the one in the build directory exactly. '
-                             "Try '%s flash monitor'." % (elf_file, ctx.info_name), ctx)
 
         idf_monitor = os.path.join(os.environ['IDF_PATH'], 'tools/idf_monitor.py')
         monitor_args = [PYTHON, idf_monitor]
@@ -123,7 +119,9 @@ def action_extensions(base_actions, project_path):
 
         if print_filter is not None:
             monitor_args += ['--print_filter', print_filter]
-        monitor_args += [elf_file]
+
+        if elf_file:
+            monitor_args += [elf_file]
 
         if encrypted:
             monitor_args += ['--encrypted']