tools/idf_monitor: add WebSocket client for IDE integration

This commit is contained in:
Roland Dobai 2020-05-28 14:47:38 +02:00 committed by bot
parent 5c783e60e1
commit e67314f646
9 changed files with 269 additions and 28 deletions

View File

@ -357,7 +357,7 @@ test_app_test_001:
artifacts:
when: always
paths:
- $CI_PROJECT_DIR/tools/test_apps/system/gdb_loadable_elf/*.log
- $CI_PROJECT_DIR/tools/test_apps/system/*/*.log
expire_in: 1 week
variables:
SETUP_TOOLS: "1"

View File

@ -24,7 +24,7 @@ class CustomProcess(object):
self.f = open(logfile, 'w')
if self.verbose:
Utility.console_log('Starting {} > {}'.format(cmd, self.f.name))
self.pexpect_proc = pexpect.spawn(cmd, timeout=60, logfile=self.f, encoding='utf-8')
self.pexpect_proc = pexpect.spawn(cmd, timeout=60, logfile=self.f, encoding='utf-8', codec_errors='ignore')
def __enter__(self):
return self

View File

@ -57,6 +57,13 @@ from distutils.version import StrictVersion
from io import open
import textwrap
import tempfile
import json
try:
import websocket
except ImportError:
# This is needed for IDE integration only.
pass
key_description = miniterm.key_description
@ -461,7 +468,8 @@ class Monitor(object):
"""
def __init__(self, serial_instance, elf_file, print_filter, make="make", encrypted=False,
toolchain_prefix=DEFAULT_TOOLCHAIN_PREFIX, eol="CRLF",
decode_coredumps=COREDUMP_DECODE_INFO):
decode_coredumps=COREDUMP_DECODE_INFO,
websocket_client=None):
super(Monitor, self).__init__()
self.event_queue = queue.Queue()
self.cmd_queue = queue.Queue()
@ -493,6 +501,7 @@ class Monitor(object):
self.make = make
self.encrypted = encrypted
self.toolchain_prefix = toolchain_prefix
self.websocket_client = websocket_client
# internal state
self._last_line_part = b""
@ -680,7 +689,16 @@ class Monitor(object):
except ValueError:
return # payload wasn't valid hex digits
if chsum == calc_chsum:
self.run_gdb()
if self.websocket_client:
yellow_print('Communicating through WebSocket')
self.websocket_client.send({'event': 'gdb_stub',
'port': self.serial.port,
'prog': self.elf_file})
yellow_print('Waiting for debug finished event')
self.websocket_client.wait([('event', 'debug_finished')])
yellow_print('Communications through WebSocket is finished')
else:
self.run_gdb()
else:
red_print("Malformed gdb message... calculated checksum %02x received %02x" % (chsum, calc_chsum))
@ -737,17 +755,27 @@ class Monitor(object):
coredump_file.write(self._coredump_buffer)
coredump_file.flush()
cmd = [sys.executable,
coredump_script,
"info_corefile",
"--core", coredump_file.name,
"--core-format", "b64",
self.elf_file
]
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
self._output_enabled = True
self._print(output)
self._output_enabled = False # Will be reenabled in check_coredump_trigger_after_print
if self.websocket_client:
self._output_enabled = True
yellow_print('Communicating through WebSocket')
self.websocket_client.send({'event': 'coredump',
'file': coredump_file.name,
'prog': self.elf_file})
yellow_print('Waiting for debug finished event')
self.websocket_client.wait([('event', 'debug_finished')])
yellow_print('Communications through WebSocket is finished')
else:
cmd = [sys.executable,
coredump_script,
"info_corefile",
"--core", coredump_file.name,
"--core-format", "b64",
self.elf_file
]
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
self._output_enabled = True
self._print(output)
self._output_enabled = False # Will be reenabled in check_coredump_trigger_after_print
except subprocess.CalledProcessError as e:
yellow_print("Failed to run espcoredump script: {}\n\n".format(e))
self._output_enabled = True
@ -936,6 +964,12 @@ def main():
help="Handling of core dumps found in serial output"
)
parser.add_argument(
'--ws',
default=os.environ.get('ESP_IDF_MONITOR_WS', None),
help="WebSocket URL for communicating with IDE tools for debugging purposes"
)
args = parser.parse_args()
# GDB uses CreateFile to open COM port, which requires the COM name to be r'\\.\COMx' if the COM
@ -974,21 +1008,112 @@ def main():
espport_val = str(args.port)
os.environ.update({espport_key: espport_val})
monitor = Monitor(serial_instance, args.elf_file.name, args.print_filter, args.make, args.encrypted,
args.toolchain_prefix, args.eol,
args.decode_coredumps)
ws = WebSocketClient(args.ws) if args.ws else None
try:
monitor = Monitor(serial_instance, args.elf_file.name, args.print_filter, args.make, args.encrypted,
args.toolchain_prefix, args.eol,
args.decode_coredumps,
ws)
yellow_print('--- idf_monitor on {p.name} {p.baudrate} ---'.format(
p=serial_instance))
yellow_print('--- Quit: {} | Menu: {} | Help: {} followed by {} ---'.format(
key_description(monitor.console_parser.exit_key),
key_description(monitor.console_parser.menu_key),
key_description(monitor.console_parser.menu_key),
key_description(CTRL_H)))
if args.print_filter != DEFAULT_PRINT_FILTER:
yellow_print('--- Print filter: {} ---'.format(args.print_filter))
yellow_print('--- idf_monitor on {p.name} {p.baudrate} ---'.format(
p=serial_instance))
yellow_print('--- Quit: {} | Menu: {} | Help: {} followed by {} ---'.format(
key_description(monitor.console_parser.exit_key),
key_description(monitor.console_parser.menu_key),
key_description(monitor.console_parser.menu_key),
key_description(CTRL_H)))
if args.print_filter != DEFAULT_PRINT_FILTER:
yellow_print('--- Print filter: {} ---'.format(args.print_filter))
monitor.main_loop()
monitor.main_loop()
finally:
if ws:
ws.close()
class WebSocketClient(object):
"""
WebSocket client used to advertise debug events to WebSocket server by sending and receiving JSON-serialized
dictionaries.
Advertisement of debug event:
{'event': 'gdb_stub', 'port': '/dev/ttyUSB1', 'prog': 'build/elf_file'} for GDB Stub, or
{'event': 'coredump', 'file': '/tmp/xy', 'prog': 'build/elf_file'} for coredump,
where 'port' is the port for the connected device, 'prog' is the full path to the ELF file and 'file' is the
generated coredump file.
Expected end of external debugging:
{'event': 'debug_finished'}
"""
RETRIES = 3
CONNECTION_RETRY_DELAY = 1
def __init__(self, url):
self.url = url
self._connect()
def _connect(self):
"""
Connect to WebSocket server at url
"""
self.close()
for _ in range(self.RETRIES):
try:
self.ws = websocket.create_connection(self.url)
break # success
except NameError:
raise RuntimeError('Please install the websocket_client package for IDE integration!')
except Exception as e:
red_print('WebSocket connection error: {}'.format(e))
time.sleep(self.CONNECTION_RETRY_DELAY)
else:
raise RuntimeError('Cannot connect to WebSocket server')
def close(self):
try:
self.ws.close()
except AttributeError:
# Not yet connected
pass
except Exception as e:
red_print('WebSocket close error: {}'.format(e))
def send(self, payload_dict):
"""
Serialize payload_dict in JSON format and send it to the server
"""
for _ in range(self.RETRIES):
try:
self.ws.send(json.dumps(payload_dict))
yellow_print('WebSocket sent: {}'.format(payload_dict))
break
except Exception as e:
red_print('WebSocket send error: {}'.format(e))
self._connect()
else:
raise RuntimeError('Cannot send to WebSocket server')
def wait(self, expect_iterable):
"""
Wait until a dictionary in JSON format is received from the server with all (key, value) tuples from
expect_iterable.
"""
for _ in range(self.RETRIES):
try:
r = self.ws.recv()
except Exception as e:
red_print('WebSocket receive error: {}'.format(e))
self._connect()
continue
obj = json.loads(r)
if all([k in obj and obj[k] == v for k, v in expect_iterable]):
yellow_print('WebSocket received: {}'.format(obj))
break
red_print('WebSocket expected: {}, received: {}'.format(dict(expect_iterable), obj))
else:
raise RuntimeError('Cannot receive from WebSocket server')
if os.name == 'nt':

View File

@ -0,0 +1,4 @@
cmake_minimum_required(VERSION 3.5)
include($ENV{IDF_PATH}/tools/cmake/project.cmake)
project(panic)

View File

@ -0,0 +1,90 @@
from __future__ import unicode_literals
from SimpleWebSocketServer import SimpleWebSocketServer, WebSocket
from tiny_test_fw import Utility
import glob
import json
import os
import re
import threading
import ttfw_idf
class IDEWSProtocol(WebSocket):
def handleMessage(self):
try:
j = json.loads(self.data)
except Exception as e:
Utility.console_log('Server ignores error: {}'.format(e), 'orange')
return
event = j.get('event')
if event and 'prog' in j and ((event == 'gdb_stub' and 'port' in j) or
(event == 'coredump' and 'file' in j)):
payload = {'event': 'debug_finished'}
self.sendMessage(json.dumps(payload))
Utility.console_log('Server sent: {}'.format(payload))
else:
Utility.console_log('Server received: {}'.format(j), 'orange')
def handleConnected(self):
Utility.console_log('{} connected to server'.format(self.address))
def handleClose(self):
Utility.console_log('{} closed the connection'.format(self.address))
class WebSocketServer(object):
HOST = '127.0.0.1'
PORT = 1123
def run(self):
server = SimpleWebSocketServer(self.HOST, self.PORT, IDEWSProtocol)
while not self.exit_event.is_set():
server.serveonce()
def __init__(self):
self.exit_event = threading.Event()
self.thread = threading.Thread(target=self.run)
self.thread.start()
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
self.exit_event.set()
self.thread.join(10)
if self.thread.is_alive():
Utility.console_log('Thread cannot be joined', 'orange')
@ttfw_idf.idf_custom_test(env_tag='test_jtag_arm', group='test-apps')
def test_monitor_ide_integration(env, extra_data):
config_files = glob.glob(os.path.join(os.path.dirname(__file__), 'sdkconfig.ci.*'))
config_names = [os.path.basename(s).replace('sdkconfig.ci.', '') for s in config_files]
rel_proj_path = 'tools/test_apps/system/monitor_ide_integration'
for name in config_names:
Utility.console_log('Checking config "{}"... '.format(name), 'green', end='')
dut = env.get_dut('panic', rel_proj_path, app_config_name=name)
monitor_path = os.path.join(dut.app.get_sdk_path(), 'tools/idf_monitor.py')
elf_path = os.path.join(dut.app.get_binary_path(rel_proj_path), 'panic.elf')
dut.start_app()
# Closing the DUT because we will reconnect with IDF Monitor
env.close_dut(dut.name)
with WebSocketServer(), ttfw_idf.CustomProcess(' '.join([monitor_path,
elf_path,
'--ws', 'ws://{}:{}'.format(WebSocketServer.HOST,
WebSocketServer.PORT)]),
logfile='monitor_{}.log'.format(name)) as p:
p.pexpect_proc.expect(re.compile(r'Guru Meditation Error'), timeout=10)
p.pexpect_proc.expect_exact('Communicating through WebSocket', timeout=5)
# "u?" is for Python 2 only in the following regular expressions.
# The elements of dictionary can be printed in different order depending on the Python version.
p.pexpect_proc.expect(re.compile(r"WebSocket sent: \{u?.*'event': u?'" + name + "'"), timeout=5)
p.pexpect_proc.expect_exact('Waiting for debug finished event', timeout=5)
p.pexpect_proc.expect(re.compile(r"WebSocket received: \{u?'event': u?'debug_finished'\}"), timeout=5)
p.pexpect_proc.expect_exact('Communications through WebSocket is finished', timeout=5)
if __name__ == '__main__':
test_monitor_ide_integration()

View File

@ -0,0 +1,2 @@
idf_component_register(SRCS "main.c"
INCLUDE_DIRS "")

View File

@ -0,0 +1,18 @@
/* Monitor-IDE integration test
This example code is in the Public Domain (or CC0 licensed, at your option.)
Unless required by applicable law or agreed to in writing, this
software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
CONDITIONS OF ANY KIND, either express or implied.
*/
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#include "esp_system.h"
void app_main(void)
{
int *p = (int *)4;
vTaskDelay(1000 / portTICK_PERIOD_MS);
*p = 0;
}

View File

@ -0,0 +1 @@
CONFIG_ESP32_ENABLE_COREDUMP_TO_UART=y

View File

@ -0,0 +1 @@
CONFIG_ESP_SYSTEM_PANIC_GDBSTUB=y