feat(tools): add 'idf.py qemu' extension

This extension allows running programs in QEMU similar to running
them on a real chip:

- 'idf.py qemu' — builds and runs the program in QEMU. User gets
  a QEMU instance launched, and can work with it as a normal QEMU
  instance.
- 'idf.py qemu monitor' — same, but QEMU is launched in the
  background, and idf_monitor runs in the foreground, showing QEMU
  output. Compared to only running 'idf.py qemu' this enables, for
  example, automatic backtrace decoding.
- 'idf.py qemu gdb' — launches QEMU in the background and opens an
  interactive GDB prompt, connecting it to QEMU.
- 'idf.py qemu --gdb monitor' and 'idf.py gdb' in another shell:
  launches QEMU in the background, keeps it suspended until GDB is
  connected, and opens idf_monitor. GDB can be used in another shell
  to debug the application.

.gitlab/ci/common.yml

@@ -140,6 +140,11 @@ variables:
     $IDF_PATH/tools/idf_tools.py --non-interactive install esp-clang
   fi
 
+  # Install QEMU if necessary
+  if [[ ! -z "$INSTALL_QEMU" ]]; then
+    $IDF_PATH/tools/idf_tools.py --non-interactive install qemu-xtensa qemu-riscv32
+  fi
+
   source ./export.sh
 
   # Custom clang

.gitlab/ci/host-test.yml

@@ -203,6 +203,7 @@ test_tools:
     expire_in: 1 week
   variables:
     LC_ALL: C.UTF-8
+    INSTALL_QEMU: 1  # for test_idf_qemu.py
   script:
     - stat=0
     - cd ${IDF_PATH}/tools/ci/test_autocomplete
@@ -210,6 +211,7 @@ test_tools:
     - cd ${IDF_PATH}/tools/test_idf_py
     - pytest --noconftest test_idf_py.py       --junitxml=${IDF_PATH}/XUNIT_IDF_PY.xml                    || stat=1
     - pytest --noconftest test_hints.py        --junitxml=${IDF_PATH}/XUNIT_HINTS.xml                     || stat=1
+    - pytest --noconftest test_idf_qemu.py     --junitxml=${IDF_PATH}/XUNIT_IDF_PY_QEMU.xml               || stat=1
     - cd ${IDF_PATH}/tools/test_mkdfu
     - pytest --noconftest test_mkdfu.py        --junitxml=${IDF_PATH}/XUNIT_MKDFU.xml                     || stat=1
     - cd ${IDF_PATH}/tools/test_sbom

docs/conf_common.py

@@ -145,6 +145,8 @@ ANA_CMPR_DOCS = ['api-reference/peripherals/ana_cmpr.rst']
 
 SPI_SLAVE_HD_DOCS = ['api-reference/peripherals/spi_slave_hd.rst']
 
+QEMU_DOCS = ['api-guides/tools/qemu.rst']
+
 ESP32_DOCS = ['api-reference/system/himem.rst',
               'api-guides/romconsole.rst',
               'api-reference/system/ipc.rst',
@@ -153,7 +155,7 @@ ESP32_DOCS = ['api-reference/system/himem.rst',
               'api-reference/peripherals/dac.rst',
               'api-reference/peripherals/sd_pullup_requirements.rst',
               'hw-reference/esp32/**',
-              'api-guides/RF_calibration.rst'] + FTDI_JTAG_DOCS
+              'api-guides/RF_calibration.rst'] + FTDI_JTAG_DOCS + QEMU_DOCS
 
 ESP32S2_DOCS = ['hw-reference/esp32s2/**',
                 'api-guides/usb-console.rst',
@@ -171,7 +173,7 @@ ESP32S3_DOCS = ['hw-reference/esp32s3/**',
 
 # No JTAG docs for this one as it gets gated on SOC_USB_SERIAL_JTAG_SUPPORTED down below.
 ESP32C3_DOCS = ['hw-reference/esp32c3/**',
-                'api-guides/RF_calibration.rst']
+                'api-guides/RF_calibration.rst'] + QEMU_DOCS
 
 ESP32C2_DOCS = ['api-guides/RF_calibration.rst']
 

docs/en/api-guides/tools/index.rst

@@ -11,3 +11,4 @@ Tools
    idf-component-manager
    idf-clang-tidy
    idf-tools
+   :esp32 or esp32c3: qemu

docs/en/api-guides/tools/qemu.rst

@@ -0,0 +1,64 @@
+QEMU Emulator
+=============
+
+Espressif maintains a `fork <https://github.com/espressif/qemu>`_ of the QEMU emulator with support for {IDF_TARGET_NAME}. This fork implements emulation of the CPU, memory, and several peripherals of {IDF_TARGET_NAME}. For more information about QEMU for {IDF_TARGET_NAME}, see the `documentation <https://github.com/espressif/esp-toolchain-docs/blob/main/qemu/README.md>`_.
+
+:doc:`idf.py<idf-py>` allows for running and debugging applications in QEMU. This is a convenient way to test applications without having to flash them to real hardware.
+
+Prerequisites
+-------------
+
+To use QEMU with ``idf.py`` you first need to install the above-mentioned fork of QEMU. ESP-IDF provides pre-built binaries for x86_64 Linux. If you are using this platform, you can install the pre-built binaries with the following command:
+
+.. code-block:: console
+
+    python $IDF_PATH/tools/idf_tools.py install qemu-xtensa qemu-riscv
+
+After installing QEMU, make sure it is added to PATH by running ``. ./export.sh`` in the IDF directory.
+
+If you are using a different platform, you need to build QEMU from source. Refer to official QEMU documentation for instructions.
+
+Usage
+-----
+
+To run an IDF application in QEMU, use the following command:
+
+.. code-block:: console
+
+    idf.py qemu monitor
+
+This command builds the application, starts QEMU and opens :doc:`IDF monitor <idf-monitor>`, connecting it to the emulated UART port. You can see the console output of the application and interact with it. IDF Monitor also provides automatic decoding of panic backtraces and UART core dumps.
+
+To debug an application in QEMU, use the following command:
+
+.. code-block:: console
+
+    idf.py qemu gdb
+
+This command builds the application, starts QEMU with the GDB server enabled, and opens an interactive GDB session. You can use GDB to debug the application as if it was running on real hardware.
+
+To see console output while debugging in QEMU, use two terminals.
+
+* In the first terminal, run:
+
+  .. code-block:: console
+
+    idf.py qemu --gdb monitor
+
+  This command starts QEMU and IDF Monitor, and tells QEMU to wait for a GDB connection.
+
+* In the second terminal, run:
+
+  .. code-block:: console
+
+    idf.py gdb
+
+  This command starts an interactive GDB sessions and connects it to QEMU. You can now debug the application, and the console output will be visible in the first terminal.
+
+It is also possible to run QEMU without the IDF Monitor:
+
+.. code-block:: console
+
+    idf.py qemu
+
+In this case, the IDF Monitor is not used, and you can interact with QEMU process directly. To switch between the emulated UART console and QEMU console ("QEMU monitor"), use Ctrl-A shortcut. For example, to exit QEMU, press Ctrl-A, then type ``q`` and press Enter. You can use the QEMU console to enter commands, such as for inspecting registers and memory.

docs/zh_CN/api-guides/tools/index.rst

@@ -11,3 +11,4 @@
    idf-component-manager
    idf-clang-tidy
    idf-tools
+   :esp32 or esp32c3: qemu

docs/zh_CN/api-guides/tools/qemu.rst

@@ -0,0 +1 @@
+.. include:: ../../../en/api-guides/tools/qemu.rst

tools/ci/executable-list.txt

@@ -109,5 +109,6 @@ tools/set-submodules-to-github.sh
 tools/test_apps/system/no_embedded_paths/check_for_file_paths.py
 tools/test_idf_py/test_hints.py
 tools/test_idf_py/test_idf_py.py
+tools/test_idf_py/test_idf_qemu.py
 tools/test_idf_tools/test_idf_tools.py
 tools/test_mkdfu/test_mkdfu.py

tools/idf.py

@@ -441,9 +441,9 @@ def init_cli(verbose_output: List=None) -> Any:
                 return None
 
         def _print_closing_message(self, args: PropertyDict, actions: _OrderedDictKeysView) -> None:
-            # print a closing message of some kind
+            # print a closing message of some kind,
-            #
+            # except if any of the following actions were requested
-            if any(t in str(actions) for t in ('flash', 'dfu', 'uf2', 'uf2-app')):
+            if any(t in str(actions) for t in ('flash', 'dfu', 'uf2', 'uf2-app', 'qemu')):
                 print('Done')
                 return
 

tools/idf_py_actions/qemu_ext.py

@@ -0,0 +1,261 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+import atexit
+import binascii
+import json
+import os
+import shutil
+import socket
+import subprocess
+import sys
+import time
+from dataclasses import dataclass
+from typing import Any, Dict, List
+
+from click.core import Context
+
+try:
+    from idf_py_actions.tools import PropertyDict, ensure_build_directory, red_print, yellow_print
+except ImportError:
+    PropertyDict = Any
+
+from idf_py_actions.tools import get_sdkconfig_value
+
+# Some ports used by QEMU. Could be made configurable, if necessary.
+# Note that port 3333 is also assumed by gdb extension.
+QEMU_PORT_SERIAL = 5555
+PYSERIAL_PORT = f'socket://localhost:{QEMU_PORT_SERIAL}'
+QEMU_PORT_GDB = 3333
+GBD_PORT = f'localhost:{QEMU_PORT_GDB}'
+
+
+@dataclass
+class QemuTarget:
+    """
+    Target-specific information related to QEMU.
+    """
+    target: str     # chip name, e.g. esp32, esp32c3
+    qemu_prog: str  # name of the QEMU binary, e.g. qemu-system-xtensa
+    install_package: str    # name of the tools.json package from which to install the QEMU binary
+    qemu_args: str          # chip-specific arguments to pass to QEMU
+    default_efuse: bytes    # default efuse values for the target
+
+
+# To generate the default eFuse values, follow the instructions in
+# https://github.com/espressif/esp-toolchain-docs/blob/main/qemu/esp32/README.md#using-esptoolpy-and-espefusepy-to-interact-with-qemu
+# and burn the eFuses which should be set by default. Then take the binary file, convert it to hex, and paste it here.
+QEMU_TARGETS: Dict[str, QemuTarget] = {
+    'esp32': QemuTarget(
+        'esp32',
+        'qemu-system-xtensa',
+        'qemu-xtensa',
+        '-M esp32 -m 4M',
+        # Chip revision 3
+        binascii.unhexlify(
+            '00000000000000000000000000800000000000000000100000000000000000000000000000000000'
+            '00000000000000000000000000000000000000000000000000000000000000000000000000000000'
+            '00000000000000000000000000000000000000000000000000000000000000000000000000000000'
+            '00000000')),
+
+    'esp32c3': QemuTarget(
+        'esp32c3',
+        'qemu-system-riscv32',
+        'qemu-riscv',
+        '-M esp32c3',
+        # Chip revision 0.3
+        binascii.unhexlify(
+            '00000000000000000000000000000000000000000000000000000000000000000000000000000c00'
+            '00000000000000000000000000000000000000000000000000000000000000000000000000000000'
+            '00000000000000000000000000000000000000000000000000000000000000000000000000000000'
+            '00000000000000000000000000000000000000000000000000000000000000000000000000000000'
+            '00000000000000000000000000000000000000000000000000000000000000000000000000000000'
+            '00000000000000000000000000000000000000000000000000000000000000000000000000000000'
+            '00000000000000000000000000000000000000000000000000000000000000000000000000000000'
+            '00000000000000000000000000000000000000000000000000000000000000000000000000000000'
+            '00000000000000000000000000000000000000000000000000000000000000000000000000000000'
+            '00000000000000000000000000000000000000000000000000000000000000000000000000000000'
+            '00000000000000000000000000000000000000000000000000000000000000000000000000000000'
+            '00000000000000000000000000000000000000000000000000000000000000000000000000000000'
+            '00000000000000000000000000000000000000000000000000000000000000000000000000000000'
+            '00000000000000000000000000000000000000000000000000000000000000000000000000000000'
+            '00000000000000000000000000000000000000000000000000000000000000000000000000000000'
+            '00000000000000000000000000000000000000000000000000000000000000000000000000000000'
+            '00000000000000000000000000000000000000000000000000000000000000000000000000000000'
+            '00000000000000000000000000000000000000000000000000000000000000000000000000000000'
+            '00000000000000000000000000000000000000000000000000000000000000000000000000000000'
+            '00000000000000000000000000000000000000000000000000000000000000000000000000000000'
+            '00000000000000000000000000000000000000000000000000000000000000000000000000000000'
+            '00000000000000000000000000000000000000000000000000000000000000000000000000000000'
+            '00000000000000000000000000000000000000000000000000000000000000000000000000000000'
+            '00000000000000000000000000000000000000000000000000000000000000000000000000000000'
+            '00000000000000000000000000000000000000000000000000000000000000000000000000000000'
+            '000000000000000000000000000000000000000000000000')),
+}
+
+
+class QemuTaskRunOptions:
+    """
+    Some options related to QEMU execution, which depend on the presence of other tasks: gdb and monitor.
+    """
+    def __init__(self) -> None:
+        self.bg_mode = False
+        self.wait_for_gdb = False
+        self.wait_for_monitor = False
+
+
+def wait_for_socket(port: int, timeout_sec: float = 10.0) -> None:
+    """
+    Helper function, waits for a socket to be open.
+    Used to detect that QEMU has started up and ready to receive connections.
+    """
+    start_time = time.time()
+    end_time = start_time + timeout_sec
+    while time.time() < end_time:
+        try:
+            with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+                s.connect(('localhost', port))
+                return  # connection successful
+        except OSError:
+            time.sleep(0.1)
+            continue
+    red_print(f'Timed out waiting for port {port} to be open')
+    raise SystemExit(1)
+
+
+def action_extensions(base_actions: Dict, project_path: str) -> Dict:
+    # Shared state between "global_callback" and "qemu" action.
+    # Stores options which depend on the presence of other tasks.
+    options = QemuTaskRunOptions()
+
+    def global_callback(ctx: Context, global_args: Dict, tasks: List) -> None:
+        # This callback lets us customize QEMU launch arguments depending on the presence of other tasks.
+        def have_task(name: str) -> bool:
+            return any(task.name == name for task in tasks)
+
+        have_qemu = have_task('qemu')
+        have_gdb = have_task('gdb')
+        have_monitor = have_task('monitor')
+
+        if have_qemu:
+            if have_gdb and have_monitor:
+                red_print('Cannot run qemu with both gdb and monitor in the same terminal')
+                raise SystemExit(1)
+            if have_gdb:
+                options.wait_for_gdb = True
+                options.bg_mode = True
+            if have_monitor:
+                options.wait_for_monitor = True
+                options.bg_mode = True
+                yellow_print(f'Running qemu on {PYSERIAL_PORT}')
+                global_args['port'] = PYSERIAL_PORT
+
+    def _get_project_desc(args: PropertyDict, ctx: Context) -> Any:
+        desc_path = os.path.join(args.build_dir, 'project_description.json')
+        if not os.path.exists(desc_path):
+            ensure_build_directory(args, ctx.info_name)
+        with open(desc_path, 'r') as f:
+            project_desc = json.load(f)
+        return project_desc
+
+    def qemu(action: str, ctx: Context, args: PropertyDict, qemu_extra_args: str, gdb: bool) -> None:
+        project_desc = _get_project_desc(args, ctx)
+
+        # Determine the target and check if we have the necessary QEMU binary
+        target = project_desc['target']
+        qemu_target_info = QEMU_TARGETS.get(target)
+        if not qemu_target_info:
+            red_print(f'QEMU is not supported for target {target}')
+            raise SystemExit(1)
+        if not shutil.which(qemu_target_info.qemu_prog):
+            red_print(f'{qemu_target_info.qemu_prog} is not installed. Please install it using '
+                      f'"python $IDF_PATH/tools/idf_tools.py install {qemu_target_info.install_package}" '
+                      'or build it from source if the pre-built version is not available for your platform.')
+            raise SystemExit(1)
+
+        # Generate flash image and efuse image
+        flash_size = get_sdkconfig_value(project_desc['config_file'], 'CONFIG_ESPTOOLPY_FLASHSIZE')
+        bin_path = os.path.join(args.build_dir, 'flash_image.bin')
+        yellow_print(f'Generating flash image: {bin_path}')
+        subprocess.check_call([
+            sys.executable, '-m', 'esptool', f'--chip={target}', 'merge_bin', f'--output={bin_path}',
+            f'--fill-flash-size={flash_size}', '@flash_args'], cwd=args.build_dir)
+
+        efuse_bin_path = os.path.join(args.build_dir, 'qemu_efuse.bin')
+        yellow_print(f'Generating efuse image: {efuse_bin_path}')
+        with open(efuse_bin_path, 'wb') as f:
+            f.write(qemu_target_info.default_efuse)
+
+        # Prepare QEMU launch arguments
+        qemu_args = [qemu_target_info.qemu_prog, '-nographic']
+        qemu_args += qemu_target_info.qemu_args.split(' ')
+        qemu_args += [
+            '-drive', f'file={bin_path},if=mtd,format=raw',
+            '-drive', f'file={efuse_bin_path},if=none,format=raw,id=efuse', '-global', f'driver=nvram.{target}.efuse,property=drive,value=efuse',
+            '-global', f'driver=timer.{target}.timg,property=wdt_disable,value=true',
+        ]
+        if '-nic' not in qemu_extra_args:
+            qemu_args += ['-nic', 'user,model=open_eth']
+
+        if options.wait_for_gdb or gdb:
+            qemu_args += ['-gdb', f'tcp::{QEMU_PORT_GDB}', '-S']
+
+        if qemu_extra_args:
+            qemu_args += qemu_extra_args.split(' ')
+
+        # Launch QEMU!
+        if not options.bg_mode:
+            yellow_print('Running qemu (fg): ' + ' '.join(qemu_args))
+            subprocess.run(qemu_args)
+        else:
+            if options.wait_for_monitor:
+                qemu_args += ['-serial', f'tcp::{QEMU_PORT_SERIAL},server']
+            else:
+                qemu_args += ['-serial', f'tcp::{QEMU_PORT_SERIAL},server,nowait']
+
+            yellow_print('Running qemu (bg): ' + ' '.join(qemu_args))
+            qemu_proc = subprocess.Popen(qemu_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+            wait_for_socket(QEMU_PORT_SERIAL)
+
+            def cleanup_qemu() -> None:
+                if qemu_proc:
+                    qemu_proc.terminate()
+                    qemu_proc.wait()
+            atexit.register(cleanup_qemu)
+            if qemu_proc.poll() is not None:
+                yellow_print('QEMU exited with error')
+                if qemu_proc.stderr is not None:
+                    yellow_print('Stderr output was:')
+                    yellow_print(qemu_proc.stderr.read().decode('utf-8'))
+                sys.exit(1)
+
+            if gdb and not options.wait_for_gdb:
+                yellow_print('Waiting for GDB to connect. You can now run "idf.py gdb" in another terminal window.')
+
+    qemu_actions = {
+        'global_action_callbacks': [global_callback],
+        'actions': {
+            'qemu': {
+                'callback': qemu,
+                'help': 'Run QEMU.',
+                'dependencies': ['all'],
+                'options': [
+                    {
+                        'names': ['--qemu-extra-args'],
+                        'help': 'Extra arguments to pass to QEMU',
+                        'hidden': False,
+                        'is_flag': False,
+                        'default': '',
+                    },
+                    {
+                        'names': ['--gdb'],
+                        'help': ('Wait for gdb to connect. '
+                                 'Use this option to run "idf.py qemu --gdb monitor" in one terminal window '
+                                 'and "idf.py gdb" in another. The program will start running when gdb connects.'),
+                        'is_flag': True,
+                        'default': False,
+                    }
+                ]
+            }
+        }
+    }
+    return qemu_actions

tools/test_idf_py/test_idf_qemu.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+#
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+import os
+import sys
+import tempfile
+import unittest
+
+import pexpect
+
+
+class IdfPyQemuTest(unittest.TestCase):
+    def test_idf_qemu(self) -> None:
+        build_dir = tempfile.mkdtemp()
+        idf_path = os.environ['IDF_PATH']
+        hello_world_dir = os.path.join(idf_path, 'examples', 'get-started', 'hello_world')
+        idf_py = os.path.join(idf_path, 'tools', 'idf.py')
+        args = [idf_py, '-C', hello_world_dir, '-B', build_dir,
+                'qemu', '--qemu-extra-args', '-no-reboot', 'monitor']
+        logfile_name = os.path.join(os.environ['IDF_PATH'], 'qemu_log.out')
+        with open(logfile_name, 'w+b') as logfile, \
+             pexpect.spawn(sys.executable, args=args, logfile=logfile) as child:
+            child.expect('Executing action: all')
+            logging.info('Waiting for the build to finish...')
+            child.expect('Executing action: qemu', timeout=120)
+            child.expect('Generating flash image:')
+            child.expect('Generating efuse image:')
+            child.expect('Executing action: monitor')
+            child.expect('Hello world!')
+            child.expect('Restarting in 0 seconds', timeout=20)
+            child.expect('Restarting now.')
+
+
+if __name__ == '__main__':
+    unittest.main()