feat(tools): Add QEMU 8.0.0_20230522 to tools.json

Process wildcards in the install and download lists of idf_tools Fix the install and download handlers to get common behaviour
2024-10-05 20:47:46 -04:00 · 2023-09-14 12:55:24 +07:00 · 2023-09-14 12:55:24 +07:00 · be79c75b64
commit be79c75b64
parent 6b1f40b9bf
7 changed files with 278 additions and 94 deletions
--- a/.gitlab/ci/host-test.yml
+++ b/.gitlab/ci/host-test.yml
@ -135,10 +135,10 @@ test_idf_tools:
    entrypoint: [""]  # use system python3. no extra pip package installed
  script:
    # Tools must be downloaded for testing
-    - python3 ${IDF_PATH}/tools/idf_tools.py download
+    - python3 ${IDF_PATH}/tools/idf_tools.py download required qemu-riscv32 qemu-xtensa
    - cd ${IDF_PATH}/tools/test_idf_tools
    - python3 -m pip install jsonschema
-    - python3 ./test_idf_tools.py
+    - python3 ./test_idf_tools.py -v
    - python3 ./test_idf_tools_python_env.py

 .test_efuse_table_on_host_template:
--- a/docs/en/api-guides/tools/idf-tools-notes.inc
+++ b/docs/en/api-guides/tools/idf-tools-notes.inc
@ -1,5 +1,7 @@
 .. This file gets included from auto-generated part of idf-tools.rst.
 .. Comments "tool-NAME-notes" act as delimiters.
+..
+.. This is a padding to have the same line numbers as zh_CN version>


 .. tool-xtensa-esp-elf-gdb-notes
@ -77,6 +79,18 @@ On Linux and macOS, it is recommended to install ninja using the OS-specific pac
 .. tool-esp-rom-elfs-notes


+---
+
+.. tool-qemu-xtensa-notes
+
+Some ESP-specific instructions for running QEMU for Xtensa chips are here: https://github.com/espressif/esp-toolchain-docs/blob/main/qemu/esp32/README.md
+
+---
+
+.. tool-qemu-riscv32-notes
+
+Some ESP-specific instructions for running QEMU for RISC-V chips are here: https://github.com/espressif/esp-toolchain-docs/blob/main/qemu/esp32c3/README.md
+
 ---

 .. tool-idf-python-notes
--- a/docs/zh_CN/api-guides/tools/idf-tools-notes.inc
+++ b/docs/zh_CN/api-guides/tools/idf-tools-notes.inc
@ -46,7 +46,7 @@

 .. tool-cmake-notes

-On Linux and macOS, it is recommended to install CMake using the OS package manager. However, for convenience it is possible to install CMake using idf_tools.py along with the other tools.
+On Linux and macOS, it is recommended to install CMake using the OS-specific package manager (like apt, yum, brew, etc.). However, for convenience it is possible to install CMake using idf_tools.py along with the other tools.

 ---

@ -57,7 +57,7 @@ On Linux and macOS, it is recommended to install CMake using the OS package mana

 .. tool-ninja-notes

-On Linux and macOS, it is recommended to install ninja using the OS package manager. However, for convenience it is possible to install ninja using idf_tools.py along with the other tools.
+On Linux and macOS, it is recommended to install ninja using the OS-specific package manager (like apt, yum, brew, etc.). However, for convenience it is possible to install ninja using idf_tools.py along with the other tools.

 ---

@ -79,6 +79,18 @@ On Linux and macOS, it is recommended to install ninja using the OS package mana
 .. tool-esp-rom-elfs-notes


+---
+
+.. tool-qemu-xtensa-notes
+
+Some ESP-specific instructions for running QEMU for Xtensa chips are here: https://github.com/espressif/esp-toolchain-docs/blob/main/qemu/esp32/README.md
+
+---
+
+.. tool-qemu-riscv32-notes
+
+Some ESP-specific instructions for running QEMU for RISC-V chips are here: https://github.com/espressif/esp-toolchain-docs/blob/main/qemu/esp32c3/README.md
+
 ---

 .. tool-idf-python-notes
--- a/tools/docker/Dockerfile
+++ b/tools/docker/Dockerfile
@ -7,6 +7,7 @@ RUN : \
  && apt-get install -y \
    apt-utils \
    bison \
+    bzip2 \
    ca-certificates \
    ccache \
    check \
@ -18,7 +19,10 @@ RUN : \
    lcov \
    libbsd-dev \
    libffi-dev \
+    libglib2.0-0 \
    libncurses-dev \
+    libpixman-1-0 \
+    libslirp0 \
    libusb-1.0-0-dev \
    make \
    ninja-build \
@ -77,6 +81,7 @@ RUN echo IDF_CHECKOUT_REF=$IDF_CHECKOUT_REF IDF_CLONE_BRANCH_OR_TAG=$IDF_CLONE_B
 RUN : \
  && update-ca-certificates --fresh \
  && $IDF_PATH/tools/idf_tools.py --non-interactive install required --targets=${IDF_INSTALL_TARGETS} \
+  && $IDF_PATH/tools/idf_tools.py --non-interactive install qemu* --targets=${IDF_INSTALL_TARGETS} \
  && $IDF_PATH/tools/idf_tools.py --non-interactive install cmake \
  && $IDF_PATH/tools/idf_tools.py --non-interactive install-python-env \
  && rm -rf $IDF_TOOLS_PATH/dist \
@ -89,35 +94,6 @@ ENV IDF_PYTHON_CHECK_CONSTRAINTS=no
 # Ccache is installed, enable it by default
 ENV IDF_CCACHE_ENABLE=1

-# Install QEMU runtime dependencies
-RUN : \
-  && apt-get update && apt-get install -y -q \
-    bzip2 \
-    libglib2.0-0 \
-    libpixman-1-0 \
-    libslirp0 \
-  && rm -rf /var/lib/apt/lists/* \
-  && :
-
-# Install QEMU
-ARG QEMU_VER=develop_8.0.0_20230522
-ARG QEMU_RISCV32_DIST=esp-qemu-riscv32-softmmu-${QEMU_VER}-x86_64-linux-gnu.tar.bz2
-ARG QEMU_RISCV32_SHA256=bc7607720ff3d7e3d39f3e1810b8795f376f4b9cf3783c8f2ed3f7f14ba74717
-ARG QEMU_XTENSA_DIST=esp-qemu-xtensa-softmmu-${QEMU_VER}-x86_64-linux-gnu.tar.bz2
-ARG QEMU_XTENSA_SHA256=a7e5e779fd593cb15f6d197034dc2fb427ed9165a4743e2febc6f6a47dfcc618
-
-RUN bash -c ': \
-  && wget --no-verbose https://github.com/espressif/qemu/releases/download/esp-${QEMU_VER//_/-}/${QEMU_RISCV32_DIST} \
-  && echo "${QEMU_RISCV32_SHA256} *${QEMU_RISCV32_DIST}" | sha256sum --check --strict - \
-  && tar -xf ${QEMU_RISCV32_DIST} -C /opt \
-  && rm ${QEMU_RISCV32_DIST} \
-  && wget --no-verbose https://github.com/espressif/qemu/releases/download/esp-${QEMU_VER//_/-}/${QEMU_XTENSA_DIST} \
-  && echo "${QEMU_XTENSA_SHA256} *${QEMU_XTENSA_DIST}" | sha256sum --check --strict - \
-  && tar -xf ${QEMU_XTENSA_DIST} -C /opt \
-  && rm ${QEMU_XTENSA_DIST} \
-  '
-ENV PATH=/opt/qemu/bin:${PATH}
-
 COPY entrypoint.sh /opt/esp/entrypoint.sh
 ENTRYPOINT [ "/opt/esp/entrypoint.sh" ]
 CMD [ "/bin/bash" ]
--- a/tools/idf_tools.py
+++ b/tools/idf_tools.py
@ -33,6 +33,7 @@ import contextlib
 import copy
 import datetime
 import errno
+import fnmatch
 import functools
 import hashlib
 import json
@ -194,7 +195,16 @@ class Platforms:
        return Platforms.get(found_alias)


-CURRENT_PLATFORM = Platforms.get(PYTHON_PLATFORM)
+def parse_platform_arg(platform_str):  # type: (str) -> str
+    platform = Platforms.get(platform_str)
+    if platform is None:
+        fatal(f'unknown platform: {platform}')
+        raise SystemExit(1)
+    return platform
+
+
+CURRENT_PLATFORM = parse_platform_arg(PYTHON_PLATFORM)
+

 EXPORT_SHELL = 'shell'
 EXPORT_KEY_VALUE = 'key-value'
@ -387,6 +397,8 @@ def unpack(filename, destination):  # type: (str, str) -> None
        archive_obj = tarfile.open(filename, 'r:gz')  # type: Union[TarFile, ZipFile]
    elif filename.endswith(('.tar.xz')):
        archive_obj = tarfile.open(filename, 'r:xz')
+    elif filename.endswith(('.tar.bz2')):
+        archive_obj = tarfile.open(filename, 'r:bz2')
    elif filename.endswith('zip'):
        archive_obj = ZipFile(filename)
    else:
@ -714,6 +726,13 @@ class IDFTool(object):
    def get_supported_targets(self):  # type: ()  -> list[str]
        return self._current_options.supported_targets  # type: ignore

+    def is_supported_for_any_of_targets(self, targets):  # type: (list[str]) -> bool
+        """
+        Checks whether the tool is suitable for at least one of the specified targets.
+        """
+        supported_targets = self.get_supported_targets()
+        return (any(item in targets for item in supported_targets) or supported_targets == ['all'])
+
    def compatible_with_platform(self):  # type: () -> bool
        return any([v.compatible_with_platform() for v in self.versions.values()])

@ -1395,24 +1414,66 @@ def get_python_env_path() -> Tuple[str, str, str, str]:
    return idf_python_env_path, idf_python_export_path, virtualenv_python, idf_version


-def add_and_check_targets(idf_env_obj, targets_str):  # type: (IDFEnv, str) -> list[str]
+def parse_tools_arg(tools_str):  # type: (List[str]) -> List[str]
    """
-    Define targets from targets_str, check that the target names are valid and add them to idf_env_obj
+    Base parsing "tools" argumets: all, required, etc
+    """
+    if not tools_str:
+        return ['required']
+    else:
+        return tools_str
+
+
+def expand_tools_arg(tools_spec, overall_tools, targets):  # type: (list[str], OrderedDict, list[str]) -> list[str]
+    """ Expand list of tools 'tools_spec' in according:
+        - a tool is in the 'overall_tools' list
+        - consider metapackages like "required" and "all"
+        - process wildcards in tool names
+        - a tool supports chips from 'targets'
+    """
+    tools = []
+    # Filtering tools if they are in overall_tools
+    # Processing wildcards if possible
+    for tool_pattern in tools_spec:
+        tools.extend([k for k, _ in overall_tools.items() if fnmatch.fnmatch(k,tool_pattern) and k not in tools])
+
+    # Processing "metapackage"
+    if 'required' in tools_spec:
+        tools.extend([k for k, v in overall_tools.items() if v.get_install_type() == IDFTool.INSTALL_ALWAYS and k not in tools])
+
+    elif 'all' in tools_spec:
+        tools.extend([k for k, v in overall_tools.items() if v.get_install_type() != IDFTool.INSTALL_NEVER and k not in tools])
+
+    # Filtering by ESP_targets
+    tools = [k for k in tools if overall_tools[k].is_supported_for_any_of_targets(targets)]
+    return tools
+
+
+def parse_targets_arg(targets_str):  # type: (str) -> List[str]
+    """
+    Parse and check if targets_str is a valid list of targets and return a target list
    """
    targets_from_tools_json = get_all_targets_from_tools_json()
    invalid_targets = []

    targets_str = targets_str.lower()
    targets = targets_str.replace('-', '').split(',')
-    if targets != ['all']:
+    if targets == ['all']:
+        return targets_from_tools_json
+    else:
        invalid_targets = [t for t in targets if t not in targets_from_tools_json]
        if invalid_targets:
            warn('Targets: "{}" are not supported. Only allowed options are: {}.'.format(', '.join(invalid_targets), ', '.join(targets_from_tools_json)))
            raise SystemExit(1)
-        idf_env_obj.get_active_idf_record().extend_targets(targets)
-    else:
-        idf_env_obj.get_active_idf_record().extend_targets(targets_from_tools_json)
+        return targets

+
+def add_and_check_targets(idf_env_obj, targets_str):  # type: (IDFEnv, str) -> list[str]
+    """
+    Define targets from targets_str, check that the target names are valid and add them to idf_env_obj
+    """
+    targets = parse_targets_arg(targets_str)
+    idf_env_obj.get_active_idf_record().extend_targets(targets)
    return idf_env_obj.get_active_idf_record().targets


@ -1789,12 +1850,7 @@ def apply_github_assets_option(idf_download_url):  # type: (str) -> str


 def get_tools_spec_and_platform_info(selected_platform, targets, tools_spec,
-                                     quiet=False):  # type: (Optional[str], list[str], list[str], bool) -> Tuple[list[str], Dict[str, IDFTool]]
-    selected_platform = Platforms.get(selected_platform)
-    if selected_platform is None:
-        fatal(f'unknown platform: {selected_platform}')
-        raise SystemExit(1)
-
+                                     quiet=False):  # type: (str, list[str], list[str], bool) -> Tuple[list[str], Dict[str, IDFTool]]
    # If this function is not called from action_download, but is used just for detecting active tools, info about downloading is unwanted.
    global global_quiet
    try:
@ -1806,21 +1862,8 @@ def get_tools_spec_and_platform_info(selected_platform, targets, tools_spec,
            tool_for_platform = tool_obj.copy_for_platform(selected_platform)
            tools_info_for_platform[name] = tool_for_platform

-        if not tools_spec or 'required' in tools_spec:
-            # Downloading tools for all ESP_targets required by the operating system.
-            tools_spec = [k for k, v in tools_info_for_platform.items() if v.get_install_type() == IDFTool.INSTALL_ALWAYS]
-            # Filtering tools user defined list of ESP_targets
-            if 'all' not in targets:
-                def is_tool_selected(tool):  # type: (IDFTool) -> bool
-                    supported_targets = tool.get_supported_targets()
-                    return (any(item in targets for item in supported_targets) or supported_targets == ['all'])
-                tools_spec = [k for k in tools_spec if is_tool_selected(tools_info[k])]
-            info('Downloading tools for {}: {}'.format(selected_platform, ', '.join(tools_spec)))
-
-        # Downloading tools for all ESP_targets (MacOS, Windows, Linux)
-        elif 'all' in tools_spec:
-            tools_spec = [k for k, v in tools_info_for_platform.items() if v.get_install_type() != IDFTool.INSTALL_NEVER]
-            info('Downloading tools for {}: {}'.format(selected_platform, ', '.join(tools_spec)))
+        tools_spec = expand_tools_arg(tools_spec, tools_info_for_platform, targets)
+        info('Downloading tools for {}: {}'.format(selected_platform, ', '.join(tools_spec)))
    finally:
        global_quiet = old_global_quiet

@ -1828,10 +1871,11 @@ def get_tools_spec_and_platform_info(selected_platform, targets, tools_spec,


 def action_download(args):  # type: ignore
-    tools_spec = args.tools
+    tools_spec = parse_tools_arg(args.tools)
+
    targets = []  # type: list[str]
-    # Downloading tools required for defined ESP_targets
-    if 'required' in tools_spec:
+    # Saving IDFEnv::targets for selected ESP_targets if all tools have been specified
+    if 'required' in tools_spec or 'all' in tools_spec:
        idf_env_obj = IDFEnv.get_idf_env()
        targets = add_and_check_targets(idf_env_obj, args.targets)
        try:
@ -1840,9 +1884,13 @@ def action_download(args):  # type: ignore
            if args.targets in targets:
                targets.remove(args.targets)
            warn('Downloading tools for targets was not successful with error: {}'.format(err))
+    # Taking into account ESP_targets but not saving them for individual tools (specified list of tools)
+    else:
+        targets = parse_targets_arg(args.targets)

-    tools_spec, tools_info_for_platform = get_tools_spec_and_platform_info(args.platform, targets, args.tools)
+    platform = parse_platform_arg(args.platform)

+    tools_spec, tools_info_for_platform = get_tools_spec_and_platform_info(platform, targets, tools_spec)
    for tool_spec in tools_spec:
        if '@' not in tool_spec:
            tool_name = tool_spec
@ -1864,18 +1912,17 @@ def action_download(args):  # type: ignore
        tool_spec = '{}@{}'.format(tool_name, tool_version)

        info('Downloading {}'.format(tool_spec))
-        _idf_tool_obj = tool_obj.versions[tool_version].get_download_for_platform(args.platform)
+        _idf_tool_obj = tool_obj.versions[tool_version].get_download_for_platform(platform)
        _idf_tool_obj.url = get_idf_download_url_apply_mirrors(args, _idf_tool_obj.url)

        tool_obj.download(tool_version)


 def action_install(args):  # type: ignore
-    tools_info = load_tools_info()
-    tools_spec = args.tools  # type: ignore
+    tools_spec = parse_tools_arg(args.tools)
+
    targets = []  # type: list[str]
-    info('Current system platform: {}'.format(CURRENT_PLATFORM))
-    # No single tool '<tool_name>@<version>' was defined, install whole toolchains
+    # Saving IDFEnv::targets for selected ESP_targets if all tools have been specified
    if 'required' in tools_spec or 'all' in tools_spec:
        idf_env_obj = IDFEnv.get_idf_env()
        targets = add_and_check_targets(idf_env_obj, args.targets)
@ -1886,23 +1933,14 @@ def action_install(args):  # type: ignore
                targets.remove(args.targets)
            warn('Installing targets was not successful with error: {}'.format(err))
        info('Selected targets are: {}'.format(', '.join(targets)))
+    # Taking into account ESP_targets but not saving them for individual tools (specified list of tools)
+    else:
+        targets = parse_targets_arg(args.targets)

-        # Installing tools for defined ESP_targets
-        if 'required' in tools_spec:
-            tools_spec = [k for k, v in tools_info.items() if v.get_install_type() == IDFTool.INSTALL_ALWAYS]
-            # If only some ESP_targets are defined, filter tools for those
-            if len(get_all_targets_from_tools_json()) != len(targets):
-                def is_tool_selected(tool):  # type: (IDFTool) -> bool
-                    supported_targets = tool.get_supported_targets()
-                    return (any(item in targets for item in supported_targets) or supported_targets == ['all'])
-                tools_spec = [k for k in tools_spec if is_tool_selected(tools_info[k])]
-            info('Installing tools: {}'.format(', '.join(tools_spec)))
-
-        # Installing all available tools for all operating systems (MacOS, Windows, Linux)
-        else:
-            tools_spec = [k for k, v in tools_info.items() if v.get_install_type() != IDFTool.INSTALL_NEVER]
-            info('Installing tools: {}'.format(', '.join(tools_spec)))
-
+    info('Current system platform: {}'.format(CURRENT_PLATFORM))
+    tools_info = load_tools_info()
+    tools_spec = expand_tools_arg(tools_spec, tools_info, targets)
+    info('Installing tools: {}'.format(', '.join(tools_spec)))
    for tool_spec in tools_spec:
        if '@' not in tool_spec:
            tool_name = tool_spec
@ -2541,6 +2579,7 @@ def main(argv):  # type: (list[str]) -> None
    install.add_argument('tools', metavar='TOOL', nargs='*', default=['required'],
                         help='Tools to install. ' +
                         'To install a specific version use <tool_name>@<version> syntax. ' +
+                         'To install tools by pattern use wildcards in <tool_name_pattern> . ' +
                         'Use empty or \'required\' to install required tools, not optional ones. ' +
                         'Use \'all\' to install all tools, including the optional ones.')
    install.add_argument('--targets', default='all', help='A comma separated list of desired chip targets for installing.' +
@ -2551,6 +2590,7 @@ def main(argv):  # type: (list[str]) -> None
    download.add_argument('tools', metavar='TOOL', nargs='*', default=['required'],
                          help='Tools to download. ' +
                          'To download a specific version use <tool_name>@<version> syntax. ' +
+                          'To download tools by pattern use wildcards in <tool_name_pattern> . ' +
                          'Use empty or \'required\' to download required tools, not optional ones. ' +
                          'Use \'all\' to download all tools, including the optional ones.')
    download.add_argument('--targets', default='all', help='A comma separated list of desired chip targets for installing.' +
--- a/tools/test_idf_tools/test_idf_tools.py
+++ b/tools/test_idf_tools/test_idf_tools.py
@ -46,6 +46,8 @@ XTENSA_ESP32S3_ELF = 'xtensa-esp32s3-elf'
 XTENSA_ESP_GDB = 'xtensa-esp-elf-gdb'
 RISCV_ESP_GDB = 'riscv32-esp-elf-gdb'
 ESP_ROM_ELFS = 'esp-rom-elfs'
+QEMU_RISCV = 'qemu-riscv32'
+QEMU_XTENSA = 'qemu-xtensa'


 def get_version_dict():
@ -72,6 +74,19 @@ XTENSA_ESP32S3_ELF_VERSION = version_dict[XTENSA_ESP32S3_ELF]
 XTENSA_ESP_GDB_VERSION = version_dict[XTENSA_ESP_GDB]
 RISCV_ESP_GDB_VERSION = version_dict[RISCV_ESP_GDB]
 ESP_ROM_ELFS_VERSION = version_dict[ESP_ROM_ELFS]
+QEMU_RISCV_VERSION = version_dict[QEMU_RISCV]
+QEMU_XTENSA_VERSION = version_dict[QEMU_XTENSA]
+
+
+# There are some complex search patterns to detect download snippets
+
+# Avoiding an ambiguity with a substring 'riscv32-esp-elf' in the `riscv32-esp-elf-gdb`
+# (removing esp- prefix from version)
+RISCV_ELF_ARCHIVE_PATTERN = RISCV_ELF + '-' \
+    + (RISCV_ELF_VERSION[len('esp-'):] if RISCV_ELF_VERSION.startswith('esp-') else RISCV_ELF_VERSION)
+
+QEMU_RISCV_ARCHIVE_PATTERN = 'esp-' + QEMU_RISCV
+QEMU_XTENSA_ARCHIVE_PATTERN = 'esp-' + QEMU_XTENSA


 class TestUsage(unittest.TestCase):
@ -148,7 +163,7 @@ class TestUsage(unittest.TestCase):
        required_tools_installed = 9
        output = self.run_idf_tools_with_action(['install'])
        self.assert_tool_installed(output, OPENOCD, OPENOCD_VERSION)
-        self.assert_tool_installed(output, RISCV_ELF, RISCV_ELF_VERSION)
+        self.assert_tool_installed(output, RISCV_ELF, RISCV_ELF_VERSION, RISCV_ELF_ARCHIVE_PATTERN)
        self.assert_tool_installed(output, XTENSA_ESP32_ELF, XTENSA_ESP32_ELF_VERSION)
        self.assert_tool_installed(output, XTENSA_ESP32S2_ELF, XTENSA_ESP32S2_ELF_VERSION)
        self.assert_tool_installed(output, XTENSA_ESP32S3_ELF, XTENSA_ESP32S3_ELF_VERSION)
@ -228,7 +243,7 @@ class TestUsage(unittest.TestCase):
        self.assert_tool_installed(output, OPENOCD, OPENOCD_VERSION)
        self.assert_tool_installed(output, ESP32ULP, ESP32ULP_VERSION)
        self.assert_tool_installed(output, XTENSA_ESP_GDB, XTENSA_ESP_GDB_VERSION)
-        self.assert_tool_not_installed(output, RISCV_ELF, RISCV_ELF_VERSION)
+        self.assert_tool_not_installed(output, RISCV_ELF, RISCV_ELF_VERSION, RISCV_ELF_ARCHIVE_PATTERN)
        self.assert_tool_not_installed(output, XTENSA_ESP32S2_ELF, XTENSA_ESP32S2_ELF_VERSION)
        self.assert_tool_not_installed(output, XTENSA_ESP32S3_ELF, XTENSA_ESP32S3_ELF_VERSION)
        self.assert_tool_not_installed(output, RISCV_ESP_GDB, RISCV_ESP_GDB_VERSION)
@ -267,7 +282,7 @@ class TestUsage(unittest.TestCase):
        required_tools_installed = 4
        output = self.run_idf_tools_with_action(['install', '--targets=esp32c3'])
        self.assert_tool_installed(output, OPENOCD, OPENOCD_VERSION)
-        self.assert_tool_installed(output, RISCV_ELF, RISCV_ELF_VERSION)
+        self.assert_tool_installed(output, RISCV_ELF, RISCV_ELF_VERSION, RISCV_ELF_ARCHIVE_PATTERN)
        self.assert_tool_installed(output, RISCV_ESP_GDB, RISCV_ESP_GDB_VERSION)
        self.assert_tool_not_installed(output, XTENSA_ESP32_ELF, XTENSA_ESP32_ELF_VERSION)
        self.assert_tool_not_installed(output, XTENSA_ESP32S2_ELF, XTENSA_ESP32S2_ELF_VERSION)
@ -307,7 +322,7 @@ class TestUsage(unittest.TestCase):
        output = self.run_idf_tools_with_action(['install', '--targets=esp32s2'])
        self.assert_tool_installed(output, XTENSA_ESP32S2_ELF, XTENSA_ESP32S2_ELF_VERSION)
        self.assert_tool_installed(output, OPENOCD, OPENOCD_VERSION)
-        self.assert_tool_installed(output, RISCV_ELF, RISCV_ELF_VERSION)
+        self.assert_tool_installed(output, RISCV_ELF, RISCV_ELF_VERSION, RISCV_ELF_ARCHIVE_PATTERN)
        self.assert_tool_installed(output, ESP32ULP, ESP32ULP_VERSION)
        self.assert_tool_installed(output, XTENSA_ESP_GDB, XTENSA_ESP_GDB_VERSION)
        self.assert_tool_installed(output, ESP_ROM_ELFS, ESP_ROM_ELFS_VERSION)
@ -348,7 +363,7 @@ class TestUsage(unittest.TestCase):
        output = self.run_idf_tools_with_action(['install', '--targets=esp32s3'])
        self.assert_tool_installed(output, XTENSA_ESP32S3_ELF, XTENSA_ESP32S3_ELF_VERSION)
        self.assert_tool_installed(output, OPENOCD, OPENOCD_VERSION)
-        self.assert_tool_installed(output, RISCV_ELF, RISCV_ELF_VERSION)
+        self.assert_tool_installed(output, RISCV_ELF, RISCV_ELF_VERSION, RISCV_ELF_ARCHIVE_PATTERN)
        self.assert_tool_installed(output, ESP32ULP, ESP32ULP_VERSION)
        self.assert_tool_installed(output, XTENSA_ESP_GDB, XTENSA_ESP_GDB_VERSION)
        self.assert_tool_installed(output, ESP_ROM_ELFS, ESP_ROM_ELFS_VERSION)
@ -385,6 +400,69 @@ class TestUsage(unittest.TestCase):
        self.assertIn('%s/tools/esp-rom-elfs/%s/' %
                      (self.temp_tools_dir, ESP_ROM_ELFS_VERSION), output)

+    # a different test for qemu because of "on_request"
+    def test_tools_for_qemu_with_required(self):
+        required_tools_installed = 11
+        output = self.run_idf_tools_with_action(['install', 'required', 'qemu-xtensa', 'qemu-riscv32'])
+        self.assert_tool_installed(output, OPENOCD, OPENOCD_VERSION)
+        self.assert_tool_installed(output, RISCV_ELF, RISCV_ELF_VERSION, RISCV_ELF_ARCHIVE_PATTERN)
+        self.assert_tool_installed(output, XTENSA_ESP32_ELF, XTENSA_ESP32_ELF_VERSION)
+        self.assert_tool_installed(output, XTENSA_ESP32S2_ELF, XTENSA_ESP32S2_ELF_VERSION)
+        self.assert_tool_installed(output, XTENSA_ESP32S3_ELF, XTENSA_ESP32S3_ELF_VERSION)
+        self.assert_tool_installed(output, ESP32ULP, ESP32ULP_VERSION)
+        self.assert_tool_installed(output, XTENSA_ESP_GDB, XTENSA_ESP_GDB_VERSION)
+        self.assert_tool_installed(output, RISCV_ESP_GDB, RISCV_ESP_GDB_VERSION)
+        self.assert_tool_installed(output, ESP_ROM_ELFS, ESP_ROM_ELFS_VERSION)
+        self.assert_tool_installed(output, QEMU_RISCV, QEMU_RISCV_VERSION, QEMU_RISCV_ARCHIVE_PATTERN)
+        self.assert_tool_installed(output, QEMU_XTENSA, QEMU_XTENSA_VERSION, QEMU_XTENSA_ARCHIVE_PATTERN)
+        self.assertIn('Destination: {}'.format(os.path.join(self.temp_tools_dir, 'dist')), output)
+        self.assertEqual(required_tools_installed, output.count('Done'))
+
+    def test_tools_for_wildcards1(self):
+        required_tools_installed = 2
+        output = self.run_idf_tools_with_action(['install', '*gdb*'])
+        self.assert_tool_not_installed(output, OPENOCD, OPENOCD_VERSION)
+        self.assert_tool_not_installed(output, RISCV_ELF, RISCV_ELF_VERSION,RISCV_ELF_ARCHIVE_PATTERN)
+        self.assert_tool_installed(output, RISCV_ESP_GDB, RISCV_ESP_GDB_VERSION)
+        self.assert_tool_not_installed(output, XTENSA_ESP32_ELF, XTENSA_ESP32_ELF_VERSION)
+        self.assert_tool_not_installed(output, XTENSA_ESP32S2_ELF, XTENSA_ESP32S2_ELF_VERSION)
+        self.assert_tool_not_installed(output, XTENSA_ESP32S3_ELF, XTENSA_ESP32S3_ELF_VERSION)
+        self.assert_tool_not_installed(output, ESP32ULP, ESP32ULP_VERSION)
+        self.assert_tool_installed(output, XTENSA_ESP_GDB, XTENSA_ESP_GDB_VERSION)
+        self.assert_tool_not_installed(output, ESP_ROM_ELFS, ESP_ROM_ELFS_VERSION)
+        self.assertIn('Destination: {}'.format(os.path.join(self.temp_tools_dir, 'dist')), output)
+        self.assertEqual(required_tools_installed, output.count('Done'))
+
+    def test_tools_for_wildcards2(self):
+        required_tools_installed = 1
+        output = self.run_idf_tools_with_action(['install', '*gdb*', '--targets=esp32c3'])
+        self.assert_tool_not_installed(output, OPENOCD, OPENOCD_VERSION)
+        self.assert_tool_not_installed(output, RISCV_ELF, RISCV_ELF_VERSION, RISCV_ELF_ARCHIVE_PATTERN)
+        self.assert_tool_installed(output, RISCV_ESP_GDB, RISCV_ESP_GDB_VERSION)
+        self.assert_tool_not_installed(output, XTENSA_ESP32_ELF, XTENSA_ESP32_ELF_VERSION)
+        self.assert_tool_not_installed(output, XTENSA_ESP32S2_ELF, XTENSA_ESP32S2_ELF_VERSION)
+        self.assert_tool_not_installed(output, XTENSA_ESP32S3_ELF, XTENSA_ESP32S3_ELF_VERSION)
+        self.assert_tool_not_installed(output, ESP32ULP, ESP32ULP_VERSION)
+        self.assert_tool_not_installed(output, XTENSA_ESP_GDB, XTENSA_ESP_GDB_VERSION)
+        self.assert_tool_not_installed(output, ESP_ROM_ELFS, ESP_ROM_ELFS_VERSION)
+        self.assertIn('Destination: {}'.format(os.path.join(self.temp_tools_dir, 'dist')), output)
+        self.assertEqual(required_tools_installed, output.count('Done'))
+
+    def test_tools_for_wildcards3(self):
+        required_tools_installed = 1
+        output = self.run_idf_tools_with_action(['install', '*gdb*', '--targets=esp32s3'])
+        self.assert_tool_not_installed(output, OPENOCD, OPENOCD_VERSION)
+        self.assert_tool_not_installed(output, RISCV_ELF, RISCV_ELF_VERSION, RISCV_ELF_ARCHIVE_PATTERN)
+        self.assert_tool_not_installed(output, RISCV_ESP_GDB, RISCV_ESP_GDB_VERSION)
+        self.assert_tool_not_installed(output, XTENSA_ESP32_ELF, XTENSA_ESP32_ELF_VERSION)
+        self.assert_tool_not_installed(output, XTENSA_ESP32S2_ELF, XTENSA_ESP32S2_ELF_VERSION)
+        self.assert_tool_not_installed(output, XTENSA_ESP32S3_ELF, XTENSA_ESP32S3_ELF_VERSION)
+        self.assert_tool_not_installed(output, ESP32ULP, ESP32ULP_VERSION)
+        self.assert_tool_installed(output, XTENSA_ESP_GDB, XTENSA_ESP_GDB_VERSION)
+        self.assert_tool_not_installed(output, ESP_ROM_ELFS, ESP_ROM_ELFS_VERSION)
+        self.assertIn('Destination: {}'.format(os.path.join(self.temp_tools_dir, 'dist')), output)
+        self.assertEqual(required_tools_installed, output.count('Done'))
+
    def test_uninstall_option(self):
        self.run_idf_tools_with_action(['install', '--targets=esp32'])

@ -427,6 +505,7 @@ class TestMaintainer(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        idf_path = os.getenv('IDF_PATH')
+        assert idf_path, 'IDF_PATH needs to be set to run this test'
        cls.tools_old = os.path.join(idf_path, 'tools/tools.json')
        cls.tools_new = os.path.join(idf_path, 'tools/tools.new.json')
        cls.test_tool_name = 'xtensa-esp32-elf'
@ -436,9 +515,6 @@ class TestMaintainer(unittest.TestCase):

    def test_json_rewrite(self):
        idf_tools.main(['rewrite'])
-        idf_path = os.getenv('IDF_PATH')
-        if not idf_path:
-            self.fail('IDF_PATH needs to be set to run this test')
        with open(self.tools_old, 'r') as f:
            json_old = f.read()
        with open(self.tools_new, 'r') as f:
--- a/tools/tools.json
+++ b/tools/tools.json
@ -1003,6 +1003,72 @@
          "status": "recommended"
        }
      ]
+    },
+    {
+      "description": "QEMU for Xtensa",
+      "export_paths": [
+        [
+          "qemu",
+          "bin"
+        ]
+      ],
+      "export_vars": {},
+      "info_url": "https://github.com/espressif/qemu",
+      "install": "on_request",
+      "license": "GPL-2.0-only",
+      "name": "qemu-xtensa",
+      "supported_targets": [
+        "esp32"
+      ],
+      "version_cmd": [
+        "qemu-system-xtensa",
+        "--version"
+      ],
+      "version_regex": "QEMU emulator version ([a-z0-9.-_]+)",
+      "versions": [
+        {
+          "linux-amd64": {
+            "sha256": "a7e5e779fd593cb15f6d197034dc2fb427ed9165a4743e2febc6f6a47dfcc618",
+            "size": 45962695,
+            "url": "https://github.com/espressif/qemu/releases/download/esp-develop-8.0.0-20230522/esp-qemu-xtensa-softmmu-develop_8.0.0_20230522-x86_64-linux-gnu.tar.bz2"
+          },
+          "name": "8.0.0",
+          "status": "recommended"
+        }
+      ]
+    },
+    {
+      "description": "QEMU for RISC-V",
+      "export_paths": [
+        [
+          "qemu",
+          "bin"
+        ]
+      ],
+      "export_vars": {},
+      "info_url": "https://github.com/espressif/qemu",
+      "install": "on_request",
+      "license": "GPL-2.0-only",
+      "name": "qemu-riscv32",
+      "supported_targets": [
+        "esp32c3"
+      ],
+      "version_cmd": [
+        "qemu-system-riscv32",
+        "--version"
+      ],
+      "version_regex": "QEMU emulator version ([a-z0-9.-_]+)",
+      "versions": [
+        {
+          "linux-amd64": {
+            "sha256": "bc7607720ff3d7e3d39f3e1810b8795f376f4b9cf3783c8f2ed3f7f14ba74717",
+            "size": 47175493,
+            "url": "https://github.com/espressif/qemu/releases/download/esp-develop-8.0.0-20230522/esp-qemu-riscv32-softmmu-develop_8.0.0_20230522-x86_64-linux-gnu.tar.bz2"
+          },
+          "name": "8.0.0",
+          "status": "recommended"
+        }
+      ]
    }
  ],
  "version": 1