Merge branch 'feature/clippy' into 'master'

idf.py: Add automated hints on how to resolve errors

Closes IDF-4511, IDF-4512, and IDF-4631

See merge request espressif/esp-idf!16998

docs/en/api-guides/tools/idf-py.rst

@@ -31,7 +31,7 @@ be created can be specified by the ``--path`` option.
 Create a new component: create-component
 ----------------------------------------
 
-This command creates a new component, which will have a minimum set of files 
+This command creates a new component, which will have a minimum set of files
 necessary for building.
 
 .. code-block:: bash
@@ -84,7 +84,7 @@ Build the project: build
 ------------------------
 
 .. code-block:: bash
-  
+
   idf.py build
 
 Running this command will build the project found in the current directory. This can involve multiple steps:
@@ -95,7 +95,7 @@ Running this command will build the project found in the current directory. This
 
 Building is incremental so if no source files or configuration has changed since the last build, nothing will be done.
 
-Additionally, the command can be run with ``app``, ``bootloader`` and 
+Additionally, the command can be run with ``app``, ``bootloader`` and
 ``partition-table`` arguments to build only the app, bootloader or partition table
 as applicable.
 
@@ -139,10 +139,17 @@ You can use ``-p`` and ``-b`` options to set serial port name and flasher baud r
 
 .. note:: The environment variables ``ESPPORT`` and ``ESPBAUD`` can be used to set default values for the ``-p`` and ``-b`` options, respectively. Providing these options on the command line overrides the default.
 
-Similarly to the ``build`` command, the command can be run with ``app``, 
+Similarly to the ``build`` command, the command can be run with ``app``,
 ``bootloader`` and ``partition-table`` arguments to flash only the app, bootloader
 or partition table as applicable.
 
+Hints on how to resolve errors
+==============================
+
+``idf.py`` will try to suggest hints on how to resolve errors. It works with a database of hints stored in :idf_file:`tools/idf_py_actions/hints.yml` and the hints will be printed if a match is found for the given error. ``idf.py menuconfig`` is not supported by automatic hints on resolving errors.
+
+The ``--no-hints`` argument of ``idf.py`` can be used to turn the hints off in case they are not desired.
+
 Important notes
 ===============
 
@@ -204,7 +211,7 @@ Reconfigure the project: reconfigure
   idf.py reconfigure
 
 This command re-runs CMake_ even if it doesn't seem to need re-running.
-This isn't necessary during normal usage, but can be useful after adding/removing 
+This isn't necessary during normal usage, but can be useful after adding/removing
 files from the source tree, or when modifying CMake cache variables.
 For example, ``idf.py -DNAME='VALUE' reconfigure`` can be used to set variable ``NAME`` in CMake cache to value ``VALUE``.
 
@@ -233,6 +240,7 @@ Note that some older versions of CCache may exhibit bugs on some platforms, so i
 
 - ``-v`` flag causes both ``idf.py`` and the build system to produce verbose build output. This can be useful for debugging build problems.
 - ``--cmake-warn-uninitialized`` (or ``-w``) will cause CMake to print uninitialized variable warnings found in the project directory only. This only controls CMake variable warnings inside CMake itself, not other types of build warnings. This option can also be set permanently by setting the ``IDF_CMAKE_WARN_UNINITIALIZED`` environment variable to a non-zero value.
+- ``--no-hints`` flag to disable hints on resolving errors and disable capturing output.
 
 .. _cmake: https://cmake.org
 .. _ninja: https://ninja-build.org

tools/ci/mypy_ignore_list.txt

@@ -210,7 +210,6 @@ tools/find_apps.py
 tools/find_build_apps/common.py
 tools/gen_esp_err_to_name.py
 tools/gen_soc_caps_kconfig/test/test_gen_soc_caps_kconfig.py
-tools/idf_py_actions/tools.py
 tools/kconfig_new/confgen.py
 tools/kconfig_new/confserver.py
 tools/kconfig_new/gen_kconfig_doc.py

tools/idf_monitor.py

@@ -84,15 +84,16 @@ class Monitor:
         websocket_client=None,  # type: Optional[WebSocketClient]
         enable_address_decoding=True,  # type: bool
         timestamps=False,  # type: bool
-        timestamp_format=''  # type: str
+        timestamp_format='',  # type: str
+        force_color=False  # type: bool
     ):
         self.event_queue = queue.Queue()  # type: queue.Queue
         self.cmd_queue = queue.Queue()  # type: queue.Queue
         self.console = miniterm.Console()
-
-        sys.stderr = get_converter(sys.stderr, decode_output=True)
-        self.console.output = get_converter(self.console.output)
-        self.console.byte_output = get_converter(self.console.byte_output)
+        # if the variable is set ANSI will be printed even if we do not print to terminal
+        sys.stderr = get_converter(sys.stderr, decode_output=True, force_color=force_color)
+        self.console.output = get_converter(self.console.output, force_color=force_color)
+        self.console.byte_output = get_converter(self.console.byte_output, force_color=force_color)
 
         self.elf_file = elf_file or ''
         self.elf_exists = os.path.exists(self.elf_file)
@@ -353,7 +354,8 @@ def main() -> None:
                       ws,
                       not args.disable_address_decoding,
                       args.timestamps,
-                      args.timestamp_format)
+                      args.timestamp_format,
+                      args.force_color)
 
         yellow_print('--- Quit: {} | Menu: {} | Help: {} followed by {} ---'.format(
             key_description(monitor.console_parser.exit_key),

tools/idf_monitor_base/ansi_color_converter.py

@@ -28,14 +28,15 @@ if os.name == 'nt':
     SetConsoleTextAttribute = ctypes.windll.kernel32.SetConsoleTextAttribute  # type: ignore
 
 
-def get_converter(orig_output_method=None, decode_output=False):
-    # type: (Any[TextIO, Optional[TextIOBase]], bool) -> Union[ANSIColorConverter, Optional[TextIOBase]]
+def get_converter(orig_output_method=None, decode_output=False, force_color=False):
+    # type: (Any[TextIO, Optional[TextIOBase]], bool, bool) -> Union[ANSIColorConverter, Optional[TextIOBase]]
     """
     Returns an ANSIColorConverter on Windows and the original output method (orig_output_method) on other platforms.
-    The ANSIColorConverter with decode_output=True will decode the bytes before passing them to the output.
+    The ANSIColorConverter with decode_output=True will decode the bytes before passing them to the output
+    The ANSIColorConverter with force_color=True will be forced to convert ANSI in windows format
     """
     if os.name == 'nt':
-        return ANSIColorConverter(orig_output_method, decode_output)
+        return ANSIColorConverter(orig_output_method, decode_output, force_color)
     return orig_output_method
 
 
@@ -50,12 +51,13 @@ class ANSIColorConverter(object):
     least-bad working solution, as winpty doesn't support any "passthrough" mode for raw output.
     """
 
-    def __init__(self, output=None, decode_output=False):
-        # type: (TextIOBase, bool) -> None
+    def __init__(self, output=None, decode_output=False, force_color=False):
+        # type: (TextIOBase, bool, bool) -> None
         self.output = output
         self.decode_output = decode_output
         self.handle = GetStdHandle(STD_ERROR_HANDLE if self.output == sys.stderr else STD_OUTPUT_HANDLE)
         self.matched = b''
+        self.force_color = force_color  # always print ANSI for colors if true
 
     def _output_write(self, data):  # type: (Union[str, bytes]) -> None
         try:
@@ -89,12 +91,12 @@ class ANSIColorConverter(object):
                 self.matched = b
             elif (length == 1 and b == b'[') or (1 < length < 7):
                 self.matched += b
-                if self.matched == ANSI_NORMAL.encode('latin-1'):  # reset console
+                if not self.force_color and self.matched == ANSI_NORMAL.encode('latin-1'):  # reset console
                     # Flush is required only with Python3 - switching color before it is printed would mess up the console
                     self.flush()
                     SetConsoleTextAttribute(self.handle, FOREGROUND_GREY)
                     self.matched = b''
-                elif len(self.matched) == 7:  # could be an ANSI sequence
+                elif self.force_color or len(self.matched) == 7:  # could be an ANSI sequence
                     m = re.match(RE_ANSI_COLOR, self.matched)
                     if m is not None:
                         color = ANSI_TO_WINDOWS_COLOR[int(m.group(2))]

tools/idf_monitor_base/argument_parser.py

@@ -114,4 +114,10 @@ def get_parser():  # type: () -> argparse.ArgumentParser
         help='Set a strftime()-compatible timestamp format'
     )
 
+    parser.add_argument(
+        '--force-color',
+        help='Always colored monitor output, even if output is redirected.',
+        default=False,
+        action='store_true')
+
     return parser

tools/idf_monitor_base/output_helpers.py

@@ -14,6 +14,7 @@ ANSI_NORMAL = '\033[0m'
 def color_print(message, color, newline='\n'):  # type: (str, str, Optional[str]) -> None
     """ Print a message to stderr with colored highlighting """
     sys.stderr.write('%s%s%s%s' % (color, message, ANSI_NORMAL, newline))
+    sys.stderr.flush()
 
 
 def normal_print(message):  # type: (str) -> None

tools/idf_py_actions/constants.py

@@ -4,17 +4,23 @@ import collections
 import multiprocessing
 import os
 import platform
+from typing import Dict, Union
 
-GENERATORS = collections.OrderedDict([
+GENERATORS: Dict[str, Union[str, Dict, list]] = collections.OrderedDict([
     # - command: build command line
     # - version: version command line
     # - dry_run: command to run in dry run mode
     # - verbose_flag: verbose flag
+    # - force_progression: one liner status of the progress
+    # - envvar: environment variables
     ('Ninja', {
         'command': ['ninja'],
         'version': ['ninja', '--version'],
         'dry_run': ['ninja', '-n'],
-        'verbose_flag': '-v'
+        'verbose_flag': '-v',
+        # as opposed to printing the status updates each in a in new line
+        'force_progression': True,
+        'envvar': {}
     }),
 ])
 
@@ -23,7 +29,10 @@ if os.name != 'nt':
     GENERATORS['Unix Makefiles'] = {'command': [MAKE_CMD, '-j', str(multiprocessing.cpu_count() + 2)],
                                     'version': [MAKE_CMD, '--version'],
                                     'dry_run': [MAKE_CMD, '-n'],
-                                    'verbose_flag': 'VERBOSE=1'}
+                                    'verbose_flag': 'VERBOSE=1',
+                                    'force_progression': False,
+                                    # CLICOLOR_FORCE if set forcing make to print ANSI escape sequence
+                                    'envvar': {'CLICOLOR_FORCE': '1'}}
 
 URL_TO_DOC = 'https://docs.espressif.com/projects/esp-idf'
 

tools/idf_py_actions/core_ext.py

@@ -18,7 +18,7 @@ from idf_py_actions.constants import GENERATORS, PREVIEW_TARGETS, SUPPORTED_TARG
 from idf_py_actions.errors import FatalError
 from idf_py_actions.global_options import global_options
 from idf_py_actions.tools import (PropertyDict, TargetChoice, ensure_build_directory, get_target, idf_version,
-                                  merge_action_lists, realpath, run_target)
+                                  merge_action_lists, print_hints, realpath, run_target)
 
 
 def action_extensions(base_actions: Dict, project_path: str) -> Any:
@@ -29,8 +29,9 @@ def action_extensions(base_actions: Dict, project_path: str) -> Any:
         Calls ensure_build_directory() which will run cmake to generate a build
         directory (with the specified generator) as needed.
         """
+        hints = not args.no_hints
         ensure_build_directory(args, ctx.info_name)
-        run_target(target_name, args)
+        run_target(target_name, args, force_progression=GENERATORS[args.generator].get('force_progression', False), hints=hints)
 
     def size_target(target_name: str, ctx: Context, args: PropertyDict) -> None:
         """
@@ -40,11 +41,13 @@ def action_extensions(base_actions: Dict, project_path: str) -> Any:
 
         """
 
-        def tool_error_handler(e: int) -> None:
-            pass
+        def tool_error_handler(e: int, stdout: str, stderr: str) -> None:
+            print_hints(stdout, stderr)
 
+        hints = not args.no_hints
         ensure_build_directory(args, ctx.info_name)
-        run_target('all', args, custom_error_handler=tool_error_handler)
+        run_target('all', args, force_progression=GENERATORS[args.generator].get('force_progression', False),
+                   custom_error_handler=tool_error_handler, hints=hints)
         run_target(target_name, args)
 
     def list_build_system_targets(target_name: str, ctx: Context, args: PropertyDict) -> None:
@@ -61,6 +64,7 @@ def action_extensions(base_actions: Dict, project_path: str) -> Any:
             # This encoding step is required only in Python 2.
             style = style.encode(sys.getfilesystemencoding() or 'utf-8')
         os.environ['MENUCONFIG_STYLE'] = style
+        args.no_hints = True
         build_target(target_name, ctx, args)
 
     def fallback_target(target_name: str, ctx: Context, args: PropertyDict) -> None:
@@ -304,6 +308,12 @@ def action_extensions(base_actions: Dict, project_path: str) -> Any:
                 'hidden': True,
                 'default': False,
             },
+            {
+                'names': ['--no-hints'],
+                'help': 'Disable hints on how to resolve errors and logging.',
+                'is_flag': True,
+                'default': False
+            }
         ],
         'global_action_callbacks': [validate_root_options],
     }

tools/idf_py_actions/hints.yml

@@ -0,0 +1,29 @@
+#  -
+#     re: Regular expression of error to search
+#     hint: Message of the hint. Optionally, it is possible to use '{}' at the place where the matched group from 're' should be inserted. This requires 'match_to_output: True'.
+#     match_to_output: (False by default) see the description of 'hint'.
+
+
+# Rules to write regex for hints on how to resolve errors
+# - Do not use more than one whitespace in a row. The script automatically merges several whitespaces into one when capturing output
+# - Do not use \n in your regex. They are all automatically deletes by the script when capturing output
+-
+    re: "error: implicit declaration of function '(\\w+)'"
+    hint: "Maybe you forgot to import {} library(s) in header file or add the necessary REQURIES component. Try to add missing libraries to your project header file or check idf_component_register(REQUIRES ...) section in your component CmakeList.txt file. For more information run 'idf.py docs -sp api-guides/build-system.html'."
+    match_to_output: True
+
+-
+    re: "The CMAKE_[A-Z]+_COMPILER: [\\w+-]+ is not a full path and was not found in the PATH\\."
+    hint: "Try to reinstall the toolchain for the chip that you trying to use. \nFor more information run 'idf.py docs -sp get-started/#installation' and follow the instructions for your system"
+
+-
+    re: "CMake Error: The current CMakeCache\\.txt directory .* is different than the directory .* where CMakeCache\\.txt was created\\."
+    hint: "Run 'idf.py fullclean' and try the build again."
+
+-
+    re: "CMake Error at .* \\(message\\): Could not create symbolic link for: error\\.c --> Cannot create a file when that file already exists\\."
+    hint: "Run 'idf.py fullclean' and try the build again."
+
+-
+    re: "ImportError: bad magic number in 'kconfiglib':"
+    hint: "Run 'idf.py python-clean', and try again"

tools/idf_py_actions/serial_ext.py

@@ -10,7 +10,7 @@ import click
 from idf_monitor_base.output_helpers import yellow_print
 from idf_py_actions.errors import FatalError, NoSerialPortFoundError
 from idf_py_actions.global_options import global_options
-from idf_py_actions.tools import PropertyDict, ensure_build_directory, get_sdkconfig_value, run_target, run_tool
+from idf_py_actions.tools import PropertyDict, RunTool, ensure_build_directory, get_sdkconfig_value, run_target
 
 PYTHON = sys.executable
 
@@ -83,7 +83,7 @@ def action_extensions(base_actions: Dict, project_path: str) -> Dict:
         return result
 
     def monitor(action: str, ctx: click.core.Context, args: PropertyDict, print_filter: str, monitor_baud: str, encrypted: bool,
-                no_reset: bool, timestamps: bool, timestamp_format: str) -> None:
+                no_reset: bool, timestamps: bool, timestamp_format: str, force_color: bool) -> None:
         """
         Run idf_monitor.py to watch build output
         """
@@ -149,10 +149,14 @@ def action_extensions(base_actions: Dict, project_path: str) -> Dict:
         if timestamp_format:
             monitor_args += ['--timestamp-format', timestamp_format]
 
+        if force_color or os.name == 'nt':
+            monitor_args += ['--force-color']
+
         idf_py = [PYTHON] + _get_commandline_options(ctx)  # commands to re-run idf.py
         monitor_args += ['-m', ' '.join("'%s'" % a for a in idf_py)]
+        hints = not args.no_hints
 
-        run_tool('idf_monitor', monitor_args, args.project_dir)
+        RunTool('idf_monitor', monitor_args, args.project_dir, build_dir=args.build_dir, hints=hints)()
 
     def flash(action: str, ctx: click.core.Context, args: PropertyDict) -> None:
         """
@@ -171,7 +175,7 @@ def action_extensions(base_actions: Dict, project_path: str) -> Dict:
         ensure_build_directory(args, ctx.info_name)
         esptool_args = _get_esptool_args(args)
         esptool_args += ['erase_flash']
-        run_tool('esptool.py', esptool_args, args.build_dir)
+        RunTool('esptool.py', esptool_args, args.build_dir)()
 
     def global_callback(ctx: click.core.Context, global_args: Dict, tasks: PropertyDict) -> None:
         encryption = any([task.name in ('encrypted-flash', 'encrypted-app-flash') for task in tasks])
@@ -285,6 +289,10 @@ def action_extensions(base_actions: Dict, project_path: str) -> Dict:
                         'help': ('Set the formatting of timestamps compatible with strftime(). '
                                  'For example, "%Y-%m-%d %H:%M:%S".'),
                         'default': None
+                    }, {
+                        'names': ['--force-color'],
+                        'is_flag': True,
+                        'help': 'Always print ANSI for colors',
                     }
 
                 ],

tools/idf_py_actions/tools.py

@@ -1,19 +1,24 @@
 # SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
+import asyncio
 import os
 import re
 import subprocess
 import sys
+from asyncio.subprocess import Process
 from io import open
-from typing import Any, List
+from types import FunctionType
+from typing import Any, Dict, List, Optional, TextIO, Tuple
 
 import click
+import yaml
+from idf_monitor_base.output_helpers import yellow_print
 
 from .constants import GENERATORS
 from .errors import FatalError
 
 
-def executable_exists(args):
+def executable_exists(args: List) -> bool:
     try:
         subprocess.check_output(args)
         return True
@@ -22,7 +27,7 @@ def executable_exists(args):
         return False
 
 
-def realpath(path):
+def realpath(path: str) -> str:
     """
     Return the cannonical path with normalized case.
 
@@ -32,7 +37,7 @@ def realpath(path):
     return os.path.normcase(os.path.realpath(path))
 
 
-def _idf_version_from_cmake():
+def _idf_version_from_cmake() -> Optional[str]:
     version_path = os.path.join(os.environ['IDF_PATH'], 'tools/cmake/version.cmake')
     regex = re.compile(r'^\s*set\s*\(\s*IDF_VERSION_([A-Z]{5})\s+(\d+)')
     ver = {}
@@ -50,17 +55,17 @@ def _idf_version_from_cmake():
         return None
 
 
-def get_target(path, sdkconfig_filename='sdkconfig'):
+def get_target(path: str, sdkconfig_filename: str='sdkconfig') -> Optional[str]:
     path = os.path.join(path, sdkconfig_filename)
     return get_sdkconfig_value(path, 'CONFIG_IDF_TARGET')
 
 
-def idf_version():
+def idf_version() -> Optional[str]:
     """Print version of ESP-IDF"""
 
     #  Try to get version from git:
     try:
-        version = subprocess.check_output([
+        version: Optional[str] = subprocess.check_output([
             'git',
             '--git-dir=%s' % os.path.join(os.environ['IDF_PATH'], '.git'),
             '--work-tree=%s' % os.environ['IDF_PATH'],
@@ -74,56 +79,180 @@ def idf_version():
     return version
 
 
-def run_tool(tool_name, args, cwd, env=dict(), custom_error_handler=None):
-    def quote_arg(arg):
-        " Quote 'arg' if necessary "
-        if ' ' in arg and not (arg.startswith('"') or arg.startswith("'")):
-            return "'" + arg + "'"
-        return arg
+def print_hints(*filenames: str) -> None:
+    """Getting output files and printing hints on how to resolve errors based on the output."""
+    with open(os.path.join(os.path.dirname(__file__), 'hints.yml'), 'r') as file:
+        hints = yaml.safe_load(file)
+    for file_name in filenames:
+        with open(file_name, 'r') as file:
+            output = ' '.join(line.strip() for line in file if line.strip())
+        for hint in hints:
+            try:
+                match = re.compile(hint['re']).findall(output)
+            except KeyError:
+                raise KeyError("Argument 're' missing in {}. Check hints.yml file.".format(hint))
+            except re.error as e:
+                raise re.error('{} from hints.yml have {} problem. Check hints.yml file.'.format(hint['re'], e))
+            if match:
+                extra_info = ', '.join(match) if hint.get('match_to_output', '') else ''
+                try:
+                    yellow_print(' '.join(['HINT:', hint['hint'].format(extra_info)]))
+                except KeyError:
+                    raise KeyError("Argument 'hint' missing in {}. Check hints.yml file.".format(hint))
 
-    args = [str(arg) for arg in args]
-    display_args = ' '.join(quote_arg(arg) for arg in args)
-    print('Running %s in directory %s' % (tool_name, quote_arg(cwd)))
-    print('Executing "%s"...' % str(display_args))
 
-    env_copy = dict(os.environ)
-    env_copy.update(env)
+def fit_text_in_terminal(out: str) -> str:
+    """Fit text in terminal, if the string is not fit replace center with `...`"""
+    space_for_dots = 3  # Space for "..."
+    terminal_width, _ = os.get_terminal_size()
+    if terminal_width <= space_for_dots:
+        # if the wide of the terminal is too small just print dots
+        return '.' * terminal_width
+    if len(out) >= terminal_width:
+        elide_size = (terminal_width - space_for_dots) // 2
+        # cut out the middle part of the output if it does not fit in the terminal
+        return '...'.join([out[:elide_size], out[len(out) - elide_size:]])
+    return out
 
-    if sys.version_info[0] < 3:
-        # The subprocess lib cannot accept environment variables as "unicode". Convert to str.
-        # This encoding step is required only in Python 2.
-        for (key, val) in env_copy.items():
-            if not isinstance(val, str):
-                env_copy[key] = val.encode(sys.getfilesystemencoding() or 'utf-8')
 
-    try:
+class RunTool:
+    def __init__(self, tool_name: str, args: List, cwd: str, env: Dict=None, custom_error_handler: FunctionType=None, build_dir: str=None,
+                 hints: bool=False, force_progression: bool=False) -> None:
+        self.tool_name = tool_name
+        self.args = args
+        self.cwd = cwd
+        self.env = env
+        self.custom_error_handler = custom_error_handler
+        # build_dir sets by tools that do not use build directory as cwd
+        self.build_dir = build_dir or cwd
+        self.hints = hints
+        self.force_progression = force_progression
+
+    def __call__(self) -> None:
+        def quote_arg(arg: str) -> str:
+            """ Quote the `arg` with whitespace in them because it can cause problems when we call it from a subprocess."""
+            if re.match(r"^(?![\'\"]).*\s.*", arg):
+                return ''.join(["'", arg, "'"])
+            return arg
+
+        self.args = [str(arg) for arg in self.args]
+        display_args = ' '.join(quote_arg(arg) for arg in self.args)
+        print('Running %s in directory %s' % (self.tool_name, quote_arg(self.cwd)))
+        print('Executing "%s"...' % str(display_args))
+
+        env_copy = dict(os.environ)
+        env_copy.update(self.env or {})
+
+        process, stderr_output_file, stdout_output_file = asyncio.run(self.run_command(self.args, env_copy))
+        if process.returncode == 0:
+            return
+
+        if self.custom_error_handler:
+            self.custom_error_handler(process.returncode, stderr_output_file, stdout_output_file)
+            return
+
+        if stderr_output_file and stdout_output_file:
+            print_hints(stderr_output_file, stdout_output_file)
+            raise FatalError('{} failed with exit code {}, output of the command is in the {} and {}'.format(self.tool_name, process.returncode,
+                             stderr_output_file, stdout_output_file))
+
+        raise FatalError('{} failed with exit code {}'.format(self.tool_name, process.returncode))
+
+    async def run_command(self, cmd: List, env_copy: Dict) -> Tuple[Process, Optional[str], Optional[str]]:
+        """ Run the `cmd` command with capturing stderr and stdout from that function and return returncode
+        and of the command, the id of the process, paths to captured output """
+        if not self.hints:
+            p = await asyncio.create_subprocess_exec(*cmd, env=env_copy, cwd=self.cwd)
+            await p.wait()  # added for avoiding None returncode
+            return p, None, None
+        log_dir_name = 'log'
+        try:
+            os.mkdir(os.path.join(self.build_dir, log_dir_name))
+        except FileExistsError:
+            pass
         # Note: we explicitly pass in os.environ here, as we may have set IDF_PATH there during startup
-        subprocess.check_call(args, env=env_copy, cwd=cwd)
-    except subprocess.CalledProcessError as e:
-        if custom_error_handler:
-            custom_error_handler(e)
-        else:
-            raise FatalError('%s failed with exit code %d' % (tool_name, e.returncode))
+        # limit was added for avoiding error in idf.py confserver
+        p = await asyncio.create_subprocess_exec(*cmd, env=env_copy, limit=1024 * 128, cwd=self.cwd, stdout=asyncio.subprocess.PIPE,
+                                                 stderr=asyncio.subprocess.PIPE, )
+        stderr_output_file = os.path.join(self.build_dir, log_dir_name, f'idf_py_stderr_output_{p.pid}')
+        stdout_output_file = os.path.join(self.build_dir, log_dir_name, f'idf_py_stdout_output_{p.pid}')
+        if p.stderr and p.stdout:  # it only to avoid None type in p.std
+            await asyncio.gather(
+                self.read_and_write_stream(p.stderr, stderr_output_file, sys.stderr),
+                self.read_and_write_stream(p.stdout, stdout_output_file))
+        await p.wait()  # added for avoiding None returncode
+        return p, stderr_output_file, stdout_output_file
+
+    async def read_and_write_stream(self, input_stream: asyncio.StreamReader, output_filename: str,
+                                    output_stream: TextIO=sys.stdout) -> None:
+        """read the output of the `input_stream` and then write it into `output_filename` and `output_stream`"""
+        def delete_ansi_escape(text: str) -> str:
+            ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
+            return ansi_escape.sub('', text)
+
+        def prepare_for_print(out: bytes) -> str:
+            # errors='ignore' is here because some chips produce some garbage bytes
+            result = out.decode(errors='ignore')
+            if not output_stream.isatty():
+                # delete escape sequence if we printing in environments where ANSI coloring is disabled
+                return delete_ansi_escape(result)
+            return result
+
+        def print_progression(output: str) -> None:
+            # Print a new line on top of the previous line
+            sys.stdout.write('\x1b[K')
+            print('\r', end='')
+            print(fit_text_in_terminal(output.strip('\n\r')), end='', file=output_stream)
+
+        try:
+            with open(output_filename, 'w') as output_file:
+                while True:
+                    out = await input_stream.readline()
+                    if not out:
+                        break
+                    output = prepare_for_print(out)
+                    output_file.write(output)
+
+                    # print output in progression way but only the progression related (that started with '[') and if verbose flag is not set
+                    if self.force_progression and output[0] == '[' and '-v' not in self.args:
+                        print_progression(output)
+                    else:
+                        print(output, end='', file=output_stream)
+        except (RuntimeError, EnvironmentError) as e:
+            yellow_print('WARNING: The exception {} was raised and we can\'t capture all your {} and '
+                         'hints on how to resolve errors can be not accurate.'.format(e, output_stream.name.strip('<>')))
 
 
-def run_target(target_name, args, env=dict(), custom_error_handler=None):
+def run_tool(*args: Any, **kwargs: Any) -> None:
+    # Added in case some one use run_tool externally in a idf.py extensions
+    return RunTool(*args, **kwargs)()
+
+
+def run_target(target_name: str, args: 'PropertyDict', env: Optional[Dict]=None,
+               custom_error_handler: FunctionType=None, force_progression: bool=False, hints: bool=False) -> None:
+    """Run target in build directory."""
+    if env is None:
+        env = {}
+
     generator_cmd = GENERATORS[args.generator]['command']
+    env.update(GENERATORS[args.generator]['envvar'])
 
     if args.verbose:
         generator_cmd += [GENERATORS[args.generator]['verbose_flag']]
 
-    run_tool(generator_cmd[0], generator_cmd + [target_name], args.build_dir, env, custom_error_handler)
+    RunTool(generator_cmd[0], generator_cmd + [target_name], args.build_dir, env, custom_error_handler, hints=hints,
+            force_progression=force_progression)()
 
 
-def _strip_quotes(value, regexp=re.compile(r"^\"(.*)\"$|^'(.*)'$|^(.*)$")):
+def _strip_quotes(value: str, regexp: re.Pattern=re.compile(r"^\"(.*)\"$|^'(.*)'$|^(.*)$")) -> Optional[str]:
     """
     Strip quotes like CMake does during parsing cache entries
     """
-
-    return [x for x in regexp.match(value).groups() if x is not None][0].rstrip()
+    matching_values = regexp.match(value)
+    return [x for x in matching_values.groups() if x is not None][0].rstrip() if matching_values is not None else None
 
 
-def _parse_cmakecache(path):
+def _parse_cmakecache(path: str) -> Dict:
     """
     Parse the CMakeCache file at 'path'.
 
@@ -142,7 +271,7 @@ def _parse_cmakecache(path):
     return result
 
 
-def _new_cmakecache_entries(cache_path, new_cache_entries):
+def _new_cmakecache_entries(cache_path: str, new_cache_entries: List) -> bool:
     if not os.path.exists(cache_path):
         return True
 
@@ -158,7 +287,7 @@ def _new_cmakecache_entries(cache_path, new_cache_entries):
     return False
 
 
-def _detect_cmake_generator(prog_name):
+def _detect_cmake_generator(prog_name: str) -> Any:
     """
     Find the default cmake generator, if none was specified. Raises an exception if no valid generator is found.
     """
@@ -168,7 +297,7 @@ def _detect_cmake_generator(prog_name):
     raise FatalError("To use %s, either the 'ninja' or 'GNU make' build tool must be available in the PATH" % prog_name)
 
 
-def ensure_build_directory(args, prog_name, always_run_cmake=False):
+def ensure_build_directory(args: 'PropertyDict', prog_name: str, always_run_cmake: bool=False) -> None:
     """Check the build directory exists and that cmake has been run there.
 
     If this isn't the case, create the build directory (if necessary) and
@@ -220,7 +349,8 @@ def ensure_build_directory(args, prog_name, always_run_cmake=False):
                 cmake_args += ['-D' + d for d in args.define_cache_entry]
             cmake_args += [project_dir]
 
-            run_tool('cmake', cmake_args, cwd=args.build_dir)
+            hints = not args.no_hints
+            RunTool('cmake', cmake_args, cwd=args.build_dir, hints=hints)()
         except Exception:
             # don't allow partially valid CMakeCache.txt files,
             # to keep the "should I run cmake?" logic simple
@@ -251,8 +381,8 @@ def ensure_build_directory(args, prog_name, always_run_cmake=False):
         pass  # if cmake failed part way, CMAKE_HOME_DIRECTORY may not be set yet
 
 
-def merge_action_lists(*action_lists):
-    merged_actions = {
+def merge_action_lists(*action_lists: Dict) -> Dict:
+    merged_actions: Dict = {
         'global_options': [],
         'actions': {},
         'global_action_callbacks': [],
@@ -264,7 +394,7 @@ def merge_action_lists(*action_lists):
     return merged_actions
 
 
-def get_sdkconfig_value(sdkconfig_file, key):
+def get_sdkconfig_value(sdkconfig_file: str, key: str) -> Optional[str]:
     """
     Return the value of given key from sdkconfig_file.
     If sdkconfig_file does not exist or the option is not present, returns None.
@@ -284,14 +414,14 @@ def get_sdkconfig_value(sdkconfig_file, key):
     return value
 
 
-def is_target_supported(project_path, supported_targets):
+def is_target_supported(project_path: str, supported_targets: List) -> bool:
     """
     Returns True if the active target is supported, or False otherwise.
     """
     return get_target(project_path) in supported_targets
 
 
-def _guess_or_check_idf_target(args, prog_name, cache):
+def _guess_or_check_idf_target(args: 'PropertyDict', prog_name: str, cache: Dict) -> None:
     """
     If CMakeCache.txt doesn't exist, and IDF_TARGET is not set in the environment, guess the value from
     sdkconfig or sdkconfig.defaults, and pass it to CMake in IDF_TARGET variable.