From 88527faff88edad0679f2adcc87afb1a66b6a296 Mon Sep 17 00:00:00 2001 From: Marek Fiala Date: Fri, 21 Jun 2024 17:13:18 +0200 Subject: [PATCH] feat(tools): Added Windows shells support + refactoring --- .gitlab/ci/rules.yml | 7 + activate_venv.py | 466 ---------------------- docs/en/api-guides/tools/idf-tools.rst | 14 +- docs/zh_CN/api-guides/tools/idf-tools.rst | 14 +- export.bat | 96 +---- export.fish | 17 +- export.ps1 | 96 +---- export.sh | 6 +- activate.py => tools/activate.py | 6 +- tools/ci/executable-list.txt | 2 +- tools/export_utils/activate_venv.py | 177 ++++++++ tools/export_utils/console_output.py | 69 ++++ tools/export_utils/shell_types.py | 316 +++++++++++++++ tools/export_utils/utils.py | 48 +++ 14 files changed, 694 insertions(+), 640 deletions(-) delete mode 100644 activate_venv.py rename activate.py => tools/activate.py (85%) create mode 100644 tools/export_utils/activate_venv.py create mode 100644 tools/export_utils/console_output.py create mode 100644 tools/export_utils/shell_types.py create mode 100644 tools/export_utils/utils.py diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 3685149e7f..d16fbbc9c3 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -83,6 +83,8 @@ - "tools/idf_monitor.py" + - "tools/activate.py" + - "tools/idf.py" - "tools/idf_py_actions/**/*" - "tools/test_idf_py/**/*" @@ -96,6 +98,11 @@ - "tools/test_idf_tools/**/*" - "tools/install_util.py" + - "tools/export_utils/utils.py" + - "tools/export_utils/shell_types.py" + - "tools/export_utils/console_output.py" + - "tools/export_utils/activate_venv.py" + - "tools/requirements/*" - "tools/requirements.json" - "tools/requirements_schema.json" diff --git a/activate_venv.py b/activate_venv.py deleted file mode 100644 index 646f798428..0000000000 --- a/activate_venv.py +++ /dev/null @@ -1,466 +0,0 @@ -# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import argparse -import os -import shutil -import sys -from pathlib import Path -from subprocess import run -from subprocess import SubprocessError -from tempfile import gettempdir -from tempfile import NamedTemporaryFile -from tempfile import TemporaryDirectory -from textwrap import dedent -from typing import Any -from typing import Callable -from typing import Dict -from typing import List -from typing import Optional -from typing import TextIO - -try: - # The ESP-IDF virtual environment hasn't been verified yet, so see if the rich library - # can be imported to display error and status messages nicely. - from rich.console import Console -except ImportError as e: - sys.exit(f'error: Unable to import the rich module: {e}. Please execute the install script.') - - -def status_message(msg: str, rv_on_ok: bool=False, die_on_err: bool=True) -> Callable: - def inner(func: Callable) -> Callable: - def wrapper(*args: Any, **kwargs: Any) -> Any: - eprint(f'[dark_orange]*[/dark_orange] {msg} ... ', end='') - - try: - rv = func(*args, **kwargs) - except Exception as e: - eprint('[red]FAILED[/red]') - if ARGS.debug: - raise - if not die_on_err: - return None - die(str(e)) - - if rv_on_ok: - eprint(f'[green]{rv}[/green]') - else: - eprint('[green]OK[/green]') - - return rv - return wrapper - return inner - - -class Shell(): - def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: Dict[str,str]): - self.shell = shell - self.deactivate_cmd = deactivate_cmd - self.new_esp_idf_env = new_esp_idf_env - - # TODO We are not removing the temporary activation scripts. - self.tmp_dir_path = Path(gettempdir()) / 'esp_idf_activate' - self.tmp_dir_path.mkdir(parents=True, exist_ok=True) - with NamedTemporaryFile(dir=self.tmp_dir_path, delete=False, prefix='activate_') as fd: - self.script_file_path = Path(fd.name) - debug(f'Temporary script file path: {self.script_file_path}') - - def expanded_env(self) -> Dict[str, str]: - expanded_env = self.new_esp_idf_env.copy() - - if 'PATH' not in expanded_env: - return expanded_env - - # The PATH returned by idf_tools.py export is not expanded. - # Note that for the export script, the PATH should remain unexpanded - # to ensure proper deactivation. In the export script, - # the expansion should occur after deactivation, when the PATH is adjusted. - # But it has to be expanded for processes started with the new PATH. - expanded_env['PATH'] = os.path.expandvars(expanded_env['PATH']) - return expanded_env - - def spawn(self) -> None: - # This method should likely work for all shells because we are delegating the initialization - # purely to Python os.environ. - new_env = os.environ.copy() - new_env.update(self.expanded_env()) - run([self.shell], env=new_env) - - -class UnixShell(Shell): - def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: Dict[str,str]): - super().__init__(shell, deactivate_cmd, new_esp_idf_env) - self.new_esp_idf_env['IDF_TOOLS_INSTALL_CMD'] = os.path.join(IDF_PATH, 'install.sh') - self.new_esp_idf_env['IDF_TOOLS_EXPORT_CMD'] = os.path.join(IDF_PATH, 'export.sh') - - def export_file(self, fd: TextIO) -> None: - fd.write(f'{self.deactivate_cmd}\n') - for var, value in self.new_esp_idf_env.items(): - fd.write(f'export {var}="{value}"\n') - prompt = self.get_prompt() - fd.write(f'{prompt}\n') - - def get_prompt(self) -> str: - return f'PS1="(ESP-IDF {IDF_VERSION}) $PS1"' - - def export(self) -> None: - with open(self.script_file_path, 'w') as fd: - self.export_file(fd) - fd.write((f'echo "\nDone! You can now compile ESP-IDF projects.\n' - 'Go to the project directory and run:\n\n idf.py build\n"')) - - print(f'. {self.script_file_path}') - - def click_ver(self) -> int: - return int(click.__version__.split('.')[0]) - - -class BashShell(UnixShell): - def get_bash_major(self) -> int: - env = self.expanded_env() - stdout = run_cmd(['bash', '-c', 'echo ${BASH_VERSINFO[0]}'], env=env) - bash_maj = int(stdout) - return bash_maj - - @status_message('Shell completion', die_on_err=False) - def autocompletion(self) -> str: - bash_maj = self.get_bash_major() - if bash_maj < 4: - raise RuntimeError('Autocompletion not supported') - - env = self.expanded_env() - env['LANG'] = 'en' - env['_IDF.PY_COMPLETE'] = 'bash_source' if self.click_ver() >= 8 else 'source_bash' - stdout = run_cmd([IDF_PY], env=env) - return stdout - - def export_file(self, fd: TextIO) -> None: - super().export_file(fd) - stdout = self.autocompletion() - if stdout is not None: - fd.write(f'{stdout}\n') - - def init_file(self) -> None: - with open(self.script_file_path, 'w') as fd: - # We will use the --init-file option to pass a custom rc file, which will ignore .bashrc, - # so we need to source .bashrc first. - fd.write(f'source ~/.bashrc\n') - - stdout = self.autocompletion() - if stdout is not None: - fd.write(f'{stdout}\n') - - prompt = self.get_prompt() - fd.write(f'{prompt}\n') - - def spawn(self) -> None: - self.init_file() - new_env = os.environ.copy() - new_env.update(self.expanded_env()) - run([self.shell, '--init-file', str(self.script_file_path)], env=new_env) - - -class ZshShell(UnixShell): - @status_message('Shell completion', die_on_err=False) - def autocompletion(self) -> str: - env = self.expanded_env() - env['LANG'] = 'en' - env['_IDF.PY_COMPLETE'] = 'zsh_source' if self.click_ver() >= 8 else 'source_zsh' - stdout = run_cmd([IDF_PY], env=env) - return f'autoload -Uz compinit && compinit -u\n{stdout}' - - def export_file(self, fd: TextIO) -> None: - super().export_file(fd) - stdout = self.autocompletion() - # Add autocompletion - if stdout is not None: - fd.write(f'{stdout}\n') - - def init_file(self) -> None: - # If ZDOTDIR is unset, HOME is used instead. - # https://zsh.sourceforge.io/Doc/Release/Files.html#Startup_002fShutdown-Files - zdotdir = os.environ.get('ZDOTDIR', str(Path.home())) - with open(self.script_file_path, 'w') as fd: - # We will use the ZDOTDIR env variable to load our custom script in the newly spawned shell - # so we need to source .zshrc first. - zshrc_path = Path(zdotdir) / '.zshrc' - if zshrc_path.is_file(): - fd.write(f'source {zshrc_path}\n') - - # Add autocompletion - stdout = self.autocompletion() - if stdout is not None: - fd.write(f'{stdout}\n') - - prompt = self.get_prompt() - fd.write(f'{prompt}\n') - - # TODO This might not be needed, or consider resetting it to the original value - fd.write('unset ZDOTDIR\n') - - def spawn(self) -> None: - self.init_file() - - # Create a temporary directory to use as ZDOTDIR - tmpdir = TemporaryDirectory() - tmpdir_path = Path(tmpdir.name) - debug(f'Temporary ZDOTDIR {tmpdir_path} with .zshrc file') - - # Copy init script to the custom ZDOTDIR - zshrc_path = tmpdir_path / '.zshrc' - shutil.copy(str(self.script_file_path), str(zshrc_path)) - - new_env = os.environ.copy() - new_env.update(self.expanded_env()) - # Set new ZDOTDIR in the new environment - new_env['ZDOTDIR'] = str(tmpdir_path) - - run([self.shell], env=new_env) - - -class FishShell(UnixShell): - def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: Dict[str,str]): - super().__init__(shell, deactivate_cmd, new_esp_idf_env) - self.new_esp_idf_env['IDF_TOOLS_INSTALL_CMD'] = os.path.join(IDF_PATH, 'install.fish') - self.new_esp_idf_env['IDF_TOOLS_EXPORT_CMD'] = os.path.join(IDF_PATH, 'export.fish') - - @status_message('Shell completion', die_on_err=False) - def autocompletion(self) -> str: - env = self.expanded_env() - env['LANG'] = 'en' - env['_IDF.PY_COMPLETE'] = 'fish_source' if self.click_ver() >= 8 else 'source_fish' - stdout = run_cmd([IDF_PY], env=env) - return stdout - - def get_prompt(self) -> str: - prompt = dedent(f''' - functions -c fish_prompt _old_fish_prompt - function fish_prompt - printf "(ESP-IDF {IDF_VERSION}) " - _old_fish_prompt - end - ''') - return prompt - - def export_file(self, fd: TextIO) -> None: - fd.write(f'{self.deactivate_cmd}\n') - for var, value in self.new_esp_idf_env.items(): - fd.write(f'export {var}="{value}"\n') - - # Add autocompletion - stdout = self.autocompletion() - if stdout is not None: - fd.write(f'{stdout}\n') - - # Adjust fish prompt - prompt = self.get_prompt() - fd.write(f'{prompt}\n') - - def init_file(self) -> None: - with open(self.script_file_path, 'w') as fd: - # Add autocompletion - stdout = self.autocompletion() - if stdout is not None: - fd.write(f'{stdout}\n') - # Adjust fish prompt - prompt = self.get_prompt() - fd.write(f'{prompt}\n') - - def spawn(self) -> None: - self.init_file() - new_env = os.environ.copy() - new_env.update(self.expanded_env()) - run([self.shell, f'--init-command=source {self.script_file_path}'], env=new_env) - - -SHELL_CLASSES = { - 'bash': BashShell, - 'zsh': ZshShell, - 'fish': FishShell, - 'sh': UnixShell, - 'ksh': UnixShell, - 'dash': UnixShell, - 'nu': UnixShell, -} - -SUPPORTED_SHELLS = ' '.join(SHELL_CLASSES.keys()) - -CONSOLE_STDERR = None -CONSOLE_STDOUT = None - - -def err(*args: Any, **kwargs: Any) -> None: - CONSOLE_STDERR.print('[red]error[/red]: ', *args, **kwargs) # type: ignore - - -def warn(*args: Any, **kwargs: Any) -> None: - CONSOLE_STDERR.print('[yellow]warning[/yellow]: ', *args, **kwargs) # type: ignore - - -def debug(*args: Any, **kwargs: Any) -> None: - if not ARGS.debug: - return - CONSOLE_STDERR.print('[green_yellow]debug[/green_yellow]: ', *args, **kwargs) # type: ignore - - -def die(*args: Any, **kwargs: Any) -> None: - err(*args, **kwargs) - sys.exit(1) - - -def eprint(*args: Any, **kwargs: Any) -> None: - CONSOLE_STDERR.print(*args, **kwargs) # type: ignore - - -def oprint(*args: Any, **kwargs: Any) -> None: - CONSOLE_STDOUT.print(*args, **kwargs) # type: ignore - - -def run_cmd(cmd: List[str], env: Optional[Dict[str, Any]]=None) -> str: - new_env = os.environ.copy() - if env is not None: - new_env.update(env) - - cmd_str = '"{}"'.format(' '.join(cmd)) - try: - p = run(cmd, env=new_env, text=True, capture_output=True) - except (OSError, SubprocessError) as e: - raise RuntimeError(f'Command {cmd_str} failed: {e}') - - stdout = p.stdout.strip() - stderr = p.stderr.strip() - if p.returncode: - raise RuntimeError(f'Command {cmd_str} failed with error code {p.returncode}\n{stdout}\n{stderr}') - - return stdout - - -parser = argparse.ArgumentParser(prog='activate', - description='Activate ESP-IDF environment') -parser.add_argument('-s', '--shell', - metavar='SHELL', - default=os.environ.get('ESP_IDF_SHELL', None), - help='Explicitly specify shell to start. For example bash, zsh, powershell.exe, cmd.exe') -parser.add_argument('-l', '--list', - action='store_true', - help=('List supported shells.')) -parser.add_argument('-e', '--export', - action='store_true', - help=('Generate commands to run in the terminal.')) -parser.add_argument('-n', '--no-color', - action='store_true', - help=('Disable ANSI color escape sequences.')) -parser.add_argument('-d', '--debug', - action='store_true', - help=('Enable debug information.')) -parser.add_argument('-q', '--quiet', - action='store_true', - help=('Suppress all output.')) - -ARGS = parser.parse_args() - -CONSOLE_STDERR = Console(stderr=True, quiet=ARGS.quiet, no_color=ARGS.no_color) -CONSOLE_STDOUT = Console(quiet=ARGS.quiet, no_color=ARGS.no_color) - -if ARGS.list: - oprint(SUPPORTED_SHELLS) - sys.exit() - -# The activate.py script sets the following environment variables -IDF_PATH = os.environ['IDF_PATH'] -IDF_VERSION = os.environ['ESP_IDF_VERSION'] -IDF_PYTHON_ENV_PATH = os.environ['IDF_PYTHON_ENV_PATH'] -IDF_TOOLS_PY = os.path.join(IDF_PATH, 'tools', 'idf_tools.py') -IDF_PY = os.path.join(IDF_PATH, 'tools', 'idf.py') - -eprint(f'[dark_orange]Activating ESP-IDF {IDF_VERSION}') -debug(f'IDF_PATH {IDF_PATH}') -debug(f'IDF_PYTHON_ENV_PATH {IDF_PYTHON_ENV_PATH}') - - -@status_message('Checking python version', rv_on_ok=True) -def check_python_version() -> str: - # Check the Python version within a virtual environment - python_version_checker = os.path.join(IDF_PATH, 'tools', 'python_version_checker.py') - run_cmd([sys.executable, python_version_checker]) - ver = sys.version_info[:3] - return f'{ver[0]}.{ver[1]}.{ver[2]}' - - -@status_message('Checking python dependencies') -def check_python_dependencies() -> None: - # Check Python dependencies within the virtual environment - run_cmd([sys.executable, IDF_TOOLS_PY, 'check-python-dependencies']) - - -check_python_version() -check_python_dependencies() -# TODO Report installed tools that are not currently used by active ESP-IDF version - -# From this point forward, we are functioning within a fully validated ESP-IDF environment. - -# TODO Verify the architectures supported by psutils. We might need to create a wheel for it or -# substitute it with ps and tasklist commands. -import psutil # noqa: E402 -import click # noqa: E402 - - -@status_message('Deactivating the current ESP-IDF environment') -def get_deactivate_cmd() -> str: - # Get previous ESP-IDF system environment variables - cmd = [sys.executable, IDF_TOOLS_PY, 'export', '--deactivate'] - stdout = run_cmd(cmd) - return stdout - - -@status_message('Establishing a new ESP-IDF environment') -def get_idf_env() -> Dict[str,str]: - # Get ESP-IDF system environment variables - extra_paths_list = [os.path.join('components', 'espcoredump'), - os.path.join('components', 'partition_table'), - os.path.join('components', 'app_update')] - extra_paths = ':'.join([os.path.join(IDF_PATH, path) for path in extra_paths_list]) - cmd = [sys.executable, IDF_TOOLS_PY, 'export', '--format', 'key-value', '--add_paths_extras', extra_paths] - stdout = run_cmd(cmd) - - # idf_tools.py might not export certain environment variables if they are already set - idf_env: Dict[str, Any] = { - 'IDF_PATH': os.environ['IDF_PATH'], - 'ESP_IDF_VERSION': os.environ['ESP_IDF_VERSION'], - 'IDF_PYTHON_ENV_PATH': os.environ['IDF_PYTHON_ENV_PATH'], - } - - for line in stdout.splitlines(): - var, val = line.split('=') - idf_env[var] = val - - if 'PATH' in idf_env: - idf_env['PATH'] = ':'.join([extra_paths, idf_env['PATH']]) - - return idf_env - - -@status_message('Identifying shell', rv_on_ok=True) -def detect_shell() -> str: - if ARGS.shell is not None: - return str(ARGS.shell) - ppid = psutil.Process(os.getpid()).ppid() - # Look for grandparent, because we started from activate.py. - pppid = psutil.Process(ppid).ppid() - return str(psutil.Process(pppid).name()) - - -deactivate_cmd = get_deactivate_cmd() -new_esp_idf_env = get_idf_env() -detected_shell = detect_shell() - -if detected_shell not in SHELL_CLASSES: - die(f'"{detected_shell}" shell is not among the supported options: "{SUPPORTED_SHELLS}"') - -shell = SHELL_CLASSES[detected_shell](detected_shell, deactivate_cmd, new_esp_idf_env) - -if ARGS.export: - shell.export() - sys.exit() - -shell.spawn() -eprint(f'[dark_orange]ESP-IDF environment exited.') diff --git a/docs/en/api-guides/tools/idf-tools.rst b/docs/en/api-guides/tools/idf-tools.rst index 5d30f70316..425f0abb09 100644 --- a/docs/en/api-guides/tools/idf-tools.rst +++ b/docs/en/api-guides/tools/idf-tools.rst @@ -186,7 +186,19 @@ Since the installed tools are not permanently added to the user or system ``PATH ``export.sh`` may be used with shells other than Bash (such as zsh). However, in this case, it is required to set the ``IDF_PATH`` environment variable before running the script. When used in Bash, the script guesses the ``IDF_PATH`` value from its own location. -In addition to calling ``idf_tools.py``, these scripts list the directories that have been added to the ``PATH``. +activate.py +~~~~~~~~~~~ + +The environment setup is handled by the underlying ``tools/activate.py`` Python script. This script performs all necessary preparations and checks, generating a temporary file that is subsequently sourced by the export script. + +``activate.py`` can also function as a standalone command. When run, it launches a new child shell with an ESP-IDF environment, which can be utilized and then exited with the ``exit`` command. Upon exiting the child shell, you will return to the parent shell from which the script was initially executed. + +Additionally, the specific behavior of the ``activate.py`` script can be modified with various options, such as spawning a specific shell with ESP-IDF using the ``--shell`` option. For more information on available options, use the ``activate.py --help`` command. + +.. note:: + + When using ``activate.py`` on Windows, it should be executed with ``python activate.py``. This ensures the script runs in the current terminal window rather than launching a new one that closes immediately. + Other Installation Methods -------------------------- diff --git a/docs/zh_CN/api-guides/tools/idf-tools.rst b/docs/zh_CN/api-guides/tools/idf-tools.rst index 7cbb700966..f614ccae00 100644 --- a/docs/zh_CN/api-guides/tools/idf-tools.rst +++ b/docs/zh_CN/api-guides/tools/idf-tools.rst @@ -186,7 +186,19 @@ ESP-IDF 的根目录中提供了针对不同 shell 的用户安装脚本，包 ``export.sh`` 可以在除了 Bash 外的其他 shell（如 zsh）中使用。但在这种情况下，必须在运行脚本前设置 ``IDF_PATH`` 环境变量。在 Bash 中使用时，脚本会从当前目录猜测 ``IDF_PATH`` 的值。 -除了调用 ``idf_tools.py``，这些脚本还会列出已经添加到 ``PATH`` 的目录。 +activate.py +~~~~~~~~~~~ + +环境设置由底层的 ``tools/activate.py`` 脚本处理。该脚本用于执行所有必要的准备和检查，并生成一个临时文件，之后供导出脚本使用。 + +``activate.py`` 也可以作为独立命令运行。执行该脚本时，会启动一个新的子 shell 并加载 ESP-IDF 环境。使用 ``exit`` 命令可以退出子 shell，并退回至最初执行该脚本的父 shell。 + +此外，``activate.py`` 脚本的具体行为可以通过各种选项进行修改，例如使用 ``--shell`` 选项可以生成特定的 ESP-IDF shell。若想了解更多有关可用选项的详细信息，请使用 ``activate.py --help`` 命令。 + +.. note:: + + 在 Windows 系统中使用 ``activate.py`` 脚本时，应执行 ``python activate.py`` 命令。这可以确保脚本在当前终端窗口中运行，而不是启动一个立即关闭的新窗口。 + 其他安装方法 -------------------------- diff --git a/export.bat b/export.bat index c37c6060d0..e9cf9df0e1 100644 --- a/export.bat +++ b/export.bat @@ -25,73 +25,25 @@ if not "%MISSING_REQUIREMENTS%" == "" goto :__error_missing_requirements set IDF_PATH=%~dp0 set IDF_PATH=%IDF_PATH:~0,-1% -echo Checking Python compatibility -python.exe "%IDF_PATH%\tools\python_version_checker.py" - -set "IDF_TOOLS_PY_PATH=%IDF_PATH%\tools\idf_tools.py" -set "IDF_TOOLS_JSON_PATH=%IDF_PATH%\tools\tools.json" -set "IDF_TOOLS_EXPORT_CMD=%IDF_PATH%\export.bat" -set "IDF_TOOLS_INSTALL_CMD=%IDF_PATH%\install.bat" -echo Setting IDF_PATH: %IDF_PATH% -echo. - -set "OLD_PATH=%PATH%" -echo Adding ESP-IDF tools to PATH... -:: Export tool paths and environment variables. -:: It is possible to do this without a temporary file (running idf_tools.py from for /r command), -:: but that way it is impossible to get the exit code of idf_tools.py. -set "IDF_TOOLS_EXPORTS_FILE=%TEMP%\idf_export_vars.tmp" -python.exe "%IDF_PATH%\tools\idf_tools.py" export --format key-value >"%IDF_TOOLS_EXPORTS_FILE%" -if %errorlevel% neq 0 ( - set SCRIPT_EXIT_CODE=%errorlevel% - goto :__end +if not exist "%IDF_PATH%\tools\idf.py" ( + set SCRIPT_EXIT_CODE=1 + goto :__missing_file +) +if not exist "%IDF_PATH%\tools\idf_tools.py" ( + set SCRIPT_EXIT_CODE=1 + goto :__missing_file +) +if not exist "%IDF_PATH%\tools\activate.py" ( + set SCRIPT_EXIT_CODE=1 + goto :__missing_file ) -for /f "usebackq tokens=1,2 eol=# delims==" %%a in ("%IDF_TOOLS_EXPORTS_FILE%") do ( - call set "%%a=%%b" - ) -:: This removes OLD_PATH substring from PATH, leaving only the paths which have been added, -:: and prints semicolon-delimited components of the path on separate lines -call set PATH_ADDITIONS=%%PATH:%OLD_PATH%=%% -if "%PATH_ADDITIONS%"=="" call :__print_nothing_added -if not "%PATH_ADDITIONS%"=="" echo %PATH_ADDITIONS:;=&echo. % - -DOSKEY idf.py=python.exe "%IDF_PATH%\tools\idf.py" $* -DOSKEY esptool.py=python.exe "%IDF_PATH%\components\esptool_py\esptool\esptool.py" $* -DOSKEY espefuse.py=python.exe "%IDF_PATH%\components\esptool_py\esptool\espefuse.py" $* -DOSKEY espsecure.py=python.exe "%IDF_PATH%\components\esptool_py\esptool\espsecure.py" $* -DOSKEY otatool.py=python.exe "%IDF_PATH%\components\app_update\otatool.py" $* -DOSKEY parttool.py=python.exe "%IDF_PATH%\components\partition_table\parttool.py" $* - -echo Checking if Python packages are up to date... -python.exe "%IDF_PATH%\tools\idf_tools.py" check-python-dependencies -if %errorlevel% neq 0 ( - set SCRIPT_EXIT_CODE=%errorlevel% - goto :__end -) - -python.exe "%IDF_PATH%\tools\idf_tools.py" uninstall --dry-run > UNINSTALL_OUTPUT -SET /p UNINSTALL=nul 2>nul -) -set IDF_TOOLS_EXPORTS_FILE= -set IDF_TOOLS_EXPORT_CMD= -set IDF_TOOLS_INSTALL_CMD= -set IDF_TOOLS_PY_PATH= -set IDF_TOOLS_JSON_PATH= -set OLD_PATH= -set PATH_ADDITIONS= set MISSING_REQUIREMENTS= -set UNINSTALL= +set activate= exit /b %SCRIPT_EXIT_CODE% diff --git a/export.fish b/export.fish index cdff5798e0..283fa66471 100644 --- a/export.fish +++ b/export.fish @@ -5,5 +5,18 @@ function unset set --erase $argv end -set script_dir (dirname (realpath (status -f))) -eval ("$script_dir"/activate.py --export) +set idf_path (dirname (realpath (status -f))) + +if not test -f "$idf_path/tools/idf.py" + or not test -f "$idf_path/tools/idf_tools.py" + or not test -f "$idf_path/tools/activate.py" + echo "Could not detect IDF_PATH. Please set it before sourcing this script:" + echo " export IDF_PATH=(add path here)" + set -e idf_path + exit 1 +end + +source "$idf_path"/tools/detect_python.fish + +eval ("$idf_path"/tools/activate.py --export) +set -e idf_path diff --git a/export.ps1 b/export.ps1 index d519b2296d..76df9bd81b 100644 --- a/export.ps1 +++ b/export.ps1 @@ -1,92 +1,20 @@ #!/usr/bin/env pwsh -$S = [IO.Path]::PathSeparator # path separator. WIN:';', UNIX:":" -$IDF_PATH = "$PSScriptRoot" +$idf_path = "$PSScriptRoot" -Write-Output "Setting IDF_PATH: $IDF_PATH" -$env:IDF_PATH = "$IDF_PATH" +if (-not (Test-Path "$idf_path/tools/idf.py") -or + -not (Test-Path "$idf_path/tools/idf_tools.py") -or + -not (Test-Path "$idf_path/tools/activate.py")) { -Write-Output "Checking Python compatibility" -python "$IDF_PATH/tools/python_version_checker.py" + Write-Output "Could not detect IDF_PATH. Please set it before running this script:" + Write-Output ' $env:IDF_PATH=(add path here)' -Write-Output "Adding ESP-IDF tools to PATH..." -$OLD_PATH = $env:PATH.split($S) | Select-Object -Unique # array without duplicates -# using idf_tools.py to get $envars_array to set -$envars_raw = python "$IDF_PATH/tools/idf_tools.py" export --format key-value -if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # if error + $env:IDF_PATH = "" -$envars_array = @() # will be filled like: -# [ -# [vname1, vval1], [vname2, vval2], ... -# ] -foreach ($line in $envars_raw) { - $pair = $line.split("=") # split in name, val - $var_name = $pair[0].Trim() # trim spaces on the ends of the name - $var_val = $pair[1].Trim() # trim spaces on the ends of the val - $envars_array += (, ($var_name, $var_val)) + exit 1 } -if ($null -eq $IsWindows) { - # $IsWindows was added in PowerShell Core 6 and PowerShell 7 together with multi-platform support. # I.E. if this - # internal variable is not set then PowerShell 5 is used and # the platform cannot be # anything else than Windows. - $Windows = $true -} - -foreach ($pair in $envars_array) { - # setting the values - $var_name = $pair[0].Trim() # trim spaces on the ends of the name - $var_val = $pair[1].Trim() # trim spaces on the ends of the val - if ($var_name -eq "PATH") { - # trim "%PATH%" or "`$PATH" - if ($IsWindows -or $Windows) { - $var_val = $var_val.Trim($S + "%PATH%") - } else { - $var_val = $var_val.Trim($S + "`$PATH") - } - # apply - $env:PATH = $var_val + $S + $env:PATH - } else { - New-Item -Path "env:$var_name" -Value "$var_val" -Force - } -} - -# Allow calling some IDF python tools without specifying the full path -function idf.py { &python "$IDF_PATH\tools\idf.py" $args } -function espefuse.py { &python "$IDF_PATH\components\esptool_py\esptool\espefuse.py" $args } -function espsecure.py { &python "$IDF_PATH\components\esptool_py\esptool\espsecure.py" $args } -function otatool.py { &python "$IDF_PATH\components\app_update\otatool.py" $args } -function parttool.py { &python "$IDF_PATH\components\partition_table\parttool.py" $args } - -#Compare Path's OLD vs. NEW -$NEW_PATH = $env:PATH.split($S) | Select-Object -Unique # array without duplicates -$dif_Path = Compare-Object -ReferenceObject $OLD_PATH -DifferenceObject $NEW_PATH -PassThru -if ($null -ne $dif_Path) { - Write-Output "`nAdded to PATH`n-------------" - Write-Output $dif_Path -} else { - Write-Output "No directories added to PATH:" - Write-Output $OLD_PATH -} - - -Write-Output "Checking if Python packages are up to date..." - -Start-Process -Wait -NoNewWindow -FilePath "python" -Args "`"$IDF_PATH/tools/idf_tools.py`" check-python-dependencies" -if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # if error - -$uninstall = python "$IDF_PATH/tools/idf_tools.py" uninstall --dry-run - -if (![string]::IsNullOrEmpty($uninstall)){ - Write-Output "" - Write-Output "Detected installed tools that are not currently used by active ESP-IDF version." - Write-Output "$uninstall" - Write-Output "For free up even more space, remove installation packages of those tools. Use option 'python.exe $IDF_PATH\tools\idf_tools.py uninstall --remove-archives'." - Write-Output "" -} - -Write-Output " -Done! You can now compile ESP-IDF projects. -Go to the project directory and run: - idf.py build - -" +$idf_exports = python "$idf_path/tools/activate.py" --export +# The dot sourcing is added here in PowerShell since +# Win PSAnalyzer complains about using `Invoke-Expression` command +. $idf_exports diff --git a/export.sh b/export.sh index 02646293b5..f4eb6334e3 100644 --- a/export.sh +++ b/export.sh @@ -28,20 +28,18 @@ fi if [ ! -f "${idf_path}/tools/idf.py" ] || [ ! -f "${idf_path}/tools/idf_tools.py" ] || - [ ! -f "${idf_path}/activate.py" ] + [ ! -f "${idf_path}/tools/activate.py" ] then - # Echo command here is not used for printing to the terminal, but as non-empty return value from function. echo "Could not detect IDF_PATH. Please set it before sourcing this script:" echo " export IDF_PATH=(add path here)" unset idf_path return 1 fi -# TODO Maybe we can use "command -v" to check just for python and python3 . "${idf_path}/tools/detect_python.sh" # Evaluate the ESP-IDF environment set up by the activate.py script. -idf_exports=$("$ESP_PYTHON" "${idf_path}/activate.py" --export) +idf_exports=$("$ESP_PYTHON" "${idf_path}/tools/activate.py" --export) eval "${idf_exports}" unset idf_path return 0 diff --git a/activate.py b/tools/activate.py similarity index 85% rename from activate.py rename to tools/activate.py index 40e1e78156..8c0c021a5d 100755 --- a/activate.py +++ b/tools/activate.py @@ -17,8 +17,8 @@ def die(msg: str) -> None: sys.exit(f'error: {msg}') -idf_path = os.path.realpath(os.path.dirname(__file__)) -idf_tools_path = os.path.join(idf_path, 'tools') +idf_tools_path = os.path.realpath(os.path.dirname(__file__)) +idf_path = os.path.dirname(idf_tools_path) sys.path.insert(0, idf_tools_path) try: @@ -38,6 +38,6 @@ os.environ['IDF_PYTHON_ENV_PATH'] = idf_python_env_path os.environ['ESP_IDF_VERSION'] = idf_version try: - run([virtualenv_python, os.path.join(idf_path, 'activate_venv.py')] + sys.argv[1:], check=True) + run([virtualenv_python, os.path.join(idf_path, 'tools', 'export_utils', 'activate_venv.py')] + sys.argv[1:], check=True) except (OSError, SubprocessError): die(f'Activation script failed') diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index d235422c16..6524d2b249 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -1,4 +1,3 @@ -activate.py components/app_update/otatool.py components/efuse/efuse_table_gen.py components/efuse/test_efuse_host/efuse_tests.py @@ -48,6 +47,7 @@ examples/system/ota/otatool/otatool_example.py examples/system/ota/otatool/otatool_example.sh install.fish install.sh +tools/activate.py tools/check_python_dependencies.py tools/ci/build_template_app.sh tools/ci/check_api_violation.sh diff --git a/tools/export_utils/activate_venv.py b/tools/export_utils/activate_venv.py new file mode 100644 index 0000000000..6014cb5fe3 --- /dev/null +++ b/tools/export_utils/activate_venv.py @@ -0,0 +1,177 @@ +# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +import argparse +import os +import sys +from typing import Any +from typing import Dict + +from console_output import CONSOLE_STDERR +from console_output import CONSOLE_STDOUT +from console_output import debug +from console_output import die +from console_output import eprint +from console_output import oprint +from console_output import status_message +from shell_types import SHELL_CLASSES +from shell_types import SUPPORTED_SHELLS +from utils import conf +from utils import run_cmd + + +def parse_arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser(prog='activate', + description='Activate ESP-IDF environment', + epilog='On Windows, run `python activate.py` to execute this script in the current terminal window.') + parser.add_argument('-s', '--shell', + metavar='SHELL', + default=os.environ.get('ESP_IDF_SHELL', None), + help='Explicitly specify shell to start. For example bash, zsh, powershell.exe, cmd.exe') + parser.add_argument('-l', '--list', + action='store_true', + help=('List supported shells.')) + parser.add_argument('-e', '--export', + action='store_true', + help=('Generate commands to run in the terminal.')) + parser.add_argument('-n', '--no-color', + action='store_true', + help=('Disable ANSI color escape sequences.')) + parser.add_argument('-d', '--debug', + action='store_true', + help=('Enable debug information.')) + parser.add_argument('-q', '--quiet', + action='store_true', + help=('Suppress all output.')) + + return parser.parse_args() + + +@status_message('Checking python version', rv_on_ok=True) +def check_python_version() -> str: + # Check the Python version within a virtual environment + python_version_checker = os.path.join(conf.IDF_PATH, 'tools', 'python_version_checker.py') + run_cmd([sys.executable, python_version_checker]) + ver = sys.version_info + return f'{ver[0]}.{ver[1]}.{ver[2]}' + + +@status_message('Checking python dependencies') +def check_python_dependencies() -> None: + # Check Python dependencies within the virtual environment + run_cmd([sys.executable, conf.IDF_TOOLS_PY, 'check-python-dependencies']) + + +@status_message('Deactivating the current ESP-IDF environment (if any)') +def get_deactivate_cmd() -> str: + # Get previous ESP-IDF system environment variables + cmd = [sys.executable, conf.IDF_TOOLS_PY, 'export', '--deactivate'] + stdout: str = run_cmd(cmd) + return stdout + + +@status_message('Establishing a new ESP-IDF environment') +def get_idf_env() -> Dict[str,str]: + # Get ESP-IDF system environment variables + extra_paths_list = [os.path.join('components', 'espcoredump'), + os.path.join('components', 'partition_table'), + os.path.join('components', 'app_update')] + extra_paths = os.pathsep.join([os.path.join(conf.IDF_PATH, path) for path in extra_paths_list]) + cmd = [sys.executable, conf.IDF_TOOLS_PY, 'export', '--format', 'key-value', '--add_paths_extras', extra_paths] + stdout = run_cmd(cmd) + + # idf_tools.py might not export certain environment variables if they are already set + idf_env: Dict[str, Any] = { + 'IDF_PATH': os.environ['IDF_PATH'], + 'ESP_IDF_VERSION': os.environ['ESP_IDF_VERSION'], + 'IDF_PYTHON_ENV_PATH': os.environ['IDF_PYTHON_ENV_PATH'], + } + + for line in stdout.splitlines(): + var, val = line.split('=') + idf_env[var] = val + + if 'PATH' in idf_env: + idf_env['PATH'] = os.pathsep.join([extra_paths, idf_env['PATH']]) + + return idf_env + + +@status_message('Identifying shell', rv_on_ok=True) +def detect_shell(args: Any) -> str: + import psutil + + if args.shell is not None: + return str(args.shell) + + current_pid = os.getpid() + detected_shell_name = '' + while True: + parent_pid = psutil.Process(current_pid).ppid() + parent_name = psutil.Process(parent_pid).name() + if not parent_name.startswith('python'): + detected_shell_name = parent_name + conf.DETECTED_SHELL_PATH = psutil.Process(parent_pid).exe() + break + current_pid = parent_pid + + return detected_shell_name + + +@status_message('Detecting outdated tools in system', rv_on_ok=True) +def print_uninstall_msg() -> Any: + stdout = run_cmd([sys.executable, conf.IDF_TOOLS_PY, 'uninstall', '--dry-run']) + if stdout: + python_cmd = 'python.exe' if sys.platform == 'win32' else 'python' + msg = (f'Found tools that are not used by active ESP-IDF version.\n' + f'[bright_cyan]{stdout}\n' + f'To free up even more space, remove installation packages of those tools.\n' + f'Use option {python_cmd} {conf.IDF_TOOLS_PY} uninstall --remove-archives.') + else: + msg = 'OK - no outdated tools found' + + return msg + + +def main() -> None: + args = parse_arguments() + + # Setup parsed arguments + CONSOLE_STDERR.no_color = args.no_color + CONSOLE_STDOUT.no_color = args.no_color + CONSOLE_STDERR.quiet = args.quiet + CONSOLE_STDOUT.quiet = args.quiet + # Fill config global holder + conf.ARGS = args + + if conf.ARGS.list: + oprint(SUPPORTED_SHELLS) + sys.exit() + + eprint(f'[dark_orange]Activating ESP-IDF {conf.IDF_VERSION}') + debug(f'IDF_PATH {conf.IDF_PATH}') + debug(f'IDF_PYTHON_ENV_PATH {conf.IDF_PYTHON_ENV_PATH}') + + check_python_version() + check_python_dependencies() + + deactivate_cmd = get_deactivate_cmd() + new_esp_idf_env = get_idf_env() + detected_shell = detect_shell(conf.ARGS) + print_uninstall_msg() + + if detected_shell not in SHELL_CLASSES: + die(f'"{detected_shell}" shell is not among the supported options: "{SUPPORTED_SHELLS}"') + + shell = SHELL_CLASSES[detected_shell](detected_shell, deactivate_cmd, new_esp_idf_env) + + if conf.ARGS.export: + shell.export() + sys.exit() + + eprint(f'[dark_orange]Starting new \'{shell.shell}\' shell with ESP-IDF environment... (use "exit" command to quit)') + shell.spawn() + eprint(f'[dark_orange]ESP-IDF environment exited.') + + +if __name__ == '__main__': + main() diff --git a/tools/export_utils/console_output.py b/tools/export_utils/console_output.py new file mode 100644 index 0000000000..03d8cb8453 --- /dev/null +++ b/tools/export_utils/console_output.py @@ -0,0 +1,69 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +import sys +from typing import Any +from typing import Callable + +from utils import conf + +try: + # The ESP-IDF virtual environment hasn't been verified yet, so see if the rich library + # can be imported to display error and status messages nicely. + from rich.console import Console +except ImportError as e: + sys.exit(f'error: Unable to import the rich module: {e}. Please execute the install script.') + +CONSOLE_STDERR = Console(stderr=True, width=255) +CONSOLE_STDOUT = Console(width=255) + + +def status_message(msg: str, rv_on_ok: bool=False, die_on_err: bool=True) -> Callable: + def inner(func: Callable) -> Callable: + def wrapper(*args: Any, **kwargs: Any) -> Any: + eprint(f'[dark_orange]*[/dark_orange] {msg} ... ', end='') + + try: + rv = func(*args, **kwargs) + except Exception as e: + eprint('[red]FAILED[/red]') + if conf.ARGS.debug: + raise + if not die_on_err: + return None + die(str(e)) + + if rv_on_ok: + eprint(f'[green]{rv}[/green]') + else: + eprint('[green]OK[/green]') + + return rv + return wrapper + return inner + + +def err(*args: Any, **kwargs: Any) -> None: + CONSOLE_STDERR.print('[red]error[/red]: ', *args, **kwargs) # type: ignore + + +def warn(*args: Any, **kwargs: Any) -> None: + CONSOLE_STDERR.print('[yellow]warning[/yellow]: ', *args, **kwargs) # type: ignore + + +def debug(*args: Any, **kwargs: Any) -> None: + if not conf.ARGS.debug: + return + CONSOLE_STDERR.print('[green_yellow]debug[/green_yellow]: ', *args, **kwargs) # type: ignore + + +def die(*args: Any, **kwargs: Any) -> None: + err(*args, **kwargs) + sys.exit(1) + + +def eprint(*args: Any, **kwargs: Any) -> None: + CONSOLE_STDERR.print(*args, **kwargs) # type: ignore + + +def oprint(*args: Any, **kwargs: Any) -> None: + CONSOLE_STDOUT.print(*args, **kwargs) # type: ignore diff --git a/tools/export_utils/shell_types.py b/tools/export_utils/shell_types.py new file mode 100644 index 0000000000..c1f0c4e690 --- /dev/null +++ b/tools/export_utils/shell_types.py @@ -0,0 +1,316 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +import os +import re +import shutil +import sys +from pathlib import Path +from subprocess import run +from tempfile import gettempdir +from tempfile import NamedTemporaryFile +from tempfile import TemporaryDirectory +from typing import Dict +from typing import List +from typing import TextIO +from typing import Union + +import click +from console_output import debug +from console_output import status_message +from utils import conf +from utils import run_cmd + + +class Shell(): + def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: Dict[str,str]): + self.shell = shell + self.deactivate_cmd = deactivate_cmd + self.new_esp_idf_env = new_esp_idf_env + self.tmp_dir_path = Path(gettempdir()) / 'esp_idf_activate' + if not conf.ARGS.debug and os.path.exists(self.tmp_dir_path): + # Do not cleanup temporary directory when debugging + shutil.rmtree(self.tmp_dir_path) + self.tmp_dir_path.mkdir(parents=True, exist_ok=True) + + def export(self) -> None: + raise NotImplementedError('Subclass must implement abstract method "export"') + + def expanded_env(self) -> Dict[str, str]: + expanded_env = self.new_esp_idf_env.copy() + + if 'PATH' not in expanded_env: + return expanded_env + + # The PATH returned by idf_tools.py export is not expanded. + # Note that for the export script, the PATH should remain unexpanded + # to ensure proper deactivation. In the export script, + # the expansion should occur after deactivation, when the PATH is adjusted. + # But it has to be expanded for processes started with the new PATH. + expanded_env['PATH'] = os.path.expandvars(expanded_env['PATH']) + return expanded_env + + def spawn(self) -> None: + # This method should likely work for all shells because we are delegating the initialization + # purely to Python os.environ. + new_env = os.environ.copy() + new_env.update(self.expanded_env()) + run([self.shell], env=new_env) + + +class UnixShell(Shell): + def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: Dict[str,str]): + super().__init__(shell, deactivate_cmd, new_esp_idf_env) + + with NamedTemporaryFile(dir=self.tmp_dir_path, delete=False, prefix='activate_') as fd: + self.script_file_path = Path(fd.name) + debug(f'Temporary script file path: {self.script_file_path}') + + self.new_esp_idf_env['IDF_TOOLS_INSTALL_CMD'] = os.path.join(conf.IDF_PATH, 'install.sh') + self.new_esp_idf_env['IDF_TOOLS_EXPORT_CMD'] = os.path.join(conf.IDF_PATH, 'export.sh') + + def autocompletion(self) -> None: + # Basic POSIX shells does not support autocompletion + return None + + def init_file(self) -> None: + with open(self.script_file_path, 'w') as fd: + self.export_file(fd) + + def export_file(self, fd: TextIO) -> None: + fd.write(f'{self.deactivate_cmd}\n') + for var, value in self.new_esp_idf_env.items(): + fd.write(f'export {var}="{value}"\n') + stdout = self.autocompletion() # type: ignore + if stdout is not None: + fd.write(f'{stdout}\n') + fd.write((f'echo "\nDone! You can now compile ESP-IDF projects.\n' + 'Go to the project directory and run:\n\n idf.py build"\n')) + + def export(self) -> None: + self.init_file() + print(f'. {self.script_file_path}') + + def click_ver(self) -> int: + return int(click.__version__.split('.')[0]) + + +class BashShell(UnixShell): + def get_bash_major_minor(self) -> float: + env = self.expanded_env() + bash_interpreter = conf.DETECTED_SHELL_PATH if conf.DETECTED_SHELL_PATH else 'bash' + stdout = run_cmd([bash_interpreter, '-c', 'echo ${BASH_VERSINFO[0]}.${BASH_VERSINFO[1]}'], env=env) + bash_maj_min = float(stdout) + return bash_maj_min + + @status_message('Shell completion', die_on_err=False) + def autocompletion(self) -> str: + bash_maj_min = self.get_bash_major_minor() + # Click supports bash version >= 4.4 + # https://click.palletsprojects.com/en/8.1.x/changes/#version-8-0-0 + if bash_maj_min < 4.4: + raise RuntimeError('Autocompletion not supported') + + env = self.expanded_env() + env['LANG'] = 'en' + env['_IDF.PY_COMPLETE'] = 'bash_source' if self.click_ver() >= 8 else 'source_bash' + stdout: str = run_cmd([sys.executable, conf.IDF_PY], env=env) + return stdout + + def init_file(self) -> None: + with open(self.script_file_path, 'w') as fd: + # We will use the --init-file option to pass a custom rc file, which will ignore .bashrc, + # so we need to source .bashrc first. + bashrc_path = os.path.expanduser('~/.bashrc') + if os.path.isfile(bashrc_path): + fd.write(f'source {bashrc_path}\n') + self.export_file(fd) + + def spawn(self) -> None: + self.init_file() + new_env = os.environ.copy() + new_env.update(self.expanded_env()) + run([self.shell, '--init-file', str(self.script_file_path)], env=new_env) + + +class ZshShell(UnixShell): + @status_message('Shell completion', die_on_err=False) + def autocompletion(self) -> str: + env = self.expanded_env() + env['LANG'] = 'en' + env['_IDF.PY_COMPLETE'] = 'zsh_source' if self.click_ver() >= 8 else 'source_zsh' + stdout = run_cmd([sys.executable, conf.IDF_PY], env=env) + return f'autoload -Uz compinit && compinit -u\n{stdout}' + + def init_file(self) -> None: + # If ZDOTDIR is unset, HOME is used instead. + # https://zsh.sourceforge.io/Doc/Release/Files.html#Startup_002fShutdown-Files + zdotdir = os.environ.get('ZDOTDIR', str(Path.home())) + with open(self.script_file_path, 'w') as fd: + # We will use the ZDOTDIR env variable to load our custom script in the newly spawned shell + # so we need to source .zshrc first. + zshrc_path = Path(zdotdir) / '.zshrc' + if zshrc_path.is_file(): + fd.write(f'source {zshrc_path}\n') + + self.export_file(fd) + + def spawn(self) -> None: + self.init_file() + + # Create a temporary directory to use as ZDOTDIR + tmpdir = TemporaryDirectory() + tmpdir_path = Path(tmpdir.name) + debug(f'Temporary ZDOTDIR {tmpdir_path} with .zshrc file') + + # Copy init script to the custom ZDOTDIR + zshrc_path = tmpdir_path / '.zshrc' + shutil.copy(str(self.script_file_path), str(zshrc_path)) + + new_env = os.environ.copy() + new_env.update(self.expanded_env()) + # Set new ZDOTDIR in the new environment + new_env['ZDOTDIR'] = str(tmpdir_path) + + run([self.shell], env=new_env) + + +class FishShell(UnixShell): + def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: Dict[str,str]): + super().__init__(shell, deactivate_cmd, new_esp_idf_env) + self.new_esp_idf_env['IDF_TOOLS_INSTALL_CMD'] = os.path.join(conf.IDF_PATH, 'install.fish') + self.new_esp_idf_env['IDF_TOOLS_EXPORT_CMD'] = os.path.join(conf.IDF_PATH, 'export.fish') + + @status_message('Shell completion', die_on_err=False) + def autocompletion(self) -> str: + env = self.expanded_env() + env['LANG'] = 'en' + env['_IDF.PY_COMPLETE'] = 'fish_source' if self.click_ver() >= 8 else 'source_fish' + stdout: str = run_cmd([sys.executable, conf.IDF_PY], env=env) + return stdout + + def spawn(self) -> None: + self.init_file() + new_env = os.environ.copy() + new_env.update(self.expanded_env()) + run([self.shell, f'--init-command=source {self.script_file_path}'], env=new_env) + + +class PowerShell(Shell): + def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: Dict[str,str]): + super().__init__(shell, deactivate_cmd, new_esp_idf_env) + + with NamedTemporaryFile(dir=self.tmp_dir_path, delete=False, prefix='activate_', suffix='.ps1') as fd: + self.script_file_path = Path(fd.name) + debug(f'Temporary script file path: {self.script_file_path}') + + self.new_esp_idf_env['IDF_TOOLS_INSTALL_CMD'] = os.path.join(conf.IDF_PATH, 'install.ps1') + self.new_esp_idf_env['IDF_TOOLS_EXPORT_CMD'] = os.path.join(conf.IDF_PATH, 'export.ps1') + + def get_functions(self) -> str: + return '\n'.join([ + r'function idf.py { &python "$Env:IDF_PATH\tools\idf.py" $args }', + r'function global:esptool.py { &python -m esptool $args }', + r'function global:espefuse.py { &python -m espefuse $args }', + r'function global:espsecure.py { &python -m espsecure $args }', + r'function global:otatool.py { &python "$Env:IDF_PATH\components\app_update\otatool.py" $args }', + r'function global:parttool.py { &python "$Env:IDF_PATH\components\partition_table\parttool.py" $args }', + ]) + + def export(self) -> None: + self.init_file() + # Powershell is the only Shell class that does not return the script name in dot sourcing style + # since PSAnalyzer complains about using `InvokeExpression` command + print(f'{self.script_file_path}') + + def init_file(self) -> None: + with open(self.script_file_path, 'w') as fd: + # fd.write(f'{self.deactivate_cmd}\n') TODO in upcoming task IDF-10292 + for var, value in self.new_esp_idf_env.items(): + if var == 'PATH': + value = re.sub(r'(%PATH%|\$PATH)', r'$Env:PATH', value) + fd.write(f'$Env:{var}="{value}"\n') + functions = self.get_functions() + fd.write(f'{functions}\n') + fd.write((f'echo "\nDone! You can now compile ESP-IDF projects.\n' + 'Go to the project directory and run:\n\n idf.py build\n"')) + + def spawn(self) -> None: + self.init_file() + new_env = os.environ.copy() + new_env.update(self.expanded_env()) + arguments = ['-NoExit', '-Command', f'{self.script_file_path}'] + cmd: Union[str, List[str]] = [self.shell] + arguments + run(cmd, env=new_env) + + +class WinCmd(Shell): + def __init__(self, shell: str, deactivate_cmd: str, new_esp_idf_env: Dict[str,str]): + super().__init__(shell, deactivate_cmd, new_esp_idf_env) + + with NamedTemporaryFile(dir=self.tmp_dir_path, delete=False, prefix='activate_', suffix='.bat') as fd: + self.script_file_path = Path(fd.name) + debug(f'Temporary script file path: {self.script_file_path}') + + self.new_esp_idf_env['IDF_TOOLS_INSTALL_CMD'] = os.path.join(conf.IDF_PATH, 'install.bat') + self.new_esp_idf_env['IDF_TOOLS_EXPORT_CMD'] = os.path.join(conf.IDF_PATH, 'export.bat') + self.new_esp_idf_env['IDF_TOOLS_JSON_PATH'] = os.path.join(conf.IDF_PATH, 'tools', 'tools.json') + self.new_esp_idf_env['IDF_TOOLS_PY_PATH'] = conf.IDF_TOOLS_PY + + def get_functions(self) -> str: + return '\n'.join([ + r'DOSKEY idf.py=python.exe "%IDF_PATH%\tools\idf.py" $*', + r'DOSKEY esptool.py=python.exe -m esptool $*', + r'DOSKEY espefuse.py=python.exe -m espefuse $*', + r'DOSKEY espsecure.py=python.exe -m espsecure $*', + r'DOSKEY otatool.py=python.exe "%IDF_PATH%\components\app_update\otatool.py" $*', + r'DOSKEY parttool.py=python.exe "%IDF_PATH%\components\partition_table\parttool.py" $*', + ]) + + def export(self) -> None: + self.init_file() + print(f'call {self.script_file_path}') + + def init_file(self) -> None: + with open(self.script_file_path, 'w') as fd: + fd.write('@echo off\n') + # fd.write(f'{self.deactivate_cmd}\n') TODO in upcoming task IDF-10292 + for var, value in self.new_esp_idf_env.items(): + fd.write(f'set {var}={value}\n') + functions = self.get_functions() + fd.write(f'{functions}\n') + fd.write('\n'.join([ + 'echo.', + 'echo Done! You can now compile ESP-IDF projects.', + 'echo Go to the project directory and run:', + 'echo.', + 'echo idf.py build', + 'echo.', + ])) + + def spawn(self) -> None: + self.init_file() + new_env = os.environ.copy() + new_env.update(self.expanded_env()) + arguments = ['/k', f'{self.script_file_path}'] + cmd: Union[str, List[str]] = [self.shell] + arguments + cmd = ' '.join(cmd) + run(cmd, env=new_env) + + +SHELL_CLASSES = { + 'bash': BashShell, + 'zsh': ZshShell, + 'fish': FishShell, + 'sh': UnixShell, + 'ksh': UnixShell, + 'dash': UnixShell, + 'nu': UnixShell, + 'pwsh.exe': PowerShell, + 'pwsh': PowerShell, + 'powershell.exe': PowerShell, + 'powershell': PowerShell, + 'cmd.exe': WinCmd, + 'cmd': WinCmd +} + +SUPPORTED_SHELLS = ' '.join(SHELL_CLASSES.keys()) diff --git a/tools/export_utils/utils.py b/tools/export_utils/utils.py new file mode 100644 index 0000000000..53ba07a852 --- /dev/null +++ b/tools/export_utils/utils.py @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +import argparse +import os +from subprocess import run +from subprocess import SubprocessError +from typing import Any +from typing import Dict +from typing import List +from typing import Optional + + +class Config: + """ + Config serves as global hodler for variables used across modules + It holds also arguments from command line + """ + def __init__(self) -> None: + self.IDF_PATH = os.environ['IDF_PATH'] + self.IDF_VERSION = os.environ['ESP_IDF_VERSION'] + self.IDF_PYTHON_ENV_PATH = os.environ['IDF_PYTHON_ENV_PATH'] + self.IDF_TOOLS_PY = os.path.join(self.IDF_PATH, 'tools', 'idf_tools.py') + self.IDF_PY = os.path.join(self.IDF_PATH, 'tools', 'idf.py') + self.ARGS: Optional[argparse.Namespace] = None + self.DETECTED_SHELL_PATH: str = '' + + +# Global variable instance +conf = Config() + + +def run_cmd(cmd: List[str], env: Optional[Dict[str, Any]]=None) -> str: + new_env = os.environ.copy() + if env is not None: + new_env.update(env) + + cmd_str = '"{}"'.format(' '.join(cmd)) + try: + p = run(cmd, env=new_env, text=True, capture_output=True) + except (OSError, SubprocessError) as e: + raise RuntimeError(f'Command {cmd_str} failed: {e}') + + stdout: str = p.stdout.strip() + stderr: str = p.stderr.strip() + if p.returncode: + raise RuntimeError(f'Command {cmd_str} failed with error code {p.returncode}\n{stdout}\n{stderr}') + + return stdout