Merge branch 'feature/python_features' into 'master'

Tools: Introduce features into the Python package management system & manage package versions outside of ESP-IDF repo

Closes IDF-4343

See merge request espressif/esp-idf!15451
This commit is contained in:
Roland Dobai 2022-01-17 17:27:40 +00:00
commit 036bd3eb26
25 changed files with 441 additions and 166 deletions

View File

@ -72,6 +72,13 @@ variables:
CI_AUTO_TEST_SCRIPT_REPO_BRANCH: "ci/v4.1"
PYTEST_EMBEDDED_TAG: "v0.4.5"
# cache python dependencies
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
cache:
paths:
- .cache/pip
.setup_tools_unless_target_test: &setup_tools_unless_target_test |
if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
export IDF_MIRROR_PREFIX_MAP=
@ -95,6 +102,7 @@ before_script:
- source tools/ci/configure_ci_environment.sh
- *setup_tools_unless_target_test
- fetch_submodules
- $IDF_PATH/tools/idf_tools.py install-python-env
# used for check scripts which we want to run unconditionally
.before_script_no_sync_submodule:
@ -103,6 +111,7 @@ before_script:
- source tools/ci/utils.sh
- source tools/ci/setup_python.sh
- source tools/ci/configure_ci_environment.sh
- $IDF_PATH/tools/idf_tools.py install-python-env
.before_script_minimal:
before_script:
@ -133,6 +142,7 @@ before_script:
- source tools/ci/configure_ci_environment.sh
- *setup_tools_unless_target_test
- fetch_submodules
- $IDF_PATH/tools/idf_tools.py install-python-env
- cd /tmp
- retry_failed git clone --depth 1 --branch $PYTEST_EMBEDDED_TAG https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/idf/pytest-embedded.git
- cd pytest-embedded && bash foreach.sh install

View File

@ -199,7 +199,7 @@
/tools/unit-test-app/ @esp-idf-codeowners/system @esp-idf-codeowners/tools
requirements.txt @esp-idf-codeowners/tools
requirements.*.txt @esp-idf-codeowners/tools
# sort-order-reset

View File

@ -215,8 +215,7 @@ test_idf_tools:
- cd ${IDF_PATH}/tools/test_idf_tools
- ./test_idf_tools.py
# Test for create virtualenv. It must be invoked from Python, not from virtualenv.
- cd ${IDF_PATH}/tools
- python3 ./idf_tools.py install-python-env
- python3 ./test_idf_tools_python_env.py
.test_efuse_table_on_host_template:
extends: .host_test_template

View File

@ -124,6 +124,7 @@
- "tools/tools_schema.json"
- "tools/idf_tools.py"
- "tools/test_idf_tools/**/*"
- "tools/install_util.py"
- "tools/mkdfu.py"
- "tools/test_mkdfu/**/*"

View File

@ -228,7 +228,7 @@ It is also possible to execute the described debugging tools conveniently from `
4. ``idf.py gdbgui``
Starts `gdbgui <https://www.gdbgui.com>`_ debugger frontend enabling out-of-the-box debugging in a browser window.
Starts `gdbgui <https://www.gdbgui.com>`_ debugger frontend enabling out-of-the-box debugging in a browser window. Please run the install script with the "--enable-gdbgui" argument in order to make this option supported, e.g. ``install.sh --enable-gdbgui``.
It is possible to combine these debugging actions on a single command line allowing convenient setup of blocking and non-blocking actions in one step. ``idf.py`` implements a simple logic to move the background actions (such as openocd) to the beginning and the interactive ones (such as gdb, monitor) to the end of the action list.

View File

@ -101,6 +101,10 @@ Any mirror server can be used provided the URL matches the ``github.com`` downlo
* ``check``: For each tool, checks whether the tool is available in the system path and in ``IDF_TOOLS_PATH``.
* ``install-python-env``: Create Python virtual environment and install the required Python packages.
* ``check-python-dependencies``: Checks if all required Python packages are installed.
.. _idf-tools-install:
Install scripts

View File

@ -50,7 +50,7 @@ DOSKEY otatool.py=python.exe "%IDF_PATH%\components\app_update\otatool.py" $*
DOSKEY parttool.py=python.exe "%IDF_PATH%\components\partition_table\parttool.py" $*
echo Checking if Python packages are up to date...
python.exe "%IDF_PATH%\tools\check_python_dependencies.py"
python.exe "%IDF_PATH%\tools\idf_tools.py" check-python-dependencies
if %errorlevel% neq 0 goto :__end
echo.

View File

@ -19,7 +19,7 @@ function __main
eval "$idf_exports"
echo "Checking if Python packages are up to date..."
python "$IDF_PATH"/tools/check_python_dependencies.py || return 1
python "$IDF_PATH"/tools/idf_tools.py check-python-dependencies || return 1
# Allow calling some IDF python tools without specifying the full path
# "$IDF_PATH"/tools is already added by 'idf_tools.py export'

View File

@ -69,7 +69,7 @@ if ($dif_Path -ne $null) {
Write-Output "Checking if Python packages are up to date..."
Start-Process -Wait -NoNewWindow -FilePath "python" -Args "`"$IDF_PATH/tools/check_python_dependencies.py`""
Start-Process -Wait -NoNewWindow -FilePath "python" -Args "`"$IDF_PATH/tools/idf_tools.py`" check-python-dependencies"
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # if error
Write-Output "

View File

@ -96,7 +96,7 @@ __main() {
__verbose "Using Python interpreter in $(which python)"
__verbose "Checking if Python packages are up to date..."
python "${IDF_PATH}/tools/check_python_dependencies.py" || return 1
python "${IDF_PATH}/tools/idf_tools.py" check-python-dependencies || return 1
# Allow calling some IDF python tools without specifying the full path

View File

@ -17,15 +17,16 @@ if not "%MISSING_REQUIREMENTS%" == "" goto :error_missing_requirements
set IDF_PATH=%~dp0
set IDF_PATH=%IDF_PATH:~0,-1%
set TARGETS="all"
if NOT "%1"=="" set TARGETS=%*
for /f "delims=" %%i in ('python.exe "%IDF_PATH%\tools\install_util.py" extract targets "%*"') do set TARGETS=%%i
echo Installing ESP-IDF tools
python.exe "%IDF_PATH%\tools\idf_tools.py" install --targets=%TARGETS%
if %errorlevel% neq 0 goto :end
for /f "delims=" %%i in ('python.exe "%IDF_PATH%\tools\install_util.py" extract features "%*"') do set FEATURES=%%i
echo Setting up Python environment
python.exe "%IDF_PATH%\tools\idf_tools.py" install-python-env
python.exe "%IDF_PATH%\tools\idf_tools.py" install-python-env --features=%FEATURES%
if %errorlevel% neq 0 goto :end
echo All done! You can now run:

View File

@ -7,17 +7,16 @@ set -x IDF_PATH $basedir
echo "Detecting the Python interpreter"
source "$IDF_PATH"/tools/detect_python.fish
if not set -q argv[1]
set TARGETS "all"
else
set TARGETS $argv[1]
end
set TARGETS ("$ESP_PYTHON" "$IDF_PATH"/tools/install_util.py extract targets $argv) || exit 1
echo "Installing ESP-IDF tools"
"$ESP_PYTHON" "$IDF_PATH"/tools/idf_tools.py install --targets=$TARGETS
or exit 1
set FEATURES ("$ESP_PYTHON" "$IDF_PATH"/tools/install_util.py extract features $argv) || exit 1
echo "Installing Python environment and packages"
"$ESP_PYTHON" "$IDF_PATH"/tools/idf_tools.py install-python-env
"$ESP_PYTHON" "$IDF_PATH"/tools/idf_tools.py install-python-env --features=$FEATURES
echo "All done! You can now run:"
echo ""

View File

@ -1,18 +1,16 @@
#!/usr/bin/env pwsh
$IDF_PATH = $PSScriptRoot
if($args.count -eq 0){
$TARGETS = "all"
}else
{
$TARGETS = $args[0] -join ','
}
$TARGETS = (python "$IDF_PATH/tools/install_util.py" extract targets "$args")
Write-Output "Installing ESP-IDF tools"
Start-Process -Wait -NoNewWindow -FilePath "python" -Args "$IDF_PATH/tools/idf_tools.py install --targets=${TARGETS}"
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # if error
$FEATURES = (python "$IDF_PATH/tools/install_util.py" extract features "$args")
Write-Output "Setting up Python environment"
Start-Process -Wait -NoNewWindow -FilePath "python" -Args "$IDF_PATH/tools/idf_tools.py install-python-env"
Start-Process -Wait -NoNewWindow -FilePath "python" -Args "$IDF_PATH/tools/idf_tools.py install-python-env --features=${FEATURES}"
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE} # if error

View File

@ -10,16 +10,15 @@ export IDF_PATH
echo "Detecting the Python interpreter"
. "${IDF_PATH}/tools/detect_python.sh"
if [ "$#" -eq 0 ]; then
TARGETS="all"
else
TARGETS=$1
fi
TARGETS=`"${ESP_PYTHON}" "${IDF_PATH}/tools/install_util.py" extract targets "$@"`
echo "Installing ESP-IDF tools"
"${ESP_PYTHON}" "${IDF_PATH}/tools/idf_tools.py" install --targets=${TARGETS}
FEATURES=`"${ESP_PYTHON}" "${IDF_PATH}/tools/install_util.py" extract features "$@"`
echo "Installing Python environment and packages"
"${ESP_PYTHON}" "${IDF_PATH}/tools/idf_tools.py" install-python-env
"${ESP_PYTHON}" "${IDF_PATH}/tools/idf_tools.py" install-python-env --features=${FEATURES}
echo "All done! You can now run:"
echo ""

23
requirements.core.txt Normal file
View File

@ -0,0 +1,23 @@
# Python package requirements for ESP-IDF. These are the so called core features which are installed in all systems.
setuptools
click
pyserial
future
cryptography
pyparsing
pyelftools
idf-component-manager
# esptool dependencies (see components/esptool_py/esptool/setup.py)
reedsolo
bitstring
ecdsa
# espcoredump dependencies
construct
pygdbmi
# kconfig and menuconfig dependencies
kconfiglib
windows-curses; sys_platform == 'win32'

4
requirements.gdbgui.txt Normal file
View File

@ -0,0 +1,4 @@
# Python package requirements for gdbgui support ESP-IDF.
# This feature can be enabled by running "install.{sh,bat,ps1,fish} --enable-gdbgui"
gdbgui

View File

@ -1,41 +0,0 @@
# This is a list of python packages needed for ESP-IDF. This file is used with pip.
# Please see the Get Started section of the ESP-IDF Programming Guide for further information.
#
setuptools>=21
# The setuptools package is required to install source distributions and on some systems is not installed by default.
# Please keep it as the first item of this list. Version 21 is required to handle PEP 508 environment markers.
#
click>=7.0
pyserial>=3.3
future>=0.15.2
cryptography>=2.1.4
--only-binary cryptography
# Only binary for cryptography is here to make it work on ARMv7 architecture
# We do have cryptography binary on https://dl.espressif.com/pypi for ARM
# On https://pypi.org/ are no ARM binaries as standard now
pyparsing>=3.0.3 # https://github.com/pyparsing/pyparsing/issues/319 is fixed in 3.0.3
pyelftools>=0.22
idf-component-manager>=0.2.99-beta
gdbgui==0.13.2.0
# 0.13.2.1 supports Python 3.6+ only
# Windows is not supported since 0.14.0.0. See https://github.com/cs01/gdbgui/issues/348
pygdbmi<=0.9.0.2
# The pygdbmi required max version 0.9.0.2 since 0.9.0.3 is not compatible with latest gdbgui (>=0.13.2.0)
# A compatible Socket.IO should be used. See https://github.com/miguelgrinberg/python-socketio/issues/578
python-socketio<5
# esptool requirements (see components/esptool_py/esptool/setup.py)
reedsolo>=1.5.3,<=1.5.4
bitstring>=3.1.6
ecdsa>=0.16.0
# espcoredump requirements
# This is the last version supports both 2.7 and 3.4
construct==2.10.54
# kconfig & menuconfig support
kconfiglib==13.7.1
windows-curses; sys_platform == 'win32'

View File

@ -1,6 +1,6 @@
#!/usr/bin/env python
#
# SPDX-FileCopyrightText: 2018-2021 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2018-2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import argparse
@ -8,69 +8,80 @@ import os
import re
import sys
PYTHON_PACKAGE_RE = re.compile(r'[^<>=~]+')
try:
import pkg_resources
except Exception:
except ImportError:
print('pkg_resources cannot be imported probably because the pip package is not installed and/or using a '
'legacy Python interpreter. Please refer to the Get Started section of the ESP-IDF Programming Guide for '
'setting up the required packages.')
sys.exit(1)
def escape_backslash(path):
if sys.platform == 'win32':
# escaped backslashes are necessary in order to be able to copy-paste the printed path
return path.replace('\\', '\\\\')
else:
return path
if __name__ == '__main__':
idf_path = os.getenv('IDF_PATH')
default_requirements_path = os.path.join(idf_path, 'requirements.txt') # type: ignore
parser = argparse.ArgumentParser(description='ESP-IDF Python package dependency checker')
parser.add_argument('--requirements', '-r',
help='Path to the requirements file',
default=default_requirements_path)
help='Path to a requirements file (can be used multiple times)',
action='append', default=[])
parser.add_argument('--constraints', '-c', default=[],
help='Path to a constraints file (can be used multiple times)',
action='append')
args = parser.parse_args()
required_set = set()
for req_path in args.requirements:
with open(req_path) as f:
required_set |= set(i for i in map(str.strip, f.readlines()) if len(i) > 0 and not i.startswith('#'))
constr_dict = {} # for example package_name -> package_name==1.0
for const_path in args.constraints:
with open(const_path) as f:
for con in [i for i in map(str.strip, f.readlines()) if len(i) > 0 and not i.startswith('#')]:
if con.startswith('file://'):
con = os.path.basename(con)
elif con.startswith('--only-binary'):
continue
elif con.startswith('-e') and '#egg=' in con: # version control URLs, take the egg= part at the end only
con_m = re.search(r'#egg=([^\s]+)', con)
if not con_m:
print('Malformed input. Cannot find name in {}'.format(con))
sys.exit(1)
con = con_m[1]
name_m = PYTHON_PACKAGE_RE.search(con)
if not name_m:
print('Malformed input. Cannot find name in {}'.format(con))
sys.exit(1)
constr_dict[name_m[0]] = con
# We need to constrain package dependencies as well. So all installed packages need to be checked.
# For example package A requires package B. We have only A in our requirements. But the newest version of B could
# broke at some time and in that case we add a constraint for B (on the server) but don't have to update the
# requirement file (in the ESP-IDF repo).
required_set |= set(i.key for i in pkg_resources.working_set)
not_satisfied = []
with open(args.requirements) as f:
for line in f:
line = line.strip()
# pkg_resources.require() cannot handle the full requirements file syntax so we need to make
# adjustments for options which we use.
if line.startswith('file://'):
line = os.path.basename(line)
if line.startswith('--only-binary'):
continue
if line.startswith('-e') and '#egg=' in line: # version control URLs, take the egg= part at the end only
line = re.search(r'#egg=([^\s]+)', line).group(1) # type: ignore
try:
pkg_resources.require(line)
except Exception:
not_satisfied.append(line)
for requirement in required_set:
# If there is a version-specific constraint for the requirement then use it. Otherwise, just use the
# requirement as is.
to_require = constr_dict.get(requirement, requirement)
try:
pkg_resources.require(to_require)
except pkg_resources.ResolutionError:
not_satisfied.append(to_require)
if len(not_satisfied) > 0:
print('The following Python requirements are not satisfied:')
for requirement in not_satisfied:
print(requirement)
if os.path.realpath(args.requirements) != os.path.realpath(default_requirements_path):
# we're using this script to check non-default requirements.txt, so tell the user to run pip
print('Please check the documentation for the feature you are using, or run "%s -m pip install -r %s"' % (sys.executable, args.requirements))
elif os.environ.get('IDF_PYTHON_ENV_PATH'):
print(os.linesep.join(not_satisfied))
if 'IDF_PYTHON_ENV_PATH' in os.environ:
# We are running inside a private virtual environment under IDF_TOOLS_PATH,
# ask the user to run install.bat again.
if sys.platform == 'win32':
install_script = 'install.bat'
else:
install_script = 'install.sh'
print('To install the missing packages, please run "%s"' % os.path.join(idf_path, install_script)) # type: ignore
install_script = 'install.bat' if sys.platform == 'win32' else 'install.sh'
print('To install the missing packages, please run "{}"'.format(install_script))
else:
print('Please follow the instructions found in the "Set up the tools" section of '
'ESP-IDF Getting Started Guide')
'ESP-IDF Getting Started Guide.')
print('Diagnostic information:')
idf_python_env_path = os.environ.get('IDF_PYTHON_ENV_PATH')
@ -81,4 +92,4 @@ if __name__ == '__main__':
print(' PATH: {}'.format(os.getenv('PATH')))
sys.exit(1)
print('Python requirements from {} are satisfied.'.format(args.requirements))
print('Python requirements are satisfied.')

View File

@ -148,7 +148,6 @@ examples/wifi/iperf/iperf_test.py
tools/ble/lib_ble_client.py
tools/ble/lib_gap.py
tools/ble/lib_gatt.py
tools/check_python_dependencies.py
tools/check_term.py
tools/ci/check_artifacts_expire_time.py
tools/ci/check_callgraph.py

View File

@ -282,7 +282,7 @@ function(__build_check_python)
idf_build_get_property(python PYTHON)
idf_build_get_property(idf_path IDF_PATH)
message(STATUS "Checking Python dependencies...")
execute_process(COMMAND "${python}" "${idf_path}/tools/check_python_dependencies.py"
execute_process(COMMAND "${python}" "${idf_path}/tools/idf_tools.py" "check-python-dependencies"
RESULT_VARIABLE result)
if(result EQUAL 1)
# check_python_dependencies returns error code 1 on failure

View File

@ -1,6 +1,6 @@
#!/usr/bin/env python
#
# SPDX-FileCopyrightText: 2019-2021 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2019-2022 Espressif Systems (Shanghai) CO LTD
#
# SPDX-License-Identifier: Apache-2.0
#
@ -95,7 +95,8 @@ def check_environment():
out = subprocess.check_output(
[
os.environ['PYTHON'],
os.path.join(os.environ['IDF_PATH'], 'tools', 'check_python_dependencies.py'),
os.path.join(os.environ['IDF_PATH'], 'tools', 'idf_tools.py'),
'check-python-dependencies',
],
env=os.environ,
)

View File

@ -217,7 +217,8 @@ def action_extensions(base_actions, project_path):
process = subprocess.Popen(args, stdout=gdbgui_out, stderr=subprocess.STDOUT, bufsize=1, env=env)
except Exception as e:
print(e)
raise FatalError('Error starting gdbgui. Please make sure gdbgui can be started', ctx)
raise FatalError('Error starting gdbgui. Please make sure gdbgui has been installed with '
'"install.{sh,bat,ps1,fish} --enable-gdbgui" and can be started.', ctx)
processes['gdbgui'] = process
processes['gdbgui_outfile'] = gdbgui_out

View File

@ -88,6 +88,7 @@ DOWNLOAD_RETRY_COUNT = 3
URL_PREFIX_MAP_SEPARATOR = ','
IDF_TOOLS_INSTALL_CMD = os.environ.get('IDF_TOOLS_INSTALL_CMD')
IDF_TOOLS_EXPORT_CMD = os.environ.get('IDF_TOOLS_INSTALL_CMD')
IDF_DL_URL = 'https://dl.espressif.com/dl/esp-idf'
PYTHON_PLATFORM = platform.system() + '-' + platform.machine()
@ -361,6 +362,31 @@ def urlretrieve_ctx(url, filename, reporthook=None, data=None, context=None):
return result
def download(url, destination): # type: (str, str) -> None
info('Downloading {} to {}'.format(os.path.basename(url), destination))
try:
ctx = None
# For dl.espressif.com, add the ISRG x1 root certificate.
# This works around the issue with outdated certificate stores in some installations.
if 'dl.espressif.com' in url:
try:
ctx = ssl.create_default_context()
ctx.load_verify_locations(cadata=ISRG_X1_ROOT_CERT)
except AttributeError:
# no ssl.create_default_context or load_verify_locations cadata argument
# in Python <=2.7.8
pass
urlretrieve_ctx(url, destination, report_progress if not global_non_interactive else None, context=ctx)
sys.stdout.write('\rDone\n')
except Exception as e:
# urlretrieve could throw different exceptions, e.g. IOError when the server is down
# Errors are ignored because the downloaded file is checked a couple of lines later.
warn('Download failure {}'.format(e))
finally:
sys.stdout.flush()
# Sometimes renaming a directory on Windows (randomly?) causes a PermissionError.
# This is confirmed to be a workaround:
# https://github.com/espressif/esp-idf/issues/3819#issuecomment-515167118
@ -680,29 +706,9 @@ class IDFTool(object):
return
downloaded = False
local_temp_path = local_path + '.tmp'
for retry in range(DOWNLOAD_RETRY_COUNT):
local_temp_path = local_path + '.tmp'
info('Downloading {} to {}'.format(archive_name, local_temp_path))
try:
ctx = None
# For dl.espressif.com, add the ISRG x1 root certificate.
# This works around the issue with outdated certificate stores in some installations.
if 'dl.espressif.com' in url:
try:
ctx = ssl.create_default_context()
ctx.load_verify_locations(cadata=ISRG_X1_ROOT_CERT)
except AttributeError:
# no ssl.create_default_context or load_verify_locations cadata argument
# in Python <=2.7.8
pass
urlretrieve_ctx(url, local_temp_path, report_progress if not global_non_interactive else None, context=ctx)
sys.stdout.write('\rDone\n')
except Exception as e:
# urlretrieve could throw different exceptions, e.g. IOError when the server is down
# Errors are ignored because the downloaded file is checked a couple of lines later.
warn('Download failure {}'.format(e))
sys.stdout.flush()
download(url, local_temp_path)
if not os.path.isfile(local_temp_path) or not self.check_download_file(download_obj, local_temp_path):
warn('Failed to download {} to {}'.format(url, local_temp_path))
continue
@ -969,7 +975,7 @@ def dump_tools_json(tools_info): # type: ignore
return json.dumps(file_json, indent=2, separators=(',', ': '), sort_keys=True)
def get_python_env_path(): # type: () -> Tuple[str, str, str]
def get_python_env_path(): # type: () -> Tuple[str, str, str, str]
python_ver_major_minor = '{}.{}'.format(sys.version_info.major, sys.version_info.minor)
version_file_path = os.path.join(global_idf_path, 'version.txt') # type: ignore
@ -1020,7 +1026,7 @@ def get_python_env_path(): # type: () -> Tuple[str, str, str]
idf_python_export_path = os.path.join(idf_python_env_path, subdir)
virtualenv_python = os.path.join(idf_python_export_path, python_exe)
return idf_python_env_path, idf_python_export_path, virtualenv_python
return idf_python_env_path, idf_python_export_path, virtualenv_python, idf_version
def get_idf_env(): # type: () -> Any
@ -1037,29 +1043,34 @@ def get_idf_env(): # type: () -> Any
os.rename(idf_env_file_path, os.path.join(os.path.dirname(idf_env_file_path), (filename + '_failed' + ending)))
info('Creating {}' .format(idf_env_file_path))
return {'idfSelectedId': 'sha', 'idfInstalled': {'sha': {'targets': {}}}}
return {'idfSelectedId': 'sha', 'idfInstalled': {'sha': {'targets': []}}}
def export_targets_to_idf_env_json(targets): # type: (list[str]) -> None
def export_into_idf_env_json(targets, features): # type: (Optional[list[str]], Optional[list[str]]) -> None
idf_env_json = get_idf_env()
targets = list(set(targets + get_user_defined_targets()))
targets = list(set(targets + get_requested_targets_and_features()[0])) if targets else None
for env in idf_env_json['idfInstalled']:
if env == idf_env_json['idfSelectedId']:
idf_env_json['idfInstalled'][env]['targets'] = targets
update_with = []
if targets:
update_with += [('targets', targets)]
if features:
update_with += [('features', features)]
idf_env_json['idfInstalled'][env].update(update_with)
break
try:
if global_idf_tools_path: # mypy fix for Optional[str] in the next call
# the directory doesn't exist if this is run on a clean system the first time
mkdir_p(global_idf_tools_path)
with open(os.path.join(global_idf_tools_path, IDF_ENV_FILE), 'w') as w: # type: ignore
json.dump(idf_env_json, w, indent=4)
with open(os.path.join(global_idf_tools_path, IDF_ENV_FILE), 'w') as w:
json.dump(idf_env_json, w, indent=4)
except (IOError, OSError):
warn('File {} can not be created. '.format(os.path.join(global_idf_tools_path, IDF_ENV_FILE))) # type: ignore
def clean_targets(targets_str): # type: (str) -> list[str]
def add_and_save_targets(targets_str): # type: (str) -> list[str]
targets_from_tools_json = get_all_targets_from_tools_json()
invalid_targets = []
@ -1072,26 +1083,44 @@ def clean_targets(targets_str): # type: (str) -> list[str]
raise SystemExit(1)
# removing duplicates
targets = list(set(targets))
export_targets_to_idf_env_json(targets)
export_into_idf_env_json(targets, None)
else:
export_targets_to_idf_env_json(targets_from_tools_json)
export_into_idf_env_json(targets_from_tools_json, None)
return targets
def get_user_defined_targets(): # type: () -> list[str]
def feature_to_requirements_path(feature): # type: (str) -> str
return os.path.join(global_idf_path or '', 'requirements.{}.txt'.format(feature))
def add_and_save_features(features_str): # type: (str) -> list[str]
_, features = get_requested_targets_and_features()
for new_feature_candidate in features_str.split(','):
if os.path.isfile(feature_to_requirements_path(new_feature_candidate)):
features += [new_feature_candidate]
features = list(set(features + ['core'])) # remove duplicates
export_into_idf_env_json(None, features)
return features
def get_requested_targets_and_features(): # type: () -> tuple[list[str], list[str]]
try:
with open(os.path.join(global_idf_tools_path, IDF_ENV_FILE), 'r') as idf_env_file: # type: ignore
idf_env_json = json.load(idf_env_file)
except OSError:
# warn('File {} was not found. Installing tools for all esp targets.'.format(os.path.join(global_idf_tools_path, IDF_ENV_FILE))) # type: ignore
return []
return [], []
targets = []
features = []
for env in idf_env_json['idfInstalled']:
if env == idf_env_json['idfSelectedId']:
targets = idf_env_json['idfInstalled'][env]['targets']
env_dict = idf_env_json['idfInstalled'][env]
targets = env_dict.get('targets', [])
features = env_dict.get('features', [])
break
return targets
return targets, features
def get_all_targets_from_tools_json(): # type: () -> list[str]
@ -1108,7 +1137,7 @@ def get_all_targets_from_tools_json(): # type: () -> list[str]
def filter_tools_info(tools_info): # type: (OrderedDict[str, IDFTool]) -> OrderedDict[str,IDFTool]
targets = get_user_defined_targets()
targets, _ = get_requested_targets_and_features()
if not targets:
return tools_info
else:
@ -1240,7 +1269,7 @@ def action_export(args): # type: ignore
export_vars[k] = v
current_path = os.getenv('PATH')
idf_python_env_path, idf_python_export_path, virtualenv_python = get_python_env_path()
idf_python_env_path, idf_python_export_path, virtualenv_python, _ = get_python_env_path()
if os.path.exists(virtualenv_python):
idf_python_env_path = to_shell_specific_paths([idf_python_env_path])[0]
if os.getenv('IDF_PYTHON_ENV_PATH') != idf_python_env_path:
@ -1349,7 +1378,7 @@ def action_download(args): # type: ignore
targets = [] # type: list[str]
# Installing only single tools, no targets are specified.
if 'required' in tools_spec:
targets = clean_targets(args.targets)
targets = add_and_save_targets(args.targets)
if args.platform not in PLATFORM_FROM_NAME:
fatal('unknown platform: {}' % args.platform)
@ -1409,8 +1438,8 @@ def action_install(args): # type: ignore
targets = [] # type: list[str]
# Installing only single tools, no targets are specified.
if 'required' in tools_spec:
targets = clean_targets(args.targets)
info('Selected targets are: {}' .format(', '.join(get_user_defined_targets())))
targets = add_and_save_targets(args.targets)
info('Selected targets are: {}' .format(', '.join(get_requested_targets_and_features()[0])))
if not tools_spec or 'required' in tools_spec:
# Installing tools for all ESP_targets required by the operating system.
@ -1475,9 +1504,42 @@ def get_wheels_dir(): # type: () -> Optional[str]
return wheels_dir
def get_requirements(new_features): # type: (str) -> list[str]
features = add_and_save_features(new_features)
return [feature_to_requirements_path(feature) for feature in features]
def get_constraints(idf_version): # type: (str) -> str
constraint_file = 'espidf.constraints.v{}.txt'.format(idf_version)
constraint_path = os.path.join(os.path.expanduser(IDF_TOOLS_PATH_DEFAULT), constraint_file)
constraint_url = '/'.join([IDF_DL_URL, constraint_file])
temp_path = constraint_path + '.tmp'
mkdir_p(os.path.dirname(temp_path))
for _ in range(DOWNLOAD_RETRY_COUNT):
download(constraint_url, temp_path)
if not os.path.isfile(temp_path):
warn('Failed to download {} to {}'.format(constraint_url, temp_path))
continue
if os.path.isfile(constraint_path):
# Windows cannot rename to existing file. It needs to be deleted.
os.remove(constraint_path)
rename_with_retry(temp_path, constraint_path)
return constraint_path
if os.path.isfile(constraint_path):
warn('Failed to download, retry count has expired, using a previously downloaded version')
return constraint_path
else:
fatal('Failed to download, and retry count has expired')
raise DownloadError()
def action_install_python_env(args): # type: ignore
use_constraints = not args.no_constraints
reinstall = args.reinstall
idf_python_env_path, _, virtualenv_python = get_python_env_path()
idf_python_env_path, _, virtualenv_python, idf_version = get_python_env_path()
is_virtualenv = hasattr(sys, 'real_prefix') or (hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix)
if is_virtualenv and (not os.path.exists(idf_python_env_path) or reinstall):
@ -1549,8 +1611,12 @@ def action_install_python_env(args): # type: ignore
warn('Found PIP_USER="yes" in the environment. Disabling PIP_USER in this shell to install packages into a virtual environment.')
env_copy['PIP_USER'] = 'no'
run_args = [virtualenv_python, '-m', 'pip', 'install', '--no-warn-script-location']
requirements_txt = os.path.join(global_idf_path, 'requirements.txt')
run_args += ['-r', requirements_txt]
requirements_file_list = get_requirements(args.features)
for requirement_file in requirements_file_list:
run_args += ['-r', requirement_file]
if use_constraints:
constraint_file = get_constraints(idf_version)
run_args += ['--upgrade', '--constraint', constraint_file]
if args.extra_wheels_dir:
run_args += ['--find-links', args.extra_wheels_dir]
if args.no_index:
@ -1562,10 +1628,58 @@ def action_install_python_env(args): # type: ignore
if wheels_dir is not None:
run_args += ['--find-links', wheels_dir]
info('Installing Python packages from {}'.format(requirements_txt))
info('Installing Python packages')
if use_constraints:
info(' Constraint file: {}'.format(constraint_file))
info(' Requirement files:')
info(os.linesep.join(' - {}'.format(path) for path in requirements_file_list))
subprocess.check_call(run_args, stdout=sys.stdout, stderr=sys.stderr, env=env_copy)
def action_check_python_dependencies(args): # type: ignore
use_constraints = not args.no_constraints
req_paths = get_requirements('') # no new features -> just detect the existing ones
_, _, virtualenv_python, idf_version = get_python_env_path()
if not os.path.isfile(virtualenv_python):
fatal('{} doesn\'t exist! Please run the install script or "idf_tools.py install-python-env" in order to '
'create it'.format(virtualenv_python))
raise SystemExit(1)
if use_constraints:
constr_path = get_constraints(idf_version)
info('Constraint file: {}'.format(constr_path))
info('Requirement files:')
info(os.linesep.join(' - {}'.format(path) for path in req_paths))
info('Python being checked: {}'.format(virtualenv_python))
# The dependency checker will be invoked with virtualenv_python. idf_tools.py could have been invoked with a
# different one, therefore, importing is not a suitable option.
dep_check_cmd = [virtualenv_python,
os.path.join(global_idf_path,
'tools',
'check_python_dependencies.py')]
if use_constraints:
dep_check_cmd += ['-c', constr_path]
for req_path in req_paths:
dep_check_cmd += ['-r', req_path]
try:
ret = subprocess.run(dep_check_cmd)
if ret and ret.returncode:
# returncode is a negative number and system exit output is usually expected be positive.
raise SystemExit(-ret.returncode)
except FileNotFoundError:
# Python environment not yet created
fatal('Requirements are not satisfied!')
raise SystemExit(1)
def action_add_version(args): # type: ignore
tools_info = load_tools_info()
tool_name = args.tool
@ -1771,6 +1885,11 @@ def main(argv): # type: (list[str]) -> None
'to use during installation')
install_python_env.add_argument('--extra-wheels-url', help='Additional URL with wheels', default='https://dl.espressif.com/pypi')
install_python_env.add_argument('--no-index', help='Work offline without retrieving wheels index')
install_python_env.add_argument('--features', default='core', help='A comma separated list of desired features for installing.'
' It defaults to installing just the core funtionality.')
install_python_env.add_argument('--no-constraints', action='store_true', default=False,
help='Disable constraint settings. Use with care and only when you want to manage '
'package versions by yourself.')
if IDF_MAINTAINER:
add_version = subparsers.add_parser('add-version', help='Add or update download info for a version')
@ -1790,6 +1909,12 @@ def main(argv): # type: (list[str]) -> None
help='Output file name')
gen_doc.add_argument('--heading-underline-char', help='Character to use when generating RST sections', default='~')
check_python_dependencies = subparsers.add_parser('check-python-dependencies',
help='Check that all required Python packages are installed.')
check_python_dependencies.add_argument('--no-constraints', action='store_true', default=False,
help='Disable constraint settings. Use with care and only when you want '
'to manage package versions by yourself.')
args = parser.parse_args(argv)
if args.action is None:

70
tools/install_util.py Normal file
View File

@ -0,0 +1,70 @@
#!/usr/bin/env python
# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
#
# SPDX-License-Identifier: Apache-2.0
# This script is used from the $IDF_PATH/install.* scripts. This way the argument parsing can be done at one place and
# doesn't have to be implemented for all shells.
import argparse
from itertools import chain
try:
import python_version_checker
# check the Python version before it will fail with an exception on syntax or package incompatibility.
python_version_checker.check()
except RuntimeError as e:
print(e)
raise SystemExit(1)
def action_extract_features(args: str) -> None:
"""
Command line arguments starting with "--enable-" are features. This function selects those and prints them.
"""
features = ['core'] # "core" features should be always installed
if args:
arg_prefix = '--enable-'
features += [arg[len(arg_prefix):] for arg in args.split() if arg.startswith(arg_prefix)]
print(','.join(features))
def action_extract_targets(args: str) -> None:
"""
Command line arguments starting with "esp" are chip targets. This function selects those and prints them.
"""
target_sep = ','
targets = []
if args:
target_args = (arg for arg in args.split() if arg.lower().startswith('esp'))
# target_args can be comma-separated lists of chip targets
targets = list(chain.from_iterable(commalist.split(target_sep) for commalist in target_args))
print(target_sep.join(targets or ['all']))
def main() -> None:
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers(dest='action', required=True)
extract = subparsers.add_parser('extract', help='Process arguments and extract part of it')
extract.add_argument('type', choices=['targets', 'features'])
extract.add_argument('str-to-parse', nargs='?')
args, unknown_args = parser.parse_known_args()
# standalone "--enable-" won't be included into str-to-parse
action_func = globals()['action_{}_{}'.format(args.action, args.type)]
str_to_parse = vars(args)['str-to-parse'] or ''
action_func(' '.join(chain([str_to_parse], unknown_args)))
if __name__ == '__main__':
main()

View File

@ -0,0 +1,71 @@
# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import os
import shutil
import subprocess
import sys
import unittest
from typing import List
try:
import idf_tools
except ImportError:
sys.path.append('..')
import idf_tools
IDF_PATH = os.environ.get('IDF_PATH', '../..')
TOOLS_DIR = os.environ.get('IDF_TOOLS_PATH') or os.path.expanduser(idf_tools.IDF_TOOLS_PATH_DEFAULT)
PYTHON_DIR = os.path.join(TOOLS_DIR, 'python_env')
REQ_SATISFIED = 'Python requirements are satisfied'
REQ_CORE = '- {}/requirements.core.txt'.format(IDF_PATH)
REQ_GDBGUI = '- {}/requirements.gdbgui.txt'.format(IDF_PATH)
CONSTR = 'Constraint file: {}/espidf.constraints'.format(TOOLS_DIR)
class TestPythonInstall(unittest.TestCase):
def setUp(self): # type: () -> None
if os.path.isdir(PYTHON_DIR):
shutil.rmtree(PYTHON_DIR)
if os.path.isfile(os.path.join(TOOLS_DIR, 'idf-env.json')):
os.remove(os.path.join(TOOLS_DIR, 'idf-env.json'))
def run_idf_tools(self, extra_args): # type: (List[str]) -> str
args = [sys.executable, '../idf_tools.py'] + extra_args
ret = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=120)
return ret.stdout.decode('utf-8', 'ignore')
def test_default_arguments(self): # type: () -> None
output = self.run_idf_tools(['check-python-dependencies'])
self.assertNotIn(REQ_SATISFIED, output)
self.assertIn('bin/python doesn\'t exist', output)
output = self.run_idf_tools(['install-python-env'])
self.assertIn(CONSTR, output)
self.assertIn(REQ_CORE, output)
self.assertNotIn(REQ_GDBGUI, output)
output = self.run_idf_tools(['check-python-dependencies'])
self.assertIn(REQ_SATISFIED, output)
def test_opt_argument(self): # type: () -> None
output = self.run_idf_tools(['install-python-env', '--features', 'gdbgui'])
self.assertIn(CONSTR, output)
self.assertIn(REQ_CORE, output)
self.assertIn(REQ_GDBGUI, output)
output = self.run_idf_tools(['install-python-env'])
# The gdbgui should be installed as well because the feature is is stored in the JSON file
self.assertIn(CONSTR, output)
self.assertIn(REQ_CORE, output)
self.assertIn(REQ_GDBGUI, output)
def test_no_constraints(self): # type: () -> None
output = self.run_idf_tools(['install-python-env', '--no-constraints'])
self.assertNotIn(CONSTR, output)
self.assertIn(REQ_CORE, output)
if __name__ == '__main__':
unittest.main()