mirror of
https://github.com/espressif/esp-idf.git
synced 2024-09-20 00:36:01 -04:00
Merge branch 'feature/python_features' into 'master'
Tools: Introduce features into the Python package management system & manage package versions outside of ESP-IDF repo Closes IDF-4343 See merge request espressif/esp-idf!15451
This commit is contained in:
commit
036bd3eb26
@ -72,6 +72,13 @@ variables:
|
|||||||
CI_AUTO_TEST_SCRIPT_REPO_BRANCH: "ci/v4.1"
|
CI_AUTO_TEST_SCRIPT_REPO_BRANCH: "ci/v4.1"
|
||||||
PYTEST_EMBEDDED_TAG: "v0.4.5"
|
PYTEST_EMBEDDED_TAG: "v0.4.5"
|
||||||
|
|
||||||
|
# cache python dependencies
|
||||||
|
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
|
||||||
|
|
||||||
|
cache:
|
||||||
|
paths:
|
||||||
|
- .cache/pip
|
||||||
|
|
||||||
.setup_tools_unless_target_test: &setup_tools_unless_target_test |
|
.setup_tools_unless_target_test: &setup_tools_unless_target_test |
|
||||||
if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
|
if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
|
||||||
export IDF_MIRROR_PREFIX_MAP=
|
export IDF_MIRROR_PREFIX_MAP=
|
||||||
@ -95,6 +102,7 @@ before_script:
|
|||||||
- source tools/ci/configure_ci_environment.sh
|
- source tools/ci/configure_ci_environment.sh
|
||||||
- *setup_tools_unless_target_test
|
- *setup_tools_unless_target_test
|
||||||
- fetch_submodules
|
- fetch_submodules
|
||||||
|
- $IDF_PATH/tools/idf_tools.py install-python-env
|
||||||
|
|
||||||
# used for check scripts which we want to run unconditionally
|
# used for check scripts which we want to run unconditionally
|
||||||
.before_script_no_sync_submodule:
|
.before_script_no_sync_submodule:
|
||||||
@ -103,6 +111,7 @@ before_script:
|
|||||||
- source tools/ci/utils.sh
|
- source tools/ci/utils.sh
|
||||||
- source tools/ci/setup_python.sh
|
- source tools/ci/setup_python.sh
|
||||||
- source tools/ci/configure_ci_environment.sh
|
- source tools/ci/configure_ci_environment.sh
|
||||||
|
- $IDF_PATH/tools/idf_tools.py install-python-env
|
||||||
|
|
||||||
.before_script_minimal:
|
.before_script_minimal:
|
||||||
before_script:
|
before_script:
|
||||||
@ -133,6 +142,7 @@ before_script:
|
|||||||
- source tools/ci/configure_ci_environment.sh
|
- source tools/ci/configure_ci_environment.sh
|
||||||
- *setup_tools_unless_target_test
|
- *setup_tools_unless_target_test
|
||||||
- fetch_submodules
|
- fetch_submodules
|
||||||
|
- $IDF_PATH/tools/idf_tools.py install-python-env
|
||||||
- cd /tmp
|
- cd /tmp
|
||||||
- retry_failed git clone --depth 1 --branch $PYTEST_EMBEDDED_TAG https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/idf/pytest-embedded.git
|
- retry_failed git clone --depth 1 --branch $PYTEST_EMBEDDED_TAG https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/idf/pytest-embedded.git
|
||||||
- cd pytest-embedded && bash foreach.sh install
|
- cd pytest-embedded && bash foreach.sh install
|
||||||
|
@ -199,7 +199,7 @@
|
|||||||
|
|
||||||
/tools/unit-test-app/ @esp-idf-codeowners/system @esp-idf-codeowners/tools
|
/tools/unit-test-app/ @esp-idf-codeowners/system @esp-idf-codeowners/tools
|
||||||
|
|
||||||
requirements.txt @esp-idf-codeowners/tools
|
requirements.*.txt @esp-idf-codeowners/tools
|
||||||
|
|
||||||
# sort-order-reset
|
# sort-order-reset
|
||||||
|
|
||||||
|
@ -215,8 +215,7 @@ test_idf_tools:
|
|||||||
- cd ${IDF_PATH}/tools/test_idf_tools
|
- cd ${IDF_PATH}/tools/test_idf_tools
|
||||||
- ./test_idf_tools.py
|
- ./test_idf_tools.py
|
||||||
# Test for create virtualenv. It must be invoked from Python, not from virtualenv.
|
# Test for create virtualenv. It must be invoked from Python, not from virtualenv.
|
||||||
- cd ${IDF_PATH}/tools
|
- python3 ./test_idf_tools_python_env.py
|
||||||
- python3 ./idf_tools.py install-python-env
|
|
||||||
|
|
||||||
.test_efuse_table_on_host_template:
|
.test_efuse_table_on_host_template:
|
||||||
extends: .host_test_template
|
extends: .host_test_template
|
||||||
|
@ -124,6 +124,7 @@
|
|||||||
- "tools/tools_schema.json"
|
- "tools/tools_schema.json"
|
||||||
- "tools/idf_tools.py"
|
- "tools/idf_tools.py"
|
||||||
- "tools/test_idf_tools/**/*"
|
- "tools/test_idf_tools/**/*"
|
||||||
|
- "tools/install_util.py"
|
||||||
|
|
||||||
- "tools/mkdfu.py"
|
- "tools/mkdfu.py"
|
||||||
- "tools/test_mkdfu/**/*"
|
- "tools/test_mkdfu/**/*"
|
||||||
|
@ -228,7 +228,7 @@ It is also possible to execute the described debugging tools conveniently from `
|
|||||||
|
|
||||||
4. ``idf.py gdbgui``
|
4. ``idf.py gdbgui``
|
||||||
|
|
||||||
Starts `gdbgui <https://www.gdbgui.com>`_ debugger frontend enabling out-of-the-box debugging in a browser window.
|
Starts `gdbgui <https://www.gdbgui.com>`_ debugger frontend enabling out-of-the-box debugging in a browser window. Please run the install script with the "--enable-gdbgui" argument in order to make this option supported, e.g. ``install.sh --enable-gdbgui``.
|
||||||
|
|
||||||
|
|
||||||
It is possible to combine these debugging actions on a single command line allowing convenient setup of blocking and non-blocking actions in one step. ``idf.py`` implements a simple logic to move the background actions (such as openocd) to the beginning and the interactive ones (such as gdb, monitor) to the end of the action list.
|
It is possible to combine these debugging actions on a single command line allowing convenient setup of blocking and non-blocking actions in one step. ``idf.py`` implements a simple logic to move the background actions (such as openocd) to the beginning and the interactive ones (such as gdb, monitor) to the end of the action list.
|
||||||
|
@ -101,6 +101,10 @@ Any mirror server can be used provided the URL matches the ``github.com`` downlo
|
|||||||
|
|
||||||
* ``check``: For each tool, checks whether the tool is available in the system path and in ``IDF_TOOLS_PATH``.
|
* ``check``: For each tool, checks whether the tool is available in the system path and in ``IDF_TOOLS_PATH``.
|
||||||
|
|
||||||
|
* ``install-python-env``: Create Python virtual environment and install the required Python packages.
|
||||||
|
|
||||||
|
* ``check-python-dependencies``: Checks if all required Python packages are installed.
|
||||||
|
|
||||||
.. _idf-tools-install:
|
.. _idf-tools-install:
|
||||||
|
|
||||||
Install scripts
|
Install scripts
|
||||||
|
@ -50,7 +50,7 @@ DOSKEY otatool.py=python.exe "%IDF_PATH%\components\app_update\otatool.py" $*
|
|||||||
DOSKEY parttool.py=python.exe "%IDF_PATH%\components\partition_table\parttool.py" $*
|
DOSKEY parttool.py=python.exe "%IDF_PATH%\components\partition_table\parttool.py" $*
|
||||||
|
|
||||||
echo Checking if Python packages are up to date...
|
echo Checking if Python packages are up to date...
|
||||||
python.exe "%IDF_PATH%\tools\check_python_dependencies.py"
|
python.exe "%IDF_PATH%\tools\idf_tools.py" check-python-dependencies
|
||||||
if %errorlevel% neq 0 goto :__end
|
if %errorlevel% neq 0 goto :__end
|
||||||
|
|
||||||
echo.
|
echo.
|
||||||
|
@ -19,7 +19,7 @@ function __main
|
|||||||
eval "$idf_exports"
|
eval "$idf_exports"
|
||||||
|
|
||||||
echo "Checking if Python packages are up to date..."
|
echo "Checking if Python packages are up to date..."
|
||||||
python "$IDF_PATH"/tools/check_python_dependencies.py || return 1
|
python "$IDF_PATH"/tools/idf_tools.py check-python-dependencies || return 1
|
||||||
|
|
||||||
# Allow calling some IDF python tools without specifying the full path
|
# Allow calling some IDF python tools without specifying the full path
|
||||||
# "$IDF_PATH"/tools is already added by 'idf_tools.py export'
|
# "$IDF_PATH"/tools is already added by 'idf_tools.py export'
|
||||||
|
@ -69,7 +69,7 @@ if ($dif_Path -ne $null) {
|
|||||||
|
|
||||||
Write-Output "Checking if Python packages are up to date..."
|
Write-Output "Checking if Python packages are up to date..."
|
||||||
|
|
||||||
Start-Process -Wait -NoNewWindow -FilePath "python" -Args "`"$IDF_PATH/tools/check_python_dependencies.py`""
|
Start-Process -Wait -NoNewWindow -FilePath "python" -Args "`"$IDF_PATH/tools/idf_tools.py`" check-python-dependencies"
|
||||||
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # if error
|
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # if error
|
||||||
|
|
||||||
Write-Output "
|
Write-Output "
|
||||||
|
@ -96,7 +96,7 @@ __main() {
|
|||||||
|
|
||||||
__verbose "Using Python interpreter in $(which python)"
|
__verbose "Using Python interpreter in $(which python)"
|
||||||
__verbose "Checking if Python packages are up to date..."
|
__verbose "Checking if Python packages are up to date..."
|
||||||
python "${IDF_PATH}/tools/check_python_dependencies.py" || return 1
|
python "${IDF_PATH}/tools/idf_tools.py" check-python-dependencies || return 1
|
||||||
|
|
||||||
|
|
||||||
# Allow calling some IDF python tools without specifying the full path
|
# Allow calling some IDF python tools without specifying the full path
|
||||||
|
@ -17,15 +17,16 @@ if not "%MISSING_REQUIREMENTS%" == "" goto :error_missing_requirements
|
|||||||
set IDF_PATH=%~dp0
|
set IDF_PATH=%~dp0
|
||||||
set IDF_PATH=%IDF_PATH:~0,-1%
|
set IDF_PATH=%IDF_PATH:~0,-1%
|
||||||
|
|
||||||
set TARGETS="all"
|
for /f "delims=" %%i in ('python.exe "%IDF_PATH%\tools\install_util.py" extract targets "%*"') do set TARGETS=%%i
|
||||||
if NOT "%1"=="" set TARGETS=%*
|
|
||||||
|
|
||||||
echo Installing ESP-IDF tools
|
echo Installing ESP-IDF tools
|
||||||
python.exe "%IDF_PATH%\tools\idf_tools.py" install --targets=%TARGETS%
|
python.exe "%IDF_PATH%\tools\idf_tools.py" install --targets=%TARGETS%
|
||||||
if %errorlevel% neq 0 goto :end
|
if %errorlevel% neq 0 goto :end
|
||||||
|
|
||||||
|
for /f "delims=" %%i in ('python.exe "%IDF_PATH%\tools\install_util.py" extract features "%*"') do set FEATURES=%%i
|
||||||
|
|
||||||
echo Setting up Python environment
|
echo Setting up Python environment
|
||||||
python.exe "%IDF_PATH%\tools\idf_tools.py" install-python-env
|
python.exe "%IDF_PATH%\tools\idf_tools.py" install-python-env --features=%FEATURES%
|
||||||
if %errorlevel% neq 0 goto :end
|
if %errorlevel% neq 0 goto :end
|
||||||
|
|
||||||
echo All done! You can now run:
|
echo All done! You can now run:
|
||||||
|
11
install.fish
11
install.fish
@ -7,17 +7,16 @@ set -x IDF_PATH $basedir
|
|||||||
echo "Detecting the Python interpreter"
|
echo "Detecting the Python interpreter"
|
||||||
source "$IDF_PATH"/tools/detect_python.fish
|
source "$IDF_PATH"/tools/detect_python.fish
|
||||||
|
|
||||||
if not set -q argv[1]
|
set TARGETS ("$ESP_PYTHON" "$IDF_PATH"/tools/install_util.py extract targets $argv) || exit 1
|
||||||
set TARGETS "all"
|
|
||||||
else
|
|
||||||
set TARGETS $argv[1]
|
|
||||||
end
|
|
||||||
echo "Installing ESP-IDF tools"
|
echo "Installing ESP-IDF tools"
|
||||||
"$ESP_PYTHON" "$IDF_PATH"/tools/idf_tools.py install --targets=$TARGETS
|
"$ESP_PYTHON" "$IDF_PATH"/tools/idf_tools.py install --targets=$TARGETS
|
||||||
or exit 1
|
or exit 1
|
||||||
|
|
||||||
|
set FEATURES ("$ESP_PYTHON" "$IDF_PATH"/tools/install_util.py extract features $argv) || exit 1
|
||||||
|
|
||||||
echo "Installing Python environment and packages"
|
echo "Installing Python environment and packages"
|
||||||
"$ESP_PYTHON" "$IDF_PATH"/tools/idf_tools.py install-python-env
|
"$ESP_PYTHON" "$IDF_PATH"/tools/idf_tools.py install-python-env --features=$FEATURES
|
||||||
|
|
||||||
echo "All done! You can now run:"
|
echo "All done! You can now run:"
|
||||||
echo ""
|
echo ""
|
||||||
|
12
install.ps1
12
install.ps1
@ -1,18 +1,16 @@
|
|||||||
#!/usr/bin/env pwsh
|
#!/usr/bin/env pwsh
|
||||||
$IDF_PATH = $PSScriptRoot
|
$IDF_PATH = $PSScriptRoot
|
||||||
|
|
||||||
if($args.count -eq 0){
|
$TARGETS = (python "$IDF_PATH/tools/install_util.py" extract targets "$args")
|
||||||
$TARGETS = "all"
|
|
||||||
}else
|
|
||||||
{
|
|
||||||
$TARGETS = $args[0] -join ','
|
|
||||||
}
|
|
||||||
Write-Output "Installing ESP-IDF tools"
|
Write-Output "Installing ESP-IDF tools"
|
||||||
Start-Process -Wait -NoNewWindow -FilePath "python" -Args "$IDF_PATH/tools/idf_tools.py install --targets=${TARGETS}"
|
Start-Process -Wait -NoNewWindow -FilePath "python" -Args "$IDF_PATH/tools/idf_tools.py install --targets=${TARGETS}"
|
||||||
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # if error
|
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # if error
|
||||||
|
|
||||||
|
$FEATURES = (python "$IDF_PATH/tools/install_util.py" extract features "$args")
|
||||||
|
|
||||||
Write-Output "Setting up Python environment"
|
Write-Output "Setting up Python environment"
|
||||||
Start-Process -Wait -NoNewWindow -FilePath "python" -Args "$IDF_PATH/tools/idf_tools.py install-python-env"
|
Start-Process -Wait -NoNewWindow -FilePath "python" -Args "$IDF_PATH/tools/idf_tools.py install-python-env --features=${FEATURES}"
|
||||||
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE} # if error
|
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE} # if error
|
||||||
|
|
||||||
|
|
||||||
|
11
install.sh
11
install.sh
@ -10,16 +10,15 @@ export IDF_PATH
|
|||||||
echo "Detecting the Python interpreter"
|
echo "Detecting the Python interpreter"
|
||||||
. "${IDF_PATH}/tools/detect_python.sh"
|
. "${IDF_PATH}/tools/detect_python.sh"
|
||||||
|
|
||||||
if [ "$#" -eq 0 ]; then
|
TARGETS=`"${ESP_PYTHON}" "${IDF_PATH}/tools/install_util.py" extract targets "$@"`
|
||||||
TARGETS="all"
|
|
||||||
else
|
|
||||||
TARGETS=$1
|
|
||||||
fi
|
|
||||||
echo "Installing ESP-IDF tools"
|
echo "Installing ESP-IDF tools"
|
||||||
"${ESP_PYTHON}" "${IDF_PATH}/tools/idf_tools.py" install --targets=${TARGETS}
|
"${ESP_PYTHON}" "${IDF_PATH}/tools/idf_tools.py" install --targets=${TARGETS}
|
||||||
|
|
||||||
|
FEATURES=`"${ESP_PYTHON}" "${IDF_PATH}/tools/install_util.py" extract features "$@"`
|
||||||
|
|
||||||
echo "Installing Python environment and packages"
|
echo "Installing Python environment and packages"
|
||||||
"${ESP_PYTHON}" "${IDF_PATH}/tools/idf_tools.py" install-python-env
|
"${ESP_PYTHON}" "${IDF_PATH}/tools/idf_tools.py" install-python-env --features=${FEATURES}
|
||||||
|
|
||||||
echo "All done! You can now run:"
|
echo "All done! You can now run:"
|
||||||
echo ""
|
echo ""
|
||||||
|
23
requirements.core.txt
Normal file
23
requirements.core.txt
Normal file
@ -0,0 +1,23 @@
|
|||||||
|
# Python package requirements for ESP-IDF. These are the so called core features which are installed in all systems.
|
||||||
|
|
||||||
|
setuptools
|
||||||
|
click
|
||||||
|
pyserial
|
||||||
|
future
|
||||||
|
cryptography
|
||||||
|
pyparsing
|
||||||
|
pyelftools
|
||||||
|
idf-component-manager
|
||||||
|
|
||||||
|
# esptool dependencies (see components/esptool_py/esptool/setup.py)
|
||||||
|
reedsolo
|
||||||
|
bitstring
|
||||||
|
ecdsa
|
||||||
|
|
||||||
|
# espcoredump dependencies
|
||||||
|
construct
|
||||||
|
pygdbmi
|
||||||
|
|
||||||
|
# kconfig and menuconfig dependencies
|
||||||
|
kconfiglib
|
||||||
|
windows-curses; sys_platform == 'win32'
|
4
requirements.gdbgui.txt
Normal file
4
requirements.gdbgui.txt
Normal file
@ -0,0 +1,4 @@
|
|||||||
|
# Python package requirements for gdbgui support ESP-IDF.
|
||||||
|
# This feature can be enabled by running "install.{sh,bat,ps1,fish} --enable-gdbgui"
|
||||||
|
|
||||||
|
gdbgui
|
@ -1,41 +0,0 @@
|
|||||||
# This is a list of python packages needed for ESP-IDF. This file is used with pip.
|
|
||||||
# Please see the Get Started section of the ESP-IDF Programming Guide for further information.
|
|
||||||
#
|
|
||||||
setuptools>=21
|
|
||||||
# The setuptools package is required to install source distributions and on some systems is not installed by default.
|
|
||||||
# Please keep it as the first item of this list. Version 21 is required to handle PEP 508 environment markers.
|
|
||||||
#
|
|
||||||
click>=7.0
|
|
||||||
pyserial>=3.3
|
|
||||||
future>=0.15.2
|
|
||||||
|
|
||||||
cryptography>=2.1.4
|
|
||||||
--only-binary cryptography
|
|
||||||
# Only binary for cryptography is here to make it work on ARMv7 architecture
|
|
||||||
# We do have cryptography binary on https://dl.espressif.com/pypi for ARM
|
|
||||||
# On https://pypi.org/ are no ARM binaries as standard now
|
|
||||||
|
|
||||||
pyparsing>=3.0.3 # https://github.com/pyparsing/pyparsing/issues/319 is fixed in 3.0.3
|
|
||||||
pyelftools>=0.22
|
|
||||||
idf-component-manager>=0.2.99-beta
|
|
||||||
|
|
||||||
gdbgui==0.13.2.0
|
|
||||||
# 0.13.2.1 supports Python 3.6+ only
|
|
||||||
# Windows is not supported since 0.14.0.0. See https://github.com/cs01/gdbgui/issues/348
|
|
||||||
pygdbmi<=0.9.0.2
|
|
||||||
# The pygdbmi required max version 0.9.0.2 since 0.9.0.3 is not compatible with latest gdbgui (>=0.13.2.0)
|
|
||||||
# A compatible Socket.IO should be used. See https://github.com/miguelgrinberg/python-socketio/issues/578
|
|
||||||
python-socketio<5
|
|
||||||
|
|
||||||
# esptool requirements (see components/esptool_py/esptool/setup.py)
|
|
||||||
reedsolo>=1.5.3,<=1.5.4
|
|
||||||
bitstring>=3.1.6
|
|
||||||
ecdsa>=0.16.0
|
|
||||||
|
|
||||||
# espcoredump requirements
|
|
||||||
# This is the last version supports both 2.7 and 3.4
|
|
||||||
construct==2.10.54
|
|
||||||
|
|
||||||
# kconfig & menuconfig support
|
|
||||||
kconfiglib==13.7.1
|
|
||||||
windows-curses; sys_platform == 'win32'
|
|
@ -1,6 +1,6 @@
|
|||||||
#!/usr/bin/env python
|
#!/usr/bin/env python
|
||||||
#
|
#
|
||||||
# SPDX-FileCopyrightText: 2018-2021 Espressif Systems (Shanghai) CO LTD
|
# SPDX-FileCopyrightText: 2018-2022 Espressif Systems (Shanghai) CO LTD
|
||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
@ -8,69 +8,80 @@ import os
|
|||||||
import re
|
import re
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
|
PYTHON_PACKAGE_RE = re.compile(r'[^<>=~]+')
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import pkg_resources
|
import pkg_resources
|
||||||
except Exception:
|
except ImportError:
|
||||||
print('pkg_resources cannot be imported probably because the pip package is not installed and/or using a '
|
print('pkg_resources cannot be imported probably because the pip package is not installed and/or using a '
|
||||||
'legacy Python interpreter. Please refer to the Get Started section of the ESP-IDF Programming Guide for '
|
'legacy Python interpreter. Please refer to the Get Started section of the ESP-IDF Programming Guide for '
|
||||||
'setting up the required packages.')
|
'setting up the required packages.')
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
def escape_backslash(path):
|
|
||||||
if sys.platform == 'win32':
|
|
||||||
# escaped backslashes are necessary in order to be able to copy-paste the printed path
|
|
||||||
return path.replace('\\', '\\\\')
|
|
||||||
else:
|
|
||||||
return path
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
idf_path = os.getenv('IDF_PATH')
|
|
||||||
|
|
||||||
default_requirements_path = os.path.join(idf_path, 'requirements.txt') # type: ignore
|
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(description='ESP-IDF Python package dependency checker')
|
parser = argparse.ArgumentParser(description='ESP-IDF Python package dependency checker')
|
||||||
parser.add_argument('--requirements', '-r',
|
parser.add_argument('--requirements', '-r',
|
||||||
help='Path to the requirements file',
|
help='Path to a requirements file (can be used multiple times)',
|
||||||
default=default_requirements_path)
|
action='append', default=[])
|
||||||
|
parser.add_argument('--constraints', '-c', default=[],
|
||||||
|
help='Path to a constraints file (can be used multiple times)',
|
||||||
|
action='append')
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
required_set = set()
|
||||||
|
for req_path in args.requirements:
|
||||||
|
with open(req_path) as f:
|
||||||
|
required_set |= set(i for i in map(str.strip, f.readlines()) if len(i) > 0 and not i.startswith('#'))
|
||||||
|
|
||||||
|
constr_dict = {} # for example package_name -> package_name==1.0
|
||||||
|
for const_path in args.constraints:
|
||||||
|
with open(const_path) as f:
|
||||||
|
for con in [i for i in map(str.strip, f.readlines()) if len(i) > 0 and not i.startswith('#')]:
|
||||||
|
if con.startswith('file://'):
|
||||||
|
con = os.path.basename(con)
|
||||||
|
elif con.startswith('--only-binary'):
|
||||||
|
continue
|
||||||
|
elif con.startswith('-e') and '#egg=' in con: # version control URLs, take the egg= part at the end only
|
||||||
|
con_m = re.search(r'#egg=([^\s]+)', con)
|
||||||
|
if not con_m:
|
||||||
|
print('Malformed input. Cannot find name in {}'.format(con))
|
||||||
|
sys.exit(1)
|
||||||
|
con = con_m[1]
|
||||||
|
|
||||||
|
name_m = PYTHON_PACKAGE_RE.search(con)
|
||||||
|
if not name_m:
|
||||||
|
print('Malformed input. Cannot find name in {}'.format(con))
|
||||||
|
sys.exit(1)
|
||||||
|
constr_dict[name_m[0]] = con
|
||||||
|
|
||||||
|
# We need to constrain package dependencies as well. So all installed packages need to be checked.
|
||||||
|
# For example package A requires package B. We have only A in our requirements. But the newest version of B could
|
||||||
|
# broke at some time and in that case we add a constraint for B (on the server) but don't have to update the
|
||||||
|
# requirement file (in the ESP-IDF repo).
|
||||||
|
required_set |= set(i.key for i in pkg_resources.working_set)
|
||||||
|
|
||||||
not_satisfied = []
|
not_satisfied = []
|
||||||
with open(args.requirements) as f:
|
for requirement in required_set:
|
||||||
for line in f:
|
# If there is a version-specific constraint for the requirement then use it. Otherwise, just use the
|
||||||
line = line.strip()
|
# requirement as is.
|
||||||
# pkg_resources.require() cannot handle the full requirements file syntax so we need to make
|
to_require = constr_dict.get(requirement, requirement)
|
||||||
# adjustments for options which we use.
|
try:
|
||||||
if line.startswith('file://'):
|
pkg_resources.require(to_require)
|
||||||
line = os.path.basename(line)
|
except pkg_resources.ResolutionError:
|
||||||
if line.startswith('--only-binary'):
|
not_satisfied.append(to_require)
|
||||||
continue
|
|
||||||
if line.startswith('-e') and '#egg=' in line: # version control URLs, take the egg= part at the end only
|
|
||||||
line = re.search(r'#egg=([^\s]+)', line).group(1) # type: ignore
|
|
||||||
try:
|
|
||||||
pkg_resources.require(line)
|
|
||||||
except Exception:
|
|
||||||
not_satisfied.append(line)
|
|
||||||
|
|
||||||
if len(not_satisfied) > 0:
|
if len(not_satisfied) > 0:
|
||||||
print('The following Python requirements are not satisfied:')
|
print('The following Python requirements are not satisfied:')
|
||||||
for requirement in not_satisfied:
|
print(os.linesep.join(not_satisfied))
|
||||||
print(requirement)
|
if 'IDF_PYTHON_ENV_PATH' in os.environ:
|
||||||
if os.path.realpath(args.requirements) != os.path.realpath(default_requirements_path):
|
|
||||||
# we're using this script to check non-default requirements.txt, so tell the user to run pip
|
|
||||||
print('Please check the documentation for the feature you are using, or run "%s -m pip install -r %s"' % (sys.executable, args.requirements))
|
|
||||||
elif os.environ.get('IDF_PYTHON_ENV_PATH'):
|
|
||||||
# We are running inside a private virtual environment under IDF_TOOLS_PATH,
|
# We are running inside a private virtual environment under IDF_TOOLS_PATH,
|
||||||
# ask the user to run install.bat again.
|
# ask the user to run install.bat again.
|
||||||
if sys.platform == 'win32':
|
install_script = 'install.bat' if sys.platform == 'win32' else 'install.sh'
|
||||||
install_script = 'install.bat'
|
print('To install the missing packages, please run "{}"'.format(install_script))
|
||||||
else:
|
|
||||||
install_script = 'install.sh'
|
|
||||||
print('To install the missing packages, please run "%s"' % os.path.join(idf_path, install_script)) # type: ignore
|
|
||||||
else:
|
else:
|
||||||
print('Please follow the instructions found in the "Set up the tools" section of '
|
print('Please follow the instructions found in the "Set up the tools" section of '
|
||||||
'ESP-IDF Getting Started Guide')
|
'ESP-IDF Getting Started Guide.')
|
||||||
|
|
||||||
print('Diagnostic information:')
|
print('Diagnostic information:')
|
||||||
idf_python_env_path = os.environ.get('IDF_PYTHON_ENV_PATH')
|
idf_python_env_path = os.environ.get('IDF_PYTHON_ENV_PATH')
|
||||||
@ -81,4 +92,4 @@ if __name__ == '__main__':
|
|||||||
print(' PATH: {}'.format(os.getenv('PATH')))
|
print(' PATH: {}'.format(os.getenv('PATH')))
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
print('Python requirements from {} are satisfied.'.format(args.requirements))
|
print('Python requirements are satisfied.')
|
||||||
|
@ -148,7 +148,6 @@ examples/wifi/iperf/iperf_test.py
|
|||||||
tools/ble/lib_ble_client.py
|
tools/ble/lib_ble_client.py
|
||||||
tools/ble/lib_gap.py
|
tools/ble/lib_gap.py
|
||||||
tools/ble/lib_gatt.py
|
tools/ble/lib_gatt.py
|
||||||
tools/check_python_dependencies.py
|
|
||||||
tools/check_term.py
|
tools/check_term.py
|
||||||
tools/ci/check_artifacts_expire_time.py
|
tools/ci/check_artifacts_expire_time.py
|
||||||
tools/ci/check_callgraph.py
|
tools/ci/check_callgraph.py
|
||||||
|
@ -282,7 +282,7 @@ function(__build_check_python)
|
|||||||
idf_build_get_property(python PYTHON)
|
idf_build_get_property(python PYTHON)
|
||||||
idf_build_get_property(idf_path IDF_PATH)
|
idf_build_get_property(idf_path IDF_PATH)
|
||||||
message(STATUS "Checking Python dependencies...")
|
message(STATUS "Checking Python dependencies...")
|
||||||
execute_process(COMMAND "${python}" "${idf_path}/tools/check_python_dependencies.py"
|
execute_process(COMMAND "${python}" "${idf_path}/tools/idf_tools.py" "check-python-dependencies"
|
||||||
RESULT_VARIABLE result)
|
RESULT_VARIABLE result)
|
||||||
if(result EQUAL 1)
|
if(result EQUAL 1)
|
||||||
# check_python_dependencies returns error code 1 on failure
|
# check_python_dependencies returns error code 1 on failure
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
#!/usr/bin/env python
|
#!/usr/bin/env python
|
||||||
#
|
#
|
||||||
# SPDX-FileCopyrightText: 2019-2021 Espressif Systems (Shanghai) CO LTD
|
# SPDX-FileCopyrightText: 2019-2022 Espressif Systems (Shanghai) CO LTD
|
||||||
#
|
#
|
||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
#
|
#
|
||||||
@ -95,7 +95,8 @@ def check_environment():
|
|||||||
out = subprocess.check_output(
|
out = subprocess.check_output(
|
||||||
[
|
[
|
||||||
os.environ['PYTHON'],
|
os.environ['PYTHON'],
|
||||||
os.path.join(os.environ['IDF_PATH'], 'tools', 'check_python_dependencies.py'),
|
os.path.join(os.environ['IDF_PATH'], 'tools', 'idf_tools.py'),
|
||||||
|
'check-python-dependencies',
|
||||||
],
|
],
|
||||||
env=os.environ,
|
env=os.environ,
|
||||||
)
|
)
|
||||||
|
@ -217,7 +217,8 @@ def action_extensions(base_actions, project_path):
|
|||||||
process = subprocess.Popen(args, stdout=gdbgui_out, stderr=subprocess.STDOUT, bufsize=1, env=env)
|
process = subprocess.Popen(args, stdout=gdbgui_out, stderr=subprocess.STDOUT, bufsize=1, env=env)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(e)
|
print(e)
|
||||||
raise FatalError('Error starting gdbgui. Please make sure gdbgui can be started', ctx)
|
raise FatalError('Error starting gdbgui. Please make sure gdbgui has been installed with '
|
||||||
|
'"install.{sh,bat,ps1,fish} --enable-gdbgui" and can be started.', ctx)
|
||||||
|
|
||||||
processes['gdbgui'] = process
|
processes['gdbgui'] = process
|
||||||
processes['gdbgui_outfile'] = gdbgui_out
|
processes['gdbgui_outfile'] = gdbgui_out
|
||||||
|
@ -88,6 +88,7 @@ DOWNLOAD_RETRY_COUNT = 3
|
|||||||
URL_PREFIX_MAP_SEPARATOR = ','
|
URL_PREFIX_MAP_SEPARATOR = ','
|
||||||
IDF_TOOLS_INSTALL_CMD = os.environ.get('IDF_TOOLS_INSTALL_CMD')
|
IDF_TOOLS_INSTALL_CMD = os.environ.get('IDF_TOOLS_INSTALL_CMD')
|
||||||
IDF_TOOLS_EXPORT_CMD = os.environ.get('IDF_TOOLS_INSTALL_CMD')
|
IDF_TOOLS_EXPORT_CMD = os.environ.get('IDF_TOOLS_INSTALL_CMD')
|
||||||
|
IDF_DL_URL = 'https://dl.espressif.com/dl/esp-idf'
|
||||||
|
|
||||||
PYTHON_PLATFORM = platform.system() + '-' + platform.machine()
|
PYTHON_PLATFORM = platform.system() + '-' + platform.machine()
|
||||||
|
|
||||||
@ -361,6 +362,31 @@ def urlretrieve_ctx(url, filename, reporthook=None, data=None, context=None):
|
|||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def download(url, destination): # type: (str, str) -> None
|
||||||
|
info('Downloading {} to {}'.format(os.path.basename(url), destination))
|
||||||
|
try:
|
||||||
|
ctx = None
|
||||||
|
# For dl.espressif.com, add the ISRG x1 root certificate.
|
||||||
|
# This works around the issue with outdated certificate stores in some installations.
|
||||||
|
if 'dl.espressif.com' in url:
|
||||||
|
try:
|
||||||
|
ctx = ssl.create_default_context()
|
||||||
|
ctx.load_verify_locations(cadata=ISRG_X1_ROOT_CERT)
|
||||||
|
except AttributeError:
|
||||||
|
# no ssl.create_default_context or load_verify_locations cadata argument
|
||||||
|
# in Python <=2.7.8
|
||||||
|
pass
|
||||||
|
|
||||||
|
urlretrieve_ctx(url, destination, report_progress if not global_non_interactive else None, context=ctx)
|
||||||
|
sys.stdout.write('\rDone\n')
|
||||||
|
except Exception as e:
|
||||||
|
# urlretrieve could throw different exceptions, e.g. IOError when the server is down
|
||||||
|
# Errors are ignored because the downloaded file is checked a couple of lines later.
|
||||||
|
warn('Download failure {}'.format(e))
|
||||||
|
finally:
|
||||||
|
sys.stdout.flush()
|
||||||
|
|
||||||
|
|
||||||
# Sometimes renaming a directory on Windows (randomly?) causes a PermissionError.
|
# Sometimes renaming a directory on Windows (randomly?) causes a PermissionError.
|
||||||
# This is confirmed to be a workaround:
|
# This is confirmed to be a workaround:
|
||||||
# https://github.com/espressif/esp-idf/issues/3819#issuecomment-515167118
|
# https://github.com/espressif/esp-idf/issues/3819#issuecomment-515167118
|
||||||
@ -680,29 +706,9 @@ class IDFTool(object):
|
|||||||
return
|
return
|
||||||
|
|
||||||
downloaded = False
|
downloaded = False
|
||||||
|
local_temp_path = local_path + '.tmp'
|
||||||
for retry in range(DOWNLOAD_RETRY_COUNT):
|
for retry in range(DOWNLOAD_RETRY_COUNT):
|
||||||
local_temp_path = local_path + '.tmp'
|
download(url, local_temp_path)
|
||||||
info('Downloading {} to {}'.format(archive_name, local_temp_path))
|
|
||||||
try:
|
|
||||||
ctx = None
|
|
||||||
# For dl.espressif.com, add the ISRG x1 root certificate.
|
|
||||||
# This works around the issue with outdated certificate stores in some installations.
|
|
||||||
if 'dl.espressif.com' in url:
|
|
||||||
try:
|
|
||||||
ctx = ssl.create_default_context()
|
|
||||||
ctx.load_verify_locations(cadata=ISRG_X1_ROOT_CERT)
|
|
||||||
except AttributeError:
|
|
||||||
# no ssl.create_default_context or load_verify_locations cadata argument
|
|
||||||
# in Python <=2.7.8
|
|
||||||
pass
|
|
||||||
|
|
||||||
urlretrieve_ctx(url, local_temp_path, report_progress if not global_non_interactive else None, context=ctx)
|
|
||||||
sys.stdout.write('\rDone\n')
|
|
||||||
except Exception as e:
|
|
||||||
# urlretrieve could throw different exceptions, e.g. IOError when the server is down
|
|
||||||
# Errors are ignored because the downloaded file is checked a couple of lines later.
|
|
||||||
warn('Download failure {}'.format(e))
|
|
||||||
sys.stdout.flush()
|
|
||||||
if not os.path.isfile(local_temp_path) or not self.check_download_file(download_obj, local_temp_path):
|
if not os.path.isfile(local_temp_path) or not self.check_download_file(download_obj, local_temp_path):
|
||||||
warn('Failed to download {} to {}'.format(url, local_temp_path))
|
warn('Failed to download {} to {}'.format(url, local_temp_path))
|
||||||
continue
|
continue
|
||||||
@ -969,7 +975,7 @@ def dump_tools_json(tools_info): # type: ignore
|
|||||||
return json.dumps(file_json, indent=2, separators=(',', ': '), sort_keys=True)
|
return json.dumps(file_json, indent=2, separators=(',', ': '), sort_keys=True)
|
||||||
|
|
||||||
|
|
||||||
def get_python_env_path(): # type: () -> Tuple[str, str, str]
|
def get_python_env_path(): # type: () -> Tuple[str, str, str, str]
|
||||||
python_ver_major_minor = '{}.{}'.format(sys.version_info.major, sys.version_info.minor)
|
python_ver_major_minor = '{}.{}'.format(sys.version_info.major, sys.version_info.minor)
|
||||||
|
|
||||||
version_file_path = os.path.join(global_idf_path, 'version.txt') # type: ignore
|
version_file_path = os.path.join(global_idf_path, 'version.txt') # type: ignore
|
||||||
@ -1020,7 +1026,7 @@ def get_python_env_path(): # type: () -> Tuple[str, str, str]
|
|||||||
idf_python_export_path = os.path.join(idf_python_env_path, subdir)
|
idf_python_export_path = os.path.join(idf_python_env_path, subdir)
|
||||||
virtualenv_python = os.path.join(idf_python_export_path, python_exe)
|
virtualenv_python = os.path.join(idf_python_export_path, python_exe)
|
||||||
|
|
||||||
return idf_python_env_path, idf_python_export_path, virtualenv_python
|
return idf_python_env_path, idf_python_export_path, virtualenv_python, idf_version
|
||||||
|
|
||||||
|
|
||||||
def get_idf_env(): # type: () -> Any
|
def get_idf_env(): # type: () -> Any
|
||||||
@ -1037,29 +1043,34 @@ def get_idf_env(): # type: () -> Any
|
|||||||
os.rename(idf_env_file_path, os.path.join(os.path.dirname(idf_env_file_path), (filename + '_failed' + ending)))
|
os.rename(idf_env_file_path, os.path.join(os.path.dirname(idf_env_file_path), (filename + '_failed' + ending)))
|
||||||
|
|
||||||
info('Creating {}' .format(idf_env_file_path))
|
info('Creating {}' .format(idf_env_file_path))
|
||||||
return {'idfSelectedId': 'sha', 'idfInstalled': {'sha': {'targets': {}}}}
|
return {'idfSelectedId': 'sha', 'idfInstalled': {'sha': {'targets': []}}}
|
||||||
|
|
||||||
|
|
||||||
def export_targets_to_idf_env_json(targets): # type: (list[str]) -> None
|
def export_into_idf_env_json(targets, features): # type: (Optional[list[str]], Optional[list[str]]) -> None
|
||||||
idf_env_json = get_idf_env()
|
idf_env_json = get_idf_env()
|
||||||
targets = list(set(targets + get_user_defined_targets()))
|
targets = list(set(targets + get_requested_targets_and_features()[0])) if targets else None
|
||||||
|
|
||||||
for env in idf_env_json['idfInstalled']:
|
for env in idf_env_json['idfInstalled']:
|
||||||
if env == idf_env_json['idfSelectedId']:
|
if env == idf_env_json['idfSelectedId']:
|
||||||
idf_env_json['idfInstalled'][env]['targets'] = targets
|
update_with = []
|
||||||
|
if targets:
|
||||||
|
update_with += [('targets', targets)]
|
||||||
|
if features:
|
||||||
|
update_with += [('features', features)]
|
||||||
|
idf_env_json['idfInstalled'][env].update(update_with)
|
||||||
break
|
break
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if global_idf_tools_path: # mypy fix for Optional[str] in the next call
|
if global_idf_tools_path: # mypy fix for Optional[str] in the next call
|
||||||
# the directory doesn't exist if this is run on a clean system the first time
|
# the directory doesn't exist if this is run on a clean system the first time
|
||||||
mkdir_p(global_idf_tools_path)
|
mkdir_p(global_idf_tools_path)
|
||||||
with open(os.path.join(global_idf_tools_path, IDF_ENV_FILE), 'w') as w: # type: ignore
|
with open(os.path.join(global_idf_tools_path, IDF_ENV_FILE), 'w') as w:
|
||||||
json.dump(idf_env_json, w, indent=4)
|
json.dump(idf_env_json, w, indent=4)
|
||||||
except (IOError, OSError):
|
except (IOError, OSError):
|
||||||
warn('File {} can not be created. '.format(os.path.join(global_idf_tools_path, IDF_ENV_FILE))) # type: ignore
|
warn('File {} can not be created. '.format(os.path.join(global_idf_tools_path, IDF_ENV_FILE))) # type: ignore
|
||||||
|
|
||||||
|
|
||||||
def clean_targets(targets_str): # type: (str) -> list[str]
|
def add_and_save_targets(targets_str): # type: (str) -> list[str]
|
||||||
targets_from_tools_json = get_all_targets_from_tools_json()
|
targets_from_tools_json = get_all_targets_from_tools_json()
|
||||||
invalid_targets = []
|
invalid_targets = []
|
||||||
|
|
||||||
@ -1072,26 +1083,44 @@ def clean_targets(targets_str): # type: (str) -> list[str]
|
|||||||
raise SystemExit(1)
|
raise SystemExit(1)
|
||||||
# removing duplicates
|
# removing duplicates
|
||||||
targets = list(set(targets))
|
targets = list(set(targets))
|
||||||
export_targets_to_idf_env_json(targets)
|
export_into_idf_env_json(targets, None)
|
||||||
else:
|
else:
|
||||||
export_targets_to_idf_env_json(targets_from_tools_json)
|
export_into_idf_env_json(targets_from_tools_json, None)
|
||||||
return targets
|
return targets
|
||||||
|
|
||||||
|
|
||||||
def get_user_defined_targets(): # type: () -> list[str]
|
def feature_to_requirements_path(feature): # type: (str) -> str
|
||||||
|
return os.path.join(global_idf_path or '', 'requirements.{}.txt'.format(feature))
|
||||||
|
|
||||||
|
|
||||||
|
def add_and_save_features(features_str): # type: (str) -> list[str]
|
||||||
|
_, features = get_requested_targets_and_features()
|
||||||
|
for new_feature_candidate in features_str.split(','):
|
||||||
|
if os.path.isfile(feature_to_requirements_path(new_feature_candidate)):
|
||||||
|
features += [new_feature_candidate]
|
||||||
|
|
||||||
|
features = list(set(features + ['core'])) # remove duplicates
|
||||||
|
export_into_idf_env_json(None, features)
|
||||||
|
return features
|
||||||
|
|
||||||
|
|
||||||
|
def get_requested_targets_and_features(): # type: () -> tuple[list[str], list[str]]
|
||||||
try:
|
try:
|
||||||
with open(os.path.join(global_idf_tools_path, IDF_ENV_FILE), 'r') as idf_env_file: # type: ignore
|
with open(os.path.join(global_idf_tools_path, IDF_ENV_FILE), 'r') as idf_env_file: # type: ignore
|
||||||
idf_env_json = json.load(idf_env_file)
|
idf_env_json = json.load(idf_env_file)
|
||||||
except OSError:
|
except OSError:
|
||||||
# warn('File {} was not found. Installing tools for all esp targets.'.format(os.path.join(global_idf_tools_path, IDF_ENV_FILE))) # type: ignore
|
# warn('File {} was not found. Installing tools for all esp targets.'.format(os.path.join(global_idf_tools_path, IDF_ENV_FILE))) # type: ignore
|
||||||
return []
|
return [], []
|
||||||
|
|
||||||
targets = []
|
targets = []
|
||||||
|
features = []
|
||||||
for env in idf_env_json['idfInstalled']:
|
for env in idf_env_json['idfInstalled']:
|
||||||
if env == idf_env_json['idfSelectedId']:
|
if env == idf_env_json['idfSelectedId']:
|
||||||
targets = idf_env_json['idfInstalled'][env]['targets']
|
env_dict = idf_env_json['idfInstalled'][env]
|
||||||
|
targets = env_dict.get('targets', [])
|
||||||
|
features = env_dict.get('features', [])
|
||||||
break
|
break
|
||||||
return targets
|
return targets, features
|
||||||
|
|
||||||
|
|
||||||
def get_all_targets_from_tools_json(): # type: () -> list[str]
|
def get_all_targets_from_tools_json(): # type: () -> list[str]
|
||||||
@ -1108,7 +1137,7 @@ def get_all_targets_from_tools_json(): # type: () -> list[str]
|
|||||||
|
|
||||||
|
|
||||||
def filter_tools_info(tools_info): # type: (OrderedDict[str, IDFTool]) -> OrderedDict[str,IDFTool]
|
def filter_tools_info(tools_info): # type: (OrderedDict[str, IDFTool]) -> OrderedDict[str,IDFTool]
|
||||||
targets = get_user_defined_targets()
|
targets, _ = get_requested_targets_and_features()
|
||||||
if not targets:
|
if not targets:
|
||||||
return tools_info
|
return tools_info
|
||||||
else:
|
else:
|
||||||
@ -1240,7 +1269,7 @@ def action_export(args): # type: ignore
|
|||||||
export_vars[k] = v
|
export_vars[k] = v
|
||||||
|
|
||||||
current_path = os.getenv('PATH')
|
current_path = os.getenv('PATH')
|
||||||
idf_python_env_path, idf_python_export_path, virtualenv_python = get_python_env_path()
|
idf_python_env_path, idf_python_export_path, virtualenv_python, _ = get_python_env_path()
|
||||||
if os.path.exists(virtualenv_python):
|
if os.path.exists(virtualenv_python):
|
||||||
idf_python_env_path = to_shell_specific_paths([idf_python_env_path])[0]
|
idf_python_env_path = to_shell_specific_paths([idf_python_env_path])[0]
|
||||||
if os.getenv('IDF_PYTHON_ENV_PATH') != idf_python_env_path:
|
if os.getenv('IDF_PYTHON_ENV_PATH') != idf_python_env_path:
|
||||||
@ -1349,7 +1378,7 @@ def action_download(args): # type: ignore
|
|||||||
targets = [] # type: list[str]
|
targets = [] # type: list[str]
|
||||||
# Installing only single tools, no targets are specified.
|
# Installing only single tools, no targets are specified.
|
||||||
if 'required' in tools_spec:
|
if 'required' in tools_spec:
|
||||||
targets = clean_targets(args.targets)
|
targets = add_and_save_targets(args.targets)
|
||||||
|
|
||||||
if args.platform not in PLATFORM_FROM_NAME:
|
if args.platform not in PLATFORM_FROM_NAME:
|
||||||
fatal('unknown platform: {}' % args.platform)
|
fatal('unknown platform: {}' % args.platform)
|
||||||
@ -1409,8 +1438,8 @@ def action_install(args): # type: ignore
|
|||||||
targets = [] # type: list[str]
|
targets = [] # type: list[str]
|
||||||
# Installing only single tools, no targets are specified.
|
# Installing only single tools, no targets are specified.
|
||||||
if 'required' in tools_spec:
|
if 'required' in tools_spec:
|
||||||
targets = clean_targets(args.targets)
|
targets = add_and_save_targets(args.targets)
|
||||||
info('Selected targets are: {}' .format(', '.join(get_user_defined_targets())))
|
info('Selected targets are: {}' .format(', '.join(get_requested_targets_and_features()[0])))
|
||||||
|
|
||||||
if not tools_spec or 'required' in tools_spec:
|
if not tools_spec or 'required' in tools_spec:
|
||||||
# Installing tools for all ESP_targets required by the operating system.
|
# Installing tools for all ESP_targets required by the operating system.
|
||||||
@ -1475,9 +1504,42 @@ def get_wheels_dir(): # type: () -> Optional[str]
|
|||||||
return wheels_dir
|
return wheels_dir
|
||||||
|
|
||||||
|
|
||||||
|
def get_requirements(new_features): # type: (str) -> list[str]
|
||||||
|
features = add_and_save_features(new_features)
|
||||||
|
return [feature_to_requirements_path(feature) for feature in features]
|
||||||
|
|
||||||
|
|
||||||
|
def get_constraints(idf_version): # type: (str) -> str
|
||||||
|
constraint_file = 'espidf.constraints.v{}.txt'.format(idf_version)
|
||||||
|
constraint_path = os.path.join(os.path.expanduser(IDF_TOOLS_PATH_DEFAULT), constraint_file)
|
||||||
|
constraint_url = '/'.join([IDF_DL_URL, constraint_file])
|
||||||
|
temp_path = constraint_path + '.tmp'
|
||||||
|
|
||||||
|
mkdir_p(os.path.dirname(temp_path))
|
||||||
|
|
||||||
|
for _ in range(DOWNLOAD_RETRY_COUNT):
|
||||||
|
download(constraint_url, temp_path)
|
||||||
|
if not os.path.isfile(temp_path):
|
||||||
|
warn('Failed to download {} to {}'.format(constraint_url, temp_path))
|
||||||
|
continue
|
||||||
|
if os.path.isfile(constraint_path):
|
||||||
|
# Windows cannot rename to existing file. It needs to be deleted.
|
||||||
|
os.remove(constraint_path)
|
||||||
|
rename_with_retry(temp_path, constraint_path)
|
||||||
|
return constraint_path
|
||||||
|
|
||||||
|
if os.path.isfile(constraint_path):
|
||||||
|
warn('Failed to download, retry count has expired, using a previously downloaded version')
|
||||||
|
return constraint_path
|
||||||
|
else:
|
||||||
|
fatal('Failed to download, and retry count has expired')
|
||||||
|
raise DownloadError()
|
||||||
|
|
||||||
|
|
||||||
def action_install_python_env(args): # type: ignore
|
def action_install_python_env(args): # type: ignore
|
||||||
|
use_constraints = not args.no_constraints
|
||||||
reinstall = args.reinstall
|
reinstall = args.reinstall
|
||||||
idf_python_env_path, _, virtualenv_python = get_python_env_path()
|
idf_python_env_path, _, virtualenv_python, idf_version = get_python_env_path()
|
||||||
|
|
||||||
is_virtualenv = hasattr(sys, 'real_prefix') or (hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix)
|
is_virtualenv = hasattr(sys, 'real_prefix') or (hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix)
|
||||||
if is_virtualenv and (not os.path.exists(idf_python_env_path) or reinstall):
|
if is_virtualenv and (not os.path.exists(idf_python_env_path) or reinstall):
|
||||||
@ -1549,8 +1611,12 @@ def action_install_python_env(args): # type: ignore
|
|||||||
warn('Found PIP_USER="yes" in the environment. Disabling PIP_USER in this shell to install packages into a virtual environment.')
|
warn('Found PIP_USER="yes" in the environment. Disabling PIP_USER in this shell to install packages into a virtual environment.')
|
||||||
env_copy['PIP_USER'] = 'no'
|
env_copy['PIP_USER'] = 'no'
|
||||||
run_args = [virtualenv_python, '-m', 'pip', 'install', '--no-warn-script-location']
|
run_args = [virtualenv_python, '-m', 'pip', 'install', '--no-warn-script-location']
|
||||||
requirements_txt = os.path.join(global_idf_path, 'requirements.txt')
|
requirements_file_list = get_requirements(args.features)
|
||||||
run_args += ['-r', requirements_txt]
|
for requirement_file in requirements_file_list:
|
||||||
|
run_args += ['-r', requirement_file]
|
||||||
|
if use_constraints:
|
||||||
|
constraint_file = get_constraints(idf_version)
|
||||||
|
run_args += ['--upgrade', '--constraint', constraint_file]
|
||||||
if args.extra_wheels_dir:
|
if args.extra_wheels_dir:
|
||||||
run_args += ['--find-links', args.extra_wheels_dir]
|
run_args += ['--find-links', args.extra_wheels_dir]
|
||||||
if args.no_index:
|
if args.no_index:
|
||||||
@ -1562,10 +1628,58 @@ def action_install_python_env(args): # type: ignore
|
|||||||
if wheels_dir is not None:
|
if wheels_dir is not None:
|
||||||
run_args += ['--find-links', wheels_dir]
|
run_args += ['--find-links', wheels_dir]
|
||||||
|
|
||||||
info('Installing Python packages from {}'.format(requirements_txt))
|
info('Installing Python packages')
|
||||||
|
if use_constraints:
|
||||||
|
info(' Constraint file: {}'.format(constraint_file))
|
||||||
|
info(' Requirement files:')
|
||||||
|
info(os.linesep.join(' - {}'.format(path) for path in requirements_file_list))
|
||||||
subprocess.check_call(run_args, stdout=sys.stdout, stderr=sys.stderr, env=env_copy)
|
subprocess.check_call(run_args, stdout=sys.stdout, stderr=sys.stderr, env=env_copy)
|
||||||
|
|
||||||
|
|
||||||
|
def action_check_python_dependencies(args): # type: ignore
|
||||||
|
use_constraints = not args.no_constraints
|
||||||
|
req_paths = get_requirements('') # no new features -> just detect the existing ones
|
||||||
|
|
||||||
|
_, _, virtualenv_python, idf_version = get_python_env_path()
|
||||||
|
|
||||||
|
if not os.path.isfile(virtualenv_python):
|
||||||
|
fatal('{} doesn\'t exist! Please run the install script or "idf_tools.py install-python-env" in order to '
|
||||||
|
'create it'.format(virtualenv_python))
|
||||||
|
raise SystemExit(1)
|
||||||
|
|
||||||
|
if use_constraints:
|
||||||
|
constr_path = get_constraints(idf_version)
|
||||||
|
info('Constraint file: {}'.format(constr_path))
|
||||||
|
|
||||||
|
info('Requirement files:')
|
||||||
|
info(os.linesep.join(' - {}'.format(path) for path in req_paths))
|
||||||
|
|
||||||
|
info('Python being checked: {}'.format(virtualenv_python))
|
||||||
|
|
||||||
|
# The dependency checker will be invoked with virtualenv_python. idf_tools.py could have been invoked with a
|
||||||
|
# different one, therefore, importing is not a suitable option.
|
||||||
|
dep_check_cmd = [virtualenv_python,
|
||||||
|
os.path.join(global_idf_path,
|
||||||
|
'tools',
|
||||||
|
'check_python_dependencies.py')]
|
||||||
|
|
||||||
|
if use_constraints:
|
||||||
|
dep_check_cmd += ['-c', constr_path]
|
||||||
|
|
||||||
|
for req_path in req_paths:
|
||||||
|
dep_check_cmd += ['-r', req_path]
|
||||||
|
|
||||||
|
try:
|
||||||
|
ret = subprocess.run(dep_check_cmd)
|
||||||
|
if ret and ret.returncode:
|
||||||
|
# returncode is a negative number and system exit output is usually expected be positive.
|
||||||
|
raise SystemExit(-ret.returncode)
|
||||||
|
except FileNotFoundError:
|
||||||
|
# Python environment not yet created
|
||||||
|
fatal('Requirements are not satisfied!')
|
||||||
|
raise SystemExit(1)
|
||||||
|
|
||||||
|
|
||||||
def action_add_version(args): # type: ignore
|
def action_add_version(args): # type: ignore
|
||||||
tools_info = load_tools_info()
|
tools_info = load_tools_info()
|
||||||
tool_name = args.tool
|
tool_name = args.tool
|
||||||
@ -1771,6 +1885,11 @@ def main(argv): # type: (list[str]) -> None
|
|||||||
'to use during installation')
|
'to use during installation')
|
||||||
install_python_env.add_argument('--extra-wheels-url', help='Additional URL with wheels', default='https://dl.espressif.com/pypi')
|
install_python_env.add_argument('--extra-wheels-url', help='Additional URL with wheels', default='https://dl.espressif.com/pypi')
|
||||||
install_python_env.add_argument('--no-index', help='Work offline without retrieving wheels index')
|
install_python_env.add_argument('--no-index', help='Work offline without retrieving wheels index')
|
||||||
|
install_python_env.add_argument('--features', default='core', help='A comma separated list of desired features for installing.'
|
||||||
|
' It defaults to installing just the core funtionality.')
|
||||||
|
install_python_env.add_argument('--no-constraints', action='store_true', default=False,
|
||||||
|
help='Disable constraint settings. Use with care and only when you want to manage '
|
||||||
|
'package versions by yourself.')
|
||||||
|
|
||||||
if IDF_MAINTAINER:
|
if IDF_MAINTAINER:
|
||||||
add_version = subparsers.add_parser('add-version', help='Add or update download info for a version')
|
add_version = subparsers.add_parser('add-version', help='Add or update download info for a version')
|
||||||
@ -1790,6 +1909,12 @@ def main(argv): # type: (list[str]) -> None
|
|||||||
help='Output file name')
|
help='Output file name')
|
||||||
gen_doc.add_argument('--heading-underline-char', help='Character to use when generating RST sections', default='~')
|
gen_doc.add_argument('--heading-underline-char', help='Character to use when generating RST sections', default='~')
|
||||||
|
|
||||||
|
check_python_dependencies = subparsers.add_parser('check-python-dependencies',
|
||||||
|
help='Check that all required Python packages are installed.')
|
||||||
|
check_python_dependencies.add_argument('--no-constraints', action='store_true', default=False,
|
||||||
|
help='Disable constraint settings. Use with care and only when you want '
|
||||||
|
'to manage package versions by yourself.')
|
||||||
|
|
||||||
args = parser.parse_args(argv)
|
args = parser.parse_args(argv)
|
||||||
|
|
||||||
if args.action is None:
|
if args.action is None:
|
||||||
|
70
tools/install_util.py
Normal file
70
tools/install_util.py
Normal file
@ -0,0 +1,70 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
|
||||||
|
# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
|
||||||
|
# This script is used from the $IDF_PATH/install.* scripts. This way the argument parsing can be done at one place and
|
||||||
|
# doesn't have to be implemented for all shells.
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
from itertools import chain
|
||||||
|
|
||||||
|
try:
|
||||||
|
import python_version_checker
|
||||||
|
|
||||||
|
# check the Python version before it will fail with an exception on syntax or package incompatibility.
|
||||||
|
python_version_checker.check()
|
||||||
|
except RuntimeError as e:
|
||||||
|
print(e)
|
||||||
|
raise SystemExit(1)
|
||||||
|
|
||||||
|
|
||||||
|
def action_extract_features(args: str) -> None:
|
||||||
|
"""
|
||||||
|
Command line arguments starting with "--enable-" are features. This function selects those and prints them.
|
||||||
|
"""
|
||||||
|
features = ['core'] # "core" features should be always installed
|
||||||
|
|
||||||
|
if args:
|
||||||
|
arg_prefix = '--enable-'
|
||||||
|
features += [arg[len(arg_prefix):] for arg in args.split() if arg.startswith(arg_prefix)]
|
||||||
|
|
||||||
|
print(','.join(features))
|
||||||
|
|
||||||
|
|
||||||
|
def action_extract_targets(args: str) -> None:
|
||||||
|
"""
|
||||||
|
Command line arguments starting with "esp" are chip targets. This function selects those and prints them.
|
||||||
|
"""
|
||||||
|
target_sep = ','
|
||||||
|
targets = []
|
||||||
|
|
||||||
|
if args:
|
||||||
|
target_args = (arg for arg in args.split() if arg.lower().startswith('esp'))
|
||||||
|
# target_args can be comma-separated lists of chip targets
|
||||||
|
|
||||||
|
targets = list(chain.from_iterable(commalist.split(target_sep) for commalist in target_args))
|
||||||
|
|
||||||
|
print(target_sep.join(targets or ['all']))
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
|
||||||
|
subparsers = parser.add_subparsers(dest='action', required=True)
|
||||||
|
extract = subparsers.add_parser('extract', help='Process arguments and extract part of it')
|
||||||
|
|
||||||
|
extract.add_argument('type', choices=['targets', 'features'])
|
||||||
|
extract.add_argument('str-to-parse', nargs='?')
|
||||||
|
|
||||||
|
args, unknown_args = parser.parse_known_args()
|
||||||
|
# standalone "--enable-" won't be included into str-to-parse
|
||||||
|
|
||||||
|
action_func = globals()['action_{}_{}'.format(args.action, args.type)]
|
||||||
|
str_to_parse = vars(args)['str-to-parse'] or ''
|
||||||
|
action_func(' '.join(chain([str_to_parse], unknown_args)))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
71
tools/test_idf_tools/test_idf_tools_python_env.py
Normal file
71
tools/test_idf_tools/test_idf_tools_python_env.py
Normal file
@ -0,0 +1,71 @@
|
|||||||
|
# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
|
||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import unittest
|
||||||
|
from typing import List
|
||||||
|
|
||||||
|
try:
|
||||||
|
import idf_tools
|
||||||
|
except ImportError:
|
||||||
|
sys.path.append('..')
|
||||||
|
import idf_tools
|
||||||
|
|
||||||
|
IDF_PATH = os.environ.get('IDF_PATH', '../..')
|
||||||
|
TOOLS_DIR = os.environ.get('IDF_TOOLS_PATH') or os.path.expanduser(idf_tools.IDF_TOOLS_PATH_DEFAULT)
|
||||||
|
PYTHON_DIR = os.path.join(TOOLS_DIR, 'python_env')
|
||||||
|
REQ_SATISFIED = 'Python requirements are satisfied'
|
||||||
|
REQ_CORE = '- {}/requirements.core.txt'.format(IDF_PATH)
|
||||||
|
REQ_GDBGUI = '- {}/requirements.gdbgui.txt'.format(IDF_PATH)
|
||||||
|
CONSTR = 'Constraint file: {}/espidf.constraints'.format(TOOLS_DIR)
|
||||||
|
|
||||||
|
|
||||||
|
class TestPythonInstall(unittest.TestCase):
|
||||||
|
|
||||||
|
def setUp(self): # type: () -> None
|
||||||
|
if os.path.isdir(PYTHON_DIR):
|
||||||
|
shutil.rmtree(PYTHON_DIR)
|
||||||
|
if os.path.isfile(os.path.join(TOOLS_DIR, 'idf-env.json')):
|
||||||
|
os.remove(os.path.join(TOOLS_DIR, 'idf-env.json'))
|
||||||
|
|
||||||
|
def run_idf_tools(self, extra_args): # type: (List[str]) -> str
|
||||||
|
args = [sys.executable, '../idf_tools.py'] + extra_args
|
||||||
|
ret = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=120)
|
||||||
|
return ret.stdout.decode('utf-8', 'ignore')
|
||||||
|
|
||||||
|
def test_default_arguments(self): # type: () -> None
|
||||||
|
output = self.run_idf_tools(['check-python-dependencies'])
|
||||||
|
self.assertNotIn(REQ_SATISFIED, output)
|
||||||
|
self.assertIn('bin/python doesn\'t exist', output)
|
||||||
|
|
||||||
|
output = self.run_idf_tools(['install-python-env'])
|
||||||
|
self.assertIn(CONSTR, output)
|
||||||
|
self.assertIn(REQ_CORE, output)
|
||||||
|
self.assertNotIn(REQ_GDBGUI, output)
|
||||||
|
|
||||||
|
output = self.run_idf_tools(['check-python-dependencies'])
|
||||||
|
self.assertIn(REQ_SATISFIED, output)
|
||||||
|
|
||||||
|
def test_opt_argument(self): # type: () -> None
|
||||||
|
output = self.run_idf_tools(['install-python-env', '--features', 'gdbgui'])
|
||||||
|
self.assertIn(CONSTR, output)
|
||||||
|
self.assertIn(REQ_CORE, output)
|
||||||
|
self.assertIn(REQ_GDBGUI, output)
|
||||||
|
|
||||||
|
output = self.run_idf_tools(['install-python-env'])
|
||||||
|
# The gdbgui should be installed as well because the feature is is stored in the JSON file
|
||||||
|
self.assertIn(CONSTR, output)
|
||||||
|
self.assertIn(REQ_CORE, output)
|
||||||
|
self.assertIn(REQ_GDBGUI, output)
|
||||||
|
|
||||||
|
def test_no_constraints(self): # type: () -> None
|
||||||
|
output = self.run_idf_tools(['install-python-env', '--no-constraints'])
|
||||||
|
self.assertNotIn(CONSTR, output)
|
||||||
|
self.assertIn(REQ_CORE, output)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
Loading…
Reference in New Issue
Block a user