fix(tools/python_dep_check): replace deprecated pkg_resources with importlib

Closes https://github.com/espressif/esp-idf/issues/11712
This commit is contained in:
Peter Dragun 2023-06-27 16:07:23 +02:00
parent 82fde1e823
commit 77429e3ef7
4 changed files with 46 additions and 21 deletions

View File

@ -1,6 +1,6 @@
#!/usr/bin/env python #!/usr/bin/env python
# #
# SPDX-FileCopyrightText: 2018-2022 Espressif Systems (Shanghai) CO LTD # SPDX-FileCopyrightText: 2018-2023 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
import argparse import argparse
@ -9,13 +9,22 @@ import re
import sys import sys
try: try:
import pkg_resources from packaging.requirements import Requirement
from packaging.version import Version
except ImportError: except ImportError:
print('pkg_resources cannot be imported. The most common cause is a missing pip or setuptools package. ' print('packaging cannot be imported. '
'If you\'ve installed a custom Python then these packages are provided separately and have to be installed as well. ' 'If you\'ve installed a custom Python then this package is provided separately and have to be installed as well. '
'Please refer to the Get Started section of the ESP-IDF Programming Guide for setting up the required packages.') 'Please refer to the Get Started section of the ESP-IDF Programming Guide for setting up the required packages.')
sys.exit(1) sys.exit(1)
try:
from importlib.metadata import PackageNotFoundError, requires
from importlib.metadata import version as get_version
except ImportError:
# compatibility for python <=3.7
from importlib_metadata import PackageNotFoundError, requires # type: ignore
from importlib_metadata import version as get_version # type: ignore
try: try:
from typing import Set from typing import Set
except ImportError: except ImportError:
@ -59,18 +68,31 @@ if __name__ == '__main__':
if not name_m: if not name_m:
print('Malformed input. Cannot find name in {}'.format(con)) print('Malformed input. Cannot find name in {}'.format(con))
sys.exit(1) sys.exit(1)
constr_dict[name_m[0]] = con constr_dict[name_m[0]] = con.partition(' #')[0] # remove comments
not_satisfied = [] # in string form which will be printed not_satisfied = [] # in string form which will be printed
# already_checked set is used in order to avoid circular checks which would cause looping. # already_checked set is used in order to avoid circular checks which would cause looping.
already_checked = set() # type: Set[pkg_resources.Requirement] already_checked = set() # type: Set[Requirement]
# required_set contains package names in string form without version constraints. If the package has a constraint # required_set contains package names in string form without version constraints. If the package has a constraint
# specification (package name + version requirement) then use that instead. new_req_list is used to store # specification (package name + version requirement) then use that instead. new_req_list is used to store
# requirements to be checked on each level of breath-first-search of the package dependency tree. The initial # requirements to be checked on each level of breath-first-search of the package dependency tree. The initial
# version is the direct dependencies deduced from the requirements arguments of the script. # version is the direct dependencies deduced from the requirements arguments of the script.
new_req_list = [pkg_resources.Requirement.parse(constr_dict.get(i, i)) for i in required_set] new_req_list = [Requirement(constr_dict.get(i, i)) for i in required_set]
def version_check(requirement: Requirement) -> None:
# compare installed version with required
version = Version(get_version(requirement.name))
if version.base_version not in requirement.specifier:
not_satisfied.append(f"Requirement '{requirement}' was not met. Installed version: {version}")
# evaluate markers and check versions of direct requirements
for req in new_req_list[:]:
if not req.marker or req.marker.evaluate():
version_check(req)
else:
new_req_list.remove(req)
while new_req_list: while new_req_list:
req_list = new_req_list req_list = new_req_list
@ -78,20 +100,23 @@ if __name__ == '__main__':
already_checked.update(req_list) already_checked.update(req_list)
for requirement in req_list: # check one level of the dependency tree for requirement in req_list: # check one level of the dependency tree
try: try:
dependency_requirements = set(pkg_resources.get_distribution(requirement).requires()) dependency_requirements = set()
extras = list(requirement.extras) or ['']
for name in requires(requirement.name) or []:
sub_req = Requirement(name)
# check extras e.g. esptool[hsm]
for extra in extras:
# evaluate markers if present
if not sub_req.marker or sub_req.marker.evaluate(environment={'extra': extra}):
dependency_requirements.add(sub_req)
version_check(sub_req)
# dependency_requirements are the direct dependencies of "requirement". They belong to the next level # dependency_requirements are the direct dependencies of "requirement". They belong to the next level
# of the dependency tree. They will be checked only if they haven't been already. Note that the # of the dependency tree. They will be checked only if they haven't been already. Note that the
# version is taken into account as well because packages can have different requirements for a given # version is taken into account as well because packages can have different requirements for a given
# Python package. The dependencies need to be checked for all of them because they can be different. # Python package. The dependencies need to be checked for all of them because they can be different.
new_req_list.extend(dependency_requirements - already_checked) new_req_list.extend(dependency_requirements - already_checked)
except pkg_resources.ResolutionError as e: except PackageNotFoundError as e:
not_satisfied.append(' - '.join([str(requirement), str(e)])) not_satisfied.append(f"'{e}' - was not found and is required by the application")
except IndexError:
# If the requirement is not installed because of a marker (requirement.marker), for example different
# operating system or python version, then pkg_resources.get_distribution() will fail with IndexError.
# We could avoid this by checking packaging.markers.Marker(requirement.marker).evaluate() but it would
# add dependency on packaging.
pass
if len(not_satisfied) > 0: if len(not_satisfied) > 0:
print('The following Python requirements are not satisfied:') print('The following Python requirements are not satisfied:')

View File

@ -64,8 +64,7 @@ build_stage2() {
--build-log ${BUILD_LOG_CMAKE} \ --build-log ${BUILD_LOG_CMAKE} \
--size-file size.json \ --size-file size.json \
--collect-size-info size_info.txt \ --collect-size-info size_info.txt \
--default-build-targets esp32,esp32s2,esp32s3,esp32c2,esp32c3,esp32c6,esp32h2 \ --default-build-targets esp32 esp32s2 esp32s3 esp32c2 esp32c3 esp32c6 esp32h2
--ignore-warning-str "DeprecationWarning: pkg_resources is deprecated as an API"
} }
build_stage1() { build_stage1() {
@ -79,8 +78,7 @@ build_stage1() {
--build-log ${BUILD_LOG_CMAKE} \ --build-log ${BUILD_LOG_CMAKE} \
--size-file size.json \ --size-file size.json \
--collect-size-info size_info.txt \ --collect-size-info size_info.txt \
--default-build-targets esp32,esp32s2,esp32s3,esp32c2,esp32c3,esp32c6,esp32h2 \ --default-build-targets esp32 esp32s2 esp32s3 esp32c2 esp32c3 esp32c6 esp32h2
--ignore-warning-str "DeprecationWarning: pkg_resources is deprecated as an API"
} }
# Default arguments # Default arguments

View File

@ -15,4 +15,3 @@ CryptographyDeprecationWarning
Warning: \d+/\d+ app partitions are too small for binary Warning: \d+/\d+ app partitions are too small for binary
CMake Deprecation Warning at main/lib/tinyxml2/CMakeLists\.txt:11 \(cmake_policy\) CMake Deprecation Warning at main/lib/tinyxml2/CMakeLists\.txt:11 \(cmake_policy\)
The smallest .+ partition is nearly full \(\d+% free space left\)! The smallest .+ partition is nearly full \(\d+% free space left\)!
DeprecationWarning: pkg_resources is deprecated as an API

View File

@ -1,6 +1,9 @@
# Python package requirements for ESP-IDF. These are the so called core features which are installed in all systems. # Python package requirements for ESP-IDF. These are the so called core features which are installed in all systems.
setuptools setuptools
packaging
# importlib_metadata: is part of python3.8 and newer as importlib.metadata
importlib_metadata; python_version < "3.8"
click click
pyserial pyserial
cryptography cryptography