From a17ec488d9c644a37e3610445e083099568bcfa7 Mon Sep 17 00:00:00 2001 From: Peter Dragun Date: Wed, 14 Feb 2024 12:52:52 +0100 Subject: [PATCH] fix(tools): catch more general errors in python dependency checker --- .gitlab/ci/rules.yml | 2 ++ tools/check_python_dependencies.py | 19 +++++++++++-------- tools/ci/exclude_check_tools_files.txt | 1 - .../test_idf_tools_python_env.py | 9 ++++----- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 5b14bb95d1..e0937a666b 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -152,6 +152,8 @@ - "tools/split_paths_by_spaces.py" + - "tools/check_python_dependencies.py" + .patterns-windows: &patterns-windows - "tools/windows/**/*" diff --git a/tools/check_python_dependencies.py b/tools/check_python_dependencies.py index ea0e45e640..945d8e7a0d 100755 --- a/tools/check_python_dependencies.py +++ b/tools/check_python_dependencies.py @@ -1,8 +1,7 @@ #!/usr/bin/env python # -# SPDX-FileCopyrightText: 2018-2023 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2018-2024 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 - import argparse import os import re @@ -18,11 +17,11 @@ except ImportError: sys.exit(1) try: - from importlib.metadata import PackageNotFoundError, requires + from importlib.metadata import requires from importlib.metadata import version as get_version except ImportError: # compatibility for python <=3.7 - from importlib_metadata import PackageNotFoundError, requires # type: ignore + from importlib_metadata import requires # type: ignore from importlib_metadata import version as get_version # type: ignore try: @@ -92,8 +91,10 @@ if __name__ == '__main__': if not req.marker or req.marker.evaluate(): try: version_check(req) - except PackageNotFoundError as e: - not_satisfied.append(f"'{e}' - was not found and is required by the application") + except Exception as e: + # Catch general exception, because get_version may return None (https://github.com/python/cpython/issues/91216) + # log package name alongside the error message for easier debugging + not_satisfied.append(f"Error while checking requirement '{req}'. Package was not found and is required by the application: {e}") new_req_list.remove(req) else: new_req_list.remove(req) @@ -120,8 +121,10 @@ if __name__ == '__main__': # version is taken into account as well because packages can have different requirements for a given # Python package. The dependencies need to be checked for all of them because they can be different. new_req_list.extend(dependency_requirements - already_checked) - except PackageNotFoundError as e: - not_satisfied.append(f"'{e}' - was not found and is required by the application") + except Exception as e: + # Catch general exception, because get_version may return None (https://github.com/python/cpython/issues/91216) + # log package name alongside the error message for easier debugging + not_satisfied.append(f"Error while checking requirement '{req}'. Package was not found and is required by the application: {e}") if len(not_satisfied) > 0: print('The following Python requirements are not satisfied:') diff --git a/tools/ci/exclude_check_tools_files.txt b/tools/ci/exclude_check_tools_files.txt index 3b54e2c58f..f74fe7bf91 100644 --- a/tools/ci/exclude_check_tools_files.txt +++ b/tools/ci/exclude_check_tools_files.txt @@ -9,7 +9,6 @@ tools/ci/get_all_test_results.py tools/ci/idf_unity_tester.py tools/gdb_panic_server.py tools/check_term.py -tools/check_python_dependencies.py tools/python_version_checker.py tools/generate_debug_prefix_map.py tools/ci/checkout_project_ref.py diff --git a/tools/test_idf_tools/test_idf_tools_python_env.py b/tools/test_idf_tools/test_idf_tools_python_env.py index 096789cfcd..4c5de39c26 100644 --- a/tools/test_idf_tools/test_idf_tools_python_env.py +++ b/tools/test_idf_tools/test_idf_tools_python_env.py @@ -1,6 +1,5 @@ -# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 - # NOTE: unittest is by default sorting tests based on their names, # so the order if which the tests are started may be different from # the order in which they are defined. Please make sure all tests @@ -9,7 +8,6 @@ # If test needs to change global state, it should return it to the # original state after it's finished. For more information please see # https://docs.python.org/3/library/unittest.html#organizing-test-code - import inspect import os import shutil @@ -30,7 +28,8 @@ TOOLS_DIR = os.environ.get('IDF_TOOLS_PATH') or os.path.expanduser(idf_tools.IDF PYTHON_DIR = os.path.join(TOOLS_DIR, 'python_env') PYTHON_DIR_BACKUP = tempfile.mkdtemp() REQ_SATISFIED = 'Python requirements are satisfied' -REQ_MISSING = "{}' - was not found and is required by the application" +# Python3.7 has a different error message that includes also "No package metadata was found for" +REQ_MISSING = r'.*Package was not found and is required by the application: (No package metadata was found for )?{}.*' REQ_CORE = '- {}'.format(os.path.join(IDF_PATH, 'tools', 'requirements', 'requirements.core.txt')) REQ_GDBGUI = '- {}'.format(os.path.join(IDF_PATH, 'tools', 'requirements', 'requirements.gdbgui.txt')) CONSTR = 'Constraint file: {}/espidf.constraints'.format(TOOLS_DIR) @@ -347,7 +346,7 @@ class TestCheckPythonDependencies(BasePythonInstall): # check-python-dependencies should fail as the package was not installed yet output = self.run_idf_tools(['check-python-dependencies']) - self.assertIn(REQ_MISSING.format('foopackage'), output) + self.assertRegex(output, REQ_MISSING.format('foopackage')) self.assertNotIn(REQ_SATISFIED, output) def test_dev_version(self): # type: () -> None