From f0e8966ca52fe271c15d67673a405803394a94b3 Mon Sep 17 00:00:00 2001 From: Marek Fiala Date: Tue, 6 Aug 2024 13:46:53 +0200 Subject: [PATCH 1/4] feat(tools): Use custom directory in failing pytest --- tools/test_build_system/test_kconfig.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/test_build_system/test_kconfig.py b/tools/test_build_system/test_kconfig.py index ddcdba2132..181533c790 100644 --- a/tools/test_build_system/test_kconfig.py +++ b/tools/test_build_system/test_kconfig.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import logging import os @@ -9,7 +9,9 @@ from pathlib import Path from typing import Iterator import pytest -from test_build_system_helpers import IdfPyFunc, append_to_file, file_contains +from test_build_system_helpers import append_to_file +from test_build_system_helpers import file_contains +from test_build_system_helpers import IdfPyFunc @contextmanager @@ -77,6 +79,7 @@ def test_kconfig_deprecated_options(idf_py: IdfPyFunc, test_app_copy: Path) -> N @pytest.mark.usefixtures('idf_copy') +@pytest.mark.idf_copy('test_kconfig_variouos_options') def test_kconfig_multiple_and_target_specific_options(idf_py: IdfPyFunc, test_app_copy: Path) -> None: idf_path = Path(os.environ['IDF_PATH']) From ccbb415be8b1e9b52b1b5caecd5f1054575b7e36 Mon Sep 17 00:00:00 2001 From: Marek Fiala Date: Thu, 15 Aug 2024 11:53:48 +0200 Subject: [PATCH 2/4] feat(tools): add pytest mark idf_copy_with_space --- tools/test_build_system/README.md | 2 ++ tools/test_build_system/conftest.py | 4 ++++ tools/test_build_system/pytest.ini | 2 ++ tools/test_build_system/test_kconfig.py | 2 +- tools/test_build_system/test_spaces.py | 12 ++++++------ 5 files changed, 15 insertions(+), 7 deletions(-) diff --git a/tools/test_build_system/README.md b/tools/test_build_system/README.md index 0563d75ffa..30fa0e7eec 100644 --- a/tools/test_build_system/README.md +++ b/tools/test_build_system/README.md @@ -114,6 +114,8 @@ Note, `default_idf_env` sets up the environment based on the `IDF_PATH` environm Copies IDF from `IDF_PATH` into a new temporary directory. `@pytest.mark.idf_copy('name prefix')` can be used to specify the name prefix of the temporary directory. +The marker `@pytest.mark.idf_copy_with_space` can be used in combination with `idf_copy` fixture to create a temporary directory with a space in the name. + For the duration of the test, `IDF_PATH` environment variable is set to the newly created copy. ```python diff --git a/tools/test_build_system/conftest.py b/tools/test_build_system/conftest.py index 2607f06938..0b8472fccc 100644 --- a/tools/test_build_system/conftest.py +++ b/tools/test_build_system/conftest.py @@ -157,6 +157,10 @@ def test_git_template_app(func_work_dir: Path, request: FixtureRequest) -> typin @pytest.fixture def idf_copy(func_work_dir: Path, request: FixtureRequest) -> typing.Generator[Path, None, None]: copy_to = request.node.name + '_idf' + # allow overriding the destination via pytest.mark.idf_copy_with_space so the destination contain space + mark_with_space = request.node.get_closest_marker('idf_copy_with_space') + if mark_with_space: + copy_to = request.node.name + ' idf' # allow overriding the destination via pytest.mark.idf_copy() mark = request.node.get_closest_marker('idf_copy') diff --git a/tools/test_build_system/pytest.ini b/tools/test_build_system/pytest.ini index 0db692d048..433952425a 100644 --- a/tools/test_build_system/pytest.ini +++ b/tools/test_build_system/pytest.ini @@ -13,7 +13,9 @@ junit_family = xunit1 junit_logging = stdout junit_log_passing_tests = False +## !! When adding new markers, don't forget to update also the tools\test_build_system\README.md !! markers = test_app_copy: specify relative path of the app to copy, and the prefix of the destination directory name idf_copy: specify the prefix of the destination directory where IDF should be copied + idf_copy_with_space: ensures that destination directory where IDF is copied contain space force_temp_work_dir: force temporary folder as the working directory diff --git a/tools/test_build_system/test_kconfig.py b/tools/test_build_system/test_kconfig.py index 181533c790..5ed0f33f0c 100644 --- a/tools/test_build_system/test_kconfig.py +++ b/tools/test_build_system/test_kconfig.py @@ -79,7 +79,7 @@ def test_kconfig_deprecated_options(idf_py: IdfPyFunc, test_app_copy: Path) -> N @pytest.mark.usefixtures('idf_copy') -@pytest.mark.idf_copy('test_kconfig_variouos_options') +@pytest.mark.idf_copy('test_kconfig_various_options') def test_kconfig_multiple_and_target_specific_options(idf_py: IdfPyFunc, test_app_copy: Path) -> None: idf_path = Path(os.environ['IDF_PATH']) diff --git a/tools/test_build_system/test_spaces.py b/tools/test_build_system/test_spaces.py index 5a21214997..d4c540179f 100644 --- a/tools/test_build_system/test_spaces.py +++ b/tools/test_build_system/test_spaces.py @@ -21,7 +21,7 @@ def clean_app_dir(app_path: Path) -> None: shutil.rmtree(app_path / 'build', ignore_errors=True) -@pytest.mark.idf_copy('esp idf with spaces') +@pytest.mark.idf_copy_with_space def test_spaces_bundle1(idf_copy: Path) -> None: logging.info('Running test spaces bundle 1') # test spiffsgen @@ -32,7 +32,7 @@ def test_spaces_bundle1(idf_copy: Path) -> None: run_idf_py('-DIDF_TARGET=esp32s2', 'build', workdir=(idf_copy / 'examples' / 'system' / 'ulp' / 'ulp_riscv' / 'gpio')) -@pytest.mark.idf_copy('esp idf with spaces') +@pytest.mark.idf_copy_with_space def test_spaces_bundle2(idf_copy: Path) -> None: logging.info('Running test spaces bundle 2') # test flash_encryption @@ -41,7 +41,7 @@ def test_spaces_bundle2(idf_copy: Path) -> None: run_idf_py('build', workdir=(idf_copy / 'examples' / 'protocols' / 'https_x509_bundle')) -@pytest.mark.idf_copy('esp idf with spaces') +@pytest.mark.idf_copy_with_space def test_spaces_bundle3(idf_copy: Path) -> None: logging.info('Running test spaces bundle 3') secure_boot_app_path = (idf_copy / 'tools' / 'test_apps' / 'security' / 'secure_boot') @@ -67,7 +67,7 @@ def test_spaces_bundle3(idf_copy: Path) -> None: # Dummy parameter with a space in it, used so that the test directory name contains a space pytest.param('test spaces') ]) -@pytest.mark.idf_copy('esp idf with spaces') +@pytest.mark.idf_copy_with_space @pytest.mark.usefixtures('idf_copy') def test_spaces_bundle4(dummy_: str, idf_py: IdfPyFunc, test_app_copy: Path) -> None: logging.info(f'Running test spaces bundle 4 in {test_app_copy}') @@ -83,7 +83,7 @@ def test_spaces_bundle4(dummy_: str, idf_py: IdfPyFunc, test_app_copy: Path) -> @pytest.mark.skipif(sys.platform == 'win32', reason='Unix test') -@pytest.mark.idf_copy('esp idf with spaces') +@pytest.mark.idf_copy_with_space def test_install_export_unix(idf_copy: Path) -> None: logging.info('install and export setup scripts') env = dict(**os.environ) @@ -100,7 +100,7 @@ def test_install_export_unix(idf_copy: Path) -> None: @pytest.mark.skipif(sys.platform != 'win32', reason='Windows test') -@pytest.mark.idf_copy('esp idf with spaces') +@pytest.mark.idf_copy_with_space def test_install_export_win(idf_copy: Path) -> None: logging.info('install and export setup scripts') env = dict(**os.environ) From e95aff7ca9c4bc76a1b552d82acddb840b10d861 Mon Sep 17 00:00:00 2001 From: Marek Fiala Date: Fri, 16 Aug 2024 15:31:30 +0200 Subject: [PATCH 3/4] feat(tools): sanitize all fixtures using test names The parametrization include `[` and `]` in names. This is not possible to use within path. Expand the sanitization into all fixtures using request.node.name. --- tools/test_build_system/conftest.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tools/test_build_system/conftest.py b/tools/test_build_system/conftest.py index 0b8472fccc..74b96ae7a1 100644 --- a/tools/test_build_system/conftest.py +++ b/tools/test_build_system/conftest.py @@ -133,7 +133,9 @@ def test_app_copy(func_work_dir: Path, request: FixtureRequest) -> typing.Genera @pytest.fixture def test_git_template_app(func_work_dir: Path, request: FixtureRequest) -> typing.Generator[Path, None, None]: - copy_to = request.node.name + '_app' + # sanitize test name in case pytest.mark.parametrize was used + test_name_sanitized = request.node.name.replace('[', '_').replace(']', '') + copy_to = test_name_sanitized + '_app' path_to = func_work_dir / copy_to logging.debug(f'cloning git-template app to {path_to}') @@ -156,11 +158,13 @@ def test_git_template_app(func_work_dir: Path, request: FixtureRequest) -> typin @pytest.fixture def idf_copy(func_work_dir: Path, request: FixtureRequest) -> typing.Generator[Path, None, None]: - copy_to = request.node.name + '_idf' + # sanitize test name in case pytest.mark.parametrize was used + test_name_sanitized = request.node.name.replace('[', '_').replace(']', '') + copy_to = test_name_sanitized + '_idf' # allow overriding the destination via pytest.mark.idf_copy_with_space so the destination contain space mark_with_space = request.node.get_closest_marker('idf_copy_with_space') if mark_with_space: - copy_to = request.node.name + ' idf' + copy_to = test_name_sanitized + ' idf' # allow overriding the destination via pytest.mark.idf_copy() mark = request.node.get_closest_marker('idf_copy') From 364f777f96e08494e609c1ec49eaaf36d2843f96 Mon Sep 17 00:00:00 2001 From: Marek Fiala Date: Wed, 21 Aug 2024 14:33:59 +0200 Subject: [PATCH 4/4] feat(tools): Allow test_spaces_bundle4 to fail on Win --- tools/test_build_system/test_spaces.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/test_build_system/test_spaces.py b/tools/test_build_system/test_spaces.py index d4c540179f..6d4ca8c981 100644 --- a/tools/test_build_system/test_spaces.py +++ b/tools/test_build_system/test_spaces.py @@ -62,6 +62,7 @@ def test_spaces_bundle3(idf_copy: Path) -> None: workdir=secure_boot_app_path) +@pytest.mark.xfail(sys.platform == 'win32', reason='Bug with reproducible build') # Use this bundle for tests which can be done with the default build_test_app @pytest.mark.parametrize('dummy_', [ # Dummy parameter with a space in it, used so that the test directory name contains a space