Merge branch 'feature/fix_build_system_win_tests' into 'master'

feat(tools): Run skipped build system pytests on Win

Closes IDF-8582 and IDFCI-896

See merge request espressif/esp-idf!28800

.gitlab/ci/test-win.yml

@@ -73,12 +73,13 @@ test_tools_win:
     - . .\export.ps1
     - python "${SUBMODULE_FETCH_TOOL}" -s "all"
     - cd ${IDF_PATH}\tools\test_build_system
-    - pytest --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml
+    - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml
 
 pytest_build_system_win:
   extends:
     - .test_build_system_template_win
     - .rules:labels:windows_pytest_build_system
+  parallel: 2
   needs: []
   tags:
     - windows-build

tools/test_build_system/conftest.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 import datetime
 import logging
@@ -11,7 +11,11 @@ from tempfile import mkdtemp
 
 import pytest
 from _pytest.fixtures import FixtureRequest
-from test_build_system_helpers import EXT_IDF_PATH, EnvDict, IdfPyFunc, get_idf_build_env, run_idf_py
+from test_build_system_helpers import EnvDict
+from test_build_system_helpers import EXT_IDF_PATH
+from test_build_system_helpers import get_idf_build_env
+from test_build_system_helpers import IdfPyFunc
+from test_build_system_helpers import run_idf_py
 
 
 # Pytest hook used to check if the test has passed or failed, from a fixture.
@@ -161,7 +165,8 @@ def idf_copy(func_work_dir: Path, request: FixtureRequest) -> typing.Generator[P
     ignore = shutil.ignore_patterns(
         path_to.name,
         # also ignore the build directories which may be quite large
-        '**/build')
+        # plus ignore .git since it is causing trouble when removing on Windows
+        '**/build', '.git')
 
     logging.debug(f'copying {path_from} to {path_to}')
     shutil.copytree(path_from, path_to, ignore=ignore, symlinks=True)

tools/test_build_system/test_cmake.py

@@ -1,22 +1,25 @@
-# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 import logging
 import os
 import re
 import shutil
-import sys
 from pathlib import Path
 
-import pytest
-from test_build_system_helpers import EnvDict, IdfPyFunc, append_to_file, file_contains, run_cmake, run_cmake_and_build
+from test_build_system_helpers import append_to_file
+from test_build_system_helpers import EnvDict
+from test_build_system_helpers import file_contains
+from test_build_system_helpers import IdfPyFunc
+from test_build_system_helpers import run_cmake
+from test_build_system_helpers import run_cmake_and_build
 
 
-@pytest.mark.skipif(sys.platform == 'win32', reason='Failing on Windows runner. TODO')
 def test_build_custom_cmake_project(test_app_copy: Path) -> None:
+    # Test is compatible with any target. Random targets in the list are selected for performance reasons
     for target in ['esp32', 'esp32s3', 'esp32c6', 'esp32h2']:
         logging.info(f'Test build ESP-IDF as a library to a custom CMake projects for {target}')
         idf_path = Path(os.environ['IDF_PATH'])
-        run_cmake_and_build(str(idf_path / 'examples' / 'build_system' / 'cmake' / 'idf_as_lib'),
+        run_cmake_and_build(str(idf_path / 'examples' / 'build_system' / 'cmake' / 'idf_as_lib'), '-G', 'Ninja',
                             '-DCMAKE_TOOLCHAIN_FILE={}'.format(idf_path / 'tools' / 'cmake' / f'toolchain-{target}.cmake'), f'-DTARGET={target}')
         assert file_contains((test_app_copy / 'build' / 'compile_commands.json'), '"command"')
         shutil.rmtree(test_app_copy / 'build')
@@ -52,20 +55,16 @@ def test_build_cmake_library_psram_strategies(idf_py: IdfPyFunc, test_app_copy:
         (test_app_copy / 'sdkconfig').unlink()
 
 
-@pytest.mark.skipif(sys.platform == 'win32', reason='Failing on Windows runner. TODO')
-@pytest.mark.usefixtures('test_app_copy')
-@pytest.mark.usefixtures('idf_copy')
-def test_defaults_for_unspecified_idf_build_process_args(default_idf_env: EnvDict) -> None:
+def test_defaults_unspecified_build_args(idf_copy: Path) -> None:
     logging.info('Defaults set properly for unspecified idf_build_process args')
-    idf_path = Path(default_idf_env.get('IDF_PATH'))
-    idf_as_lib_path = idf_path / 'examples' / 'build_system' / 'cmake' / 'idf_as_lib'
+    idf_as_lib_path = idf_copy / 'examples' / 'build_system' / 'cmake' / 'idf_as_lib'
     append_to_file(idf_as_lib_path / 'CMakeLists.txt', '\n'.join(['idf_build_get_property(project_dir PROJECT_DIR)',
                                                                   'message("Project directory: ${project_dir}")']))
-    ret = run_cmake('..',
-                    '-DCMAKE_TOOLCHAIN_FILE={}'.format(str(idf_path / 'tools' / 'cmake' / 'toolchain-esp32.cmake')),
+    ret = run_cmake('..', '-G', 'Ninja',
+                    '-DCMAKE_TOOLCHAIN_FILE={}'.format(str(idf_copy / 'tools' / 'cmake' / 'toolchain-esp32.cmake')),
                     '-DTARGET=esp32',
                     workdir=idf_as_lib_path)
-    assert 'Project directory: {}'.format(str(idf_as_lib_path)) in ret.stderr
+    assert 'Project directory: {}'.format(str(idf_as_lib_path.as_posix())) in ret.stderr
 
 
 def test_build_example_on_host(default_idf_env: EnvDict) -> None:

tools/test_build_system/test_common.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 import json
 import logging
@@ -12,8 +12,14 @@ from pathlib import Path
 from typing import List
 
 import pytest
-from test_build_system_helpers import (EnvDict, IdfPyFunc, append_to_file, file_contains, find_python, get_snapshot,
-                                       replace_in_file, run_idf_py)
+from test_build_system_helpers import append_to_file
+from test_build_system_helpers import EnvDict
+from test_build_system_helpers import file_contains
+from test_build_system_helpers import find_python
+from test_build_system_helpers import get_snapshot
+from test_build_system_helpers import IdfPyFunc
+from test_build_system_helpers import replace_in_file
+from test_build_system_helpers import run_idf_py
 
 
 def get_subdirs_absolute_paths(path: Path) -> List[str]:
@@ -207,8 +213,7 @@ def test_fallback_to_build_system_target(idf_py: IdfPyFunc, test_app_copy: Path)
     assert msg in ret.stdout, 'Custom target did not produce expected output'
 
 
-@pytest.mark.skipif(sys.platform == 'win32', reason='Failing on Windows runner. TODO')
-def test_create_component_and_project_plus_build(idf_copy: Path) -> None:
+def test_create_component_project(idf_copy: Path) -> None:
     logging.info('Create project and component using idf.py and build it')
     run_idf_py('-C', 'projects', 'create-project', 'temp_test_project', workdir=idf_copy)
     run_idf_py('-C', 'components', 'create-component', 'temp_test_component', workdir=idf_copy)

tools/test_build_system/test_components.py

@@ -1,25 +1,25 @@
-# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-
 import json
 import logging
 import shutil
-import sys
 from pathlib import Path
 
 import pytest
-from test_build_system_helpers import EnvDict, IdfPyFunc, append_to_file, replace_in_file
+from test_build_system_helpers import append_to_file
+from test_build_system_helpers import EnvDict
+from test_build_system_helpers import IdfPyFunc
+from test_build_system_helpers import replace_in_file
 
 
-@pytest.mark.skipif(sys.platform == 'win32', reason='Failing on Windows runner. TODO')
 def test_component_extra_dirs(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
     logging.info('Setting EXTRA_COMPONENT_DIRS works')
     shutil.move(test_app_copy / 'main', test_app_copy / 'different_main' / 'main')
     replace_in_file((test_app_copy / 'CMakeLists.txt'), '# placeholder_before_include_project_cmake',
-                    'set(EXTRA_COMPONENT_DIRS {})'.format(Path('different_main', 'main')))
+                    'set(EXTRA_COMPONENT_DIRS {})'.format(Path('different_main', 'main').as_posix()))
     ret = idf_py('reconfigure')
-    assert str(test_app_copy / 'different_main' / 'main') in ret.stdout
-    assert str(test_app_copy / 'main') not in ret.stdout
+    assert str((test_app_copy / 'different_main' / 'main').as_posix()) in ret.stdout
+    assert str((test_app_copy / 'main').as_posix()) not in ret.stdout
 
 
 @pytest.mark.usefixtures('test_app_copy')
@@ -44,18 +44,16 @@ def test_component_can_not_be_empty_dir(idf_py: IdfPyFunc, test_app_copy: Path)
     assert str(empty_component_dir) not in data.get('build_component_paths')
 
 
-@pytest.mark.skipif(sys.platform == 'win32', reason='Failing on Windows runner. TODO')
 def test_component_subdirs_not_added_to_component_dirs(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
     logging.info('If a component directory is added to COMPONENT_DIRS, its subdirectories are not added')
     (test_app_copy / 'main' / 'test').mkdir(parents=True)
     (test_app_copy / 'main' / 'test' / 'CMakeLists.txt').write_text('idf_component_register()')
     idf_py('reconfigure')
     data = json.load(open(test_app_copy / 'build' / 'project_description.json', 'r'))
-    assert str(test_app_copy / 'main' / 'test') not in data.get('build_component_paths')
-    assert str(test_app_copy / 'main') in data.get('build_component_paths')
+    assert str((test_app_copy / 'main' / 'test').as_posix()) not in data.get('build_component_paths')
+    assert str((test_app_copy / 'main').as_posix()) in data.get('build_component_paths')
 
 
-@pytest.mark.skipif(sys.platform == 'win32', reason='Failing on Windows runner. TODO')
 def test_component_sibling_dirs_not_added_to_component_dirs(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
     logging.info('If a component directory is added to COMPONENT_DIRS, its sibling directories are not added')
     mycomponents_subdir = (test_app_copy / 'mycomponents')
@@ -67,8 +65,8 @@ def test_component_sibling_dirs_not_added_to_component_dirs(idf_py: IdfPyFunc, t
     (mycomponents_subdir / 'esp32' / 'CMakeLists.txt').write_text('idf_component_register()')
     idf_py('-DEXTRA_COMPONENT_DIRS={}'.format(str(mycomponents_subdir / 'mycomponent')), 'reconfigure')
     data = json.load(open(test_app_copy / 'build' / 'project_description.json', 'r'))
-    assert str(mycomponents_subdir / 'esp32') not in data.get('build_component_paths')
-    assert str(mycomponents_subdir / 'mycomponent') in data.get('build_component_paths')
+    assert str((mycomponents_subdir / 'esp32').as_posix()) not in data.get('build_component_paths')
+    assert str((mycomponents_subdir / 'mycomponent').as_posix()) in data.get('build_component_paths')
     shutil.rmtree(mycomponents_subdir / 'esp32')
 
     # now the same thing, but add a components directory
@@ -76,21 +74,19 @@ def test_component_sibling_dirs_not_added_to_component_dirs(idf_py: IdfPyFunc, t
     (test_app_copy / 'esp32' / 'CMakeLists.txt').write_text('idf_component_register()')
     idf_py('-DEXTRA_COMPONENT_DIRS={}'.format(str(mycomponents_subdir)), 'reconfigure')
     data = json.load(open(test_app_copy / 'build' / 'project_description.json', 'r'))
-    assert str(test_app_copy / 'esp32') not in data.get('build_component_paths')
-    assert str(mycomponents_subdir / 'mycomponent') in data.get('build_component_paths')
+    assert str((test_app_copy / 'esp32').as_posix()) not in data.get('build_component_paths')
+    assert str((mycomponents_subdir / 'mycomponent').as_posix()) in data.get('build_component_paths')
 
 
-@pytest.mark.skipif(sys.platform == 'win32', reason='Failing on Windows runner. TODO')
 def test_component_properties_are_set(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
     logging.info('Component properties are set')
     append_to_file(test_app_copy / 'CMakeLists.txt', '\n'.join(['',
                                                                 'idf_component_get_property(srcs main SRCS)',
                                                                 'message(STATUS SRCS:${srcs})']))
     ret = idf_py('reconfigure')
-    assert 'SRCS:{}'.format(test_app_copy / 'main' / 'build_test_app.c') in ret.stdout, 'Component properties should be set'
+    assert 'SRCS:{}'.format((test_app_copy / 'main' / 'build_test_app.c').as_posix()) in ret.stdout, 'Component properties should be set'
 
 
-@pytest.mark.skipif(sys.platform == 'win32', reason='Failing on Windows runner. TODO')
 def test_component_overriden_dir(idf_py: IdfPyFunc, test_app_copy: Path, default_idf_env: EnvDict) -> None:
     logging.info('Getting component overriden dir')
     (test_app_copy / 'components' / 'hal').mkdir(parents=True)
@@ -100,17 +96,16 @@ def test_component_overriden_dir(idf_py: IdfPyFunc, test_app_copy: Path, default
     ret = idf_py('reconfigure')
     idf_path = Path(default_idf_env.get('IDF_PATH'))
     # no registration, overrides registration as well
-    assert 'overriden_dir:{}'.format(idf_path / 'components' / 'hal') in ret.stdout, 'Failed to get overriden dir'
+    assert 'overriden_dir:{}'.format((idf_path / 'components' / 'hal').as_posix()) in ret.stdout, 'Failed to get overriden dir'
     append_to_file((test_app_copy / 'components' / 'hal' / 'CMakeLists.txt'), '\n'.join([
         '',
         'idf_component_register(KCONFIG ${overriden_dir}/Kconfig)',
         'idf_component_get_property(kconfig ${COMPONENT_NAME} KCONFIG)',
         'message(STATUS kconfig:${overriden_dir}/Kconfig)']))
     ret = idf_py('reconfigure', check=False)
-    assert 'kconfig:{}'.format(idf_path / 'components' / 'hal') in ret.stdout, 'Failed to verify original `main` directory'
+    assert 'kconfig:{}'.format((idf_path / 'components' / 'hal').as_posix()) in ret.stdout, 'Failed to verify original `main` directory'
 
 
-@pytest.mark.skipif(sys.platform == 'win32', reason='Failing on Windows runner. TODO')
 def test_components_prioritizer_over_extra_components_dir(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
     logging.info('Project components prioritized over EXTRA_COMPONENT_DIRS')
     (test_app_copy / 'extra_dir' / 'my_component').mkdir(parents=True)
@@ -119,11 +114,11 @@ def test_components_prioritizer_over_extra_components_dir(idf_py: IdfPyFunc, tes
                     '# placeholder_before_include_project_cmake',
                     'set(EXTRA_COMPONENT_DIRS extra_dir)')
     ret = idf_py('reconfigure')
-    assert str(test_app_copy / 'extra_dir' / 'my_component') in ret.stdout, 'Unable to find component specified in EXTRA_COMPONENT_DIRS'
+    assert str((test_app_copy / 'extra_dir' / 'my_component').as_posix()) in ret.stdout, 'Unable to find component specified in EXTRA_COMPONENT_DIRS'
     (test_app_copy / 'components' / 'my_component').mkdir(parents=True)
     (test_app_copy / 'components' / 'my_component' / 'CMakeLists.txt').write_text('idf_component_register()')
     ret = idf_py('reconfigure')
-    assert str(test_app_copy / 'components' / 'my_component') in ret.stdout, 'Project components should be prioritized over EXTRA_COMPONENT_DIRS'
+    assert str((test_app_copy / 'components' / 'my_component').as_posix()) in ret.stdout, 'Project components should be prioritized over EXTRA_COMPONENT_DIRS'
 
 
 def test_exclude_components_not_passed(idf_py: IdfPyFunc, test_app_copy: Path) -> None:

tools/test_build_system/test_non_default_target.py

@@ -1,14 +1,16 @@
-# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-
 import logging
 import shutil
-import sys
 from pathlib import Path
-from typing import List, Optional
+from typing import List
+from typing import Optional
 
 import pytest
-from test_build_system_helpers import EnvDict, IdfPyFunc, file_contains, run_cmake
+from test_build_system_helpers import EnvDict
+from test_build_system_helpers import file_contains
+from test_build_system_helpers import IdfPyFunc
+from test_build_system_helpers import run_cmake
 
 ESP32C3_TARGET = 'esp32c3'
 ESP32C2_TARGET = 'esp32c2'
@@ -32,7 +34,6 @@ def test_target_from_environment_cmake(default_idf_env: EnvDict) -> None:
     assert file_contains('build/CMakeCache.txt', 'IDF_TARGET:STRING={}'.format(ESP32S2_TARGET))
 
 
-@pytest.mark.skipif(sys.platform == 'win32', reason='Failing on Windows runner. TODO')
 def test_target_from_environment_idf_py(idf_py: IdfPyFunc, default_idf_env: EnvDict, test_app_copy: Path) -> None:
     def reconfigure_and_check_return_values(errmsg: str, opts: Optional[List[str]] = None) -> None:
         opts = opts or []
@@ -43,7 +44,7 @@ def test_target_from_environment_idf_py(idf_py: IdfPyFunc, default_idf_env: EnvD
     idf_py('set-target', ESP32S2_TARGET)
     default_idf_env.update({'IDF_TARGET': ESP32_TARGET})
 
-    cfg_path = (test_app_copy / 'sdkconfig')
+    cfg_path = (test_app_copy / 'sdkconfig').as_posix()
 
     logging.info("idf.py fails if IDF_TARGET settings don't match the environment")
     reconfigure_and_check_return_values("Project sdkconfig '{}' was generated for target '{}', but environment "
@@ -74,7 +75,6 @@ def test_target_from_environment_idf_py(idf_py: IdfPyFunc, default_idf_env: EnvD
                                         ['-D', 'IDF_TARGET={}'.format(ESP32_TARGET)])
 
 
-@pytest.mark.skipif(sys.platform == 'win32', reason='Failing on Windows runner. TODO')
 def test_target_consistency_cmake(default_idf_env: EnvDict, test_app_copy: Path) -> None:
     def reconfigure_and_check_return_values(errmsg: str, opts: Optional[List[str]] = None) -> None:
         opts = opts or []
@@ -84,7 +84,7 @@ def test_target_consistency_cmake(default_idf_env: EnvDict, test_app_copy: Path)
 
     run_cmake('-G', 'Ninja', '..')
 
-    cfg_path = (test_app_copy / 'sdkconfig')
+    cfg_path = (test_app_copy / 'sdkconfig').as_posix()
 
     logging.info("cmake fails if IDF_TARGET settings don't match the environment")
     default_idf_env.update({'IDF_TARGET': ESP32S2_TARGET})

tools/test_build_system/test_spaces.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 import logging
 import os
@@ -20,21 +20,21 @@ def clean_app_dir(app_path: Path) -> None:
     shutil.rmtree(app_path / 'build', ignore_errors=True)
 
 
-@pytest.mark.skipif(sys.platform == 'win32', reason='Failing on Windows runner. TODO')
 @pytest.mark.idf_copy('esp idf with spaces')
 def test_spaces_bundle1(idf_copy: Path) -> None:
     logging.info('Running test spaces bundle 1')
     # test_build
     run_idf_py('build', workdir=(idf_copy / 'examples' / 'get-started' / 'hello_world'))
-    # test build ulp_fsm
-    run_idf_py('build', workdir=(idf_copy / 'examples' / 'system' / 'ulp' / 'ulp_fsm' / 'ulp'))
-    # test build ulp_riscv
-    run_idf_py('-DIDF_TARGET=esp32s2', 'build', workdir=(idf_copy / 'examples' / 'system' / 'ulp' / 'ulp_riscv' / 'gpio'))
     # test spiffsgen
     run_idf_py('build', workdir=(idf_copy / 'examples' / 'storage' / 'spiffsgen'))
+    # bug reported in IDF-9151
+    if sys.platform != 'win32':
+        # test build ulp_fsm
+        run_idf_py('build', workdir=(idf_copy / 'examples' / 'system' / 'ulp' / 'ulp_fsm' / 'ulp'))
+        # test build ulp_riscv
+        run_idf_py('-DIDF_TARGET=esp32s2', 'build', workdir=(idf_copy / 'examples' / 'system' / 'ulp' / 'ulp_riscv' / 'gpio'))
 
 
-@pytest.mark.skipif(sys.platform == 'win32', reason='Failing on Windows runner. TODO')
 @pytest.mark.idf_copy('esp idf with spaces')
 def test_spaces_bundle2(idf_copy: Path) -> None:
     logging.info('Running test spaces bundle 2')
@@ -50,7 +50,6 @@ def test_spaces_bundle2(idf_copy: Path) -> None:
     run_idf_py('uf2', workdir=hello_world_app_path)
 
 
-@pytest.mark.skipif(sys.platform == 'win32', reason='Failing on Windows runner. TODO')
 @pytest.mark.idf_copy('esp idf with spaces')
 def test_spaces_bundle3(idf_copy: Path) -> None:
     logging.info('Running test spaces bundle 3')
@@ -89,7 +88,6 @@ def test_install_export_unix(idf_copy: Path) -> None:
     subprocess.check_call(export_cmd, env=env, shell=True, cwd=idf_copy, executable='/bin/bash')
 
 
-@pytest.mark.skipif(sys.platform == 'win32', reason='Failing on Windows runner. TODO')
 @pytest.mark.skipif(sys.platform != 'win32', reason='Windows test')
 @pytest.mark.idf_copy('esp idf with spaces')
 def test_install_export_win(idf_copy: Path) -> None: