refactor(tools): Rewrite build system unit tests to python

Rewritten parts:
 - git
 - version
 - build

tools/test_build_system/MIGRATION.md

@@ -14,13 +14,13 @@ Partial build doesn't compile anything by default | test_rebuild::test_rebuild_n
 Rebuild when app version was changed | test_rebuild.py::test_rebuild_version_change |
 Change app version | test_rebuild.py::test_rebuild_version_change |
 Re-building does not change app.bin | test_rebuild.py::test_rebuild_version_change |
-Get the version of app from git describe. Project is not inside IDF and do not have a tag only a hash commit. | |
-Get the version of app from Kconfig option | |
-Use IDF version variables in component CMakeLists.txt file | |
-Project is in ESP-IDF which has a custom tag | |
+Get the version of app from git describe. Project is not inside IDF and do not have a tag only a hash commit. | test_git.py::test_get_version_from_git_describe |
+Get the version of app from Kconfig option | test_kconfig.py::test_kconfig_get_version_from_describe |
+Use IDF version variables in component CMakeLists.txt file | test_components.py::test_version_in_component_cmakelist |
+Project is in ESP-IDF which has a custom tag | test_git.py::test_git_custom_tag |
 Moving BUILD_DIR_BASE out of tree | test_build.py::test_build_alternative_directories |
 BUILD_DIR_BASE inside default build directory | test_build.py::test_build_alternative_directories |
-Can still clean build if all text files are CRLFs | |
+Can still clean build if all text files are CRLFs | test_build.py::test_build_with_crlf_files |
 Updating rom ld file should re-link app and bootloader | test_rebuild::test_rebuild_linker |
 Updating app-only ld file should only re-link app | test_rebuild::test_rebuild_linker |
 Updating ld file should only re-link app | test_rebuild::test_rebuild_linker |
@@ -51,7 +51,7 @@ Setting EXTRA_COMPONENT_DIRS works | test_components.py::test_component_extra_di
 Non-existent paths in EXTRA_COMPONENT_DIRS are not allowed | test_components.py::test_component_nonexistent_extra_dirs_not_allowed |
 Component names may contain spaces | test_components.py::test_component_names_contain_spaces |
 sdkconfig should have contents of all files: sdkconfig, sdkconfig.defaults, sdkconfig.defaults.IDF_TARGET | test_sdkconfig.py::test_sdkconfig_contains_all_files |
-Test if it can build the example to run on host | |
+Test if it can build the example to run on host | test_cmake.py::test_build_example_on_host |
 Test build ESP-IDF as a library to a custom CMake projects for all targets | test_cmake.py::test_build_custom_cmake_project |
 Building a project with CMake library imported and PSRAM workaround, all files compile with workaround | test_cmake.py::test_build_cmake_library_psram_workaround |
 Test for external libraries in custom CMake projects with ESP-IDF components linked | test_cmake.py::test_build_custom_cmake_project |
@@ -74,7 +74,7 @@ Can set options to subcommands: print_filter for monitor | test_common.py::test_
 Fail on build time works | test_build.py::test_build_fail_on_build_time |
 Component properties are set | test_components.py::test_component_properties_are_set |
 should be able to specify multiple sdkconfig default files | test_sdkconfig.py::test_sdkconfig_multiple_default_files |
-Supports git worktree | |
+Supports git worktree | test_git.py::test_support_git_worktree |
 idf.py fallback to build system target | test_common.py::test_fallback_to_build_system_target |
 Build fails if partitions don't fit in flash | test_partition.py::test_partitions_dont_fit_in_flash |
 Warning is given if smallest partition is nearly full | test_partition.py::test_partition_nearly_full_warning |
@@ -94,7 +94,7 @@ Check that command for creating new project will fail if the target folder is no
 Check that command for creating new project will fail if the target path is file. | test_common.py::test_create_project |
 Check docs command | test_common.py::test_docs_command |
 Deprecation warning check | test_common.py::test_deprecation_warning |
-Save-defconfig checks | |
+Save-defconfig checks | test_common.py::test_save_defconfig_check |
 test_build | |
 test_build_ulp_fsm | |
 test_build_ulp_riscv | |

tools/test_build_system/conftest.py

@@ -98,6 +98,29 @@ def test_app_copy(session_work_dir: Path, request: FixtureRequest) -> typing.Gen
         shutil.rmtree(path_to, ignore_errors=True)
 
 
+@pytest.fixture
+def test_git_template_app(session_work_dir: Path, request: FixtureRequest) -> typing.Generator[Path, None, None]:
+    copy_to = request.node.name + '_app'
+    path_to = session_work_dir / copy_to
+
+    logging.debug(f'clonning git-teplate app to {path_to}')
+    path_to.mkdir()
+    # No need to clone full repository, just single master branch
+    subprocess.run(['git', 'clone', '--single-branch', '-b', 'master', '--depth', '1', 'https://github.com/espressif/esp-idf-template.git', '.'],
+                   cwd=path_to, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+    old_cwd = Path.cwd()
+    os.chdir(path_to)
+
+    yield Path(path_to)
+
+    os.chdir(old_cwd)
+
+    if should_clean_test_dir(request):
+        logging.debug('cleaning up work directory after a successful test: {}'.format(path_to))
+        shutil.rmtree(path_to, ignore_errors=True)
+
+
 @pytest.fixture
 def idf_copy(session_work_dir: Path, request: FixtureRequest) -> typing.Generator[Path, None, None]:
     copy_to = request.node.name + '_idf'

tools/test_build_system/test_build.py

@@ -3,6 +3,7 @@
 
 import logging
 import os
+import stat
 import sys
 import textwrap
 from pathlib import Path
@@ -178,3 +179,25 @@ def test_build_loadable_elf(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
     idf_py('reconfigure')
     assert (test_app_copy / 'build' / 'flasher_args.json').exists(), 'flasher_args.json should be generated in a loadable ELF build'
     idf_py('build')
+
+
+@pytest.mark.skipif(sys.platform == 'win32', reason='Windows does not support stat commands')
+def test_build_with_crlf_files(idf_py: IdfPyFunc, test_app_copy: Path, idf_copy: Path) -> None:
+    def change_files_to_crlf(path: Path) -> None:
+        for root, _, files in os.walk(path):
+            for filename in files:
+                file_path = os.path.join(root, filename)
+                # Do not modify .git directory and executable files, as Linux will fail to execute them
+                if '.git' in file_path or os.stat(file_path).st_mode & stat.S_IEXEC:
+                    continue
+                with open(file_path, 'rb') as f:
+                    data = f.read()
+                    crlf_data = data.replace(b'\n', b'\r\n')
+                with open(file_path, 'wb') as f:
+                    f.write(crlf_data)
+
+    logging.info('Can still build if all text files are CRLFs')
+    change_files_to_crlf(test_app_copy)
+    change_files_to_crlf(idf_copy)
+    idf_py('build')
+    assert_built(BOOTLOADER_BINS + APP_BINS + PARTITION_BIN)

tools/test_build_system/test_cmake.py

@@ -63,3 +63,22 @@ def test_defaults_for_unspecified_idf_build_process_args(default_idf_env: EnvDic
                     '-DTARGET=esp32',
                     workdir=idf_as_lib_path)
     assert 'Project directory: {}'.format(str(idf_as_lib_path)) in ret.stderr
+
+
+def test_build_example_on_host(default_idf_env: EnvDict) -> None:
+    logging.info('Test if it can build the example to run on host')
+    idf_path = Path(default_idf_env.get('IDF_PATH'))
+    idf_as_lib_path = Path(idf_path, 'examples', 'build_system', 'cmake', 'idf_as_lib')
+    try:
+        target = 'esp32'
+        run_cmake('..',
+                  f'-DCMAKE_TOOLCHAIN_FILE={idf_path}/tools/cmake/toolchain-{target}.cmake',
+                  f'-DTARGET={target}',
+                  '-GNinja',
+                  workdir=idf_as_lib_path)
+
+        run_cmake('--build',
+                  '.',
+                  workdir=idf_as_lib_path)
+    finally:
+        shutil.rmtree(idf_as_lib_path / 'build', ignore_errors=True)

tools/test_build_system/test_common.py

@@ -12,8 +12,8 @@ from pathlib import Path
 from typing import List
 
 import pytest
-from test_build_system_helpers import (EnvDict, IdfPyFunc, append_to_file, find_python, get_snapshot, replace_in_file,
-                                       run_idf_py)
+from test_build_system_helpers import (EnvDict, IdfPyFunc, append_to_file, file_contains, find_python, get_snapshot,
+                                       replace_in_file, run_idf_py)
 
 
 def get_subdirs_absolute_paths(path: Path) -> List[str]:
@@ -263,3 +263,26 @@ def test_deprecation_warning(idf_py: IdfPyFunc) -> None:
     ret = idf_py('efuse_common_table', check=False)
     # cmake warning
     assert 'Have you wanted to run "efuse-common-table" instead?' in ret.stdout
+
+
+def test_save_defconfig_check(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
+    logging.info('Save-defconfig checks')
+    (test_app_copy / 'sdkconfig').write_text('\n'.join(['CONFIG_COMPILER_OPTIMIZATION_SIZE=y',
+                                                        'CONFIG_ESPTOOLPY_FLASHFREQ_80M=y']))
+    idf_py('save-defconfig')
+    assert not file_contains(test_app_copy / 'sdkconfig.defaults', 'CONFIG_IDF_TARGET'), \
+        'CONFIG_IDF_TARGET should not be in sdkconfig.defaults'
+    assert file_contains(test_app_copy / 'sdkconfig.defaults', 'CONFIG_COMPILER_OPTIMIZATION_SIZE=y'), \
+        'Missing CONFIG_COMPILER_OPTIMIZATION_SIZE=y in sdkconfig.defaults'
+    assert file_contains(test_app_copy / 'sdkconfig.defaults', 'CONFIG_ESPTOOLPY_FLASHFREQ_80M=y'), \
+        'Missing CONFIG_ESPTOOLPY_FLASHFREQ_80M=y in sdkconfig.defaults'
+    idf_py('fullclean')
+    (test_app_copy / 'sdkconfig').unlink()
+    (test_app_copy / 'sdkconfig.defaults').unlink()
+    idf_py('set-target', 'esp32c3')
+    (test_app_copy / 'sdkconfig').write_text('CONFIG_PARTITION_TABLE_OFFSET=0x8001')
+    idf_py('save-defconfig')
+    assert file_contains(test_app_copy / 'sdkconfig.defaults', 'CONFIG_IDF_TARGET="esp32c3"'), \
+        'Missing CONFIG_IDF_TARGET="esp32c3" in sdkconfig.defaults'
+    assert file_contains(test_app_copy / 'sdkconfig.defaults', 'CONFIG_PARTITION_TABLE_OFFSET=0x8001'), \
+        'Missing CONFIG_PARTITION_TABLE_OFFSET=0x8001 in sdkconfig.defaults'

tools/test_build_system/test_components.py

@@ -126,3 +126,10 @@ def test_exclude_components_not_passed(idf_py: IdfPyFunc, test_app_copy: Path) -
     ret = idf_py('reconfigure', check=False)
     assert ret.returncode == 2, 'Reconfigure should have failed due to invalid syntax in idf_component.yml'
     idf_py('-DEXCLUDE_COMPONENTS=to_be_excluded', 'reconfigure')
+
+
+def test_version_in_component_cmakelist(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
+    logging.info('Use IDF version variables in component CMakeLists.txt file')
+    replace_in_file((test_app_copy / 'main' / 'CMakeLists.txt'), '# placeholder_before_idf_component_register',
+                    '\n'.join(['if (NOT IDF_VERSION_MAJOR)', ' message(FATAL_ERROR "IDF version not set")', 'endif()']))
+    idf_py('reconfigure')

tools/test_build_system/test_git.py

@@ -0,0 +1,68 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+import logging
+import os
+import re
+import shutil
+import subprocess
+import typing
+from pathlib import Path
+
+from test_build_system_helpers import EnvDict, IdfPyFunc, run_idf_py
+
+
+def run_git_cmd(*args: str,
+                workdir: Path,
+                env: typing.Optional[EnvDict] = None) -> subprocess.CompletedProcess:
+
+    cmd = ['git'] + list(args)
+    env_dict = dict(**os.environ)
+    if env:
+        env_dict.update(env)
+    logging.debug('running {} in {}'.format(' '.join(cmd), workdir))
+
+    return subprocess.run(cmd, cwd=workdir, env=env_dict,
+                          stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+
+def test_get_version_from_git_describe(test_git_template_app: Path, idf_py: IdfPyFunc) -> None:
+    logging.info('Get the version of app from git describe. Project is not inside IDF and do not have a tag only a hash commit.')
+    idf_ret = idf_py('reconfigure')
+    git_ret = run_git_cmd('describe', '--always', '--tags', '--dirty', workdir=test_git_template_app)
+    assert f'App "app-template" version: {git_ret.stdout.decode("utf-8")}' in idf_ret.stdout, 'Project version should have a hash commit'
+
+
+# In this test, the action needs to be performed in ESP-IDF that is valid git directory
+# Copying ESP-IDF is not possible
+def test_git_custom_tag() -> None:
+    try:
+        env_dict = dict(**os.environ)
+        idf_build_test_app_path = Path(env_dict['IDF_PATH'], 'tools', 'test_build_system', 'build_test_app')
+
+        logging.info('Project is in ESP-IDF which has a custom tag')
+        env_dict['GIT_COMMITTER_NAME'] = 'No One'
+        env_dict['GIT_COMMITTER_EMAIL'] = 'noone@espressif.com'
+        run_git_cmd('tag', 'mytag', '-a', '-m', 'mytag', workdir=idf_build_test_app_path, env=env_dict)
+        idf_ret = run_idf_py('reconfigure', workdir=idf_build_test_app_path)
+        assert 'App "build_test_app" version: mytag' in idf_ret.stdout, 'Project version should be the custom tag'
+        version = run_idf_py('--version', workdir=idf_build_test_app_path)
+        assert 'mytag' not in version.stdout, 'IDF version should not contain mytag'
+
+    finally:
+        run_git_cmd('tag', '-d', 'mytag', workdir=idf_build_test_app_path)
+        shutil.rmtree(idf_build_test_app_path / 'build')
+        os.unlink(idf_build_test_app_path / 'sdkconfig')
+
+
+def test_support_git_worktree(test_git_template_app: Path) -> None:
+    logging.info('Supports git worktree')
+    run_git_cmd('branch', 'test_build_system', workdir=test_git_template_app)
+    run_git_cmd('worktree', 'add', '../esp-idf-worktree-app', 'test_build_system', workdir=test_git_template_app)
+    try:
+        idf_ret_template_app = run_idf_py('reconfigure', workdir=test_git_template_app)
+        idf_ret_worktree_app = run_idf_py('reconfigure', workdir=os.path.join(os.path.dirname(test_git_template_app), 'esp-idf-worktree-app'))
+        assert (re.search(r'-- App \"app-template\".*', idf_ret_template_app.stdout).group() ==  # type: ignore
+                re.search(r'-- App \"app-template\".*', idf_ret_worktree_app.stdout).group())  # type: ignore
+    finally:
+        run_git_cmd('worktree', 'remove', '../esp-idf-worktree-app', workdir=test_git_template_app)
+        run_git_cmd('branch', '-d', 'test_build_system', workdir=test_git_template_app)

tools/test_build_system/test_kconfig.py

@@ -124,3 +124,12 @@ def test_kconfig_multiple_and_target_specific_options(idf_py: IdfPyFunc, test_ap
     idf_py('set-target', 'esp32s2')
     assert all([file_contains((test_app_copy / 'sdkconfig'), x) for x in ['CONFIG_TEST_NEW_OPTION=y',
                                                                           'CONFIG_TEST_OLD_OPTION=y']])
+
+
+def test_kconfig_get_version_from_describe(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
+    logging.info('Get the version of app from Kconfig option')
+    (test_app_copy / 'version.txt').write_text('project_version_from_txt')
+    (test_app_copy / 'sdkconfig.defaults').write_text('\n'.join(['CONFIG_APP_PROJECT_VER_FROM_CONFIG=y',
+                                                                 'CONFIG_APP_PROJECT_VER="project_version_from_Kconfig"']))
+    ret = idf_py('build')
+    assert 'App "build_test_app" version: project_version_from_Kconfig' in ret.stdout