Tools: Rewrite build system unit tests to python - other target tests

tools/test_apps/system/build_test/sdkconfig.ci.ethernet_disabled

@@ -0,0 +1,2 @@
+CONFIG_ETH_USE_SPI_ETHERNET=n
+CONFIG_ETH_USE_ESP32_EMAC=n

tools/test_build_system/MIGRATION.md

@@ -34,19 +34,19 @@ idf.py can build with Unix Makefiles | test_build.py::test_build_with_generator_
 Can build with IDF_PATH set via cmake cache not environment | test_build.py::test_build_with_cmake_and_idf_path_unset |
 Can build with IDF_PATH unset and inferred by build system | test_build.py::test_build_with_cmake_and_idf_path_unset |
 Can build with IDF_PATH unset and inferred by cmake when Kconfig needs it to be set | test_build.py::test_build_with_cmake_and_idf_path_unset |
-can build with phy_init_data | |
-can build with ethernet component disabled | |
-Compiler flags on build command line are taken into account | |
-Compiler flags cannot be overwritten | |
-Can override IDF_TARGET from environment | |
-Can set target using idf.py -D | |
-Can set target using -D as subcommand parameter for idf.py | |
-Can set target using idf.py set-target | |
-idf.py understands alternative target names | |
-Can guess target from sdkconfig, if CMakeCache does not exist | |
-Can set the default target using sdkconfig.defaults | |
-IDF_TARGET takes precedence over the value of CONFIG_IDF_TARGET in sdkconfig.defaults | |
-idf.py fails if IDF_TARGET settings don't match in sdkconfig, CMakeCache.txt, and the environment | |
+can build with phy_init_data | test_build.py::test_build_skdconfig_phy_init_data |
+can build with ethernet component disabled | | moved to test_apps/system/build_test/sdkconfig.ci.ethernet_disabled
+Compiler flags on build command line are taken into account | test_build.py::test_build_compiler_flag_in_source_file |
+Compiler flags cannot be overwritten | test_build.py::test_build_compiler_flags_no_overwriting |
+Can override IDF_TARGET from environment | test_non_default_target.py::test_target_from_environment_cmake |
+Can set target using idf.py -D | test_non_default_target.py::test_target_using_D_parameter |
+Can set target using -D as subcommand parameter for idf.py | test_non_default_target.py::test_target_using_D_parameter |
+Can set target using idf.py set-target | test_non_default_target.py::test_target_using_settarget_parameter |
+idf.py understands alternative target names | test_non_default_target.py::test_target_using_settarget_parameter_alternative_name |
+Can guess target from sdkconfig, if CMakeCache does not exist | test_non_default_target.py::test_target_using_settarget_parameter |
+Can set the default target using sdkconfig.defaults | test_non_default_target.py::test_target_using_sdkconfig |
+IDF_TARGET takes precedence over the value of CONFIG_IDF_TARGET in sdkconfig.defaults | test_non_default_target.py::test_target_precedence |
+idf.py fails if IDF_TARGET settings don't match in sdkconfig, CMakeCache.txt, and the environment | test_non_default_target.py::test_target_from_environment_idf_py |
 Setting EXTRA_COMPONENT_DIRS works | |
 Non-existent paths in EXTRA_COMPONENT_DIRS are not allowed | |
 Component names may contain spaces | |

tools/test_build_system/conftest.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 import datetime
 import logging
@@ -51,7 +51,8 @@ def fixture_session_work_dir(request: FixtureRequest) -> typing.Generator[Path,
         logging.debug(f'created temporary work directory: {work_dir}')
         clean_dir = work_dir
 
-    yield Path(work_dir)
+    # resolve allows to use relative paths with --work-dir option
+    yield Path(work_dir).resolve()
 
     if clean_dir:
         logging.debug(f'cleaning up {clean_dir}')
@@ -138,6 +139,6 @@ def fixture_default_idf_env() -> EnvDict:
 
 @pytest.fixture
 def idf_py(default_idf_env: EnvDict) -> IdfPyFunc:
-    def result(*args: str) -> subprocess.CompletedProcess:
-        return run_idf_py(*args, env=default_idf_env, workdir=os.getcwd())  # type: ignore
+    def result(*args: str, check: bool = True) -> subprocess.CompletedProcess:
+        return run_idf_py(*args, env=default_idf_env, workdir=os.getcwd(), check=check)  # type: ignore
     return result

tools/test_build_system/test_build.py

@@ -3,15 +3,15 @@
 
 import logging
 import os
-import re
 import sys
+import textwrap
 import typing
 from pathlib import Path
-from typing import List, Pattern, Union
+from typing import List, Union
 
 import pytest
-from test_build_system_helpers import (APP_BINS, BOOTLOADER_BINS, PARTITION_BIN, EnvDict, IdfPyFunc, get_idf_build_env,
-                                       replace_in_file, run_cmake)
+from test_build_system_helpers import (APP_BINS, BOOTLOADER_BINS, PARTITION_BIN, EnvDict, IdfPyFunc, append_to_file,
+                                       check_file_contains, get_idf_build_env, replace_in_file, run_cmake)
 
 
 def run_cmake_and_build(*cmake_args: str, env: typing.Optional[EnvDict] = None) -> None:
@@ -29,15 +29,6 @@ def assert_built(paths: Union[List[str], List[Path]]) -> None:
         assert os.path.exists(path)
 
 
-def check_file_contains(filename: Union[str, Path], what: Union[str, Pattern]) -> None:
-    with open(filename, 'r', encoding='utf-8') as f:
-        data = f.read()
-        if isinstance(what, str):
-            assert what in data
-        else:
-            assert re.search(what, data) is not None
-
-
 def test_build_alternative_directories(idf_py: IdfPyFunc, session_work_dir: Path, test_app_copy: Path) -> None:
     logging.info('Moving BUILD_DIR_BASE out of tree')
     alt_build_dir = session_work_dir / 'alt_build'
@@ -114,3 +105,31 @@ def test_build_with_cmake_and_idf_path_unset(idf_py: IdfPyFunc, test_app_copy: P
     replace_in_file('CMakeLists.txt', '{IDF_PATH}', '{ci_idf_path}')
     run_cmake_and_build('-G', 'Ninja', '-D', 'ci_idf_path={}'.format(idf_path), '..', env=env)
     assert_built(BOOTLOADER_BINS + APP_BINS + PARTITION_BIN)
+
+
+def test_build_skdconfig_phy_init_data(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
+    logging.info('can build with phy_init_data')
+    (test_app_copy / 'sdkconfig.defaults').touch()
+    (test_app_copy / 'sdkconfig.defaults').write_text('CONFIG_ESP32_PHY_INIT_DATA_IN_PARTITION=y')
+    idf_py('reconfigure')
+    idf_py('build')
+    assert_built(BOOTLOADER_BINS + APP_BINS + PARTITION_BIN + ['build/phy_init_data.bin'])
+
+
+def test_build_compiler_flag_in_source_file(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
+    logging.info('Compiler flags on build command line are taken into account')
+    compiler_flag = textwrap.dedent("""
+                                    #ifndef USER_FLAG
+                                    #error "USER_FLAG is not defined!"
+                                    #endif
+                                    """)
+    append_to_file((test_app_copy / 'main' / 'build_test_app.c'), compiler_flag)
+    idf_py('build', '-DCMAKE_C_FLAGS=-DUSER_FLAG')
+
+
+@pytest.mark.usefixtures('test_app_copy')
+def test_build_compiler_flags_no_overwriting(idf_py: IdfPyFunc) -> None:
+    logging.info('Compiler flags cannot be overwritten')
+    # If the compiler flags are overriden, the following build command will
+    # cause issues at link time.
+    idf_py('build', '-DCMAKE_C_FLAGS=', '-DCMAKE_CXX_FLAGS=')

tools/test_build_system/test_build_system_helpers/__init__.py

@@ -2,12 +2,12 @@
 # SPDX-License-Identifier: Apache-2.0
 from .build_constants import ALL_ARTIFACTS, APP_BINS, BOOTLOADER_BINS, JSON_METADATA, PARTITION_BIN
 from .editing import append_to_file, replace_in_file
-from .idf_utils import EXT_IDF_PATH, EnvDict, IdfPyFunc, get_idf_build_env, run_cmake, run_idf_py
+from .idf_utils import EXT_IDF_PATH, EnvDict, IdfPyFunc, check_file_contains, get_idf_build_env, run_cmake, run_idf_py
 from .snapshot import Snapshot, get_snapshot
 
 __all__ = [
     'append_to_file', 'replace_in_file',
     'get_idf_build_env', 'run_idf_py', 'EXT_IDF_PATH', 'EnvDict', 'IdfPyFunc',
     'Snapshot', 'get_snapshot', 'run_cmake', 'APP_BINS', 'BOOTLOADER_BINS',
-    'PARTITION_BIN', 'JSON_METADATA', 'ALL_ARTIFACTS'
+    'PARTITION_BIN', 'JSON_METADATA', 'ALL_ARTIFACTS', 'check_file_contains'
 ]

tools/test_build_system/test_build_system_helpers/idf_utils.py

@@ -2,11 +2,13 @@
 # SPDX-License-Identifier: Apache-2.0
 import logging
 import os
+import re
 import shutil
 import subprocess
 import sys
 import typing
 from pathlib import Path
+from typing import Pattern, Union
 
 try:
     EXT_IDF_PATH = os.environ['IDF_PATH']  # type: str
@@ -57,13 +59,15 @@ def get_idf_build_env(idf_path: str) -> EnvDict:
 def run_idf_py(*args: str,
                env: typing.Optional[EnvDict] = None,
                idf_path: typing.Optional[typing.Union[str,Path]] = None,
-               workdir: typing.Optional[str] = None) -> subprocess.CompletedProcess:
+               workdir: typing.Optional[str] = None,
+               check: bool = True) -> subprocess.CompletedProcess:
     """
     Run idf.py command with given arguments, raise an exception on failure
     :param args: arguments to pass to idf.py
     :param env: environment variables to run the build with; if not set, the default environment is used
     :param idf_path: path to the IDF copy to use; if not set, IDF_PATH from the 'env' argument is used
     :param workdir: directory where to run the build; if not set, the current directory is used
+    :param check: check process exits with a zero exit code, if false all retvals are accepted without failing the test
     """
     env_dict = dict(**os.environ)
     if env is not None:
@@ -86,7 +90,7 @@ def run_idf_py(*args: str,
     logging.debug('running {} in {}'.format(' '.join(cmd), workdir))
     return subprocess.run(
         cmd, env=env_dict, cwd=workdir,
-        check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+        check=check, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
         text=True, encoding='utf-8', errors='backslashreplace')
 
 
@@ -107,3 +111,12 @@ def run_cmake(*cmake_args: str, env: typing.Optional[EnvDict] = None) -> None:
     subprocess.check_call(
         cmd, env=env, cwd=workdir,
         stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+
+
+def check_file_contains(filename: Union[str, Path], what: Union[str, Pattern]) -> None:
+    with open(filename, 'r', encoding='utf-8') as f:
+        data = f.read()
+        if isinstance(what, str):
+            assert what in data
+        else:
+            assert re.search(what, data) is not None

tools/test_build_system/test_non_default_target.py

@@ -0,0 +1,104 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+import shutil
+from pathlib import Path
+
+import pytest
+from test_build_system_helpers import EnvDict, IdfPyFunc, check_file_contains, run_cmake
+
+ESP32S2_TARGET = 'esp32s2'
+ESP32_TARGET = 'esp32'
+
+
+def clean_app(test_app_copy: Path) -> None:
+    shutil.rmtree(test_app_copy / 'build')
+    (test_app_copy / 'sdkconfig').unlink()
+
+
+@pytest.mark.usefixtures('test_app_copy')
+def test_target_from_environment_cmake(default_idf_env: EnvDict) -> None:
+    logging.info('Can override IDF_TARGET from environment')
+    env = default_idf_env
+    env.update({'IDF_TARGET': ESP32S2_TARGET})
+    run_cmake('-G', 'Ninja', '..', env=env)
+    check_file_contains('sdkconfig', 'CONFIG_IDF_TARGET="{}"'.format(ESP32S2_TARGET))
+    check_file_contains('build/CMakeCache.txt', 'IDF_TARGET:STRING={}'.format(ESP32S2_TARGET))
+
+
+def test_target_from_environment_idf_py(idf_py: IdfPyFunc, default_idf_env: EnvDict, test_app_copy: Path) -> None:
+    def reconfigure_and_check_return_values(errmsg: str) -> None:
+        ret = idf_py('reconfigure', check=False)
+        assert ret.returncode == 2
+        assert errmsg in ret.stderr
+
+    idf_py('set-target', ESP32S2_TARGET)
+    default_idf_env.update({'IDF_TARGET': ESP32_TARGET})
+
+    logging.info("idf.py fails if IDF_TARGET settings don't match the environment")
+    reconfigure_and_check_return_values("Project sdkconfig was generated for target '{}', but environment "
+                                        "variable IDF_TARGET is set to '{}'.".format(ESP32S2_TARGET, ESP32_TARGET))
+
+    logging.info("idf.py fails if IDF_TARGET settings in CMakeCache.txt don't match the environment")
+    (test_app_copy / 'sdkconfig').write_text('CONFIG_IDF_TARGET="{}"'.format(ESP32_TARGET))
+    reconfigure_and_check_return_values("Target settings are not consistent: '{}' in the environment, "
+                                        "'{}' in CMakeCache.txt.".format(ESP32_TARGET, ESP32S2_TARGET))
+
+    logging.info("idf.py fails if IDF_TARGET settings in CMakeCache.txt don't match the sdkconfig")
+    default_idf_env.pop('IDF_TARGET')
+    reconfigure_and_check_return_values("Project sdkconfig was generated for target '{}', but CMakeCache.txt "
+                                        "contains '{}'.".format(ESP32_TARGET, ESP32S2_TARGET))
+
+
+def test_target_precedence(idf_py: IdfPyFunc, default_idf_env: EnvDict, test_app_copy: Path) -> None:
+    logging.info('IDF_TARGET takes precedence over the value of CONFIG_IDF_TARGET in sdkconfig.defaults')
+    (test_app_copy / 'sdkconfig.defaults').write_text('CONFIG_IDF_TARGET="{}"'.format(ESP32S2_TARGET))
+    default_idf_env.update({'IDF_TARGET': ESP32_TARGET})
+    idf_py('reconfigure')
+    check_file_contains('sdkconfig', 'CONFIG_IDF_TARGET="{}"'.format(ESP32_TARGET))
+    check_file_contains('sdkconfig', 'CONFIG_IDF_TARGET_{}=y'.format(ESP32_TARGET.upper()))
+    check_file_contains('build/CMakeCache.txt', 'IDF_TARGET:STRING={}'.format(ESP32_TARGET))
+
+
+def test_target_using_D_parameter(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
+    logging.info('Can set target using idf.py -D')
+    idf_py('-DIDF_TARGET={}'.format(ESP32S2_TARGET), 'reconfigure')
+    check_file_contains('sdkconfig', 'CONFIG_IDF_TARGET="{}"'.format(ESP32S2_TARGET))
+    check_file_contains('build/CMakeCache.txt', 'IDF_TARGET:STRING={}'.format(ESP32S2_TARGET))
+
+    logging.info('Can set target using -D as subcommand parameter for idf.py')
+    clean_app(test_app_copy)
+    idf_py('reconfigure', '-DIDF_TARGET={}'.format(ESP32S2_TARGET))
+    check_file_contains('sdkconfig', 'CONFIG_IDF_TARGET="{}"'.format(ESP32S2_TARGET))
+    check_file_contains('build/CMakeCache.txt', 'IDF_TARGET:STRING={}'.format(ESP32S2_TARGET))
+
+
+@pytest.mark.usefixtures('test_app_copy')
+def test_target_using_settarget_parameter_alternative_name(idf_py: IdfPyFunc) -> None:
+    logging.info('idf.py understands alternative target names')
+    idf_py('set-target', ESP32S2_TARGET.upper())
+    check_file_contains('sdkconfig', 'CONFIG_IDF_TARGET="{}"'.format(ESP32S2_TARGET))
+    check_file_contains('build/CMakeCache.txt', 'IDF_TARGET:STRING={}'.format(ESP32S2_TARGET))
+
+
+@pytest.mark.usefixtures('test_app_copy')
+def test_target_using_settarget_parameter(idf_py: IdfPyFunc) -> None:
+    logging.info('Can set target using idf.py set-target')
+    idf_py('set-target', ESP32S2_TARGET)
+    check_file_contains('sdkconfig', 'CONFIG_IDF_TARGET="{}"'.format(ESP32S2_TARGET))
+    check_file_contains('build/CMakeCache.txt', 'IDF_TARGET:STRING={}'.format(ESP32S2_TARGET))
+
+    logging.info('Can guess target from sdkconfig, if CMakeCache does not exist')
+    idf_py('fullclean')
+    idf_py('reconfigure')
+    check_file_contains('sdkconfig', 'CONFIG_IDF_TARGET="{}"'.format(ESP32S2_TARGET))
+    check_file_contains('build/CMakeCache.txt', 'IDF_TARGET:STRING={}'.format(ESP32S2_TARGET))
+
+
+def test_target_using_sdkconfig(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
+    logging.info('Can set the default target using sdkconfig.defaults')
+    (test_app_copy / 'sdkconfig.defaults').write_text('CONFIG_IDF_TARGET="{}"'.format(ESP32S2_TARGET))
+    idf_py('reconfigure')
+    check_file_contains('sdkconfig', 'CONFIG_IDF_TARGET="{}"'.format(ESP32S2_TARGET))
+    check_file_contains('sdkconfig', 'CONFIG_IDF_TARGET_{}=y'.format(ESP32S2_TARGET.upper()))