build system: add the initial set of pytest-based build system tests

.gitlab/CODEOWNERS

@@ -198,6 +198,8 @@
 
 /tools/test_apps/**/*.py             @esp-idf-codeowners/ci @esp-idf-codeowners/tools
 
+/tools/test_build_system/     @esp-idf-codeowners/tools @esp-idf-codeowners/build-config
+
 /tools/unit-test-app/         @esp-idf-codeowners/system @esp-idf-codeowners/tools
 
 # sort-order-reset

.gitlab/ci/build.yml

@@ -563,6 +563,21 @@ test_build_system_spaces:
   variables:
     SHELL_TEST_SCRIPT: test_build_system_spaces.py
 
+pytest_build_system:
+  extends: .test_build_system_template
+  artifacts:
+    paths:
+      - XUNIT_RESULT.xml
+      - test_build_system
+    when: always
+    expire_in: 2 days
+    reports:
+      junit: XUNIT_RESULT.xml
+  script:
+    - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
+    - cd ${IDF_PATH}/tools/test_build_system
+    - pytest --work-dir ${CI_PROJECT_DIR}/test_build_system --junitxml=${CI_PROJECT_DIR}/XUNIT_RESULT.xml
+
 build_docker:
   extends:
     - .before_script_minimal

.gitlab/ci/rules.yml

@@ -50,6 +50,7 @@
   - "tools/ci/test_build_system*.sh"
   - "tools/ci/test_build_system*.py"
   - "tools/ci/ci_build_apps.py"
+  - "tools/test_build_system/**/*"
 
 .patterns-custom_test: &patterns-custom_test
   - "components/espcoredump/**/*"

tools/test_build_system/MIGRATION.md

@@ -0,0 +1,107 @@
+# Migration from test_build_system_cmake.sh to pytest
+
+This table tracks migration of tests from [test_build_system_cmake.sh](../ci/test_build_system_cmake.sh) and [test_build_system_spaces.py](../ci/test_build_system_spaces.py) to pytest.
+
+When all tests are migrated to pytest, remove the original tests, corresponding CI jobs, and this file.
+
+Legacy test name | New test name | Comments
+-----------------|---------------|---------
+Initial clean build | test_rebuild::test_rebuild_no_changes |
+Updating component source file rebuilds component | test_rebuild::test_rebuild_source_files |
+Bootloader source file rebuilds bootloader | test_rebuild::test_rebuild_source_files |
+Partition CSV file rebuilds partitions | test_rebuild::test_rebuild_source_files |
+Partial build doesn't compile anything by default | test_rebuild::test_rebuild_no_changes |
+Rebuild when app version was changed | |
+Change app version | |
+Re-building does not change app.bin | |
+Get the version of app from git describe. Project is not inside IDF and do not have a tag only a hash commit. | |
+Get the version of app from Kconfig option | |
+Use IDF version variables in component CMakeLists.txt file | |
+Project is in ESP-IDF which has a custom tag | |
+Moving BUILD_DIR_BASE out of tree | |
+BUILD_DIR_BASE inside default build directory | |
+Can still clean build if all text files are CRLFs | |
+Updating rom ld file should re-link app and bootloader | test_rebuild::test_rebuild_linker |
+Updating app-only ld file should only re-link app | test_rebuild::test_rebuild_linker |
+Updating ld file should only re-link app | test_rebuild::test_rebuild_linker |
+Updating fragment file should only re-link app | test_rebuild::test_rebuild_linker |
+sdkconfig update triggers full recompile | test_rebuild::test_rebuild_source_files |
+Updating project CMakeLists.txt triggers full recompile | test_rebuild::test_rebuild_source_files |
+Can build with Ninja (no idf.py) | |
+Can build with GNU Make (no idf.py) | |
+idf.py can build with Ninja | |
+idf.py can build with Unix Makefiles | |
+Can build with IDF_PATH set via cmake cache not environment | |
+Can build with IDF_PATH unset and inferred by build system | |
+Can build with IDF_PATH unset and inferred by cmake when Kconfig needs it to be set | |
+can build with phy_init_data | |
+can build with ethernet component disabled | |
+Compiler flags on build command line are taken into account | |
+Compiler flags cannot be overwritten | |
+Can override IDF_TARGET from environment | |
+Can set target using idf.py -D | |
+Can set target using -D as subcommand parameter for idf.py | |
+Can set target using idf.py set-target | |
+idf.py understands alternative target names | |
+Can guess target from sdkconfig, if CMakeCache does not exist | |
+Can set the default target using sdkconfig.defaults | |
+IDF_TARGET takes precedence over the value of CONFIG_IDF_TARGET in sdkconfig.defaults | |
+idf.py fails if IDF_TARGET settings don't match in sdkconfig, CMakeCache.txt, and the environment | |
+Setting EXTRA_COMPONENT_DIRS works | |
+Non-existent paths in EXTRA_COMPONENT_DIRS are not allowed | |
+Component names may contain spaces | |
+sdkconfig should have contents of all files: sdkconfig, sdkconfig.defaults, sdkconfig.defaults.IDF_TARGET | |
+Test if it can build the example to run on host | |
+Test build ESP-IDF as a library to a custom CMake projects for all targets | |
+Building a project with CMake library imported and PSRAM workaround, all files compile with workaround | |
+Test for external libraries in custom CMake projects with ESP-IDF components linked | |
+Test for external libraries in custom CMake projects with PSRAM strategy $strat | |
+Cleaning Python bytecode | |
+Displays partition table when executing target partition_table | |
+Make sure a full build never runs '/usr/bin/env python' or similar | |
+Handling deprecated Kconfig options | |
+Handling deprecated Kconfig options in sdkconfig.defaults | |
+Confserver can be invoked by idf.py | |
+Check ccache is used to build | |
+Custom bootloader overrides original | |
+Empty directory not treated as a component | |
+If a component directory is added to COMPONENT_DIRS, its subdirectories are not added | |
+If a component directory is added to COMPONENT_DIRS, its sibling directories are not added | |
+toolchain prefix is set in project description file | |
+Can set options to subcommands: print_filter for monitor | |
+Fail on build time works | |
+Component properties are set | |
+should be able to specify multiple sdkconfig default files | |
+Supports git worktree | |
+idf.py fallback to build system target | |
+Build fails if partitions don't fit in flash | |
+Warning is given if smallest partition is nearly full | |
+Flash size is correctly set in the bootloader image header | |
+DFU build works | |
+UF2 build works | |
+Loadable ELF build works | |
+Defaults set properly for unspecified idf_build_process args | |
+Getting component overriden dir | |
+Overriding Kconfig | |
+Project components prioritized over EXTRA_COMPONENT_DIRS | |
+Components in EXCLUDE_COMPONENTS not passed to idf_component_manager | |
+Create project using idf.py and build it | |
+Create component using idf.py, create project using idf.py. | |
+Add the component to the created project and build the project. | |
+Check that command for creating new project will fail if the target folder is not empty. | |
+Check that command for creating new project will fail if the target path is file. | |
+Check docs command | |
+Deprecation warning check | |
+Save-defconfig checks | |
+test_build | |
+test_build_ulp_fsm | |
+test_build_ulp_riscv | |
+test_spiffsgen | |
+test_flash_encryption | |
+test_secure_boot_v1 | |
+test_secure_boot_v2 | |
+test_app_signing | |
+test_secure_boot_release_mode | |
+test_x509_cert_bundle | |
+test_dfu | |
+test_uf2 | |

tools/test_build_system/README.md

@@ -0,0 +1,166 @@
+# Tools & Build System Tests
+
+This directory contains tests for the build system and build-related tools. These tests are meant to be used both by developers and in CI. Please check the sections below for details on:
+
+- Running the tests locally
+- Debugging test failures
+- Adding new tests
+- Fixtures and helper functions
+
+## Running the tests locally
+
+1. Install pytest using `install.{sh,bat,ps1,fish} --enable-pytest`.
+1. Activate the IDF shell environment using `export.{sh,bat,ps1,fish}`.
+1. To run all the tests, go to `$IDF_PATH/tools/test_build_system` directory, then run:
+    ```
+    pytest
+    ```
+1. To run one specific test, use `-k` flag of pytest, for example
+    ```
+    pytest -k test_compile_commands_json_updated_by_reconfigure
+    ```
+1. To speed up the builds you can install Ccache and set the following environment variables:
+    ```
+    export IDF_CCACHE_ENABLE=1
+    export CCACHE_NOHASHDIR=1
+    ```
+
+## Debugging test failures
+
+If you are working on a bug fix or a feature and one of the tests starts to fail, you should try to reproduce the failure locally.
+
+1. Find the name of the failing test in the CI job log
+1. Follow the steps in the section above to run that one test
+1. By default, the fixtures which create temporary directories will remove them after the test. To prevent the directories from being removed, run `pytest` with `--work-dir /some/path` flag. The temporary directories will be created under `/some/path`, and you will be able to inspect them once the test fails.
+1. You can increase the logging level to see the commands being executed by the test by running `pytest` with `--log-cli-level DEBUG` argument.
+
+## Adding new tests
+
+1. When adding a new test, think of the developer who might have to run this test locally.
+   - Avoid adding tests which take a long time to run. Running the entire test suite should be possible!
+   - Remember that developers run these tests in their IDF work directories. Be careful with destructive actions, especially removing directories recursively. Developers might have untracked files in the directory you are removing! Prefer using the `idf_copy` fixture to make a copy of the IDF directory, when doing some modifications to IDF source. If this is too expensive, do the modification in place but make sure to clean up the changes you perform using a try/finally block. This especially applies to the newly created files.
+1. Read through the test cases and try to find a test which does something similar to what you need to test. This will usually be a good starting point. Also read through the section below, which explains fixtures and utility functions.
+1. The tests need to run on Windows, Linux and macOS. Avoid calling OS-specific programs such as `sed` or `awk` in tests. If you need to perform some complex file modification in the test case, consider writing a Python helper function for that.
+
+## Fixtures and helper functions
+
+If you aren't yet familiar with Pytest fixtures, please take a few moments and read a Pytest tutorial or watch the training, before moving on to the next section.
+
+### `test_app_copy` fixture
+
+This fixture selects the app (inside IDF) to be used by the test and copies this app to a temporary directory, recursively. The working directory is set to the root of the copied app. The directory is removed once the test is finished.
+
+```python
+def test_something(test_app_copy):
+    assert test_app_copy == os.getcwd()
+    # the current working directory now contains the copy of the test app
+```
+
+If the test case doesn't use the `test_app_copy` argument, pylint will typically warn about an unused argument, even if the fixture is actually used. To avoid the warning, use the following pattern:
+```python
+@pytest.mark.usefixtures('test_app_copy')
+def test_something(idf_py):
+    idf_py('build')
+```
+
+By default, the source app is `tools/test_build_system/build_test_app` and the destination directory name is derived from the test case name. (See more about this test app [here](#application-under-test).) This can be overridden using a `@pytest.mark.test_app_copy` decorator, as shown below. The first argument is the path of the source app. The second argument is the name of the temporary directory to create. The second argument is optional, it is mostly useful to test handling of special characters (such as spaces) in the path.
+
+```python
+@pytest.mark.test_app_copy('examples/get-started/blink', 'custom dir name')
+@pytest.mark.usefixtures('test_app_copy')
+def test_something():
+    pass
+```
+
+### `idf_py` fixture
+
+This fixture runs `idf.py` with IDF environment set up.
+
+```python
+@pytest.mark.usefixtures('test_app_copy')
+def test_something(idf_py)
+    # 1. 'test_app_copy' fixture has copied the test app into a temporary directory and
+    #    set the working directory there.
+    # 2. 'idf_py' fixture is a function which calls idf.py:
+    idf_py('fullclean')
+
+    # 3. It accepts multiple arguments and returns a subprocess.CompletedProcess
+    #    instance. It can be used to check the process output.
+    output = idf_py('-DIDF_TARGET=esp32c3', 'reconfigure')
+    assert 'CONFIG_IDF_TARGET="esp32c3"' in Path('sdkconfig').read_text()
+    assert 'Building ESP-IDF components for target esp32c3' in output.stdout
+
+    # 4. Raises subprocess.CalledProcessError on failure
+    with(pytest.raises(subprocess.CalledProcessError)) as exc_info:
+        idf_py('unknown_command')
+    assert 'command "unknown_command" is not known to idf.py' in exc_info.value.stderr
+```
+
+### `default_idf_env` fixture
+
+Returns a dictionary of environment variables required for the IDF build environment. It is similar to the output of `env` command after running the `export` script.
+
+```python
+@pytest.mark.usefixtures('test_app_copy')
+def test_cmake(default_idf_env):
+    with tempfile.TemporaryDirectory() as build_dir:
+        # default_idf_env can be passed to subprocess APIs to run other tools
+        subprocess.run(['cmake', '-B', build_dir, '.'], env=default_idf_env, check=True)
+```
+
+Note, `default_idf_env` sets up the environment based on the `IDF_PATH` environment variable set before launching `pytest`.
+
+### `idf_copy` fixture
+
+Copies IDF from `IDF_PATH` into a new temporary directory. `@pytest.mark.idf_copy('name prefix')` can be used to specify the name prefix of the temporary directory.
+
+For the duration of the test, `IDF_PATH` environment variable is set to the newly created copy.
+
+```python
+@pytest.mark.usefixtures('test_app_copy')
+def test_idf_copy(idf_copy):
+    # idf_copy is the temporary IDF copy.
+    # For example, we can check if idf.py build can work without the .git directory:
+    shutil.rmtree(os.path.join(idf_copy, '.git'), ignore_errors=True)
+    # Note that we can't use idf_py fixture, since it uses the default IDF path.
+    # We can use 'get_idf_build_env' with 'run_idf_py', instead:
+    env = get_idf_build_env(idf_copy)
+    run_idf_py('build', env=env)
+```
+
+### Build snapshots
+
+`get_snapshot(list_of_globs)` function takes a list of glob expressions, finds the files matching these expressions, and returns a `Snapshot` instance. `Snapshot` instances record file names and their modification timestamps. Two `Snapshot` instances can be compared using `assert_same` and `assert_different` methods:
+
+```python
+@pytest.mark.usefixtures('test_app_copy')
+def test_build_jsons_updated_by_reconfigure(idf_py):
+    globs = ['build/*.json']
+    
+    idf_py('reconfigure')
+    snapshot_1 = get_snapshot(globs)
+    snapshot_2 = get_snapshot(globs)
+    snapshot_2.assert_same(snapshot_1)
+
+    idf_py('reconfigure')
+    snapshot_3 = get_snapshot(globs)
+    snapshot_3.assert_different(snapshot_2)
+```
+
+### Helper functions for file modifications
+
+A few extra functions are provided to make simple file modifications:
+
+* `append_to_file(filename: typing.Union[str, Path], what: str) -> None` — appends the given string to a file.
+* `replace_in_file(filename: typing.Union[str, Path], search: str, replace: str) -> None` — searches the file for occurrences of the string `search` and replaces all of them with `replace`, then writes the result back to the file.
+
+### Application under test
+
+Most build system tests should use the included [`build_test_app`](build_test_app/), if possible.
+
+Using other test apps and examples for the purpose of testing the build system is okay as long as you keep the following in mind:
+
+* Don't use the build system tests to compile examples or test apps under a particular combination of sdkconfig options. Use the `sdkconfig.ci.*` files for that, instead.
+* Examples or test apps may be changed, renamed or removed. If you add a dependency on another example or a test app, your test case might need to be rewritten if someone has to modify or remove the example or a test app your test case depends on.
+
+For convenience, the `build_test_app` app included here provides several placeholders which can be modified using the `replace_in_file` function. You can find the placeholders by running `grep -r placeholder_ build_test_app`.

tools/test_build_system/build_test_app/CMakeLists.txt

@@ -0,0 +1,9 @@
+cmake_minimum_required(VERSION 3.16)
+
+# placeholder_before_include_project_cmake
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+
+# placeholder_after_include_project_cmake
+
+project(build_test_app)

tools/test_build_system/build_test_app/README.md

@@ -0,0 +1 @@
+Information about this test app can be found [here](../README.md#application-under-test).

tools/test_build_system/build_test_app/main/CMakeLists.txt

@@ -0,0 +1,5 @@
+# placeholder_before_idf_component_register
+
+idf_component_register(SRCS "build_test_app.c"
+                       # placeholder_inside_idf_component_register
+)

tools/test_build_system/build_test_app/main/build_test_app.c

@@ -0,0 +1,12 @@
+/*
+ * SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#include <stdio.h>
+// placeholder_before_main
+
+void app_main(void)
+{
+    // placeholder_inside_main
+}

tools/test_build_system/conftest.py

@@ -0,0 +1,142 @@
+# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+import datetime
+import logging
+import os
+import shutil
+import subprocess
+import typing
+from pathlib import Path
+from tempfile import mkdtemp
+
+import pytest
+from _pytest.fixtures import FixtureRequest
+from test_build_system_helpers import EXT_IDF_PATH, EnvDict, IdfPyFunc, get_idf_build_env, run_idf_py
+
+
+# Pytest hook used to check if the test has passed or failed, from a fixture.
+# Based on https://docs.pytest.org/en/latest/example/simple.html#making-test-result-information-available-in-fixtures
+@pytest.hookimpl(tryfirst=True, hookwrapper=True)
+def pytest_runtest_makereport(item: typing.Any, call: typing.Any) -> typing.Generator[None, pytest.TestReport, None]:  # pylint: disable=unused-argument
+    outcome = yield  # Execute all other hooks to obtain the report object
+    report = outcome.get_result()
+    if report.when == 'call' and report.passed:
+        # set an attribute which can be checked using 'should_clean_test_dir' function below
+        setattr(item, 'passed', True)
+
+
+def should_clean_test_dir(request: FixtureRequest) -> bool:
+    # Only remove the test directory if the test has passed
+    return getattr(request.node, 'passed', False)
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+    parser.addoption(
+        '--work-dir', action='store', default=None,
+        help='Directory for temporary files. If not specified, an OS-specific '
+             'temporary directory will be used.'
+    )
+
+
+@pytest.fixture(name='session_work_dir', scope='session', autouse=True)
+def fixture_session_work_dir(request: FixtureRequest) -> typing.Generator[Path, None, None]:
+    work_dir = request.config.getoption('--work-dir')
+    if work_dir:
+        work_dir = os.path.join(work_dir, datetime.datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S'))
+        logging.debug(f'using work directory: {work_dir}')
+        os.makedirs(work_dir, exist_ok=True)
+        clean_dir = None
+    else:
+        work_dir = mkdtemp()
+        logging.debug(f'created temporary work directory: {work_dir}')
+        clean_dir = work_dir
+
+    yield Path(work_dir)
+
+    if clean_dir:
+        logging.debug(f'cleaning up {clean_dir}')
+        shutil.rmtree(clean_dir, ignore_errors=True)
+
+
+@pytest.fixture
+def test_app_copy(session_work_dir: Path, request: FixtureRequest) -> typing.Generator[Path, None, None]:
+    # by default, use hello_world app and copy it to a temporary directory with
+    # the name resembling that of the test
+    copy_from = 'tools/test_build_system/build_test_app'
+    copy_to = request.node.name + '_app'
+
+    # allow overriding source and destination via pytest.mark.test_app_copy()
+    mark = request.node.get_closest_marker('test_app_copy')
+    if mark:
+        copy_from = mark.args[0]
+        if len(mark.args) > 1:
+            copy_to = mark.args[1]
+
+    path_from = Path(os.environ['IDF_PATH']) / copy_from
+    path_to = session_work_dir / copy_to
+
+    # if the new directory inside the original directory,
+    # make sure not to go into recursion.
+    ignore = shutil.ignore_patterns(
+        path_to.name,
+        # also ignore files which may be present in the work directory
+        'build', 'sdkconfig')
+
+    logging.debug(f'copying {path_from} to {path_to}')
+    shutil.copytree(path_from, path_to, ignore=ignore, symlinks=True)
+
+    old_cwd = Path.cwd()
+    os.chdir(path_to)
+
+    yield Path(path_to)
+
+    os.chdir(old_cwd)
+
+    if should_clean_test_dir(request):
+        logging.debug('cleaning up work directory after a successful test: {}'.format(path_to))
+        shutil.rmtree(path_to, ignore_errors=True)
+
+
+@pytest.fixture
+def idf_copy(session_work_dir: Path, request: FixtureRequest) -> typing.Generator[Path, None, None]:
+    copy_to = request.node.name + '_idf'
+
+    # allow overriding the destination via pytest.mark.idf_copy()
+    mark = request.node.get_closest_marker('idf_copy')
+    if mark:
+        copy_to = mark.args[0]
+
+    path_from = EXT_IDF_PATH
+    path_to = session_work_dir / copy_to
+
+    # if the new directory inside the original directory,
+    # make sure not to go into recursion.
+    ignore = shutil.ignore_patterns(
+        path_to.name,
+        # also ignore the build directories which may be quite large
+        '**/build')
+
+    logging.debug(f'copying {path_from} to {path_to}')
+    shutil.copytree(path_from, path_to, ignore=ignore, symlinks=True)
+
+    orig_idf_path = os.environ['IDF_PATH']
+
+    yield Path(path_to)
+
+    os.environ['IDF_PATH'] = orig_idf_path
+
+    if should_clean_test_dir(request):
+        logging.debug('cleaning up work directory after a successful test: {}'.format(path_to))
+        shutil.rmtree(path_to, ignore_errors=True)
+
+
+@pytest.fixture(name='default_idf_env')
+def fixture_default_idf_env() -> EnvDict:
+    return get_idf_build_env(os.environ['IDF_PATH'])  # type: ignore
+
+
+@pytest.fixture
+def idf_py(default_idf_env: EnvDict) -> IdfPyFunc:
+    def result(*args: str) -> subprocess.CompletedProcess:
+        return run_idf_py(*args, env=default_idf_env, workdir=os.getcwd())  # type: ignore
+    return result

tools/test_build_system/pytest.ini

@@ -0,0 +1,19 @@
+[pytest]
+addopts = -s -p no:pytest-embedded
+
+# log related
+log_cli = True
+log_cli_level = INFO
+log_cli_format = %(asctime)s %(levelname)s %(message)s
+log_cli_date_format = %Y-%m-%d %H:%M:%S
+
+# junit related
+junit_family = xunit1
+
+## log all to `system-out` when case fail
+junit_logging = stdout
+junit_log_passing_tests = False
+
+markers =
+    test_app_copy: specify relative path of the app to copy, and the prefix of the destination directory name
+    idf_copy: specify the prefix of the destination directory where IDF should be copied

tools/test_build_system/test_build_system_helpers/__init__.py

@@ -0,0 +1,11 @@
+# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+from .editing import append_to_file, replace_in_file
+from .idf_utils import EXT_IDF_PATH, EnvDict, IdfPyFunc, get_idf_build_env, run_idf_py
+from .snapshot import Snapshot, get_snapshot
+
+__all__ = [
+    'append_to_file', 'replace_in_file',
+    'get_idf_build_env', 'run_idf_py', 'EXT_IDF_PATH', 'EnvDict', 'IdfPyFunc',
+    'Snapshot', 'get_snapshot'
+]

tools/test_build_system/test_build_system_helpers/editing.py

@@ -0,0 +1,17 @@
+# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+import typing
+from pathlib import Path
+
+
+def append_to_file(filename: typing.Union[str, Path], what: str) -> None:
+    with open(filename, 'a', encoding='utf-8') as f:
+        f.write(what)
+
+
+def replace_in_file(filename: typing.Union[str, Path], search: str, replace: str) -> None:
+    with open(filename, 'r', encoding='utf-8') as f:
+        data = f.read()
+    result = data.replace(search, replace)
+    with open(filename, 'w', encoding='utf-8') as f:
+        f.write(result)

tools/test_build_system/test_build_system_helpers/idf_utils.py

@@ -0,0 +1,87 @@
+# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+import logging
+import os
+import shutil
+import subprocess
+import sys
+import typing
+from pathlib import Path
+
+try:
+    EXT_IDF_PATH = os.environ['IDF_PATH']  # type: str
+except KeyError:
+    print('IDF_PATH must be set before running this test', file=sys.stderr)
+    exit(1)
+
+
+EnvDict = typing.Dict[str, str]
+IdfPyFunc = typing.Callable[..., subprocess.CompletedProcess]
+
+
+def find_python(path_var: str) -> str:
+    """
+    Find python interpreter in the paths specified in the given PATH variable.
+    Returns the full path to the interpreter.
+    """
+    res = shutil.which('python', path=path_var)
+    if res is None:
+        raise ValueError('python not found')
+    return res
+
+
+def get_idf_build_env(idf_path: str) -> EnvDict:
+    """
+    Get environment variables (as set by export.sh) for the specific IDF copy
+    :param idf_path: path of the IDF copy to use
+    :return: dictionary of environment variables and their values
+    """
+    cmd = [
+        sys.executable,
+        os.path.join(idf_path, 'tools', 'idf_tools.py'),
+        'export',
+        '--format=key-value'
+    ]
+    keys_values = subprocess.check_output(cmd, stderr=subprocess.PIPE).decode()
+    env_vars = {key: os.path.expandvars(value) for key, value in
+                [line.split('=') for line in keys_values.splitlines()]}
+    # not set by idf_tools.py, normally set by export.sh
+    env_vars['IDF_PATH'] = idf_path
+
+    return env_vars
+
+
+def run_idf_py(*args: str,
+               env: typing.Optional[EnvDict] = None,
+               idf_path: typing.Optional[typing.Union[str,Path]] = None,
+               workdir: typing.Optional[str] = None) -> subprocess.CompletedProcess:
+    """
+    Run idf.py command with given arguments, raise an exception on failure
+    :param args: arguments to pass to idf.py
+    :param env: environment variables to run the build with; if not set, the default environment is used
+    :param idf_path: path to the IDF copy to use; if not set, IDF_PATH from the 'env' argument is used
+    :param workdir: directory where to run the build; if not set, the current directory is used
+    """
+    env_dict = dict(**os.environ)
+    if env is not None:
+        env_dict.update(env)
+    if not workdir:
+        workdir = os.getcwd()
+    # order: function argument -> value in env dictionary -> system environment
+    if idf_path is None:
+        idf_path = env_dict.get('IDF_PATH')
+        if not idf_path:
+            raise ValueError('IDF_PATH must be set in the env array if idf_path argument is not set')
+
+    python = find_python(env_dict['PATH'])
+
+    cmd = [
+        python,
+        os.path.join(idf_path, 'tools', 'idf.py')
+    ]
+    cmd += args  # type: ignore
+    logging.debug('running {} in {}'.format(' '.join(cmd), workdir))
+    return subprocess.run(
+        cmd, env=env_dict, cwd=workdir,
+        check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+        text=True, encoding='utf-8', errors='backslashreplace')

tools/test_build_system/test_build_system_helpers/snapshot.py

@@ -0,0 +1,61 @@
+# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+import fnmatch
+import glob
+import os
+from typing import List, Optional, Tuple, Union
+
+
+class Snapshot:
+    """
+    Helper class for working with build snapshots.
+    A snapshot is a set of files along with their modification timestamps.
+    """
+    def __init__(self, files: List[str]) -> None:
+        """Create a snapshot of the given list of files, recording their mtimes"""
+        self.info: List[Tuple[str, int]] = []
+        for file in sorted(files):
+            st = os.stat(file)
+            self.info.append((file, st.st_mtime_ns))
+
+    def assert_same(self, other: 'Snapshot') -> None:
+        """Assert that this snapshot has the same files and mtimes as the other snapshot"""
+        for my_info, other_info in zip(self.info, other.info):
+            assert my_info[0] == other_info[0], f'Snapshots mismatch, expected file {my_info[0]}, found {other_info[0]}'
+            assert my_info[1] == other_info[1], f'File {my_info[0]} timestamp has changed! diff={(other_info[1]-my_info[1])/10**9}'
+
+    def assert_different(self, other: 'Snapshot') -> None:
+        """Assert that this snapshot has the same files as the other snapshot, but all mtimes are different"""
+        for my_info, other_info in zip(self.info, other.info):
+            assert my_info[0] == other_info[0], f'Snapshots mismatch, expected file {my_info[0]}, found {other_info[0]}'
+            assert my_info[1] != other_info[1], f'File {my_info[0]} timestamp has not changed! ({my_info[1]})'
+
+
+def get_snapshot(glob_patterns: Union[str, List[str]],
+                 exclude_patterns: Optional[Union[str, List[str]]] = None) -> Snapshot:
+    """Return a snapshot including the files matched by glob_patterns, and excluding those matched by exclude_patterns"""
+    if isinstance(glob_patterns, str):
+        glob_patterns = [glob_patterns]
+
+    if isinstance(exclude_patterns, str):
+        exclude_patterns_list = [exclude_patterns]
+    elif exclude_patterns is None:
+        exclude_patterns_list = []
+    elif isinstance(exclude_patterns, list):
+        exclude_patterns_list = exclude_patterns
+    else:
+        raise ValueError(f'Unexpcted type of exclude_patterns: ({type(exclude_patterns)}')
+
+    # whether the path found by glob.glob should be excluded?
+    def should_exclude(filename: str) -> bool:
+        return os.path.isdir(filename) or \
+            any((fnmatch.fnmatch(filename, pattern) for pattern in exclude_patterns_list))
+
+    files_to_snapshot: List[str] = []
+    for pattern in glob_patterns:
+        found = filter(lambda f: not should_exclude(f),
+                       glob.glob(pattern, recursive=True))
+        if not found:
+            raise RuntimeError(f'failed to match any files with pattern {pattern}')
+        files_to_snapshot.extend(found)
+    return Snapshot(files_to_snapshot)

tools/test_build_system/test_common.py

@@ -0,0 +1,57 @@
+# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+from test_build_system_helpers import EnvDict, IdfPyFunc, get_snapshot, replace_in_file
+
+
+@pytest.mark.usefixtures('test_app_copy')
+@pytest.mark.test_app_copy('examples/get-started/blink')
+def test_compile_commands_json_updated_by_reconfigure(idf_py: IdfPyFunc) -> None:
+    output = idf_py('reconfigure')
+    assert 'Building ESP-IDF components for target esp32' in output.stdout
+    snapshot_1 = get_snapshot(['build/compile_commands.json'])
+    snapshot_2 = get_snapshot(['build/compile_commands.json'])
+    snapshot_2.assert_same(snapshot_1)
+    idf_py('reconfigure')
+    snapshot_3 = get_snapshot(['build/compile_commands.json'])
+    snapshot_3.assert_different(snapshot_2)
+
+
+@pytest.mark.usefixtures('test_app_copy')
+def test_of_test_app_copy(idf_py: IdfPyFunc) -> None:
+    p = Path('main/idf_component.yml')
+    p.write_text('syntax_error\n')
+    try:
+        with (pytest.raises(subprocess.CalledProcessError)) as exc_info:
+            idf_py('reconfigure')
+        assert 'ERROR: Unknown format of the manifest file:' in exc_info.value.stderr
+    finally:
+        p.unlink()
+
+
+@pytest.mark.usefixtures('test_app_copy')
+def test_hints_no_color_output_when_noninteractive(idf_py: EnvDict) -> None:
+    """Check that idf.py hints don't include color escape codes in non-interactive builds"""
+
+    # make the build fail in such a way that idf.py shows a hint
+    replace_in_file('main/build_test_app.c', '// placeholder_inside_main',
+                    'esp_chip_info_t chip_info; esp_chip_info(&chip_info);')
+
+    with (pytest.raises(subprocess.CalledProcessError)) as exc_info:
+        idf_py('build')
+
+    # Should not actually include a color escape sequence!
+    # Change the assert to the correct value once the bug is fixed.
+    assert '\x1b[0;33mHINT: esp_chip_info.h' in exc_info.value.stderr
+
+
+@pytest.mark.usefixtures('test_app_copy')
+def test_idf_copy(idf_copy: Path, idf_py: IdfPyFunc) -> None:
+    # idf_copy is the temporary IDF copy.
+    # For example, we can check if idf.py build can work without the .git directory:
+    shutil.rmtree(idf_copy / '.git', ignore_errors=True)
+    idf_py('build')

tools/test_build_system/test_rebuild.py

@@ -0,0 +1,139 @@
+# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+# These tests check whether the build system rebuilds some files or not
+# depending on the changes to the project.
+import logging
+import os
+from pathlib import Path
+from typing import List, Union
+
+import pytest
+from test_build_system_helpers import IdfPyFunc, get_snapshot, replace_in_file
+
+BOOTLOADER_BINS = ['build/bootloader/bootloader.elf', 'build/bootloader/bootloader.bin']
+APP_BINS = ['build/build_test_app.elf', 'build/build_test_app.bin']
+PARTITION_BIN = ['build/partition_table/partition-table.bin']
+JSON_METADATA = ['build/project_description.json', 'build/flasher_args.json', 'build/config/kconfig_menus.json', 'build/config/sdkconfig.json']
+ALL_ARTIFACTS = [
+    *BOOTLOADER_BINS,
+    *APP_BINS,
+    *PARTITION_BIN,
+    *JSON_METADATA
+]
+
+
+@pytest.mark.usefixtures('test_app_copy')
+def test_rebuild_no_changes(idf_py: IdfPyFunc) -> None:
+    logging.info('initial build')
+    idf_py('build')
+    logging.info('get the first snapshot')
+    # excluding the 'log' subdirectory here since it changes after every build
+    all_build_files = get_snapshot('build/**/*', exclude_patterns='build/log/*')
+
+    logging.info('check that all build artifacts were generated')
+    for artifact in ALL_ARTIFACTS:
+        assert Path(artifact).exists()
+
+    logging.info('build again with no changes')
+    idf_py('build')
+    # if there are no changes, nothing gets rebuilt
+    all_build_files_after_rebuild = get_snapshot('build/**/*', exclude_patterns='build/log/*')
+    all_build_files_after_rebuild.assert_same(all_build_files)
+
+
+def rebuild_and_check(idf_py: IdfPyFunc,
+                      should_be_rebuilt: Union[str, List[str]],
+                      should_not_be_rebuilt: Union[str, List[str]]) -> None:
+    """
+    Helper function for the test cases below.
+    Asserts that the files matching 'should_be_rebuilt' patterns are rebuilt
+    and files matching 'should_not_be_rebuilt' patterns aren't, after
+    touching (updating the mtime) of the given 'file_to_touch' and rebuilding.
+    """
+    snapshot_should_be_rebuilt = get_snapshot(should_be_rebuilt)
+    snapshot_should_not_be_rebuilt = get_snapshot(should_not_be_rebuilt)
+    idf_py('build')
+    snapshot_should_be_rebuilt.assert_different(get_snapshot(should_be_rebuilt))
+    snapshot_should_not_be_rebuilt.assert_same(get_snapshot(should_not_be_rebuilt))
+
+
+# For this and the following test function, there are actually multiple logical
+# tests in one test function. It would be better to have every check in a separate
+# test case, but that would mean doing a full clean build each time. Having a few
+# related checks per test function looks like a reasonable compromise.
+# If the test function grows too big, try splitting it into two or more functions.
+@pytest.mark.usefixtures('test_app_copy')
+def test_rebuild_source_files(idf_py: IdfPyFunc) -> None:
+    idf_path = Path(os.environ['IDF_PATH'])
+    logging.info('initial build')
+    idf_py('build')
+
+    logging.info('updating a component source file rebuilds only that component')
+    component_files_patterns = [
+        'build/esp-idf/esp_system/libesp_system.a',
+        'build/esp-idf/esp_system/CMakeFiles/__idf_esp_system.dir/port/cpu_start.c.obj']
+    other_files_patterns = [
+        'build/esp-idf/lwip/liblwip.a',
+        'build/esp-idf/freertos/libfreertos.a',
+        *BOOTLOADER_BINS,
+        *PARTITION_BIN]
+    (idf_path / 'components/esp_system/port/cpu_start.c').touch()
+    rebuild_and_check(idf_py,
+                      component_files_patterns, other_files_patterns)
+
+    logging.info('changing a bootloader source file rebuilds the bootloader')
+    (idf_path / 'components/bootloader/subproject/main/bootloader_start.c').touch()
+    rebuild_and_check(idf_py,
+                      BOOTLOADER_BINS, APP_BINS + PARTITION_BIN)
+
+    logging.info('changing the partitions CSV file rebuilds only the partition table')
+    (idf_path / 'components/partition_table/partitions_singleapp.csv').touch()
+    rebuild_and_check(idf_py,
+                      PARTITION_BIN, APP_BINS + BOOTLOADER_BINS)
+
+    logging.info('sdkconfig update triggers full recompile')
+    # pick on .c, .cpp, .S file which includes sdkconfig.h:
+    obj_files = [
+        'build/esp-idf/newlib/CMakeFiles/__idf_newlib.dir/newlib_init.c.obj',
+        'build/esp-idf/nvs_flash/CMakeFiles/__idf_nvs_flash.dir/src/nvs_api.cpp.obj'
+        'build/esp-idf/esp_system/CMakeFiles/__idf_esp_system.dir/port/arch/xtensa/panic_handler_asm.S.obj'
+    ]
+    sdkconfig_files = [
+        'build/config/sdkconfig.h',
+        'build/config/sdkconfig.json'
+    ]
+    replace_in_file('sdkconfig', '# CONFIG_FREERTOS_UNICORE is not set', 'CONFIG_FREERTOS_UNICORE=y')
+    rebuild_and_check(idf_py, APP_BINS + BOOTLOADER_BINS + obj_files + sdkconfig_files, PARTITION_BIN)
+
+    logging.info('Updating project CMakeLists.txt triggers app recompile')
+    replace_in_file('CMakeLists.txt',
+                    '# placeholder_after_include_project_cmake',
+                    'add_compile_options("-DUSELESS_MACRO_DOES_NOTHING=1")')
+    rebuild_and_check(idf_py, APP_BINS + obj_files, PARTITION_BIN + BOOTLOADER_BINS + sdkconfig_files)
+
+
+@pytest.mark.usefixtures('test_app_copy')
+def test_rebuild_linker(idf_py: IdfPyFunc) -> None:
+    idf_path = Path(os.environ['IDF_PATH'])
+    logging.info('initial build')
+    idf_py('build')
+
+    logging.info('Updating rom ld file should re-link app and bootloader')
+    (idf_path / 'components/esp_rom/esp32/ld/esp32.rom.ld').touch()
+    rebuild_and_check(idf_py,
+                      APP_BINS + BOOTLOADER_BINS, PARTITION_BIN)
+
+    logging.info('Updating app-only sections ld file should only re-link the app')
+    (idf_path / 'components/esp_system/ld/esp32/sections.ld.in').touch()
+    rebuild_and_check(idf_py,
+                      APP_BINS, BOOTLOADER_BINS + PARTITION_BIN)
+
+    logging.info('Updating app-only memory ld file should only re-link the app')
+    (idf_path / 'components/esp_system/ld/esp32/memory.ld.in').touch()
+    rebuild_and_check(idf_py,
+                      APP_BINS, BOOTLOADER_BINS + PARTITION_BIN)
+
+    logging.info('Updating fragment file should only re-link the app')
+    (idf_path / 'components/esp_common/common.lf').touch()
+    rebuild_and_check(idf_py,
+                      APP_BINS, BOOTLOADER_BINS + PARTITION_BIN)