test(build_system): move reproducible build test to pytest

.gitlab/ci/host-test.yml

@@ -68,22 +68,6 @@ test_ldgen_on_host:
   variables:
     LC_ALL: C.UTF-8
 
-test_reproducible_build:
-  extends: .host_test_template
-  script:
-    - ./tools/ci/test_reproducible_build.sh
-  artifacts:
-    when: on_failure
-    paths:
-      - "**/sdkconfig"
-      - "**/build*/*.bin"
-      - "**/build*/*.elf"
-      - "**/build*/*.map"
-      - "**/build*/flasher_args.json"
-      - "**/build*/*.bin"
-      - "**/build*/bootloader/*.bin"
-      - "**/build*/partition_table/*.bin"
-
 test_spiffs_on_host:
   extends: .host_test_template
   script:

.gitlab/ci/rules.yml

@@ -108,8 +108,6 @@
   - "tools/detect_python.sh"
   - "tools/detect_python.fish"
 
-  - "tools/ci/test_reproducible_build.sh"
-
   - "tools/gen_soc_caps_kconfig/*"
   - "tools/gen_soc_caps_kconfig/test/test_gen_soc_caps_kconfig.py"
 

tools/ci/executable-list.txt

@@ -80,7 +80,6 @@ tools/ci/push_to_github.sh
 tools/ci/sort_yaml.py
 tools/ci/test_autocomplete/test_autocomplete.py
 tools/ci/test_configure_ci_environment.sh
-tools/ci/test_reproducible_build.sh
 tools/docker/entrypoint.sh
 tools/esp_app_trace/logtrace_proc.py
 tools/esp_app_trace/sysviewtrace_proc.py

tools/ci/test_reproducible_build.sh

@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo
-
-for path in \
-  "examples/get-started/hello_world" \
-  "examples/bluetooth/nimble/blecent"; do
-  cd "${IDF_PATH}/${path}"
-
-  echo "CONFIG_APP_REPRODUCIBLE_BUILD=y" >sdkconfig
-
-  idf.py -B build_first fullclean build
-  idf.py -B build_second fullclean build
-
-  for item in \
-    "partition_table/partition-table.bin" \
-    "bootloader/bootloader.bin" \
-    "bootloader/bootloader.elf" \
-    "bootloader/bootloader.map" \
-    "*.bin" \
-    "*.elf" \
-    "*.map"; do
-    diff -s build_first/${item} build_second/${item} # use glob, don't use double quotes
-  done
-
-  # test gdb
-  rm -f gdb.txt
-  elf_file=$(find build_first -maxdepth 1 -iname '*.elf')
-  xtensa-esp32-elf-gdb -x build_first/prefix_map_gdbinit -ex 'set logging enabled' -ex 'set pagination off' -ex 'list app_main' -ex 'quit' "$elf_file"
-  if grep "No such file or directory" gdb.txt; then
-    exit 1
-  fi
-done

tools/test_build_system/conftest.py

@@ -95,7 +95,9 @@ def test_app_copy(func_work_dir: Path, request: FixtureRequest) -> typing.Genera
     # by default, use hello_world app and copy it to a temporary directory with
     # the name resembling that of the test
     copy_from = 'tools/test_build_system/build_test_app'
-    copy_to = request.node.name + '_app'
+    # sanitize test name in case pytest.mark.parametrize was used
+    test_name_sanitized = request.node.name.replace('[', '_').replace(']', '')
+    copy_to = test_name_sanitized + '_app'
 
     # allow overriding source and destination via pytest.mark.test_app_copy()
     mark = request.node.get_closest_marker('test_app_copy')

tools/test_build_system/test_reproducible_build.py

@@ -0,0 +1,67 @@
+# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+# This test checks the behavior of reproducible builds option.
+import logging
+import os
+import subprocess
+from pathlib import Path
+
+import pytest
+from test_build_system_helpers import append_to_file
+from test_build_system_helpers import bin_files_differ
+from test_build_system_helpers import BOOTLOADER_BINS
+from test_build_system_helpers import IdfPyFunc
+
+
+@pytest.mark.parametrize(
+    'app_name', [
+        pytest.param('blink', marks=[pytest.mark.test_app_copy('examples/get-started/blink')]),
+        pytest.param('blecent', marks=[pytest.mark.test_app_copy('examples/bluetooth/nimble/blecent')]),
+    ]
+)
+def test_reproducible_builds(app_name: str, idf_py: IdfPyFunc, test_app_copy: Path) -> None:
+    append_to_file(test_app_copy / 'sdkconfig', 'CONFIG_APP_REPRODUCIBLE_BUILD=y')
+    build_first = test_app_copy / 'build_first'
+    build_second = test_app_copy / 'build_second'
+
+    logging.info(f'Building in {build_first} directory')
+    idf_py('-B', str(build_first), 'build')
+
+    elf_file = build_first / f'{app_name}.elf'
+    logging.info(f'Checking that various paths are not included in the ELF file')
+    strings_output = subprocess.check_output(
+        ['xtensa-esp32-elf-strings', str(elf_file)],
+        encoding='utf-8'
+    )
+    idf_path = os.environ['IDF_PATH']
+    assert str(idf_path) not in strings_output, f'{idf_path} found in {elf_file}'
+    assert str(test_app_copy) not in strings_output, f'{test_app_copy} found in {elf_file}'
+
+    logging.info(f'Building in {build_second} directory')
+    idf_py('-B', str(build_second), 'build')
+
+    logging.info(f'Comparing build artifacts')
+    artifacts_to_check = [
+        f'build/{app_name}.map',
+        f'build/{app_name}.elf',
+        f'build/{app_name}.bin',
+    ] + BOOTLOADER_BINS
+
+    for artifact in artifacts_to_check:
+        path_first = artifact.replace('build/', f'{build_first}/')
+        path_second = artifact.replace('build/', f'{build_second}/')
+
+        assert not bin_files_differ(path_first, path_second), f'{path_first} and {path_second} differ'
+
+    logging.info(f'Checking that GDB works with CONFIG_APP_REPRODUCIBLE_BUILD=y')
+    gdb_output = subprocess.check_output([
+        'xtensa-esp32-elf-gdb',
+        '--batch', '--quiet',
+        '-x', f'{build_first}/prefix_map_gdbinit',
+        '-ex', 'set logging enabled',
+        '-ex', 'set pagination off',
+        '-ex', 'list app_main',
+        str(elf_file)
+    ], encoding='utf-8', stderr=subprocess.STDOUT, cwd=str(build_first))
+
+    assert 'No such file or directory' not in gdb_output, f'GDB failed to find app_main in {elf_file}:\n{gdb_output}'