diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index 5bebaa040a..a93447068c 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -64,6 +64,23 @@ test_ldgen_on_host: variables: LC_ALL: C.UTF-8 +test_reproducible_build: + extends: .host_test_template + script: + - ./tools/ci/test_reproducible_build.sh + artifacts: + when: on_failure + paths: + - "**/sdkconfig" + - "**/build*/*.bin" + - "**/build*/*.elf" + - "**/build*/*.map" + - "**/build*/flasher_args.json" + - "**/build*/*.bin" + - "**/build*/bootloader/*.bin" + - "**/build*/partition_table/*.bin" + expire_in: 1 week + .host_fuzzer_test_template: extends: - .host_test_template diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index c5a12c66e3..3a59f7f8b0 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -130,6 +130,8 @@ - "tools/detect_python.sh" - "tools/detect_python.fish" + - "tools/ci/test_reproducible_build.sh" + .patterns-windows: &patterns-windows - "tools/windows/**/*" diff --git a/CMakeLists.txt b/CMakeLists.txt index 4be17e206d..b0b9713ad6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -147,7 +147,41 @@ endif() if(NOT ${CMAKE_C_COMPILER_VERSION} VERSION_LESS 8.0.0) if(CONFIG_COMPILER_HIDE_PATHS_MACROS) list(APPEND compile_options "-fmacro-prefix-map=${CMAKE_SOURCE_DIR}=.") - list(APPEND compile_options "-fmacro-prefix-map=${IDF_PATH}=IDF") + list(APPEND compile_options "-fmacro-prefix-map=${IDF_PATH}=/IDF") + endif() + + if(CONFIG_APP_REPRODUCIBLE_BUILD) + idf_build_set_property(DEBUG_PREFIX_MAP_GDBINIT "${BUILD_DIR}/prefix_map_gdbinit") + + list(APPEND compile_options "-fdebug-prefix-map=${IDF_PATH}=/IDF") + list(APPEND compile_options "-fdebug-prefix-map=${PROJECT_DIR}=/IDF_PROJECT") + list(APPEND compile_options "-fdebug-prefix-map=${BUILD_DIR}=/IDF_BUILD") + + # component dirs + idf_build_get_property(python PYTHON) + idf_build_get_property(idf_path IDF_PATH) + idf_build_get_property(component_dirs BUILD_COMPONENT_DIRS) + + execute_process( + COMMAND ${python} + "${idf_path}/tools/generate_debug_prefix_map.py" + "${BUILD_DIR}" + "${component_dirs}" + OUTPUT_VARIABLE result + RESULT_VARIABLE ret + ) + if(NOT ret EQUAL 0) + message(FATAL_ERROR "This is a bug. Please report to https://github.com/espressif/esp-idf/issues") + endif() + + spaces2list(result) + list(LENGTH component_dirs length) + math(EXPR max_index "${length} - 1") + foreach(index RANGE ${max_index}) + list(GET component_dirs ${index} folder) + list(GET result ${index} after) + list(APPEND compile_options "-fdebug-prefix-map=${folder}=${after}") + endforeach() endif() endif() diff --git a/Kconfig b/Kconfig index 928d274106..f1ede93458 100644 --- a/Kconfig +++ b/Kconfig @@ -201,6 +201,14 @@ mainmenu "Espressif IoT Development Framework Configuration" config APP_BUILD_USE_FLASH_SECTIONS bool # Whether to place code/data into memory-mapped flash sections + config APP_REPRODUCIBLE_BUILD + bool "Enable reproducible build" + default n + select COMPILER_HIDE_PATHS_MACROS + help + If enabled, all date, time, and path information would be eliminated. A .gdbinit file would be create + automatically. (or will be append if you have one already) + endmenu # Build type source "$COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE" diff --git a/components/app_update/esp_app_desc.c b/components/app_update/esp_app_desc.c index e0fdfd6418..ae3325b1c3 100644 --- a/components/app_update/esp_app_desc.c +++ b/components/app_update/esp_app_desc.c @@ -32,7 +32,7 @@ const __attribute__((section(".rodata_desc"))) esp_app_desc_t esp_app_desc = { .secure_version = 0, #endif -#ifdef CONFIG_APP_COMPILE_TIME_DATE +#if defined(CONFIG_APP_COMPILE_TIME_DATE) && !defined(CONFIG_APP_REPRODUCIBLE_BUILD) .time = __TIME__, .date = __DATE__, #else diff --git a/components/bootloader_support/src/bootloader_init.c b/components/bootloader_support/src/bootloader_init.c index 1d73f5dc26..41538bce85 100644 --- a/components/bootloader_support/src/bootloader_init.c +++ b/components/bootloader_support/src/bootloader_init.c @@ -91,5 +91,7 @@ void bootloader_enable_random(void) void bootloader_print_banner(void) { ESP_LOGI(TAG, "ESP-IDF %s 2nd stage bootloader", IDF_VER); +#ifndef CONFIG_APP_REPRODUCIBLE_BUILD ESP_LOGI(TAG, "compile time " __TIME__); +#endif } diff --git a/tools/ci/check_copyright_ignore.txt b/tools/ci/check_copyright_ignore.txt index e4eb03a258..a531dbd3ce 100644 --- a/tools/ci/check_copyright_ignore.txt +++ b/tools/ci/check_copyright_ignore.txt @@ -3713,17 +3713,14 @@ tools/kconfig_new/test/gen_kconfig_doc/test_kconfig_out.py tools/kconfig_new/test/gen_kconfig_doc/test_target_visibility.py tools/ldgen/entity.py tools/ldgen/fragments.py -tools/ldgen/generation.py tools/ldgen/ldgen.py tools/ldgen/ldgen_common.py -tools/ldgen/linker_script.py tools/ldgen/output_commands.py tools/ldgen/samples/template.ld tools/ldgen/sdkconfig.py tools/ldgen/test/data/linker_script.ld tools/ldgen/test/test_entity.py tools/ldgen/test/test_fragments.py -tools/ldgen/test/test_generation.py tools/ldgen/test/test_output_commands.py tools/mass_mfg/mfg_gen.py tools/mkdfu.py diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index 15d65cd320..487f4f8d3d 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -66,6 +66,7 @@ tools/ci/test_autocomplete.py tools/ci/test_build_system_cmake.sh tools/ci/test_check_kconfigs.py tools/ci/test_configure_ci_environment.sh +tools/ci/test_reproducible_build.sh tools/cmake/convert_to_cmake.py tools/docker/entrypoint.sh tools/docker/hooks/build diff --git a/tools/ci/test_reproducible_build.sh b/tools/ci/test_reproducible_build.sh new file mode 100755 index 0000000000..6d71c5011b --- /dev/null +++ b/tools/ci/test_reproducible_build.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -euo + +for path in \ + "examples/get-started/hello_world" \ + "examples/bluetooth/nimble/blecent"; do + cd "${IDF_PATH}/${path}" + + echo "CONFIG_APP_REPRODUCIBLE_BUILD=y" >sdkconfig + + idf.py -B build_first fullclean build + idf.py -B build_second fullclean build + + for item in \ + "partition_table/partition-table.bin" \ + "bootloader/bootloader.bin" \ + "bootloader/bootloader.elf" \ + "bootloader/bootloader.map" \ + "*.bin" \ + "*.elf" \ + "*.map"; do + diff -s build_first/${item} build_second/${item} # use glob, don't use double quotes + done + + # test gdb + rm -f gdb.txt + elf_file=$(find build_first -maxdepth 1 -iname '*.elf') + xtensa-esp32-elf-gdb -x build_first/prefix_map_gdbinit -ex 'set logging on' -ex 'set pagination off' -ex 'list' -ex 'quit' "$elf_file" + if grep "No such file or directory" gdb.txt; then + exit 1 + fi +done diff --git a/tools/cmake/component.cmake b/tools/cmake/component.cmake index 8bdba18be2..25ed8e7d52 100644 --- a/tools/cmake/component.cmake +++ b/tools/cmake/component.cmake @@ -189,6 +189,9 @@ function(__component_add component_dir prefix) # Set Kconfig related properties on the component __kconfig_component_init(${component_target}) + + # set BUILD_COMPONENT_DIRS build property + idf_build_set_property(BUILD_COMPONENT_DIRS ${component_dir} APPEND) endfunction() # diff --git a/tools/cmake/project.cmake b/tools/cmake/project.cmake index eed3def77a..433a3257bc 100644 --- a/tools/cmake/project.cmake +++ b/tools/cmake/project.cmake @@ -111,6 +111,7 @@ function(__project_info test_components) include(${sdkconfig_cmake}) idf_build_get_property(COMPONENT_KCONFIGS KCONFIGS) idf_build_get_property(COMPONENT_KCONFIGS_PROJBUILD KCONFIG_PROJBUILDS) + idf_build_get_property(debug_prefix_map_gdbinit DEBUG_PREFIX_MAP_GDBINIT) # Write project description JSON file idf_build_get_property(build_dir BUILD_DIR) diff --git a/tools/cmake/project_description.json.in b/tools/cmake/project_description.json.in index db18650e45..d14966a181 100644 --- a/tools/cmake/project_description.json.in +++ b/tools/cmake/project_description.json.in @@ -17,5 +17,6 @@ "COMPONENT_KCONFIGS_PROJBUILD" : "${COMPONENT_KCONFIGS_PROJBUILD}" }, "build_components" : ${build_components_json}, - "build_component_paths" : ${build_component_paths_json} + "build_component_paths" : ${build_component_paths_json}, + "debug_prefix_map_gdbinit": "${debug_prefix_map_gdbinit}" } diff --git a/tools/generate_debug_prefix_map.py b/tools/generate_debug_prefix_map.py new file mode 100644 index 0000000000..d9d46b4e2f --- /dev/null +++ b/tools/generate_debug_prefix_map.py @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +# General Workflow: +# 1. read all components dirs, a semicolon-separated string (cmake list) +# 2. map the component dir with a unique prefix /COMPONENT__DIR +# 2. write the prefix mapping file to $BUILD_DIR/prefix_map_gdbinit +# 3. print the unique prefix out, a space-separated string, will be used by the build system to add compile options. + +import argparse +import os +from typing import List + + +def component_name(component_dir: str) -> str: + return '/COMPONENT_{}_DIR'.format(os.path.basename(component_dir).upper()) + + +GDB_SUBSTITUTE_PATH_FMT = 'set substitute-path {} {}\n' + + +def write_gdbinit(build_dir: str, folders: List[str]) -> None: + gdb_init_filepath = os.path.join(build_dir, 'prefix_map_gdbinit') + + with open(gdb_init_filepath, 'w') as fw: + for folder in folders: + fw.write(f'{GDB_SUBSTITUTE_PATH_FMT.format(component_name(folder), folder)}') + + +def main(build_dir: str, folders: List[str]) -> None: + write_gdbinit(build_dir, folders) + print(' '.join([component_name(folder) for folder in folders]), end='') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='print the debug-prefix-map and write to ' + '$BUILD_DIR/prefix_map_gdbinit file') + + parser.add_argument('build_dir', + help='build dir') + parser.add_argument('folders', + help='component folders, semicolon separated string') + args = parser.parse_args() + + main(args.build_dir, args.folders.split(';')) diff --git a/tools/idf_py_actions/debug_ext.py b/tools/idf_py_actions/debug_ext.py index 53b94e810a..f534b03d87 100644 --- a/tools/idf_py_actions/debug_ext.py +++ b/tools/idf_py_actions/debug_ext.py @@ -7,6 +7,7 @@ import sys import threading import time from threading import Thread +from typing import Any, Dict, List from idf_py_actions.errors import FatalError from idf_py_actions.tools import ensure_build_directory @@ -83,19 +84,6 @@ def action_extensions(base_actions, project_path): print('Failed to close/kill {}'.format(target)) processes[target] = None # to indicate this has ended - def _get_commandline_options(ctx): - """ Return all the command line options up to first action """ - # This approach ignores argument parsing done Click - result = [] - - for arg in sys.argv: - if arg in ctx.command.commands_with_aliases: - break - - result.append(arg) - - return result - def create_local_gdbinit(gdbinit, elf_file): with open(gdbinit, 'w') as f: if os.name == 'nt': @@ -188,6 +176,13 @@ def action_extensions(base_actions, project_path): processes['openocd_outfile_name'] = openocd_out_name print('OpenOCD started as a background task {}'.format(process.pid)) + def get_gdb_args(gdbinit, project_desc: Dict[str, Any]) -> List[str]: + args = ['-x={}'.format(gdbinit)] + debug_prefix_gdbinit = project_desc.get('debug_prefix_map_gdbinit') + if debug_prefix_gdbinit: + args.append('-ix={}'.format(debug_prefix_gdbinit)) + return args + def gdbui(action, ctx, args, gdbgui_port, gdbinit, require_openocd): """ Asynchronous GDB-UI target @@ -198,7 +193,17 @@ def action_extensions(base_actions, project_path): if gdbinit is None: gdbinit = os.path.join(local_dir, 'gdbinit') create_local_gdbinit(gdbinit, os.path.join(args.build_dir, project_desc['app_elf'])) - args = ['gdbgui', '-g', gdb, '--gdb-args="-x={}"'.format(gdbinit)] + + # this is a workaround for gdbgui + # gdbgui is using shlex.split for the --gdb-args option. When the input is: + # - '"-x=foo -x=bar"', would return ['foo bar'] + # - '-x=foo', would return ['-x', 'foo'] and mess up the former option '--gdb-args' + # so for one item, use extra double quotes. for more items, use no extra double quotes. + gdb_args = get_gdb_args(gdbinit, project_desc) + gdb_args = '"{}"'.format(' '.join(gdb_args)) if len(gdb_args) == 1 else ' '.join(gdb_args) + args = ['gdbgui', '-g', gdb, '--gdb-args', gdb_args] + print(args) + if gdbgui_port is not None: args += ['--port', gdbgui_port] gdbgui_out_name = os.path.join(local_dir, GDBGUI_OUT_FILE) @@ -278,10 +283,10 @@ def action_extensions(base_actions, project_path): if gdbinit is None: gdbinit = os.path.join(local_dir, 'gdbinit') create_local_gdbinit(gdbinit, elf_file) - args = [gdb, '-x={}'.format(gdbinit)] + args = [gdb, *get_gdb_args(gdbinit, project_desc)] if gdb_tui is not None: args += ['-tui'] - t = Thread(target=run_gdb, args=(args, )) + t = Thread(target=run_gdb, args=(args,)) t.start() while True: try: diff --git a/tools/ldgen/generation.py b/tools/ldgen/generation.py index 910969dd60..74a6d0ba60 100644 --- a/tools/ldgen/generation.py +++ b/tools/ldgen/generation.py @@ -1,17 +1,6 @@ # -# Copyright 2021 Espressif Systems (Shanghai) CO LTD -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 # import collections @@ -251,7 +240,7 @@ class EntityNode(): self.child_placement(entity, sections, target, flags, sections_db) def get_output_sections(self): - return sorted(self.placements.keys(), key=' '.join) + return sorted(self.placements.keys(), key=lambda x: sorted(x)) # pylint: disable=W0108 class SymbolNode(EntityNode): diff --git a/tools/ldgen/linker_script.py b/tools/ldgen/linker_script.py index c15b397a24..968ac776a5 100644 --- a/tools/ldgen/linker_script.py +++ b/tools/ldgen/linker_script.py @@ -1,17 +1,6 @@ # -# Copyright 2021 Espressif Systems (Shanghai) CO LTD -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 # import collections @@ -69,8 +58,6 @@ class LinkerScript: target = member.target rules = member.rules - del rules[:] - rules.extend(mapping_rules[target]) except KeyError: message = GenerationException.UNDEFINED_REFERENCE + " to target '" + target + "'." diff --git a/tools/ldgen/test/test_generation.py b/tools/ldgen/test/test_generation.py index 28aac1c974..26e038aafe 100755 --- a/tools/ldgen/test/test_generation.py +++ b/tools/ldgen/test/test_generation.py @@ -1,18 +1,7 @@ #!/usr/bin/env python # -# Copyright 2021 Espressif Systems (Shanghai) CO LTD -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 # import collections @@ -102,32 +91,22 @@ class GenerationTest(unittest.TestCase): def generate_default_rules(self): rules = collections.defaultdict(list) - rules['flash_text'].append(InputSectionDesc(ROOT, ['.literal', '.literal.*', '.text', '.text.*'], [])) - rules['flash_rodata'].append(InputSectionDesc(ROOT, ['.rodata', '.rodata.*'], [])) - rules['dram0_data'].append(InputSectionDesc(ROOT, ['.data', '.data.*'], [])) - rules['dram0_data'].append(InputSectionDesc(ROOT, ['.dram', '.dram.*'], [])) rules['dram0_bss'].append(InputSectionDesc(ROOT, ['.bss', '.bss.*'], [])) rules['dram0_bss'].append(InputSectionDesc(ROOT, ['COMMON'], [])) + rules['dram0_data'].append(InputSectionDesc(ROOT, ['.data', '.data.*'], [])) + rules['dram0_data'].append(InputSectionDesc(ROOT, ['.dram', '.dram.*'], [])) + rules['flash_text'].append(InputSectionDesc(ROOT, ['.literal', '.literal.*', '.text', '.text.*'], [])) + rules['flash_rodata'].append(InputSectionDesc(ROOT, ['.rodata', '.rodata.*'], [])) rules['iram0_text'].append(InputSectionDesc(ROOT, ['.iram', '.iram.*'], [])) - rules['rtc_text'].append(InputSectionDesc(ROOT, ['.rtc.text', '.rtc.literal'], [])) + rules['rtc_bss'].append(InputSectionDesc(ROOT, ['.rtc.bss'], [])) rules['rtc_data'].append(InputSectionDesc(ROOT, ['.rtc.data'], [])) rules['rtc_data'].append(InputSectionDesc(ROOT, ['.rtc.rodata'], [])) - rules['rtc_bss'].append(InputSectionDesc(ROOT, ['.rtc.bss'], [])) + rules['rtc_text'].append(InputSectionDesc(ROOT, ['.rtc.text', '.rtc.literal'], [])) return rules def compare_rules(self, expected, actual): - self.assertEqual(set(expected.keys()), set(actual.keys())) - - for target in sorted(actual.keys()): - message = 'failed target %s' % target - a_cmds = actual[target] - e_cmds = expected[target] - - self.assertEqual(len(a_cmds), len(e_cmds), message) - - for a, e in zip(a_cmds, e_cmds): - self.assertEqual(a, e, message) + self.assertEqual(expected, actual) def get_default(self, target, rules): return rules[target][0]