Merge branch 'bugfix/check_partitions_fit' into 'master'

cmake partition_table: Check binaries fit in partition spaces at build time

Closes IDF-1539 and IDFGH-2503

See merge request espressif/esp-idf!9330

components/bootloader/project_include.cmake

@@ -1,4 +1,4 @@
-set(BOOTLOADER_OFFSET CONFIG_BOOTLOADER_OFFSET_IN_FLASH)
+set(BOOTLOADER_OFFSET ${CONFIG_BOOTLOADER_OFFSET_IN_FLASH})
 
 # Do not generate flash file when building bootloader
 if(BOOTLOADER_BUILD OR NOT CONFIG_APP_BUILD_BOOTLOADER)

components/esptool_py/CMakeLists.txt

@@ -1,4 +1,4 @@
-idf_component_register(REQUIRES bootloader)
+idf_component_register(REQUIRES bootloader PRIV_REQUIRES partition_table)
 
 if(NOT BOOTLOADER_BUILD)
     idf_build_get_property(build_dir BUILD_DIR)
@@ -41,6 +41,7 @@ consist of two ota app without factory or test partitions.")
     endif()
 
     if(CONFIG_APP_BUILD_GENERATE_BINARIES)
+        # Generate flasher args files
         file(READ "flasher_args.json.in" flasher_args_content)
         string(CONFIGURE "${flasher_args_content}" flasher_args_content)
 
@@ -48,5 +49,21 @@ consist of two ota app without factory or test partitions.")
                     CONTENT "${flasher_args_content}")
         file_generate("${CMAKE_BINARY_DIR}/flasher_args.json"
                     INPUT "${CMAKE_CURRENT_BINARY_DIR}/flasher_args.json.in")
+
+        # Generate app_check_size_command target to check the app size against the partition table parameters
+        partition_table_add_check_size_target(app_check_size
+            DEPENDS gen_project_binary
+            BINARY_PATH "${build_dir}/${PROJECT_BIN}"
+            PARTITION_TYPE app)
+        add_dependencies(app app_check_size)
     endif()
+endif()  # NOT BOOTLOADER_BUILD
+
+if(BOOTLOADER_BUILD)
+  # Generate bootloader post-build check of the bootloader size against the offset
+  partition_table_add_check_bootloader_size_target(bootloader_check_size
+      DEPENDS gen_project_binary
+      BOOTLOADER_BINARY_PATH "${build_dir}/${PROJECT_BIN}"
+      RESULT bootloader_check_size_command)
+  add_dependencies(app bootloader_check_size)  # note: in the subproject, so the target is 'app'...
 endif()

components/partition_table/Kconfig.projbuild

@@ -9,21 +9,79 @@ menu "Partition Table"
             be found.
 
             The predefined partition table CSV descriptions can be found
-            in the components/partition_table directory. Otherwise it's
-            possible to create a new custom partition CSV for your application.
+            in the components/partition_table directory. These are mostly intended
+            for example and development use, it's expect that for production use you
+            will copy one of these CSV files and create a custom partition CSV for
+            your application.
 
         config PARTITION_TABLE_SINGLE_APP
             bool "Single factory app, no OTA"
+            help
+                This is the default partition table, designed to fit into a 2MB or
+                larger flash with a single 1MB app partition.
+
+                The corresponding CSV file in the IDF directory is
+                components/partition_table/partitions_singleapp.csv
+
+                This partition table is not suitable for an app that needs OTA
+                (over the air update) capability.
+        config PARTITION_TABLE_SINGLE_APP_LARGE
+            bool "Single factory app (large), no OTA"
+            help
+                This is a variation of the default partition table, that expands
+                the 1MB app partition size to 1.5MB to fit more code.
+
+                The corresponding CSV file in the IDF directory is
+                components/partition_table/partitions_singleapp_large.csv
+
+                This partition table is not suitable for an app that needs OTA
+                (over the air update) capability.
         config PARTITION_TABLE_TWO_OTA
             bool "Factory app, two OTA definitions"
+            help
+                This is a basic OTA-enabled partition table with a factory app
+                partition plus two OTA app partitions. All are 1MB, so this
+                partition table requires 4MB or larger flash size.
+
+                The corresponding CSV file in the IDF directory is
+                components/partition_table/partitions_two_ota.csv
         config PARTITION_TABLE_CUSTOM
             bool "Custom partition table CSV"
+            help
+                Specify the path to the partition table CSV to use for your project.
+
+                Consult the Partition Table section in the ESP-IDF Programmers Guide
+                for more information.
         config PARTITION_TABLE_SINGLE_APP_ENCRYPTED_NVS
             bool "Single factory app, no OTA, encrypted NVS"
             depends on !ESP32_COREDUMP_ENABLE_TO_FLASH && NVS_ENCRYPTION
+            help
+                This is a variation of the default "Single factory app, no OTA" partition table
+                that supports encrypted NVS when using flash encryption. See the Flash Encryption section
+                in the ESP-IDF Programmers Guide for more information.
+
+                The corresponding CSV file in the IDF directory is
+                components/partition_table/partitions_singleapp_encr_nvs.csv
+        config PARTITION_TABLE_SINGLE_APP_LARGE_ENC_NVS
+            bool "Single factory app (large), no OTA, encrypted NVS"
+            depends on !ESP32_COREDUMP_ENABLE_TO_FLASH && NVS_ENCRYPTION
+            help
+                This is a variation of the "Single factory app (large), no OTA" partition table
+                that supports encrypted NVS when using flash encryption. See the Flash Encryption section
+                in the ESP-IDF Programmers Guide for more information.
+
+                The corresponding CSV file in the IDF directory is
+                components/partition_table/partitions_singleapp_large_encr_nvs.csv
         config PARTITION_TABLE_TWO_OTA_ENCRYPTED_NVS
             bool "Factory app, two OTA definitions, encrypted NVS"
             depends on !ESP_COREDUMP_ENABLE_TO_FLASH && NVS_ENCRYPTION
+            help
+                This is a variation of the "Factory app, two OTA definitions" partition table
+                that supports encrypted NVS when using flash encryption. See the Flash Encryption section
+                in the ESP-IDF Programmers Guide for more information.
+
+                The corresponding CSV file in the IDF directory is
+                components/partition_table/partitions_two_ota_encr_nvs.csv
     endchoice
 
     config PARTITION_TABLE_CUSTOM_FILENAME
@@ -38,6 +96,9 @@ menu "Partition Table"
         default "partitions_singleapp.csv" if PARTITION_TABLE_SINGLE_APP && !ESP_COREDUMP_ENABLE_TO_FLASH
         default "partitions_singleapp_coredump.csv" if PARTITION_TABLE_SINGLE_APP && ESP_COREDUMP_ENABLE_TO_FLASH
         default "partitions_singleapp_encr_nvs.csv" if PARTITION_TABLE_SINGLE_APP_ENCRYPTED_NVS
+        default "partitions_singleapp_large.csv" if PARTITION_TABLE_SINGLE_APP_LARGE && !ESP_COREDUMP_ENABLE_TO_FLASH
+        default "partitions_singleapp_large_coredump.csv" if PARTITION_TABLE_SINGLE_APP_LARGE && ESP_COREDUMP_ENABLE_TO_FLASH # NOERROR
+        default "partitions_singleapp_large_encr_nvs.csv" if PARTITION_TABLE_SINGLE_APP_LARGE_ENC_NVS
         default "partitions_two_ota.csv" if PARTITION_TABLE_TWO_OTA && !ESP_COREDUMP_ENABLE_TO_FLASH
         default "partitions_two_ota_coredump.csv" if PARTITION_TABLE_TWO_OTA && ESP_COREDUMP_ENABLE_TO_FLASH
         default "partitions_two_ota_encr_nvs.csv" if PARTITION_TABLE_TWO_OTA_ENCRYPTED_NVS

components/partition_table/check_sizes.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python
+#
+# check_sizes.py is a tool run by the ESP-IDF build system
+# to check a particular binary fits in the available partitions of
+# a particular type/subtype. Can be used to check if the app binary fits in
+# all available app partitions, for example.
+#
+# (Can also check if the bootloader binary fits before the partition table.)
+#
+# Copyright 2020 Espressif Systems (Shanghai) PTE LTD
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import division, print_function, unicode_literals
+
+import argparse
+import io  # noqa: F401 # pylint: disable=unused-import
+import os
+import sys
+
+try:
+    from typing import IO  # noqa: F401 # pylint: disable=unused-import
+except ImportError:
+    pass  # used for type hinting only
+
+import gen_esp32part
+from gen_esp32part import PartitionTable, get_ptype_as_int, get_subtype_as_int
+
+allow_failures = False
+
+
+def _file_size(f):   # type: (IO) -> int
+    before = f.tell()
+    f.seek(0, 2)  # seek to end
+    result = f.tell()
+    f.seek(before)
+    return result
+
+
+def _fail(msg):   # type: (str) -> None
+    if allow_failures:
+        print('Warning: {}'.format(msg))
+    else:
+        raise SystemExit('Error: {}'.format(msg))
+
+
+def check_bootloader(partition_table_offset, bootloader_offset, binary_file):  # type: (int, int, IO) -> None
+    max_size = partition_table_offset - bootloader_offset
+    bootloader_size = _file_size(binary_file)
+    if bootloader_size > max_size:
+        msg = ('Bootloader binary size {:#x} bytes is too large for partition table offset {:#02x}. ' +
+               'Bootloader binary can be maximum {:#x} ({}) bytes unless the partition table offset ' +
+               'is increased in the Partition Table section of the project configuration menu.').format(
+                   bootloader_size, partition_table_offset, max_size, max_size)
+        _fail(msg)
+    free_size = max_size - bootloader_size
+    print('Bootloader binary size {:#x} bytes. {:#x} bytes ({}%) free.'.format(
+          bootloader_size, free_size, round(free_size * 100 / bootloader_size)))
+
+
+def check_partition(ptype, subtype, partition_table_file, bin_file):  # type: (str, str, io.IOBase, IO) -> None
+    table, _ = PartitionTable.from_file(partition_table_file)
+    ptype_str = str(ptype)
+    ptype = get_ptype_as_int(ptype)
+    partitions = [p for p in table if p.type == ptype]
+
+    if subtype is not None:
+        ptype_str += ' ({})'.format(subtype)
+        subtype = get_subtype_as_int(ptype, subtype)
+        partitions = [p for p in partitions if p.subtype == subtype]
+
+    if len(partitions) == 0:
+        print('WARNING: Partition table does not contain any partitions matching {}'.format(ptype_str))
+        return
+
+    bin_name = os.path.basename(bin_file.name)
+    bin_size = _file_size(bin_file)
+    smallest_size = min(p.size for p in partitions)
+    if smallest_size >= bin_size:
+        free_size = smallest_size - bin_size
+        print('{} binary size {:#x} bytes. Smallest {} partition is {:#x} bytes. {:#x} bytes ({}%) free.'.format(
+            bin_name, bin_size, ptype_str, smallest_size, free_size, round(free_size * 100 / smallest_size)))
+        return
+
+    too_small_partitions = [p for p in partitions if p.size < bin_size]
+    if len(partitions) == 1:
+        msg = '{} partition is'.format(ptype_str)
+    elif len(partitions) == len(too_small_partitions):
+        msg = 'All {} partitions are'.format(ptype_str)
+    else:
+        msg = '{}/{} {} partitions are'.format(len(too_small_partitions), len(partitions), ptype_str)
+    msg += ' too small for binary {} size {:#x}:'.format(bin_name, bin_size)
+    for p in too_small_partitions:
+        msg += '\n  - {} (overflow {:#x})'.format(p, bin_size - p.size)
+    if not allow_failures and len(partitions) == len(too_small_partitions):
+        # if some partitions can fit the binary then just print a warning
+        raise SystemExit('Error: ' + msg)
+    else:
+        print('Warning: ' + msg)
+
+
+def main():  # type: () -> None
+    global allow_failures  # pylint: disable=global-statement
+
+    parser = argparse.ArgumentParser(description='Check binary sizes against partition table entries')
+    parser.add_argument('--target', choices=['esp32', 'esp32s2'])
+    parser.add_argument('--allow_failures', action='store_true', help='If true, failures will print warnings but not exit with an error')
+    parser.add_argument('--offset', '-o', help='Set partition table offset', default='0x8000')
+
+    subparsers = parser.add_subparsers(dest='check_target',
+                                       help='Type of binary to check against partition table layout')
+    sp_bootloader = subparsers.add_parser('bootloader')
+    sp_bootloader.add_argument('bootloader_offset', help='Hex offset of bootloader in flash')
+    sp_bootloader.add_argument('bootloader_binary', type=argparse.FileType('rb'), help='Bootloader binary (.bin) file from build output')
+
+    sp_part = subparsers.add_parser('partition')
+    sp_part.add_argument('--type', type=str, help='Check the file size against all partitions of this type.', required=True)
+    sp_part.add_argument('--subtype', type=str, help='Optional, only check the file size against all partitions of this subtype.')
+    sp_part.add_argument('partition_table', type=argparse.FileType('rb'), help='Partition table file')
+    sp_part.add_argument('binary', type=argparse.FileType('rb'), help='Binary file which will have the size checked')
+
+    args = parser.parse_args()
+
+    gen_esp32part.quiet = True
+
+    args.offset = int(args.offset, 0)
+    gen_esp32part.offset_part_table = args.offset
+
+    if args.check_target is None:  # add_subparsers only has a 'required' argument since Python 3
+        parser.print_help()
+        sys.exit(1)
+    if args.check_target == 'bootloader':
+        check_bootloader(args.offset, int(args.bootloader_offset, 0), args.bootloader_binary)
+    else:
+        check_partition(args.type, args.subtype, args.partition_table, args.binary)
+
+
+if __name__ == '__main__':
+    main()

components/partition_table/gen_esp32part.py

@@ -48,6 +48,18 @@ TYPES = {
     'data': DATA_TYPE,
 }
 
+
+def get_ptype_as_int(ptype):
+    """ Convert a string which might be numeric or the name of a partition type to an integer """
+    try:
+        return TYPES[ptype]
+    except KeyError:
+        try:
+            return int(ptype, 0)
+        except TypeError:
+            return ptype
+
+
 # Keep this map in sync with esp_partition_subtype_t enum in esp_partition.h
 SUBTYPES = {
     APP_TYPE: {
@@ -67,6 +79,18 @@ SUBTYPES = {
     },
 }
 
+
+def get_subtype_as_int(ptype, subtype):
+    """ Convert a string which might be numeric or the name of a partition subtype to an integer """
+    try:
+        return SUBTYPES[get_ptype_as_int(ptype)][subtype]
+    except KeyError:
+        try:
+            return int(subtype, 0)
+        except TypeError:
+            return subtype
+
+
 quiet = False
 md5sum = True
 secure = False
@@ -89,6 +113,18 @@ class PartitionTable(list):
     def __init__(self):
         super(PartitionTable, self).__init__(self)
 
+    @classmethod
+    def from_file(cls, f):
+        data = f.read()
+        data_is_binary = data[0:2] == PartitionDefinition.MAGIC_BYTES
+        if data_is_binary:
+            status('Parsing binary partition input...')
+            return cls.from_binary(data), True
+
+        data = data.decode()
+        status('Parsing CSV input...')
+        return cls.from_csv(data), False
+
     @classmethod
     def from_csv(cls, csv_contents):
         res = PartitionTable()
@@ -149,20 +185,8 @@ class PartitionTable(list):
         """ Return a partition by type & subtype, returns
         None if not found """
         # convert ptype & subtypes names (if supplied this way) to integer values
-        try:
-            ptype = TYPES[ptype]
-        except KeyError:
-            try:
-                ptype = int(ptype, 0)
-            except TypeError:
-                pass
-        try:
-            subtype = SUBTYPES[int(ptype)][subtype]
-        except KeyError:
-            try:
-                subtype = int(subtype, 0)
-            except TypeError:
-                pass
+        ptype = get_ptype_as_int(ptype)
+        subtype = get_subtype_as_int(ptype, subtype)
 
         for p in self:
             if p.type == ptype and p.subtype == subtype:
@@ -471,15 +495,7 @@ def main():
     md5sum = not args.disable_md5sum
     secure = args.secure
     offset_part_table = int(args.offset, 0)
-    input = args.input.read()
-    input_is_binary = input[0:2] == PartitionDefinition.MAGIC_BYTES
-    if input_is_binary:
-        status('Parsing binary partition input...')
-        table = PartitionTable.from_binary(input)
-    else:
-        input = input.decode()
-        status('Parsing CSV input...')
-        table = PartitionTable.from_csv(input)
+    table, input_is_binary = PartitionTable.from_file(args.input)
 
     if not args.no_verify:
         status('Verifying table...')

components/partition_table/partitions_singleapp_large.csv

@@ -0,0 +1,5 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+# Note: if you have increased the bootloader size, make sure to update the offsets to avoid overlap
+nvs,      data, nvs,     ,        0x6000,
+phy_init, data, phy,     ,        0x1000,
+factory,  app,  factory, ,        1500K,

components/partition_table/partitions_singleapp_large_coredump.csv

@@ -0,0 +1,6 @@
+# Name,   Type, SubType, Offset,  Size
+# Note: if you have increased the bootloader size, make sure to update the offsets to avoid overlap
+nvs,      data, nvs,           ,  0x6000
+phy_init, data, phy,           ,  0x1000
+factory,  app,  factory,       ,  1500K
+coredump, data, coredump,      ,  64K

components/partition_table/partitions_singleapp_large_encr_nvs.csv

@@ -0,0 +1,6 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+# Note: if you have increased the bootloader size, make sure to update the offsets to avoid overlap
+nvs,      data, nvs,     ,        0x6000,
+phy_init, data, phy,     ,        0x1000,
+factory,  app,  factory, ,        1500K,
+nvs_key,  data, nvs_keys,,        0x1000, encrypted

components/partition_table/project_include.cmake

@@ -1,6 +1,11 @@
-if(NOT BOOTLOADER_BUILD)
-    set(PARTITION_TABLE_OFFSET ${CONFIG_PARTITION_TABLE_OFFSET})
+set(PARTITION_TABLE_OFFSET ${CONFIG_PARTITION_TABLE_OFFSET})
 
+set(PARTITION_TABLE_CHECK_SIZES_TOOL_PATH "${CMAKE_CURRENT_LIST_DIR}/check_sizes.py")
+
+idf_build_get_property(build_dir BUILD_DIR)
+idf_build_set_property(PARTITION_TABLE_BIN_PATH "${build_dir}/partition_table/partition-table.bin")
+
+if(NOT BOOTLOADER_BUILD)
     # Set PARTITION_CSV_PATH to the configured partition CSV file
     # absolute path
     if(CONFIG_PARTITION_TABLE_CUSTOM)
@@ -49,3 +54,57 @@ function(partition_table_get_partition_info result get_part_info_args part_info)
     endif()
     set(${result} ${info} PARENT_SCOPE)
 endfunction()
+
+# Add a custom target (see add_custom_target) that checks a given binary built by the build system
+# doesn't overflow any partitions with the given partition type and (optional) sub-type.
+#
+# Adding the target doesn't mean it gets called during the build, use add_dependencies to make another
+# target depend on this one.
+#
+# Arguments:
+# - target name - (first argument) name of the target to create
+# - DEPENDS - dependencies the target should have (i.e. whatever target generates the binary).
+# - BINARY_PATH - path to the target binary file to check
+# - PARTITION_TYPE - partition type to check against
+# - PARTITION_SUBTYPE - (optional) partition subtype to check against
+
+function(partition_table_add_check_size_target target_name)
+    # result binary_path partition_type partition_subtype
+    set(args BINARY_PATH PARTITION_TYPE PARTITION_SUBTYPE)
+    set(multi_args DEPENDS)
+    cmake_parse_arguments(CMD "" "${args}" "${multi_args}" ${ARGN})
+
+    idf_build_get_property(python PYTHON)
+    idf_build_get_property(table_bin PARTITION_TABLE_BIN_PATH)
+    if(CMD_PARTITION_SUBTYPE)
+        set(subtype_arg --subtype ${CMD_PARTITION_SUBTYPE})
+    else()
+        set(subtype_arg)
+    endif()
+    set(command ${python} ${PARTITION_TABLE_CHECK_SIZES_TOOL_PATH}
+        --offset ${PARTITION_TABLE_OFFSET}
+        partition --type ${CMD_PARTITION_TYPE} ${subtype_arg}
+        ${table_bin} ${CMD_BINARY_PATH})
+
+    add_custom_target(${target_name} COMMAND ${command} DEPENDS ${CMD_DEPENDS} partition_table_bin)
+endfunction()
+
+# Add a custom target (see add_custom_target) that checks a bootloader binary
+# built by the build system will not overlap the partition table.
+#
+# Adding the target doesn't mean it gets called during the build, use add_dependencies to make another
+# target depend on this one.
+#
+# Arguments:
+# - target name - (first argument) name of the target to create
+# - DEPENDS - dependencies the new target should have (i.e. whatever target generates the bootloader binary)
+# - BOOTLOADER_BINARY_PATH - path to bootloader binary file
+function(partition_table_add_check_bootloader_size_target target_name)
+    cmake_parse_arguments(CMD "" "BOOTLOADER_BINARY_PATH" "DEPENDS" ${ARGN})
+    idf_build_get_property(python PYTHON)
+
+    set(command ${python} ${PARTITION_TABLE_CHECK_SIZES_TOOL_PATH}
+        --offset ${PARTITION_TABLE_OFFSET}
+        bootloader ${BOOTLOADER_OFFSET} ${CMD_BOOTLOADER_BINARY_PATH})
+    add_custom_target(${target_name} COMMAND ${command} DEPENDS ${CMD_DEPENDS})
+endfunction()

components/partition_table/test_gen_esp32part_host/check_sizes_test.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+from __future__ import division, print_function
+
+import io
+import sys
+import unittest
+
+from test_utils import Py23TestCase
+
+try:
+    from typing import IO
+except ImportError:
+    pass  # only needed for type annotations
+
+try:
+    import check_sizes
+except ImportError:
+    sys.path.append('..')
+    import check_sizes
+
+try:
+    import gen_esp32part
+except ImportError:
+    sys.path.append('..')
+    import gen_esp32part
+
+
+BASE_CSV = """
+# Name,   Type, SubType, Offset,   Size, Flags
+# Note: if you have increased the bootloader size, make sure to update the offsets to avoid overlap
+nvs,      data, nvs,     ,        0x4000,
+otadata,  data, ota,     ,        0x2000,
+phy_init, data, phy,     ,        0x1000,
+
+"""
+
+
+def gen_table(factory_appsize=None, ota0_appsize=None, ota1_appsize=None):  # type: (str, str, str) -> io.BytesIO
+    """ generate a partition table binary with up to 3 app partitions with the specified sizes. """
+    csv = BASE_CSV
+    if factory_appsize:
+        csv += 'factory, app, factory, , {}\n'.format(factory_appsize)
+    if ota0_appsize:
+        csv += 'ota0, app, ota_0, , {}\n'.format(ota0_appsize)
+    if ota1_appsize:
+        csv += 'ota1, app, ota_1, , {}\n'.format(ota1_appsize)
+
+    table = gen_esp32part.PartitionTable.from_csv(csv)
+    return io.BytesIO(table.to_binary())
+
+
+def fake_file(kilobytes, name):  # type: (int, str) -> IO
+    result = io.BytesIO(b'\xEE' * kilobytes * 1024)
+    result.name = name
+    return result
+
+
+class TestAppSizes(Py23TestCase):
+
+    def test_sizes_ok(self):  # type: () -> None
+        pt = gen_table('1M')
+        app = fake_file(512, 'test.bin')
+        check_sizes.check_partition('app', None, pt, app)
+
+    def test_single_part_too_small(self):  # type: () -> None
+        pt = gen_table('1M')
+        app = fake_file(1500, 'too_big.bin')
+        with self.assertRaises(SystemExit) as e:
+            check_sizes.check_partition('app', None, pt, app)
+        self.assertIn('too small', str(e.exception))
+        self.assertIn('too_big.bin', str(e.exception))
+
+    def test_all_parts_too_small(self):  # type: () -> None
+        pt = gen_table('500K', '500K', '500K')
+        app = fake_file(501, 'just_too_big.bin')
+        with self.assertRaises(SystemExit) as e:
+            check_sizes.check_partition('app', None, pt, app)
+        self.assertIn('All', str(e.exception))
+        self.assertIn('too small', str(e.exception))
+        self.assertIn('just_too_big.bin', str(e.exception))
+
+    def test_one_part_too_small_warning(self):  # type: () -> None
+        pt = gen_table('500K', '1M', '1M')
+        app = fake_file(501, 'big.bin')
+        # this will print a warning, no easy way to verify it looks correct
+        check_sizes.check_partition('app', None, pt, app)
+
+
+class TestBootloaderSizes(Py23TestCase):
+
+    def test_sizes_ok(self):  # type: () -> None
+        bootloader = fake_file(25, 'test.bin')
+        check_sizes.check_bootloader(0x8000, 0x1000, bootloader)
+
+    def test_bootloader_too_big_default(self):  # type: () -> None
+        bootloader = fake_file(40, 'test.bin')
+        with self.assertRaises(SystemExit) as e:
+            check_sizes.check_bootloader(0x8000, 0x1000, bootloader)
+        self.assertIn('overlap', str(e.exception))
+        # move the partition table offset up, it will pass
+        check_sizes.check_bootloader(0xb000, 0x1000, bootloader)
+
+
+if __name__ == '__main__':
+    unittest.main()

components/partition_table/test_gen_esp32part_host/gen_esp32part_tests.py

@@ -10,6 +10,8 @@ import sys
 import tempfile
 import unittest
 
+from test_utils import Py23TestCase
+
 try:
     import gen_esp32part
 except ImportError:
@@ -59,18 +61,6 @@ def _strip_trailing_ffs(binary_table):
     return binary_table
 
 
-class Py23TestCase(unittest.TestCase):
-
-    def __init__(self, *args, **kwargs):
-        super(Py23TestCase, self).__init__(*args, **kwargs)
-        try:
-            self.assertRaisesRegex
-        except AttributeError:
-            # assertRaisesRegexp is deprecated in Python3 but assertRaisesRegex doesn't exist in Python2
-            # This fix is used in order to avoid using the alias from the six library
-            self.assertRaisesRegex = self.assertRaisesRegexp
-
-
 class CSVParserTests(Py23TestCase):
 
     def test_simple_partition(self):

components/partition_table/test_gen_esp32part_host/test_utils.py

@@ -0,0 +1,18 @@
+import unittest
+
+try:
+    from typing import Any
+except ImportError:
+    pass  # only needed to check type annotations
+
+
+class Py23TestCase(unittest.TestCase):
+
+    def __init__(self, *args, **kwargs):  # type: (Any, Any) -> None
+        super(Py23TestCase, self).__init__(*args, **kwargs)
+        try:
+            self.assertRaisesRegex
+        except AttributeError:
+            # assertRaisesRegexp is deprecated in Python3 but assertRaisesRegex doesn't exist in Python2
+            # This fix is used in order to avoid using the alias from the six library
+            self.assertRaisesRegex = self.assertRaisesRegexp  # type: ignore

docs/en/api-guides/partition-tables.rst

@@ -169,6 +169,24 @@ To display the contents of a binary partition table on stdout (this is how the s
 
   python gen_esp32part.py binary_partitions.bin
 
+Partition Size Checks
+---------------------
+
+The ESP-IDF build system will automatically check if generated binaries fit in the available partition space, and will fail with an error if a binary is too large.
+
+Currently these checks are performed for the following binaries:
+
+* Bootloader binary must fit in space before partition table (see :ref:`secure-boot-bootloader-size`)
+* App binary should fit in at least one partition of type "app". If the app binary doesn't fit in any app partition, the build will fail. If it only fits in some of the app partitions, a warning is printed about this.
+
+.. note::
+
+   Although the build process will fail if the size check returns an error, the binary files are still generated and can be flashed (although they may not work if they are too large for the available space.)
+
+.. note::
+
+   Build system binary size checks are only performed when using the CMake build system. When using the legacy GNU Make build system, file sizes can be checked manually or an error will be logged during boot.
+
 MD5 checksum
 ~~~~~~~~~~~~
 

docs/en/security/secure-boot-bootloader-size.rst

@@ -5,14 +5,18 @@
 Bootloader Size
 ===============
 
-
+{IDF_TARGET_DEFAULT_MAX_BOOTLOADER_SIZE:default = "0x8000 (32768)", esp32 = "0x7000 (28672)", esp32s2 = "0x7000 (28672)"}
 {IDF_TARGET_MAX_BOOTLOADER_SIZE:default = "64KB (0x10000 bytes)", esp32 = "48KB (0xC000 bytes)"}
 {IDF_TARGET_MAX_PARTITION_TABLE_OFFSET:default = "0x12000", esp32 = "0xE000"}
 .. Above is calculated as 0x1000 at start of flash + IDF_TARGET_MAX_BOOTLOADER_SIZE + 0x1000 signature sector
 
-When secure boot is enabled the bootloader app binary ``bootloader.bin`` may exceed the default bootloader size limit. This is especially likely if flash encryption is enabled as well. The default size limit is 0x7000 (28672) bytes (partition table offset 0x8000 - bootloader offset 0x1000).
+When secure boot is enabled the bootloader app binary ``bootloader.bin`` may exceed the bootloader binary size limit and overlap with the partition table. This is especially likely if flash encryption is enabled as well. When using the default :ref:`CONFIG_PARTITION_TABLE_OFFSET` value 0x8000, the size limit is {IDF_TARGET_DEFAULT_MAX_BOOTLOADER_SIZE} bytes.
 
-If the bootloader becomes too large, the {IDF_TARGET_NAME} will fail to boot - errors will be logged about either invalid partition table or invalid bootloader checksum.
+If the bootloader binary is too large, then the bootloader build will fail with an error "Bootloader binary size [..] is too large for partition table offset". If the bootloader binary is flashed anyhow then the {IDF_TARGET_NAME} will fail to boot - errors will be logged about either invalid partition table or invalid bootloader checksum.
+
+.. note::
+
+   The bootloader size check only happens in the CMake Build System, when using the legacy GNU Make Build System the size is not checked but the {IDF_TARGET_NAME} will fail to boot if bootloader is too large.
 
 When Secure Boot V2 is enabled, there is also an absolute binary size limit of {IDF_TARGET_MAX_BOOTLOADER_SIZE} (excluding the 4KB signature), because the bootloader is first loaded into a fixed size buffer for verification.
 
@@ -20,6 +24,6 @@ Options to work around this are:
 
 - Set :ref:`bootloader compiler optimization <CONFIG_BOOTLOADER_COMPILER_OPTIMIZATION>` back to "Size" if it has been changed from this default value.
 - Reduce :ref:`bootloader log level <CONFIG_BOOTLOADER_LOG_LEVEL>`. Setting log level to Warning, Error or None all significantly reduce the final binary size (but may make it harder to debug).
-- Set :ref:`partition table offset <CONFIG_PARTITION_TABLE_OFFSET>` to a higher value than 0x8000, to place the partition table later in the flash. This increases the space available for the bootloader. If the :doc:`partition table </api-guides/partition-tables>` CSV file contains explicit partition offsets, they will need changing so no partition has an offset lower than ``CONFIG_PARTITION_TABLE_OFFSET + 0x1000``. (This includes the default partition CSV files supplied with ESP-IDF.)
+- Set :ref:`CONFIG_PARTITION_TABLE_OFFSET` to a higher value than 0x8000, to place the partition table later in the flash. This increases the space available for the bootloader. If the :doc:`partition table </api-guides/partition-tables>` CSV file contains explicit partition offsets, they will need changing so no partition has an offset lower than ``CONFIG_PARTITION_TABLE_OFFSET + 0x1000``. (This includes the default partition CSV files supplied with ESP-IDF.)
 
-  Note that because of the absolute binary size limit, there is no benefit to moving the partition table any higher than offset {IDF_TARGET_MAX_PARTITION_TABLE_OFFSET}.
+Note that because of the absolute binary size limit, when using Secure Boot V2 there is no benefit to moving the partition table any higher than offset {IDF_TARGET_MAX_PARTITION_TABLE_OFFSET}.

examples/bluetooth/bluedroid/ble/blufi/sdkconfig.defaults

@@ -1,6 +1,12 @@
 # Override some defaults so BT stack is enabled
 # in this example
 
+#
+# Partition Table
+# (It's possible to fit Blufi in 1MB app partition size with some other optimizations, but
+# default config is close to 1MB.)
+CONFIG_PARTITION_TABLE_SINGLE_APP_LARGE=y
+
 #
 # BT config
 #

examples/bluetooth/bluedroid/ble/blufi/sdkconfig.defaults.esp32

@@ -25,18 +25,6 @@ CONFIG_APP_BUILD_BOOTLOADER=y
 CONFIG_APP_BUILD_USE_FLASH_SECTIONS=y
 # end of Build type
 
-#
-# Partition Table
-#
-CONFIG_PARTITION_TABLE_SINGLE_APP=y
-# CONFIG_PARTITION_TABLE_TWO_OTA is not set
-# CONFIG_PARTITION_TABLE_CUSTOM is not set
-CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv"
-CONFIG_PARTITION_TABLE_FILENAME="partitions_singleapp.csv"
-CONFIG_PARTITION_TABLE_OFFSET=0x8000
-CONFIG_PARTITION_TABLE_MD5=y
-# end of Partition Table
-
 #
 # Application manager
 #

examples/bluetooth/bluedroid/ble/blufi/sdkconfig.defaults.esp32c3

@@ -25,18 +25,6 @@ CONFIG_APP_BUILD_BOOTLOADER=y
 CONFIG_APP_BUILD_USE_FLASH_SECTIONS=y
 # end of Build type
 
-#
-# Partition Table
-#
-CONFIG_PARTITION_TABLE_SINGLE_APP=y
-# CONFIG_PARTITION_TABLE_TWO_OTA is not set
-# CONFIG_PARTITION_TABLE_CUSTOM is not set
-CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv"
-CONFIG_PARTITION_TABLE_FILENAME="partitions_singleapp.csv"
-CONFIG_PARTITION_TABLE_OFFSET=0x8000
-CONFIG_PARTITION_TABLE_MD5=y
-# end of Partition Table
-
 #
 # Application manager
 #

examples/bluetooth/bluedroid/ble/blufi/sdkconfig.defaults.esp32s3

@@ -26,18 +26,6 @@ CONFIG_APP_BUILD_BOOTLOADER=y
 CONFIG_APP_BUILD_USE_FLASH_SECTIONS=y
 # end of Build type
 
-#
-# Partition Table
-#
-CONFIG_PARTITION_TABLE_SINGLE_APP=y
-# CONFIG_PARTITION_TABLE_TWO_OTA is not set
-# CONFIG_PARTITION_TABLE_CUSTOM is not set
-CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv"
-CONFIG_PARTITION_TABLE_FILENAME="partitions_singleapp.csv"
-CONFIG_PARTITION_TABLE_OFFSET=0x8000
-CONFIG_PARTITION_TABLE_MD5=y
-# end of Partition Table
-
 #
 # Application manager
 #

examples/protocols/asio/chat_client/sdkconfig.defaults

@@ -1 +1,7 @@
 CONFIG_ESP_MAIN_TASK_STACK_SIZE=8192
+
+#
+# Partition Table
+#
+# Leave some room for larger apps without needing to reduce other features
+CONFIG_PARTITION_TABLE_SINGLE_APP_LARGE=y

examples/protocols/asio/chat_server/sdkconfig.defaults

@@ -1 +1,7 @@
 CONFIG_ESP_MAIN_TASK_STACK_SIZE=8192
+
+#
+# Partition Table
+#
+# Leave some room for larger apps without needing to reduce other features
+CONFIG_PARTITION_TABLE_SINGLE_APP_LARGE=y

examples/protocols/asio/ssl_client_server/sdkconfig.defaults

@@ -2,3 +2,9 @@ CONFIG_ASIO_SSL_SUPPORT=y
 CONFIG_PARTITION_TABLE_CUSTOM=y
 CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv"
 CONFIG_PARTITION_TABLE_FILENAME="partitions.csv"
+
+#
+# Partition Table
+#
+# Leave some room for larger apps without needing to reduce other features
+CONFIG_PARTITION_TABLE_SINGLE_APP_LARGE=y

examples/protocols/asio/tcp_echo_server/sdkconfig.defaults

@@ -1 +1,7 @@
 CONFIG_ESP_MAIN_TASK_STACK_SIZE=8192
+
+#
+# Partition Table
+#
+# Leave some room for larger apps without needing to reduce other features
+CONFIG_PARTITION_TABLE_SINGLE_APP_LARGE=y

examples/protocols/asio/udp_echo_server/sdkconfig.defaults

@@ -1 +1,7 @@
 CONFIG_ESP_MAIN_TASK_STACK_SIZE=8192
+
+#
+# Partition Table
+#
+# Leave some room for larger apps without needing to reduce other features
+CONFIG_PARTITION_TABLE_SINGLE_APP_LARGE=y

tools/ci/build_template_app.sh

@@ -17,9 +17,13 @@ gen_configs() {
     # CONFIG_COMPILER_OPTIMIZATION_NONE with flag -O0
     echo "CONFIG_COMPILER_OPTIMIZATION_NONE=y" > esp-idf-template/sdkconfig.ci.O0
     echo "CONFIG_BOOTLOADER_COMPILER_OPTIMIZATION_NONE=y" >> esp-idf-template/sdkconfig.ci.O0
+    # -O0 makes the bootloader too large to fit in the default space, otherwise(!)
+    echo "CONFIG_PARTITION_TABLE_OFFSET=0x10000" >> esp-idf-template/sdkconfig.ci.O0
+
     # CONFIG_COMPILER_OPTIMIZATION_SIZE with flag -Os
     echo "CONFIG_COMPILER_OPTIMIZATION_SIZE=y" > esp-idf-template/sdkconfig.ci.Os
     echo "CONFIG_BOOTLOADER_COMPILER_OPTIMIZATION_SIZE=y" >> esp-idf-template/sdkconfig.ci.Os
+
     # CONFIG_COMPILER_OPTIMIZATION_PERF with flag -O2
     echo "CONFIG_COMPILER_OPTIMIZATION_PERF=y" > esp-idf-template/sdkconfig.ci.O2
     echo "CONFIG_BOOTLOADER_COMPILER_OPTIMIZATION_PERF=y" >> esp-idf-template/sdkconfig.ci.O2

tools/ci/check_build_warnings.py

@@ -32,6 +32,7 @@ IGNORE_WARNS = [
         r'CryptographyDeprecationWarning',
         r'Python 3 versions older than 3.6 are not supported.',
         r'Support for Python 2 is deprecated and will be removed in future versions.',
+        r'Warning: \d+/\d+ app partitions are too small for binary'
     ]
 ]
 
@@ -49,7 +50,7 @@ def line_has_warnings(line):  # type: (str) -> bool
     return has_warnings
 
 
-def main():
+def main():  # type: () -> None
     parser = argparse.ArgumentParser(description='ESP-IDF app builder')
     parser.add_argument(
         '-v',

tools/ci/executable-list.txt

@@ -10,9 +10,11 @@ components/heap/test_multi_heap_host/test_all_configs.sh
 components/mbedtls/esp_crt_bundle/gen_crt_bundle.py
 components/mbedtls/esp_crt_bundle/test_gen_crt_bundle/test_gen_crt_bundle.py
 components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
+components/partition_table/check_sizes.py
 components/partition_table/gen_empty_partition.py
 components/partition_table/gen_esp32part.py
 components/partition_table/parttool.py
+components/partition_table/test_gen_esp32part_host/check_sizes_test.py
 components/partition_table/test_gen_esp32part_host/gen_esp32part_tests.py
 components/spiffs/spiffsgen.py
 components/ulp/esp32ulp_mapgen.py

tools/ci/mypy_ignore_list.txt

@@ -156,7 +156,6 @@ tools/ble/lib_gatt.py
 tools/check_python_dependencies.py
 tools/check_term.py
 tools/ci/check_artifacts_expire_time.py
-tools/ci/check_build_warnings.py
 tools/ci/check_callgraph.py
 tools/ci/check_codeowners.py
 tools/ci/check_deprecated_kconfigs.py

tools/test_apps/security/secure_boot/sdkconfig.defaults

@@ -0,0 +1,6 @@
+# Default Secure Boot bootloader is too large to fit in default
+# bootloader size without adding 0x1000 bytes or reducing the log level.
+#
+# If you find yourself needing to edit this in the future, it's a sign the
+# bootloader is bloating out!
+CONFIG_PARTITION_TABLE_OFFSET=0xA000