Merge branch 'feature/custom_partition_subtypes' into 'master'

gen_esp32part.py: Support custom partition types

See merge request espressif/esp-idf!18656

components/nvs_flash/host_test/nvs_page_test/main/CMakeLists.txt

@@ -5,7 +5,7 @@ idf_component_register(SRCS "nvs_page_test.cpp"
                     "${CMAKE_CURRENT_SOURCE_DIR}/../../../src"
                     PRIV_INCLUDE_DIRS
                     "${CMAKE_CURRENT_SOURCE_DIR}/../../../private_include"
-                    REQUIRES cmock nvs_flash spi_flash)
+                    REQUIRES cmock nvs_flash spi_flash partition_table)
 
 target_compile_options(${COMPONENT_LIB} PUBLIC --coverage)
 target_link_libraries(${COMPONENT_LIB} --coverage)

components/partition_table/CMakeLists.txt

@@ -36,12 +36,22 @@ else()
     set(partition_secure_opt "")
 endif()
 
+idf_build_get_property(extra_subtypes EXTRA_PARTITION_SUBTYPES)
+if(extra_subtypes)
+    # Remove all white spaces from the string
+    string(REPLACE " " "" extra_subtypes "${extra_subtypes}")
+    set(extra_partition_subtypes --extra-partition-subtypes ${extra_subtypes})
+else()
+    set(extra_partition_subtypes "")
+endif()
+
 idf_build_get_property(build_dir BUILD_DIR)
 idf_build_get_property(python PYTHON)
+idf_build_get_property(extra_subtypes EXTRA_PARTITION_SUBTYPES)
 
 set(gen_partition_table "${python}" "${CMAKE_CURRENT_SOURCE_DIR}/gen_esp32part.py" "-q"
                         "--offset" "${PARTITION_TABLE_OFFSET}" "${md5_opt}" "${flashsize_opt}"
-                        "${partition_secure_opt}")
+                        "${partition_secure_opt}" ${extra_partition_subtypes} "--")
 
 set(partition_table_display
     COMMAND ${CMAKE_COMMAND} -E echo "Partition table binary generated. Contents:"
@@ -56,8 +66,20 @@ add_custom_command(OUTPUT "${build_dir}/partition_table/${unsigned_partition_bin
     DEPENDS ${partition_csv} "${CMAKE_CURRENT_SOURCE_DIR}/gen_esp32part.py"
     VERBATIM)
 
+if(extra_subtypes)
+    set(extra_subtypes_h "${build_dir}/config/extra_partition_subtypes.inc")
+
+    add_custom_command(OUTPUT ${extra_subtypes_h}
+        COMMAND ${python} ${CMAKE_CURRENT_SOURCE_DIR}/gen_extra_subtypes_inc.py ${extra_subtypes_h} ${extra_subtypes}
+        COMMENT "Generating extra partition subtype header file"
+    )
+    add_custom_target(extra_subtype_hdr DEPENDS ${extra_subtypes_h})
+    add_dependencies(${COMPONENT_LIB} extra_subtype_hdr)
+endif()
+
 add_custom_target(partition_table_bin DEPENDS "${build_dir}/partition_table/${unsigned_partition_bin}"
-                                              "${build_dir}/partition_table/${final_partition_bin}")
+                                              "${build_dir}/partition_table/${final_partition_bin}"
+                                              )
 
 if(EXISTS ${partition_csv})
     add_custom_target(partition-table

components/partition_table/gen_esp32part.py

@@ -92,6 +92,26 @@ def get_alignment_for_type(ptype):
     return ALIGNMENT.get(ptype, ALIGNMENT[DATA_TYPE])
 
 
+def get_partition_type(ptype):
+    if ptype == 'app':
+        return APP_TYPE
+    if ptype == 'data':
+        return DATA_TYPE
+    raise InputError('Invalid partition type')
+
+
+def add_extra_subtypes(csv):
+    for line_no in csv:
+        try:
+            fields = [line.strip() for line in line_no.split(',')]
+            for subtype, subtype_values in SUBTYPES.items():
+                if (int(fields[2], 16) in subtype_values.values() and subtype == get_partition_type(fields[0])):
+                    raise ValueError('Found duplicate value in partition subtype')
+            SUBTYPES[TYPES[fields[0]]][fields[1]] = int(fields[2], 16)
+        except InputError as err:
+            raise InputError('Error parsing custom subtypes: %s' % err)
+
+
 quiet = False
 md5sum = True
 secure = False
@@ -145,7 +165,7 @@ class PartitionTable(list):
             try:
                 res.append(PartitionDefinition.from_csv(line, line_no + 1))
             except InputError as err:
-                raise InputError('Error at line %d: %s' % (line_no + 1, err))
+                raise InputError('Error at line %d: %s\nPlease check extra_partition_subtypes.inc file in build/config directory' % (line_no + 1, err))
             except Exception:
                 critical('Unexpected error parsing CSV line %d: %s' % (line_no + 1, line))
                 raise
@@ -506,6 +526,7 @@ def main():
     parser.add_argument('--quiet', '-q', help="Don't print non-critical status messages to stderr", action='store_true')
     parser.add_argument('--offset', '-o', help='Set offset partition table', default='0x8000')
     parser.add_argument('--secure', help='Require app partitions to be suitable for secure boot', action='store_true')
+    parser.add_argument('--extra-partition-subtypes', help='Extra partition subtype entries', nargs='*')
     parser.add_argument('input', help='Path to CSV or binary file to parse.', type=argparse.FileType('rb'))
     parser.add_argument('output', help='Path to output converted binary or CSV file. Will use stdout if omitted.',
                         nargs='?', default='-')
@@ -516,6 +537,9 @@ def main():
     md5sum = not args.disable_md5sum
     secure = args.secure
     offset_part_table = int(args.offset, 0)
+    if args.extra_partition_subtypes:
+        add_extra_subtypes(args.extra_partition_subtypes)
+
     table, input_is_binary = PartitionTable.from_file(args.input)
 
     if not args.no_verify:

components/partition_table/gen_extra_subtypes_inc.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+
+import argparse
+
+
+def gen_header_file(path: str, subtypes: str) -> None:
+    HDR_MESSAGE = '/* Automatically generated file. DO NOT EDIT. */\n\n'
+    PARTTOOL_USAGE = 'If you want to use parttool.py manually, please use the following as an extra argument:'
+    with open(path, 'w') as f:
+        f.write(HDR_MESSAGE)
+        if subtypes:
+            f.write('/*\n\t' + PARTTOOL_USAGE + '\n\t')
+            f.write('--extra-partition-subtypes ')
+            for line_no in subtypes:
+                f.write(line_no + ' ')
+            f.write('\n*/\n\n')
+        f.write('#pragma once\n\n')
+        for line_no in subtypes:
+            try:
+                fields = [line.strip() for line in line_no.split(',')]
+                fields[0] = fields[0].strip()
+                fields[1] = fields[1].strip()
+                fields[2] = fields[2].strip()
+                f.write('ESP_PARTITION_SUBTYPE_%s_%s = %s,\n' % (fields[0].upper(), fields[1].upper(), fields[2]))
+            except ValueError as err:
+                raise ValueError('Error parsing custom subtypes: %s' % err)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='ESP32 extra partitions utility')
+    parser.add_argument('config_dir', help='Path to config directory')
+    parser.add_argument('extra_partition_subtypes', help='Extra partition subtype entries', nargs='*')
+    args = parser.parse_args()
+
+    gen_header_file(args.config_dir, args.extra_partition_subtypes)

components/partition_table/parttool.py

@@ -3,19 +3,8 @@
 # parttool is used to perform partition level operations - reading,
 # writing, erasing and getting info about the partition.
 #
-# Copyright 2018 Espressif Systems (Shanghai) PTE LTD
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http:#www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# SPDX-FileCopyrightText: 2018-2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
 from __future__ import division, print_function
 
 import argparse
@@ -281,6 +270,7 @@ def main():
     print_partition_info_subparser.add_argument('--info', help='type of partition information to get',
                                                 choices=['name', 'type', 'subtype', 'offset', 'size', 'encrypted'], default=['offset', 'size'], nargs='+')
     print_partition_info_subparser.add_argument('--part_list', help='Get a list of partitions suitable for a given type', action='store_true')
+    print_partition_info_subparser.add_argument('--extra-partition-subtypes', help='Extra partition subtype entries', nargs='*')
 
     args = parser.parse_args()
     quiet = args.quiet
@@ -331,6 +321,9 @@ def main():
     if args.esptool_erase_args:
         target_args['esptool_erase_args'] = args.esptool_erase_args
 
+    if args.extra_partition_subtypes:
+        gen.add_extra_subtypes(args.extra_partition_subtypes)
+
     target = ParttoolTarget(**target_args)
 
     # Create the operation table and execute the operation

components/partition_table/project_include.cmake

@@ -39,12 +39,22 @@ endif()
 function(partition_table_get_partition_info result get_part_info_args part_info)
     idf_build_get_property(python PYTHON)
     idf_build_get_property(idf_path IDF_PATH)
+
+    idf_build_get_property(extra_subtypes EXTRA_PARTITION_SUBTYPES)
+    if(extra_subtypes)
+        # Remove all white spaces from the string
+        string(REPLACE " " "" extra_subtypes "${extra_subtypes}")
+        set(extra_partition_subtypes --extra-partition-subtypes ${extra_subtypes})
+    else()
+        set(extra_partition_subtypes "")
+    endif()
     separate_arguments(get_part_info_args)
     execute_process(COMMAND ${python}
         ${idf_path}/components/partition_table/parttool.py -q
         --partition-table-offset ${PARTITION_TABLE_OFFSET}
         --partition-table-file ${PARTITION_CSV_PATH}
         get_partition_info ${get_part_info_args} --info ${part_info}
+        ${extra_partition_subtypes}
         OUTPUT_VARIABLE info
         RESULT_VARIABLE exit_code
         OUTPUT_STRIP_TRAILING_WHITESPACE)

components/spi_flash/include/esp_partition.h

@@ -84,6 +84,10 @@ typedef enum {
     ESP_PARTITION_SUBTYPE_DATA_FAT = 0x81,                                    //!< FAT partition
     ESP_PARTITION_SUBTYPE_DATA_SPIFFS = 0x82,                                 //!< SPIFFS partition
 
+#if __has_include("extra_partition_subtypes.inc")
+    #include "extra_partition_subtypes.inc"
+#endif
+
     ESP_PARTITION_SUBTYPE_ANY = 0xff,                                         //!< Used to search for partitions with any subtype
 } esp_partition_subtype_t;
 

docs/en/api-guides/build-system.rst

@@ -343,6 +343,7 @@ from the component CMakeLists.txt:
 - ``IDF_VERSION_MAJOR``, ``IDF_VERSION_MINOR``, ``IDF_VERSION_PATCH``: Components of ESP-IDF version, to be used in conditional expressions. Note that this information is less precise than that provided by ``IDF_VER`` variable. ``v4.0-dev-*``, ``v4.0-beta1``, ``v4.0-rc1`` and ``v4.0`` will all have the same values of ``IDF_VERSION_*`` variables, but different ``IDF_VER`` values.
 - ``IDF_TARGET``: Name of the target for which the project is being built.
 - ``PROJECT_VER``: Project version.
+- ``EXTRA_PARTITION_SUBTYPES``: CMake list of extra partition subtypes. Each subtype description is a comma separated string with ``type_name, subtype_name, numeric_value`` format. Components may add new subtypes by appending them to this list.
 
   * If :ref:`CONFIG_APP_PROJECT_VER_FROM_CONFIG` option is set, the value of :ref:`CONFIG_APP_PROJECT_VER` will be used.
   * Else, if ``PROJECT_VER`` variable is set in project CMakeLists.txt file, its value will be used.

docs/en/api-guides/partition-tables.rst

@@ -131,6 +131,11 @@ See enum :cpp:type:`esp_partition_subtype_t` for the full list of subtypes defin
 
   Note that when writing in C++, an application-defined subtype value requires casting to type :cpp:type:`esp_partition_subtype_t` in order to use it with the :ref:`partition API<api-reference-partition-table>`.
 
+Extra Partition SubTypes
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+A component can define a new partition subtype by setting the ``EXTRA_PARTITION_SUBTYPES`` property. This property is a CMake list, each entry of which is a comma separated string with ``<type>, <subtype>, <value>`` format. The build system uses this property to add extra subtypes and creates fields named ``ESP_PARTITION_SUBTYPE_<type>_<subtype>`` in :cpp:type:`esp_partition_type_t`. The project can use this subtype to define partitions in the partitions table CSV file and use the new fields in :cpp:type:`esp_partition_type_t`.
+
 Offset & Size
 ~~~~~~~~~~~~~
 

tools/ci/executable-list.txt

@@ -20,6 +20,7 @@ components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
 components/partition_table/check_sizes.py
 components/partition_table/gen_empty_partition.py
 components/partition_table/gen_esp32part.py
+components/partition_table/gen_extra_subtypes_inc.py
 components/partition_table/parttool.py
 components/partition_table/test_gen_esp32part_host/check_sizes_test.py
 components/partition_table/test_gen_esp32part_host/gen_esp32part_tests.py

tools/test_apps/.build-test-rules.yml

@@ -1,5 +1,10 @@
 # Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
 
+tools/test_apps/build_system/custom_partition_subtypes:
+  enable:
+    - if: IDF_TARGET in ["esp32", "linux"]
+      reason: the test should be run on ESP32 and linux
+
 tools/test_apps/build_system/ldgen_test:
   disable:
     - if: IDF_TARGET == "esp32c2"

tools/test_apps/build_system/custom_partition_subtypes/CMakeLists.txt

@@ -0,0 +1,25 @@
+# The following lines of boilerplate have to be in your project's
+# CMakeLists in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.16)
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+
+project(custom_partition_subtypes)
+
+idf_build_get_property(build_dir BUILD_DIR)
+idf_build_get_property(idf_path IDF_PATH)
+idf_build_get_property(python PYTHON)
+set(blank_file ${build_dir}/blank_file.bin)
+idf_component_get_property(partition_table_dir partition_table COMPONENT_DIR)
+
+partition_table_get_partition_info(partition "--partition-type app --partition-subtype my_app1" "name")
+partition_table_get_partition_info(partition_size "--partition-type app --partition-subtype my_app1" "size")
+
+add_custom_command(OUTPUT ${blank_file}
+    COMMAND ${python} ${partition_table_dir}/gen_empty_partition.py
+    ${partition_size} ${blank_file})
+
+add_custom_target(blank_bin ALL DEPENDS ${blank_file})
+add_dependencies(flash blank_bin)
+
+esptool_py_flash_to_partition(flash "${partition}" "${blank_file}")

tools/test_apps/build_system/custom_partition_subtypes/README.md

@@ -0,0 +1,3 @@
+| Supported Targets | ESP32 | Linux |
+| ----------------- | ----- | ----- |
+

tools/test_apps/build_system/custom_partition_subtypes/components/custom/CMakeLists.txt

@@ -0,0 +1 @@
+idf_component_register()

tools/test_apps/build_system/custom_partition_subtypes/components/custom/project_include.cmake

@@ -0,0 +1,2 @@
+idf_build_set_property(EXTRA_PARTITION_SUBTYPES "app, my_app1, 0x40" APPEND)
+idf_build_set_property(EXTRA_PARTITION_SUBTYPES "app, my_app2, 0x41" APPEND)

tools/test_apps/build_system/custom_partition_subtypes/main/CMakeLists.txt

@@ -0,0 +1 @@
+idf_component_register(SRCS test_main.c)

tools/test_apps/build_system/custom_partition_subtypes/main/test_main.c

@@ -0,0 +1,27 @@
+/*
+ * SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Unlicense OR CC0-1.0
+ */
+#include <stdio.h>
+#include <esp_partition.h>
+#include <esp_log.h>
+
+static const char *TAG = "test_main";
+
+void app_main(void)
+{
+    const esp_partition_t *my_app1 = esp_partition_find_first(ESP_PARTITION_TYPE_APP, ESP_PARTITION_SUBTYPE_APP_MY_APP1, NULL);
+    if (my_app1 == NULL) {
+        ESP_LOGE(TAG, "Failed to find custom my_app1");
+    } else {
+        ESP_LOGI(TAG, "Found custom partition @ 0x%x (0x%x)", my_app1->address, my_app1->size);
+    }
+
+    const esp_partition_t *my_app2 = esp_partition_find_first(ESP_PARTITION_TYPE_APP, ESP_PARTITION_SUBTYPE_APP_MY_APP2, NULL);
+    if (my_app2 == NULL) {
+        ESP_LOGE(TAG, "Failed to find custom my_app1");
+    } else {
+        ESP_LOGI(TAG, "Found custom partition @ 0x%x (0x%x)", my_app2->address, my_app2->size);
+    }
+}

tools/test_apps/build_system/custom_partition_subtypes/partitions.csv

@@ -0,0 +1,7 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+# Note: if you have increased the bootloader size, make sure to update the offsets to avoid overlap
+nvs,      data, nvs,     ,        0x6000,
+phy_init, data, phy,     ,        0x1000,
+fctry,  app,  factory, ,        1M,
+my_app1,  app,  my_app1, ,        256K,
+my_app2,  app,  my_app2, ,        256K,

tools/test_apps/build_system/custom_partition_subtypes/sdkconfig.defaults

@@ -0,0 +1,2 @@
+CONFIG_PARTITION_TABLE_CUSTOM=y
+CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv"