Merge branch 'bugfix/ldgen_ignore_nonexistent_archives_and_obj_v4.0' into 'release/v4.0'

ldgen: check mappings (v4.0)

See merge request espressif/esp-idf!14065

components/app_trace/linker.lf

@@ -3,10 +3,11 @@ archive: libapp_trace.a
 entries: 
     app_trace (noflash)
     app_trace_util (noflash)
-    SEGGER_SYSVIEW (noflash)
+    if SYSVIEW_ENABLE = y:
-    SEGGER_RTT_esp32 (noflash)
+      SEGGER_SYSVIEW (noflash)
-    SEGGER_SYSVIEW_Config_FreeRTOS (noflash)
+      SEGGER_RTT_esp32 (noflash)
-    SEGGER_SYSVIEW_FreeRTOS (noflash)
+      SEGGER_SYSVIEW_Config_FreeRTOS (noflash)
+      SEGGER_SYSVIEW_FreeRTOS (noflash)
 
 [mapping:driver]
 archive: libdriver.a

components/esp_ringbuf/test/test_ringbuf.c

@@ -673,8 +673,8 @@ static IRAM_ATTR __attribute__((noinline)) bool iram_ringbuf_test()
 {
     bool result = true;
 
-    spi_flash_guard_get()->start(); // Disables flash cache
     RingbufHandle_t handle = xRingbufferCreate(CONT_DATA_TEST_BUFF_LEN, RINGBUF_TYPE_NOSPLIT);
+    spi_flash_guard_get()->start(); // Disables flash cache
     result = result && (handle != NULL);
     xRingbufferGetMaxItemSize(handle);
     vRingbufferDelete(handle);

components/freertos/linker.lf

@@ -5,3 +5,5 @@ entries:
     tasks:vTaskStartScheduler (default)
     port:xPortStartScheduler (default)
     port:pxPortInitialiseStack (default)
+    if FREERTOS_SUPPORT_STATIC_ALLOCATION = y:
+        queue:xQueueGenericCreateStatic (default)

components/heap/linker.lf

@@ -2,4 +2,6 @@
 archive: libheap.a
 entries:
     multi_heap (noflash)
-    multi_heap_poisoning (noflash)
+    if HEAP_POISONING_DISABLED = n:
+        multi_heap_poisoning (noflash)
+

components/lwip/linker.lf

@@ -62,7 +62,6 @@ entries:
     pbuf:pbuf_header_impl (noflash_text)
     pbuf:pbuf_header (noflash_text)
     pbuf:pbuf_free (noflash_text)
-    timeouts:sys_timeouts_mbox_fetch (noflash_text)
     udp:udp_input_local_match (noflash_text)
     udp:udp_input (noflash_text)
     udp:udp_send (noflash_text)

components/soc/include/soc/efuse_periph.h

@@ -11,6 +11,5 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
 #pragma once
 #include "soc/efuse_reg.h"

components/soc/linker.lf

@@ -5,7 +5,6 @@ entries:
     rtc_clk (noflash)
     rtc_clk_init (noflash_text)
     rtc_init (noflash_text)
-    rtc_periph (noflash_text)
     rtc_pm (noflash_text)
     rtc_sleep (noflash_text)
     rtc_time (noflash_text)

tools/ci/check_ldgen_mapping_exceptions.txt

@@ -0,0 +1,3 @@
+libc
+sha256_coredump
+gcc

tools/ci/config/build.yml

@@ -80,7 +80,8 @@ build_ssc:
 build_esp_idf_tests_make:
   extends: .build_esp_idf_unit_test_template
   variables:
-      PYTHON_VER: 3
+    PYTHON_VER: 3
+    LDGEN_CHECK_MAPPING: 1
   script:
     - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
     - export EXTRA_CXXFLAGS=${EXTRA_CFLAGS}
@@ -129,6 +130,7 @@ build_examples_make:
       - $LOG_PATH
     expire_in: 4 days
   variables:
+    LDGEN_CHECK_MAPPING: 1
     LOG_PATH: "$CI_PROJECT_DIR/log_examples_make"
   only:
     # Here both 'variables' and 'refs' conditions are given. They are combined with "AND" logic.

tools/cmake/ldgen.cmake

@@ -56,6 +56,12 @@ function(__ldgen_process_template template output)
 
     idf_build_get_property(config_env_path CONFIG_ENV_PATH)
 
+    if($ENV{LDGEN_CHECK_MAPPING})
+        set(ldgen_check "--check-mapping"
+            "--check-mapping-exceptions=${idf_path}/tools/ci/check_ldgen_mapping_exceptions.txt")
+        message(STATUS "Mapping check enabled in ldgen")
+    endif()
+
     add_custom_command(
         OUTPUT ${output}
         COMMAND ${python} ${idf_path}/tools/ldgen/ldgen.py
@@ -67,6 +73,7 @@ function(__ldgen_process_template template output)
         --env-file  "${config_env_path}"
         --libraries-file ${build_dir}/ldgen_libraries
         --objdump   ${CMAKE_OBJDUMP}
+        ${ldgen_check}
         DEPENDS     ${template} ${ldgen_fragment_files} ${ldgen_depends} ${SDKCONFIG}
     )
 

tools/ldgen/generation.py

@@ -21,7 +21,7 @@ import fnmatch
 
 from fragments import Sections, Scheme, Mapping, Fragment
 from pyparsing import Suppress, White, ParseException, Literal, Group, ZeroOrMore
-from pyparsing import Word, OneOrMore, nums, alphanums, alphas, Optional, restOfLine
+from pyparsing import Word, OneOrMore, nums, alphas, restOfLine, SkipTo
 from ldgen_common import LdGenFailure
 
 
@@ -80,7 +80,6 @@ class PlacementRule():
         def do_section_expansion(rule, section):
             if section in rule.get_section_names():
                 sections_in_obj = sections_infos.get_obj_sections(rule.archive, rule.obj)
-
                 expansions = fnmatch.filter(sections_in_obj, section)
                 return expansions
 
@@ -254,11 +253,18 @@ class GenerationModel:
 
     DEFAULT_SCHEME = "default"
 
-    def __init__(self):
+    def __init__(self, check_mappings=False, check_mapping_exceptions=None):
         self.schemes = {}
         self.sections = {}
         self.mappings = {}
 
+        self.check_mappings = check_mappings
+
+        if check_mapping_exceptions:
+            self.check_mapping_exceptions = check_mapping_exceptions
+        else:
+            self.check_mapping_exceptions = []
+
     def _add_mapping_rules(self, archive, obj, symbol, scheme_name, scheme_dict, rules):
         # Use an ordinary dictionary to raise exception on non-existing keys
         temp_dict = dict(scheme_dict)
@@ -338,6 +344,19 @@ class GenerationModel:
                 try:
                     if not (obj == Mapping.MAPPING_ALL_OBJECTS and symbol is None and
                             scheme_name == GenerationModel.DEFAULT_SCHEME):
+                        if self.check_mappings and mapping.name not in self.check_mapping_exceptions:
+                            if not obj == Mapping.MAPPING_ALL_OBJECTS:
+                                obj_sections = sections_infos.get_obj_sections(archive, obj)
+                                if not obj_sections:
+                                    message = "'%s:%s' not found" % (archive, obj)
+                                    raise GenerationException(message, mapping)
+
+                                if symbol:
+                                    obj_sym = fnmatch.filter(obj_sections, "*%s" % symbol)
+                                    if not obj_sym:
+                                        message = "'%s:%s %s' not found" % (archive, obj, symbol)
+                                        raise GenerationException(message, mapping)
+
                         self._add_mapping_rules(archive, obj, symbol, scheme_name, scheme_dictionary, mapping_rules)
                 except KeyError:
                     message = GenerationException.UNDEFINED_REFERENCE + " to scheme '" + scheme_name + "'."
@@ -589,7 +608,7 @@ class SectionsInfo(dict):
         results = None
 
         try:
-            results = parser.parseString(first_line)
+            results = parser.parseString(first_line, parseAll=True)
         except ParseException as p:
             raise ParseException("Parsing sections info for library " + sections_info_dump.name + " failed. " + p.message)
 
@@ -597,43 +616,57 @@ class SectionsInfo(dict):
         self.sections[archive] = SectionsInfo.__info(sections_info_dump.name, sections_info_dump.read())
 
     def _get_infos_from_file(self, info):
-        # Object file line: '{object}:  file format elf32-xtensa-le'
+        # {object}:  file format elf32-xtensa-le
-        object = Fragment.ENTITY.setResultsName("object") + Literal(":").suppress() + Literal("file format elf32-xtensa-le").suppress()
+        object_line = SkipTo(":").setResultsName("object") + Suppress(restOfLine)
 
-        # Sections table
+        # Sections:
-        header = Suppress(Literal("Sections:") + Literal("Idx") + Literal("Name") + Literal("Size") + Literal("VMA") +
+        # Idx Name ...
-                          Literal("LMA") + Literal("File off") + Literal("Algn"))
+        section_start = Suppress(Literal("Sections:"))
-        entry = Word(nums).suppress() + Fragment.ENTITY + Suppress(OneOrMore(Word(alphanums, exact=8)) +
+        section_header = Suppress(OneOrMore(Word(alphas)))
-                                                                   Word(nums + "*") + ZeroOrMore(Word(alphas.upper()) +
-                                                                   Optional(Literal(","))))
 
-        # Content is object file line + sections table
+        # 00 {section} 0000000 ...
-        content = Group(object + header + Group(ZeroOrMore(entry)).setResultsName("sections"))
+        #              CONTENTS, ALLOC, ....
+        section_entry = Suppress(Word(nums)) + SkipTo(' ') + Suppress(restOfLine) + \
+            Suppress(ZeroOrMore(Word(alphas) + Literal(",")) + Word(alphas))
 
+        content = Group(object_line + section_start + section_header + Group(OneOrMore(section_entry)).setResultsName("sections"))
         parser = Group(ZeroOrMore(content)).setResultsName("contents")
 
-        sections_info_text = info.content
         results = None
 
         try:
-            results = parser.parseString(sections_info_text)
+            results = parser.parseString(info.content, parseAll=True)
         except ParseException as p:
             raise ParseException("Unable to parse section info file " + info.filename + ". " + p.message)
 
         return results
 
     def get_obj_sections(self, archive, obj):
-        stored = self.sections[archive]
+        res = []
+        try:
+            stored = self.sections[archive]
 
-        # Parse the contents of the sections file
+            # Parse the contents of the sections file on-demand,
-        if not isinstance(stored, dict):
+            # save the result for later
-            parsed = self._get_infos_from_file(stored)
+            if not isinstance(stored, dict):
-            stored = dict()
+                parsed = self._get_infos_from_file(stored)
-            for content in parsed.contents:
+                stored = dict()
-                sections = list(map(lambda s: s, content.sections))
+                for content in parsed.contents:
-                stored[content.object] = sections
+                    sections = list(map(lambda s: s, content.sections))
-            self.sections[archive] = stored
+                    stored[content.object] = sections
+                self.sections[archive] = stored
 
-        for obj_key in stored.keys():
+            try:
-            if obj_key == obj + ".o" or obj_key == obj + ".c.obj":
+                res = stored[obj + ".o"]
-                return stored[obj_key]
+            except KeyError:
+                try:
+                    res = stored[obj + ".c.obj"]
+                except KeyError:
+                    try:
+                        res = stored[obj + ".cpp.obj"]
+                    except KeyError:
+                        res = stored[obj + ".S.obj"]
+        except KeyError:
+            pass
+
+        return res

tools/ldgen/ldgen.py

@@ -75,6 +75,18 @@ def main():
         "--kconfig", "-k",
         help="IDF Kconfig file")
 
+    argparser.add_argument(
+        "--check-mapping",
+        help="Perform a check if a mapping (archive, obj, symbol) exists",
+        action='store_true'
+    )
+
+    argparser.add_argument(
+        "--check-mapping-exceptions",
+        help="Mappings exempted from check",
+        type=argparse.FileType("r")
+    )
+
     argparser.add_argument(
         "--env", "-e",
         action='append', default=[],
@@ -98,6 +110,12 @@ def main():
     kconfig_file = args.kconfig
     objdump = args.objdump
 
+    check_mapping = args.check_mapping
+    if args.check_mapping_exceptions:
+        check_mapping_exceptions = [line.strip() for line in args.check_mapping_exceptions]
+    else:
+        check_mapping_exceptions = None
+
     try:
         sections_infos = SectionsInfo()
         for library in libraries_file:
@@ -107,7 +125,7 @@ def main():
                 dump.name = library
                 sections_infos.add_sections_info(dump)
 
-        generation_model = GenerationModel()
+        generation_model = GenerationModel(check_mapping, check_mapping_exceptions)
 
         _update_environment(args)  # assign args.env and args.env_file to os.environ
 

tools/ldgen/test/data/sections.info

@@ -1,4 +1,4 @@
-In archive /home/user/ãóç+ěščřžýáíé/build/esp-idf/freertos/libfreertos.a:
+In archive /home/user/build/esp-idf/freertos/libfreertos.a:
 
 FreeRTOS-openocd.c.obj:     file format elf32-xtensa-le
 

tools/ldgen/test/data/sections_parse.info

@@ -0,0 +1,19 @@
+In archive /home/user/ãóç+ěščřžýáíé/build/esp-idf/freertos/libsections_parse.a:
+
+croutine.c.obj:     file format elf32-littleriscv
+
+Sections:
+Idx Name          Size      VMA       LMA       File off  Algn
+  0 .text         00000000  00000000  00000000  00000034  2**0
+                  CONTENTS, ALLOC, LOAD, READONLY, CODE
+  1 .data         00000000  00000000  00000000  00000034  2**0
+                  CONTENTS, ALLOC, LOAD, DATA
+  2 .bss          00000000  00000000  00000000  00000034  2**0
+                  ALLOC
+
+FreeRTOS-openocd.c.obj:     file format elf32-xtensa-le // 'F' should not get included in match for 'CONTENTS, ALLOC, LOAD ...' prior
+
+Sections:
+Idx Name          Size      VMA       LMA       File off  Algn
+  0 .literal.prvCheckPendingReadyList 00000018  00000000  00000000  00000034  2**2
+                  CONTENTS, ALLOC, LOAD, RELOC, READONLY, CODE

tools/ldgen/test/test_generation.py

@@ -1405,6 +1405,19 @@ entries:
         self.assertListEqual(actual["flash_text"], expected["flash_text"])
         self.assertListEqual(actual["iram0_text"], expected["iram0_text"])
 
+    def test_sections_info_parsing(self):
+
+        self.sections_info = SectionsInfo()
+
+        with open("data/sections_parse.info") as sections_info_file_obj:
+            self.sections_info.add_sections_info(sections_info_file_obj)
+
+        sections = self.sections_info.get_obj_sections("libsections_parse.a", "croutine")
+        self.assertEqual(set(sections), set([".text", ".data", ".bss"]))
+
+        sections = self.sections_info.get_obj_sections("libsections_parse.a", "FreeRTOS-openocd")
+        self.assertEqual(set(sections), set([".literal.prvCheckPendingReadyList"]))
+
 
 if __name__ == "__main__":
     unittest.main()