Merge branch 'feature/common_fragment_grammar' into 'master'

ldgen: Common fragment grammar

See merge request idf/esp-idf!4576

components/app_trace/linker.lf

@@ -1,4 +1,4 @@
-[mapping]
+[mapping:app_trace]
 archive: libapp_trace.a
 entries: 
     app_trace (noflash)
@@ -8,10 +8,11 @@ entries:
     SEGGER_SYSVIEW_Config_FreeRTOS (noflash)
     SEGGER_SYSVIEW_FreeRTOS (noflash)
 
-[mapping]
+[mapping:driver]
 archive: libdriver.a
 entries:
-    : SYSVIEW_TS_SOURCE_TIMER_00 = y || SYSVIEW_TS_SOURCE_TIMER_01 = y 
+    if SYSVIEW_TS_SOURCE_TIMER_00 = y || SYSVIEW_TS_SOURCE_TIMER_01 = y
-      || SYSVIEW_TS_SOURCE_TIMER_10 = y || SYSVIEW_TS_SOURCE_TIMER_11 = y
+       || SYSVIEW_TS_SOURCE_TIMER_10 = y || SYSVIEW_TS_SOURCE_TIMER_11 = y:
       timer (noflash)
-
+    else:
+      * (default)

components/esp32/linker.lf

@@ -1,14 +1,14 @@
-[mapping]
+[mapping:esp32]
 archive: libesp32.a
 entries:
     panic (noflash)
 
-[mapping]
+[mapping:gcc]
 archive: libgcc.a
 entries:
     lib2funcs (noflash_text)
 
-[mapping]
+[mapping:gcov]
 archive: libgcov.a
 entries:
     * (noflash)

components/esp_ringbuf/linker.lf

@@ -1,4 +1,4 @@
-[mapping]
+[mapping:esp_ringbuf]
 archive: libesp_ringbuf.a
 entries:
     * (noflash_text)

components/esp_wifi/linker.lf

@@ -1,22 +1,21 @@
-[mapping]
+[mapping:phy]
 archive: libphy.a
 entries:
     * (noflash_data)
 
-[mapping]
+[mapping:rtc]
 archive: librtc.a
 entries:
     * (noflash_text)
 
-[mapping]
+[mapping:pp]
 archive: libpp.a
 entries:
-    : ESP32_WIFI_IRAM_OPT = y
+    if ESP32_WIFI_IRAM_OPT = y:
         * (wifi_iram)
 
-[mapping]
+[mapping:net80211]
 archive: libnet80211.a
 entries:
-    : ESP32_WIFI_IRAM_OPT = y
+    if ESP32_WIFI_IRAM_OPT = y:
         * (wifi_iram)
-

components/espcoredump/linker.lf

@@ -1,4 +1,4 @@
-[mapping]
+[mapping:espcoredump]
 archive: libespcoredump.a
 entries: 
     core_dump_uart (noflash_text)

components/freertos/linker.lf

@@ -1,4 +1,4 @@
-[mapping]
+[mapping:freertos]
 archive: libfreertos.a
 entries:
     * (noflash_text)

components/heap/linker.lf

@@ -1,4 +1,4 @@
-[mapping]
+[mapping:heap]
 archive: libheap.a
 entries:
     multi_heap (noflash)

components/lwip/linker.lf

@@ -1,7 +1,7 @@
-[mapping]
+[mapping:lwip]
 archive: liblwip.a
 entries:
-    :LWIP_IRAM_OPTIMIZATION = y
+  if LWIP_IRAM_OPTIMIZATION = y:
     ethernetif:ethernet_low_level_output (noflash_text)
     ethernetif:ethernetif_input (noflash_text)
     wlanif:low_level_output (noflash_text)
@@ -55,3 +55,5 @@ entries:
     pbuf:pbuf_header_impl (noflash_text)
     pbuf:pbuf_header (noflash_text)
     ethernet:ethernet_input (noflash_text)
+  else:
+    * (default)

components/newlib/esp32-spiram-rom-functions.lf

@@ -8,10 +8,10 @@
 # Note: We currently never link libg-psram-workaround.a, so no rules
 # are generated for this library
 
-[mapping]
+[mapping:libc_psram_workaround]
 archive: libc-psram-workaround.a
 entries:
-  : SPIRAM_CACHE_WORKAROUND = y
+  if SPIRAM_CACHE_WORKAROUND = y:
     lib_a-utoa (noflash)
     lib_a-longjmp (noflash)
     lib_a-setjmp (noflash)
@@ -141,4 +141,6 @@ entries:
     lib_a-strdup_r (noflash)
     lib_a-system (noflash)
     lib_a-strndup_r (noflash)
+  else:
+    * (default)
 

components/soc/linker.lf

@@ -1,4 +1,4 @@
-[mapping]
+[mapping:soc]
 archive: libsoc.a
 entries: 
     cpu_util (noflash_text)

components/spi_flash/linker.lf

@@ -1,4 +1,4 @@
-[mapping]
+[mapping:spi_flash]
 archive: libspi_flash.a
 entries: 
     spi_flash_rom_patch (noflash_text)

components/vfs/test/linker.lf

@@ -1,4 +1,4 @@
-[mapping]
+[mapping:vfs]
 archive: libvfs.a
 entries:
     vfs:esp_vfs_open (noflash)

components/xtensa/linker.lf

@@ -1,9 +1,9 @@
-[mapping]
+[mapping:xtensa]
 archive: libxtensa.a
 entries:
     eri (noflash_text)
 
-[mapping]
+[mapping:hal]
 archive: libhal.a
 entries:
     * (noflash_text)

docs/en/api-guides/linker-script-generation.rst

@@ -5,131 +5,131 @@ Linker Script Generation
 Overview
 --------
 
-There are several :ref:`memory regions<memory-layout>` where code and data can be placed. Usually, code and read-only data are placed in flash regions,
+There are several :ref:`memory regions<memory-layout>` where code and data can be placed. Code and read-only data are placed by default in flash,
-writable data in RAM, etc. A common action is changing where code/data are mapped by default, say placing critical code/rodata in RAM for performance
+writable data in RAM, etc. However, it is sometimes necessary to change these default placements. For example, it may
-reasons or placing code/data/rodata in RTC memory for use in a wake stub or the ULP coprocessor.
+be necessary to place critical code in RAM for performance reasons or to place code in RTC memory for use in a wake stub or the ULP coprocessor.
 
-IDF provides the ability for defining these placements at the component level using the linker script generation mechanism. The component presents
+With the linker script generation mechanism, it is possible to specify these placements at the component level within ESP-IDF. The component presents
-how it would like to map the input sections of its object files (or even functions/data) through :ref:`linker fragment files<ldgen-fragment-files>`. During app build, 
+information on how it would like to place its symbols, objects or the entire archive. During build the information presented by the components are collected,
-the linker fragment files are collected, parsed and processed; and the :ref:`linker script template<ldgen-script-templates>` is augmented with
+parsed and processed; and the placement rules generated is used to link the app.
-information generated from the fragment files to produce the final linker script. This linker script is then used for the linking
-the final app binary.
 
 Quick Start
 ------------
 
-This section presents a guide for quickly placing code/data to RAM and RTC memory; as well as demonstrating how to make these placements 
+This section presents a guide for quickly placing code/data to RAM and RTC memory - placements ESP-IDF provides out-of-the-box.
-dependent on project configuration values. In a true quick start fashion, this section glosses over terms and concepts that will be discussed 
-at a later part of the document. However, whenever it does so, it provides a link to the relevant section on the first mention.
 
-.. _ldgen-add-fragment-file :
+For this guide, suppose we have the following::
 
-Preparation
+    - components/
-^^^^^^^^^^^
+                    - my_component/                              
+                                    - CMakeLists.txt
+                                    - component.mk
+                                    - Kconfig                                
+                                    - src/
+                                          - my_src1.c                          
+                                          - my_src2.c                          
+                                          - my_src3.c                       
+                                    - my_linker_fragment_file.lf          
+
+
+- a component named ``my_component`` that is archived as library ``libmy_component.a`` during build
+- three source files archived under the library, ``my_src1.c``, ``my_src2.c`` and ``my_src3.c`` which are compiled as ``my_src1.o``, ``my_src2.o`` and ``my_src3.o``, respectively
+- under ``my_src1.o``, the function ``my_function1`` is defined; under ``my_src2.o``, the function ``my_function2`` is defined
+- there exist bool-type config ``PERFORMANCE_MODE`` (y/n) and int type config ``PERFORMANCE_LEVEL`` (with range 0-3) in my_component's Kconfig
+
+
+Creating and Specifying a Linker Fragment File
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Before anything else, a linker fragment file needs to be created. A linker fragment file
+is simply a text file with a ``.lf`` extension upon which the desired placements will be written.
+After creating the file, it is then necessary to present it to the build system. The instructions for the build systems
+supported by ESP-IDF are as follows:
 
 Make
 """"
 
-Create a linker fragment file inside the component directory, which is just a text file with a .lf extension. In order for the build system to collect your fragment file,
+In the component's ``component.mk`` file, set the variable ``COMPONENT_ADD_LDFRAGMENTS`` to the path of the created linker
-add an entry to it from the component, set the variable ``COMPONENT_ADD_LDFRAGMENTS`` to your linker file/s before the ``register_component`` call.
+fragment file. The path can either be an absolute path or a relative path from the component directory.
 
 .. code-block:: make
 
-    # file paths relative to component Makefile 
+    COMPONENT_ADD_LDFRAGMENTS += "my_linker_fragment_file.lf"
-    COMPONENT_ADD_LDFRAGMENTS += "path/to/linker_fragment_file.lf" "path/to/another_linker_fragment_file.lf"
 
 CMake
 """""
 
-For CMake set the variable ``COMPONENT_ADD_LDFRAGMENTS`` to your linker file/s before the ``register_component`` call.
+In the component's ``CMakeLists.txt`` file, set the variable ``COMPONENT_ADD_LDFRAGMENTS`` to the path of the created linker
+fragment file before the ``register_component`` call. The path can either be an absolute path or a relative path from the component directory.
 
 .. code-block:: cmake
 
-    # file paths relative to CMakeLists.txt  
+    set(COMPONENT_ADD_LDFRAGMENTS "my_linker_fragment_file.lf")
-    set(COMPONENT_ADD_LDFRAGMENTS "path/to/linker_fragment_file.lf" "path/to/another_linker_fragment_file.lf")
-
     register_component()
 
-It is also possible to specify fragment files from the project CMakeLists.txt or component project_include.cmake using the function `ldgen_add_fragment_files`::
-
-    ldgen_add_fragment_files(target files ...)
-
 
 Specifying placements
 ^^^^^^^^^^^^^^^^^^^^^
 
-This mechanism allows specifying placement of the following entities:
+It is possible to specify placements at the following levels of granularity:
 
-    - one or multiple object files within the component
+    - object file (``.obj`` or ``.o`` files)
-    - one or multiple function/variable using their names
+    - symbol (function/variable)
-    - the entire component library
+    - archive (``.a`` files)
-
-For the following text, suppose we have the following:
-
-    - a component named ``component`` that is archived as library ``libcomponent.a`` during build
-    - three object files archived under the library, ``object1.o``, ``object2.o`` and ``object3.o``
-    - under ``object1.o``, the function ``function1`` is defined; under ``object2.o``, the function ``function2`` is defined
-    - there exist configuration ``PERFORMANCE_MODE`` and ``PERFORMANCE_LEVEL`` in one of the IDF KConfig files, with the set value indicated by entries ``CONFIG_PERFORMANCE_MODE`` and ``CONFIG_PERFORMANCE_LEVEL`` in the project sdkconfig
-
-In the created linker fragment file, we write:
-
-.. code-block:: none
-
-    [mapping]
-    archive: libcomponent.a
-    entries:
-
-This creates an empty :ref:`mapping fragment<ldgen-mapping-fragment>`, which doesn't do anything yet. During linking the :ref:`default placements<ldgen-default-placements>` 
-will still be used for ``libcomponent.a``, unless the ``entries`` key is populated.
 
 .. _ldgen-placing-object-files :
 
 Placing object files
 """"""""""""""""""""
 
-Suppose the entirety of ``object1.o``  is performance-critical, so it is desirable to place it in RAM. On the other hand, suppose all of ``object2.o`` contains things to be executed coming out of deep sleep, so it needs to be put under RTC memory. We can write:
+Suppose the entirety of ``my_src1.o`` is performance-critical, so it is desirable to place it in RAM. 
+On the other hand, the entirety of ``my_src2.o`` contains symbols needed coming out of deep sleep, so it needs to be put under RTC memory.
+In the the linker fragment file, we can write:
 
 .. code-block:: none
 
-    [mapping]
+    [mapping:my_component]
-    archive: libcomponent.a
+    archive: libmy_component.a
     entries:
-        object1 (noflash)     # places all code / read-only data under IRAM/ DRAM
+        my_src1 (noflash)     # places all my_src1 code/read-only data under IRAM/DRAM
-        object2 (rtc)         # places all code/ data and read-only data under RTC fast memory/ RTC slow memory
+        my_src2 (rtc)         # places all my_src2 code/ data and read-only data under RTC fast memory/RTC slow memory
 
-What happens to ``object3.o``? Since it is not specified, default placements are used for ``object3.o``.
+What happens to ``my_src3.o``? Since it is not specified, default placements are used for ``my_src3.o``. More on default placements
+:ref:`here<ldgen-default-placements>`. 
 
-Placing functions/data using their names
+Placing symbols
-""""""""""""""""""""""""""""""""""""""""
+""""""""""""""""
 
-Continuing our example, suppose that among functions defined under ``object1.o``, only ``function1`` is performance-critical; and under ``object2.o``,
+Continuing our example, suppose that among functions defined under ``object1.o``, only ``my_function1`` is performance-critical; and under ``object2.o``,
-only ``function2`` needs to execute after the chip comes out of deep sleep. This could be accomplished by writing:
+only ``my_function2`` needs to execute after the chip comes out of deep sleep. This could be accomplished by writing:
 
 .. code-block:: none
 
-    [mapping]
+    [mapping:my_component]
-    archive: libcomponent.a
+    archive: libmy_component.a
     entries:
-        object1:function1 (noflash) 
+        my_src1:my_function1 (noflash)
-        object2:function2 (rtc) 
+        my_src2:my_function2 (rtc)
 
-The default placements are used for the rest of the functions in ``object1.o`` and ``object2.o`` and the entire ``object3.o``. Something similar
+The default placements are used for the rest of the functions in ``my_src1.o`` and ``my_src2.o`` and the entire ``object3.o``. Something similar
-can be achieved for placing data by writing the variable name instead of the function name after ``:``.
+can be achieved for placing data by writing the variable name instead of the function name, like so::
+
+       my_src1:my_variable (noflash)
 
 .. warning::
 
-    There are :ref:`limitations<ldgen-type1-limitations>` in placing code/data using their symbol names. In order to ensure proper placements, an alternative would be to group
+    There are :ref:`limitations<ldgen-symbol-granularity-placements>` in placing code/data at symbol granularity. In order to ensure proper placements, an alternative would be to group
-    relevant code and data into source files, and :ref:`use object file placement<ldgen-placing-object-files>`.
+    relevant code and data into source files, and :ref:`use object-granularity placements<ldgen-placing-object-files>`.
 
-Placing entire component
+Placing entire archive
-""""""""""""""""""""""""
+"""""""""""""""""""""""
 
-In this example, suppose that the entire component needs to be placed in RAM. This can be written as:
+In this example, suppose that the entire component archive needs to be placed in RAM. This can be written as:
 
 .. code-block:: none
 
-    [mapping]
+    [mapping:my_component]
-    archive: libcomponent.a
+    archive: libmy_component.a
     entries:
         * (noflash)
 
@@ -137,67 +137,65 @@ Similarly, this places the entire component in RTC memory:
 
 .. code-block:: none
 
-    [mapping]
+    [mapping:my_component]
-    archive: libcomponent.a
+    archive: libmy_component.a
     entries:
         * (rtc)
 
 Configuration-dependent placements
 """"""""""""""""""""""""""""""""""
 
-Suppose that the entire component library should only be placed when ``CONFIG_PERFORMANCE_MODE == y`` in the sdkconfig. This could be written as:
+Suppose that the entire component library should only have special placement when a certain condition is true; for example, when ``CONFIG_PERFORMANCE_MODE == y``.
+This could be written as:
 
 .. code-block:: none
 
-    [mapping]
+    [mapping:my_component]
-    archive: libcomponent.a
+    archive: libmy_component.a
     entries:
-        : PERFORMANCE_MODE = y
+        if PERFORMANCE_MODE = y:
             * (noflash)
+        else:
+            * (default)
 
-In pseudocode, this translates to:
+For a more complex config-dependent placement, suppose the following requirements: when ``CONFIG_PERFORMANCE_LEVEL == 1``, only ``object1.o`` is put in RAM;
-
-.. code-block:: none
-
-    if PERFORMANCE_MODE = y
-        place entire libcomponent.a in RAM
-    else
-        use default placements
-
-It is also possible to have multiple conditions to test. Suppose the following requirements: when ``CONFIG_PERFORMANCE_LEVEL == 1``, only ``object1.o`` is put in RAM;
 when ``CONFIG_PERFORMANCE_LEVEL == 2``, ``object1.o`` and ``object2.o``; and when ``CONFIG_PERFORMANCE_LEVEL == 3`` all object files under the archive
 are to be put into RAM. When these three are false however, put entire library in RTC memory. This scenario is a bit contrived, but,
 it can be written as:
 
 .. code-block:: none
 
-    [mapping]
+    [mapping:my_component]
-    archive: libcomponent.a
+    archive: libmy_component.a
     entries:
-        : PERFORMANCE_LEVEL = 3
+        if PERFORMANCE_LEVEL = 1:
-        * (noflash)
+            my_src1 (noflash)
-        : PERFORMANCE_LEVEL = 2
+        elif PERFORMANCE_LEVEL = 2:
-        object1 (noflash)
+            my_src1 (noflash)
-        object2 (noflash)
+            my_src2 (noflash)
-        : PERFORMANCE_LEVEL = 1
+        elif PERFORMANCE_LEVEL = 3:
-        object1 (noflash)
+            my_src1 (noflash)
-        : default
+            my_src2 (noflash)
+            my_src3 (noflash)
+        else:
             * (rtc)
 
-Which reads:
+Nesting condition-checking is also possible. The following is equivalent to the snippet above:
 
 .. code-block:: none
 
-    if CONFIG_PERFORMANCE_LEVEL == 3
+    [mapping:my_component]
-        place entire libcomponent.a in RAM
+    archive: libmy_component.a
-    else if CONFIG_PERFORMANCE_LEVEL == 2
+    entries:
-        only place object1.o and object2.o in RAM
+        if PERFORMANCE_LEVEL <= 3 && PERFORMANCE_LEVEL > 0:
-    else if CONFIG_PERFORMANCE_LEVEL == 1
+            if PERFORMANCE_LEVEL >= 1:
-        only place object1.o in RAM
+                object1 (noflash)
-    else
+                if PERFORMANCE_LEVEL >= 2:
-        place entire libcomponent.a in RTC memory 
+                    object2 (noflash)
-
+                    if PERFORMANCE_LEVEL >= 3:
-The conditions test :ref:`support other operations<ldgen-condition-entries>`.
+                        object2 (noflash)
+        else:
+            * (rtc)
 
 .. _ldgen-default-placements:
 
@@ -205,55 +203,161 @@ The 'default' placements
 ^^^^^^^^^^^^^^^^^^^^^^^^
 
 Up until this point, the term  'default placements' has been mentioned as fallback placements for when the
-placement rules ``rtc`` and ``noflash`` are not specified. The tokens ``noflash`` or ``rtc`` are not merely keywords known by the mechanism, but are actually 
+placement rules ``rtc`` and ``noflash`` are not specified. It is important to note that the tokens ``noflash`` or ``rtc`` are not merely keywords, but are actually
-objects called :ref:`scheme fragments<ldgen-scheme-fragment>` that are specified by the user. Due to the commonness of these placement use cases,
+entities called fragments, specifically :ref:`schemes<ldgen-scheme-fragment>`.
-they are pre-defined in IDF.
 
-Similarly, there exists a ``default`` scheme fragment which defines what the default placement rules should be, which is discussed :ref:`here<ldgen-default-scheme>`.
+In the same manner as ``rtc`` and ``noflash`` are schemes, there exists a ``default`` scheme which defines what the default placement rules should be.
+As the name suggests, it is where code and data are usually placed, i.e. code/constants is placed in flash, variables
+placed in RAM, etc.  More on the default scheme :ref:`here<ldgen-default-scheme>`.
 
 .. note::
-    For an example of an IDF component using this feature, see :component_file:`freertos/CMakeLists.txt`. The ``freertos`` component uses this 
+    For an example of an ESP-IDF component using the linker script generation mechanism, see :component_file:`freertos/CMakeLists.txt`. 
-    mechanism to place all code, literal and rodata of all of its object files to the instruction RAM memory region for performance reasons.
+    ``freertos`` uses this to place its object files to the instruction RAM for performance reasons.
 
-This marks the end of the quick start guide. The following text discusses this mechanism in a little bit more detail, such its components, essential concepts,
+This marks the end of the quick start guide. The following text discusses the internals of the mechanism in a little bit more detail.
-the syntax, how it is integrated with the build system, etc. The following sections should be helpful in creating custom mappings or modifying default 
+The following sections should be helpful in creating custom placements or modifying default behavior.
-behavior.
 
-Components
+Linker Script Generation Internals
-----------
+----------------------------------
 
-.. _ldgen-fragment-files :
+Linking is the last step in the process of turning C/C++ source files into an executable. It is performed by the toolchain's linker, and accepts
+linker scripts which specify code/data placements, among other things. With the linker script generation mechanism, this process is no different, except
+that the linker script passed to the linker is dynamically generated from: (1) the collected :ref:`linker fragment files<ldgen-linker-fragment-files>` and 
+(2) :ref:`linker script template<ldgen-linker-script-template>`.
+
+.. note::
+
+    The tool that implements the linker script generation mechanism lives under :idf:`tools/ldgen`.
+
+.. _ldgen-linker-fragment-files :
 
 Linker Fragment Files
 ^^^^^^^^^^^^^^^^^^^^^
 
-The fragment files contain objects called 'fragments'. These fragments contain pieces of information which, when put together, form
+As mentioned in the quick start guide, fragment files are simple text files with the ``.lf`` extension containing the desired placements. This is a simplified
-placement rules that tell where to place sections of object files in the output binary.
+description of what fragment files contain, however. What fragment files actually contain are 'fragments'. Fragments are entities which contain pieces of information which, when put together, form
+placement rules that tell where to place sections of object files in the output binary. There are three types of fragments: :ref:`sections<ldgen-sections-fragment>`, 
+:ref:`scheme<ldgen-scheme-fragment>` and :ref:`mapping<ldgen-mapping-fragment>`.
 
-Another way of putting it is that processing linker fragment files aims to create the section placement rules inside GNU LD ``SECTIONS`` command. 
+Grammar
-Where to collect and put these section placement rules is represented internally as a ``target`` token.
+"""""""
 
-The three types of fragments are discussed below.
+The three fragment types share a common grammar:
+
+.. code-block:: none
+
+    [type:name]
+    key: value
+    key:
+        value
+        value
+        value
+        ...
+
+- type: Corresponds to the fragment type, can either be ``sections``, ``scheme`` or ``mapping``.
+- name: The name of the fragment, should be unique for the specified fragment type.
+- key, value: Contents of the fragment; each fragment type may support different keys and different grammars for the key values.
 
 .. note::
 
-    Fragments have a name property (except mapping fragments) and are known globally. 
+    In cases where multiple fragments of the same type and name are encountered, an exception is thrown.
-    Fragment naming follows C variable naming rules, i.e. case sensitive, must begin with a letter or underscore, alphanumeric/underscore after
+
-    initial characters are allowed, no spaces/special characters. Each type of fragment has its own namespace. In cases where multiple fragments
+.. note::
-    of the same type and name are encountered, an exception is thrown.
+
+    The only valid characters for fragment names and keys are alphanumeric characters and underscore.
+
+
+.. _ldgen-condition-checking :
+
+**Condition Checking**
+
+Condition checking enable the linker script generation to be configuration-aware. Depending on whether expressions involving configuration values
+are true or not, a particular set of values for a key can be used. The evaluation uses ``eval_string`` from :idf_file:`tools/kconfig_new/kconfiglib.py` 
+and adheres to its required syntax and limitations. Supported operators are as follows:
+
+    - comparison
+        - LessThan ``<``
+        - LessThanOrEqualTo ``<=``
+        - MoreThan ``>``
+        - MoreThanOrEqualTo ``>=``
+        - Equal ``=``
+        - NotEqual ``!=``
+    - logical
+        - Or ``||``
+        - And ``&&``
+        - Negation ``!``
+    - grouping
+        - Parenthesis ``()``
+
+Condition checking behaves as you would expect an ``if...elseif/elif...else`` block in other languages. Condition-checking is possible
+for both key values and entire fragments. The two sample fragments below are equivalent:
+
+.. code-block:: none
+
+    # Value for keys is dependent on config
+    [type:name]
+    key_1:
+        if CONDITION = y:
+            value_1
+        else:
+            value_2
+    key_2:
+        if CONDITION = y:
+            value_a
+        else:
+            value_b
+
+.. code-block:: none
+
+    # Entire fragment definition is dependent on config
+    if CONDITION = y:
+        [type:name]
+        key_1:
+            value_1
+        key_2:
+            value_b
+    else:
+        [type:name]
+        key_1:
+            value_2
+        key_2:
+            value_b
+
+
+**Comments**
+
+Comment in linker fragment files begin with ``#``. Like in other languages, comment are used to provide helpful descriptions and documentation
+and are ignored during processing. 
+
+Compatibility with ESP-IDF v3.x Linker Script Fragment Files
+""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+
+ESP-IDF v4.0 brings some changes to the linker script fragment file grammar:
+
+- indentation is enforced and improperly indented fragment files generate a parse exception; this was not enforced in the old version but previous documentation
+    and examples demonstrates properly indented grammar
+- move to ``if...elif...else`` structure for conditionals, with the ability to nest checks and place entire fragments themselves inside conditionals
+- mapping fragments now requires a name like other fragment types
+
+Linker script generator should be able to parse ESP-IDF v3.x linker fragment files that are indented properly (as demonstrated by
+the ESP-IDF v3.x version of this document). Backward compatibility with the previous mapping fragment grammar (optional
+name and the old grammar for conditionals) has also been retained but with a deprecation warning. Users should switch to the newer grammar discussed
+in this document as support for the old grammar is planned to be removed in the future.
+
+Note that linker fragment files using the new ESP-IDF v4.0 grammar is not supported on ESP-IDF v3.x, however.
+
+Types
+"""""
 
 .. _ldgen-sections-fragment :
 
-I. Sections
+**Sections**
-"""""""""""
 
 Sections fragments defines a list of object file sections that the GCC compiler emits. It may be a default section (e.g. ``.text``, ``.data``) or
 it may be user defined section through the ``__attribute__`` keyword.
 
 The use of an optional '+' indicates the inclusion of the section in the list, as well as sections that start with it. This is the preferred method over listing both explicitly.
 
-**Syntax**
-
 .. code-block:: none
 
     [sections:name]
@@ -262,7 +366,7 @@ The use of an optional '+' indicates the inclusion of the section in the list, a
         .section
         ...
 
-**Example**
+Example:
 
 .. code-block:: none
 
@@ -282,13 +386,10 @@ The use of an optional '+' indicates the inclusion of the section in the list, a
 
 .. _ldgen-scheme-fragment :
 
-II. Scheme
+**Scheme**
-""""""""""
 
 Scheme fragments define what ``target`` a sections fragment is assigned to.
 
-**Syntax**
-
 .. code-block:: none
 
     [scheme:name]
@@ -297,7 +398,7 @@ Scheme fragments define what ``target`` a sections fragment is assigned to.
         sections -> target
         ...
 
-**Example**
+Example:
 
 .. code-block:: none
 
@@ -308,7 +409,7 @@ Scheme fragments define what ``target`` a sections fragment is assigned to.
 
 .. _ldgen-default-scheme:
 
-**The** ``default`` **scheme**
+The ``default`` scheme
 
 There exists a special scheme with the name ``default``. This scheme is special because catch-all placement rules are generated from
 its entries. This means that, if one of its entries is ``text -> flash_text``, the placement rule
@@ -328,49 +429,26 @@ These catch-all rules then effectively serve as fallback rules for those whose m
 
 .. _ldgen-mapping-fragment :
 
-III. Mapping
+**Mapping**
-""""""""""""
 
-Mapping fragments define what scheme fragment to use for mappable entities, i.e. object files, function names, variable names. There are two types of entries
+Mapping fragments define what scheme fragment to use for mappable entities, i.e. object files, function names, variable names, archives.
-for this fragment: mapping entries and condition entries.
-
-.. note::
-
-    Mapping fragments have no explicit name property. Internally, the name is constructed from the value of the archive entry. 
-
-**Syntax**
 
 .. code-block:: none
 
-    [mapping]
+    [mapping:name]
     archive: archive                # output archive file name, as built (i.e. libxxx.a)
     entries:
-        : condition                 # condition entry, non-default
+        object:symbol (scheme)      # symbol granularity
-        object:symbol (scheme)      # mapping entry, Type I
+        object (scheme)             # object granularity
-        object (scheme)             # mapping entry, Type II
+        * (scheme)                  # archive granularity
-        * (scheme)                  # mapping entry, Type III
 
-        # optional separation/comments, for readability
+There are three levels of placement granularity:
 
-        : default                   # condition entry, default
+    - symbol: The object file name and symbol name are specified. The symbol name can be a function name or a variable name.
-        * (scheme)                  # mapping entry, Type III
+    - object: Only the object file name is specified.
+    - archive: ``*`` is specified, which is a short-hand for all the object files under the archive.
 
-.. _ldgen-mapping-entries :
+To know what an entry means, let us expand a sample object-granularity placement:
-
-**Mapping Entries**
-
-There are three types of mapping entries:
-
-    ``Type I``
-        The object file name and symbol name are specified. The symbol name can be a function name or a variable name.
-
-    ``Type II``
-        Only the object file name is specified.
-
-    ``Type III``
-        ``*`` is specified, which is a short-hand for all the object files under the archive.
-
-To know what a mapping entry means, let us expand a ``Type II`` entry. Originally: 
 
 .. code-block:: none
 
@@ -398,13 +476,23 @@ Expanding the sections fragment with its entries definition:
 
             ...)           # and so on
 
-.. _ldgen-type1-limitations :
+Example:
 
-**On** ``Type I`` **Mapping Entries**
+.. code-block:: none
 
-``Type I`` mapping entry is possible due to compiler flags ``-ffunction-sections`` and ``-ffdata-sections``. If the user opts to remove these flags, then
+    [mapping:map]
-the ``Type I`` mapping will not work. Furthermore, even if the user does not opt to compile without these flags, there are still limitations 
+    archive: libfreertos.a
-as the implementation is dependent on the emitted output sections.
+    entries:
+        * (noflash)
+
+.. _ldgen-symbol-granularity-placements :
+
+On Symbol-Granularity Placements
+""""""""""""""""""""""""""""""""
+
+Symbol granularity placements is possible due to compiler flags ``-ffunction-sections`` and ``-ffdata-sections``. ESP-IDF compiles with these flags by default.
+If the user opts to remove these flags, then the symbol-granularity placements will not work. Furthermore, even with the presence of these flags, there are still other limitations to keep in mind
+due to the dependence on the compiler's emitted output sections.
 
 For example, with ``-ffunction-sections``, separate sections are emitted for each function; with section names predictably constructed i.e. ``.text.{func_name}``
 and ``.literal.{func_name}``. This is not the case for string literals within the function, as they go to pooled or generated section names.
@@ -412,38 +500,7 @@ and ``.literal.{func_name}``. This is not the case for string literals within th
 With ``-fdata-sections``, for global scope data the compiler predictably emits either ``.data.{var_name}``, ``.rodata.{var_name}`` or ``.bss.{var_name}``; and so ``Type I`` mapping entry works for these.
 However, this is not the case for static data declared in function scope, as the generated section name is a result of mangling the variable name with some other information.
 
-.. _ldgen-condition-entries :
+.. _ldgen-linker-script-template :
-
-**Condition Entries**
-
-Condition entries enable the linker script generation to be configuration-aware. Depending on whether expressions involving configuration values
-are true or not, a particular set of mapping entries can be used. The evaluation uses ``eval_string`` from :idf_file:`tools/kconfig_new/kconfiglib.py` and adheres to its required syntax and limitations.
-
-All mapping entries defined after a condition entry until the next one or the end of the mapping fragment belongs to that condition entry. During processing 
-conditions are tested sequentially, and the mapping entries under the first condition that evaluates to ``TRUE`` are used.
-
-A default condition can be defined (though every mapping contains an implicit, empty one), whose mapping entries get used in the event no conditions evaluates to ``TRUE``.
-
-**Example**
-
-.. code-block:: none
-
-    [scheme:noflash]
-    entries:
-        text -> iram0_text
-        rodata -> dram0_data
-
-    [mapping:lwip]
-    archive: liblwip.a
-    entries:
-        : LWIP_IRAM_OPTIMIZATION = y         # if CONFIG_LWIP_IRAM_OPTIMIZATION is set to 'y' in sdkconfig
-        ip4:ip4_route_src_hook (noflash)     # map ip4.o:ip4_route_src_hook, ip4.o:ip4_route_src and
-        ip4:ip4_route_src (noflash)          # ip4.o:ip4_route using the noflash scheme, which puts 
-        ip4:ip4_route (noflash)              # them in RAM
-        
-        : default                            # else no special mapping rules apply
-
-.. _ldgen-script-templates :
 
 Linker Script Template
 ^^^^^^^^^^^^^^^^^^^^^^
@@ -451,15 +508,13 @@ Linker Script Template
 The linker script template is the skeleton in which the generated placement rules are put into. It is an otherwise ordinary linker script, with a specific marker syntax
 that indicates where the generated placement rules are placed.
 
-**Syntax**
-
 To reference the placement rules collected under a ``target`` token, the following syntax is used:
 
 .. code-block:: none
 
     mapping[target]
 
-**Example**
+Example:
 
 The example below is an excerpt from a possible linker script template. It defines an output section ``.iram0.text``, and inside is a marker referencing
 the target ``iram0_text``.
@@ -530,19 +585,9 @@ Then the corresponding excerpt from the generated linker script will be as follo
     it too is placed wherever ``iram0_text`` is referenced by a marker. Since it is a rule generated from the default scheme, it comes first
     among all other rules collected under the same target name.
 
+.. note::
 
-Integration with Build System
+    The linker script template currently used is :component:`esp32/ld/esp32.project.ld.in`, specified by the ``esp32`` component; the
------------------------------
+    generated output script is put under its build directory.
 
-The linker script generation occurs during application build, before the final output binary is linked. The tool that implements the mechanism 
-lives under ``$(IDF_PATH)/tools/ldgen``.
 
-Linker Script Template
-^^^^^^^^^^^^^^^^^^^^^^
-Currently, the linker script template used is :component:`esp32/ld/esp32.project.ld.in`, and is used only for the app build. The generated output script is 
-put under the build directory of the same component. Modifying this linker script template triggers a re-link of the app binary. 
-
-Linker Fragment File
-^^^^^^^^^^^^^^^^^^^^
-Any component can add a fragment file to the build. In order to add a fragment file to process, set COMPONENT_ADD_LDFRAGMENTS or use the function ``ldgen_add_fragment_files`` (CMake only) as mentioned :ref:`here <ldgen-add-fragment-file>`.
-Modifying any fragment file presented to the build system triggers a re-link of the app binary.

tools/ldgen/fragments.py

@@ -13,9 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
-import re
 import os
+import re
 
 from sdkconfig import SDKConfig
 from pyparsing import OneOrMore
@@ -23,49 +22,181 @@ from pyparsing import restOfLine
 from pyparsing import alphanums
 from pyparsing import Word
 from pyparsing import alphas
-from pyparsing import ParseBaseException
+from pyparsing import ParseFatalException
 from pyparsing import Suppress
 from pyparsing import Group
 from pyparsing import Literal
 from pyparsing import ZeroOrMore
 from pyparsing import Optional
 from pyparsing import originalTextFor
-from common import LdGenFailure
+from pyparsing import Forward
+from pyparsing import indentedBlock
+from collections import namedtuple
+import abc
 
 
-class FragmentFileModel():
+KeyGrammar = namedtuple("KeyGrammar", "grammar min max required")
+
+
+class FragmentFile():
     """
     Fragment file internal representation. Parses and stores instances of the fragment definitions
     contained within the file.
     """
 
-    def __init__(self, fragment_file):
+    def __init__(self, fragment_file, sdkconfig):
+        try:
+            fragment_file = open(fragment_file, "r")
+        except TypeError:
+            pass
+
         path = os.path.realpath(fragment_file.name)
 
-        sections = Sections.get_fragment_grammar()
+        indent_stack = [1]
-        scheme = Scheme.get_fragment_grammar()
-        mapping = Mapping.get_fragment_grammar()
 
-        # Each fragment file is composed of sections, scheme or mapping fragments. The grammar
+        class parse_ctx:
-        # for each of those objects are defined it the respective classes
+            fragment = None  # current fragment
-        parser = OneOrMore(sections | scheme | mapping)
+            key = ""  # current key
+            keys = list()  # list of keys parsed
+            key_grammar = None  # current key grammar
 
-        # Set any text beginnning with # as comment
+            @staticmethod
-        parser.ignore("#" + restOfLine)
+            def reset():
+                parse_ctx.fragment_instance = None
+                parse_ctx.key = ""
+                parse_ctx.keys = list()
+                parse_ctx.key_grammar = None
+
+        def fragment_type_parse_action(toks):
+            parse_ctx.reset()
+            parse_ctx.fragment = FRAGMENT_TYPES[toks[0]]()  # create instance of the fragment
+            return None
+
+        def expand_conditionals(toks, stmts):
+            try:
+                stmt = toks["value"]
+                stmts.append(stmt)
+            except KeyError:
+                try:
+                    conditions = toks["conditional"]
+                    for condition in conditions:
+                        try:
+                            _toks = condition[1]
+                            _cond = condition[0]
+                            if sdkconfig.evaluate_expression(_cond):
+                                expand_conditionals(_toks, stmts)
+                                break
+                        except IndexError:
+                            expand_conditionals(condition[0], stmts)
+                except KeyError:
+                    for tok in toks:
+                        expand_conditionals(tok, stmts)
+
+        def key_body_parsed(pstr, loc, toks):
+            stmts = list()
+            expand_conditionals(toks, stmts)
+
+            if parse_ctx.key_grammar.min and len(stmts) < parse_ctx.key_grammar.min:
+                raise ParseFatalException(pstr, loc, "fragment requires at least %d values for key '%s'" %
+                                          (parse_ctx.key_grammar.min, parse_ctx.key))
+
+            if parse_ctx.key_grammar.max and len(stmts) > parse_ctx.key_grammar.max:
+                raise ParseFatalException(pstr, loc, "fragment requires at most %d values for key '%s'" %
+                                          (parse_ctx.key_grammar.max, parse_ctx.key))
 
             try:
+                parse_ctx.fragment.set_key_value(parse_ctx.key, stmts)
+            except Exception as e:
+                raise ParseFatalException(pstr, loc, "unable to add key '%s'; %s" % (parse_ctx.key, e.message))
+            return None
+
+        key = Word(alphanums + "_") + Suppress(":")
+        key_stmt = Forward()
+
+        condition_block = indentedBlock(key_stmt, indent_stack)
+        key_stmts = OneOrMore(condition_block)
+        key_body = Suppress(key) + key_stmts
+        key_body.setParseAction(key_body_parsed)
+
+        condition = originalTextFor(SDKConfig.get_expression_grammar()).setResultsName("condition")
+        if_condition = Group(Suppress("if") + condition + Suppress(":") + condition_block)
+        elif_condition = Group(Suppress("elif") + condition + Suppress(":") + condition_block)
+        else_condition = Group(Suppress("else") + Suppress(":") + condition_block)
+        conditional = (if_condition + Optional(OneOrMore(elif_condition)) + Optional(else_condition)).setResultsName("conditional")
+
+        def key_parse_action(pstr, loc, toks):
+            key = toks[0]
+
+            if key in parse_ctx.keys:
+                raise ParseFatalException(pstr, loc, "duplicate key '%s' value definition" % parse_ctx.key)
+
+            parse_ctx.key = key
+            parse_ctx.keys.append(key)
+
+            try:
+                parse_ctx.key_grammar = parse_ctx.fragment.get_key_grammars()[key]
+                key_grammar = parse_ctx.key_grammar.grammar
+            except KeyError:
+                raise ParseFatalException(pstr, loc, "key '%s' is not supported by fragment" % key)
+            except Exception as e:
+                raise ParseFatalException(pstr, loc, "unable to parse key '%s'; %s" % (key, e.message))
+
+            key_stmt << (conditional | Group(key_grammar).setResultsName("value"))
+
+            return None
+
+        def name_parse_action(pstr, loc, toks):
+            parse_ctx.fragment.name = toks[0]
+
+        key.setParseAction(key_parse_action)
+
+        ftype = Word(alphas).setParseAction(fragment_type_parse_action)
+        fid = Suppress(":") + Word(alphanums + "_.").setResultsName("name")
+        fid.setParseAction(name_parse_action)
+        header = Suppress("[") + ftype + fid + Suppress("]")
+
+        def fragment_parse_action(pstr, loc, toks):
+            key_grammars = parse_ctx.fragment.get_key_grammars()
+            required_keys = set([k for (k,v) in key_grammars.items() if v.required])
+            present_keys = required_keys.intersection(set(parse_ctx.keys))
+            if present_keys != required_keys:
+                raise ParseFatalException(pstr, loc, "required keys %s for fragment not found" %
+                                          list(required_keys - present_keys))
+            return parse_ctx.fragment
+
+        fragment_stmt = Forward()
+        fragment_block = indentedBlock(fragment_stmt, indent_stack)
+
+        fragment_if_condition = Group(Suppress("if") + condition + Suppress(":") + fragment_block)
+        fragment_elif_condition = Group(Suppress("elif") + condition + Suppress(":") + fragment_block)
+        fragment_else_condition = Group(Suppress("else") + Suppress(":") + fragment_block)
+        fragment_conditional = (fragment_if_condition + Optional(OneOrMore(fragment_elif_condition)) +
+                                Optional(fragment_else_condition)).setResultsName("conditional")
+
+        fragment = (header + OneOrMore(indentedBlock(key_body, indent_stack, False))).setResultsName("value")
+        fragment.setParseAction(fragment_parse_action)
+        fragment.ignore("#" + restOfLine)
+
+        deprecated_mapping = DeprecatedMapping.get_fragment_grammar(sdkconfig, fragment_file.name).setResultsName("value")
+
+        fragment_stmt << (Group(deprecated_mapping) | Group(fragment) | Group(fragment_conditional))
+
+        def fragment_stmt_parsed(pstr, loc, toks):
+            stmts = list()
+            expand_conditionals(toks, stmts)
+            return stmts
+
+        parser = ZeroOrMore(fragment_stmt)
+        parser.setParseAction(fragment_stmt_parsed)
+
         self.fragments = parser.parseFile(fragment_file, parseAll=True)
-        except ParseBaseException as e:
-            # the actual parse error is kind of useless for normal users, so just point to the location of
-            # the error
-            raise LdGenFailure("Parse error in linker fragment %s: error at line %d col %d (char %d)" % (
-                fragment_file.name, e.lineno, e.column, e.loc))
 
         for fragment in self.fragments:
             fragment.path = path
 
 
-class Fragment:
+class Fragment():
+    __metaclass__ = abc.ABCMeta
     """
     Encapsulates a fragment as defined in the generator syntax. Sets values common to all fragment and performs processing
     such as checking the validity of the fragment name and getting the entry values.
@@ -74,22 +205,20 @@ class Fragment:
     IDENTIFIER = Word(alphas + "_", alphanums + "_")
     ENTITY = Word(alphanums + ".-_$")
 
-    def __init__(self, name, entries):
+    @abc.abstractmethod
-        self.path = None
+    def set_key_value(self, key, parse_results):
-        self.name = name
+        pass
-        self.entries = entries
+
+    @abc.abstractmethod
+    def get_key_grammars(self):
+        pass
 
 
 class Sections(Fragment):
 
-    def __init__(self, name, entries):
+    grammars = {
-        Fragment.__init__(self, name, entries)
+        "entries": KeyGrammar(Word(alphanums + "+.").setResultsName("section"), 1, None, True)
-        self._process_entries()
+    }
-
-    def _process_entries(self):
-        # Quietly ignore duplicate entries
-        self.entries = set(self.entries)
-        self.entries = list(self.entries)
 
     """
     Utility function that returns a list of sections given a sections fragment entry,
@@ -110,20 +239,14 @@ class Sections(Fragment):
             else:
                 return (sections_entry, None)
 
-    @staticmethod
+    def set_key_value(self, key, parse_results):
-    def get_fragment_grammar():
+        if key == "entries":
-        name = Fragment.IDENTIFIER
+            self.entries = set()
-        header = Suppress("[") + Suppress("sections") + Suppress(":") + name.setResultsName("name") + Suppress("]")
+            for result in parse_results:
-        entry = Word(alphanums + "+" + ".")
+                self.entries.add(result["section"])
-        entries = Suppress("entries") + Suppress(":") + Group(OneOrMore(entry)).setResultsName("entries")
 
-        sections = Group(header + entries)
+    def get_key_grammars(self):
-
+        return self.__class__.grammars
-        sections.setParseAction(lambda t: Sections(t[0].name, t[0].entries))
-
-        sections.ignore("#" + restOfLine)
-
-        return sections
 
 
 class Scheme(Fragment):
@@ -131,38 +254,19 @@ class Scheme(Fragment):
     Encapsulates a scheme fragment, which defines what target input sections are placed under.
     """
 
-    def __init__(self, name, items):
+    grammars = {
-        Fragment.__init__(self, name, items)
+        "entries": KeyGrammar(Fragment.IDENTIFIER.setResultsName("sections") + Suppress("->") +
-        self._process_entries()
+                              Fragment.IDENTIFIER.setResultsName("target"), 1, None, True)
+    }
 
-    def _process_entries(self):
+    def set_key_value(self, key, parse_results):
-        processed = set()
+        if key == "entries":
+            self.entries = set()
+            for result in parse_results:
+                self.entries.add((result["sections"], result["target"]))
 
-        # Store entries as a set of tuples. Quietly ignores duplicate entries.
+    def get_key_grammars(self):
-        for entry in self.entries:
+        return self.__class__.grammars
-            processed.add((entry.sections, entry.target))
-
-        self.entries = processed
-
-    @staticmethod
-    def get_fragment_grammar():
-        name = Fragment.IDENTIFIER
-        header = Suppress("[") + Suppress("scheme") + Suppress(":") + name.setResultsName("name") + Suppress("]")
-
-        # Scheme entry in the form 'sections -> target'
-        sections = Fragment.IDENTIFIER
-        target = Fragment.IDENTIFIER
-        entry = Group(sections.setResultsName("sections") + Suppress("->") + target.setResultsName("target"))
-
-        entries = Suppress("entries") + Suppress(":") + Group(OneOrMore(entry)).setResultsName("entries")
-
-        scheme = Group(header + entries)
-
-        scheme.setParseAction(lambda t: Scheme(t[0].name, t[0].entries))
-
-        scheme.ignore("#" + restOfLine)
-
-        return scheme
 
 
 class Mapping(Fragment):
@@ -170,63 +274,70 @@ class Mapping(Fragment):
     Encapsulates a mapping fragment, which defines what targets the input sections of mappable entties are placed under.
     """
 
+    MAPPING_ALL_OBJECTS = "*"
+
+    def set_key_value(self, key, parse_results):
+        if key == "archive":
+            self.archive = parse_results[0]["archive"]
+        elif key == "entries":
+            self.entries = set()
+
+            for result in parse_results:
+                obj = None
+                symbol = None
+                scheme = None
+
+                try:
+                    obj = result["object"]
+                except KeyError:
+                    pass
+
+                try:
+                    symbol = result["symbol"]
+                except KeyError:
+                    pass
+
+                try:
+                    scheme = result["scheme"]
+                except KeyError:
+                    pass
+
+                self.entries.add((obj, symbol, scheme))
+
+    def get_key_grammars(self):
+        # There are three possible patterns for mapping entries:
+        #       obj:symbol (scheme)
+        #       obj (scheme)
+        #       * (scheme)
+        obj = Fragment.ENTITY.setResultsName("object")
+        symbol = Suppress(":") + Fragment.IDENTIFIER.setResultsName("symbol")
+        scheme = Suppress("(") + Fragment.IDENTIFIER.setResultsName("scheme") + Suppress(")")
+
+        pattern1 = obj + symbol + scheme
+        pattern2 = obj + scheme
+        pattern3 = Literal(Mapping.MAPPING_ALL_OBJECTS).setResultsName("object") + scheme
+
+        entry = pattern1 | pattern2 | pattern3
+
+        grammars = {
+            "archive": KeyGrammar(Fragment.ENTITY.setResultsName("archive"), 1, 1, True),
+            "entries": KeyGrammar(entry, 1, None, True)
+        }
+
+        return grammars
+
+
+class DeprecatedMapping():
+    """
+    Encapsulates a mapping fragment, which defines what targets the input sections of mappable entties are placed under.
+    """
+
     # Name of the default condition entry
     DEFAULT_CONDITION = "default"
     MAPPING_ALL_OBJECTS = "*"
 
-    def __init__(self, archive, entries):
-        self.archive = archive
-
-        # Generate name from archive value by replacing all non-alphanumeric
-        # characters with underscore
-        name = Mapping.get_mapping_name_from_archive(self.archive)
-        Fragment.__init__(self, name, entries)
-
-        self._process_entries()
-
-    def _create_mappings_set(self, mappings):
-        mapping_set = set()
-
-        for mapping in mappings:
-            obj = mapping.object
-            symbol = mapping.symbol
-            scheme = mapping.scheme
-
-            if symbol == "":
-                symbol = None
-
-            # Quietly handle duplicate definitions under the same condition
-            mapping_set.add((obj, symbol, scheme))
-
-        return mapping_set
-
-    def _process_entries(self):
-        processed = []
-
-        for normal_group in self.entries[0]:
-            # Get the original string of the condition
-            condition  = next(iter(normal_group.condition.asList())).strip()
-            mappings = self._create_mappings_set(normal_group[1])
-
-            processed.append((condition, mappings))
-
-        default_group = self.entries[1]
-
-        if len(default_group) > 1:
-            mappings = self._create_mappings_set(default_group[1])
-        else:
-            mappings = self._create_mappings_set(default_group[0])
-
-        processed.append(("default", mappings))
-
-        self.entries = processed
-
     @staticmethod
-    def get_mapping_name_from_archive(archive):
+    def get_fragment_grammar(sdkconfig, fragment_file):
-        return re.sub(r"[^0-9a-zA-Z]+", "_", archive)
-
-    @staticmethod
-    def get_fragment_grammar():
 
         # Match header [mapping]
         header = Suppress("[") + Suppress("mapping") + Suppress("]")
@@ -251,7 +362,7 @@ class Mapping(Fragment):
         mapping_entries = Group(ZeroOrMore(mapping_entry)).setResultsName("mappings")
 
         normal_condition = Suppress(":") + originalTextFor(SDKConfig.get_expression_grammar())
-        default_condition = Optional(Suppress(":") + Literal(Mapping.DEFAULT_CONDITION))
+        default_condition = Optional(Suppress(":") + Literal(DeprecatedMapping.DEFAULT_CONDITION))
 
         normal_group = Group(normal_condition.setResultsName("condition") + mapping_entries)
         default_group = Group(default_condition + mapping_entries).setResultsName("default_group")
@@ -263,9 +374,48 @@ class Mapping(Fragment):
         entries = Suppress("entries") + Suppress(":") + (normal_groups + default_group).setResultsName("entries")
 
         mapping = Group(header + archive + entries)
-
-        mapping.setParseAction(lambda t: Mapping(t[0].archive, t[0].entries))
-
         mapping.ignore("#" + restOfLine)
 
+        def parsed_deprecated_mapping(pstr, loc, toks):
+            fragment = Mapping()
+            fragment.archive = toks[0].archive
+            fragment.name = re.sub(r"[^0-9a-zA-Z]+", "_", fragment.archive)
+
+            fragment.entries = set()
+            condition_true = False
+            for entries in toks[0].entries[0]:
+                condition  = next(iter(entries.condition.asList())).strip()
+                condition_val = sdkconfig.evaluate_expression(condition)
+
+                if condition_val:
+                    for entry in entries[1]:
+                        fragment.entries.add((entry.object, None if entry.symbol == '' else entry.symbol, entry.scheme))
+                    condition_true = True
+                    break
+
+            if not fragment.entries and not condition_true:
+                try:
+                    entries = toks[0].entries[1][1]
+                except IndexError:
+                    entries = toks[0].entries[1][0]
+                for entry in entries:
+                    fragment.entries.add((entry.object, None if entry.symbol == '' else entry.symbol, entry.scheme))
+
+            if not fragment.entries:
+                fragment.entries.add(("*", None, "default"))
+
+            dep_warning = str(ParseFatalException(pstr, loc,
+                              "Warning: Deprecated old-style mapping fragment parsed in file %s." % fragment_file))
+
+            print(dep_warning)
+            return fragment
+
+        mapping.setParseAction(parsed_deprecated_mapping)
         return mapping
+
+
+FRAGMENT_TYPES = {
+    "sections": Sections,
+    "scheme": Scheme,
+    "mapping": Mapping
+}

tools/ldgen/generation.py

@@ -321,7 +321,7 @@ class GenerationModel:
 
         return scheme_dictionary
 
-    def generate_rules(self, sdkconfig, sections_infos):
+    def generate_rules(self, sections_infos):
         placement_rules = collections.defaultdict(list)
 
         scheme_dictionary = self._build_scheme_dictionary()
@@ -334,25 +334,12 @@ class GenerationModel:
 
         # Generate rules based on mapping fragments
         for mapping in self.mappings.values():
-            for (condition, entries) in mapping.entries:
-                condition_true = False
-
-                # Only non-default condition are evaluated agains sdkconfig model
-                if condition != Mapping.DEFAULT_CONDITION:
-                    try:
-                        condition_true = sdkconfig.evaluate_expression(condition)
-                    except Exception as e:
-                        raise GenerationException(e.message, mapping)
-                else:
-                    condition_true = True
-
-                if condition_true:
-
             mapping_rules = list()
-
             archive = mapping.archive
-                    for (obj, symbol, scheme_name) in entries:
+            for (obj, symbol, scheme_name) in mapping.entries:
                 try:
+                    if not (obj == Mapping.MAPPING_ALL_OBJECTS and symbol is None and
+                            scheme_name == GenerationModel.DEFAULT_SCHEME):
                         self._add_mapping_rules(archive, obj, symbol, scheme_name, scheme_dictionary, mapping_rules)
                 except KeyError:
                     message = GenerationException.UNDEFINED_REFERENCE + " to scheme '" + scheme_name + "'."
@@ -360,8 +347,6 @@ class GenerationModel:
 
             all_mapping_rules[mapping.name] = mapping_rules
 
-                    break   # Exit on first condition that evaluates to true
-
         # Detect rule conflicts
         for mapping_rules in all_mapping_rules.items():
             self._detect_conflicts(mapping_rules)
@@ -397,8 +382,7 @@ class GenerationModel:
                 if intersections and rule_a.maps_same_entities_as(rule_b):
                     rules_string = str([str(rule_a), str(rule_b)])
                     message = "Rules " + rules_string + " map sections " + str(list(intersections)) + " into multiple targets."
-                    mapping = self.mappings[Mapping.get_mapping_name_from_archive(archive)]
+                    raise GenerationException(message)
-                    raise GenerationException(message, mapping)
 
     def _create_extra_rules(self, rules):
         # This function generates extra rules for symbol specific rules. The reason for generating extra rules is to isolate,
@@ -523,7 +507,7 @@ class TemplateModel:
                 # Does not match marker syntax
                 self.members.append(line)
 
-    def fill(self, mapping_rules, sdkconfig):
+    def fill(self, mapping_rules):
         for member in self.members:
             target = None
             try:

tools/ldgen/ldgen.py

@@ -19,10 +19,11 @@ import argparse
 import sys
 import tempfile
 
-from fragments import FragmentFileModel
+from fragments import FragmentFile
 from sdkconfig import SDKConfig
 from generation import GenerationModel, TemplateModel, SectionsInfo
 from common import LdGenFailure
+from pyparsing import ParseException, ParseFatalException
 
 
 def main():
@@ -52,13 +53,11 @@ def main():
 
     argparser.add_argument(
         "--config", "-c",
-        help="Project configuration",
+        help="Project configuration")
-        type=argparse.FileType("r"))
 
     argparser.add_argument(
         "--kconfig", "-k",
-        help="IDF Kconfig file",
+        help="IDF Kconfig file")
-        type=argparse.FileType("r"))
 
     argparser.add_argument(
         "--env", "-e",
@@ -89,15 +88,22 @@ def main():
 
         generation_model = GenerationModel()
 
+        sdkconfig = SDKConfig(kconfig_file, config_file, args.env)
+
         for fragment_file in fragment_files:
-            fragment_file = FragmentFileModel(fragment_file)
+            try:
+                fragment_file = FragmentFile(fragment_file, sdkconfig)
+            except (ParseException, ParseFatalException) as e:
+                # ParseException is raised on incorrect grammar
+                # ParseFatalException is raised on correct grammar, but inconsistent contents (ex. duplicate
+                # keys, key unsupported by fragment, unexpected number of values, etc.)
+                raise LdGenFailure("failed to parse %s\n%s" % (fragment_file.name, str(e)))
             generation_model.add_fragments_from_file(fragment_file)
 
-        sdkconfig = SDKConfig(kconfig_file, config_file, args.env)
+        mapping_rules = generation_model.generate_rules(sections_infos)
-        mapping_rules = generation_model.generate_rules(sdkconfig, sections_infos)
 
         script_model = TemplateModel(input_file)
-        script_model.fill(mapping_rules, sdkconfig)
+        script_model.fill(mapping_rules)
 
         with tempfile.TemporaryFile("w+") as output:
             script_model.write(output)

tools/ldgen/samples/mappings.lf

@@ -1,62 +1,62 @@
 
-[mapping]
+[mapping:heap]
 archive: libheap.a
 entries:
     multi_heap (noflash)
     multi_heap_poisoning (noflash)
 
-[mapping]
+[mapping:soc]
 archive: libsoc.a
 entries: 
     * (noflash)
 
-[mapping]
+[mapping:freertos]
 archive: libfreertos.a
 entries:
     * (noflash)
 
-[mapping]
+[mapping:esp32]
 archive: libesp32.a
 entries: 
     core_dump (noflash)
     panic (noflash)
 
-[mapping]
+[mapping:app_trace]
 archive: libapp_trace.a
 entries: 
     * (noflash)
 
-[mapping]
+[mapping:xtensa_debug_module]
 archive: libxtensa-debug-module.a
 entries: 
     eri (noflash)
 
-[mapping]
+[mapping:phy]
 archive: libphy.a
 entries: 
     * (noflash_data)
 
-[mapping]
+[mapping:rtc]
 archive: librtc.a
 entries: 
     * (noflash)
 
-[mapping]
+[mapping:hal]
 archive: libhal.a
 entries: 
     * (noflash)
 
-[mapping]
+[mapping:gcc]
 archive: libgcc.a
 entries: 
     lib2funcs (noflash)
 
-[mapping]
+[mapping:spi_flash]
 archive: libspi_flash.a
 entries: 
     spi_flash_rom_patch (noflash)
 
-[mapping]
+[mapping:gcov]
 archive: libgcov.a
 entries: 
     * (noflash)

tools/ldgen/sdkconfig.py

@@ -15,7 +15,7 @@
 #
 
 import os
-from pyparsing import Word, printables, Combine, Literal, hexnums, quotedString, Optional, nums, removeQuotes, oneOf, Group, infixNotation, opAssoc
+from pyparsing import Word, alphanums, printables, Combine, Literal, hexnums, quotedString, Optional, nums, removeQuotes, oneOf, Group, infixNotation, opAssoc
 
 import sys
 try:
@@ -34,11 +34,11 @@ class SDKConfig:
     """
 
     # A configuration entry is in the form CONFIG=VALUE. Definitions of components of that grammar
-    IDENTIFIER = Word(printables.upper())
+    IDENTIFIER = Word(alphanums.upper() + "_")
 
     HEX = Combine("0x" + Word(hexnums)).setParseAction(lambda t:int(t[0], 16))
     DECIMAL = Combine(Optional(Literal("+") | Literal("-")) + Word(nums)).setParseAction(lambda t:int(t[0]))
-    LITERAL = Word(printables)
+    LITERAL = Word(printables.replace(":", ""))
     QUOTED_LITERAL = quotedString.setParseAction(removeQuotes)
 
     VALUE = HEX | DECIMAL | LITERAL | QUOTED_LITERAL
@@ -53,8 +53,8 @@ class SDKConfig:
             value = " ".join(value.split())
             os.environ[name] = value
 
-        self.config = kconfiglib.Kconfig(kconfig_file.name)
+        self.config = kconfiglib.Kconfig(kconfig_file)
-        self.config.load_config(sdkconfig_file.name)
+        self.config.load_config(sdkconfig_file)
 
     def evaluate_expression(self, expression):
         result = self.config.eval_string(expression)
@@ -64,7 +64,7 @@ class SDKConfig:
         elif result == 2:  # y
             return True
         else:  # m
-            raise Exception("Unsupported config expression result.")
+            raise Exception("unsupported config expression result")
 
     @staticmethod
     def get_expression_grammar():

tools/ldgen/test/data/Kconfig

@@ -1,7 +1,19 @@
 menu "Test config"
 
     config PERFORMANCE_LEVEL
-        int "Performance level"
+        int
         range 0 3
+        prompt "Performance level"
 
+    config A
+        bool
+        default "y"
+
+    config B
+        bool
+        default "n"
+
+    config C
+        bool
+        default "y"
 endmenu