alex/esp-idf
alex/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf.git synced 2024-10-05 20:47:46 -04:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Merge branch 'feature/common_fragment_grammar' into 'master'

Browse Source 
ldgen: Common fragment grammar

See merge request idf/esp-idf!4576
This commit is contained in:
Angus Gratton 2019-04-05 07:19:22 +08:00
commit 041c784ff8
22 changed files with 2144 additions and 1490 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
components/app_trace/linker.lf
View File

@ -1,4 +1,4 @@
[mapping]
[mapping:app_trace]
archive: libapp_trace.a
entries:
app_trace (noflash)
@ -8,10 +8,11 @@ entries:
SEGGER_SYSVIEW_Config_FreeRTOS (noflash)
SEGGER_SYSVIEW_FreeRTOS (noflash)
[mapping]
[mapping:driver]
archive: libdriver.a
entries:
: SYSVIEW_TS_SOURCE_TIMER_00 = y || SYSVIEW_TS_SOURCE_TIMER_01 = y
|| SYSVIEW_TS_SOURCE_TIMER_10 = y || SYSVIEW_TS_SOURCE_TIMER_11 = y
timer (noflash)
if SYSVIEW_TS_SOURCE_TIMER_00 = y || SYSVIEW_TS_SOURCE_TIMER_01 = y
|| SYSVIEW_TS_SOURCE_TIMER_10 = y || SYSVIEW_TS_SOURCE_TIMER_11 = y:
timer (noflash)
else:
* (default)

6
components/esp32/linker.lf
View File

@ -1,14 +1,14 @@
[mapping]
[mapping:esp32]
archive: libesp32.a
entries:
panic (noflash)
[mapping]
[mapping:gcc]
archive: libgcc.a
entries:
lib2funcs (noflash_text)
[mapping]
[mapping:gcov]
archive: libgcov.a
entries:
* (noflash)

2
components/esp_ringbuf/linker.lf
View File

@ -1,4 +1,4 @@
[mapping]
[mapping:esp_ringbuf]
archive: libesp_ringbuf.a
entries:
* (noflash_text)

17
components/esp_wifi/linker.lf
View File

@ -1,22 +1,21 @@
[mapping]
[mapping:phy]
archive: libphy.a
entries:
* (noflash_data)
[mapping]
[mapping:rtc]
archive: librtc.a
entries:
* (noflash_text)
[mapping]
[mapping:pp]
archive: libpp.a
entries:
: ESP32_WIFI_IRAM_OPT = y
* (wifi_iram)
if ESP32_WIFI_IRAM_OPT = y:
* (wifi_iram)
[mapping]
[mapping:net80211]
archive: libnet80211.a
entries:
: ESP32_WIFI_IRAM_OPT = y
* (wifi_iram)
if ESP32_WIFI_IRAM_OPT = y:
* (wifi_iram)

2
components/espcoredump/linker.lf
View File

@ -1,4 +1,4 @@
[mapping]
[mapping:espcoredump]
archive: libespcoredump.a
entries:
core_dump_uart (noflash_text)

2
components/freertos/linker.lf
View File

@ -1,4 +1,4 @@
[mapping]
[mapping:freertos]
archive: libfreertos.a
entries:
* (noflash_text)

2
components/heap/linker.lf
View File

@ -1,4 +1,4 @@
[mapping]
[mapping:heap]
archive: libheap.a
entries:
multi_heap (noflash)

6
components/lwip/linker.lf
View File

@ -1,7 +1,7 @@
[mapping]
[mapping:lwip]
archive: liblwip.a
entries:
:LWIP_IRAM_OPTIMIZATION = y
if LWIP_IRAM_OPTIMIZATION = y:
ethernetif:ethernet_low_level_output (noflash_text)
ethernetif:ethernetif_input (noflash_text)
wlanif:low_level_output (noflash_text)
@ -55,3 +55,5 @@ entries:
pbuf:pbuf_header_impl (noflash_text)
pbuf:pbuf_header (noflash_text)
ethernet:ethernet_input (noflash_text)
else:
* (default)

264
components/newlib/esp32-spiram-rom-functions.lf
View File

@ -8,137 +8,139 @@
# Note: We currently never link libg-psram-workaround.a, so no rules
# are generated for this library
[mapping]
[mapping:libc_psram_workaround]
archive: libc-psram-workaround.a
entries:
: SPIRAM_CACHE_WORKAROUND = y
lib_a-utoa (noflash)
lib_a-longjmp (noflash)
lib_a-setjmp (noflash)
lib_a-abs (noflash)
lib_a-div (noflash)
lib_a-labs (noflash)
lib_a-ldiv (noflash)
lib_a-quorem (noflash)
lib_a-utoa (noflash)
lib_a-itoa (noflash)
lib_a-atoi (noflash)
lib_a-atol (noflash)
lib_a-strtol (noflash)
lib_a-strtoul (noflash)
lib_a-wcrtomb (noflash)
lib_a-fvwrite (noflash)
lib_a-wbuf (noflash)
lib_a-wsetup (noflash)
lib_a-fputwc (noflash)
lib_a-wctomb_r (noflash)
lib_a-ungetc (noflash)
lib_a-makebuf (noflash)
lib_a-fflush (noflash)
lib_a-refill (noflash)
lib_a-s_fpclassify (noflash)
lib_a-asctime (noflash)
lib_a-ctime (noflash)
lib_a-ctime_r (noflash)
lib_a-lcltime (noflash)
lib_a-lcltime_r (noflash)
lib_a-gmtime (noflash)
lib_a-gmtime_r (noflash)
lib_a-strftime (noflash)
lib_a-mktime (noflash)
lib_a-syswrite (noflash)
lib_a-tzset_r (noflash)
lib_a-tzset (noflash)
lib_a-toupper (noflash)
lib_a-tolower (noflash)
lib_a-toascii (noflash)
lib_a-systimes (noflash)
lib_a-time (noflash)
lib_a-gettzinfo (noflash)
lib_a-strupr (noflash)
lib_a-asctime_r (noflash)
lib_a-bzero (noflash)
lib_a-close (noflash)
lib_a-creat (noflash)
lib_a-environ (noflash)
lib_a-fclose (noflash)
lib_a-isalnum (noflash)
lib_a-isalpha (noflash)
lib_a-isascii (noflash)
lib_a-isblank (noflash)
lib_a-iscntrl (noflash)
lib_a-isdigit (noflash)
lib_a-isgraph (noflash)
lib_a-islower (noflash)
lib_a-isprint (noflash)
lib_a-ispunct (noflash)
lib_a-isspace (noflash)
lib_a-isupper (noflash)
lib_a-memccpy (noflash)
lib_a-memchr (noflash)
lib_a-memcmp (noflash)
lib_a-memcpy (noflash)
lib_a-memmove (noflash)
lib_a-memrchr (noflash)
lib_a-memset (noflash)
lib_a-open (noflash)
lib_a-rand (noflash)
lib_a-rand_r (noflash)
lib_a-read (noflash)
lib_a-rshift (noflash)
lib_a-sbrk (noflash)
lib_a-srand (noflash)
lib_a-strcasecmp (noflash)
lib_a-strcasestr (noflash)
lib_a-strcat (noflash)
lib_a-strchr (noflash)
lib_a-strcmp (noflash)
lib_a-strcoll (noflash)
lib_a-strcpy (noflash)
lib_a-strcspn (noflash)
lib_a-strdup (noflash)
lib_a-strlcat (noflash)
lib_a-strlcpy (noflash)
lib_a-strlen (noflash)
lib_a-strlwr (noflash)
lib_a-strncasecmp (noflash)
lib_a-strncat (noflash)
lib_a-strncmp (noflash)
lib_a-strncpy (noflash)
lib_a-strndup (noflash)
lib_a-strnlen (noflash)
lib_a-strrchr (noflash)
lib_a-strsep (noflash)
lib_a-strspn (noflash)
lib_a-strstr (noflash)
lib_a-strtok_r (noflash)
lib_a-strupr (noflash)
lib_a-stdio (noflash)
lib_a-syssbrk (noflash)
lib_a-sysclose (noflash)
lib_a-sysopen (noflash)
creat (noflash)
lib_a-sysread (noflash)
lib_a-syswrite (noflash)
lib_a-impure (noflash)
lib_a-tzvars (noflash)
lib_a-sf_nan (noflash)
lib_a-tzcalc_limits (noflash)
lib_a-month_lengths (noflash)
lib_a-timelocal (noflash)
lib_a-findfp (noflash)
lock (noflash)
lib_a-getenv_r (noflash)
isatty (noflash)
lib_a-fwalk (noflash)
lib_a-getenv_r (noflash)
lib_a-tzlock (noflash)
lib_a-ctype_ (noflash)
lib_a-sccl (noflash)
lib_a-strptime (noflash)
lib_a-envlock (noflash)
lib_a-raise (noflash)
lib_a-strdup_r (noflash)
lib_a-system (noflash)
lib_a-strndup_r (noflash)
if SPIRAM_CACHE_WORKAROUND = y:
lib_a-utoa (noflash)
lib_a-longjmp (noflash)
lib_a-setjmp (noflash)
lib_a-abs (noflash)
lib_a-div (noflash)
lib_a-labs (noflash)
lib_a-ldiv (noflash)
lib_a-quorem (noflash)
lib_a-utoa (noflash)
lib_a-itoa (noflash)
lib_a-atoi (noflash)
lib_a-atol (noflash)
lib_a-strtol (noflash)
lib_a-strtoul (noflash)
lib_a-wcrtomb (noflash)
lib_a-fvwrite (noflash)
lib_a-wbuf (noflash)
lib_a-wsetup (noflash)
lib_a-fputwc (noflash)
lib_a-wctomb_r (noflash)
lib_a-ungetc (noflash)
lib_a-makebuf (noflash)
lib_a-fflush (noflash)
lib_a-refill (noflash)
lib_a-s_fpclassify (noflash)
lib_a-asctime (noflash)
lib_a-ctime (noflash)
lib_a-ctime_r (noflash)
lib_a-lcltime (noflash)
lib_a-lcltime_r (noflash)
lib_a-gmtime (noflash)
lib_a-gmtime_r (noflash)
lib_a-strftime (noflash)
lib_a-mktime (noflash)
lib_a-syswrite (noflash)
lib_a-tzset_r (noflash)
lib_a-tzset (noflash)
lib_a-toupper (noflash)
lib_a-tolower (noflash)
lib_a-toascii (noflash)
lib_a-systimes (noflash)
lib_a-time (noflash)
lib_a-gettzinfo (noflash)
lib_a-strupr (noflash)
lib_a-asctime_r (noflash)
lib_a-bzero (noflash)
lib_a-close (noflash)
lib_a-creat (noflash)
lib_a-environ (noflash)
lib_a-fclose (noflash)
lib_a-isalnum (noflash)
lib_a-isalpha (noflash)
lib_a-isascii (noflash)
lib_a-isblank (noflash)
lib_a-iscntrl (noflash)
lib_a-isdigit (noflash)
lib_a-isgraph (noflash)
lib_a-islower (noflash)
lib_a-isprint (noflash)
lib_a-ispunct (noflash)
lib_a-isspace (noflash)
lib_a-isupper (noflash)
lib_a-memccpy (noflash)
lib_a-memchr (noflash)
lib_a-memcmp (noflash)
lib_a-memcpy (noflash)
lib_a-memmove (noflash)
lib_a-memrchr (noflash)
lib_a-memset (noflash)
lib_a-open (noflash)
lib_a-rand (noflash)
lib_a-rand_r (noflash)
lib_a-read (noflash)
lib_a-rshift (noflash)
lib_a-sbrk (noflash)
lib_a-srand (noflash)
lib_a-strcasecmp (noflash)
lib_a-strcasestr (noflash)
lib_a-strcat (noflash)
lib_a-strchr (noflash)
lib_a-strcmp (noflash)
lib_a-strcoll (noflash)
lib_a-strcpy (noflash)
lib_a-strcspn (noflash)
lib_a-strdup (noflash)
lib_a-strlcat (noflash)
lib_a-strlcpy (noflash)
lib_a-strlen (noflash)
lib_a-strlwr (noflash)
lib_a-strncasecmp (noflash)
lib_a-strncat (noflash)
lib_a-strncmp (noflash)
lib_a-strncpy (noflash)
lib_a-strndup (noflash)
lib_a-strnlen (noflash)
lib_a-strrchr (noflash)
lib_a-strsep (noflash)
lib_a-strspn (noflash)
lib_a-strstr (noflash)
lib_a-strtok_r (noflash)
lib_a-strupr (noflash)
lib_a-stdio (noflash)
lib_a-syssbrk (noflash)
lib_a-sysclose (noflash)
lib_a-sysopen (noflash)
creat (noflash)
lib_a-sysread (noflash)
lib_a-syswrite (noflash)
lib_a-impure (noflash)
lib_a-tzvars (noflash)
lib_a-sf_nan (noflash)
lib_a-tzcalc_limits (noflash)
lib_a-month_lengths (noflash)
lib_a-timelocal (noflash)
lib_a-findfp (noflash)
lock (noflash)
lib_a-getenv_r (noflash)
isatty (noflash)
lib_a-fwalk (noflash)
lib_a-getenv_r (noflash)
lib_a-tzlock (noflash)
lib_a-ctype_ (noflash)
lib_a-sccl (noflash)
lib_a-strptime (noflash)
lib_a-envlock (noflash)
lib_a-raise (noflash)
lib_a-strdup_r (noflash)
lib_a-system (noflash)
lib_a-strndup_r (noflash)
else:
* (default)

2
components/soc/linker.lf
View File

@ -1,4 +1,4 @@
[mapping]
[mapping:soc]
archive: libsoc.a
entries:
cpu_util (noflash_text)

2
components/spi_flash/linker.lf
View File

@ -1,4 +1,4 @@
[mapping]
[mapping:spi_flash]
archive: libspi_flash.a
entries:
spi_flash_rom_patch (noflash_text)

2
components/vfs/test/linker.lf
View File

@ -1,4 +1,4 @@
[mapping]
[mapping:vfs]
archive: libvfs.a
entries:
vfs:esp_vfs_open (noflash)

4
components/xtensa/linker.lf
View File

@ -1,9 +1,9 @@
[mapping]
[mapping:xtensa]
archive: libxtensa.a
entries:
eri (noflash_text)
[mapping]
[mapping:hal]
archive: libhal.a
entries:
* (noflash_text)

535
docs/en/api-guides/linker-script-generation.rst
View File

@ -5,131 +5,131 @@ Linker Script Generation
Overview
--------
There are several :ref:`memory regions<memory-layout>` where code and data can be placed. Usually, code and read-only data are placed in flash regions,
writable data in RAM, etc. A common action is changing where code/data are mapped by default, say placing critical code/rodata in RAM for performance
reasons or placing code/data/rodata in RTC memory for use in a wake stub or the ULP coprocessor.
There are several :ref:`memory regions<memory-layout>` where code and data can be placed. Code and read-only data are placed by default in flash,
writable data in RAM, etc. However, it is sometimes necessary to change these default placements. For example, it may
be necessary to place critical code in RAM for performance reasons or to place code in RTC memory for use in a wake stub or the ULP coprocessor.
IDF provides the ability for defining these placements at the component level using the linker script generation mechanism. The component presents
how it would like to map the input sections of its object files (or even functions/data) through :ref:`linker fragment files<ldgen-fragment-files>`. During app build,
the linker fragment files are collected, parsed and processed; and the :ref:`linker script template<ldgen-script-templates>` is augmented with
information generated from the fragment files to produce the final linker script. This linker script is then used for the linking
the final app binary.
With the linker script generation mechanism, it is possible to specify these placements at the component level within ESP-IDF. The component presents
information on how it would like to place its symbols, objects or the entire archive. During build the information presented by the components are collected,
parsed and processed; and the placement rules generated is used to link the app.
Quick Start
------------
This section presents a guide for quickly placing code/data to RAM and RTC memory; as well as demonstrating how to make these placements
dependent on project configuration values. In a true quick start fashion, this section glosses over terms and concepts that will be discussed
at a later part of the document. However, whenever it does so, it provides a link to the relevant section on the first mention.
This section presents a guide for quickly placing code/data to RAM and RTC memory - placements ESP-IDF provides out-of-the-box.
.. _ldgen-add-fragment-file :
For this guide, suppose we have the following::
Preparation
^^^^^^^^^^^
- components/
- my_component/
- CMakeLists.txt
- component.mk
- Kconfig
- src/
- my_src1.c
- my_src2.c
- my_src3.c
- my_linker_fragment_file.lf
- a component named ``my_component`` that is archived as library ``libmy_component.a`` during build
- three source files archived under the library, ``my_src1.c``, ``my_src2.c`` and ``my_src3.c`` which are compiled as ``my_src1.o``, ``my_src2.o`` and ``my_src3.o``, respectively
- under ``my_src1.o``, the function ``my_function1`` is defined; under ``my_src2.o``, the function ``my_function2`` is defined
- there exist bool-type config ``PERFORMANCE_MODE`` (y/n) and int type config ``PERFORMANCE_LEVEL`` (with range 0-3) in my_component's Kconfig
Creating and Specifying a Linker Fragment File
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Before anything else, a linker fragment file needs to be created. A linker fragment file
is simply a text file with a ``.lf`` extension upon which the desired placements will be written.
After creating the file, it is then necessary to present it to the build system. The instructions for the build systems
supported by ESP-IDF are as follows:
Make
""""
Create a linker fragment file inside the component directory, which is just a text file with a .lf extension. In order for the build system to collect your fragment file,
add an entry to it from the component, set the variable ``COMPONENT_ADD_LDFRAGMENTS`` to your linker file/s before the ``register_component`` call.
In the component's ``component.mk`` file, set the variable ``COMPONENT_ADD_LDFRAGMENTS`` to the path of the created linker
fragment file. The path can either be an absolute path or a relative path from the component directory.
.. code-block:: make
# file paths relative to component Makefile
COMPONENT_ADD_LDFRAGMENTS += "path/to/linker_fragment_file.lf" "path/to/another_linker_fragment_file.lf"
COMPONENT_ADD_LDFRAGMENTS += "my_linker_fragment_file.lf"
CMake
"""""
For CMake set the variable ``COMPONENT_ADD_LDFRAGMENTS`` to your linker file/s before the ``register_component`` call.
In the component's ``CMakeLists.txt`` file, set the variable ``COMPONENT_ADD_LDFRAGMENTS`` to the path of the created linker
fragment file before the ``register_component`` call. The path can either be an absolute path or a relative path from the component directory.
.. code-block:: cmake
# file paths relative to CMakeLists.txt
set(COMPONENT_ADD_LDFRAGMENTS "path/to/linker_fragment_file.lf" "path/to/another_linker_fragment_file.lf")
set(COMPONENT_ADD_LDFRAGMENTS "my_linker_fragment_file.lf")
register_component()
It is also possible to specify fragment files from the project CMakeLists.txt or component project_include.cmake using the function `ldgen_add_fragment_files`::
ldgen_add_fragment_files(target files ...)
Specifying placements
^^^^^^^^^^^^^^^^^^^^^
This mechanism allows specifying placement of the following entities:
It is possible to specify placements at the following levels of granularity:
- one or multiple object files within the component
- one or multiple function/variable using their names
- the entire component library
For the following text, suppose we have the following:
- a component named ``component`` that is archived as library ``libcomponent.a`` during build
- three object files archived under the library, ``object1.o``, ``object2.o`` and ``object3.o``
- under ``object1.o``, the function ``function1`` is defined; under ``object2.o``, the function ``function2`` is defined
- there exist configuration ``PERFORMANCE_MODE`` and ``PERFORMANCE_LEVEL`` in one of the IDF KConfig files, with the set value indicated by entries ``CONFIG_PERFORMANCE_MODE`` and ``CONFIG_PERFORMANCE_LEVEL`` in the project sdkconfig
In the created linker fragment file, we write:
.. code-block:: none
[mapping]
archive: libcomponent.a
entries:
This creates an empty :ref:`mapping fragment<ldgen-mapping-fragment>`, which doesn't do anything yet. During linking the :ref:`default placements<ldgen-default-placements>`
will still be used for ``libcomponent.a``, unless the ``entries`` key is populated.
- object file (``.obj`` or ``.o`` files)
- symbol (function/variable)
- archive (``.a`` files)
.. _ldgen-placing-object-files :
Placing object files
""""""""""""""""""""
Suppose the entirety of ``object1.o`` is performance-critical, so it is desirable to place it in RAM. On the other hand, suppose all of ``object2.o`` contains things to be executed coming out of deep sleep, so it needs to be put under RTC memory. We can write:
Suppose the entirety of ``my_src1.o`` is performance-critical, so it is desirable to place it in RAM.
On the other hand, the entirety of ``my_src2.o`` contains symbols needed coming out of deep sleep, so it needs to be put under RTC memory.
In the the linker fragment file, we can write:
.. code-block:: none
[mapping]
archive: libcomponent.a
[mapping:my_component]
archive: libmy_component.a
entries:
object1 (noflash) # places all code / read-only data under IRAM/ DRAM
object2 (rtc) # places all code/ data and read-only data under RTC fast memory/ RTC slow memory
my_src1 (noflash) # places all my_src1 code/read-only data under IRAM/DRAM
my_src2 (rtc) # places all my_src2 code/ data and read-only data under RTC fast memory/RTC slow memory
What happens to ``object3.o``? Since it is not specified, default placements are used for ``object3.o``.
What happens to ``my_src3.o``? Since it is not specified, default placements are used for ``my_src3.o``. More on default placements
:ref:`here<ldgen-default-placements>`.
Placing functions/data using their names
""""""""""""""""""""""""""""""""""""""""
Placing symbols
""""""""""""""""
Continuing our example, suppose that among functions defined under ``object1.o``, only ``function1`` is performance-critical; and under ``object2.o``,
only ``function2`` needs to execute after the chip comes out of deep sleep. This could be accomplished by writing:
Continuing our example, suppose that among functions defined under ``object1.o``, only ``my_function1`` is performance-critical; and under ``object2.o``,
only ``my_function2`` needs to execute after the chip comes out of deep sleep. This could be accomplished by writing:
.. code-block:: none
[mapping]
archive: libcomponent.a
[mapping:my_component]
archive: libmy_component.a
entries:
object1:function1 (noflash)
object2:function2 (rtc)
my_src1:my_function1 (noflash)
my_src2:my_function2 (rtc)
The default placements are used for the rest of the functions in ``object1.o`` and ``object2.o`` and the entire ``object3.o``. Something similar
can be achieved for placing data by writing the variable name instead of the function name after ``:``.
The default placements are used for the rest of the functions in ``my_src1.o`` and ``my_src2.o`` and the entire ``object3.o``. Something similar
can be achieved for placing data by writing the variable name instead of the function name, like so::
my_src1:my_variable (noflash)
.. warning::
There are :ref:`limitations<ldgen-type1-limitations>` in placing code/data using their symbol names. In order to ensure proper placements, an alternative would be to group
relevant code and data into source files, and :ref:`use object file placement<ldgen-placing-object-files>`.
There are :ref:`limitations<ldgen-symbol-granularity-placements>` in placing code/data at symbol granularity. In order to ensure proper placements, an alternative would be to group
relevant code and data into source files, and :ref:`use object-granularity placements<ldgen-placing-object-files>`.
Placing entire component
""""""""""""""""""""""""
Placing entire archive
"""""""""""""""""""""""
In this example, suppose that the entire component needs to be placed in RAM. This can be written as:
In this example, suppose that the entire component archive needs to be placed in RAM. This can be written as:
.. code-block:: none
[mapping]
archive: libcomponent.a
[mapping:my_component]
archive: libmy_component.a
entries:
* (noflash)
@ -137,122 +137,226 @@ Similarly, this places the entire component in RTC memory:
.. code-block:: none
[mapping]
archive: libcomponent.a
[mapping:my_component]
archive: libmy_component.a
entries:
* (rtc)
Configuration-dependent placements
""""""""""""""""""""""""""""""""""
Suppose that the entire component library should only be placed when ``CONFIG_PERFORMANCE_MODE == y`` in the sdkconfig. This could be written as:
Suppose that the entire component library should only have special placement when a certain condition is true; for example, when ``CONFIG_PERFORMANCE_MODE == y``.
This could be written as:
.. code-block:: none
[mapping]
archive: libcomponent.a
[mapping:my_component]
archive: libmy_component.a
entries:
: PERFORMANCE_MODE = y
* (noflash)
if PERFORMANCE_MODE = y:
* (noflash)
else:
* (default)
In pseudocode, this translates to:
.. code-block:: none
if PERFORMANCE_MODE = y
place entire libcomponent.a in RAM
else
use default placements
It is also possible to have multiple conditions to test. Suppose the following requirements: when ``CONFIG_PERFORMANCE_LEVEL == 1``, only ``object1.o`` is put in RAM;
For a more complex config-dependent placement, suppose the following requirements: when ``CONFIG_PERFORMANCE_LEVEL == 1``, only ``object1.o`` is put in RAM;
when ``CONFIG_PERFORMANCE_LEVEL == 2``, ``object1.o`` and ``object2.o``; and when ``CONFIG_PERFORMANCE_LEVEL == 3`` all object files under the archive
are to be put into RAM. When these three are false however, put entire library in RTC memory. This scenario is a bit contrived, but,
are to be put into RAM. When these three are false however, put entire library in RTC memory. This scenario is a bit contrived, but,
it can be written as:
.. code-block:: none
[mapping]
archive: libcomponent.a
[mapping:my_component]
archive: libmy_component.a
entries:
: PERFORMANCE_LEVEL = 3
* (noflash)
: PERFORMANCE_LEVEL = 2
object1 (noflash)
object2 (noflash)
: PERFORMANCE_LEVEL = 1
object1 (noflash)
: default
* (rtc)
if PERFORMANCE_LEVEL = 1:
my_src1 (noflash)
elif PERFORMANCE_LEVEL = 2:
my_src1 (noflash)
my_src2 (noflash)
elif PERFORMANCE_LEVEL = 3:
my_src1 (noflash)
my_src2 (noflash)
my_src3 (noflash)
else:
* (rtc)
Which reads:
Nesting condition-checking is also possible. The following is equivalent to the snippet above:
.. code-block:: none
if CONFIG_PERFORMANCE_LEVEL == 3
place entire libcomponent.a in RAM
else if CONFIG_PERFORMANCE_LEVEL == 2
only place object1.o and object2.o in RAM
else if CONFIG_PERFORMANCE_LEVEL == 1
only place object1.o in RAM
else
place entire libcomponent.a in RTC memory
The conditions test :ref:`support other operations<ldgen-condition-entries>`.
[mapping:my_component]
archive: libmy_component.a
entries:
if PERFORMANCE_LEVEL <= 3 && PERFORMANCE_LEVEL > 0:
if PERFORMANCE_LEVEL >= 1:
object1 (noflash)
if PERFORMANCE_LEVEL >= 2:
object2 (noflash)
if PERFORMANCE_LEVEL >= 3:
object2 (noflash)
else:
* (rtc)
.. _ldgen-default-placements:
The 'default' placements
^^^^^^^^^^^^^^^^^^^^^^^^
Up until this point, the term 'default placements' has been mentioned as fallback placements for when the
placement rules ``rtc`` and ``noflash`` are not specified. The tokens ``noflash`` or ``rtc`` are not merely keywords known by the mechanism, but are actually
objects called :ref:`scheme fragments<ldgen-scheme-fragment>` that are specified by the user. Due to the commonness of these placement use cases,
they are pre-defined in IDF.
Up until this point, the term 'default placements' has been mentioned as fallback placements for when the
placement rules ``rtc`` and ``noflash`` are not specified. It is important to note that the tokens ``noflash`` or ``rtc`` are not merely keywords, but are actually
entities called fragments, specifically :ref:`schemes<ldgen-scheme-fragment>`.
Similarly, there exists a ``default`` scheme fragment which defines what the default placement rules should be, which is discussed :ref:`here<ldgen-default-scheme>`.
In the same manner as ``rtc`` and ``noflash`` are schemes, there exists a ``default`` scheme which defines what the default placement rules should be.
As the name suggests, it is where code and data are usually placed, i.e. code/constants is placed in flash, variables
placed in RAM, etc. More on the default scheme :ref:`here<ldgen-default-scheme>`.
.. note::
For an example of an IDF component using this feature, see :component_file:`freertos/CMakeLists.txt`. The ``freertos`` component uses this
mechanism to place all code, literal and rodata of all of its object files to the instruction RAM memory region for performance reasons.
For an example of an ESP-IDF component using the linker script generation mechanism, see :component_file:`freertos/CMakeLists.txt`.
``freertos`` uses this to place its object files to the instruction RAM for performance reasons.
This marks the end of the quick start guide. The following text discusses this mechanism in a little bit more detail, such its components, essential concepts,
the syntax, how it is integrated with the build system, etc. The following sections should be helpful in creating custom mappings or modifying default
behavior.
This marks the end of the quick start guide. The following text discusses the internals of the mechanism in a little bit more detail.
The following sections should be helpful in creating custom placements or modifying default behavior.
Components
----------
Linker Script Generation Internals
----------------------------------
.. _ldgen-fragment-files :
Linking is the last step in the process of turning C/C++ source files into an executable. It is performed by the toolchain's linker, and accepts
linker scripts which specify code/data placements, among other things. With the linker script generation mechanism, this process is no different, except
that the linker script passed to the linker is dynamically generated from: (1) the collected :ref:`linker fragment files<ldgen-linker-fragment-files>` and
(2) :ref:`linker script template<ldgen-linker-script-template>`.
.. note::
The tool that implements the linker script generation mechanism lives under :idf:`tools/ldgen`.
.. _ldgen-linker-fragment-files :
Linker Fragment Files
^^^^^^^^^^^^^^^^^^^^^
The fragment files contain objects called 'fragments'. These fragments contain pieces of information which, when put together, form
placement rules that tell where to place sections of object files in the output binary.
As mentioned in the quick start guide, fragment files are simple text files with the ``.lf`` extension containing the desired placements. This is a simplified
description of what fragment files contain, however. What fragment files actually contain are 'fragments'. Fragments are entities which contain pieces of information which, when put together, form
placement rules that tell where to place sections of object files in the output binary. There are three types of fragments: :ref:`sections<ldgen-sections-fragment>`,
:ref:`scheme<ldgen-scheme-fragment>` and :ref:`mapping<ldgen-mapping-fragment>`.
Another way of putting it is that processing linker fragment files aims to create the section placement rules inside GNU LD ``SECTIONS`` command.
Where to collect and put these section placement rules is represented internally as a ``target`` token.
Grammar
"""""""
The three types of fragments are discussed below.
The three fragment types share a common grammar:
.. code-block:: none
[type:name]
key: value
key:
value
value
value
...
- type: Corresponds to the fragment type, can either be ``sections``, ``scheme`` or ``mapping``.
- name: The name of the fragment, should be unique for the specified fragment type.
- key, value: Contents of the fragment; each fragment type may support different keys and different grammars for the key values.
.. note::
Fragments have a name property (except mapping fragments) and are known globally.
Fragment naming follows C variable naming rules, i.e. case sensitive, must begin with a letter or underscore, alphanumeric/underscore after
initial characters are allowed, no spaces/special characters. Each type of fragment has its own namespace. In cases where multiple fragments
of the same type and name are encountered, an exception is thrown.
In cases where multiple fragments of the same type and name are encountered, an exception is thrown.
.. note::
The only valid characters for fragment names and keys are alphanumeric characters and underscore.
.. _ldgen-condition-checking :
**Condition Checking**
Condition checking enable the linker script generation to be configuration-aware. Depending on whether expressions involving configuration values
are true or not, a particular set of values for a key can be used. The evaluation uses ``eval_string`` from :idf_file:`tools/kconfig_new/kconfiglib.py`
and adheres to its required syntax and limitations. Supported operators are as follows:
- comparison
- LessThan ``<``
- LessThanOrEqualTo ``<=``
- MoreThan ``>``
- MoreThanOrEqualTo ``>=``
- Equal ``=``
- NotEqual ``!=``
- logical
- Or ``||``
- And ``&&``
- Negation ``!``
- grouping
- Parenthesis ``()``
Condition checking behaves as you would expect an ``if...elseif/elif...else`` block in other languages. Condition-checking is possible
for both key values and entire fragments. The two sample fragments below are equivalent:
.. code-block:: none
# Value for keys is dependent on config
[type:name]
key_1:
if CONDITION = y:
value_1
else:
value_2
key_2:
if CONDITION = y:
value_a
else:
value_b
.. code-block:: none
# Entire fragment definition is dependent on config
if CONDITION = y:
[type:name]
key_1:
value_1
key_2:
value_b
else:
[type:name]
key_1:
value_2
key_2:
value_b
**Comments**
Comment in linker fragment files begin with ``#``. Like in other languages, comment are used to provide helpful descriptions and documentation
and are ignored during processing.
Compatibility with ESP-IDF v3.x Linker Script Fragment Files
""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
ESP-IDF v4.0 brings some changes to the linker script fragment file grammar:
- indentation is enforced and improperly indented fragment files generate a parse exception; this was not enforced in the old version but previous documentation
and examples demonstrates properly indented grammar
- move to ``if...elif...else`` structure for conditionals, with the ability to nest checks and place entire fragments themselves inside conditionals
- mapping fragments now requires a name like other fragment types
Linker script generator should be able to parse ESP-IDF v3.x linker fragment files that are indented properly (as demonstrated by
the ESP-IDF v3.x version of this document). Backward compatibility with the previous mapping fragment grammar (optional
name and the old grammar for conditionals) has also been retained but with a deprecation warning. Users should switch to the newer grammar discussed
in this document as support for the old grammar is planned to be removed in the future.
Note that linker fragment files using the new ESP-IDF v4.0 grammar is not supported on ESP-IDF v3.x, however.
Types
"""""
.. _ldgen-sections-fragment :
I. Sections
"""""""""""
**Sections**
Sections fragments defines a list of object file sections that the GCC compiler emits. It may be a default section (e.g. ``.text``, ``.data``) or
it may be user defined section through the ``__attribute__`` keyword.
Sections fragments defines a list of object file sections that the GCC compiler emits. It may be a default section (e.g. ``.text``, ``.data``) or
it may be user defined section through the ``__attribute__`` keyword.
The use of an optional '+' indicates the inclusion of the section in the list, as well as sections that start with it. This is the preferred method over listing both explicitly.
**Syntax**
The use of an optional '+' indicates the inclusion of the section in the list, as well as sections that start with it. This is the preferred method over listing both explicitly.
.. code-block:: none
@ -262,7 +366,7 @@ The use of an optional '+' indicates the inclusion of the section in the list, a
.section
...
**Example**
Example:
.. code-block:: none
@ -282,12 +386,9 @@ The use of an optional '+' indicates the inclusion of the section in the list, a
.. _ldgen-scheme-fragment :
II. Scheme
""""""""""
**Scheme**
Scheme fragments define what ``target`` a sections fragment is assigned to.
**Syntax**
Scheme fragments define what ``target`` a sections fragment is assigned to.
.. code-block:: none
@ -297,7 +398,7 @@ Scheme fragments define what ``target`` a sections fragment is assigned to.
sections -> target
...
**Example**
Example:
.. code-block:: none
@ -308,69 +409,46 @@ Scheme fragments define what ``target`` a sections fragment is assigned to.
.. _ldgen-default-scheme:
**The** ``default`` **scheme**
The ``default`` scheme
There exists a special scheme with the name ``default``. This scheme is special because catch-all placement rules are generated from
its entries. This means that, if one of its entries is ``text -> flash_text``, the placement rule
its entries. This means that, if one of its entries is ``text -> flash_text``, the placement rule
.. code-block:: none
*(.literal .literal.* .text .text.*)
will be generated for the target ``flash_text``.
will be generated for the target ``flash_text``.
These catch-all rules then effectively serve as fallback rules for those whose mappings were not specified.
These catch-all rules then effectively serve as fallback rules for those whose mappings were not specified.
.. note::
The ``default scheme`` is defined in :component:`esp32/ld/esp32_fragments.lf`. The ``noflash`` and ``rtc`` scheme fragments which are
The ``default scheme`` is defined in :component:`esp32/ld/esp32_fragments.lf`. The ``noflash`` and ``rtc`` scheme fragments which are
built-in schemes referenced in the quick start guide are also defined in this file.
.. _ldgen-mapping-fragment :
III. Mapping
""""""""""""
**Mapping**
Mapping fragments define what scheme fragment to use for mappable entities, i.e. object files, function names, variable names. There are two types of entries
for this fragment: mapping entries and condition entries.
.. note::
Mapping fragments have no explicit name property. Internally, the name is constructed from the value of the archive entry.
**Syntax**
Mapping fragments define what scheme fragment to use for mappable entities, i.e. object files, function names, variable names, archives.
.. code-block:: none
[mapping]
[mapping:name]
archive: archive # output archive file name, as built (i.e. libxxx.a)
entries:
: condition # condition entry, non-default
object:symbol (scheme) # mapping entry, Type I
object (scheme) # mapping entry, Type II
* (scheme) # mapping entry, Type III
object:symbol (scheme) # symbol granularity
object (scheme) # object granularity
* (scheme) # archive granularity
# optional separation/comments, for readability
There are three levels of placement granularity:
: default # condition entry, default
* (scheme) # mapping entry, Type III
- symbol: The object file name and symbol name are specified. The symbol name can be a function name or a variable name.
- object: Only the object file name is specified.
- archive: ``*`` is specified, which is a short-hand for all the object files under the archive.
.. _ldgen-mapping-entries :
**Mapping Entries**
There are three types of mapping entries:
``Type I``
The object file name and symbol name are specified. The symbol name can be a function name or a variable name.
``Type II``
Only the object file name is specified.
``Type III``
``*`` is specified, which is a short-hand for all the object files under the archive.
To know what a mapping entry means, let us expand a ``Type II`` entry. Originally:
To know what an entry means, let us expand a sample object-granularity placement:
.. code-block:: none
@ -380,8 +458,8 @@ Then expanding the scheme fragment from its entries definitions, we have:
.. code-block:: none
object (sections -> target,
sections -> target,
object (sections -> target,
sections -> target,
...)
Expanding the sections fragment with its entries definition:
@ -391,59 +469,38 @@ Expanding the sections fragment with its entries definition:
object (.section, # given this object file
.section, # put its sections listed here at this
... -> target, # target
.section,
.section, # same should be done for these sections
... -> target,
... -> target,
...) # and so on
.. _ldgen-type1-limitations :
Example:
**On** ``Type I`` **Mapping Entries**
.. code-block:: none
``Type I`` mapping entry is possible due to compiler flags ``-ffunction-sections`` and ``-ffdata-sections``. If the user opts to remove these flags, then
the ``Type I`` mapping will not work. Furthermore, even if the user does not opt to compile without these flags, there are still limitations
as the implementation is dependent on the emitted output sections.
[mapping:map]
archive: libfreertos.a
entries:
* (noflash)
.. _ldgen-symbol-granularity-placements :
On Symbol-Granularity Placements
""""""""""""""""""""""""""""""""
Symbol granularity placements is possible due to compiler flags ``-ffunction-sections`` and ``-ffdata-sections``. ESP-IDF compiles with these flags by default.
If the user opts to remove these flags, then the symbol-granularity placements will not work. Furthermore, even with the presence of these flags, there are still other limitations to keep in mind
due to the dependence on the compiler's emitted output sections.
For example, with ``-ffunction-sections``, separate sections are emitted for each function; with section names predictably constructed i.e. ``.text.{func_name}``
and ``.literal.{func_name}``. This is not the case for string literals within the function, as they go to pooled or generated section names.
With ``-fdata-sections``, for global scope data the compiler predictably emits either ``.data.{var_name}``, ``.rodata.{var_name}`` or ``.bss.{var_name}``; and so ``Type I`` mapping entry works for these.
With ``-fdata-sections``, for global scope data the compiler predictably emits either ``.data.{var_name}``, ``.rodata.{var_name}`` or ``.bss.{var_name}``; and so ``Type I`` mapping entry works for these.
However, this is not the case for static data declared in function scope, as the generated section name is a result of mangling the variable name with some other information.
.. _ldgen-condition-entries :
**Condition Entries**
Condition entries enable the linker script generation to be configuration-aware. Depending on whether expressions involving configuration values
are true or not, a particular set of mapping entries can be used. The evaluation uses ``eval_string`` from :idf_file:`tools/kconfig_new/kconfiglib.py` and adheres to its required syntax and limitations.
All mapping entries defined after a condition entry until the next one or the end of the mapping fragment belongs to that condition entry. During processing
conditions are tested sequentially, and the mapping entries under the first condition that evaluates to ``TRUE`` are used.
A default condition can be defined (though every mapping contains an implicit, empty one), whose mapping entries get used in the event no conditions evaluates to ``TRUE``.
**Example**
.. code-block:: none
[scheme:noflash]
entries:
text -> iram0_text
rodata -> dram0_data
[mapping:lwip]
archive: liblwip.a
entries:
: LWIP_IRAM_OPTIMIZATION = y # if CONFIG_LWIP_IRAM_OPTIMIZATION is set to 'y' in sdkconfig
ip4:ip4_route_src_hook (noflash) # map ip4.o:ip4_route_src_hook, ip4.o:ip4_route_src and
ip4:ip4_route_src (noflash) # ip4.o:ip4_route using the noflash scheme, which puts
ip4:ip4_route (noflash) # them in RAM
: default # else no special mapping rules apply
.. _ldgen-script-templates :
.. _ldgen-linker-script-template :
Linker Script Template
^^^^^^^^^^^^^^^^^^^^^^
@ -451,15 +508,13 @@ Linker Script Template
The linker script template is the skeleton in which the generated placement rules are put into. It is an otherwise ordinary linker script, with a specific marker syntax
that indicates where the generated placement rules are placed.
**Syntax**
To reference the placement rules collected under a ``target`` token, the following syntax is used:
.. code-block:: none
mapping[target]
**Example**
Example:
The example below is an excerpt from a possible linker script template. It defines an output section ``.iram0.text``, and inside is a marker referencing
the target ``iram0_text``.
@ -530,19 +585,9 @@ Then the corresponding excerpt from the generated linker script will be as follo
it too is placed wherever ``iram0_text`` is referenced by a marker. Since it is a rule generated from the default scheme, it comes first
among all other rules collected under the same target name.
.. note::
Integration with Build System
-----------------------------
The linker script template currently used is :component:`esp32/ld/esp32.project.ld.in`, specified by the ``esp32`` component; the
generated output script is put under its build directory.
The linker script generation occurs during application build, before the final output binary is linked. The tool that implements the mechanism
lives under ``$(IDF_PATH)/tools/ldgen``.
Linker Script Template
^^^^^^^^^^^^^^^^^^^^^^
Currently, the linker script template used is :component:`esp32/ld/esp32.project.ld.in`, and is used only for the app build. The generated output script is
put under the build directory of the same component. Modifying this linker script template triggers a re-link of the app binary.
Linker Fragment File
^^^^^^^^^^^^^^^^^^^^
Any component can add a fragment file to the build. In order to add a fragment file to process, set COMPONENT_ADD_LDFRAGMENTS or use the function ``ldgen_add_fragment_files`` (CMake only) as mentioned :ref:`here <ldgen-add-fragment-file>`.
Modifying any fragment file presented to the build system triggers a re-link of the app binary.

416
tools/ldgen/fragments.py
View File

@ -13,9 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
import re
import os
import re
from sdkconfig import SDKConfig
from pyparsing import OneOrMore
@ -23,49 +22,181 @@ from pyparsing import restOfLine
from pyparsing import alphanums
from pyparsing import Word
from pyparsing import alphas
from pyparsing import ParseBaseException
from pyparsing import ParseFatalException
from pyparsing import Suppress
from pyparsing import Group
from pyparsing import Literal
from pyparsing import ZeroOrMore
from pyparsing import Optional
from pyparsing import originalTextFor
from common import LdGenFailure
from pyparsing import Forward
from pyparsing import indentedBlock
from collections import namedtuple
import abc
class FragmentFileModel():
KeyGrammar = namedtuple("KeyGrammar", "grammar min max required")
class FragmentFile():
"""
Fragment file internal representation. Parses and stores instances of the fragment definitions
contained within the file.
"""
def __init__(self, fragment_file):
def __init__(self, fragment_file, sdkconfig):
try:
fragment_file = open(fragment_file, "r")
except TypeError:
pass
path = os.path.realpath(fragment_file.name)
sections = Sections.get_fragment_grammar()
scheme = Scheme.get_fragment_grammar()
mapping = Mapping.get_fragment_grammar()
indent_stack = [1]
# Each fragment file is composed of sections, scheme or mapping fragments. The grammar
# for each of those objects are defined it the respective classes
parser = OneOrMore(sections | scheme | mapping)
class parse_ctx:
fragment = None # current fragment
key = "" # current key
keys = list() # list of keys parsed
key_grammar = None # current key grammar
# Set any text beginnning with # as comment
parser.ignore("#" + restOfLine)
@staticmethod
def reset():
parse_ctx.fragment_instance = None
parse_ctx.key = ""
parse_ctx.keys = list()
parse_ctx.key_grammar = None
try:
self.fragments = parser.parseFile(fragment_file, parseAll=True)
except ParseBaseException as e:
# the actual parse error is kind of useless for normal users, so just point to the location of
# the error
raise LdGenFailure("Parse error in linker fragment %s: error at line %d col %d (char %d)" % (
fragment_file.name, e.lineno, e.column, e.loc))
def fragment_type_parse_action(toks):
parse_ctx.reset()
parse_ctx.fragment = FRAGMENT_TYPES[toks[0]]() # create instance of the fragment
return None
def expand_conditionals(toks, stmts):
try:
stmt = toks["value"]
stmts.append(stmt)
except KeyError:
try:
conditions = toks["conditional"]
for condition in conditions:
try:
_toks = condition[1]
_cond = condition[0]
if sdkconfig.evaluate_expression(_cond):
expand_conditionals(_toks, stmts)
break
except IndexError:
expand_conditionals(condition[0], stmts)
except KeyError:
for tok in toks:
expand_conditionals(tok, stmts)
def key_body_parsed(pstr, loc, toks):
stmts = list()
expand_conditionals(toks, stmts)
if parse_ctx.key_grammar.min and len(stmts) < parse_ctx.key_grammar.min:
raise ParseFatalException(pstr, loc, "fragment requires at least %d values for key '%s'" %
(parse_ctx.key_grammar.min, parse_ctx.key))
if parse_ctx.key_grammar.max and len(stmts) > parse_ctx.key_grammar.max:
raise ParseFatalException(pstr, loc, "fragment requires at most %d values for key '%s'" %
(parse_ctx.key_grammar.max, parse_ctx.key))
try:
parse_ctx.fragment.set_key_value(parse_ctx.key, stmts)
except Exception as e:
raise ParseFatalException(pstr, loc, "unable to add key '%s'; %s" % (parse_ctx.key, e.message))
return None
key = Word(alphanums + "_") + Suppress(":")
key_stmt = Forward()
condition_block = indentedBlock(key_stmt, indent_stack)
key_stmts = OneOrMore(condition_block)
key_body = Suppress(key) + key_stmts
key_body.setParseAction(key_body_parsed)
condition = originalTextFor(SDKConfig.get_expression_grammar()).setResultsName("condition")
if_condition = Group(Suppress("if") + condition + Suppress(":") + condition_block)
elif_condition = Group(Suppress("elif") + condition + Suppress(":") + condition_block)
else_condition = Group(Suppress("else") + Suppress(":") + condition_block)
conditional = (if_condition + Optional(OneOrMore(elif_condition)) + Optional(else_condition)).setResultsName("conditional")
def key_parse_action(pstr, loc, toks):
key = toks[0]
if key in parse_ctx.keys:
raise ParseFatalException(pstr, loc, "duplicate key '%s' value definition" % parse_ctx.key)
parse_ctx.key = key
parse_ctx.keys.append(key)
try:
parse_ctx.key_grammar = parse_ctx.fragment.get_key_grammars()[key]
key_grammar = parse_ctx.key_grammar.grammar
except KeyError:
raise ParseFatalException(pstr, loc, "key '%s' is not supported by fragment" % key)
except Exception as e:
raise ParseFatalException(pstr, loc, "unable to parse key '%s'; %s" % (key, e.message))
key_stmt << (conditional | Group(key_grammar).setResultsName("value"))
return None
def name_parse_action(pstr, loc, toks):
parse_ctx.fragment.name = toks[0]
key.setParseAction(key_parse_action)
ftype = Word(alphas).setParseAction(fragment_type_parse_action)
fid = Suppress(":") + Word(alphanums + "_.").setResultsName("name")
fid.setParseAction(name_parse_action)
header = Suppress("[") + ftype + fid + Suppress("]")
def fragment_parse_action(pstr, loc, toks):
key_grammars = parse_ctx.fragment.get_key_grammars()
required_keys = set([k for (k,v) in key_grammars.items() if v.required])
present_keys = required_keys.intersection(set(parse_ctx.keys))
if present_keys != required_keys:
raise ParseFatalException(pstr, loc, "required keys %s for fragment not found" %
list(required_keys - present_keys))
return parse_ctx.fragment
fragment_stmt = Forward()
fragment_block = indentedBlock(fragment_stmt, indent_stack)
fragment_if_condition = Group(Suppress("if") + condition + Suppress(":") + fragment_block)
fragment_elif_condition = Group(Suppress("elif") + condition + Suppress(":") + fragment_block)
fragment_else_condition = Group(Suppress("else") + Suppress(":") + fragment_block)
fragment_conditional = (fragment_if_condition + Optional(OneOrMore(fragment_elif_condition)) +
Optional(fragment_else_condition)).setResultsName("conditional")
fragment = (header + OneOrMore(indentedBlock(key_body, indent_stack, False))).setResultsName("value")
fragment.setParseAction(fragment_parse_action)
fragment.ignore("#" + restOfLine)
deprecated_mapping = DeprecatedMapping.get_fragment_grammar(sdkconfig, fragment_file.name).setResultsName("value")
fragment_stmt << (Group(deprecated_mapping) | Group(fragment) | Group(fragment_conditional))
def fragment_stmt_parsed(pstr, loc, toks):
stmts = list()
expand_conditionals(toks, stmts)
return stmts
parser = ZeroOrMore(fragment_stmt)
parser.setParseAction(fragment_stmt_parsed)
self.fragments = parser.parseFile(fragment_file, parseAll=True)
for fragment in self.fragments:
fragment.path = path
class Fragment:
class Fragment():
__metaclass__ = abc.ABCMeta
"""
Encapsulates a fragment as defined in the generator syntax. Sets values common to all fragment and performs processing
such as checking the validity of the fragment name and getting the entry values.
@ -74,22 +205,20 @@ class Fragment:
IDENTIFIER = Word(alphas + "_", alphanums + "_")
ENTITY = Word(alphanums + ".-_$")
def __init__(self, name, entries):
self.path = None
self.name = name
self.entries = entries
@abc.abstractmethod
def set_key_value(self, key, parse_results):
pass
@abc.abstractmethod
def get_key_grammars(self):
pass
class Sections(Fragment):
def __init__(self, name, entries):
Fragment.__init__(self, name, entries)
self._process_entries()
def _process_entries(self):
# Quietly ignore duplicate entries
self.entries = set(self.entries)
self.entries = list(self.entries)
grammars = {
"entries": KeyGrammar(Word(alphanums + "+.").setResultsName("section"), 1, None, True)
}
"""
Utility function that returns a list of sections given a sections fragment entry,
@ -110,20 +239,14 @@ class Sections(Fragment):
else:
return (sections_entry, None)
@staticmethod
def get_fragment_grammar():
name = Fragment.IDENTIFIER
header = Suppress("[") + Suppress("sections") + Suppress(":") + name.setResultsName("name") + Suppress("]")
entry = Word(alphanums + "+" + ".")
entries = Suppress("entries") + Suppress(":") + Group(OneOrMore(entry)).setResultsName("entries")
def set_key_value(self, key, parse_results):
if key == "entries":
self.entries = set()
for result in parse_results:
self.entries.add(result["section"])
sections = Group(header + entries)
sections.setParseAction(lambda t: Sections(t[0].name, t[0].entries))
sections.ignore("#" + restOfLine)
return sections
def get_key_grammars(self):
return self.__class__.grammars
class Scheme(Fragment):
@ -131,38 +254,19 @@ class Scheme(Fragment):
Encapsulates a scheme fragment, which defines what target input sections are placed under.
"""
def __init__(self, name, items):
Fragment.__init__(self, name, items)
self._process_entries()
grammars = {
"entries": KeyGrammar(Fragment.IDENTIFIER.setResultsName("sections") + Suppress("->") +
Fragment.IDENTIFIER.setResultsName("target"), 1, None, True)
}
def _process_entries(self):
processed = set()
def set_key_value(self, key, parse_results):
if key == "entries":
self.entries = set()
for result in parse_results:
self.entries.add((result["sections"], result["target"]))
# Store entries as a set of tuples. Quietly ignores duplicate entries.
for entry in self.entries:
processed.add((entry.sections, entry.target))
self.entries = processed
@staticmethod
def get_fragment_grammar():
name = Fragment.IDENTIFIER
header = Suppress("[") + Suppress("scheme") + Suppress(":") + name.setResultsName("name") + Suppress("]")
# Scheme entry in the form 'sections -> target'
sections = Fragment.IDENTIFIER
target = Fragment.IDENTIFIER
entry = Group(sections.setResultsName("sections") + Suppress("->") + target.setResultsName("target"))
entries = Suppress("entries") + Suppress(":") + Group(OneOrMore(entry)).setResultsName("entries")
scheme = Group(header + entries)
scheme.setParseAction(lambda t: Scheme(t[0].name, t[0].entries))
scheme.ignore("#" + restOfLine)
return scheme
def get_key_grammars(self):
return self.__class__.grammars
class Mapping(Fragment):
@ -170,63 +274,70 @@ class Mapping(Fragment):
Encapsulates a mapping fragment, which defines what targets the input sections of mappable entties are placed under.
"""
MAPPING_ALL_OBJECTS = "*"
def set_key_value(self, key, parse_results):
if key == "archive":
self.archive = parse_results[0]["archive"]
elif key == "entries":
self.entries = set()
for result in parse_results:
obj = None
symbol = None
scheme = None
try:
obj = result["object"]
except KeyError:
pass
try:
symbol = result["symbol"]
except KeyError:
pass
try:
scheme = result["scheme"]
except KeyError:
pass
self.entries.add((obj, symbol, scheme))
def get_key_grammars(self):
# There are three possible patterns for mapping entries:
# obj:symbol (scheme)
# obj (scheme)
# * (scheme)
obj = Fragment.ENTITY.setResultsName("object")
symbol = Suppress(":") + Fragment.IDENTIFIER.setResultsName("symbol")
scheme = Suppress("(") + Fragment.IDENTIFIER.setResultsName("scheme") + Suppress(")")
pattern1 = obj + symbol + scheme
pattern2 = obj + scheme
pattern3 = Literal(Mapping.MAPPING_ALL_OBJECTS).setResultsName("object") + scheme
entry = pattern1 | pattern2 | pattern3
grammars = {
"archive": KeyGrammar(Fragment.ENTITY.setResultsName("archive"), 1, 1, True),
"entries": KeyGrammar(entry, 1, None, True)
}
return grammars
class DeprecatedMapping():
"""
Encapsulates a mapping fragment, which defines what targets the input sections of mappable entties are placed under.
"""
# Name of the default condition entry
DEFAULT_CONDITION = "default"
MAPPING_ALL_OBJECTS = "*"
def __init__(self, archive, entries):
self.archive = archive
# Generate name from archive value by replacing all non-alphanumeric
# characters with underscore
name = Mapping.get_mapping_name_from_archive(self.archive)
Fragment.__init__(self, name, entries)
self._process_entries()
def _create_mappings_set(self, mappings):
mapping_set = set()
for mapping in mappings:
obj = mapping.object
symbol = mapping.symbol
scheme = mapping.scheme
if symbol == "":
symbol = None
# Quietly handle duplicate definitions under the same condition
mapping_set.add((obj, symbol, scheme))
return mapping_set
def _process_entries(self):
processed = []
for normal_group in self.entries[0]:
# Get the original string of the condition
condition = next(iter(normal_group.condition.asList())).strip()
mappings = self._create_mappings_set(normal_group[1])
processed.append((condition, mappings))
default_group = self.entries[1]
if len(default_group) > 1:
mappings = self._create_mappings_set(default_group[1])
else:
mappings = self._create_mappings_set(default_group[0])
processed.append(("default", mappings))
self.entries = processed
@staticmethod
def get_mapping_name_from_archive(archive):
return re.sub(r"[^0-9a-zA-Z]+", "_", archive)
@staticmethod
def get_fragment_grammar():
def get_fragment_grammar(sdkconfig, fragment_file):
# Match header [mapping]
header = Suppress("[") + Suppress("mapping") + Suppress("]")
@ -251,7 +362,7 @@ class Mapping(Fragment):
mapping_entries = Group(ZeroOrMore(mapping_entry)).setResultsName("mappings")
normal_condition = Suppress(":") + originalTextFor(SDKConfig.get_expression_grammar())
default_condition = Optional(Suppress(":") + Literal(Mapping.DEFAULT_CONDITION))
default_condition = Optional(Suppress(":") + Literal(DeprecatedMapping.DEFAULT_CONDITION))
normal_group = Group(normal_condition.setResultsName("condition") + mapping_entries)
default_group = Group(default_condition + mapping_entries).setResultsName("default_group")
@ -263,9 +374,48 @@ class Mapping(Fragment):
entries = Suppress("entries") + Suppress(":") + (normal_groups + default_group).setResultsName("entries")
mapping = Group(header + archive + entries)
mapping.setParseAction(lambda t: Mapping(t[0].archive, t[0].entries))
mapping.ignore("#" + restOfLine)
def parsed_deprecated_mapping(pstr, loc, toks):
fragment = Mapping()
fragment.archive = toks[0].archive
fragment.name = re.sub(r"[^0-9a-zA-Z]+", "_", fragment.archive)
fragment.entries = set()
condition_true = False
for entries in toks[0].entries[0]:
condition = next(iter(entries.condition.asList())).strip()
condition_val = sdkconfig.evaluate_expression(condition)
if condition_val:
for entry in entries[1]:
fragment.entries.add((entry.object, None if entry.symbol == '' else entry.symbol, entry.scheme))
condition_true = True
break
if not fragment.entries and not condition_true:
try:
entries = toks[0].entries[1][1]
except IndexError:
entries = toks[0].entries[1][0]
for entry in entries:
fragment.entries.add((entry.object, None if entry.symbol == '' else entry.symbol, entry.scheme))
if not fragment.entries:
fragment.entries.add(("*", None, "default"))
dep_warning = str(ParseFatalException(pstr, loc,
"Warning: Deprecated old-style mapping fragment parsed in file %s." % fragment_file))
print(dep_warning)
return fragment
mapping.setParseAction(parsed_deprecated_mapping)
return mapping
FRAGMENT_TYPES = {
"sections": Sections,
"scheme": Scheme,
"mapping": Mapping
}

44
tools/ldgen/generation.py
View File

@ -321,7 +321,7 @@ class GenerationModel:
return scheme_dictionary
def generate_rules(self, sdkconfig, sections_infos):
def generate_rules(self, sections_infos):
placement_rules = collections.defaultdict(list)
scheme_dictionary = self._build_scheme_dictionary()
@ -334,33 +334,18 @@ class GenerationModel:
# Generate rules based on mapping fragments
for mapping in self.mappings.values():
for (condition, entries) in mapping.entries:
condition_true = False
mapping_rules = list()
archive = mapping.archive
for (obj, symbol, scheme_name) in mapping.entries:
try:
if not (obj == Mapping.MAPPING_ALL_OBJECTS and symbol is None and
scheme_name == GenerationModel.DEFAULT_SCHEME):
self._add_mapping_rules(archive, obj, symbol, scheme_name, scheme_dictionary, mapping_rules)
except KeyError:
message = GenerationException.UNDEFINED_REFERENCE + " to scheme '" + scheme_name + "'."
raise GenerationException(message, mapping)
# Only non-default condition are evaluated agains sdkconfig model
if condition != Mapping.DEFAULT_CONDITION:
try:
condition_true = sdkconfig.evaluate_expression(condition)
except Exception as e:
raise GenerationException(e.message, mapping)
else:
condition_true = True
if condition_true:
mapping_rules = list()
archive = mapping.archive
for (obj, symbol, scheme_name) in entries:
try:
self._add_mapping_rules(archive, obj, symbol, scheme_name, scheme_dictionary, mapping_rules)
except KeyError:
message = GenerationException.UNDEFINED_REFERENCE + " to scheme '" + scheme_name + "'."
raise GenerationException(message, mapping)
all_mapping_rules[mapping.name] = mapping_rules
break # Exit on first condition that evaluates to true
all_mapping_rules[mapping.name] = mapping_rules
# Detect rule conflicts
for mapping_rules in all_mapping_rules.items():
@ -397,8 +382,7 @@ class GenerationModel:
if intersections and rule_a.maps_same_entities_as(rule_b):
rules_string = str([str(rule_a), str(rule_b)])
message = "Rules " + rules_string + " map sections " + str(list(intersections)) + " into multiple targets."
mapping = self.mappings[Mapping.get_mapping_name_from_archive(archive)]
raise GenerationException(message, mapping)
raise GenerationException(message)
def _create_extra_rules(self, rules):
# This function generates extra rules for symbol specific rules. The reason for generating extra rules is to isolate,
@ -523,7 +507,7 @@ class TemplateModel:
# Does not match marker syntax
self.members.append(line)
def fill(self, mapping_rules, sdkconfig):
def fill(self, mapping_rules):
for member in self.members:
target = None
try:

24
tools/ldgen/ldgen.py
View File

@ -19,10 +19,11 @@ import argparse
import sys
import tempfile
from fragments import FragmentFileModel
from fragments import FragmentFile
from sdkconfig import SDKConfig
from generation import GenerationModel, TemplateModel, SectionsInfo
from common import LdGenFailure
from pyparsing import ParseException, ParseFatalException
def main():
@ -52,13 +53,11 @@ def main():
argparser.add_argument(
"--config", "-c",
help="Project configuration",
type=argparse.FileType("r"))
help="Project configuration")
argparser.add_argument(
"--kconfig", "-k",
help="IDF Kconfig file",
type=argparse.FileType("r"))
help="IDF Kconfig file")
argparser.add_argument(
"--env", "-e",
@ -89,15 +88,22 @@ def main():
generation_model = GenerationModel()
sdkconfig = SDKConfig(kconfig_file, config_file, args.env)
for fragment_file in fragment_files:
fragment_file = FragmentFileModel(fragment_file)
try:
fragment_file = FragmentFile(fragment_file, sdkconfig)
except (ParseException, ParseFatalException) as e:
# ParseException is raised on incorrect grammar
# ParseFatalException is raised on correct grammar, but inconsistent contents (ex. duplicate
# keys, key unsupported by fragment, unexpected number of values, etc.)
raise LdGenFailure("failed to parse %s\n%s" % (fragment_file.name, str(e)))
generation_model.add_fragments_from_file(fragment_file)
sdkconfig = SDKConfig(kconfig_file, config_file, args.env)
mapping_rules = generation_model.generate_rules(sdkconfig, sections_infos)
mapping_rules = generation_model.generate_rules(sections_infos)
script_model = TemplateModel(input_file)
script_model.fill(mapping_rules, sdkconfig)
script_model.fill(mapping_rules)
with tempfile.TemporaryFile("w+") as output:
script_model.write(output)

24
tools/ldgen/samples/mappings.lf
View File

@ -1,62 +1,62 @@
[mapping]
[mapping:heap]
archive: libheap.a
entries:
multi_heap (noflash)
multi_heap_poisoning (noflash)
[mapping]
[mapping:soc]
archive: libsoc.a
entries:
* (noflash)
[mapping]
[mapping:freertos]
archive: libfreertos.a
entries:
* (noflash)
[mapping]
[mapping:esp32]
archive: libesp32.a
entries:
core_dump (noflash)
panic (noflash)
[mapping]
[mapping:app_trace]
archive: libapp_trace.a
entries:
* (noflash)
[mapping]
[mapping:xtensa_debug_module]
archive: libxtensa-debug-module.a
entries:
eri (noflash)
[mapping]
[mapping:phy]
archive: libphy.a
entries:
* (noflash_data)
[mapping]
[mapping:rtc]
archive: librtc.a
entries:
* (noflash)
[mapping]
[mapping:hal]
archive: libhal.a
entries:
* (noflash)
[mapping]
[mapping:gcc]
archive: libgcc.a
entries:
lib2funcs (noflash)
[mapping]
[mapping:spi_flash]
archive: libspi_flash.a
entries:
spi_flash_rom_patch (noflash)
[mapping]
[mapping:gcov]
archive: libgcov.a
entries:
* (noflash)

12
tools/ldgen/sdkconfig.py
View File

@ -15,7 +15,7 @@
#
import os
from pyparsing import Word, printables, Combine, Literal, hexnums, quotedString, Optional, nums, removeQuotes, oneOf, Group, infixNotation, opAssoc
from pyparsing import Word, alphanums, printables, Combine, Literal, hexnums, quotedString, Optional, nums, removeQuotes, oneOf, Group, infixNotation, opAssoc
import sys
try:
@ -34,11 +34,11 @@ class SDKConfig:
"""
# A configuration entry is in the form CONFIG=VALUE. Definitions of components of that grammar
IDENTIFIER = Word(printables.upper())
IDENTIFIER = Word(alphanums.upper() + "_")
HEX = Combine("0x" + Word(hexnums)).setParseAction(lambda t:int(t[0], 16))
DECIMAL = Combine(Optional(Literal("+") | Literal("-")) + Word(nums)).setParseAction(lambda t:int(t[0]))
LITERAL = Word(printables)
LITERAL = Word(printables.replace(":", ""))
QUOTED_LITERAL = quotedString.setParseAction(removeQuotes)
VALUE = HEX | DECIMAL | LITERAL | QUOTED_LITERAL
@ -53,8 +53,8 @@ class SDKConfig:
value = " ".join(value.split())
os.environ[name] = value
self.config = kconfiglib.Kconfig(kconfig_file.name)
self.config.load_config(sdkconfig_file.name)
self.config = kconfiglib.Kconfig(kconfig_file)
self.config.load_config(sdkconfig_file)
def evaluate_expression(self, expression):
result = self.config.eval_string(expression)
@ -64,7 +64,7 @@ class SDKConfig:
elif result == 2: # y
return True
else: # m
raise Exception("Unsupported config expression result.")
raise Exception("unsupported config expression result")
@staticmethod
def get_expression_grammar():

14
tools/ldgen/test/data/Kconfig
View File

@ -1,7 +1,19 @@
menu "Test config"
config PERFORMANCE_LEVEL
int "Performance level"
int
range 0 3
prompt "Performance level"
config A
bool
default "y"
config B
bool
default "n"
config C
bool
default "y"
endmenu

1419
tools/ldgen/test/test_fragments.py
View File

File diff suppressed because it is too large Load Diff

822
tools/ldgen/test/test_generation.py
View File

File diff suppressed because it is too large Load Diff