Merge branch 'refactor/ldgen' into 'master'

refactor: ldgen

Closes IDF-605 and IDFGH-6271

See merge request espressif/esp-idf!16509
This commit is contained in:
Fu Hanxi 2022-01-10 09:15:44 +00:00
commit 60c5b37bfe
32 changed files with 951 additions and 1729 deletions

View File

@ -56,11 +56,12 @@ variables:
# Docker images
BOT_DOCKER_IMAGE_TAG: ":latest"
ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env:v4.4-1-v5"
ESP_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-env:v4.4-1"
AFL_FUZZER_TEST_IMAGE: "$CI_DOCKER_REGISTRY/afl-fuzzer-test:v4.4-1-1"
CLANG_STATIC_ANALYSIS_IMAGE: "${CI_DOCKER_REGISTRY}/clang-static-analysis:v4.4-1-2"
ESP_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/esp-env-v5.0:1"
AFL_FUZZER_TEST_IMAGE: "${CI_DOCKER_REGISTRY}/afl-fuzzer-test-v5.0:1-1"
CLANG_STATIC_ANALYSIS_IMAGE: "${CI_DOCKER_REGISTRY}/clang-static-analysis-v5.0:1-1"
ESP_IDF_DOC_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/esp-idf-doc-env-v5.0:1-1"
SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:3"
LINUX_SHELL_IMAGE: "${CI_DOCKER_REGISTRY}/linux-shells:2"
# target test config file, used by assign test job
CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/.gitlab/ci/target-test.yml"

View File

@ -57,10 +57,8 @@ test_ldgen_on_host:
extends: .host_test_template
script:
- cd tools/ldgen/test
- ./test_fragments.py
- ./test_generation.py
- ./test_entity.py
- ./test_output_commands.py
- export PYTHONPATH=$PYTHONPATH:..
- python -m unittest
variables:
LC_ALL: C.UTF-8
@ -317,7 +315,7 @@ test_mkuf2:
test_autocomplete:
extends: .host_test_template
image: $CI_DOCKER_REGISTRY/linux-shells:1
image: $LINUX_SHELL_IMAGE
artifacts:
when: on_failure
paths:
@ -328,7 +326,7 @@ test_autocomplete:
test_detect_python:
extends: .host_test_template
image: $CI_DOCKER_REGISTRY/linux-shells:1
image: $LINUX_SHELL_IMAGE
script:
- cd ${IDF_PATH}
- shellcheck -s sh tools/detect_python.sh

View File

@ -151,6 +151,7 @@ disable=print-statement,
too-many-branches,
too-many-statements,
ungrouped-imports, # since we have isort in pre-commit
no-name-in-module, # since we have flake8 to check this
# Enable the message, report, category or checker with the given id(s). You can
# either give multiple identifier separated by comma (,) or put this option

View File

@ -11,26 +11,21 @@ else
elf_dir=$1
fi
if ! command -v coverage &> /dev/null; then
echo "coverage could not be found, please install it ('pip install coverage')"
exit 1
fi
SUPPORTED_TARGETS=("esp32" "esp32s2" "esp32c3" "esp32s3" )
res=0
coverage erase
python -m coverage erase
for chip in "${SUPPORTED_TARGETS[@]}"; do
{
echo "run b64 decoding tests on $chip"
coverage run -a --source=corefile ../espcoredump.py --chip="$chip" --gdb-timeout-sec 5 info_corefile -m -t b64 -c "${chip}/coredump.b64" -s "${chip}/core.elf" "${elf_dir}/${chip}.elf" &>"${chip}/output" &&
python -m coverage run -a --source=corefile ../espcoredump.py --chip="$chip" --gdb-timeout-sec 5 info_corefile -m -t b64 -c "${chip}/coredump.b64" -s "${chip}/core.elf" "${elf_dir}/${chip}.elf" &>"${chip}/output" &&
diff "${chip}/expected_output" "${chip}/output" &&
coverage run -a --source=corefile ../espcoredump.py --chip="$chip" --gdb-timeout-sec 5 info_corefile -m -t elf -c "${chip}/core.elf" "${elf_dir}/${chip}.elf" &>"${chip}/output2" &&
python -m coverage run -a --source=corefile ../espcoredump.py --chip="$chip" --gdb-timeout-sec 5 info_corefile -m -t elf -c "${chip}/core.elf" "${elf_dir}/${chip}.elf" &>"${chip}/output2" &&
diff "${chip}/expected_output" "${chip}/output2"
} || {
echo 'The test for espcoredump has failed!'
res=1
}
done
coverage run -a --source=corefile ./test_espcoredump.py
coverage report ../corefile/*.py ../espcoredump.py
python -m coverage run -a --source=corefile ./test_espcoredump.py
python -m coverage report ../corefile/*.py ../espcoredump.py
exit $res

View File

@ -1,17 +1,10 @@
// Copyright 2020 Espressif Systems (Shanghai) CO LTD
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
* SPDX-FileCopyrightText: 2020-2022 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
#include <features.h>
#include "esp_netif_lwip_internal.h"
esp_err_t esp_netif_get_ip_info(esp_netif_t *esp_netif, esp_netif_ip_info_t *ip_info)

View File

@ -3,3 +3,4 @@
#define __warning__ deprecated
#define IRAM_ATTR
#define __ESP_ATTR_H__
#include <features.h>

View File

@ -1,16 +1,8 @@
// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
* SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
#ifndef _ESP32_COMPAT_H_
#define _ESP32_COMPAT_H_
@ -20,6 +12,7 @@
#define _ESP_TASK_H_
#ifdef USE_BSD_STRING
#include <features.h>
#include <bsd/string.h>
#endif
#include <stdint.h>

View File

@ -24,16 +24,16 @@ This section presents a guide for quickly placing code/data to RAM and RTC memor
For this guide, suppose we have the following::
- components/
- my_component/
- CMakeLists.txt
- component.mk
- Kconfig
- src/
- my_src1.c
- my_src2.c
- my_src3.c
- my_linker_fragment_file.lf
component
└── my_component
└── CMakeLists.txt
├── component.mk
├── Kconfig
├── src/
│ ├── my_src1.c
│ ├── my_src2.c
│ └── my_src3.c
└── my_linker_fragment_file.lf
- a component named ``my_component`` that is archived as library ``libmy_component.a`` during build
- three source files archived under the library, ``my_src1.c``, ``my_src2.c`` and ``my_src3.c`` which are compiled as ``my_src1.o``, ``my_src2.o`` and ``my_src3.o``, respectively
@ -71,7 +71,7 @@ Placing object files
""""""""""""""""""""
Suppose the entirety of ``my_src1.o`` is performance-critical, so it is desirable to place it in RAM. On the other hand, the entirety of ``my_src2.o`` contains symbols needed coming out of deep sleep, so it needs to be put under RTC memory.
In the the linker fragment file, we can write:
In the linker fragment file, we can write:
.. code-block:: none
@ -125,6 +125,9 @@ Similarly, this places the entire component in RTC memory:
entries:
* (rtc)
.. _ldgen-conditional-placements:
Configuration-dependent placements
""""""""""""""""""""""""""""""""""
@ -224,6 +227,9 @@ The three fragment types share a common grammar:
- type: Corresponds to the fragment type, can either be ``sections``, ``scheme`` or ``mapping``.
- name: The name of the fragment, should be unique for the specified fragment type.
- key, value: Contents of the fragment; each fragment type may support different keys and different grammars for the key values.
- For :ref:`sections<ldgen-sections-fragment>` and :ref:`scheme<ldgen-scheme-fragment>`, the only supported key is ``entries``
- For :ref:`mappings<ldgen-mapping-fragment>`, both ``archive`` and ``entries`` are supported.
.. note::
@ -286,24 +292,10 @@ Condition checking behaves as you would expect an ``if...elseif/elif...else`` bl
key_2:
value_b
**Comments**
Comment in linker fragment files begin with ``#``. Like in other languages, comment are used to provide helpful descriptions and documentation and are ignored during processing.
Compatibility with ESP-IDF v3.x Linker Script Fragment Files
""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
ESP-IDF v4.0 brings some changes to the linker script fragment file grammar:
- indentation is enforced and improperly indented fragment files generate a parse exception; this was not enforced in the old version but previous documentation and examples demonstrates properly indented grammar
- move to ``if...elif...else`` structure for conditionals, with the ability to nest checks and place entire fragments themselves inside conditionals
- mapping fragments now requires a name like other fragment types
Linker script generator should be able to parse ESP-IDF v3.x linker fragment files that are indented properly (as demonstrated by the ESP-IDF v3.x version of this document). Backward compatibility with the previous mapping fragment grammar (optional name and the old grammar for conditionals) has also been retained but with a deprecation warning. Users should switch to the newer grammar discussed in this document as support for the old grammar is planned to be removed in the future.
Note that linker fragment files using the new ESP-IDF v4.0 grammar is not supported on ESP-IDF v3.x, however.
Types
"""""
@ -608,3 +600,14 @@ Then the corresponding excerpt from the generated linker script will be as follo
Rule generated from the default scheme entry ``iram -> iram0_text``. Since the default scheme specifies an ``iram -> iram0_text`` entry, it too is placed wherever ``iram0_text`` is referenced by a marker. Since it is a rule generated from the default scheme, it comes first among all other rules collected under the same target name.
The linker script template currently used is :component_file:`esp_system/ld/{IDF_TARGET_PATH_NAME}/sections.ld.in`; the generated output script ``sections.ld`` is put under its build directory.
.. _ldgen-migrate-lf-grammar :
Migrate to ESP-IDF v5.0 Linker Script Fragment Files Grammar
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
The old grammar supported in ESP-IDF v3.x would be dropped in ESP-IDF v5.0. Here are a few notes on how to migrate properly:
1. Now indentation is enforced and improperly indented fragment files would generate a runtime parse exception. This was not enforced in the old version but previous documentation and examples demonstrate properly indented grammar.
2. Migrate the old condition entry to the ``if...elif...else`` structure for conditionals. You can refer to the :ref:`earlier chapter<ldgen-conditional-placements>` for detailed grammar.
3. mapping fragments now requires a name like other fragment types.

View File

@ -1,4 +1,12 @@
Migrate Build System to ESP-IDF 5.0
===================================
Migrating from make to cmake
----------------------------
Please follow the :ref:`build system <migrating_from_make>` guide for migrating make-based projects no longer supported in ESP-IDF v5.0.
Update fragment file grammar
----------------------------
Please follow the :ref:`migrate linker script fragment files grammar<ldgen-migrate-lf-grammar>` chapter for migrating v3.x grammar to the new one.

View File

@ -24,16 +24,16 @@
假设用户有::
- components/
- my_component/
- CMakeLists.txt
- component.mk
- Kconfig
- src/
- my_src1.c
- my_src2.c
- my_src3.c
- my_linker_fragment_file.lf
component
└── my_component
└── CMakeLists.txt
├── component.mk
├── Kconfig
├── src/
│ ├── my_src1.c
│ ├── my_src2.c
│ └── my_src3.c
└── my_linker_fragment_file.lf
- 名为 ``my_component`` 的组件,在构建过程中存储为 ``libmy_component.a`` 库文件
- 库文件包含的三个源文件:``my_src1.c````my_src2.c````my_src3.c``,编译后分别为 ``my_src1.o````my_src2.o````my_src3.o``
@ -125,6 +125,8 @@
entries:
* (rtc)
.. _ldgen-conditional-placements :
根据具体配置存放
""""""""""""""""""""
@ -225,6 +227,9 @@
- 名称:片段名称,指定片段类型的片段名称应唯一。
- 键值:片段内容。每个片段类型可支持不同的键值和不同的键值语法。
- 在 ``````协议`` 中,仅支持 ``entries`` 键。
- 在 ``映射`` 中,键支持 ``archive````entries``
.. note::
多个片段的类型和名称相同时会引发异常。
@ -286,24 +291,10 @@
key_2:
value_b
**注释**
链接器片段文件中的注释以 ``#`` 开头。和在其他语言中一样,注释提供了有用的描述和资料,在处理过程中会被忽略。
与 ESP-IDF v3.x 链接器脚本片段文件兼容
""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
ESP-IDF v4.0 变更了链接器脚本片段文件使用的一些语法:
- 必须缩进,缩进不当的文件会产生解析异常;旧版本不强制缩进,但之前的文档和示例均遵循了正确的缩进语法
- 条件改用 ``if...elif...else`` 结构,可以嵌套检查,将完整片段置于条件内
- 映射片段和其他片段类型一样,需有名称
链接器脚本生成器可解析 ESP-IDF v3.x 版本中缩进正确的链接器片段文件(如 ESP-IDF v3.x 版本中的本文件所示),依然可以向后兼容此前的映射片段语法(可选名称和条件的旧语法),但是会有弃用警告。用户应换成本文档介绍的新语法,因为旧语法将在未来停用。
请注意ESP-IDF v3.x 不支持使用 ESP-IDF v4.0 新语法的链接器片段文件。
类型
"""""""
@ -608,3 +599,14 @@ ESP-IDF v4.0 变更了链接器脚本片段文件使用的一些语法:
这是根据默认协议条目 ``iram -> iram0_text`` 生成的规则。默认协议指定了 ``iram -> iram0_text`` 条目,因此生成的规则同样也放在被 ``iram0_text`` 标记的地方。由于该规则是根据默认协议生成的,因此在同一目标下收集的所有规则下排在第一位。
目前使用的链接器脚本模板是 :component_file:`esp_system/ld/{IDF_TARGET_PATH_NAME}/sections.ld.in`,生成的脚本存放在构建目录下。
.. _ldgen-migrate-lf-grammar :
将链接器脚本片段文件语法迁移至 ESP-IDF v5.0 适应版本
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ESP-IDF v5.0 中将不再支持 ESP-IDF v3.x 中链接器脚本片段文件的旧式语法。在迁移的过程中需注意以下几点:
- 必须缩进,缩进不当的文件会产生解析异常;旧版本不强制缩进,但之前的文档和示例均遵循了正确的缩进语法
- 条件改用 ``if...elif...else`` 结构,可以参照 :ref:`之前的章节<ldgen-conditional-placements>`
- 映射片段和其他片段类型一样,需有名称

View File

@ -15,7 +15,7 @@ cryptography>=2.1.4
# We do have cryptography binary on https://dl.espressif.com/pypi for ARM
# On https://pypi.org/ are no ARM binaries as standard now
pyparsing>=2.0.3,<2.4.0
pyparsing>=3.0.3 # https://github.com/pyparsing/pyparsing/issues/319 is fixed in 3.0.3
pyelftools>=0.22
idf-component-manager>=0.2.99-beta

View File

@ -35,7 +35,8 @@ IGNORE_WARNS = [
r'changes choice state',
r'crosstool_version_check\.cmake',
r'CryptographyDeprecationWarning',
r'Warning: \d+/\d+ app partitions are too small for binary'
r'Warning: \d+/\d+ app partitions are too small for binary',
r'CMake Deprecation Warning at main/lib/tinyxml2/CMakeLists\.txt:11 \(cmake_policy\)',
]
]

View File

@ -1243,7 +1243,6 @@ components/lwip/test_afl_host/dhcp_di.h
components/lwip/test_afl_host/dhcpserver_di.h
components/lwip/test_afl_host/dns_di.h
components/lwip/test_afl_host/esp_attr.h
components/lwip/test_afl_host/esp_netif_loopback_mock.c
components/lwip/test_afl_host/network_mock.c
components/lwip/test_afl_host/no_warn_host.h
components/lwip/test_afl_host/test_dhcp_client.c
@ -1350,7 +1349,6 @@ components/mdns/mdns_networking_lwip.c
components/mdns/private_include/mdns_networking.h
components/mdns/test/test_mdns.c
components/mdns/test_afl_fuzz_host/esp32_mock.c
components/mdns/test_afl_fuzz_host/esp32_mock.h
components/mdns/test_afl_fuzz_host/esp_attr.h
components/mdns/test_afl_fuzz_host/esp_netif_mock.c
components/mdns/test_afl_fuzz_host/mdns_di.h
@ -3028,8 +3026,6 @@ tools/ldgen/output_commands.py
tools/ldgen/samples/template.ld
tools/ldgen/sdkconfig.py
tools/ldgen/test/data/linker_script.ld
tools/ldgen/test/test_entity.py
tools/ldgen/test/test_output_commands.py
tools/mass_mfg/mfg_gen.py
tools/mkdfu.py
tools/mkuf2.py

View File

@ -233,13 +233,13 @@ tools/kconfig_new/test/confgen/test_confgen.py
tools/kconfig_new/test/confserver/test_confserver.py
tools/kconfig_new/test/gen_kconfig_doc/test_kconfig_out.py
tools/kconfig_new/test/gen_kconfig_doc/test_target_visibility.py
tools/ldgen/fragments.py
tools/ldgen/generation.py
tools/ldgen/ldgen.py
tools/ldgen/ldgen_common.py
tools/ldgen/linker_script.py
tools/ldgen/output_commands.py
tools/ldgen/sdkconfig.py
tools/ldgen/ldgen/entity.py
tools/ldgen/ldgen/fragments.py
tools/ldgen/ldgen/generation.py
tools/ldgen/ldgen/linker_script.py
tools/ldgen/ldgen/output_commands.py
tools/ldgen/ldgen/sdkconfig.py
tools/ldgen/test/test_entity.py
tools/ldgen/test/test_fragments.py
tools/ldgen/test/test_generation.py

View File

@ -1,8 +1,8 @@
#!/usr/bin/env bash
{ coverage debug sys \
&& coverage erase &> output \
&& coverage run -a $IDF_PATH/tools/esp_app_trace/logtrace_proc.py adc_log.trc test.elf &>> output \
{ python -m coverage debug sys \
&& python -m coverage erase &> output \
&& python -m coverage run -a $IDF_PATH/tools/esp_app_trace/logtrace_proc.py adc_log.trc test.elf &>> output \
&& diff output expected_output \
&& coverage report \
&& python -m coverage report \
; } || { echo 'The test for logtrace_proc has failed. Please examine the artifacts.' ; exit 1; }

View File

@ -1,29 +1,29 @@
#!/usr/bin/env bash
{ coverage debug sys \
&& coverage erase &> output \
&& coverage run -a $IDF_PATH/tools/esp_app_trace/sysviewtrace_proc.py -d -p -b test.elf cpu0.svdat cpu1.svdat &>> output \
{ python -m coverage debug sys \
&& python -m coverage erase &> output \
&& python -m coverage run -a $IDF_PATH/tools/esp_app_trace/sysviewtrace_proc.py -d -p -b test.elf cpu0.svdat cpu1.svdat &>> output \
&& diff output expected_output \
&& coverage report \
&& python -m coverage report \
; } || { echo 'The test for sysviewtrace_proc has failed. Please examine the artifacts.' ; exit 1; }
{ coverage debug sys \
&& coverage erase &> output.json \
&& coverage run -a $IDF_PATH/tools/esp_app_trace/sysviewtrace_proc.py -j -b test.elf cpu0.svdat cpu1.svdat &>> output.json \
{ python -m coverage debug sys \
&& python -m coverage erase &> output.json \
&& python -m coverage run -a $IDF_PATH/tools/esp_app_trace/sysviewtrace_proc.py -j -b test.elf cpu0.svdat cpu1.svdat &>> output.json \
&& diff output.json expected_output.json \
&& coverage report \
&& python -m coverage report \
; } || { echo 'The test for sysviewtrace_proc JSON functionality has failed. Please examine the artifacts.' ; exit 1; }
{ coverage debug sys \
&& coverage erase &> output \
&& coverage run -a $IDF_PATH/tools/esp_app_trace/sysviewtrace_proc.py -d -p -b sysview_tracing_heap_log.elf heap_log_mcore.svdat &>> output \
{ python -m coverage debug sys \
&& python -m coverage erase &> output \
&& python -m coverage run -a $IDF_PATH/tools/esp_app_trace/sysviewtrace_proc.py -d -p -b sysview_tracing_heap_log.elf heap_log_mcore.svdat &>> output \
&& diff output expected_output_mcore \
&& coverage report \
&& python -m coverage report \
; } || { echo 'The test for mcore sysviewtrace_proc functionality has failed. Please examine the artifacts.' ; exit 1; }
{ coverage debug sys \
&& coverage erase &> output.json \
&& coverage run -a $IDF_PATH/tools/esp_app_trace/sysviewtrace_proc.py -j -b sysview_tracing_heap_log.elf heap_log_mcore.svdat &>> output.json \
{ python -m coverage debug sys \
&& python -m coverage erase &> output.json \
&& python -m coverage run -a $IDF_PATH/tools/esp_app_trace/sysviewtrace_proc.py -j -b sysview_tracing_heap_log.elf heap_log_mcore.svdat &>> output.json \
&& diff output.json expected_output_mcore.json \
&& coverage report \
&& python -m coverage report \
; } || { echo 'The test for mcore sysviewtrace_proc JSON functionality has failed. Please examine the artifacts.' ; exit 1; }

View File

@ -1,607 +0,0 @@
#
# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
#
import abc
import os
import re
from collections import namedtuple
from enum import Enum
from entity import Entity
from pyparsing import (Combine, Forward, Group, Keyword, Literal, OneOrMore, Optional, Or, ParseFatalException,
Suppress, Word, ZeroOrMore, alphanums, alphas, delimitedList, indentedBlock, nums,
originalTextFor, restOfLine)
from sdkconfig import SDKConfig
class FragmentFile():
"""
Processes a fragment file and stores all parsed fragments. For
more information on how this class interacts with classes for the different fragment types,
see description of Fragment.
"""
def __init__(self, fragment_file, sdkconfig):
try:
fragment_file = open(fragment_file, 'r')
except TypeError:
pass
path = os.path.realpath(fragment_file.name)
indent_stack = [1]
class parse_ctx:
fragment = None # current fragment
key = '' # current key
keys = list() # list of keys parsed
key_grammar = None # current key grammar
@staticmethod
def reset():
parse_ctx.fragment_instance = None
parse_ctx.key = ''
parse_ctx.keys = list()
parse_ctx.key_grammar = None
def fragment_type_parse_action(toks):
parse_ctx.reset()
parse_ctx.fragment = FRAGMENT_TYPES[toks[0]]() # create instance of the fragment
return None
def expand_conditionals(toks, stmts):
try:
stmt = toks['value']
stmts.append(stmt)
except KeyError:
try:
conditions = toks['conditional']
for condition in conditions:
try:
_toks = condition[1]
_cond = condition[0]
if sdkconfig.evaluate_expression(_cond):
expand_conditionals(_toks, stmts)
break
except IndexError:
expand_conditionals(condition[0], stmts)
except KeyError:
for tok in toks:
expand_conditionals(tok, stmts)
def key_body_parsed(pstr, loc, toks):
stmts = list()
expand_conditionals(toks, stmts)
if parse_ctx.key_grammar.min and len(stmts) < parse_ctx.key_grammar.min:
raise ParseFatalException(pstr, loc, "fragment requires at least %d values for key '%s'" %
(parse_ctx.key_grammar.min, parse_ctx.key))
if parse_ctx.key_grammar.max and len(stmts) > parse_ctx.key_grammar.max:
raise ParseFatalException(pstr, loc, "fragment requires at most %d values for key '%s'" %
(parse_ctx.key_grammar.max, parse_ctx.key))
try:
parse_ctx.fragment.set_key_value(parse_ctx.key, stmts)
except Exception as e:
raise ParseFatalException(pstr, loc, "unable to add key '%s'; %s" % (parse_ctx.key, str(e)))
return None
key = Word(alphanums + '_') + Suppress(':')
key_stmt = Forward()
condition_block = indentedBlock(key_stmt, indent_stack)
key_stmts = OneOrMore(condition_block)
key_body = Suppress(key) + key_stmts
key_body.setParseAction(key_body_parsed)
condition = originalTextFor(SDKConfig.get_expression_grammar()).setResultsName('condition')
if_condition = Group(Suppress('if') + condition + Suppress(':') + condition_block)
elif_condition = Group(Suppress('elif') + condition + Suppress(':') + condition_block)
else_condition = Group(Suppress('else') + Suppress(':') + condition_block)
conditional = (if_condition + Optional(OneOrMore(elif_condition)) + Optional(else_condition)).setResultsName('conditional')
def key_parse_action(pstr, loc, toks):
key = toks[0]
if key in parse_ctx.keys:
raise ParseFatalException(pstr, loc, "duplicate key '%s' value definition" % parse_ctx.key)
parse_ctx.key = key
parse_ctx.keys.append(key)
try:
parse_ctx.key_grammar = parse_ctx.fragment.get_key_grammars()[key]
key_grammar = parse_ctx.key_grammar.grammar
except KeyError:
raise ParseFatalException(pstr, loc, "key '%s' is not supported by fragment" % key)
except Exception as e:
raise ParseFatalException(pstr, loc, "unable to parse key '%s'; %s" % (key, str(e)))
key_stmt << (conditional | Group(key_grammar).setResultsName('value'))
return None
def name_parse_action(pstr, loc, toks):
parse_ctx.fragment.name = toks[0]
key.setParseAction(key_parse_action)
ftype = Word(alphas).setParseAction(fragment_type_parse_action)
fid = Suppress(':') + Word(alphanums + '_.').setResultsName('name')
fid.setParseAction(name_parse_action)
header = Suppress('[') + ftype + fid + Suppress(']')
def fragment_parse_action(pstr, loc, toks):
key_grammars = parse_ctx.fragment.get_key_grammars()
required_keys = set([k for (k,v) in key_grammars.items() if v.required])
present_keys = required_keys.intersection(set(parse_ctx.keys))
if present_keys != required_keys:
raise ParseFatalException(pstr, loc, 'required keys %s for fragment not found' %
list(required_keys - present_keys))
return parse_ctx.fragment
fragment_stmt = Forward()
fragment_block = indentedBlock(fragment_stmt, indent_stack)
fragment_if_condition = Group(Suppress('if') + condition + Suppress(':') + fragment_block)
fragment_elif_condition = Group(Suppress('elif') + condition + Suppress(':') + fragment_block)
fragment_else_condition = Group(Suppress('else') + Suppress(':') + fragment_block)
fragment_conditional = (fragment_if_condition + Optional(OneOrMore(fragment_elif_condition)) +
Optional(fragment_else_condition)).setResultsName('conditional')
fragment = (header + OneOrMore(indentedBlock(key_body, indent_stack, False))).setResultsName('value')
fragment.setParseAction(fragment_parse_action)
fragment.ignore('#' + restOfLine)
deprecated_mapping = DeprecatedMapping.get_fragment_grammar(sdkconfig, fragment_file.name).setResultsName('value')
fragment_stmt << (Group(deprecated_mapping) | Group(fragment) | Group(fragment_conditional))
def fragment_stmt_parsed(pstr, loc, toks):
stmts = list()
expand_conditionals(toks, stmts)
return stmts
parser = ZeroOrMore(fragment_stmt)
parser.setParseAction(fragment_stmt_parsed)
self.fragments = parser.parseFile(fragment_file, parseAll=True)
for fragment in self.fragments:
fragment.path = path
class Fragment():
"""
Base class for a fragment that can be parsed from a fragment file. All fragments
share the common grammar:
[type:name]
key1:value1
key2:value2
...
Supporting a new fragment type means deriving a concrete class which specifies
key-value pairs that the fragment supports and what to do with the parsed key-value pairs.
The new fragment must also be appended to FRAGMENT_TYPES, specifying the
keyword for the type and the derived class.
The key of the key-value pair is a simple keyword string. Other parameters
that describe the key-value pair is specified in Fragment.KeyValue:
1. grammar - pyparsing grammar to parse the value of key-value pair
2. min - the minimum number of value in the key entry, None means no minimum
3. max - the maximum number of value in the key entry, None means no maximum
4. required - if the key-value pair is required in the fragment
Setting min=max=1 means that the key has a single value.
FragmentFile provides conditional expression evaluation, enforcing
the parameters for Fragment.Keyvalue.
"""
__metaclass__ = abc.ABCMeta
KeyValue = namedtuple('KeyValue', 'grammar min max required')
IDENTIFIER = Word(alphas + '_', alphanums + '_')
ENTITY = Word(alphanums + '.-_$+')
@abc.abstractmethod
def set_key_value(self, key, parse_results):
pass
@abc.abstractmethod
def get_key_grammars(self):
pass
class Sections(Fragment):
"""
Fragment which contains list of input sections.
[sections:<name>]
entries:
.section1
.section2
...
"""
# Unless quoted, symbol names start with a letter, underscore, or point
# and may include any letters, underscores, digits, points, and hyphens.
GNU_LD_SYMBOLS = Word(alphas + '_.', alphanums + '._-')
entries_grammar = Combine(GNU_LD_SYMBOLS + Optional('+'))
grammars = {
'entries': Fragment.KeyValue(entries_grammar.setResultsName('section'), 1, None, True)
}
"""
Utility function that returns a list of sections given a sections fragment entry,
with the '+' notation and symbol concatenation handled automatically.
"""
@staticmethod
def get_section_data_from_entry(sections_entry, symbol=None):
if not symbol:
sections = list()
sections.append(sections_entry.replace('+', ''))
sections.append(sections_entry.replace('+', '.*'))
return sections
else:
if sections_entry.endswith('+'):
section = sections_entry.replace('+', '.*')
expansion = section.replace('.*', '.' + symbol)
return (section, expansion)
else:
return (sections_entry, None)
def set_key_value(self, key, parse_results):
if key == 'entries':
self.entries = set()
for result in parse_results:
self.entries.add(result['section'])
def get_key_grammars(self):
return self.__class__.grammars
class Scheme(Fragment):
"""
Fragment which defines where the input sections defined in a Sections fragment
is going to end up, the target. The targets are markers in a linker script template
(see LinkerScript in linker_script.py).
[scheme:<name>]
entries:
sections1 -> target1
...
"""
grammars = {
'entries': Fragment.KeyValue(Fragment.IDENTIFIER.setResultsName('sections') + Suppress('->') +
Fragment.IDENTIFIER.setResultsName('target'), 1, None, True)
}
def set_key_value(self, key, parse_results):
if key == 'entries':
self.entries = set()
for result in parse_results:
self.entries.add((result['sections'], result['target']))
def get_key_grammars(self):
return self.__class__.grammars
class Mapping(Fragment):
"""
Fragment which attaches a scheme to entities (see Entity in entity.py), specifying where the input
sections of the entity will end up.
[mapping:<name>]
archive: lib1.a
entries:
obj1:symbol1 (scheme1); section1 -> target1 KEEP SURROUND(sym1) ...
obj2 (scheme2)
...
Ultimately, an `entity (scheme)` entry generates an
input section description (see https://sourceware.org/binutils/docs/ld/Input-Section.html)
in the output linker script. It is possible to attach 'flags' to the
`entity (scheme)` to generate different output commands or to
emit additional keywords in the generated input section description. The
input section description, as well as other output commands, is defined in
output_commands.py.
"""
class Flag():
PRE_POST = (Optional(Suppress(',') + Suppress('pre').setParseAction(lambda: True).setResultsName('pre')) +
Optional(Suppress(',') + Suppress('post').setParseAction(lambda: True).setResultsName('post')))
class Surround(Flag):
def __init__(self, symbol):
self.symbol = symbol
self.pre = True
self.post = True
@staticmethod
def get_grammar():
# SURROUND(symbol)
#
# '__symbol_start', '__symbol_end' is generated before and after
# the corresponding input section description, respectively.
grammar = (Keyword('SURROUND').suppress() +
Suppress('(') +
Fragment.IDENTIFIER.setResultsName('symbol') +
Suppress(')'))
grammar.setParseAction(lambda tok: Mapping.Surround(tok.symbol))
return grammar
def __eq__(self, other):
return (isinstance(other, Mapping.Surround) and
self.symbol == other.symbol)
class Align(Flag):
def __init__(self, alignment, pre=True, post=False):
self.alignment = alignment
self.pre = pre
self.post = post
@staticmethod
def get_grammar():
# ALIGN(alignment, [, pre, post]).
#
# Generates alignment command before and/or after the corresponding
# input section description, depending whether pre, post or
# both are specified.
grammar = (Keyword('ALIGN').suppress() +
Suppress('(') +
Word(nums).setResultsName('alignment') +
Mapping.Flag.PRE_POST +
Suppress(')'))
def on_parse(tok):
alignment = int(tok.alignment)
if tok.pre == '' and tok.post == '':
res = Mapping.Align(alignment)
elif tok.pre != '' and tok.post == '':
res = Mapping.Align(alignment, tok.pre)
elif tok.pre == '' and tok.post != '':
res = Mapping.Align(alignment, False, tok.post)
else:
res = Mapping.Align(alignment, tok.pre, tok.post)
return res
grammar.setParseAction(on_parse)
return grammar
def __eq__(self, other):
return (isinstance(other, Mapping.Align) and
self.alignment == other.alignment and
self.pre == other.pre and
self.post == other.post)
class Keep(Flag):
def __init__(self):
pass
@staticmethod
def get_grammar():
# KEEP()
#
# Surrounds input section description with KEEP command.
grammar = Keyword('KEEP()').setParseAction(Mapping.Keep)
return grammar
def __eq__(self, other):
return isinstance(other, Mapping.Keep)
class Sort(Flag):
class Type(Enum):
NAME = 0
ALIGNMENT = 1
INIT_PRIORITY = 2
def __init__(self, first, second=None):
self.first = first
self.second = second
@staticmethod
def get_grammar():
# SORT([sort_by_first, sort_by_second])
#
# where sort_by_first, sort_by_second = {name, alignment, init_priority}
#
# Emits SORT_BY_NAME, SORT_BY_ALIGNMENT or SORT_BY_INIT_PRIORITY
# depending on arguments. Nested sort follows linker script rules.
keywords = Keyword('name') | Keyword('alignment') | Keyword('init_priority')
grammar = (Keyword('SORT').suppress() + Suppress('(') +
keywords.setResultsName('first') +
Optional(Suppress(',') + keywords.setResultsName('second')) + Suppress(')'))
grammar.setParseAction(lambda tok: Mapping.Sort(tok.first, tok.second if tok.second != '' else None))
return grammar
def __eq__(self, other):
return (isinstance(other, Mapping.Sort) and
self.first == other.first and
self.second == other.second)
def __init__(self):
Fragment.__init__(self)
self.entries = set()
# k = (obj, symbol, scheme)
# v = list((section, target), Mapping.Flag))
self.flags = dict()
self.deprecated = False
def set_key_value(self, key, parse_results):
if key == 'archive':
self.archive = parse_results[0]['archive']
elif key == 'entries':
for result in parse_results:
obj = None
symbol = None
scheme = None
obj = result['object']
try:
symbol = result['symbol']
except KeyError:
pass
scheme = result['scheme']
mapping = (obj, symbol, scheme)
self.entries.add(mapping)
try:
parsed_flags = result['sections_target_flags']
except KeyError:
parsed_flags = []
if parsed_flags:
entry_flags = []
for pf in parsed_flags:
entry_flags.append((pf.sections, pf.target, list(pf.flags)))
try:
existing_flags = self.flags[mapping]
except KeyError:
existing_flags = list()
self.flags[mapping] = existing_flags
existing_flags.extend(entry_flags)
def get_key_grammars(self):
# There are three possible patterns for mapping entries:
# obj:symbol (scheme)
# obj (scheme)
# * (scheme)
# Flags can be specified for section->target in the scheme specified, ex:
# obj (scheme); section->target SURROUND(symbol), section2->target2 ALIGN(4)
obj = Fragment.ENTITY.setResultsName('object')
symbol = Suppress(':') + Fragment.IDENTIFIER.setResultsName('symbol')
scheme = Suppress('(') + Fragment.IDENTIFIER.setResultsName('scheme') + Suppress(')')
# The flags are specified for section->target in the scheme specified
sections_target = Scheme.grammars['entries'].grammar
flag = Or([f.get_grammar() for f in [Mapping.Keep, Mapping.Align, Mapping.Surround, Mapping.Sort]])
section_target_flags = Group(sections_target + Group(OneOrMore(flag)).setResultsName('flags'))
pattern1 = obj + symbol
pattern2 = obj
pattern3 = Literal(Entity.ALL).setResultsName('object')
entry = ((pattern1 | pattern2 | pattern3) + scheme +
Optional(Suppress(';') + delimitedList(section_target_flags).setResultsName('sections_target_flags')))
grammars = {
'archive': Fragment.KeyValue(Or([Fragment.ENTITY, Word(Entity.ALL)]).setResultsName('archive'), 1, 1, True),
'entries': Fragment.KeyValue(entry, 0, None, True)
}
return grammars
class DeprecatedMapping():
"""
Mapping fragment with old grammar in versions older than ESP-IDF v4.0. Does not conform to
requirements of the Fragment class and thus is limited when it comes to conditional expression
evaluation.
"""
# Name of the default condition entry
DEFAULT_CONDITION = 'default'
@staticmethod
def get_fragment_grammar(sdkconfig, fragment_file):
# Match header [mapping]
header = Suppress('[') + Suppress('mapping') + Suppress(']')
# There are three possible patterns for mapping entries:
# obj:symbol (scheme)
# obj (scheme)
# * (scheme)
obj = Fragment.ENTITY.setResultsName('object')
symbol = Suppress(':') + Fragment.IDENTIFIER.setResultsName('symbol')
scheme = Suppress('(') + Fragment.IDENTIFIER.setResultsName('scheme') + Suppress(')')
pattern1 = Group(obj + symbol + scheme)
pattern2 = Group(obj + scheme)
pattern3 = Group(Literal(Entity.ALL).setResultsName('object') + scheme)
mapping_entry = pattern1 | pattern2 | pattern3
# To simplify parsing, classify groups of condition-mapping entry into two types: normal and default
# A normal grouping is one with a non-default condition. The default grouping is one which contains the
# default condition
mapping_entries = Group(ZeroOrMore(mapping_entry)).setResultsName('mappings')
normal_condition = Suppress(':') + originalTextFor(SDKConfig.get_expression_grammar())
default_condition = Optional(Suppress(':') + Literal(DeprecatedMapping.DEFAULT_CONDITION))
normal_group = Group(normal_condition.setResultsName('condition') + mapping_entries)
default_group = Group(default_condition + mapping_entries).setResultsName('default_group')
normal_groups = Group(ZeroOrMore(normal_group)).setResultsName('normal_groups')
# Any mapping fragment definition can have zero or more normal group and only one default group as a last entry.
archive = Suppress('archive') + Suppress(':') + Fragment.ENTITY.setResultsName('archive')
entries = Suppress('entries') + Suppress(':') + (normal_groups + default_group).setResultsName('entries')
mapping = Group(header + archive + entries)
mapping.ignore('#' + restOfLine)
def parsed_deprecated_mapping(pstr, loc, toks):
fragment = Mapping()
fragment.archive = toks[0].archive
fragment.name = re.sub(r'[^0-9a-zA-Z]+', '_', fragment.archive)
fragment.deprecated = True
fragment.entries = set()
condition_true = False
for entries in toks[0].entries[0]:
condition = next(iter(entries.condition.asList())).strip()
condition_val = sdkconfig.evaluate_expression(condition)
if condition_val:
for entry in entries[1]:
fragment.entries.add((entry.object, None if entry.symbol == '' else entry.symbol, entry.scheme))
condition_true = True
break
if not fragment.entries and not condition_true:
try:
entries = toks[0].entries[1][1]
except IndexError:
entries = toks[0].entries[1][0]
for entry in entries:
fragment.entries.add((entry.object, None if entry.symbol == '' else entry.symbol, entry.scheme))
if not fragment.entries:
fragment.entries.add(('*', None, 'default'))
dep_warning = str(ParseFatalException(pstr, loc,
'Warning: Deprecated old-style mapping fragment parsed in file %s.' % fragment_file))
print(dep_warning)
return fragment
mapping.setParseAction(parsed_deprecated_mapping)
return mapping
FRAGMENT_TYPES = {
'sections': Sections,
'scheme': Scheme,
'mapping': Mapping
}

View File

@ -1,6 +1,6 @@
#!/usr/bin/env python
#
# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
#
@ -13,13 +13,13 @@ import sys
import tempfile
from io import StringIO
from entity import EntityDB
from fragments import FragmentFile
from generation import Generation
from ldgen_common import LdGenFailure
from linker_script import LinkerScript
from ldgen.entity import EntityDB
from ldgen.fragments import parse_fragment_file
from ldgen.generation import Generation
from ldgen.ldgen_common import LdGenFailure
from ldgen.linker_script import LinkerScript
from ldgen.sdkconfig import SDKConfig
from pyparsing import ParseException, ParseFatalException
from sdkconfig import SDKConfig
try:
import confgen
@ -148,12 +148,12 @@ def main():
for fragment_file in fragment_files:
try:
fragment_file = FragmentFile(fragment_file, sdkconfig)
fragment_file = parse_fragment_file(fragment_file, sdkconfig)
except (ParseException, ParseFatalException) as e:
# ParseException is raised on incorrect grammar
# ParseFatalException is raised on correct grammar, but inconsistent contents (ex. duplicate
# keys, key unsupported by fragment, unexpected number of values, etc.)
raise LdGenFailure('failed to parse %s\n%s' % (fragment_file.name, str(e)))
raise LdGenFailure('failed to parse %s\n%s' % (fragment_file, str(e)))
generation_model.add_fragments_from_file(fragment_file)
mapping_rules = generation_model.generate(sections_infos)

View File

@ -1,17 +1,6 @@
#
# Copyright 2021 Espressif Systems (Shanghai) CO LTD
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
#
import collections
@ -21,11 +10,11 @@ from enum import Enum
from functools import total_ordering
from pyparsing import (Group, Literal, OneOrMore, ParseException, SkipTo, Suppress, White, Word, ZeroOrMore, alphas,
nums, restOfLine)
nums, rest_of_line)
@total_ordering
class Entity():
class Entity:
"""
An entity refers to a library, object, symbol whose input
sections can be placed or excluded from placement.
@ -60,7 +49,7 @@ class Entity():
else:
raise ValueError("Invalid arguments '(%s, %s, %s)'" % (archive, obj, symbol))
self.archive = archive
self.archive = archive
self.obj = obj
self.symbol = symbol
@ -93,7 +82,7 @@ class Entity():
return '%s:%s %s' % self.__repr__()
def __repr__(self):
return (self.archive, self.obj, self.symbol)
return self.archive, self.obj, self.symbol
def __getitem__(self, spec):
res = None
@ -108,7 +97,7 @@ class Entity():
return res
class EntityDB():
class EntityDB:
"""
Collection of entities extracted from libraries known in the build.
Allows retrieving a list of archives, a list of object files in an archive
@ -127,11 +116,10 @@ class EntityDB():
archive_path = (Literal('In archive').suppress() +
White().suppress() +
# trim the colon and line ending characters from archive_path
restOfLine.setResultsName('archive_path').setParseAction(lambda s, loc, toks: s.rstrip(':\n\r ')))
rest_of_line.set_results_name('archive_path').set_parse_action(
lambda s, loc, toks: s.rstrip(':\n\r ')))
parser = archive_path
results = None
try:
results = parser.parseString(first_line, parseAll=True)
except ParseException as p:
@ -142,7 +130,7 @@ class EntityDB():
def _get_infos_from_file(self, info):
# {object}: file format elf32-xtensa-le
object_line = SkipTo(':').setResultsName('object') + Suppress(restOfLine)
object_line = SkipTo(':').set_results_name('object') + Suppress(rest_of_line)
# Sections:
# Idx Name ...
@ -151,13 +139,14 @@ class EntityDB():
# 00 {section} 0000000 ...
# CONTENTS, ALLOC, ....
section_entry = Suppress(Word(nums)) + SkipTo(' ') + Suppress(restOfLine) + \
Suppress(ZeroOrMore(Word(alphas) + Literal(',')) + Word(alphas))
section_entry = (Suppress(Word(nums)) + SkipTo(' ') + Suppress(rest_of_line)
+ Suppress(ZeroOrMore(Word(alphas) + Literal(',')) + Word(alphas)))
content = Group(object_line + section_start + section_header + Group(OneOrMore(section_entry)).setResultsName('sections'))
parser = Group(ZeroOrMore(content)).setResultsName('contents')
results = None
content = Group(object_line
+ section_start
+ section_header
+ Group(OneOrMore(section_entry)).set_results_name('sections'))
parser = Group(ZeroOrMore(content)).set_results_name('contents')
try:
results = parser.parseString(info.content, parseAll=True)
@ -192,7 +181,9 @@ class EntityDB():
def _match_obj(self, archive, obj):
objs = self.get_objects(archive)
match_objs = fnmatch.filter(objs, obj + '.o') + fnmatch.filter(objs, obj + '.*.obj') + fnmatch.filter(objs, obj + '.obj')
match_objs = (fnmatch.filter(objs, obj + '.o')
+ fnmatch.filter(objs, obj + '.*.obj')
+ fnmatch.filter(objs, obj + '.obj'))
if len(match_objs) > 1:
raise ValueError("Multiple matches for object: '%s: %s': %s" % (archive, obj, str(match_objs)))

View File

@ -0,0 +1,473 @@
#
# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
#
from typing import Any, Dict, List, Optional, Set, Tuple, Union
from pyparsing import (Combine, Forward, Group, IndentedBlock, Keyword, LineEnd, Literal, OneOrMore, Opt,
ParseFatalException, SkipTo, Suppress, Word, ZeroOrMore, alphanums, alphas, delimited_list,
nums, rest_of_line)
class Empty:
"""
Return `Empty()` when the sdkconfig does not meet the conditional statements.
"""
def __repr__(self):
return '<EMPTY>'
def __bool__(self):
return False
class Fragment:
"""
Base class for a fragment that can be parsed from a fragment file.
"""
IDENTIFIER = Word(alphas + '_', alphanums + '_')
ENTITY = Word(alphanums + '.-_$+')
def __init__(self, name: str, entries: Set[Union[str, Tuple[str]]]):
self.name = name
self.entries = entries
def __repr__(self):
return str(self.__dict__)
class Sections(Fragment):
"""
Fragment which contains list of input sections.
[sections:<name>]
entries:
.section1
.section2
...
"""
# Unless quoted, symbol names start with a letter, underscore, or point
# and may include any letters, underscores, digits, points, and hyphens.
ENTRY = Combine(Word(alphas + '_.', alphanums + '._-') + Opt('+')) + LineEnd().suppress()
@staticmethod
def parse_entry(toks):
# section
return toks[0]
@staticmethod
def parse(s, loc, toks):
this = toks[0]
name = this[0]
entries = {entry for entry in this[1] if entry}
if not entries:
raise ParseFatalException(s, loc, 'Sections entries shouldn\'t be empty')
return Sections(name, entries)
@staticmethod
def get_section_data_from_entry(sections_entry, symbol=None):
"""
Returns a list of sections given a sections fragment entry,
with the '+' notation and symbol concatenation handled automatically.
"""
if not symbol:
sections = list()
sections.append(sections_entry.replace('+', ''))
sections.append(sections_entry.replace('+', '.*'))
return sections
else:
if sections_entry.endswith('+'):
section = sections_entry.replace('+', '.*')
expansion = section.replace('.*', '.' + symbol)
return section, expansion
else:
return sections_entry, None
class Scheme(Fragment):
"""
Fragment which defines where the input sections defined in a Sections fragment
is going to end up, the target. The targets are markers in a linker script template
(see LinkerScript in linker_script.py).
[scheme:<name>]
entries:
sections1 -> target1
...
"""
ENTRY = Fragment.IDENTIFIER + Suppress('->') + Fragment.IDENTIFIER + LineEnd().suppress()
@staticmethod
def parse_entry(toks):
# section, target
return toks[0], toks[1]
@staticmethod
def parse(s, loc, toks):
this = toks[0]
name = this[0]
entries = {entry for entry in this[1] if entry}
if not entries:
raise ParseFatalException(s, loc, 'Scheme entries shouldn\'t be empty')
return Scheme(name, entries)
class EntryFlag:
def __repr__(self):
return str(self.__dict__)
class Surround(EntryFlag):
"""
SURROUND(symbol)
'__symbol_start', '__symbol_end' is generated before and after
the corresponding input section description, respectively.
"""
SURROUND = (Keyword('SURROUND').suppress()
+ Suppress('(')
+ Fragment.IDENTIFIER
+ Suppress(')'))
def __init__(self, symbol: str):
self.symbol = symbol
self.pre = True
self.post = True
def __eq__(self, other):
if isinstance(other, Surround):
if self.symbol == other.symbol and self.pre == other.pre and self.post == other.post:
return True
return False
@staticmethod
def parse(toks):
return Surround(toks[0])
class Align(EntryFlag):
"""
ALIGN(alignment, [, pre, post]).
Generates alignment command before and/or after the corresponding
input section description, depending on whether pre, post or
both are specified.
"""
PRE = Opt(Suppress(',') + Suppress('pre')).set_results_name('pre')
POST = Opt(Suppress(',') + Suppress('post')).set_results_name('post')
ALIGN = (Keyword('ALIGN').suppress()
+ Suppress('(')
+ Word(nums)
+ PRE
+ POST
+ Suppress(')'))
def __init__(self, alignment, pre=True, post=False):
self.alignment = alignment
self.pre = pre
self.post = post
def __eq__(self, other):
if isinstance(other, Align):
if self.alignment == other.alignment and self.pre == other.pre and self.post == other.post:
return True
return False
@staticmethod
def parse(toks):
alignment = int(toks[0])
if toks.post == '':
return Align(alignment)
if toks.pre == '' and toks.post != '':
return Align(alignment, False, True)
return Align(alignment, True, True)
class Keep(EntryFlag):
"""
KEEP()
Surrounds input section description with KEEP command.
"""
KEEP = Keyword('KEEP()')
def __eq__(self, other):
if isinstance(other, Keep):
return True
return False
@staticmethod
def parse():
return Keep()
class Sort(EntryFlag):
"""
SORT([sort_by_first, sort_by_second])
where sort_by_first, sort_by_second = {name, alignment, init_priority}
Emits SORT_BY_NAME, SORT_BY_ALIGNMENT or SORT_BY_INIT_PRIORITY
depending on arguments. Nested sort follows linker script rules.
"""
_keywords = Keyword('name') | Keyword('alignment') | Keyword('init_priority')
SORT = (Keyword('SORT').suppress()
+ Suppress('(')
+ _keywords.set_results_name('first')
+ Opt(Suppress(',') + _keywords.set_results_name('second'))
+ Suppress(')'))
def __init__(self, first: str, second: Optional[str] = None):
self.first = first
self.second = second
def __eq__(self, other):
if isinstance(other, Sort):
if self.first == other.first and self.second == other.second:
return True
return False
@staticmethod
def parse(toks):
return Sort(toks.first, toks.second or None)
class Flag:
_section_target = Fragment.IDENTIFIER + Suppress('->') + Fragment.IDENTIFIER
_flag = (Surround.SURROUND.set_parse_action(Surround.parse)
| Align.ALIGN.set_parse_action(Align.parse)
| Keep.KEEP.set_parse_action(Keep.parse)
| Sort.SORT.set_parse_action(Sort.parse))
FLAG = _section_target + OneOrMore(_flag)
def __init__(self, section: str, target: str, flags: List[EntryFlag]):
self.section = section
self.target = target
self.flags = flags
def __eq__(self, other):
if isinstance(other, Flag):
if self.section == other.section and self.target == other.target and len(self.flags) == len(other.flags):
for i, j in zip(self.flags, other.flags):
if i != j:
break
else:
return True
return False
@staticmethod
def parse(toks):
return Flag(toks[0], toks[1], toks[2:])
def __repr__(self):
return str(self.__dict__)
class Mapping(Fragment):
"""
Fragment which attaches a scheme to entities (see Entity in entity.py), specifying where the input
sections of the entity will end up.
[mapping:<name>]
archive: lib1.a
entries:
obj1:symbol1 (scheme1); section1 -> target1 KEEP SURROUND(sym1) ...
obj2 (scheme2)
...
Ultimately, an `entity (scheme)` entry generates an
input section description (see https://sourceware.org/binutils/docs/ld/Input-Section.html)
in the output linker script. It is possible to attach 'flags' to the
`entity (scheme)` to generate different output commands or to
emit additional keywords in the generated input section description. The
input section description, as well as other output commands, is defined in
output_commands.py.
"""
_any = Literal('*')
_obj = Word(alphas + '_', alphanums + '-_').set_results_name('object')
_sym = Fragment.IDENTIFIER.set_results_name('symbol')
# There are three possible patterns for mapping entries:
# obj:symbol (scheme)
# obj (scheme)
# * (scheme)
_entry = (((_obj + Opt(Suppress(':') + _sym)) | _any.set_results_name('object'))
+ Suppress('(')
+ Fragment.IDENTIFIER.set_results_name('section')
+ Suppress(')'))
ENTRY = _entry + LineEnd().suppress()
ARCHIVE = (Word(alphanums + '.-_$+') | Literal('*')) + LineEnd().suppress()
# Flags can be specified for section->target in the scheme specified, ex:
# obj (scheme);
# section->target SURROUND(symbol),
# section2->target2 ALIGN(4)
ENTRY_WITH_FLAG = (_entry + Suppress(';')
+ delimited_list(Flag.FLAG.set_parse_action(Flag.parse)))
def __init__(self, archive: str, flags: Dict[Any, Flag], *args, **kwargs):
super().__init__(*args, **kwargs)
self.archive = archive
self.flags = flags
@staticmethod
def parse_archive(s, loc, toks):
this = toks[0][0]
if len(this) != 1:
raise ParseFatalException(s, loc, 'Could only specify one archive file in one mapping fragment')
return this[0]
@staticmethod
def parse_entry(toks):
return toks.object, toks.symbol or None, toks.section
@staticmethod
def parse_entry_with_flag(toks):
entry = toks.object, toks.symbol or None, toks.section
return {
entry: [tok for tok in toks if isinstance(tok, Flag)]
}
@staticmethod
def parse_entries(toks):
return toks[0]
@staticmethod
def parse(toks):
this = toks[0]
name = this[0]
archive = this[1]
entries_or_dict_with_flags = this[2]
entries = set()
flags = dict()
for item in entries_or_dict_with_flags:
if isinstance(item, Empty):
continue
elif isinstance(item, dict): # entry with flags
for k, v in item.items():
entries.add(k)
if k in flags:
flags[k].extend(v)
else:
flags[k] = v
else:
entries.add(item)
return Mapping(archive=archive, name=name, entries=entries, flags=flags)
class FragmentFile:
"""
Processes a fragment file and stores all parsed fragments. For
more information on how this class interacts with classes for the different fragment types,
see description of Fragment.
"""
def __init__(self, fragments: List[Fragment]):
self.path = None # assign later, couldn't pass extra argument while parsing
self.fragments: List[Fragment] = fragments
def __repr__(self):
return str(self.__dict__)
def parse_fragment_file(path, sdkconfig):
def parse_conditional(toks):
this = toks[0]
for stmt in this:
if stmt[0] in ['if', 'elif']: # if/elif
if sdkconfig.evaluate_expression(stmt.condition):
return stmt[-1]
else: # else
return stmt[-1]
return Empty()
def get_conditional_stmt(_stmt):
condition = SkipTo(':').set_results_name('condition') + Suppress(':')
_suite = IndentedBlock(_stmt)
if_decl = Literal('if') + condition
elif_decl = Literal('elif') + condition
else_decl = Literal('else:')
if_ = Group(if_decl + _suite)
elif_ = Group(elif_decl + _suite)
else_ = Group(else_decl + _suite)
return Group(if_ + Opt(OneOrMore(elif_)) + Opt(else_)).set_parse_action(parse_conditional)
def get_suite(_stmt):
__stmt = Forward()
__conditional = get_conditional_stmt(__stmt)
__stmt <<= (comment
| _stmt
| __conditional)
return IndentedBlock(__stmt)
def parse(toks):
return FragmentFile([tok for tok in toks if not isinstance(tok, Empty)])
# comment
comment = (Literal('#') + rest_of_line).set_parse_action(lambda s, l, t: Empty())
# section
section_entry = Sections.ENTRY.set_parse_action(Sections.parse_entry)
section_entries_suite = get_suite(section_entry)
section_header = Suppress('[sections:') + Fragment.IDENTIFIER + Suppress(']') + LineEnd().suppress()
section = Group(section_header
+ Suppress('entries:')
+ section_entries_suite).set_parse_action(Sections.parse)
# scheme
scheme_entry = Scheme.ENTRY.set_parse_action(Scheme.parse_entry)
scheme_entries_suite = get_suite(scheme_entry)
scheme_header = Suppress('[scheme:') + Fragment.IDENTIFIER + Suppress(']') + LineEnd().suppress()
scheme = Group(scheme_header
+ Suppress('entries:')
+ scheme_entries_suite).set_parse_action(Scheme.parse)
# mapping
mapping_archive = Mapping.ARCHIVE
mapping_archive_suite = get_suite(mapping_archive)
mapping_entry = Mapping.ENTRY.set_parse_action(Mapping.parse_entry)
mapping_entry_with_flag = Mapping.ENTRY_WITH_FLAG.set_parse_action(Mapping.parse_entry_with_flag)
mapping_entries_suite = get_suite(mapping_entry | mapping_entry_with_flag)
mapping_header = Suppress('[mapping:') + Fragment.IDENTIFIER + Suppress(']')
mapping = Group(mapping_header
+ Group(Suppress('archive:')
+ mapping_archive_suite).set_parse_action(Mapping.parse_archive)
+ Group(Suppress('entries:')
+ mapping_entries_suite).set_parse_action(Mapping.parse_entries)
).set_parse_action(Mapping.parse)
# highest level
fragment = (section
| scheme
| mapping
| get_conditional_stmt(section | scheme | mapping))
parser = ZeroOrMore(fragment).ignore(comment).set_parse_action(parse)
fragment_file = parser.parse_file(path, parse_all=True)[0]
fragment_file.path = path
return fragment_file

View File

@ -1,5 +1,5 @@
#
# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
#
@ -8,13 +8,13 @@ import fnmatch
import itertools
from collections import namedtuple
from entity import Entity
from fragments import Mapping, Scheme, Sections
from ldgen_common import LdGenFailure
from output_commands import AlignAtAddress, InputSectionDesc, SymbolAtAddress
from .entity import Entity
from .fragments import Keep, Scheme, Sections, Sort, Surround
from .ldgen_common import LdGenFailure
from .output_commands import AlignAtAddress, InputSectionDesc, SymbolAtAddress
class Placement():
class Placement:
"""
A Placement is an assignment of an entity's input sections to a target
in the output linker script - a precursor to the input section description.
@ -97,7 +97,7 @@ class Placement():
self.subplacements.add(subplacement)
class EntityNode():
class EntityNode:
"""
Node in entity tree. An EntityNode
is created from an Entity (see entity.py).
@ -134,12 +134,12 @@ class EntityNode():
def add_child(self, entity):
child_specificity = self.entity.specificity.value + 1
assert(child_specificity <= Entity.Specificity.SYMBOL.value)
assert (child_specificity <= Entity.Specificity.SYMBOL.value)
name = entity[Entity.Specificity(child_specificity)]
assert(name and name != Entity.ALL)
assert (name and name != Entity.ALL)
child = [c for c in self.children if c.name == name]
assert(len(child) <= 1)
assert (len(child) <= 1)
if not child:
child = self.child_t(self, name)
@ -174,7 +174,7 @@ class EntityNode():
for sections in self.get_output_sections():
placement = self.placements[sections]
if placement.is_significant():
assert(placement.node == self)
assert (placement.node == self)
keep = False
sort = None
@ -183,16 +183,16 @@ class EntityNode():
placement_flags = placement.flags if placement.flags is not None else []
for flag in placement_flags:
if isinstance(flag, Mapping.Keep):
if isinstance(flag, Keep):
keep = True
elif isinstance(flag, Mapping.Sort):
elif isinstance(flag, Sort):
sort = (flag.first, flag.second)
else: # SURROUND or ALIGN
surround_type.append(flag)
for flag in surround_type:
if flag.pre:
if isinstance(flag, Mapping.Surround):
if isinstance(flag, Surround):
commands[placement.target].append(SymbolAtAddress('_%s_start' % flag.symbol))
else: # ALIGN
commands[placement.target].append(AlignAtAddress(flag.alignment))
@ -202,11 +202,12 @@ class EntityNode():
placement_sections = frozenset(placement.sections)
command_sections = sections if sections == placement_sections else placement_sections
command = InputSectionDesc(placement.node.entity, command_sections, [e.node.entity for e in placement.exclusions], keep, sort)
command = InputSectionDesc(placement.node.entity, command_sections,
[e.node.entity for e in placement.exclusions], keep, sort)
commands[placement.target].append(command)
# Generate commands for intermediate, non-explicit exclusion placements here, so that they can be enclosed by
# flags that affect the parent placement.
# Generate commands for intermediate, non-explicit exclusion placements here,
# so that they can be enclosed by flags that affect the parent placement.
for subplacement in placement.subplacements:
if not subplacement.flags and not subplacement.explicit:
command = InputSectionDesc(subplacement.node.entity, subplacement.sections,
@ -215,7 +216,7 @@ class EntityNode():
for flag in surround_type:
if flag.post:
if isinstance(flag, Mapping.Surround):
if isinstance(flag, Surround):
commands[placement.target].append(SymbolAtAddress('_%s_end' % flag.symbol))
else: # ALIGN
commands[placement.target].append(AlignAtAddress(flag.alignment))
@ -248,6 +249,7 @@ class SymbolNode(EntityNode):
Entities at depth=3. Represents entities with archive, object
and symbol specified.
"""
def __init__(self, parent, name):
EntityNode.__init__(self, parent, name)
self.entity = Entity(self.parent.parent.name, self.parent.name)
@ -270,6 +272,7 @@ class ObjectNode(EntityNode):
An intermediate placement on this node is created, if one does not exist,
and is the one excluded from its basis placement.
"""
def __init__(self, parent, name):
EntityNode.__init__(self, parent, name)
self.child_t = SymbolNode
@ -334,6 +337,7 @@ class ArchiveNode(EntityNode):
"""
Entities at depth=1. Represents entities with archive specified.
"""
def __init__(self, parent, name):
EntityNode.__init__(self, parent, name)
self.child_t = ObjectNode
@ -345,6 +349,7 @@ class RootNode(EntityNode):
Single entity at depth=0. Represents entities with no specific members
specified.
"""
def __init__(self):
EntityNode.__init__(self, None, Entity.ALL)
self.child_t = ArchiveNode
@ -382,7 +387,7 @@ class Generation:
for (sections_name, target_name) in scheme.entries:
# Get the sections under the bucket 'target_name'. If this bucket does not exist
# is is created automatically
# is created automatically
sections_in_bucket = sections_bucket[target_name]
try:
@ -433,9 +438,9 @@ class Generation:
entity = Entity(archive, obj, symbol)
# Check the entity exists
if (self.check_mappings and
entity.specificity.value > Entity.Specificity.ARCHIVE.value and
mapping.name not in self.check_mapping_exceptions):
if (self.check_mappings
and entity.specificity.value > Entity.Specificity.ARCHIVE.value
and mapping.name not in self.check_mapping_exceptions):
if not entities.check_exists(entity):
message = "'%s' not found" % str(entity)
raise GenerationException(message, mapping)
@ -444,11 +449,11 @@ class Generation:
flags = mapping.flags[(obj, symbol, scheme_name)]
# Check if all section->target defined in the current
# scheme.
for (s, t, f) in flags:
if (t not in scheme_dictionary[scheme_name].keys() or
s not in [_s.name for _s in scheme_dictionary[scheme_name][t]]):
message = "%s->%s not defined in scheme '%s'" % (s, t, scheme_name)
for flag in flags:
if (flag.target not in scheme_dictionary[scheme_name].keys()
or flag.section not in
[_s.name for _s in scheme_dictionary[scheme_name][flag.target]]):
message = "%s->%s not defined in scheme '%s'" % (flag.section, flag.target, scheme_name)
raise GenerationException(message, mapping)
else:
flags = None
@ -460,9 +465,9 @@ class Generation:
_flags = []
if flags:
for (s, t, f) in flags:
if (s, t) == (section.name, target):
_flags.extend(f)
for flag in flags:
if (flag.section, flag.target) == (section.name, target):
_flags.extend(flag.flags)
sections_str = get_section_strs(section)
@ -477,18 +482,18 @@ class Generation:
entity_mappings[key] = Generation.EntityMapping(entity, sections_str, target, _flags)
else:
# Check for conflicts.
if (target != existing.target):
if target != existing.target:
raise GenerationException('Sections mapped to multiple targets.', mapping)
# Combine flags here if applicable, to simplify
# insertion logic.
if (_flags or existing.flags):
if ((_flags and not existing.flags) or (not _flags and existing.flags)):
if _flags or existing.flags:
if (_flags and not existing.flags) or (not _flags and existing.flags):
_flags.extend(existing.flags)
entity_mappings[key] = Generation.EntityMapping(entity,
sections_str,
target, _flags)
elif (_flags == existing.flags):
elif _flags == existing.flags:
pass
else:
raise GenerationException('Conflicting flags specified.', mapping)
@ -517,26 +522,22 @@ class Generation:
def add_fragments_from_file(self, fragment_file):
for fragment in fragment_file.fragments:
dict_to_append_to = None
if isinstance(fragment, Mapping) and fragment.deprecated and fragment.name in self.mappings.keys():
self.mappings[fragment.name].entries |= fragment.entries
if isinstance(fragment, Scheme):
dict_to_append_to = self.schemes
elif isinstance(fragment, Sections):
dict_to_append_to = self.placements
else:
if isinstance(fragment, Scheme):
dict_to_append_to = self.schemes
elif isinstance(fragment, Sections):
dict_to_append_to = self.placements
else:
dict_to_append_to = self.mappings
dict_to_append_to = self.mappings
# Raise exception when the fragment of the same type is already in the stored fragments
if fragment.name in dict_to_append_to.keys():
stored = dict_to_append_to[fragment.name].path
new = fragment.path
message = "Duplicate definition of fragment '%s' found in %s and %s." % (fragment.name, stored, new)
raise GenerationException(message)
# Raise exception when the fragment of the same type is already in the stored fragments
if fragment.name in dict_to_append_to:
stored = dict_to_append_to[fragment.name].path
new = fragment.path
message = "Duplicate definition of fragment '%s' found in %s and %s." % (
fragment.name, stored, new)
raise GenerationException(message)
dict_to_append_to[fragment.name] = fragment
dict_to_append_to[fragment.name] = fragment
class GenerationException(LdGenFailure):

View File

@ -0,0 +1,9 @@
#
# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
#
class LdGenFailure(RuntimeError):
"""
Parent class for any ldgen runtime failure which is due to input data
"""

View File

@ -1,15 +1,16 @@
#
# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
#
import collections
import os
from fragments import Fragment
from generation import GenerationException
from pyparsing import ParseException, Suppress, White
from .fragments import Fragment
from .generation import GenerationException
class LinkerScript:
"""
@ -32,24 +33,21 @@ class LinkerScript:
lines = template_file.readlines()
target = Fragment.IDENTIFIER
reference = Suppress('mapping') + Suppress('[') + target.setResultsName('target') + Suppress(']')
pattern = White(' \t').setResultsName('indent') + reference
reference = Suppress('mapping') + Suppress('[') + target + Suppress(']')
pattern = White(' \t') + reference
# Find the markers in the template file line by line. If line does not match marker grammar,
# set it as a literal to be copied as is to the output file.
for line in lines:
try:
parsed = pattern.parseString(line)
indent = parsed.indent
target = parsed.target
marker = LinkerScript.Marker(target, indent, [])
self.members.append(marker)
parsed = pattern.parse_string(line)
except ParseException:
# Does not match marker syntax
self.members.append(line)
else:
indent, target = parsed
marker = LinkerScript.Marker(target, indent, [])
self.members.append(marker)
def fill(self, mapping_rules):
for member in self.members:

View File

@ -1,26 +1,15 @@
#
# Copyright 2021 Espressif Systems (Shanghai) CO LTD
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
#
from entity import Entity
from .entity import Entity
# Contains classes for output section commands referred to in
# https://www.acrc.bris.ac.uk/acrc/RedHat/rhel-ld-en-4/sections.html#OUTPUT-SECTION-DESCRIPTION.
class AlignAtAddress():
class AlignAtAddress:
"""
Outputs assignment of builtin function ALIGN to current
position:
@ -42,7 +31,7 @@ class AlignAtAddress():
self.alignment == other.alignment)
class SymbolAtAddress():
class SymbolAtAddress:
"""
Outputs assignment of builtin function ABSOLUTE to a symbol
for current position:
@ -65,7 +54,7 @@ class SymbolAtAddress():
self.symbol == other.symbol)
class InputSectionDesc():
class InputSectionDesc:
"""
Outputs an input section description as described in
https://www.acrc.bris.ac.uk/acrc/RedHat/rhel-ld-en-4/sections.html#INPUT-SECTION.
@ -76,7 +65,7 @@ class InputSectionDesc():
"""
def __init__(self, entity, sections, exclusions=None, keep=False, sort=None):
assert(entity.specificity != Entity.Specificity.SYMBOL)
assert (entity.specificity != Entity.Specificity.SYMBOL)
self.entity = entity
self.sections = set(sections)
@ -84,8 +73,8 @@ class InputSectionDesc():
self.exclusions = set()
if exclusions:
assert(not [e for e in exclusions if e.specificity == Entity.Specificity.SYMBOL or
e.specificity == Entity.Specificity.NONE])
assert (not [e for e in exclusions if e.specificity == Entity.Specificity.SYMBOL or
e.specificity == Entity.Specificity.NONE])
self.exclusions = set(exclusions)
else:
self.exclusions = set()

View File

@ -0,0 +1,26 @@
#
# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
#
import kconfiglib
class SDKConfig:
"""
Evaluates conditional expressions based on the build's sdkconfig and Kconfig files.
"""
def __init__(self, kconfig_file, sdkconfig_file):
self.config = kconfiglib.Kconfig(kconfig_file)
self.config.load_config(sdkconfig_file)
self.config.warn = False # eval_string may contain un-declared symbol
def evaluate_expression(self, expression):
result = self.config.eval_string(expression)
if result == 0: # n
return False
elif result == 2: # y
return True
else: # m
raise Exception('unsupported config expression result')

View File

@ -1,23 +0,0 @@
#
# Copyright 2021 Espressif Systems (Shanghai) CO LTD
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
class LdGenFailure(RuntimeError):
"""
Parent class for any ldgen runtime failure which is due to input data
"""
def __init__(self, message):
super(LdGenFailure, self).__init__(message)

View File

@ -1,73 +0,0 @@
#
# Copyright 2021 Espressif Systems (Shanghai) CO LTD
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import kconfiglib
from pyparsing import (Combine, Group, Literal, Optional, Word, alphanums, hexnums, infixNotation, nums, oneOf,
opAssoc, printables, quotedString, removeQuotes)
class SDKConfig:
"""
Evaluates conditional expressions based on the build's sdkconfig and Kconfig files.
This also defines the grammar of conditional expressions.
"""
# A configuration entry is in the form CONFIG=VALUE. Definitions of components of that grammar
IDENTIFIER = Word(alphanums.upper() + '_')
HEX = Combine('0x' + Word(hexnums)).setParseAction(lambda t:int(t[0], 16))
DECIMAL = Combine(Optional(Literal('+') | Literal('-')) + Word(nums)).setParseAction(lambda t:int(t[0]))
LITERAL = Word(printables.replace(':', ''))
QUOTED_LITERAL = quotedString.setParseAction(removeQuotes)
VALUE = HEX | DECIMAL | LITERAL | QUOTED_LITERAL
# Operators supported by the expression evaluation
OPERATOR = oneOf(['=', '!=', '>', '<', '<=', '>='])
def __init__(self, kconfig_file, sdkconfig_file):
self.config = kconfiglib.Kconfig(kconfig_file)
self.config.load_config(sdkconfig_file)
def evaluate_expression(self, expression):
result = self.config.eval_string(expression)
if result == 0: # n
return False
elif result == 2: # y
return True
else: # m
raise Exception('unsupported config expression result')
@staticmethod
def get_expression_grammar():
identifier = SDKConfig.IDENTIFIER.setResultsName('identifier')
operator = SDKConfig.OPERATOR.setResultsName('operator')
value = SDKConfig.VALUE.setResultsName('value')
test_binary = identifier + operator + value
test_single = identifier
test = test_binary | test_single
condition = Group(Optional('(').suppress() + test + Optional(')').suppress())
grammar = infixNotation(condition, [
('!', 1, opAssoc.RIGHT),
('&&', 2, opAssoc.LEFT),
('||', 2, opAssoc.LEFT)])
return grammar

View File

@ -1,29 +1,19 @@
#!/usr/bin/env python
# coding=utf-8
#
# Copyright 2018-2020 Espressif Systems (Shanghai) CO LTD
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# SPDX-FileCopyrightText: 2018-2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
#
import os
import sys
import unittest
try:
from entity import Entity, EntityDB
from ldgen.entity import Entity, EntityDB
except ImportError:
sys.path.append('../')
from entity import Entity, EntityDB
sys.path.append(os.path.dirname(os.path.dirname(__file__)))
from ldgen.entity import Entity, EntityDB
class EntityTest(unittest.TestCase):

File diff suppressed because it is too large Load Diff

View File

@ -1,6 +1,6 @@
#!/usr/bin/env python
#
# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
#
@ -10,20 +10,23 @@ import os
import sys
import tempfile
import unittest
try:
from generation import Generation, GenerationException
except ImportError:
sys.path.append('../')
from generation import Generation, GenerationException
from io import StringIO
from entity import Entity, EntityDB
from fragments import FragmentFile
from linker_script import LinkerScript
from output_commands import AlignAtAddress, InputSectionDesc, SymbolAtAddress
from sdkconfig import SDKConfig
try:
from ldgen.entity import Entity, EntityDB
from ldgen.fragments import parse_fragment_file
from ldgen.generation import Generation, GenerationException
from ldgen.linker_script import LinkerScript
from ldgen.output_commands import AlignAtAddress, InputSectionDesc, SymbolAtAddress
from ldgen.sdkconfig import SDKConfig
except ImportError:
sys.path.append(os.path.dirname(os.path.dirname(__file__)))
from ldgen.entity import Entity, EntityDB
from ldgen.fragments import parse_fragment_file
from ldgen.generation import Generation, GenerationException
from ldgen.linker_script import LinkerScript
from ldgen.output_commands import AlignAtAddress, InputSectionDesc, SymbolAtAddress
from ldgen.sdkconfig import SDKConfig
ROOT = Entity('*')
@ -58,9 +61,8 @@ class GenerationTest(unittest.TestCase):
self.sdkconfig = SDKConfig('data/Kconfig', 'data/sdkconfig')
with open('data/base.lf') as fragment_file_obj:
fragment_file = FragmentFile(fragment_file_obj, self.sdkconfig)
self.generation.add_fragments_from_file(fragment_file)
fragment_file = parse_fragment_file('data/base.lf', self.sdkconfig)
self.generation.add_fragments_from_file(fragment_file)
self.entities = EntityDB()
@ -78,7 +80,7 @@ class GenerationTest(unittest.TestCase):
def add_fragments(self, text):
fragment_file = self.create_fragment_file(text)
fragment_file = FragmentFile(fragment_file, self.sdkconfig)
fragment_file = parse_fragment_file(fragment_file, self.sdkconfig)
self.generation.add_fragments_from_file(fragment_file)
def write(self, expected, actual):
@ -1062,43 +1064,6 @@ entries:
with self.assertRaises(GenerationException):
self.generation.generate(self.entities)
def test_disambiguated_obj(self):
# Test command generation for disambiguated entry. Should produce similar
# results to test_nondefault_mapping_symbol.
mapping = u"""
[mapping:test]
archive: libfreertos.a
entries:
port.c:xPortGetTickRateHz (noflash) #1
"""
port = Entity('libfreertos.a', 'port.c')
self.add_fragments(mapping)
actual = self.generation.generate(self.entities)
expected = self.generate_default_rules()
flash_text = expected['flash_text']
iram0_text = expected['iram0_text']
# Generate exclusion in flash_text A
flash_text[0].exclusions.add(port)
# Generate intermediate command B
# List all relevant sections except the symbol
# being mapped
port_sections = self.entities.get_sections('libfreertos.a', 'port.c')
filtered_sections = fnmatch.filter(port_sections, '.literal.*')
filtered_sections.extend(fnmatch.filter(port_sections, '.text.*'))
filtered_sections = [s for s in filtered_sections if not s.endswith('xPortGetTickRateHz')]
filtered_sections.append('.text')
flash_text.append(InputSectionDesc(port, set(filtered_sections), []))
# Input section commands in iram_text for #1 C
iram0_text.append(InputSectionDesc(port, set(['.text.xPortGetTickRateHz', '.literal.xPortGetTickRateHz']), []))
self.compare_rules(expected, actual)
def test_root_mapping_fragment_conflict(self):
# Test that root mapping fragments are also checked for
# conflicts.
@ -1258,84 +1223,6 @@ entries:
self.compare_rules(expected, actual)
def test_conditional_on_scheme_legacy_mapping_00(self):
# Test use of conditional scheme on legacy mapping fragment grammar.
mapping = u"""
[mapping]
archive: lib.a
entries:
* (cond_noflash)
"""
self._test_conditional_on_scheme(0, mapping)
def test_conditional_on_scheme_legacy_mapping_01(self):
# Test use of conditional scheme on legacy mapping fragment grammar.
mapping = u"""
[mapping]
archive: lib.a
entries:
* (cond_noflash)
"""
self._test_conditional_on_scheme(0, mapping)
def test_conditional_entries_legacy_mapping_fragment(self):
# Test conditional entries on legacy mapping fragment grammar.
mapping = u"""
[mapping:default]
archive: *
entries:
* (default)
[mapping]
archive: lib.a
entries:
: PERFORMANCE_LEVEL = 0
: PERFORMANCE_LEVEL = 1
obj1 (noflash)
: PERFORMANCE_LEVEL = 2
obj1 (noflash)
obj2 (noflash)
: PERFORMANCE_LEVEL = 3
obj1 (noflash)
obj2 (noflash)
obj3 (noflash)
"""
self.test_conditional_mapping(mapping)
def test_multiple_fragment_same_lib_conditional_legacy(self):
# Test conditional entries on legacy mapping fragment grammar
# across multiple fragments.
mapping = u"""
[mapping:default]
archive: *
entries:
* (default)
[mapping]
archive: lib.a
entries:
: PERFORMANCE_LEVEL = 0
: PERFORMANCE_LEVEL = 1
obj1 (noflash)
: PERFORMANCE_LEVEL = 2
obj1 (noflash)
: PERFORMANCE_LEVEL = 3
obj1 (noflash)
[mapping]
archive: lib.a
entries:
: PERFORMANCE_LEVEL = 1
obj1 (noflash) # ignore duplicate definition
: PERFORMANCE_LEVEL = 2
obj2 (noflash)
: PERFORMANCE_LEVEL = 3
obj2 (noflash)
obj3 (noflash)
"""
self.test_conditional_mapping(mapping)
def test_multiple_fragment_same_lib_conditional(self):
# Test conditional entries on new mapping fragment grammar.
# across multiple fragments.

View File

@ -1,30 +1,21 @@
#!/usr/bin/env python
#
# Copyright 2021 Espressif Systems (Shanghai) CO LTD
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
#
import os
import sys
import unittest
try:
from output_commands import AlignAtAddress, InputSectionDesc, SymbolAtAddress
from ldgen.entity import Entity
from ldgen.output_commands import AlignAtAddress, InputSectionDesc, SymbolAtAddress
except ImportError:
sys.path.append('../')
from output_commands import InputSectionDesc, SymbolAtAddress, AlignAtAddress
sys.path.append(os.path.dirname(os.path.dirname(__file__)))
from ldgen.entity import Entity
from ldgen.output_commands import AlignAtAddress, InputSectionDesc, SymbolAtAddress
from entity import Entity
SECTIONS = ['.text', '.text.*', '.literal', '.literal.*']