From 172854a85097aa09170bb2682f7aeb5b54545d98 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Fri, 24 Dec 2021 09:41:47 +0800 Subject: [PATCH] refactor: rewrite ldgen fragment file parser closes https://github.com/espressif/esp-idf/issues/7940 --- .gitlab/ci/host-test.yml | 5 +- .pylintrc | 1 + requirements.txt | 2 +- tools/ci/mypy_ignore_list.txt | 1 - tools/ldgen/ldgen.py | 8 +- tools/ldgen/ldgen/entity.py | 31 +- tools/ldgen/ldgen/fragments.py | 873 ++++++++++------------- tools/ldgen/ldgen/generation.py | 72 +- tools/ldgen/ldgen/ldgen_common.py | 6 +- tools/ldgen/ldgen/linker_script.py | 19 +- tools/ldgen/ldgen/output_commands.py | 8 +- tools/ldgen/ldgen/sdkconfig.py | 40 +- tools/ldgen/test/test_entity.py | 5 +- tools/ldgen/test/test_fragments.py | 798 +++++---------------- tools/ldgen/test/test_generation.py | 151 +--- tools/ldgen/test/test_output_commands.py | 5 +- 16 files changed, 654 insertions(+), 1371 deletions(-) diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index 2eca5b01e8..4e9a5604cb 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -57,10 +57,7 @@ test_ldgen_on_host: extends: .host_test_template script: - cd tools/ldgen/test - - ./test_fragments.py - - ./test_generation.py - - ./test_entity.py - - ./test_output_commands.py + - python -m unittest variables: LC_ALL: C.UTF-8 diff --git a/.pylintrc b/.pylintrc index 019a561e44..8b57c7b550 100644 --- a/.pylintrc +++ b/.pylintrc @@ -151,6 +151,7 @@ disable=print-statement, too-many-branches, too-many-statements, ungrouped-imports, # since we have isort in pre-commit + no-name-in-module, # since we have flake8 to check this # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff --git a/requirements.txt b/requirements.txt index 45e43877b7..5141f4481b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ cryptography>=2.1.4 # We do have cryptography binary on https://dl.espressif.com/pypi for ARM # On https://pypi.org/ are no ARM binaries as standard now -pyparsing>=2.0.3,<2.4.0 +pyparsing>=3.0.3 # https://github.com/pyparsing/pyparsing/issues/319 is fixed in 3.0.3 pyelftools>=0.22 idf-component-manager>=0.2.99-beta diff --git a/tools/ci/mypy_ignore_list.txt b/tools/ci/mypy_ignore_list.txt index 264682cbf4..bb3e0404c0 100644 --- a/tools/ci/mypy_ignore_list.txt +++ b/tools/ci/mypy_ignore_list.txt @@ -237,7 +237,6 @@ tools/ldgen/ldgen.py tools/ldgen/ldgen/entity.py tools/ldgen/ldgen/fragments.py tools/ldgen/ldgen/generation.py -tools/ldgen/ldgen/ldgen_common.py tools/ldgen/ldgen/linker_script.py tools/ldgen/ldgen/output_commands.py tools/ldgen/ldgen/sdkconfig.py diff --git a/tools/ldgen/ldgen.py b/tools/ldgen/ldgen.py index bc9bc6c3e8..767b34bbc7 100755 --- a/tools/ldgen/ldgen.py +++ b/tools/ldgen/ldgen.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # @@ -14,7 +14,7 @@ import tempfile from io import StringIO from ldgen.entity import EntityDB -from ldgen.fragments import FragmentFile +from ldgen.fragments import parse_fragment_file from ldgen.generation import Generation from ldgen.ldgen_common import LdGenFailure from ldgen.linker_script import LinkerScript @@ -148,12 +148,12 @@ def main(): for fragment_file in fragment_files: try: - fragment_file = FragmentFile(fragment_file, sdkconfig) + fragment_file = parse_fragment_file(fragment_file, sdkconfig) except (ParseException, ParseFatalException) as e: # ParseException is raised on incorrect grammar # ParseFatalException is raised on correct grammar, but inconsistent contents (ex. duplicate # keys, key unsupported by fragment, unexpected number of values, etc.) - raise LdGenFailure('failed to parse %s\n%s' % (fragment_file.name, str(e))) + raise LdGenFailure('failed to parse %s\n%s' % (fragment_file, str(e))) generation_model.add_fragments_from_file(fragment_file) mapping_rules = generation_model.generate(sections_infos) diff --git a/tools/ldgen/ldgen/entity.py b/tools/ldgen/ldgen/entity.py index 3e020e2d7d..35021a51b5 100644 --- a/tools/ldgen/ldgen/entity.py +++ b/tools/ldgen/ldgen/entity.py @@ -1,5 +1,5 @@ # -# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # @@ -10,7 +10,7 @@ from enum import Enum from functools import total_ordering from pyparsing import (Group, Literal, OneOrMore, ParseException, SkipTo, Suppress, White, Word, ZeroOrMore, alphas, - nums, restOfLine) + nums, rest_of_line) @total_ordering @@ -82,7 +82,7 @@ class Entity: return '%s:%s %s' % self.__repr__() def __repr__(self): - return (self.archive, self.obj, self.symbol) + return self.archive, self.obj, self.symbol def __getitem__(self, spec): res = None @@ -97,7 +97,7 @@ class Entity: return res -class EntityDB(): +class EntityDB: """ Collection of entities extracted from libraries known in the build. Allows retrieving a list of archives, a list of object files in an archive @@ -116,12 +116,10 @@ class EntityDB(): archive_path = (Literal('In archive').suppress() + White().suppress() + # trim the colon and line ending characters from archive_path - restOfLine.setResultsName('archive_path').setParseAction( + rest_of_line.set_results_name('archive_path').set_parse_action( lambda s, loc, toks: s.rstrip(':\n\r '))) parser = archive_path - results = None - try: results = parser.parseString(first_line, parseAll=True) except ParseException as p: @@ -132,7 +130,7 @@ class EntityDB(): def _get_infos_from_file(self, info): # {object}: file format elf32-xtensa-le - object_line = SkipTo(':').setResultsName('object') + Suppress(restOfLine) + object_line = SkipTo(':').set_results_name('object') + Suppress(rest_of_line) # Sections: # Idx Name ... @@ -141,12 +139,14 @@ class EntityDB(): # 00 {section} 0000000 ... # CONTENTS, ALLOC, .... - section_entry = Suppress(Word(nums)) + SkipTo(' ') + Suppress(restOfLine) + \ - Suppress(ZeroOrMore(Word(alphas) + Literal(',')) + Word(alphas)) + section_entry = (Suppress(Word(nums)) + SkipTo(' ') + Suppress(rest_of_line) + + Suppress(ZeroOrMore(Word(alphas) + Literal(',')) + Word(alphas))) - content = Group( - object_line + section_start + section_header + Group(OneOrMore(section_entry)).setResultsName('sections')) - parser = Group(ZeroOrMore(content)).setResultsName('contents') + content = Group(object_line + + section_start + + section_header + + Group(OneOrMore(section_entry)).set_results_name('sections')) + parser = Group(ZeroOrMore(content)).set_results_name('contents') try: results = parser.parseString(info.content, parseAll=True) @@ -181,8 +181,9 @@ class EntityDB(): def _match_obj(self, archive, obj): objs = self.get_objects(archive) - match_objs = fnmatch.filter(objs, obj + '.o') + fnmatch.filter(objs, obj + '.*.obj') + fnmatch.filter(objs, - obj + '.obj') + match_objs = (fnmatch.filter(objs, obj + '.o') + + fnmatch.filter(objs, obj + '.*.obj') + + fnmatch.filter(objs, obj + '.obj')) if len(match_objs) > 1: raise ValueError("Multiple matches for object: '%s: %s': %s" % (archive, obj, str(match_objs))) diff --git a/tools/ldgen/ldgen/fragments.py b/tools/ldgen/ldgen/fragments.py index a1b7133b43..ea17f65b06 100644 --- a/tools/ldgen/ldgen/fragments.py +++ b/tools/ldgen/ldgen/fragments.py @@ -1,220 +1,40 @@ # -# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # -import abc -import os -import re -from collections import namedtuple -from enum import Enum -from pyparsing import (Combine, Forward, Group, Keyword, Literal, OneOrMore, Optional, Or, ParseFatalException, - Suppress, Word, ZeroOrMore, alphanums, alphas, delimitedList, indentedBlock, nums, - originalTextFor, restOfLine) +from typing import Any, Dict, List, Optional, Set, Tuple, Union -from .entity import Entity -from .sdkconfig import SDKConfig +from pyparsing import (Combine, Forward, Group, IndentedBlock, Keyword, LineEnd, Literal, OneOrMore, Opt, + ParseFatalException, SkipTo, Suppress, Word, ZeroOrMore, alphanums, alphas, delimited_list, + nums, rest_of_line) -class FragmentFile(): +class Empty: """ - Processes a fragment file and stores all parsed fragments. For - more information on how this class interacts with classes for the different fragment types, - see description of Fragment. + Return `Empty()` when the sdkconfig does not meet the conditional statements. """ - def __init__(self, fragment_file, sdkconfig): - try: - fragment_file = open(fragment_file, 'r') - except TypeError: - pass + def __repr__(self): + return '' - path = os.path.realpath(fragment_file.name) - - indent_stack = [1] - - class parse_ctx: - fragment = None # current fragment - key = '' # current key - keys = list() # list of keys parsed - key_grammar = None # current key grammar - - @staticmethod - def reset(): - parse_ctx.fragment_instance = None - parse_ctx.key = '' - parse_ctx.keys = list() - parse_ctx.key_grammar = None - - def fragment_type_parse_action(toks): - parse_ctx.reset() - parse_ctx.fragment = FRAGMENT_TYPES[toks[0]]() # create instance of the fragment - return None - - def expand_conditionals(toks, stmts): - try: - stmt = toks['value'] - stmts.append(stmt) - except KeyError: - try: - conditions = toks['conditional'] - for condition in conditions: - try: - _toks = condition[1] - _cond = condition[0] - if sdkconfig.evaluate_expression(_cond): - expand_conditionals(_toks, stmts) - break - except IndexError: - expand_conditionals(condition[0], stmts) - except KeyError: - for tok in toks: - expand_conditionals(tok, stmts) - - def key_parse_action(pstr, loc, toks): - key = toks[0] - - if key in parse_ctx.keys: - raise ParseFatalException(pstr, loc, "duplicate key '%s' value definition" % parse_ctx.key) - - parse_ctx.key = key - parse_ctx.keys.append(key) - - try: - parse_ctx.key_grammar = parse_ctx.fragment.get_key_grammars()[key] - key_grammar = parse_ctx.key_grammar.grammar - except KeyError: - raise ParseFatalException(pstr, loc, "key '%s' is not supported by fragment" % key) - except Exception as e: - raise ParseFatalException(pstr, loc, "unable to parse key '%s'; %s" % (key, str(e))) - - key_stmt << (conditional | Group(key_grammar).setResultsName('value')) - - return None - - def key_body_parsed(pstr, loc, toks): - stmts = list() - expand_conditionals(toks, stmts) - - if parse_ctx.key_grammar.min and len(stmts) < parse_ctx.key_grammar.min: - raise ParseFatalException(pstr, loc, "fragment requires at least %d values for key '%s'" % - (parse_ctx.key_grammar.min, parse_ctx.key)) - - if parse_ctx.key_grammar.max and len(stmts) > parse_ctx.key_grammar.max: - raise ParseFatalException(pstr, loc, "fragment requires at most %d values for key '%s'" % - (parse_ctx.key_grammar.max, parse_ctx.key)) - - try: - parse_ctx.fragment.set_key_value(parse_ctx.key, stmts) - except Exception as e: - raise ParseFatalException(pstr, loc, "unable to add key '%s'; %s" % (parse_ctx.key, str(e))) - return None - - key = (Word(alphanums + '_') + Suppress(':')).setParseAction(key_parse_action) - key_stmt = Forward() - - condition_block = indentedBlock(key_stmt, indent_stack) - key_stmts = OneOrMore(condition_block) - key_body = Suppress(key) + key_stmts - key_body.setParseAction(key_body_parsed) - - condition = originalTextFor(SDKConfig.get_expression_grammar()).setResultsName('condition') - if_condition = Group(Suppress('if') + condition + Suppress(':') + condition_block) - elif_condition = Group(Suppress('elif') + condition + Suppress(':') + condition_block) - else_condition = Group(Suppress('else') + Suppress(':') + condition_block) - conditional = (if_condition + Optional(OneOrMore(elif_condition)) + Optional(else_condition)).setResultsName( - 'conditional') - - def name_parse_action(pstr, loc, toks): - parse_ctx.fragment.name = toks[0] - - ftype = Word(alphas).setParseAction(fragment_type_parse_action) - fid = Suppress(':') + Word(alphanums + '_.').setResultsName('name') - fid.setParseAction(name_parse_action) - header = Suppress('[') + ftype + fid + Suppress(']') - - def fragment_parse_action(pstr, loc, toks): - key_grammars = parse_ctx.fragment.get_key_grammars() - required_keys = set([k for (k, v) in key_grammars.items() if v.required]) - present_keys = required_keys.intersection(set(parse_ctx.keys)) - if present_keys != required_keys: - raise ParseFatalException(pstr, loc, 'required keys %s for fragment not found' % - list(required_keys - present_keys)) - return parse_ctx.fragment - - fragment_stmt = Forward() - fragment_block = indentedBlock(fragment_stmt, indent_stack) - - fragment_if_condition = Group(Suppress('if') + condition + Suppress(':') + fragment_block) - fragment_elif_condition = Group(Suppress('elif') + condition + Suppress(':') + fragment_block) - fragment_else_condition = Group(Suppress('else') + Suppress(':') + fragment_block) - fragment_conditional = (fragment_if_condition + Optional(OneOrMore(fragment_elif_condition)) + - Optional(fragment_else_condition)).setResultsName('conditional') - - fragment = (header + OneOrMore(indentedBlock(key_body, indent_stack, False))).setResultsName('value') - fragment.setParseAction(fragment_parse_action) - fragment.ignore('#' + restOfLine) - - deprecated_mapping = DeprecatedMapping.get_fragment_grammar(sdkconfig, fragment_file.name).setResultsName( - 'value') - - fragment_stmt << (Group(deprecated_mapping) | Group(fragment) | Group(fragment_conditional)) - - def fragment_stmt_parsed(pstr, loc, toks): - stmts = list() - expand_conditionals(toks, stmts) - return stmts - - parser = ZeroOrMore(fragment_stmt).setParseAction(fragment_stmt_parsed) - - self.fragments = parser.parseFile(fragment_file, parseAll=True) - - for fragment in self.fragments: - fragment.path = path + def __bool__(self): + return False class Fragment: """ - Base class for a fragment that can be parsed from a fragment file. All fragments - share the common grammar: - - [type:name] - key1:value1 - key2:value2 - ... - - Supporting a new fragment type means deriving a concrete class which specifies - key-value pairs that the fragment supports and what to do with the parsed key-value pairs. - - The new fragment must also be appended to FRAGMENT_TYPES, specifying the - keyword for the type and the derived class. - - The key of the key-value pair is a simple keyword string. Other parameters - that describe the key-value pair is specified in Fragment.KeyValue: - 1. grammar - pyparsing grammar to parse the value of key-value pair - 2. min - the minimum number of value in the key entry, None means no minimum - 3. max - the maximum number of value in the key entry, None means no maximum - 4. required - if the key-value pair is required in the fragment - - Setting min=max=1 means that the key has a single value. - - FragmentFile provides conditional expression evaluation, enforcing - the parameters for Fragment.Keyvalue. + Base class for a fragment that can be parsed from a fragment file. """ - __metaclass__ = abc.ABCMeta - - KeyValue = namedtuple('KeyValue', 'grammar min max required') - IDENTIFIER = Word(alphas + '_', alphanums + '_') ENTITY = Word(alphanums + '.-_$+') - @abc.abstractmethod - def set_key_value(self, key, parse_results): - pass + def __init__(self, name: str, entries: Set[Union[str, Tuple[str]]]): + self.name = name + self.entries = entries - @abc.abstractmethod - def get_key_grammars(self): - pass + def __repr__(self): + return str(self.__dict__) class Sections(Fragment): @@ -227,24 +47,33 @@ class Sections(Fragment): .section2 ... """ - # Unless quoted, symbol names start with a letter, underscore, or point # and may include any letters, underscores, digits, points, and hyphens. - GNU_LD_SYMBOLS = Word(alphas + '_.', alphanums + '._-') + ENTRY = Combine(Word(alphas + '_.', alphanums + '._-') + Opt('+')) + LineEnd().suppress() - entries_grammar = Combine(GNU_LD_SYMBOLS + Optional('+')) + @staticmethod + def parse_entry(toks): + # section + return toks[0] - grammars = { - 'entries': Fragment.KeyValue(entries_grammar.setResultsName('section'), 1, None, True) - } + @staticmethod + def parse(s, loc, toks): + this = toks[0] - """ - Utility function that returns a list of sections given a sections fragment entry, - with the '+' notation and symbol concatenation handled automatically. - """ + name = this[0] + entries = {entry for entry in this[1] if entry} + + if not entries: + raise ParseFatalException(s, loc, 'Sections entries shouldn\'t be empty') + + return Sections(name, entries) @staticmethod def get_section_data_from_entry(sections_entry, symbol=None): + """ + Returns a list of sections given a sections fragment entry, + with the '+' notation and symbol concatenation handled automatically. + """ if not symbol: sections = list() sections.append(sections_entry.replace('+', '')) @@ -254,18 +83,9 @@ class Sections(Fragment): if sections_entry.endswith('+'): section = sections_entry.replace('+', '.*') expansion = section.replace('.*', '.' + symbol) - return (section, expansion) + return section, expansion else: - return (sections_entry, None) - - def set_key_value(self, key, parse_results): - if key == 'entries': - self.entries = set() - for result in parse_results: - self.entries.add(result['section']) - - def get_key_grammars(self): - return self.__class__.grammars + return sections_entry, None class Scheme(Fragment): @@ -279,20 +99,184 @@ class Scheme(Fragment): sections1 -> target1 ... """ + ENTRY = Fragment.IDENTIFIER + Suppress('->') + Fragment.IDENTIFIER + LineEnd().suppress() - grammars = { - 'entries': Fragment.KeyValue(Fragment.IDENTIFIER.setResultsName('sections') + Suppress('->') + - Fragment.IDENTIFIER.setResultsName('target'), 1, None, True) - } + @staticmethod + def parse_entry(toks): + # section, target + return toks[0], toks[1] - def set_key_value(self, key, parse_results): - if key == 'entries': - self.entries = set() - for result in parse_results: - self.entries.add((result['sections'], result['target'])) + @staticmethod + def parse(s, loc, toks): + this = toks[0] - def get_key_grammars(self): - return self.__class__.grammars + name = this[0] + entries = {entry for entry in this[1] if entry} + + if not entries: + raise ParseFatalException(s, loc, 'Scheme entries shouldn\'t be empty') + + return Scheme(name, entries) + + +class EntryFlag: + def __repr__(self): + return str(self.__dict__) + + +class Surround(EntryFlag): + """ + SURROUND(symbol) + + '__symbol_start', '__symbol_end' is generated before and after + the corresponding input section description, respectively. + """ + SURROUND = (Keyword('SURROUND').suppress() + + Suppress('(') + + Fragment.IDENTIFIER + + Suppress(')')) + + def __init__(self, symbol: str): + self.symbol = symbol + self.pre = True + self.post = True + + def __eq__(self, other): + if isinstance(other, Surround): + if self.symbol == other.symbol and self.pre == other.pre and self.post == other.post: + return True + + return False + + @staticmethod + def parse(toks): + return Surround(toks[0]) + + +class Align(EntryFlag): + """ + ALIGN(alignment, [, pre, post]). + + Generates alignment command before and/or after the corresponding + input section description, depending on whether pre, post or + both are specified. + """ + PRE = Opt(Suppress(',') + Suppress('pre')).set_results_name('pre') + POST = Opt(Suppress(',') + Suppress('post')).set_results_name('post') + + ALIGN = (Keyword('ALIGN').suppress() + + Suppress('(') + + Word(nums) + + PRE + + POST + + Suppress(')')) + + def __init__(self, alignment, pre=True, post=False): + self.alignment = alignment + self.pre = pre + self.post = post + + def __eq__(self, other): + if isinstance(other, Align): + if self.alignment == other.alignment and self.pre == other.pre and self.post == other.post: + return True + + return False + + @staticmethod + def parse(toks): + alignment = int(toks[0]) + if toks.post == '': + return Align(alignment) + + if toks.pre == '' and toks.post != '': + return Align(alignment, False, True) + + return Align(alignment, True, True) + + +class Keep(EntryFlag): + """ + KEEP() + + Surrounds input section description with KEEP command. + """ + KEEP = Keyword('KEEP()') + + def __eq__(self, other): + if isinstance(other, Keep): + return True + + return False + + @staticmethod + def parse(): + return Keep() + + +class Sort(EntryFlag): + """ + SORT([sort_by_first, sort_by_second]) + + where sort_by_first, sort_by_second = {name, alignment, init_priority} + + Emits SORT_BY_NAME, SORT_BY_ALIGNMENT or SORT_BY_INIT_PRIORITY + depending on arguments. Nested sort follows linker script rules. + """ + _keywords = Keyword('name') | Keyword('alignment') | Keyword('init_priority') + SORT = (Keyword('SORT').suppress() + + Suppress('(') + + _keywords.set_results_name('first') + + Opt(Suppress(',') + _keywords.set_results_name('second')) + + Suppress(')')) + + def __init__(self, first: str, second: Optional[str] = None): + self.first = first + self.second = second + + def __eq__(self, other): + if isinstance(other, Sort): + if self.first == other.first and self.second == other.second: + return True + + return False + + @staticmethod + def parse(toks): + return Sort(toks.first, toks.second or None) + + +class Flag: + _section_target = Fragment.IDENTIFIER + Suppress('->') + Fragment.IDENTIFIER + _flag = (Surround.SURROUND.set_parse_action(Surround.parse) + | Align.ALIGN.set_parse_action(Align.parse) + | Keep.KEEP.set_parse_action(Keep.parse) + | Sort.SORT.set_parse_action(Sort.parse)) + + FLAG = _section_target + OneOrMore(_flag) + + def __init__(self, section: str, target: str, flags: List[EntryFlag]): + self.section = section + self.target = target + self.flags = flags + + def __eq__(self, other): + if isinstance(other, Flag): + if self.section == other.section and self.target == other.target and len(self.flags) == len(other.flags): + for i, j in zip(self.flags, other.flags): + if i != j: + break + else: + return True + + return False + + @staticmethod + def parse(toks): + return Flag(toks[0], toks[1], toks[2:]) + + def __repr__(self): + return str(self.__dict__) class Mapping(Fragment): @@ -316,295 +300,174 @@ class Mapping(Fragment): output_commands.py. """ - class Flag(): - PRE_POST = (Optional(Suppress(',') + Suppress('pre').setParseAction(lambda: True).setResultsName('pre')) + - Optional(Suppress(',') + Suppress('post').setParseAction(lambda: True).setResultsName('post'))) + _any = Literal('*') + _obj = Word(alphas + '_', alphanums + '-_').set_results_name('object') + _sym = Fragment.IDENTIFIER.set_results_name('symbol') - class Surround(Flag): - def __init__(self, symbol): - self.symbol = symbol - self.pre = True - self.post = True + # There are three possible patterns for mapping entries: + # obj:symbol (scheme) + # obj (scheme) + # * (scheme) + _entry = (((_obj + Opt(Suppress(':') + _sym)) | _any.set_results_name('object')) + + Suppress('(') + + Fragment.IDENTIFIER.set_results_name('section') + + Suppress(')')) - @staticmethod - def get_grammar(): - # SURROUND(symbol) - # - # '__symbol_start', '__symbol_end' is generated before and after - # the corresponding input section description, respectively. - grammar = (Keyword('SURROUND').suppress() + - Suppress('(') + - Fragment.IDENTIFIER.setResultsName('symbol') + - Suppress(')')) + ENTRY = _entry + LineEnd().suppress() + ARCHIVE = (Word(alphanums + '.-_$+') | Literal('*')) + LineEnd().suppress() - grammar.setParseAction(lambda tok: Mapping.Surround(tok.symbol)) - return grammar + # Flags can be specified for section->target in the scheme specified, ex: + # obj (scheme); + # section->target SURROUND(symbol), + # section2->target2 ALIGN(4) + ENTRY_WITH_FLAG = (_entry + Suppress(';') + + delimited_list(Flag.FLAG.set_parse_action(Flag.parse))) - def __eq__(self, other): - return (isinstance(other, Mapping.Surround) and - self.symbol == other.symbol) - - class Align(Flag): - - def __init__(self, alignment, pre=True, post=False): - self.alignment = alignment - self.pre = pre - self.post = post - - @staticmethod - def get_grammar(): - # ALIGN(alignment, [, pre, post]). - # - # Generates alignment command before and/or after the corresponding - # input section description, depending whether pre, post or - # both are specified. - grammar = (Keyword('ALIGN').suppress() + - Suppress('(') + - Word(nums).setResultsName('alignment') + - Mapping.Flag.PRE_POST + - Suppress(')')) - - def on_parse(tok): - alignment = int(tok.alignment) - if tok.pre == '' and tok.post == '': - res = Mapping.Align(alignment) - elif tok.pre != '' and tok.post == '': - res = Mapping.Align(alignment, tok.pre) - elif tok.pre == '' and tok.post != '': - res = Mapping.Align(alignment, False, tok.post) - else: - res = Mapping.Align(alignment, tok.pre, tok.post) - return res - - grammar.setParseAction(on_parse) - return grammar - - def __eq__(self, other): - return (isinstance(other, Mapping.Align) and - self.alignment == other.alignment and - self.pre == other.pre and - self.post == other.post) - - class Keep(Flag): - - def __init__(self): - pass - - @staticmethod - def get_grammar(): - # KEEP() - # - # Surrounds input section description with KEEP command. - grammar = Keyword('KEEP()').setParseAction(Mapping.Keep) - return grammar - - def __eq__(self, other): - return isinstance(other, Mapping.Keep) - - class Sort(Flag): - class Type(Enum): - NAME = 0 - ALIGNMENT = 1 - INIT_PRIORITY = 2 - - def __init__(self, first, second=None): - self.first = first - self.second = second - - @staticmethod - def get_grammar(): - # SORT([sort_by_first, sort_by_second]) - # - # where sort_by_first, sort_by_second = {name, alignment, init_priority} - # - # Emits SORT_BY_NAME, SORT_BY_ALIGNMENT or SORT_BY_INIT_PRIORITY - # depending on arguments. Nested sort follows linker script rules. - keywords = Keyword('name') | Keyword('alignment') | Keyword('init_priority') - grammar = (Keyword('SORT').suppress() + Suppress('(') + - keywords.setResultsName('first') + - Optional(Suppress(',') + keywords.setResultsName('second')) + Suppress(')')) - - grammar.setParseAction(lambda tok: Mapping.Sort(tok.first, tok.second if tok.second != '' else None)) - return grammar - - def __eq__(self, other): - return (isinstance(other, Mapping.Sort) and - self.first == other.first and - self.second == other.second) - - def __init__(self): - Fragment.__init__(self) - self.entries = set() - # k = (obj, symbol, scheme) - # v = list((section, target), Mapping.Flag)) - self.flags = dict() - self.deprecated = False - - def set_key_value(self, key, parse_results): - if key == 'archive': - self.archive = parse_results[0]['archive'] - elif key == 'entries': - for result in parse_results: - obj = None - symbol = None - scheme = None - - obj = result['object'] - - try: - symbol = result['symbol'] - except KeyError: - pass - - scheme = result['scheme'] - - mapping = (obj, symbol, scheme) - self.entries.add(mapping) - - try: - parsed_flags = result['sections_target_flags'] - except KeyError: - parsed_flags = [] - - if parsed_flags: - entry_flags = [] - for pf in parsed_flags: - entry_flags.append((pf.sections, pf.target, list(pf.flags))) - - try: - existing_flags = self.flags[mapping] - except KeyError: - existing_flags = list() - self.flags[mapping] = existing_flags - - existing_flags.extend(entry_flags) - - def get_key_grammars(self): - # There are three possible patterns for mapping entries: - # obj:symbol (scheme) - # obj (scheme) - # * (scheme) - # Flags can be specified for section->target in the scheme specified, ex: - # obj (scheme); section->target SURROUND(symbol), section2->target2 ALIGN(4) - obj = Fragment.ENTITY.setResultsName('object') - symbol = Suppress(':') + Fragment.IDENTIFIER.setResultsName('symbol') - scheme = Suppress('(') + Fragment.IDENTIFIER.setResultsName('scheme') + Suppress(')') - - # The flags are specified for section->target in the scheme specified - sections_target = Scheme.grammars['entries'].grammar - - flag = Or([f.get_grammar() for f in [Mapping.Keep, Mapping.Align, Mapping.Surround, Mapping.Sort]]) - - section_target_flags = Group(sections_target + Group(OneOrMore(flag)).setResultsName('flags')) - - pattern1 = obj + symbol - pattern2 = obj - pattern3 = Literal(Entity.ALL).setResultsName('object') - - entry = ((pattern1 | pattern2 | pattern3) + scheme + - Optional(Suppress(';') + delimitedList(section_target_flags).setResultsName('sections_target_flags'))) - - grammars = { - 'archive': Fragment.KeyValue(Or([Fragment.ENTITY, Word(Entity.ALL)]).setResultsName('archive'), 1, 1, - True), - 'entries': Fragment.KeyValue(entry, 0, None, True) - } - - return grammars - - -class DeprecatedMapping(): - """ - Mapping fragment with old grammar in versions older than ESP-IDF v4.0. Does not conform to - requirements of the Fragment class and thus is limited when it comes to conditional expression - evaluation. - """ - - # Name of the default condition entry - DEFAULT_CONDITION = 'default' + def __init__(self, archive: str, flags: Dict[Any, Flag], *args, **kwargs): + super().__init__(*args, **kwargs) + self.archive = archive + self.flags = flags @staticmethod - def get_fragment_grammar(sdkconfig, fragment_file): + def parse_archive(s, loc, toks): + this = toks[0][0] + if len(this) != 1: + raise ParseFatalException(s, loc, 'Could only specify one archive file in one mapping fragment') - # Match header [mapping] - header = Suppress('[') + Suppress('mapping') + Suppress(']') + return this[0] - # There are three possible patterns for mapping entries: - # obj:symbol (scheme) - # obj (scheme) - # * (scheme) - obj = Fragment.ENTITY.setResultsName('object') - symbol = Suppress(':') + Fragment.IDENTIFIER.setResultsName('symbol') - scheme = Suppress('(') + Fragment.IDENTIFIER.setResultsName('scheme') + Suppress(')') + @staticmethod + def parse_entry(toks): + return toks.object, toks.symbol or None, toks.section - pattern1 = Group(obj + symbol + scheme) - pattern2 = Group(obj + scheme) - pattern3 = Group(Literal(Entity.ALL).setResultsName('object') + scheme) + @staticmethod + def parse_entry_with_flag(toks): + entry = toks.object, toks.symbol or None, toks.section + return { + entry: [tok for tok in toks if isinstance(tok, Flag)] + } - mapping_entry = pattern1 | pattern2 | pattern3 + @staticmethod + def parse_entries(toks): + return toks[0] - # To simplify parsing, classify groups of condition-mapping entry into two types: normal and default - # A normal grouping is one with a non-default condition. The default grouping is one which contains the - # default condition - mapping_entries = Group(ZeroOrMore(mapping_entry)).setResultsName('mappings') + @staticmethod + def parse(toks): + this = toks[0] - normal_condition = Suppress(':') + originalTextFor(SDKConfig.get_expression_grammar()) - default_condition = Optional(Suppress(':') + Literal(DeprecatedMapping.DEFAULT_CONDITION)) + name = this[0] + archive = this[1] + entries_or_dict_with_flags = this[2] - normal_group = Group(normal_condition.setResultsName('condition') + mapping_entries) - default_group = Group(default_condition + mapping_entries).setResultsName('default_group') + entries = set() + flags = dict() + for item in entries_or_dict_with_flags: + if isinstance(item, Empty): + continue + elif isinstance(item, dict): # entry with flags + for k, v in item.items(): + entries.add(k) + if k in flags: + flags[k].extend(v) + else: + flags[k] = v + else: + entries.add(item) - normal_groups = Group(ZeroOrMore(normal_group)).setResultsName('normal_groups') - - # Any mapping fragment definition can have zero or more normal group and only one default group as a last entry. - archive = Suppress('archive') + Suppress(':') + Fragment.ENTITY.setResultsName('archive') - entries = Suppress('entries') + Suppress(':') + (normal_groups + default_group).setResultsName('entries') - - mapping = Group(header + archive + entries) - mapping.ignore('#' + restOfLine) - - def parsed_deprecated_mapping(pstr, loc, toks): - fragment = Mapping() - fragment.archive = toks[0].archive - fragment.name = re.sub(r'[^0-9a-zA-Z]+', '_', fragment.archive) - fragment.deprecated = True - - fragment.entries = set() - condition_true = False - for entries in toks[0].entries[0]: - condition = next(iter(entries.condition.asList())).strip() - condition_val = sdkconfig.evaluate_expression(condition) - - if condition_val: - for entry in entries[1]: - fragment.entries.add( - (entry.object, None if entry.symbol == '' else entry.symbol, entry.scheme)) - condition_true = True - break - - if not fragment.entries and not condition_true: - try: - entries = toks[0].entries[1][1] - except IndexError: - entries = toks[0].entries[1][0] - for entry in entries: - fragment.entries.add((entry.object, None if entry.symbol == '' else entry.symbol, entry.scheme)) - - if not fragment.entries: - fragment.entries.add(('*', None, 'default')) - - dep_warning = str(ParseFatalException(pstr, loc, - 'Warning: Deprecated old-style mapping fragment parsed in file %s.' % fragment_file)) - - print(dep_warning) - return fragment - - mapping.setParseAction(parsed_deprecated_mapping) - return mapping + return Mapping(archive=archive, name=name, entries=entries, flags=flags) -FRAGMENT_TYPES = { - 'sections': Sections, - 'scheme': Scheme, - 'mapping': Mapping -} +class FragmentFile: + """ + Processes a fragment file and stores all parsed fragments. For + more information on how this class interacts with classes for the different fragment types, + see description of Fragment. + """ + + def __init__(self, fragments: List[Fragment]): + self.path = None # assign later, couldn't pass extra argument while parsing + self.fragments: List[Fragment] = fragments + + def __repr__(self): + return str(self.__dict__) + + +def parse_fragment_file(path, sdkconfig): + def parse_conditional(toks): + this = toks[0] + for stmt in this: + if stmt[0] in ['if', 'elif']: # if/elif + if sdkconfig.evaluate_expression(stmt.condition): + return stmt[-1] + else: # else + return stmt[-1] + + return Empty() + + def get_conditional_stmt(_stmt): + condition = SkipTo(':').set_results_name('condition') + Suppress(':') + _suite = IndentedBlock(_stmt) + + if_decl = Literal('if') + condition + elif_decl = Literal('elif') + condition + else_decl = Literal('else:') + if_ = Group(if_decl + _suite) + elif_ = Group(elif_decl + _suite) + else_ = Group(else_decl + _suite) + return Group(if_ + Opt(OneOrMore(elif_)) + Opt(else_)).set_parse_action(parse_conditional) + + def get_suite(_stmt): + __stmt = Forward() + __conditional = get_conditional_stmt(__stmt) + __stmt <<= (comment + | _stmt + | __conditional) + return IndentedBlock(__stmt) + + def parse(toks): + return FragmentFile([tok for tok in toks if not isinstance(tok, Empty)]) + + # comment + comment = (Literal('#') + rest_of_line).set_parse_action(lambda s, l, t: Empty()) + + # section + section_entry = Sections.ENTRY.set_parse_action(Sections.parse_entry) + section_entries_suite = get_suite(section_entry) + section_header = Suppress('[sections:') + Fragment.IDENTIFIER + Suppress(']') + LineEnd().suppress() + section = Group(section_header + + Suppress('entries:') + + section_entries_suite).set_parse_action(Sections.parse) + + # scheme + scheme_entry = Scheme.ENTRY.set_parse_action(Scheme.parse_entry) + scheme_entries_suite = get_suite(scheme_entry) + scheme_header = Suppress('[scheme:') + Fragment.IDENTIFIER + Suppress(']') + LineEnd().suppress() + scheme = Group(scheme_header + + Suppress('entries:') + + scheme_entries_suite).set_parse_action(Scheme.parse) + # mapping + mapping_archive = Mapping.ARCHIVE + mapping_archive_suite = get_suite(mapping_archive) + + mapping_entry = Mapping.ENTRY.set_parse_action(Mapping.parse_entry) + mapping_entry_with_flag = Mapping.ENTRY_WITH_FLAG.set_parse_action(Mapping.parse_entry_with_flag) + mapping_entries_suite = get_suite(mapping_entry | mapping_entry_with_flag) + + mapping_header = Suppress('[mapping:') + Fragment.IDENTIFIER + Suppress(']') + mapping = Group(mapping_header + + Group(Suppress('archive:') + + mapping_archive_suite).set_parse_action(Mapping.parse_archive) + + Group(Suppress('entries:') + + mapping_entries_suite).set_parse_action(Mapping.parse_entries) + ).set_parse_action(Mapping.parse) + + # highest level + fragment = (section + | scheme + | mapping + | get_conditional_stmt(section | scheme | mapping)) + parser = ZeroOrMore(fragment).ignore(comment).set_parse_action(parse) + fragment_file = parser.parse_file(path, parse_all=True)[0] + fragment_file.path = path + + return fragment_file diff --git a/tools/ldgen/ldgen/generation.py b/tools/ldgen/ldgen/generation.py index 888883dccf..f4e87d849e 100644 --- a/tools/ldgen/ldgen/generation.py +++ b/tools/ldgen/ldgen/generation.py @@ -1,5 +1,5 @@ # -# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # @@ -9,7 +9,7 @@ import itertools from collections import namedtuple from .entity import Entity -from .fragments import Mapping, Scheme, Sections +from .fragments import Keep, Scheme, Sections, Sort, Surround from .ldgen_common import LdGenFailure from .output_commands import AlignAtAddress, InputSectionDesc, SymbolAtAddress @@ -97,7 +97,7 @@ class Placement: self.subplacements.add(subplacement) -class EntityNode(): +class EntityNode: """ Node in entity tree. An EntityNode is created from an Entity (see entity.py). @@ -183,16 +183,16 @@ class EntityNode(): placement_flags = placement.flags if placement.flags is not None else [] for flag in placement_flags: - if isinstance(flag, Mapping.Keep): + if isinstance(flag, Keep): keep = True - elif isinstance(flag, Mapping.Sort): + elif isinstance(flag, Sort): sort = (flag.first, flag.second) else: # SURROUND or ALIGN surround_type.append(flag) for flag in surround_type: if flag.pre: - if isinstance(flag, Mapping.Surround): + if isinstance(flag, Surround): commands[placement.target].append(SymbolAtAddress('_%s_start' % flag.symbol)) else: # ALIGN commands[placement.target].append(AlignAtAddress(flag.alignment)) @@ -206,8 +206,8 @@ class EntityNode(): [e.node.entity for e in placement.exclusions], keep, sort) commands[placement.target].append(command) - # Generate commands for intermediate, non-explicit exclusion placements here, so that they can be enclosed by - # flags that affect the parent placement. + # Generate commands for intermediate, non-explicit exclusion placements here, + # so that they can be enclosed by flags that affect the parent placement. for subplacement in placement.subplacements: if not subplacement.flags and not subplacement.explicit: command = InputSectionDesc(subplacement.node.entity, subplacement.sections, @@ -216,7 +216,7 @@ class EntityNode(): for flag in surround_type: if flag.post: - if isinstance(flag, Mapping.Surround): + if isinstance(flag, Surround): commands[placement.target].append(SymbolAtAddress('_%s_end' % flag.symbol)) else: # ALIGN commands[placement.target].append(AlignAtAddress(flag.alignment)) @@ -387,7 +387,7 @@ class Generation: for (sections_name, target_name) in scheme.entries: # Get the sections under the bucket 'target_name'. If this bucket does not exist - # is is created automatically + # is created automatically sections_in_bucket = sections_bucket[target_name] try: @@ -449,10 +449,11 @@ class Generation: flags = mapping.flags[(obj, symbol, scheme_name)] # Check if all section->target defined in the current # scheme. - for (s, t, f) in flags: - if (t not in scheme_dictionary[scheme_name].keys() - or s not in [_s.name for _s in scheme_dictionary[scheme_name][t]]): - message = "%s->%s not defined in scheme '%s'" % (s, t, scheme_name) + for flag in flags: + if (flag.target not in scheme_dictionary[scheme_name].keys() + or flag.section not in + [_s.name for _s in scheme_dictionary[scheme_name][flag.target]]): + message = "%s->%s not defined in scheme '%s'" % (flag.section, flag.target, scheme_name) raise GenerationException(message, mapping) else: flags = None @@ -464,9 +465,9 @@ class Generation: _flags = [] if flags: - for (s, t, f) in flags: - if (s, t) == (section.name, target): - _flags.extend(f) + for flag in flags: + if (flag.section, flag.target) == (section.name, target): + _flags.extend(flag.flags) sections_str = get_section_strs(section) @@ -481,18 +482,18 @@ class Generation: entity_mappings[key] = Generation.EntityMapping(entity, sections_str, target, _flags) else: # Check for conflicts. - if (target != existing.target): + if target != existing.target: raise GenerationException('Sections mapped to multiple targets.', mapping) # Combine flags here if applicable, to simplify # insertion logic. - if (_flags or existing.flags): - if ((_flags and not existing.flags) or (not _flags and existing.flags)): + if _flags or existing.flags: + if (_flags and not existing.flags) or (not _flags and existing.flags): _flags.extend(existing.flags) entity_mappings[key] = Generation.EntityMapping(entity, sections_str, target, _flags) - elif (_flags == existing.flags): + elif _flags == existing.flags: pass else: raise GenerationException('Conflicting flags specified.', mapping) @@ -521,25 +522,22 @@ class Generation: def add_fragments_from_file(self, fragment_file): for fragment in fragment_file.fragments: - if isinstance(fragment, Mapping) and fragment.deprecated and fragment.name in self.mappings.keys(): - self.mappings[fragment.name].entries |= fragment.entries + if isinstance(fragment, Scheme): + dict_to_append_to = self.schemes + elif isinstance(fragment, Sections): + dict_to_append_to = self.placements else: - if isinstance(fragment, Scheme): - dict_to_append_to = self.schemes - elif isinstance(fragment, Sections): - dict_to_append_to = self.placements - else: - dict_to_append_to = self.mappings + dict_to_append_to = self.mappings - # Raise exception when the fragment of the same type is already in the stored fragments - if fragment.name in dict_to_append_to.keys(): - stored = dict_to_append_to[fragment.name].path - new = fragment.path - message = "Duplicate definition of fragment '%s' found in %s and %s." % ( - fragment.name, stored, new) - raise GenerationException(message) + # Raise exception when the fragment of the same type is already in the stored fragments + if fragment.name in dict_to_append_to: + stored = dict_to_append_to[fragment.name].path + new = fragment.path + message = "Duplicate definition of fragment '%s' found in %s and %s." % ( + fragment.name, stored, new) + raise GenerationException(message) - dict_to_append_to[fragment.name] = fragment + dict_to_append_to[fragment.name] = fragment class GenerationException(LdGenFailure): diff --git a/tools/ldgen/ldgen/ldgen_common.py b/tools/ldgen/ldgen/ldgen_common.py index 779206d58c..ba4346cb14 100644 --- a/tools/ldgen/ldgen/ldgen_common.py +++ b/tools/ldgen/ldgen/ldgen_common.py @@ -1,13 +1,9 @@ # -# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # - class LdGenFailure(RuntimeError): """ Parent class for any ldgen runtime failure which is due to input data """ - - def __init__(self, message): - super(LdGenFailure, self).__init__(message) diff --git a/tools/ldgen/ldgen/linker_script.py b/tools/ldgen/ldgen/linker_script.py index 94e9e1ae5c..59405e2fc9 100644 --- a/tools/ldgen/ldgen/linker_script.py +++ b/tools/ldgen/ldgen/linker_script.py @@ -1,5 +1,5 @@ # -# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # @@ -33,24 +33,21 @@ class LinkerScript: lines = template_file.readlines() target = Fragment.IDENTIFIER - reference = Suppress('mapping') + Suppress('[') + target.setResultsName('target') + Suppress(']') - pattern = White(' \t').setResultsName('indent') + reference + reference = Suppress('mapping') + Suppress('[') + target + Suppress(']') + pattern = White(' \t') + reference # Find the markers in the template file line by line. If line does not match marker grammar, # set it as a literal to be copied as is to the output file. for line in lines: try: - parsed = pattern.parseString(line) - - indent = parsed.indent - target = parsed.target - - marker = LinkerScript.Marker(target, indent, []) - - self.members.append(marker) + parsed = pattern.parse_string(line) except ParseException: # Does not match marker syntax self.members.append(line) + else: + indent, target = parsed + marker = LinkerScript.Marker(target, indent, []) + self.members.append(marker) def fill(self, mapping_rules): for member in self.members: diff --git a/tools/ldgen/ldgen/output_commands.py b/tools/ldgen/ldgen/output_commands.py index 1708891fd7..7df4a85210 100644 --- a/tools/ldgen/ldgen/output_commands.py +++ b/tools/ldgen/ldgen/output_commands.py @@ -1,5 +1,5 @@ # -# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # @@ -9,7 +9,7 @@ from .entity import Entity # https://www.acrc.bris.ac.uk/acrc/RedHat/rhel-ld-en-4/sections.html#OUTPUT-SECTION-DESCRIPTION. -class AlignAtAddress(): +class AlignAtAddress: """ Outputs assignment of builtin function ALIGN to current position: @@ -31,7 +31,7 @@ class AlignAtAddress(): self.alignment == other.alignment) -class SymbolAtAddress(): +class SymbolAtAddress: """ Outputs assignment of builtin function ABSOLUTE to a symbol for current position: @@ -54,7 +54,7 @@ class SymbolAtAddress(): self.symbol == other.symbol) -class InputSectionDesc(): +class InputSectionDesc: """ Outputs an input section description as described in https://www.acrc.bris.ac.uk/acrc/RedHat/rhel-ld-en-4/sections.html#INPUT-SECTION. diff --git a/tools/ldgen/ldgen/sdkconfig.py b/tools/ldgen/ldgen/sdkconfig.py index 06f0cae38c..7899ae5f91 100644 --- a/tools/ldgen/ldgen/sdkconfig.py +++ b/tools/ldgen/ldgen/sdkconfig.py @@ -1,35 +1,19 @@ # -# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # import kconfiglib -from pyparsing import (Combine, Group, Literal, Optional, Word, alphanums, hexnums, infixNotation, nums, oneOf, - opAssoc, printables, quotedString, removeQuotes) class SDKConfig: """ Evaluates conditional expressions based on the build's sdkconfig and Kconfig files. - This also defines the grammar of conditional expressions. """ - - # A configuration entry is in the form CONFIG=VALUE. Definitions of components of that grammar - IDENTIFIER = Word(alphanums.upper() + '_') - - HEX = Combine('0x' + Word(hexnums)).setParseAction(lambda t: int(t[0], 16)) - DECIMAL = Combine(Optional(Literal('+') | Literal('-')) + Word(nums)).setParseAction(lambda t: int(t[0])) - LITERAL = Word(printables.replace(':', '')) - QUOTED_LITERAL = quotedString.setParseAction(removeQuotes) - - VALUE = HEX | DECIMAL | LITERAL | QUOTED_LITERAL - - # Operators supported by the expression evaluation - OPERATOR = oneOf(['=', '!=', '>', '<', '<=', '>=']) - def __init__(self, kconfig_file, sdkconfig_file): self.config = kconfiglib.Kconfig(kconfig_file) self.config.load_config(sdkconfig_file) + self.config.warn = False # eval_string may contain un-declared symbol def evaluate_expression(self, expression): result = self.config.eval_string(expression) @@ -40,23 +24,3 @@ class SDKConfig: return True else: # m raise Exception('unsupported config expression result') - - @staticmethod - def get_expression_grammar(): - identifier = SDKConfig.IDENTIFIER.setResultsName('identifier') - operator = SDKConfig.OPERATOR.setResultsName('operator') - value = SDKConfig.VALUE.setResultsName('value') - - test_binary = identifier + operator + value - test_single = identifier - - test = test_binary | test_single - - condition = Group(Optional('(').suppress() + test + Optional(')').suppress()) - - grammar = infixNotation(condition, [ - ('!', 1, opAssoc.RIGHT), - ('&&', 2, opAssoc.LEFT), - ('||', 2, opAssoc.LEFT)]) - - return grammar diff --git a/tools/ldgen/test/test_entity.py b/tools/ldgen/test/test_entity.py index 7b27f75155..0f297157a8 100755 --- a/tools/ldgen/test/test_entity.py +++ b/tools/ldgen/test/test_entity.py @@ -1,17 +1,18 @@ #!/usr/bin/env python # coding=utf-8 # -# SPDX-FileCopyrightText: 2018-2021 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2018-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # +import os import sys import unittest try: from ldgen.entity import Entity, EntityDB except ImportError: - sys.path.append('../') + sys.path.append(os.path.dirname(os.path.dirname(__file__))) from ldgen.entity import Entity, EntityDB diff --git a/tools/ldgen/test/test_fragments.py b/tools/ldgen/test/test_fragments.py index f735a8ec24..df1665cd72 100755 --- a/tools/ldgen/test/test_fragments.py +++ b/tools/ldgen/test/test_fragments.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # @@ -10,44 +10,18 @@ import tempfile import unittest from io import StringIO -from pyparsing import ParseException, ParseFatalException, Word, alphanums - try: - from ldgen.fragments import FRAGMENT_TYPES, Fragment, FragmentFile, Mapping + from ldgen.fragments import Align, Flag, Keep, Sort, Surround, parse_fragment_file from ldgen.sdkconfig import SDKConfig except ImportError: - sys.path.append('../') - from ldgen.fragments import FRAGMENT_TYPES, Fragment, FragmentFile, Mapping + sys.path.append(os.path.dirname(os.path.dirname(__file__))) + from ldgen.fragments import Align, Flag, Keep, Sort, Surround, parse_fragment_file from ldgen.sdkconfig import SDKConfig - -class SampleFragment(Fragment): - - grammars = { - 'key_1': Fragment.KeyValue(Word(alphanums + '_').setResultsName('value'), 0, None, True), - 'key_2': Fragment.KeyValue(Word(alphanums + '_').setResultsName('value'), 0, None, False), - 'key_3': Fragment.KeyValue(Word(alphanums + '_').setResultsName('value'), 3, 5, False) - } - - def set_key_value(self, key, parse_results): - if key == 'key_1': - self.key_1 = list() - for result in parse_results: - self.key_1.append(result['value']) - elif key == 'key_2': - self.key_2 = list() - for result in parse_results: - self.key_2.append(result['value']) - - def get_key_grammars(self): - return self.__class__.grammars - - -FRAGMENT_TYPES['test'] = SampleFragment +from pyparsing import ParseException, ParseFatalException class FragmentTest(unittest.TestCase): - def setUp(self): with tempfile.NamedTemporaryFile(delete=False) as f: self.kconfigs_source_file = os.path.join(tempfile.gettempdir(), f.name) @@ -79,46 +53,24 @@ class FragmentTest(unittest.TestCase): def test_basic(self): test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: +[sections:test] +entries: value_1 value_2 # comments should be ignored value_3 # this is a comment as well -key_2: value_a - + value_a # this is the last comment """) - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) - self.assertEqual(len(fragment_file.fragments[0].key_1), 3) - self.assertEqual(fragment_file.fragments[0].key_1[0], 'value_1') - self.assertEqual(fragment_file.fragments[0].key_1[1], 'value_2') - self.assertEqual(fragment_file.fragments[0].key_1[2], 'value_3') - self.assertEqual(len(fragment_file.fragments[0].key_2), 1) - self.assertEqual(fragment_file.fragments[0].key_2[0], 'value_a') - - def test_duplicate_keys(self): - test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: value_1 -key_1: value_a -""") - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) - - def test_empty_key(self): - test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: -""") - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) + self.assertEqual(fragment_file.fragments[0].name, 'test') + self.assertEqual(fragment_file.fragments[0].entries, {'value_1', 'value_2', 'value_3', 'value_a'}) def test_conditional(self): test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: +[sections:test] +entries: value_1 if A = y: value_2 @@ -128,15 +80,14 @@ key_1: if B = n: value_5 """) - fragment_file = FragmentFile(test_fragment, self.sdkconfig) - self.assertEqual(fragment_file.fragments[0].key_1[0], 'value_1') - self.assertEqual(fragment_file.fragments[0].key_1[1], 'value_2') - self.assertEqual(fragment_file.fragments[0].key_1[2], 'value_3') - self.assertEqual(fragment_file.fragments[0].key_1[3], 'value_5') + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) + + self.assertEqual(fragment_file.fragments[0].name, 'test') + self.assertEqual(fragment_file.fragments[0].entries, {'value_1', 'value_2', 'value_3', 'value_5'}) test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: +[sections:test] +entries: value_1 if B = y: value_2 @@ -148,15 +99,14 @@ key_1: value_5 value_6 """) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) - fragment_file = FragmentFile(test_fragment, self.sdkconfig) - self.assertEqual(fragment_file.fragments[0].key_1[0], 'value_1') - self.assertEqual(fragment_file.fragments[0].key_1[1], 'value_3') - self.assertEqual(fragment_file.fragments[0].key_1[2], 'value_6') + self.assertEqual(fragment_file.fragments[0].name, 'test') + self.assertEqual(fragment_file.fragments[0].entries, {'value_1', 'value_3', 'value_6'}) test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: +[sections:test] +entries: value_1 if A = y: value_2 @@ -168,304 +118,206 @@ key_1: value_5 value_6 value_7 -key_2: - value_a - if B != y: - value_b """) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) - fragment_file = FragmentFile(test_fragment, self.sdkconfig) - self.assertEqual(fragment_file.fragments[0].key_1[0], 'value_1') - self.assertEqual(fragment_file.fragments[0].key_1[1], 'value_2') - self.assertEqual(fragment_file.fragments[0].key_1[2], 'value_4') - self.assertEqual(fragment_file.fragments[0].key_1[3], 'value_5') - self.assertEqual(fragment_file.fragments[0].key_1[4], 'value_6') - self.assertEqual(fragment_file.fragments[0].key_1[5], 'value_7') - self.assertEqual(fragment_file.fragments[0].key_2[0], 'value_a') - self.assertEqual(fragment_file.fragments[0].key_2[1], 'value_b') + self.assertEqual(fragment_file.fragments[0].name, 'test') + self.assertEqual(fragment_file.fragments[0].entries, + {'value_1', 'value_2', 'value_4', 'value_5', 'value_6', 'value_7'}) test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: +[sections:test] +entries: if A = n: value_2 """) - - fragment_file = FragmentFile(test_fragment, self.sdkconfig) - self.assertEqual(len(fragment_file.fragments[0].key_1), 0) + with self.assertRaises(ParseFatalException): + parse_fragment_file(test_fragment, self.sdkconfig) def test_empty_file(self): test_fragment = self.create_fragment_file(u""" - - - - """) - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) self.assertEqual(len(fragment_file.fragments), 0) def test_setting_indent(self): test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: +[sections:test] +entries: value_1 value_2 value_3 """) - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) - self.assertEqual(len(fragment_file.fragments[0].key_1), 3) - self.assertEqual(fragment_file.fragments[0].key_1[0], 'value_1') - self.assertEqual(fragment_file.fragments[0].key_1[1], 'value_2') - self.assertEqual(fragment_file.fragments[0].key_1[2], 'value_3') + self.assertEqual(fragment_file.fragments[0].name, 'test') + self.assertEqual(fragment_file.fragments[0].entries, {'value_1', 'value_2', 'value_3'}) + def test_settings_unmatch_indent(self): test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: +[sections:test] +entries: value_1 value_2 # first element dictates indent value_3 """) - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) - - def test_values_num_limit(self): - test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: - value_a -key_3: - value_1 - value_2 - value_3 -""") - fragment_file = FragmentFile(test_fragment, self.sdkconfig) - - test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: - value_a -key_3: - value_1 - value_2 - value_3 - value_4 -""") - fragment_file = FragmentFile(test_fragment, self.sdkconfig) - self.assertEqual(len(fragment_file.fragments), 1) - - test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: - value_a -key_3: - value_1 - value_2 - value_3 - value_4 - value_5 -""") - fragment_file = FragmentFile(test_fragment, self.sdkconfig) - self.assertEqual(len(fragment_file.fragments), 1) - - test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: - value_a -key_3: - value_1 - value_2 -""") - - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) - - test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: - value_a -key_3: - value_1 - value_2 - value_3 - value_4 - value_5 - value_6 -""") - - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) + with self.assertRaises(ParseException): + parse_fragment_file(test_fragment, self.sdkconfig) def test_unsupported_key(self): test_fragment = self.create_fragment_file(u""" -[test:test] +[sections:test] key_1: value_a -key_4: - value_1 """) - - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) + with self.assertRaises(ParseException): + parse_fragment_file(test_fragment, self.sdkconfig) def test_empty_fragment(self): test_fragment = self.create_fragment_file(u""" -[test:test] +[sections:test] """) - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) def test_empty_conditional(self): test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: +[sections:test] +entries: if B = y: else: value_1 """) - - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) + with self.assertRaises(ParseException): + parse_fragment_file(test_fragment, self.sdkconfig) test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: +[sections:test] +entries: if B = y: value_1 else B = y: """) - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: +[sections:test] +entries: if B = y: value_1 elif B = y: else: value_2 """) - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) def test_out_of_order_conditional(self): test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: +[sections:test] +entries: elif B = y: value_1 else: value_2 """) - - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) + with self.assertRaises(ParseException): + parse_fragment_file(test_fragment, self.sdkconfig) test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: +[sections:test] +entries: else: value_2 """) - - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) - - def test_required_keys(self): - test_fragment = self.create_fragment_file(u""" -[test:test] -key_2: - value_1 -""") - - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) + with self.assertRaises(ParseException): + parse_fragment_file(test_fragment, self.sdkconfig) def test_multiple_fragments(self): test_fragment = self.create_fragment_file(u""" -[test:test1] -key_1: +[sections:test1] +entries: value_1 -[test:test2] -key_1: - value_2 +[scheme:test2] +entries: + section -> target """) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) - fragment_file = FragmentFile(test_fragment, self.sdkconfig) - - self.assertEqual(len(fragment_file.fragments), 2) - self.assertEqual(fragment_file.fragments[0].key_1[0], 'value_1') - self.assertEqual(fragment_file.fragments[1].key_1[0], 'value_2') + self.assertEqual(fragment_file.fragments[0].name, 'test1') + self.assertEqual(fragment_file.fragments[0].entries, {'value_1'}) + self.assertEqual(fragment_file.fragments[1].name, 'test2') + self.assertEqual(fragment_file.fragments[1].entries, {('section', 'target')}) def test_whole_conditional_fragment(self): test_fragment = self.create_fragment_file(u""" if B = y: - [test:test1] - key_1: + [sections:test1] + entries: value_1 else: - [test:test2] - key_1: + [sections:test2] + entries: value_2 if A = y: - [test:test3] - key_1: + [sections:test3] + entries: value_3 if C = y: value_6 - [test:test4] - key_1: + [sections:test4] + entries: value_4 -[test:test5] -key_1: +[sections:test5] +entries: value_5 """) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) - fragment_file = FragmentFile(test_fragment, self.sdkconfig) self.assertEqual(len(fragment_file.fragments), 4) self.assertEqual(fragment_file.fragments[0].name, 'test2') + self.assertEqual(fragment_file.fragments[0].entries, {'value_2'}) self.assertEqual(fragment_file.fragments[1].name, 'test3') - self.assertEqual(fragment_file.fragments[1].key_1[1], 'value_6') + self.assertEqual(fragment_file.fragments[1].entries, {'value_3', 'value_6'}) self.assertEqual(fragment_file.fragments[2].name, 'test4') + self.assertEqual(fragment_file.fragments[2].entries, {'value_4'}) self.assertEqual(fragment_file.fragments[3].name, 'test5') + self.assertEqual(fragment_file.fragments[3].entries, {'value_5'}) def test_equivalent_conditional_fragment(self): test_fragment1 = self.create_fragment_file(u""" if A = y: - [test:test1] - key_1: + [sections:test1] + entries: value_1 else: - [test:test2] - key_1: + [sections:test2] + entries: value_2 """) + fragment_file1 = parse_fragment_file(test_fragment1, self.sdkconfig) - fragment_file1 = FragmentFile(test_fragment1, self.sdkconfig) - self.assertEqual(len(fragment_file1.fragments), 1) - self.assertEqual(fragment_file1.fragments[0].key_1[0], 'value_1') + self.assertEqual(fragment_file1.fragments[0].name, 'test1') + self.assertEqual(fragment_file1.fragments[0].entries, {'value_1'}) test_fragment2 = self.create_fragment_file(u""" -[test:test1] -key_1: +[sections:test1] +entries: if A = y: value_1 else: value_2 """) + fragment_file2 = parse_fragment_file(test_fragment2, self.sdkconfig) - fragment_file2 = FragmentFile(test_fragment2, self.sdkconfig) - self.assertEqual(len(fragment_file2.fragments), 1) - self.assertEqual(fragment_file2.fragments[0].key_1[0], 'value_1') + self.assertEqual(fragment_file2.fragments[0].name, 'test1') + self.assertEqual(fragment_file2.fragments[0].entries, {'value_1'}) class SectionsTest(FragmentTest): @@ -477,8 +329,7 @@ entries: .section1 .section2 """) - - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) self.assertEqual(fragment_file.fragments[0].entries, {'.section1', '.section2'}) def test_duplicate_entries(self): @@ -490,8 +341,7 @@ entries: .section3 .section2 """) - - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) self.assertEqual(fragment_file.fragments[0].entries, {'.section1', '.section2', '.section3'}) def test_empty_entries(self): @@ -499,9 +349,8 @@ entries: [sections:test] entries: """) - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) test_fragment = self.create_fragment_file(u""" [sections:test] @@ -509,12 +358,10 @@ entries: if B = y: .section1 """) - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) def test_entries_grammar(self): - test_fragment = self.create_fragment_file(u""" [sections:test] entries: @@ -522,8 +369,7 @@ entries: valid2. .valid3_- """) - - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) self.assertEqual(fragment_file.fragments[0].entries, {'_valid1', 'valid2.', '.valid3_-'}) @@ -533,18 +379,16 @@ entries: entries: 1invalid """) - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) test_fragment = self.create_fragment_file(u""" [sections:test] entries: -invalid """) - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) # + notation test_fragment = self.create_fragment_file(u""" @@ -552,8 +396,7 @@ entries: entries: valid+ """) - - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) self.assertEqual(fragment_file.fragments[0].entries, {'valid+'}) @@ -562,9 +405,8 @@ entries: entries: inva+lid+ """) - - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) + with self.assertRaises(ParseException): + parse_fragment_file(test_fragment, self.sdkconfig) class SchemeTest(FragmentTest): @@ -577,7 +419,7 @@ entries: sections2 -> target2 """) - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) self.assertEqual(fragment_file.fragments[0].entries, {('sections1', 'target1'), ('sections2', 'target2')}) @@ -591,7 +433,7 @@ entries: sections2 -> target2 """) - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) self.assertEqual(fragment_file.fragments[0].entries, {('sections1', 'target1'), ('sections2', 'target2')}) @@ -601,9 +443,8 @@ entries: [scheme:test] entries: """) - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) test_fragment = self.create_fragment_file(u""" [scheme:test] @@ -611,9 +452,8 @@ entries: if B = y: sections1 -> target1 """) - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) def test_improper_grammar(self): test_fragment = self.create_fragment_file(u""" @@ -623,7 +463,7 @@ entries: """) with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) class MappingTest(FragmentTest): @@ -646,7 +486,7 @@ entries: ('obj_2', None, 'noflash'), ('*', None, 'noflash')} - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) self.assertEqual(expected, fragment_file.fragments[0].entries) def test_archive(self): @@ -656,9 +496,8 @@ archive: entries: * (default) """) - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) test_fragment = self.create_fragment_file(u""" [mapping:test] @@ -668,9 +507,8 @@ archive: entries: * (default) """) - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) def test_archive_allowed_names(self): test_fragment = self.create_fragment_file(u""" @@ -680,8 +518,7 @@ archive: entries: * (default) """) - - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) self.assertEqual('libstdc++.a', fragment_file.fragments[0].archive) def test_empty_entries(self): @@ -696,7 +533,7 @@ entries: expected = set() - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) self.assertEqual(expected, fragment_file.fragments[0].entries) test_fragment = self.create_fragment_file(u""" @@ -707,7 +544,7 @@ entries: """) with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) def test_duplicate_entries(self): test_fragment = self.create_fragment_file(u""" @@ -721,7 +558,7 @@ entries: expected = {('obj', 'symbol', 'noflash')} - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) self.assertEqual(expected, fragment_file.fragments[0].entries) def test_invalid_grammar(self): @@ -730,18 +567,16 @@ entries: archive: lib.a """) - - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) + with self.assertRaises(ParseException): + parse_fragment_file(test_fragment, self.sdkconfig) test_fragment = self.create_fragment_file(u""" [mapping:test] entries: * (default) """) - - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) + with self.assertRaises(ParseException): + parse_fragment_file(test_fragment, self.sdkconfig) test_fragment = self.create_fragment_file(u""" [mapping:test] @@ -749,9 +584,8 @@ archive: lib.a entries: obj: (noflash) """) - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) test_fragment = self.create_fragment_file(u""" [mapping:test] @@ -759,9 +593,8 @@ archive: lib.a entries: obj: () """) - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) test_fragment = self.create_fragment_file(u""" [mapping:test] @@ -769,9 +602,8 @@ archive: lib.a entries: obj:symbol """) - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) test_fragment = self.create_fragment_file(u""" [mapping:test] @@ -779,9 +611,8 @@ archive: lib.a entries: (noflash) """) - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) test_fragment = self.create_fragment_file(u""" [mapping:test] @@ -789,9 +620,8 @@ archive: lib.a entries: obj:* (noflash) """) - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) test_fragment = self.create_fragment_file(u""" [mapping:test] @@ -799,9 +629,8 @@ archive: lib.a entries: :symbol (noflash) """) - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) test_fragment = self.create_fragment_file(u""" [mapping:test] @@ -809,9 +638,8 @@ archive: lib.a entries: *:symbol (noflash) """) - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) def test_keep_flag(self): # Test parsing combinations and orders of flags @@ -823,12 +651,12 @@ entries: text->flash_text KEEP(), rodata->flash_rodata KEEP() KEEP() """) - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) fragment = fragment_file.fragments[0] - expected = [('text', 'flash_text', [Mapping.Keep()]), - ('rodata', 'flash_rodata', [Mapping.Keep(), Mapping.Keep()])] + expected = [Flag('text', 'flash_text', [Keep()]), + Flag('rodata', 'flash_rodata', [Keep(), Keep()])] actual = fragment.flags[('obj1', None, 'default')] self.assertEqual(expected, actual) @@ -847,14 +675,14 @@ entries: common->dram0_bss ALIGN(8, pre, post) ALIGN(8) """) - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) fragment = fragment_file.fragments[0] - expected = [('text', 'flash_text', [Mapping.Align(8, True, False)]), - ('rodata', 'flash_rodata', [Mapping.Align(8, True, False)]), - ('data', 'dram0_data', [Mapping.Align(8, True, True)]), - ('bss', 'dram0_bss', [Mapping.Align(8, False, True)]), - ('common', 'dram0_bss', [Mapping.Align(8, True, True), Mapping.Align(8, True, False)])] + expected = [Flag('text', 'flash_text', [Align(8, True, False)]), + Flag('rodata', 'flash_rodata', [Align(8, True, False)]), + Flag('data', 'dram0_data', [Align(8, True, True)]), + Flag('bss', 'dram0_bss', [Align(8, False, True)]), + Flag('common', 'dram0_bss', [Align(8, True, True), Align(8, True, False)])] actual = fragment.flags[('obj1', None, 'default')] self.assertEqual(expected, actual) @@ -868,8 +696,8 @@ entries: text->iram0_text ALIGN(8, post, pre) """) - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) + with self.assertRaises(ParseException): + parse_fragment_file(test_fragment, self.sdkconfig) def test_sort_flag(self): # Test parsing combinations and orders of flags @@ -887,27 +715,19 @@ entries: dram->dram0_data SORT(alignment, alignment) """) - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) fragment = fragment_file.fragments[0] - expected = [('text', 'flash_text', [Mapping.Sort('name')]), - ('rodata', 'flash_rodata', [Mapping.Sort('alignment')]), - ('data', 'dram0_data', [Mapping.Sort('init_priority')]), - ('bss', 'dram0_bss', [Mapping.Sort('name', 'alignment')]), - ('common', 'dram0_bss', [Mapping.Sort('alignment', 'name')]), - ('iram', 'iram0_text', [Mapping.Sort('name', 'name')]), - ('dram', 'dram0_data', [Mapping.Sort('alignment', 'alignment')])] + expected = [Flag('text', 'flash_text', [Sort('name')]), + Flag('rodata', 'flash_rodata', [Sort('alignment')]), + Flag('data', 'dram0_data', [Sort('init_priority')]), + Flag('bss', 'dram0_bss', [Sort('name', 'alignment')]), + Flag('common', 'dram0_bss', [Sort('alignment', 'name')]), + Flag('iram', 'iram0_text', [Sort('name', 'name')]), + Flag('dram', 'dram0_data', [Sort('alignment', 'alignment')])] actual = fragment.flags[('obj1', None, 'default')] self.assertEqual(expected, actual) - test_fragment = self.create_fragment_file(u""" -[mapping:map] -archive: libmain.a -entries: - obj1 (default) - text->iram0_text SORT(name) SORT(alignment) -""") - def test_surround_flag(self): # Test parsing combinations and orders of flags test_fragment = self.create_fragment_file(u""" @@ -917,11 +737,10 @@ entries: obj1 (default); text->flash_text SURROUND(sym1) """) - - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) fragment = fragment_file.fragments[0] - expected = [('text', 'flash_text', [Mapping.Surround('sym1')])] + expected = [Flag('text', 'flash_text', [Surround('sym1')])] actual = fragment.flags[('obj1', None, 'default')] self.assertEqual(expected, actual) @@ -935,21 +754,21 @@ entries: text->flash_text ALIGN(4) KEEP() SURROUND(sym1) ALIGN(8) SORT(name), rodata->flash_rodata KEEP() ALIGN(4) KEEP() SURROUND(sym1) ALIGN(8) ALIGN(4) SORT(name) """) - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) fragment = fragment_file.fragments[0] - expected = [('text', 'flash_text', [Mapping.Align(4, True, False), - Mapping.Keep(), - Mapping.Surround('sym1'), - Mapping.Align(8, True, False), - Mapping.Sort('name')]), - ('rodata', 'flash_rodata', [Mapping.Keep(), - Mapping.Align(4, True, False), - Mapping.Keep(), - Mapping.Surround('sym1'), - Mapping.Align(8, True, False), - Mapping.Align(4, True, False), - Mapping.Sort('name')])] + expected = [Flag('text', 'flash_text', [Align(4, True, False), + Keep(), + Surround('sym1'), + Align(8, True, False), + Sort('name')]), + Flag('rodata', 'flash_rodata', [Keep(), + Align(4, True, False), + Keep(), + Surround('sym1'), + Align(8, True, False), + Align(4, True, False), + Sort('name')])] actual = fragment.flags[('obj1', None, 'default')] self.assertEqual(expected, actual) @@ -965,17 +784,17 @@ entries: text->flash_text ALIGN(4) KEEP() SURROUND(sym1) SORT(name), text->flash_text ALIGN(4) KEEP() SURROUND(sym1) SORT(name) """) - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) fragment = fragment_file.fragments[0] - expected = [('text', 'flash_text', [Mapping.Align(4, True, False), - Mapping.Keep(), - Mapping.Surround('sym1'), - Mapping.Sort('name')]), - ('text', 'flash_text', [Mapping.Align(4, True, False), - Mapping.Keep(), - Mapping.Surround('sym1'), - Mapping.Sort('name')])] + expected = [Flag('text', 'flash_text', [Align(4, True, False), + Keep(), + Surround('sym1'), + Sort('name')]), + Flag('text', 'flash_text', [Align(4, True, False), + Keep(), + Surround('sym1'), + Sort('name')])] actual = fragment.flags[('obj1', None, 'default')] self.assertEqual(expected, actual) @@ -993,261 +812,20 @@ entries: obj1 (default); text->flash_text ALIGN(4) KEEP() SURROUND(sym1) SORT(name) """) - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) fragment = fragment_file.fragments[0] - expected = [('text', 'flash_text', [Mapping.Align(4, True, False), - Mapping.Keep(), - Mapping.Surround('sym1'), - Mapping.Sort('name')]), - ('text', 'flash_text', [Mapping.Align(4, True, False), - Mapping.Keep(), - Mapping.Surround('sym1'), - Mapping.Sort('name')])] + expected = [Flag('text', 'flash_text', [Align(4, True, False), + Keep(), + Surround('sym1'), + Sort('name')]), + Flag('text', 'flash_text', [Align(4, True, False), + Keep(), + Surround('sym1'), + Sort('name')])] actual = fragment.flags[('obj1', None, 'default')] self.assertEqual(expected, actual) -class DeprecatedMappingTest(FragmentTest): - - def test_valid_grammar(self): - test_fragment = self.create_fragment_file(u""" -[mapping] -archive: lib.a -entries: - obj:symbol (noflash) - # Comments should not matter - obj (noflash) - # Nor should whitespace - obj : symbol_2 ( noflash ) - obj_2 ( noflash ) - * (noflash) -""") - fragment_file = FragmentFile(test_fragment, self.sdkconfig) - self.assertEqual('lib.a', fragment_file.fragments[0].archive) - self.assertEqual('lib_a', fragment_file.fragments[0].name) - - expected = {('obj', 'symbol', 'noflash'), - ('obj', None, 'noflash'), - ('obj', 'symbol_2', 'noflash'), - ('obj_2', None, 'noflash'), - ('*', None, 'noflash') - } - - self.assertEqual(expected, fragment_file.fragments[0].entries) - - def test_explicit_blank_default_w_others(self): - test_fragment = self.create_fragment_file(u""" -[mapping] -archive: lib.a -entries: - : A = n - obj_a (noflash) - : default -""") - fragment_file = FragmentFile(test_fragment, self.sdkconfig) - expected = {('*', None, 'default')} - - self.assertEqual(expected, fragment_file.fragments[0].entries) - - def test_implicit_blank_default_w_others(self): - test_fragment = self.create_fragment_file(u""" -[mapping] -archive: lib.a -entries: - : A = n - obj_a (noflash) -""") - - fragment_file = FragmentFile(test_fragment, self.sdkconfig) - expected = {('*', None, 'default')} - - self.assertEqual(expected, fragment_file.fragments[0].entries) - - def test_explicit_blank_default(self): - test_fragment = self.create_fragment_file(u""" -[mapping] -archive: lib.a -entries: - : default -""") - fragment_file = FragmentFile(test_fragment, self.sdkconfig) - expected = {('*', None, 'default')} - - self.assertEqual(expected, fragment_file.fragments[0].entries) - - def test_implicit_blank_default(self): - test_fragment = self.create_fragment_file(u""" -[mapping] -archive: lib.a -entries: - : default -""") - fragment_file = FragmentFile(test_fragment, self.sdkconfig) - expected = {('*', None, 'default')} - - self.assertEqual(expected, fragment_file.fragments[0].entries) - - def test_multiple_entries(self): - test_fragment = self.create_fragment_file(u""" -[mapping] -archive: lib.a -entries: - : A = n - obj_a1 (noflash) - obj_a2 (noflash) - : B = n - obj_b1 (noflash) - obj_b2 (noflash) - obj_b3 (noflash) - : C = n - obj_c1 (noflash) -""") - - fragment_file = FragmentFile(test_fragment, self.sdkconfig) - expected = {('obj_b1', None, 'noflash'), - ('obj_b2', None, 'noflash'), - ('obj_b3', None, 'noflash')} - self.assertEqual(expected, fragment_file.fragments[0].entries) - - def test_blank_entries(self): - test_fragment = self.create_fragment_file(u""" -[mapping] -archive: lib.a -entries: - : A = n - obj_a (noflash) - : B = n - : C = n - obj_c (noflash) - : default - obj (noflash) -""") - fragment_file = FragmentFile(test_fragment, self.sdkconfig) - expected = {('*', None, 'default')} - self.assertEqual(expected, fragment_file.fragments[0].entries) - - def test_blank_first_condition(self): - test_fragment = self.create_fragment_file(u""" -[mapping] -archive: lib.a -entries: - obj_a (noflash) - : CONFIG_B = y - obj_b (noflash) -""") - - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) - - def test_nonlast_default_1(self): - test_fragment = self.create_fragment_file(u""" -[mapping] -archive: lib.a -entries: - : default - obj_a (noflash) - : CONFIG_A = y - obj_A (noflash) -""") - - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) - - def test_nonlast_default_2(self): - test_fragment = self.create_fragment_file(u""" -[mapping] -archive: lib.a -entries: - : A = y - obj_A (noflash) - : default - obj_a (noflash) - : B = y - obj_B (noflash -""") - - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) - - def test_nonlast_default_3(self): - test_fragment = self.create_fragment_file(u""" -[mapping] -archive: lib.a -entries: - : A = y - obj_A (noflash) - : - obj_a (noflash) - : B = y - obj_B (noflash -""") - - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) - - def test_duplicate_default_1(self): - test_fragment = self.create_fragment_file(u""" -[mapping] -archive: lib.a -entries: - : CONFIG_A = y - obj_A (noflash) - : default - obj_a (noflash) - : CONFIG_B = y - obj_B (noflash) - : default - obj_a (noflash) -""") - - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) - - def test_duplicate_default_2(self): - test_fragment = self.create_fragment_file(u""" -[mapping] -archive: lib.a -entries: - : CONFIG_A = y - obj_A (noflash) - : CONFIG_B = y - obj_a (noflash) - : default - obj_B (noflash) - : - obj_a (noflash) -""") - - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) - - def test_mixed_deprecated_mapping(self): - test_fragment = self.create_fragment_file(u""" -[mapping] -archive: lib.a -entries: - : A = n - obj_A (noflash) - : default - obj_B (noflash) - - -[mapping:test] -archive: lib.a -entries: - if A = n: - obj_A (noflash) - else: - obj_B (noflash) -""") - - fragment_file = FragmentFile(test_fragment, self.sdkconfig) - self.assertEqual(2, len(fragment_file.fragments)) - - self.assertEqual(fragment_file.fragments[0].entries, - fragment_file.fragments[1].entries) - - if __name__ == '__main__': unittest.main() diff --git a/tools/ldgen/test/test_generation.py b/tools/ldgen/test/test_generation.py index b4482ae0e5..2ccfd7d70b 100755 --- a/tools/ldgen/test/test_generation.py +++ b/tools/ldgen/test/test_generation.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # @@ -10,20 +10,23 @@ import os import sys import tempfile import unittest - -try: - from ldgen.generation import Generation, GenerationException -except ImportError: - sys.path.append('../') - from ldgen.generation import Generation, GenerationException - from io import StringIO -from ldgen.entity import Entity, EntityDB -from ldgen.fragments import FragmentFile -from ldgen.linker_script import LinkerScript -from ldgen.output_commands import AlignAtAddress, InputSectionDesc, SymbolAtAddress -from ldgen.sdkconfig import SDKConfig +try: + from ldgen.entity import Entity, EntityDB + from ldgen.fragments import parse_fragment_file + from ldgen.generation import Generation, GenerationException + from ldgen.linker_script import LinkerScript + from ldgen.output_commands import AlignAtAddress, InputSectionDesc, SymbolAtAddress + from ldgen.sdkconfig import SDKConfig +except ImportError: + sys.path.append(os.path.dirname(os.path.dirname(__file__))) + from ldgen.entity import Entity, EntityDB + from ldgen.fragments import parse_fragment_file + from ldgen.generation import Generation, GenerationException + from ldgen.linker_script import LinkerScript + from ldgen.output_commands import AlignAtAddress, InputSectionDesc, SymbolAtAddress + from ldgen.sdkconfig import SDKConfig ROOT = Entity('*') @@ -58,9 +61,8 @@ class GenerationTest(unittest.TestCase): self.sdkconfig = SDKConfig('data/Kconfig', 'data/sdkconfig') - with open('data/base.lf') as fragment_file_obj: - fragment_file = FragmentFile(fragment_file_obj, self.sdkconfig) - self.generation.add_fragments_from_file(fragment_file) + fragment_file = parse_fragment_file('data/base.lf', self.sdkconfig) + self.generation.add_fragments_from_file(fragment_file) self.entities = EntityDB() @@ -78,7 +80,7 @@ class GenerationTest(unittest.TestCase): def add_fragments(self, text): fragment_file = self.create_fragment_file(text) - fragment_file = FragmentFile(fragment_file, self.sdkconfig) + fragment_file = parse_fragment_file(fragment_file, self.sdkconfig) self.generation.add_fragments_from_file(fragment_file) def write(self, expected, actual): @@ -1062,43 +1064,6 @@ entries: with self.assertRaises(GenerationException): self.generation.generate(self.entities) - def test_disambiguated_obj(self): - # Test command generation for disambiguated entry. Should produce similar - # results to test_nondefault_mapping_symbol. - mapping = u""" -[mapping:test] -archive: libfreertos.a -entries: - port.c:xPortGetTickRateHz (noflash) #1 -""" - port = Entity('libfreertos.a', 'port.c') - self.add_fragments(mapping) - actual = self.generation.generate(self.entities) - expected = self.generate_default_rules() - - flash_text = expected['flash_text'] - iram0_text = expected['iram0_text'] - - # Generate exclusion in flash_text A - flash_text[0].exclusions.add(port) - - # Generate intermediate command B - # List all relevant sections except the symbol - # being mapped - port_sections = self.entities.get_sections('libfreertos.a', 'port.c') - filtered_sections = fnmatch.filter(port_sections, '.literal.*') - filtered_sections.extend(fnmatch.filter(port_sections, '.text.*')) - - filtered_sections = [s for s in filtered_sections if not s.endswith('xPortGetTickRateHz')] - filtered_sections.append('.text') - - flash_text.append(InputSectionDesc(port, set(filtered_sections), [])) - - # Input section commands in iram_text for #1 C - iram0_text.append(InputSectionDesc(port, set(['.text.xPortGetTickRateHz', '.literal.xPortGetTickRateHz']), [])) - - self.compare_rules(expected, actual) - def test_root_mapping_fragment_conflict(self): # Test that root mapping fragments are also checked for # conflicts. @@ -1258,84 +1223,6 @@ entries: self.compare_rules(expected, actual) - def test_conditional_on_scheme_legacy_mapping_00(self): - # Test use of conditional scheme on legacy mapping fragment grammar. - mapping = u""" -[mapping] -archive: lib.a -entries: - * (cond_noflash) -""" - self._test_conditional_on_scheme(0, mapping) - - def test_conditional_on_scheme_legacy_mapping_01(self): - # Test use of conditional scheme on legacy mapping fragment grammar. - mapping = u""" -[mapping] -archive: lib.a -entries: - * (cond_noflash) -""" - self._test_conditional_on_scheme(0, mapping) - - def test_conditional_entries_legacy_mapping_fragment(self): - # Test conditional entries on legacy mapping fragment grammar. - mapping = u""" -[mapping:default] -archive: * -entries: - * (default) - -[mapping] -archive: lib.a -entries: - : PERFORMANCE_LEVEL = 0 - : PERFORMANCE_LEVEL = 1 - obj1 (noflash) - : PERFORMANCE_LEVEL = 2 - obj1 (noflash) - obj2 (noflash) - : PERFORMANCE_LEVEL = 3 - obj1 (noflash) - obj2 (noflash) - obj3 (noflash) -""" - self.test_conditional_mapping(mapping) - - def test_multiple_fragment_same_lib_conditional_legacy(self): - # Test conditional entries on legacy mapping fragment grammar - # across multiple fragments. - mapping = u""" -[mapping:default] -archive: * -entries: - * (default) - -[mapping] -archive: lib.a -entries: - : PERFORMANCE_LEVEL = 0 - : PERFORMANCE_LEVEL = 1 - obj1 (noflash) - : PERFORMANCE_LEVEL = 2 - obj1 (noflash) - : PERFORMANCE_LEVEL = 3 - obj1 (noflash) - -[mapping] -archive: lib.a -entries: - : PERFORMANCE_LEVEL = 1 - obj1 (noflash) # ignore duplicate definition - : PERFORMANCE_LEVEL = 2 - obj2 (noflash) - : PERFORMANCE_LEVEL = 3 - obj2 (noflash) - obj3 (noflash) -""" - - self.test_conditional_mapping(mapping) - def test_multiple_fragment_same_lib_conditional(self): # Test conditional entries on new mapping fragment grammar. # across multiple fragments. diff --git a/tools/ldgen/test/test_output_commands.py b/tools/ldgen/test/test_output_commands.py index 291ef9094a..4a6723626d 100755 --- a/tools/ldgen/test/test_output_commands.py +++ b/tools/ldgen/test/test_output_commands.py @@ -1,9 +1,10 @@ #!/usr/bin/env python # -# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # +import os import sys import unittest @@ -11,7 +12,7 @@ try: from ldgen.entity import Entity from ldgen.output_commands import AlignAtAddress, InputSectionDesc, SymbolAtAddress except ImportError: - sys.path.append('../') + sys.path.append(os.path.dirname(os.path.dirname(__file__))) from ldgen.entity import Entity from ldgen.output_commands import AlignAtAddress, InputSectionDesc, SymbolAtAddress