From 84d5cc1c179c5306600233b55af3dcd04492c11e Mon Sep 17 00:00:00 2001 From: Roland Dobai Date: Mon, 21 Oct 2019 13:44:19 +0200 Subject: [PATCH] Generate source files for kconfiglib from the build system --- docs/conf_common.py | 9 +++ make/ldgen.mk | 4 +- make/project_config.mk | 14 +++- tools/cmake/kconfig.cmake | 10 +++ tools/kconfig_new/confgen.py | 22 ----- tools/kconfig_new/confserver.py | 1 - tools/kconfig_new/prepare_kconfig_files.py | 94 ++++++++++++++++++++++ tools/kconfig_new/test/test_confserver.py | 3 + tools/ldgen/sdkconfig.py | 3 - tools/ldgen/test/test_fragments.py | 3 + tools/ldgen/test/test_generation.py | 3 + 11 files changed, 135 insertions(+), 31 deletions(-) create mode 100644 tools/kconfig_new/prepare_kconfig_files.py diff --git a/docs/conf_common.py b/docs/conf_common.py index 4c992f57f8..708321af77 100644 --- a/docs/conf_common.py +++ b/docs/conf_common.py @@ -103,6 +103,15 @@ sdkconfig_renames = [r for r in sdkconfig_renames if "esp32s2beta" not in r] kconfigs_source_path = '{}/inc/kconfigs_source.in'.format(builddir) kconfig_projbuilds_source_path = '{}/inc/kconfig_projbuilds_source.in'.format(builddir) +prepare_kconfig_files_args = [sys.executable, + "../../tools/kconfig_new/prepare_kconfig_files.py", + "--env", "COMPONENT_KCONFIGS={}".format(" ".join(kconfigs)), + "--env", "COMPONENT_KCONFIGS_PROJBUILD={}".format(" ".join(kconfig_projbuilds)), + "--env", "COMPONENT_KCONFIGS_SOURCE_FILE={}".format(kconfigs_source_path), + "--env", "COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE={}".format(kconfig_projbuilds_source_path), + ] +subprocess.check_call(prepare_kconfig_files_args) + confgen_args = [sys.executable, "../../tools/kconfig_new/confgen.py", "--kconfig", "../../Kconfig", diff --git a/make/ldgen.mk b/make/ldgen.mk index 561f591cb4..bd03e20670 100644 --- a/make/ldgen.mk +++ b/make/ldgen.mk @@ -9,7 +9,7 @@ define ldgen_process_template $(BUILD_DIR_BASE)/ldgen_libraries: $(LDGEN_LIBRARIES) $(IDF_PATH)/make/ldgen.mk printf "$(foreach info,$(LDGEN_LIBRARIES),$(subst \,/,$(shell cygpath -m $(info)))\n)" > $(BUILD_DIR_BASE)/ldgen_libraries -$(2): $(1) $(LDGEN_FRAGMENT_FILES) $(SDKCONFIG) $(BUILD_DIR_BASE)/ldgen_libraries +$(2): $(1) $(LDGEN_FRAGMENT_FILES) $(SDKCONFIG) $(BUILD_DIR_BASE)/ldgen_libraries | prepare_kconfig_files @echo 'Generating $(notdir $(2))' $(PYTHON) $(IDF_PATH)/tools/ldgen/ldgen.py \ --input $(1) \ @@ -30,7 +30,7 @@ define ldgen_process_template $(BUILD_DIR_BASE)/ldgen_libraries: $(LDGEN_LIBRARIES) $(IDF_PATH)/make/ldgen.mk printf "$(foreach library,$(LDGEN_LIBRARIES),$(library)\n)" > $(BUILD_DIR_BASE)/ldgen_libraries -$(2): $(1) $(LDGEN_FRAGMENT_FILES) $(SDKCONFIG) $(BUILD_DIR_BASE)/ldgen_libraries +$(2): $(1) $(LDGEN_FRAGMENT_FILES) $(SDKCONFIG) $(BUILD_DIR_BASE)/ldgen_libraries | prepare_kconfig_files @echo 'Generating $(notdir $(2))' $(PYTHON) $(IDF_PATH)/tools/ldgen/ldgen.py \ --input $(1) \ diff --git a/make/project_config.mk b/make/project_config.mk index 912d207367..b05a4a461a 100644 --- a/make/project_config.mk +++ b/make/project_config.mk @@ -70,6 +70,14 @@ SDKCONFIG_DEFAULTS_FILES := $(shell cygpath -m $(SDKCONFIG_DEFAULTS_FILES)) endif DEFAULTS_ARG := $(foreach f,$(SDKCONFIG_DEFAULTS_FILES),--defaults $(f)) +prepare_kconfig_files: + mkdir -p $(BUILD_DIR_BASE) + $(PYTHON) $(IDF_PATH)/tools/kconfig_new/prepare_kconfig_files.py \ + --env "COMPONENT_KCONFIGS=$(strip $(COMPONENT_KCONFIGS))" \ + --env "COMPONENT_KCONFIGS_PROJBUILD=$(strip $(COMPONENT_KCONFIGS_PROJBUILD))" \ + --env "COMPONENT_KCONFIGS_SOURCE_FILE=$(COMPONENT_KCONFIGS_SOURCE_FILE)" \ + --env "COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE=$(COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE)" + # macro for running confgen.py define RunConfGen mkdir -p $(BUILD_DIR_BASE)/include/config @@ -118,7 +126,7 @@ ifndef MAKE_RESTARTS # depend on any prerequisite that may cause a make restart as part of # the prerequisite's own recipe. -menuconfig: $(KCONFIG_TOOL_DIR)/mconf-idf | check_python_dependencies +menuconfig: $(KCONFIG_TOOL_DIR)/mconf-idf | check_python_dependencies prepare_kconfig_files $(summary) MENUCONFIG ifdef BATCH_BUILD @echo "Can't run interactive configuration inside non-interactive build process." @@ -135,13 +143,13 @@ else endif # defconfig creates a default config, based on SDKCONFIG_DEFAULTS if present -defconfig: | check_python_dependencies +defconfig: | check_python_dependencies prepare_kconfig_files $(summary) DEFCONFIG $(call RunConfGen) # if neither defconfig or menuconfig are requested, use the GENCONFIG rule to # ensure generated config files are up to date -$(SDKCONFIG_MAKEFILE) $(BUILD_DIR_BASE)/include/sdkconfig.h: $(SDKCONFIG) $(COMPONENT_KCONFIGS) $(COMPONENT_KCONFIGS_PROJBUILD) | check_python_dependencies $(call prereq_if_explicit,defconfig) $(call prereq_if_explicit,menuconfig) +$(SDKCONFIG_MAKEFILE) $(BUILD_DIR_BASE)/include/sdkconfig.h: $(SDKCONFIG) $(COMPONENT_KCONFIGS) $(COMPONENT_KCONFIGS_PROJBUILD) | check_python_dependencies prepare_kconfig_files $(call prereq_if_explicit,defconfig) $(call prereq_if_explicit,menuconfig) $(summary) GENCONFIG $(call RunConfGen) touch $(SDKCONFIG_MAKEFILE) $(BUILD_DIR_BASE)/include/sdkconfig.h # ensure newer than sdkconfig diff --git a/tools/cmake/kconfig.cmake b/tools/cmake/kconfig.cmake index e6226f6edb..3bc476b255 100644 --- a/tools/cmake/kconfig.cmake +++ b/tools/cmake/kconfig.cmake @@ -175,6 +175,10 @@ function(__kconfig_generate_config sdkconfig sdkconfig_defaults) idf_build_get_property(root_sdkconfig_rename __ROOT_SDKCONFIG_RENAME) idf_build_get_property(python PYTHON) + set(prepare_kconfig_files_command + ${python} ${idf_path}/tools/kconfig_new/prepare_kconfig_files.py + --env-file ${config_env_path}) + set(confgen_basecommand ${python} ${idf_path}/tools/kconfig_new/confgen.py --kconfig ${root_kconfig} @@ -195,6 +199,8 @@ function(__kconfig_generate_config sdkconfig sdkconfig_defaults) idf_build_get_property(output_sdkconfig __OUTPUT_SDKCONFIG) if(output_sdkconfig) + execute_process( + COMMAND ${prepare_kconfig_files_command}) execute_process( COMMAND ${confgen_basecommand} --output header ${sdkconfig_header} @@ -204,6 +210,8 @@ function(__kconfig_generate_config sdkconfig sdkconfig_defaults) --output config ${sdkconfig} RESULT_VARIABLE config_result) else() + execute_process( + COMMAND ${prepare_kconfig_files_command}) execute_process( COMMAND ${confgen_basecommand} --output header ${sdkconfig_header} @@ -254,6 +262,7 @@ function(__kconfig_generate_config sdkconfig sdkconfig_defaults) add_custom_target(menuconfig ${menuconfig_depends} # create any missing config file, with defaults if necessary + COMMAND ${prepare_kconfig_files_command} COMMAND ${confgen_basecommand} --env "IDF_TARGET=${idf_target}" --output config ${sdkconfig} COMMAND ${CMAKE_COMMAND} -E env "COMPONENT_KCONFIGS_SOURCE_FILE=${kconfigs_path}" @@ -273,6 +282,7 @@ function(__kconfig_generate_config sdkconfig sdkconfig_defaults) # Custom target to run confserver.py from the build tool add_custom_target(confserver + COMMAND ${prepare_kconfig_files_command} COMMAND ${PYTHON} ${IDF_PATH}/tools/kconfig_new/confserver.py --env-file ${config_env_path} --kconfig ${IDF_PATH}/Kconfig diff --git a/tools/kconfig_new/confgen.py b/tools/kconfig_new/confgen.py index 0aff083fc2..185aa2903d 100755 --- a/tools/kconfig_new/confgen.py +++ b/tools/kconfig_new/confgen.py @@ -175,27 +175,6 @@ class DeprecatedOptions(object): f_o.write('#define {}{} {}{}\n'.format(self.config_prefix, dep_opt, self.config_prefix, new_opt)) -def prepare_source_files(): - """ - Prepares source files which are sourced from the main Kconfig because upstream kconfiglib doesn't support sourcing - a file list. - """ - - def _dequote(var): - return var[1:-1] if len(var) > 0 and (var[0], var[-1]) == ('"',) * 2 else var - - def _write_source_file(config_var, config_file): - with open(config_file, "w") as f: - f.write('\n'.join(['source "{}"'.format(path) for path in _dequote(config_var).split()])) - - try: - _write_source_file(os.environ['COMPONENT_KCONFIGS'], os.environ['COMPONENT_KCONFIGS_SOURCE_FILE']) - _write_source_file(os.environ['COMPONENT_KCONFIGS_PROJBUILD'], os.environ['COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE']) - except KeyError as e: - print('Error:', e, 'is not defined!') - raise - - def dict_enc_for_env(dic, encoding=sys.getfilesystemencoding() or 'utf-8'): """ This function can be deleted after dropping support for Python 2. @@ -266,7 +245,6 @@ def main(): env = json.load(args.env_file) os.environ.update(dict_enc_for_env(env)) - prepare_source_files() config = kconfiglib.Kconfig(args.kconfig) config.warn_assign_redun = False config.warn_assign_override = False diff --git a/tools/kconfig_new/confserver.py b/tools/kconfig_new/confserver.py index fbe5c78a0d..92f25c76a9 100755 --- a/tools/kconfig_new/confserver.py +++ b/tools/kconfig_new/confserver.py @@ -75,7 +75,6 @@ def main(): def run_server(kconfig, sdkconfig, sdkconfig_rename, default_version=MAX_PROTOCOL_VERSION): - confgen.prepare_source_files() config = kconfiglib.Kconfig(kconfig) sdkconfig_renames = [sdkconfig_rename] if sdkconfig_rename else [] sdkconfig_renames += os.environ.get("COMPONENT_SDKCONFIG_RENAMES", "").split() diff --git a/tools/kconfig_new/prepare_kconfig_files.py b/tools/kconfig_new/prepare_kconfig_files.py new file mode 100644 index 0000000000..913c27c9c8 --- /dev/null +++ b/tools/kconfig_new/prepare_kconfig_files.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python +# +# Copyright 2019 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function +from __future__ import unicode_literals +from io import open +import argparse +import json +import sys + + +def _prepare_source_files(env_dict): + """ + Prepares source files which are sourced from the main Kconfig because upstream kconfiglib doesn't support sourcing + a file list. The inputs are the same environment variables which are used by kconfiglib: + - COMPONENT_KCONFIGS, + - COMPONENT_KCONFIGS_SOURCE_FILE, + - COMPONENT_KCONFIGS_PROJBUILD, + - COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE. + + The outputs are written into files pointed by the value of + - COMPONENT_KCONFIGS_SOURCE_FILE, + - COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE, + + After running this function, COMPONENT_KCONFIGS_SOURCE_FILE and COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE will + contain a list of source statements based on the content of COMPONENT_KCONFIGS and COMPONENT_KCONFIGS_PROJBUILD, + respectively. For example, if COMPONENT_KCONFIGS="var1 var2 var3" and + COMPONENT_KCONFIGS_SOURCE_FILE="/path/file.txt" then the content of file /path/file.txt will be: + source "var1" + source "var2" + source "var3" + """ + + def _dequote(var): + return var[1:-1] if len(var) > 0 and (var[0], var[-1]) == ('"',) * 2 else var + + def _write_source_file(config_var, config_file): + new_content = '\n'.join(['source "{}"'.format(path) for path in _dequote(config_var).split()]) + try: + with open(config_file, 'r', encoding='utf-8') as f: + old_content = f.read() + except Exception: + # File doesn't exist or other issue + old_content = '' + + if new_content != old_content: + # write or rewrite file only if it is necessary + with open(config_file, 'w', encoding='utf-8') as f: + f.write(new_content) + + try: + _write_source_file(env_dict['COMPONENT_KCONFIGS'], env_dict['COMPONENT_KCONFIGS_SOURCE_FILE']) + _write_source_file(env_dict['COMPONENT_KCONFIGS_PROJBUILD'], env_dict['COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE']) + except KeyError as e: + print('Error:', e, 'is not defined!') + sys.exit(1) + + +if __name__ == '__main__': + + parser = argparse.ArgumentParser(description='Kconfig Source File Generator') + + parser.add_argument('--env', action='append', default=[], + help='Environment value', metavar='NAME=VAL') + + parser.add_argument('--env-file', type=argparse.FileType('r'), + help='Optional file to load environment variables from. Contents ' + 'should be a JSON object where each key/value pair is a variable.') + + args = parser.parse_args() + + try: + env = dict([(name, value) for (name, value) in (e.split("=", 1) for e in args.env)]) + except ValueError: + print("--env arguments must each contain =.") + sys.exit(1) + + if args.env_file is not None: + env.update(json.load(args.env_file)) + + _prepare_source_files(env) diff --git a/tools/kconfig_new/test/test_confserver.py b/tools/kconfig_new/test/test_confserver.py index 84a9caa453..e7da1f8c80 100755 --- a/tools/kconfig_new/test/test_confserver.py +++ b/tools/kconfig_new/test/test_confserver.py @@ -66,6 +66,9 @@ def main(): cmdline = re.sub(r' +', ' ', cmdline) + # prepare_kconfig_files.py doesn't have to be called because COMPONENT_KCONFIGS and + # COMPONENT_KCONFIGS_PROJBUILD are empty + print("Running: %s" % cmdline) p = pexpect.spawn(cmdline, timeout=30, logfile=args.logfile, echo=False, use_poll=True, maxread=1) diff --git a/tools/ldgen/sdkconfig.py b/tools/ldgen/sdkconfig.py index e4476d592a..28ea6b0f45 100644 --- a/tools/ldgen/sdkconfig.py +++ b/tools/ldgen/sdkconfig.py @@ -26,8 +26,6 @@ except Exception: sys.path.insert(0, kconfig_new_dir) import kconfiglib -import confgen - class SDKConfig: """ @@ -49,7 +47,6 @@ class SDKConfig: OPERATOR = oneOf(["=", "!=", ">", "<", "<=", ">="]) def __init__(self, kconfig_file, sdkconfig_file): - confgen.prepare_source_files() self.config = kconfiglib.Kconfig(kconfig_file) self.config.load_config(sdkconfig_file) diff --git a/tools/ldgen/test/test_fragments.py b/tools/ldgen/test/test_fragments.py index 04080df43d..f0b5cf574f 100755 --- a/tools/ldgen/test/test_fragments.py +++ b/tools/ldgen/test/test_fragments.py @@ -69,6 +69,9 @@ class FragmentTest(unittest.TestCase): os.environ['COMPONENT_KCONFIGS'] = '' os.environ['COMPONENT_KCONFIGS_PROJBUILD'] = '' + # prepare_kconfig_files.py doesn't have to be called because COMPONENT_KCONFIGS and + # COMPONENT_KCONFIGS_PROJBUILD are empty + self.sdkconfig = SDKConfig("data/Kconfig", "data/sdkconfig") def tearDown(self): diff --git a/tools/ldgen/test/test_generation.py b/tools/ldgen/test/test_generation.py index 9846d8e30b..7be3c3ad5d 100755 --- a/tools/ldgen/test/test_generation.py +++ b/tools/ldgen/test/test_generation.py @@ -56,6 +56,9 @@ class GenerationModelTest(unittest.TestCase): os.environ['COMPONENT_KCONFIGS'] = '' os.environ['COMPONENT_KCONFIGS_PROJBUILD'] = '' + # prepare_kconfig_files.py doesn't have to be called because COMPONENT_KCONFIGS and + # COMPONENT_KCONFIGS_PROJBUILD are empty + self.sdkconfig = SDKConfig("data/Kconfig", "data/sdkconfig") with open("data/sample.lf") as fragment_file_obj: