Generate source files for kconfiglib from the build system

docs/conf_common.py

@@ -103,6 +103,15 @@ sdkconfig_renames = [r for r in sdkconfig_renames if "esp32s2beta" not in r]
 kconfigs_source_path = '{}/inc/kconfigs_source.in'.format(builddir)
 kconfig_projbuilds_source_path = '{}/inc/kconfig_projbuilds_source.in'.format(builddir)
 
+prepare_kconfig_files_args = [sys.executable,
+                              "../../tools/kconfig_new/prepare_kconfig_files.py",
+                              "--env", "COMPONENT_KCONFIGS={}".format(" ".join(kconfigs)),
+                              "--env", "COMPONENT_KCONFIGS_PROJBUILD={}".format(" ".join(kconfig_projbuilds)),
+                              "--env", "COMPONENT_KCONFIGS_SOURCE_FILE={}".format(kconfigs_source_path),
+                              "--env", "COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE={}".format(kconfig_projbuilds_source_path),
+                              ]
+subprocess.check_call(prepare_kconfig_files_args)
+
 confgen_args = [sys.executable,
                 "../../tools/kconfig_new/confgen.py",
                 "--kconfig", "../../Kconfig",

make/ldgen.mk

@@ -9,7 +9,7 @@ define ldgen_process_template
 $(BUILD_DIR_BASE)/ldgen_libraries: $(LDGEN_LIBRARIES) $(IDF_PATH)/make/ldgen.mk
 	printf "$(foreach info,$(LDGEN_LIBRARIES),$(subst \,/,$(shell cygpath -m $(info)))\n)" > $(BUILD_DIR_BASE)/ldgen_libraries
 
-$(2): $(1) $(LDGEN_FRAGMENT_FILES) $(SDKCONFIG) $(BUILD_DIR_BASE)/ldgen_libraries
+$(2): $(1) $(LDGEN_FRAGMENT_FILES) $(SDKCONFIG) $(BUILD_DIR_BASE)/ldgen_libraries | prepare_kconfig_files
 	@echo 'Generating $(notdir $(2))'
 	$(PYTHON) $(IDF_PATH)/tools/ldgen/ldgen.py \
 		--input         $(1) \
@@ -30,7 +30,7 @@ define ldgen_process_template
 $(BUILD_DIR_BASE)/ldgen_libraries: $(LDGEN_LIBRARIES) $(IDF_PATH)/make/ldgen.mk
 	printf "$(foreach library,$(LDGEN_LIBRARIES),$(library)\n)" > $(BUILD_DIR_BASE)/ldgen_libraries
 
-$(2): $(1) $(LDGEN_FRAGMENT_FILES) $(SDKCONFIG) $(BUILD_DIR_BASE)/ldgen_libraries
+$(2): $(1) $(LDGEN_FRAGMENT_FILES) $(SDKCONFIG) $(BUILD_DIR_BASE)/ldgen_libraries | prepare_kconfig_files
 	@echo 'Generating $(notdir $(2))'
 	$(PYTHON) $(IDF_PATH)/tools/ldgen/ldgen.py \
 		--input         $(1) \

make/project_config.mk

@@ -70,6 +70,14 @@ SDKCONFIG_DEFAULTS_FILES := $(shell cygpath -m $(SDKCONFIG_DEFAULTS_FILES))
 endif
 DEFAULTS_ARG := $(foreach f,$(SDKCONFIG_DEFAULTS_FILES),--defaults $(f))
 
+prepare_kconfig_files:
+	mkdir -p $(BUILD_DIR_BASE)
+	$(PYTHON) $(IDF_PATH)/tools/kconfig_new/prepare_kconfig_files.py \
+		--env "COMPONENT_KCONFIGS=$(strip $(COMPONENT_KCONFIGS))" \
+		--env "COMPONENT_KCONFIGS_PROJBUILD=$(strip $(COMPONENT_KCONFIGS_PROJBUILD))" \
+		--env "COMPONENT_KCONFIGS_SOURCE_FILE=$(COMPONENT_KCONFIGS_SOURCE_FILE)" \
+		--env "COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE=$(COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE)"
+
 # macro for running confgen.py
 define RunConfGen
 	mkdir -p $(BUILD_DIR_BASE)/include/config
@@ -118,7 +126,7 @@ ifndef MAKE_RESTARTS
 # depend on any prerequisite that may cause a make restart as part of
 # the prerequisite's own recipe.
 
-menuconfig: $(KCONFIG_TOOL_DIR)/mconf-idf | check_python_dependencies
+menuconfig: $(KCONFIG_TOOL_DIR)/mconf-idf | check_python_dependencies prepare_kconfig_files
 	$(summary) MENUCONFIG
 ifdef BATCH_BUILD
 	@echo "Can't run interactive configuration inside non-interactive build process."
@@ -135,13 +143,13 @@ else
 endif
 
 # defconfig creates a default config, based on SDKCONFIG_DEFAULTS if present
-defconfig: | check_python_dependencies
+defconfig: | check_python_dependencies prepare_kconfig_files
 	$(summary) DEFCONFIG
 	$(call RunConfGen)
 
 # if neither defconfig or menuconfig are requested, use the GENCONFIG rule to
 # ensure generated config files are up to date
-$(SDKCONFIG_MAKEFILE) $(BUILD_DIR_BASE)/include/sdkconfig.h: $(SDKCONFIG) $(COMPONENT_KCONFIGS) $(COMPONENT_KCONFIGS_PROJBUILD) | check_python_dependencies $(call prereq_if_explicit,defconfig) $(call prereq_if_explicit,menuconfig)
+$(SDKCONFIG_MAKEFILE) $(BUILD_DIR_BASE)/include/sdkconfig.h: $(SDKCONFIG) $(COMPONENT_KCONFIGS) $(COMPONENT_KCONFIGS_PROJBUILD) | check_python_dependencies prepare_kconfig_files $(call prereq_if_explicit,defconfig) $(call prereq_if_explicit,menuconfig)
 	$(summary) GENCONFIG
 	$(call RunConfGen)
 	touch $(SDKCONFIG_MAKEFILE) $(BUILD_DIR_BASE)/include/sdkconfig.h  # ensure newer than sdkconfig

tools/cmake/kconfig.cmake

@@ -175,6 +175,10 @@ function(__kconfig_generate_config sdkconfig sdkconfig_defaults)
     idf_build_get_property(root_sdkconfig_rename __ROOT_SDKCONFIG_RENAME)
     idf_build_get_property(python PYTHON)
 
+    set(prepare_kconfig_files_command
+        ${python} ${idf_path}/tools/kconfig_new/prepare_kconfig_files.py
+        --env-file ${config_env_path})
+
     set(confgen_basecommand
         ${python} ${idf_path}/tools/kconfig_new/confgen.py
         --kconfig ${root_kconfig}
@@ -195,6 +199,8 @@ function(__kconfig_generate_config sdkconfig sdkconfig_defaults)
 
     idf_build_get_property(output_sdkconfig __OUTPUT_SDKCONFIG)
     if(output_sdkconfig)
+        execute_process(
+            COMMAND ${prepare_kconfig_files_command})
         execute_process(
             COMMAND ${confgen_basecommand}
             --output header ${sdkconfig_header}
@@ -204,6 +210,8 @@ function(__kconfig_generate_config sdkconfig sdkconfig_defaults)
             --output config ${sdkconfig}
             RESULT_VARIABLE config_result)
     else()
+        execute_process(
+            COMMAND ${prepare_kconfig_files_command})
         execute_process(
             COMMAND ${confgen_basecommand}
             --output header ${sdkconfig_header}
@@ -254,6 +262,7 @@ function(__kconfig_generate_config sdkconfig sdkconfig_defaults)
     add_custom_target(menuconfig
         ${menuconfig_depends}
         # create any missing config file, with defaults if necessary
+        COMMAND ${prepare_kconfig_files_command}
         COMMAND ${confgen_basecommand} --env "IDF_TARGET=${idf_target}" --output config ${sdkconfig}
         COMMAND ${CMAKE_COMMAND} -E env
         "COMPONENT_KCONFIGS_SOURCE_FILE=${kconfigs_path}"
@@ -273,6 +282,7 @@ function(__kconfig_generate_config sdkconfig sdkconfig_defaults)
 
     # Custom target to run confserver.py from the build tool
     add_custom_target(confserver
+        COMMAND ${prepare_kconfig_files_command}
         COMMAND ${PYTHON} ${IDF_PATH}/tools/kconfig_new/confserver.py
         --env-file ${config_env_path}
         --kconfig ${IDF_PATH}/Kconfig

tools/kconfig_new/confgen.py

@@ -175,27 +175,6 @@ class DeprecatedOptions(object):
                         f_o.write('#define {}{} {}{}\n'.format(self.config_prefix, dep_opt, self.config_prefix, new_opt))
 
 
-def prepare_source_files():
-    """
-    Prepares source files which are sourced from the main Kconfig because upstream kconfiglib doesn't support sourcing
-    a file list.
-    """
-
-    def _dequote(var):
-        return var[1:-1] if len(var) > 0 and (var[0], var[-1]) == ('"',) * 2 else var
-
-    def _write_source_file(config_var, config_file):
-        with open(config_file, "w") as f:
-            f.write('\n'.join(['source "{}"'.format(path) for path in _dequote(config_var).split()]))
-
-    try:
-        _write_source_file(os.environ['COMPONENT_KCONFIGS'], os.environ['COMPONENT_KCONFIGS_SOURCE_FILE'])
-        _write_source_file(os.environ['COMPONENT_KCONFIGS_PROJBUILD'], os.environ['COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE'])
-    except KeyError as e:
-        print('Error:', e, 'is not defined!')
-        raise
-
-
 def dict_enc_for_env(dic, encoding=sys.getfilesystemencoding() or 'utf-8'):
     """
     This function can be deleted after dropping support for Python 2.
@@ -266,7 +245,6 @@ def main():
         env = json.load(args.env_file)
         os.environ.update(dict_enc_for_env(env))
 
-    prepare_source_files()
     config = kconfiglib.Kconfig(args.kconfig)
     config.warn_assign_redun = False
     config.warn_assign_override = False

tools/kconfig_new/confserver.py

@@ -75,7 +75,6 @@ def main():
 
 
 def run_server(kconfig, sdkconfig, sdkconfig_rename, default_version=MAX_PROTOCOL_VERSION):
-    confgen.prepare_source_files()
     config = kconfiglib.Kconfig(kconfig)
     sdkconfig_renames = [sdkconfig_rename] if sdkconfig_rename else []
     sdkconfig_renames += os.environ.get("COMPONENT_SDKCONFIG_RENAMES", "").split()

tools/kconfig_new/prepare_kconfig_files.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python
+#
+# Copyright 2019 Espressif Systems (Shanghai) PTE LTD
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+from __future__ import unicode_literals
+from io import open
+import argparse
+import json
+import sys
+
+
+def _prepare_source_files(env_dict):
+    """
+    Prepares source files which are sourced from the main Kconfig because upstream kconfiglib doesn't support sourcing
+    a file list. The inputs are the same environment variables which are used by kconfiglib:
+        - COMPONENT_KCONFIGS,
+        - COMPONENT_KCONFIGS_SOURCE_FILE,
+        - COMPONENT_KCONFIGS_PROJBUILD,
+        - COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE.
+
+    The outputs are written into files pointed by the value of
+        - COMPONENT_KCONFIGS_SOURCE_FILE,
+        - COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE,
+
+    After running this function, COMPONENT_KCONFIGS_SOURCE_FILE and COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE will
+    contain a list of source statements based on the content of COMPONENT_KCONFIGS and COMPONENT_KCONFIGS_PROJBUILD,
+    respectively. For example, if COMPONENT_KCONFIGS="var1 var2 var3" and
+    COMPONENT_KCONFIGS_SOURCE_FILE="/path/file.txt" then the content of file /path/file.txt will be:
+        source "var1"
+        source "var2"
+        source "var3"
+    """
+
+    def _dequote(var):
+        return var[1:-1] if len(var) > 0 and (var[0], var[-1]) == ('"',) * 2 else var
+
+    def _write_source_file(config_var, config_file):
+        new_content = '\n'.join(['source "{}"'.format(path) for path in _dequote(config_var).split()])
+        try:
+            with open(config_file, 'r', encoding='utf-8') as f:
+                old_content = f.read()
+        except Exception:
+            # File doesn't exist or other issue
+            old_content = ''
+
+        if new_content != old_content:
+            # write or rewrite file only if it is necessary
+            with open(config_file, 'w', encoding='utf-8') as f:
+                f.write(new_content)
+
+    try:
+        _write_source_file(env_dict['COMPONENT_KCONFIGS'], env_dict['COMPONENT_KCONFIGS_SOURCE_FILE'])
+        _write_source_file(env_dict['COMPONENT_KCONFIGS_PROJBUILD'], env_dict['COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE'])
+    except KeyError as e:
+        print('Error:', e, 'is not defined!')
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Kconfig Source File Generator')
+
+    parser.add_argument('--env', action='append', default=[],
+                        help='Environment value', metavar='NAME=VAL')
+
+    parser.add_argument('--env-file', type=argparse.FileType('r'),
+                        help='Optional file to load environment variables from. Contents '
+                             'should be a JSON object where each key/value pair is a variable.')
+
+    args = parser.parse_args()
+
+    try:
+        env = dict([(name, value) for (name, value) in (e.split("=", 1) for e in args.env)])
+    except ValueError:
+        print("--env arguments must each contain =.")
+        sys.exit(1)
+
+    if args.env_file is not None:
+        env.update(json.load(args.env_file))
+
+    _prepare_source_files(env)

tools/kconfig_new/test/test_confserver.py

@@ -66,6 +66,9 @@ def main():
 
         cmdline = re.sub(r' +', ' ', cmdline)
 
+        # prepare_kconfig_files.py doesn't have to be called because COMPONENT_KCONFIGS and
+        # COMPONENT_KCONFIGS_PROJBUILD are empty
+
         print("Running: %s" % cmdline)
         p = pexpect.spawn(cmdline, timeout=30, logfile=args.logfile, echo=False, use_poll=True, maxread=1)
 

tools/ldgen/sdkconfig.py

@@ -26,8 +26,6 @@ except Exception:
     sys.path.insert(0, kconfig_new_dir)
     import kconfiglib
 
-import confgen
-
 
 class SDKConfig:
     """
@@ -49,7 +47,6 @@ class SDKConfig:
     OPERATOR = oneOf(["=", "!=", ">", "<", "<=", ">="])
 
     def __init__(self, kconfig_file, sdkconfig_file):
-        confgen.prepare_source_files()
         self.config = kconfiglib.Kconfig(kconfig_file)
         self.config.load_config(sdkconfig_file)
 

tools/ldgen/test/test_fragments.py

@@ -69,6 +69,9 @@ class FragmentTest(unittest.TestCase):
         os.environ['COMPONENT_KCONFIGS'] = ''
         os.environ['COMPONENT_KCONFIGS_PROJBUILD'] = ''
 
+        # prepare_kconfig_files.py doesn't have to be called because COMPONENT_KCONFIGS and
+        # COMPONENT_KCONFIGS_PROJBUILD are empty
+
         self.sdkconfig = SDKConfig("data/Kconfig", "data/sdkconfig")
 
     def tearDown(self):

tools/ldgen/test/test_generation.py

@@ -56,6 +56,9 @@ class GenerationModelTest(unittest.TestCase):
         os.environ['COMPONENT_KCONFIGS'] = ''
         os.environ['COMPONENT_KCONFIGS_PROJBUILD'] = ''
 
+        # prepare_kconfig_files.py doesn't have to be called because COMPONENT_KCONFIGS and
+        # COMPONENT_KCONFIGS_PROJBUILD are empty
+
         self.sdkconfig = SDKConfig("data/Kconfig", "data/sdkconfig")
 
         with open("data/sample.lf") as fragment_file_obj: