Merge branch 'bugfix/regenerate_sdkconfig_fails_v3.1' into 'release/v3.1'

Fix issues with regenerating sdkconfig on modification (backport v3.1)

See merge request idf/esp-idf!4170

docs/conf_common.py

@@ -53,7 +53,6 @@ confgen_args = ["python",
                 "../../tools/kconfig_new/confgen.py",
                 "--kconfig", "../../Kconfig",
                 "--config", temp_sdkconfig_path,
-                "--create-config-if-missing",
                 "--env", "COMPONENT_KCONFIGS={}".format(kconfigs),
                 "--env", "COMPONENT_KCONFIGS_PROJBUILD={}".format(kconfig_projbuilds),
                 "--output", "docs", kconfig_inc_path + '.in'

tools/ci/test_build_system.sh

@@ -265,6 +265,17 @@ function run_tests()
     make EXTRA_COMPONENT_DIRS=$PWD/mycomponents || failure "EXTRA_COMPONENT_DIRS has added a sibling directory"
     rm -rf esp32
     rm -rf mycomponents
+
+    print_status "sdkconfig should have contents both files: sdkconfig and sdkconfig.defaults"
+    make clean > /dev/null;
+    rm -f sdkconfig.defaults;
+    rm -f sdkconfig;
+    echo "CONFIG_PARTITION_TABLE_OFFSET=0x10000" >> sdkconfig.defaults;
+    echo "CONFIG_PARTITION_TABLE_TWO_OTA=y" >> sdkconfig;
+    make defconfig > /dev/null;
+    grep "CONFIG_PARTITION_TABLE_OFFSET=0x10000" sdkconfig || failure "The define from sdkconfig.defaults should be into sdkconfig"
+    grep "CONFIG_PARTITION_TABLE_TWO_OTA=y" sdkconfig || failure "The define from sdkconfig should be into sdkconfig"
+
     print_status "All tests completed"
     if [ -n "${FAILURES}" ]; then
         echo "Some failures were detected:"

tools/ci/test_build_system_cmake.sh

@@ -238,6 +238,17 @@ EOF
     export PATH="$OLDPATH"
     rm ./python
 
+    print_status "sdkconfig should have contents both files: sdkconfig and sdkconfig.defaults"
+    idf.py clean > /dev/null;
+    idf.py fullclean > /dev/null;
+    rm -f sdkconfig.defaults;
+    rm -f sdkconfig;
+    echo "CONFIG_PARTITION_TABLE_OFFSET=0x10000" >> sdkconfig.defaults;
+    echo "CONFIG_PARTITION_TABLE_TWO_OTA=y" >> sdkconfig;
+    idf.py reconfigure > /dev/null;
+    grep "CONFIG_PARTITION_TABLE_OFFSET=0x10000" sdkconfig || failure "The define from sdkconfig.defaults should be into sdkconfig"
+    grep "CONFIG_PARTITION_TABLE_TWO_OTA=y" sdkconfig || failure "The define from sdkconfig should be into sdkconfig"
+
     print_status "All tests completed"
     if [ -n "${FAILURES}" ]; then
         echo "Some failures were detected:"

tools/cmake/kconfig.cmake

@@ -105,7 +105,6 @@ function(kconfig_process_config)
         --kconfig ${ROOT_KCONFIG}
         --config ${SDKCONFIG}
         ${defaults_arg}
-        --create-config-if-missing
         --env "COMPONENT_KCONFIGS=${kconfigs}"
         --env "COMPONENT_KCONFIGS_PROJBUILD=${kconfigs_projbuild}"
         --env "IDF_CMAKE=y")
@@ -138,12 +137,24 @@ function(kconfig_process_config)
     # makes sdkconfig.h and skdconfig.cmake
     #
     # This happens during the cmake run not during the build
-    execute_process(COMMAND ${confgen_basecommand}
+    if(NOT BOOTLOADER_BUILD)
-        --output header ${SDKCONFIG_HEADER}
+        execute_process(
-        --output cmake ${SDKCONFIG_CMAKE}
+            COMMAND ${confgen_basecommand}
-        --output json ${SDKCONFIG_JSON}
+            --output header ${SDKCONFIG_HEADER}
-        --output json_menus ${KCONFIG_JSON_MENUS}
+            --output cmake ${SDKCONFIG_CMAKE}
-        RESULT_VARIABLE config_result)
+            --output json ${SDKCONFIG_JSON}
+            --output json_menus ${KCONFIG_JSON_MENUS}
+            --output config ${SDKCONFIG} # only generate config at the top-level project
+            RESULT_VARIABLE config_result)
+    else()
+        execute_process(
+            COMMAND ${confgen_basecommand}
+            --output header ${SDKCONFIG_HEADER}
+            --output cmake ${SDKCONFIG_CMAKE}
+            --output json ${SDKCONFIG_JSON}
+            --output json_menus ${KCONFIG_JSON_MENUS}
+            RESULT_VARIABLE config_result)
+    endif()
     if(config_result)
         message(FATAL_ERROR "Failed to run confgen.py (${confgen_basecommand}). Error ${config_result}")
     endif()

tools/kconfig_new/confgen.py

@@ -49,10 +49,6 @@ def main():
                         nargs='?',
                         default=None)
 
-    parser.add_argument('--create-config-if-missing',
-                        help='If set, a new config file will be saved if the old one is not found',
-                        action='store_true')
-
     parser.add_argument('--kconfig',
                         help='KConfig file with config item definitions',
                         required=True)
@@ -82,6 +78,8 @@ def main():
         os.environ[name] = value
 
     config = kconfiglib.Kconfig(args.kconfig)
+    config.disable_redun_warnings()
+    config.disable_override_warnings()
 
     if args.defaults is not None:
         # always load defaults first, so any items which are not defined in that config
@@ -90,26 +88,22 @@ def main():
             raise RuntimeError("Defaults file not found: %s" % args.defaults)
         config.load_config(args.defaults)
 
-    if args.config is not None:
+    # If config file previously exists, load it
-        if os.path.exists(args.config):
+    if args.config and os.path.exists(args.config):
-            config.load_config(args.config)
+        config.load_config(args.config, replace=False)
-        elif args.create_config_if_missing:
-            print("Creating config file %s..." % args.config)
-            config.write_config(args.config)
-        elif args.default is None:
-            raise RuntimeError("Config file not found: %s" % args.config)
 
-        for output_type, filename in args.output:
+    # Output the files specified in the arguments
-            temp_file = tempfile.mktemp(prefix="confgen_tmp")
+    for output_type, filename in args.output:
+        temp_file = tempfile.mktemp(prefix="confgen_tmp")
+        try:
+            output_function = OUTPUT_FORMATS[output_type]
+            output_function(config, temp_file)
+            update_if_changed(temp_file, filename)
+        finally:
             try:
-                output_function = OUTPUT_FORMATS[output_type]
+                os.remove(temp_file)
-                output_function(config, temp_file)
+            except OSError:
-                update_if_changed(temp_file, filename)
+                pass
-            finally:
-                try:
-                    os.remove(temp_file)
-                except OSError:
-                    pass
 
 
 def write_config(config, filename):
@@ -262,6 +256,7 @@ def write_json_menus(config, filename):
 def update_if_changed(source, destination):
     with open(source, "r") as f:
         source_contents = f.read()
+
     if os.path.exists(destination):
         with open(destination, "r") as f:
             dest_contents = f.read()

tools/kconfig_new/kconfiglib.py

@@ -500,6 +500,8 @@ class Kconfig(object):
     __slots__ = (
         "_choices",
         "_print_undef_assign",
+        "_print_override",
+        "_print_redun_assign",
         "_print_warnings",
         "_set_re_match",
         "_unset_re_match",
@@ -575,6 +577,7 @@ class Kconfig(object):
 
         self._print_warnings = warn
         self._print_undef_assign = False
+        self._print_redun_assign = self._print_override = True
 
         self.syms = {}
         self.const_syms = {}
@@ -826,10 +829,12 @@ class Kconfig(object):
                         display_val = val
                         display_user_val = sym.user_value
 
-                    self._warn('{} set more than once. Old value: "{}", new '
+                    msg = '{} set more than once. Old value: "{}", new value: "{}".'.format(name, display_user_val, display_val)
-                               'value: "{}".'
+
-                               .format(name, display_user_val, display_val),
+                    if display_user_val == display_val:
-                               filename, linenr)
+                        self._warn_redun_assign(msg, filename, linenr)
+                    else:
+                        self._warn_override(msg, filename, linenr)
 
                 sym.set_value(val)
 
@@ -1057,6 +1062,36 @@ class Kconfig(object):
         """
         self._print_undef_assign = False
 
+    def enable_redun_warnings(self):
+        """
+        Enables warnings for redundant assignments to symbols. Printed to
+        stderr. Enabled by default.
+        """
+        self._print_redun_assign = True
+
+    def disable_redun_warnings(self):
+        """
+        See enable_redun_warnings().
+        """
+        self._print_redun_assign = False
+
+    def enable_override_warnings(self):
+        """
+        Enables warnings for duplicated assignments in .config files that set
+        different values (e.g. CONFIG_FOO=m followed by CONFIG_FOO=y, where
+        the last value set is used).
+
+        These warnings are enabled by default. Disabling them might be helpful
+        in certain cases when merging configurations.
+        """
+        self._print_override = True
+
+    def disable_override_warnings(self):
+        """
+        See enable_override_warnings().
+        """
+        self._print_override = False
+
     def __repr__(self):
         """
         Returns a string with information about the Kconfig object when it is
@@ -1071,6 +1106,8 @@ class Kconfig(object):
             "warnings " + ("enabled" if self._print_warnings else "disabled"),
             "undef. symbol assignment warnings " +
                 ("enabled" if self._print_undef_assign else "disabled"),
+            "redundant symbol assignment warnings " +
+                ("enabled" if self._print_redun_assign else "disabled")
         )))
 
     #
@@ -2150,6 +2187,19 @@ class Kconfig(object):
             'attempt to assign the value "{}" to the undefined symbol {}' \
             .format(val, name), filename, linenr)
 
+    def _warn_redun_assign(self, msg, filename=None, linenr=None):
+        """
+        See the class documentation.
+        """
+        if self._print_redun_assign:
+            _stderr_msg("warning: " + msg, filename, linenr)
+
+    def _warn_override(self, msg, filename=None, linenr=None):
+        """
+        See the class documentation.
+        """
+        if self._print_override:
+            _stderr_msg("warning: " + msg, filename, linenr)
 
 class Symbol(object):
     """