diff --git a/components/esp_hw_support/CMakeLists.txt b/components/esp_hw_support/CMakeLists.txt
index e5eced1814..79f4e11bb5 100644
--- a/components/esp_hw_support/CMakeLists.txt
+++ b/components/esp_hw_support/CMakeLists.txt
@@ -17,6 +17,11 @@ if(NOT BOOTLOADER_BUILD)
                      "mac_addr.c"
                      "sleep_modes.c")
      list(APPEND priv_requires esp_ipc)
+
+     if(NOT CMAKE_BUILD_EARLY_EXPANSION)
+          set_source_files_properties("${CMAKE_CURRENT_LIST_DIR}/sleep_modes.c" PROPERTIES
+                                      COMPILE_FLAGS "-fno-jump-tables -fno-tree-switch-conversion")
+     endif()
 endif()
 
 idf_component_register(SRCS ${srcs}
diff --git a/components/esp_system/CMakeLists.txt b/components/esp_system/CMakeLists.txt
index 66f22aad0b..ce9950d498 100644
--- a/components/esp_system/CMakeLists.txt
+++ b/components/esp_system/CMakeLists.txt
@@ -52,10 +52,6 @@ else()
         PROPERTIES COMPILE_FLAGS
         -fno-stack-protector)
 
-    if(NOT CMAKE_BUILD_EARLY_EXPANSION)
-        set_source_files_properties("${CMAKE_CURRENT_LIST_DIR}/sleep_modes.c" PROPERTIES
-                                    COMPILE_FLAGS "-fno-jump-tables -fno-tree-switch-conversion")
-    endif()
 endif()
 
 if(CONFIG_IDF_ENV_FPGA)