diff --git a/components/bt/CMakeLists.txt b/components/bt/CMakeLists.txt
index df0da6a316..ac8962d148 100644
--- a/components/bt/CMakeLists.txt
+++ b/components/bt/CMakeLists.txt
@@ -705,14 +705,14 @@ if(CONFIG_BT_ENABLED)
         target_link_libraries(${COMPONENT_LIB} INTERFACE
                 "-L${CMAKE_CURRENT_LIST_DIR}/controller/lib_esp32c3_family/esp32s3")
         target_link_libraries(${COMPONENT_LIB} PUBLIC btdm_app)
-    elseif(CONFIG_IDF_TARGET_ESP32H2)
+    elseif(CONFIG_IDF_TARGET_ESP32H2 AND CONFIG_BT_CONTROLLER_ENABLED)
         if(CONFIG_IDF_TARGET_ESP32H2_BETA_VERSION_1)
             add_prebuilt_library(nimblelib "controller/lib_esp32h2/esp32h2-bt-lib/beta1/libble_app.a")
         elseif(CONFIG_IDF_TARGET_ESP32H2_BETA_VERSION_2)
             add_prebuilt_library(nimblelib "controller/lib_esp32h2/esp32h2-bt-lib/beta2/libble_app.a")
         endif()
         target_link_libraries(${COMPONENT_LIB} PRIVATE nimblelib)
-    elseif(CONFIG_IDF_TARGET_ESP32C2)
+    elseif(CONFIG_IDF_TARGET_ESP32C2 AND CONFIG_BT_CONTROLLER_ENABLED)
         add_prebuilt_library(nimblelib "controller/lib_esp32c2/esp32c2-bt-lib/libble_app.a")
         target_link_libraries(${COMPONENT_LIB} PRIVATE nimblelib)
     endif()